Merge pull request #65 from bothub-it/staging

Staging

Pipfile

@@ -13,7 +13,7 @@ bothub_backend = {ref = "1.0.12",git = "https://github.com/Ilhasoft/bothub-backe
 celery = "==4.3.0"
 gunicorn = "==19.9.0"
 gevent = "==1.4.0"
-bothub_nlp_celery = {ref = "0.1.26",git = "https://github.com/Ilhasoft/bothub-nlp-celery"}
+bothub_nlp_celery = {ref = "0.1.28",git = "https://github.com/Ilhasoft/bothub-nlp-celery"}
 django-environ = "==0.4.5"
 redis = "==3.5.3"
 kombu = "==4.5.0"

bothub_nlp_api/handlers/debug_parse.py

@@ -1,8 +1,7 @@
 from bothub_nlp_celery.actions import ACTION_DEBUG_PARSE, queue_name
 from bothub_nlp_celery.app import celery_app
 from bothub_nlp_celery.tasks import TASK_NLU_DEBUG_PARSE_TEXT
-from bothub_nlp_celery.utils import ALGORITHM_TO_LANGUAGE_MODEL
-from bothub_nlp_celery import settings as celery_settings
+from bothub_nlp_celery.utils import get_language_model
 
 from bothub_nlp_api import settings
 from bothub_nlp_api.utils import AuthorizationIsRequired
@@ -43,22 +42,7 @@ def _debug_parse(authorization, text, language, repository_version=None):
     if not update.get("version"):
         raise ValidationError("This repository has never been trained")
 
-    chosen_algorithm = update.get("algorithm")
-    # chosen_algorithm = choose_best_algorithm(update.get("language"))
-    model = ALGORITHM_TO_LANGUAGE_MODEL[chosen_algorithm]
-
-    if (model == "SPACY" and language not in celery_settings.SPACY_LANGUAGES) or (
-        model == "BERT" and language not in celery_settings.BERT_LANGUAGES
-    ):
-        model = None
-
-    # Send parse to SPACY worker to use name_entities (only if BERT not in use)
-    if (
-        (update.get("use_name_entities"))
-        and (model is None)
-        and (language in celery_settings.SPACY_LANGUAGES)
-    ):
-        model = "SPACY"
+    model = get_language_model(update)
 
     answer_task = celery_app.send_task(
         TASK_NLU_DEBUG_PARSE_TEXT,

bothub_nlp_api/handlers/evaluate.py

@@ -1,20 +1,24 @@
 from bothub_nlp_celery.actions import ACTION_EVALUATE, queue_name
 from bothub_nlp_celery.app import celery_app
 from bothub_nlp_celery.tasks import TASK_NLU_EVALUATE_UPDATE
-from bothub_nlp_celery.utils import ALGORITHM_TO_LANGUAGE_MODEL
-from bothub_nlp_celery import settings as celery_settings
+from bothub_nlp_celery.utils import get_language_model
 
 from .. import settings
 from ..utils import AuthorizationIsRequired
 from ..utils import DEFAULT_LANGS_PRIORITY
 from ..utils import ValidationError, get_repository_authorization
 from ..utils import backend
+from ..utils import send_job_train_ai_platform
+import time
 
 EVALUATE_STATUS_EVALUATED = "evaluated"
+EVALUATE_STATUS_PROCESSING = "processing"
 EVALUATE_STATUS_FAILED = "failed"
 
 
-def evaluate_handler(authorization, language, repository_version=None):
+def evaluate_handler(
+    authorization, language, repository_version=None, cross_validation=False
+):
     if language and (
         language not in settings.SUPPORTED_LANGUAGES.keys()
         and language not in DEFAULT_LANGS_PRIORITY.keys()
@@ -35,40 +39,50 @@ def evaluate_handler(authorization, language, repository_version=None):
     if not update.get("update"):
         raise ValidationError("This repository has never been trained")
 
-    chosen_algorithm = update.get("algorithm")
-    # chosen_algorithm = choose_best_algorithm(update.get("language"))
-    model = ALGORITHM_TO_LANGUAGE_MODEL[chosen_algorithm]
-    if (model == "SPACY" and language not in celery_settings.SPACY_LANGUAGES) or (
-        model == "BERT" and language not in celery_settings.BERT_LANGUAGES
-    ):
-        model = None
-
-    # Send evaluate to SPACY worker to use name_entities (only if BERT not in use)
-    if (
-        (update.get("use_name_entities"))
-        and (model is None)
-        and (language in celery_settings.SPACY_LANGUAGES)
-    ):
-        model = "SPACY"
+    model = get_language_model(update)
 
     try:
-        evaluate_task = celery_app.send_task(
-            TASK_NLU_EVALUATE_UPDATE,
-            args=[
-                update.get("repository_version"),
-                update.get("user_id"),
-                repository_authorization,
-            ],
-            queue=queue_name(update.get("language"), ACTION_EVALUATE, model),
-        )
-        evaluate_task.wait()
-        evaluate = evaluate_task.result
+        evaluate = None
+        if cross_validation is False:
+            evaluate_task = celery_app.send_task(
+                TASK_NLU_EVALUATE_UPDATE,
+                args=[
+                    update.get("repository_version"),
+                    update.get("user_id"),
+                    repository_authorization,
+                    cross_validation,
+                ],
+                queue=queue_name(update.get("language"), ACTION_EVALUATE, model),
+            )
+            evaluate_task.wait()
+            evaluate = evaluate_task.result
+        else:
+            job_id = f'bothub_{settings.ENVIRONMENT}_evaluate_{str(update.get("repository_version"))}_{language}_{str(int(time.time()))}'
+            send_job_train_ai_platform(
+                jobId=job_id,
+                repository_version=str(update.get("repository_version")),
+                by_id=str(update.get("user_id")),
+                repository_authorization=str(repository_authorization),
+                language=language,
+                type_model=model,
+                operation="evaluate",
+            )
+            backend().request_backend_save_queue_id(
+                update_id=str(update.get("repository_version")),
+                repository_authorization=str(repository_authorization),
+                task_id=job_id,
+                from_queue=0,
+            )
+
         evaluate_report = {
             "language": language,
-            "status": EVALUATE_STATUS_EVALUATED,
+            "status": EVALUATE_STATUS_PROCESSING,
             "repository_version": update.get("repository_version"),
-            "evaluate_id": evaluate.get("id"),
-            "evaluate_version": evaluate.get("version"),
+            "evaluate_id": evaluate.get("id") if evaluate is not None else None,
+            "evaluate_version": evaluate.get("version")
+            if evaluate is not None
+            else None,
+            "cross_validation": cross_validation,
         }
     except Exception as e:
         evaluate_report = {"status": EVALUATE_STATUS_FAILED, "error": str(e)}

bothub_nlp_api/handlers/intent_sentence_suggestion.py

@@ -0,0 +1,59 @@
+from bothub_nlp_celery.actions import ACTION_INTENT_SENTENCE_SUGGESTION, queue_name
+from bothub_nlp_celery.app import celery_app
+from bothub_nlp_celery.tasks import TASK_NLU_INTENT_SENTENCE_SUGGESTION_TEXT
+from bothub_nlp_api.utils import get_repository_authorization
+from bothub_nlp_api.utils import backend
+from bothub_nlp_api import settings
+from bothub_nlp_api.utils import ValidationError
+from bothub_nlp_api.utils import AuthorizationIsRequired
+
+
+def _intent_sentence_suggestion(
+    authorization,
+    language,
+    intent,
+    n_sentences_to_generate,
+    percentage_to_replace,
+    repository_version=None,
+):
+    print(authorization)
+    from ..utils import DEFAULT_LANGS_PRIORITY
+
+    if language and (
+        language not in settings.SUPPORTED_LANGUAGES.keys()
+        and language not in DEFAULT_LANGS_PRIORITY.keys()
+    ):
+        raise ValidationError("Language '{}' not supported by now.".format(language))
+
+    repository_authorization = get_repository_authorization(authorization)
+    if not repository_authorization:
+        raise AuthorizationIsRequired()
+
+    try:
+        update = backend().request_backend_parse(
+            repository_authorization, language, repository_version
+        )
+    except Exception:
+        update = {}
+    answer_task = celery_app.send_task(
+        TASK_NLU_INTENT_SENTENCE_SUGGESTION_TEXT,
+        args=[
+            update.get("repository_version"),
+            repository_authorization,
+            intent,
+            percentage_to_replace,
+            n_sentences_to_generate,
+        ],
+        queue=queue_name(language, ACTION_INTENT_SENTENCE_SUGGESTION, "SPACY"),
+    )
+    answer_task.wait()
+    answer = answer_task.result
+    answer.update(
+        {
+            "language": language,
+            "n_sentences_to_generate": n_sentences_to_generate,
+            "percentage_to_replace": percentage_to_replace,
+            "intent": intent,
+        }
+    )
+    return answer

bothub_nlp_api/handlers/parse.py

@@ -5,8 +5,7 @@
 from bothub_nlp_celery.actions import ACTION_PARSE, queue_name
 from bothub_nlp_celery.app import celery_app
 from bothub_nlp_celery.tasks import TASK_NLU_PARSE_TEXT
-from bothub_nlp_celery.utils import ALGORITHM_TO_LANGUAGE_MODEL
-from bothub_nlp_celery import settings as celery_settings
+from bothub_nlp_celery.utils import get_language_model
 
 from bothub_nlp_api import settings
 from bothub_nlp_api.utils import AuthorizationIsRequired
@@ -94,23 +93,7 @@ def _parse(
     if not update.get("version"):
         raise ValidationError("This repository has never been trained")
 
-    chosen_algorithm = update.get("algorithm")
-    # chosen_algorithm = choose_best_algorithm(update.get("language"))
-    model = ALGORITHM_TO_LANGUAGE_MODEL[chosen_algorithm]
-
-    language = update.get("language")
-    if (model == "SPACY" and language not in celery_settings.SPACY_LANGUAGES) or (
-        model == "BERT" and language not in celery_settings.BERT_LANGUAGES
-    ):
-        model = None
-
-    # Send parse to SPACY worker to use name_entities (only if BERT not in use)
-    if (
-        (update.get("use_name_entities"))
-        and (model is None)
-        and (language in celery_settings.SPACY_LANGUAGES)
-    ):
-        model = "SPACY"
+    model = get_language_model(update)
 
     answer_task = celery_app.send_task(
         TASK_NLU_PARSE_TEXT,

bothub_nlp_api/handlers/score_calculation.py

@@ -0,0 +1,32 @@
+from bothub_nlp_celery.actions import ACTION_SCORE_CALCULATION, queue_name
+
+from ..utils import backend, get_repository_authorization
+from ..utils import AuthorizationIsRequired
+
+
+from bothub_nlp_celery.app import celery_app
+from bothub_nlp_celery.tasks import TASK_NLU_SCORE_CALCULATION
+
+
+def score_handler(authorization, repository_version, language):
+
+    repository_authorization = get_repository_authorization(authorization)
+
+    if not repository_authorization:
+        raise AuthorizationIsRequired()
+
+    try:
+        update = backend().request_backend_train(
+            repository_authorization, language, repository_version
+        )
+    except Exception:
+        update = {}
+
+    answer_task = celery_app.send_task(
+        TASK_NLU_SCORE_CALCULATION,
+        args=[update.get("current_version_id"), repository_authorization],
+        queue=queue_name(language, ACTION_SCORE_CALCULATION),
+    )
+    answer_task.wait()
+
+    return answer_task.result