Merge pull request #702 from NatLibFi/upgrade-to-connexion3

Upgrade to Connexion3

Dockerfile

@@ -54,4 +54,6 @@ RUN groupadd -g 998 annif_user && \
     chown -R annif_user:annif_user /annif-projects /Annif/tests/data
 USER annif_user
 
+ENV GUNICORN_CMD_ARGS="--worker-class uvicorn.workers.UvicornWorker"
+
 CMD annif

annif/__init__.py

@@ -7,19 +7,20 @@
 import os.path
 from typing import TYPE_CHECKING
 
+from flask import Flask
+
 logging.basicConfig()
 logger = logging.getLogger("annif")
 logger.setLevel(level=logging.INFO)
 
 import annif.backend  # noqa
 
 if TYPE_CHECKING:
-    from flask.app import Flask
+    from connexion.apps.flask import FlaskApp
 
 
 def create_flask_app(config_name: str | None = None) -> Flask:
     """Create a Flask app to be used by the CLI."""
-    from flask import Flask
 
     _set_tensorflow_loglevel()
 
@@ -31,29 +32,41 @@ def create_flask_app(config_name: str | None = None) -> Flask:
     return app
 
 
-def create_app(config_name: str | None = None) -> Flask:
+def create_cx_app(config_name: str | None = None) -> FlaskApp:
     """Create a Connexion app to be used for the API."""
-    # 'cxapp' here is the Connexion application that has a normal Flask app
-    # as a property (cxapp.app)
     import connexion
-    from flask_cors import CORS
+    from connexion.datastructures import MediaTypeDict
+    from connexion.middleware import MiddlewarePosition
+    from connexion.validators import FormDataValidator, MultiPartFormDataValidator
+    from starlette.middleware.cors import CORSMiddleware
 
+    import annif.registry
     from annif.openapi.validation import CustomRequestBodyValidator
 
     specdir = os.path.join(os.path.dirname(__file__), "openapi")
-    cxapp = connexion.App(__name__, specification_dir=specdir)
+    cxapp = connexion.FlaskApp(__name__, specification_dir=specdir)
     config_name = _get_config_name(config_name)
     logger.debug(f"creating connexion app with configuration {config_name}")
     cxapp.app.config.from_object(config_name)
     cxapp.app.config.from_envvar("ANNIF_SETTINGS", silent=True)
 
     validator_map = {
-        "body": CustomRequestBodyValidator,
+        "body": MediaTypeDict(
+            {
+                "*/*json": CustomRequestBodyValidator,
+                "application/x-www-form-urlencoded": FormDataValidator,
+                "multipart/form-data": MultiPartFormDataValidator,
+            }
+        ),
     }
     cxapp.add_api("annif.yaml", validator_map=validator_map)
 
     # add CORS support
-    CORS(cxapp.app)
+    cxapp.add_middleware(
+        CORSMiddleware,
+        position=MiddlewarePosition.BEFORE_EXCEPTION,
+        allow_origins=["*"],
+    )
 
     if cxapp.app.config["INITIALIZE_PROJECTS"]:
         annif.registry.initialize_projects(cxapp.app)
@@ -64,8 +77,11 @@ def create_app(config_name: str | None = None) -> Flask:
 
     cxapp.app.register_blueprint(bp)
 
-    # return the Flask app
-    return cxapp.app
+    # return the Connexion app
+    return cxapp
+
+
+create_app = create_cx_app  # Alias to allow starting directly with uvicorn run
 
 
 def _get_config_name(config_name: str | None) -> str:

annif/cli.py

@@ -25,15 +25,12 @@
 logger = annif.logger
 click_log.basic_config(logger)
 
-
-if len(sys.argv) > 1 and sys.argv[1] in ("run", "routes"):
-    create_app = annif.create_app  # Use Flask with Connexion
-else:
-    # Connexion is not needed for most CLI commands, use plain Flask
-    create_app = annif.create_flask_app
-
-cli = FlaskGroup(create_app=create_app, add_version_option=False)
+create_app = annif.create_flask_app
+cli = FlaskGroup(
+    create_app=create_app, add_default_commands=False, add_version_option=False
+)
 cli = click.version_option(message="%(version)s")(cli)
+cli.params = [opt for opt in cli.params if opt.name not in ("env_file", "app")]
 
 
 @cli.command("list-projects")
@@ -442,6 +439,21 @@ def run_eval(
         )
 
 
+@cli.command("run")
+@click.option("--port", type=int, default=5000)
+@click.option("--log-level")
+@click_log.simple_verbosity_option(logger, default="ERROR")
+def run_app(**kwargs):
+    """
+    Run Annif in server mode for development.
+    \f
+    The server is for development purposes only.
+    """
+    kwargs = {k: v for k, v in kwargs.items() if v is not None}
+    cxapp = annif.create_cx_app()
+    cxapp.run(**kwargs)
+
+
 FILTER_BATCH_MAX_LIMIT = 15
 OPTIMIZE_METRICS = ["Precision (doc avg)", "Recall (doc avg)", "F1 score (doc avg)"]
 

annif/openapi/annif.yaml

@@ -174,7 +174,9 @@ paths:
       responses:
         "204":
           description: successful operation
-          content: {}
+          content:
+            application/json:
+              {}
         "404":
           $ref: '#/components/responses/NotFound'
         "503":

annif/openapi/validation.py

@@ -3,48 +3,37 @@
 from __future__ import annotations
 
 import logging
+from typing import Any
 
-import jsonschema
-from connexion import decorators
 from connexion.exceptions import BadRequestProblem
-from connexion.utils import is_null
+from connexion.json_schema import format_error_with_path
+from connexion.validators import JSONRequestBodyValidator
+from jsonschema.exceptions import ValidationError
 
 logger = logging.getLogger("openapi.validation")
 
 
-class CustomRequestBodyValidator(decorators.validation.RequestBodyValidator):
+class CustomRequestBodyValidator(JSONRequestBodyValidator):
     """Custom request body validator that overrides the default error message for the
-    'maxItems' validator for the 'documents' property."""
+    'maxItems' validator for the 'documents' property to prevent logging request body
+    with the contents of all documents."""
 
     def __init__(self, *args, **kwargs) -> None:
         super().__init__(*args, **kwargs)
 
-    def validate_schema(
-        self,
-        data: list | dict,
-        url: str,
-    ) -> None:
-        """Validate the request body against the schema."""
-
-        if self.is_null_value_valid and is_null(data):
-            return None  # pragma: no cover
-
+    def _validate(self, body: Any) -> dict | None:
         try:
-            self.validator.validate(data)
-        except jsonschema.ValidationError as exception:
+            return self._validator.validate(body)
+        except ValidationError as exception:
             if exception.validator == "maxItems" and list(exception.schema_path) == [
                 "properties",
                 "documents",
                 "maxItems",
             ]:
                 exception.message = "too many items"
-
-            error_path_msg = self._error_path_message(exception=exception)
+            error_path_msg = format_error_with_path(exception=exception)
             logger.error(
-                "{url} validation error: {error}{error_path_msg}".format(
-                    url=url, error=exception.message, error_path_msg=error_path_msg
-                ),
+                f"Validation error: {exception.message}{error_path_msg}",
                 extra={"validator": "body"},
             )
             raise BadRequestProblem(detail=f"{exception.message}{error_path_msg}")
-        return None

annif/rest.py

@@ -14,8 +14,6 @@
 from annif.project import Access
 
 if TYPE_CHECKING:
-    from datetime import datetime
-
     from connexion.lifecycle import ConnexionResponse
 
     from annif.corpus.subject import SubjectIndex
@@ -43,10 +41,11 @@ def server_error(
     )
 
 
-def show_info() -> dict[str, str]:
+def show_info() -> tuple:
     """return version of annif and a title for the api according to OpenAPI spec"""
 
-    return {"title": "Annif REST API", "version": importlib.metadata.version("annif")}
+    result = {"title": "Annif REST API", "version": importlib.metadata.version("annif")}
+    return result, 200, {"Content-Type": "application/json"}
 
 
 def language_not_supported_error(lang: str) -> ConnexionResponse:
@@ -59,15 +58,16 @@ def language_not_supported_error(lang: str) -> ConnexionResponse:
     )
 
 
-def list_projects() -> dict[str, list[dict[str, str | dict | bool | datetime | None]]]:
+def list_projects() -> tuple:
     """return a dict with projects formatted according to OpenAPI spec"""
 
-    return {
+    result = {
         "projects": [
             proj.dump()
             for proj in annif.registry.get_projects(min_access=Access.public).values()
         ]
     }
+    return result, 200, {"Content-Type": "application/json"}
 
 
 def show_project(
@@ -79,7 +79,7 @@ def show_project(
         project = annif.registry.get_project(project_id, min_access=Access.hidden)
     except ValueError:
         return project_not_found_error(project_id)
-    return project.dump()
+    return project.dump(), 200, {"Content-Type": "application/json"}
 
 
 def _suggestion_to_dict(
@@ -124,7 +124,7 @@ def suggest(
 
     if _is_error(result):
         return result
-    return result[0]
+    return result[0], 200, {"Content-Type": "application/json"}
 
 
 def suggest_batch(
@@ -142,7 +142,7 @@ def suggest_batch(
         return result
     for document_results, document in zip(result, documents):
         document_results["document_id"] = document.get("document_id")
-    return result
+    return result, 200, {"Content-Type": "application/json"}
 
 
 def _suggest(
@@ -214,4 +214,4 @@ def learn(
     except AnnifException as err:
         return server_error(err)
 
-    return None, 204
+    return None, 204, {"Content-Type": "application/json"}

annif/templates/home.html

@@ -306,7 +306,7 @@ <h2 class="mt-4" id="suggestions">Suggested subjects</h2>\
         return;
       }
       var this_ = this;
-      var formData = new FormData();
+      var formData = new URLSearchParams();
       formData.append('text', this_.text);
       formData.append('limit', this_.limit);
       this_.loading = true;

docs/source/commands.rst

@@ -121,7 +121,7 @@ Subject index administration
 
    N/A
 
-.. click:: flask.cli:run_command
+.. click:: annif.cli:run_app
    :prog: annif run
 
 **REST equivalent**

pyproject.toml

@@ -30,9 +30,7 @@ classifiers = [
 [tool.poetry.dependencies]
 python = ">=3.9,<3.12"
 
-connexion = { version = "2.14.2", extras = ["swagger-ui"] }
-flask = "2.2.*"
-flask-cors = "4.0.*"
+connexion = { version = "~3.0.5", extras = ["flask", "uvicorn", "swagger-ui"] }
 click = "8.1.*"
 click-log = "0.4.*"
 joblib = "1.3.*"

tests/conftest.py

@@ -15,34 +15,39 @@
 
 
 @pytest.fixture(scope="module")
-def app():
+def cxapp():
     # make sure the dummy vocab is in place because many tests depend on it
     subjfile = os.path.join(os.path.dirname(__file__), "corpora", "dummy-subjects.csv")
-    app = annif.create_app(config_name="annif.default_config.TestingConfig")
-    with app.app_context():
+    cxapp = annif.create_app(config_name="annif.default_config.TestingConfig")
+    with cxapp.app.app_context():
         project = annif.registry.get_project("dummy-en")
         # the vocab is needed for both English and Finnish language projects
         vocab = annif.corpus.SubjectFileCSV(subjfile)
         project.vocab.load_vocabulary(vocab)
-    return app
+    return cxapp
+
+
+@pytest.fixture(scope="module")
+def app(cxapp):
+    return cxapp.app
 
 
 @pytest.fixture(scope="module")
 def app_with_initialize():
-    app = annif.create_app(config_name="annif.default_config.TestingInitializeConfig")
-    return app
+    cxapp = annif.create_app(config_name="annif.default_config.TestingInitializeConfig")
+    return cxapp.app
 
 
 @pytest.fixture(scope="module")
 @unittest.mock.patch.dict(os.environ, {"ANNIF_PROJECTS_INIT": ".*-fi"})
 def app_with_initialize_fi_projects():
-    app = annif.create_app(config_name="annif.default_config.TestingInitializeConfig")
-    return app
+    cxapp = annif.create_app(config_name="annif.default_config.TestingInitializeConfig")
+    return cxapp.app
 
 
 @pytest.fixture
-def app_client(app):
-    with app.test_client() as app_client:
+def app_client(cxapp):
+    with cxapp.test_client() as app_client:
         yield app_client
 
 

tests/test_cli.py

@@ -1051,25 +1051,19 @@ def test_version_option():
     assert result.output.strip() == version.strip()
 
 
-def test_run():
-    result = runner.invoke(annif.cli.cli, ["run", "--help"])
+@mock.patch("connexion.FlaskApp.run")
+def test_run(run):
+    result = runner.invoke(annif.cli.cli, ["run"])
     assert not result.exception
     assert result.exit_code == 0
-    assert "Run a local development server." in result.output
-
-
-def test_routes_with_flask_app():
-    # When using plain Flask only the static endpoint exists
-    result = runner.invoke(annif.cli.cli, ["routes"])
-    assert re.search(r"static\s+GET\s+\/static\/\<path:filename\>", result.output)
-    assert not re.search(r"app.home\s+GET\s+\/", result.output)
+    assert run.called
 
 
-def test_routes_with_connexion_app():
-    # When using Connexion all endpoints exist
-    result = os.popen("python annif/cli.py routes").read()
-    assert re.search(r"static\s+GET\s+\/static\/<path:filename>", result)
-    assert re.search(r"app.home\s+GET\s+\/", result)
+def test_run_help():
+    result = runner.invoke(annif.cli.cli, ["run", "--help"])
+    assert not result.exception
+    assert result.exit_code == 0
+    assert "Run Annif in server mode for development." in result.output
 
 
 def test_completion_script_generation():