diff --git a/.github/CONTRIBUTING.md b/.github/CONTRIBUTING.md index 9ec0a14..67cc42c 100644 --- a/.github/CONTRIBUTING.md +++ b/.github/CONTRIBUTING.md @@ -208,7 +208,7 @@ After saving files, changes need to be committed to the Git repository. - Python 3 (modern Python) was used. Python 2 (legacy Python) is nearing its [end of life](https://pythonclock.org/). - Python code was linted with [Flake8](https://flake8.readthedocs.io/en/latest/) and autoformatted with [Black](https://black.readthedocs.io/en/stable/). - Git pre-commit hooks have been installed for the [Black autoformatter](https://black.readthedocs.io/en/stable/version_control_integration.html) and [Flake8 linter](https://flake8.pycqa.org/en/latest/user/using-hooks.html). -- Within Python modules, `import` statements are organized automatically by [isort](https://timothycrosley.github.io/isort/). +- Within Python modules, `import` statements are organized automatically by [isort](https://pycqa.github.io/isort/). - In general, a [Pythonic](https://docs.python-guide.org/writing/style/) code style following the [Zen of Python](https://www.python.org/dev/peps/pep-0020/) was used. [Foolish consistency](https://pep8.org) was avoided. ### Python virtual environment tools diff --git a/README.md b/README.md index 1c78d41..b5fc6ca 100644 --- a/README.md +++ b/README.md @@ -27,6 +27,8 @@ Brendon Smith ([br3ndonland](https://github.com/br3ndonland/)) - [Logging](#logging) - [Development](#development) - [Code style](#code-style) + - [Testing with pytest](#testing-with-pytest) + - [GitHub Actions workflows](#github-actions-workflows) - [Building development images](#building-development-images) - [Running development containers](#running-development-containers) - [Configuring Docker for GitHub Container Registry](#configuring-docker-for-github-container-registry) @@ -338,7 +340,7 @@ For more information on Python logging configuration, see the [Python `logging` ### Code style - Python code is formatted with [Black](https://black.readthedocs.io/en/stable/). Configuration for Black is stored in _[pyproject.toml](pyproject.toml)_. -- Python imports are organized automatically with [isort](https://timothycrosley.github.io/isort/). +- Python imports are organized automatically with [isort](https://pycqa.github.io/isort/). - The isort package organizes imports in three sections: 1. Standard library 2. Dependencies @@ -347,6 +349,48 @@ For more information on Python logging configuration, see the [Python `logging` - You can run isort from the command line with `poetry run isort .`. - Configuration for isort is stored in _[pyproject.toml](pyproject.toml)_. - Other web code (JSON, Markdown, YAML) is formatted with [Prettier](https://prettier.io/). +- Code style is enforced with [pre-commit](https://pre-commit.com/), which runs [Git hooks](https://www.git-scm.com/book/en/v2/Customizing-Git-Git-Hooks). + + - Configuration is stored in _[.pre-commit-config.yaml](.pre-commit-config.yaml)_. + - Pre-commit can run locally before each commit (hence "pre-commit"), or on different Git events like `pre-push`. + - Pre-commit is installed in the Poetry environment. To use: + + ```sh + # after running `poetry install` + path/to/inboard + ❯ poetry shell + + # install hooks that run before each commit + path/to/inboard + .venv ❯ pre-commit install + + # and/or install hooks that run before each push + path/to/inboard + .venv ❯ pre-commit install --hook-type pre-push + ``` + + - Pre-commit is also useful as a CI tool. The [hooks](.github/workflows/hooks.yml) GitHub Actions workflow runs pre-commit hooks with [GitHub Actions](https://github.com/features/actions). + +### Testing with pytest + +- Tests are in the _tests/_ directory. +- Run tests by [invoking `pytest` from the command-line](https://docs.pytest.org/en/stable/usage.html) in the root directory of the repo. +- [pytest](https://docs.pytest.org/en/latest/) features used include: + - [fixtures](https://docs.pytest.org/en/latest/fixture.html) + - [monkeypatch](https://docs.pytest.org/en/latest/monkeypatch.html) + - [parametrize](https://docs.pytest.org/en/latest/parametrize.html) + - [`tmp_path`](https://docs.pytest.org/en/latest/tmpdir.html) +- [pytest plugins](https://docs.pytest.org/en/stable/plugins.html) include: + - [pytest-cov](https://github.com/pytest-dev/pytest-cov) + - [pytest-mock](https://github.com/pytest-dev/pytest-mock) +- [pytest configuration](https://docs.pytest.org/en/stable/customize.html) is in _[pyproject.toml](pyproject.toml)_. +- [FastAPI testing](https://fastapi.tiangolo.com/tutorial/testing/) and [Starlette testing](https://www.starlette.io/testclient/) rely on the [Starlette `TestClient`](https://www.starlette.io/testclient/), which uses [Requests](https://requests.readthedocs.io/en/master/) under the hood. +- Test coverage results are reported when invoking `pytest` from the command-line. To see interactive HTML coverage reports, invoke pytest with `pytest --cov-report=html`. +- Test coverage reports are generated within GitHub Actions workflows by [pytest-cov](https://github.com/pytest-dev/pytest-cov) with [coverage.py](https://github.com/nedbat/coveragepy), and uploaded to [Codecov](https://docs.codecov.io/docs) using [codecov/codecov-action](https://github.com/marketplace/actions/codecov). Codecov is then integrated into pull requests with the [Codecov GitHub app](https://github.com/marketplace/codecov). + +### GitHub Actions workflows + +[GitHub Actions](https://github.com/features/actions) is a continuous integration/continuous deployment (CI/CD) service that runs on GitHub repos. It replaces other services like Travis CI. Actions are grouped into workflows and stored in _.github/workflows_. See my [GitHub Actions Gist](https://gist.github.com/br3ndonland/f9c753eb27381f97336aa21b8d932be6) for more info on GitHub Actions. ### Building development images diff --git a/inboard/__init__.py b/inboard/__init__.py index 525b209..48bd4a8 100644 --- a/inboard/__init__.py +++ b/inboard/__init__.py @@ -5,10 +5,10 @@ from importlib.metadata import version -def package_version() -> str: +def package_version(package: str = __package__) -> str: """Calculate version number based on pyproject.toml""" try: - return version(__package__) + return version(package) except Exception: return "Package not found." diff --git a/inboard/app/base/main.py b/inboard/app/base/main.py index eaa7284..36385a4 100644 --- a/inboard/app/base/main.py +++ b/inboard/app/base/main.py @@ -24,11 +24,14 @@ async def __call__( } ) version = f"{sys.version_info.major}.{sys.version_info.minor}" - server = "Uvicorn" if bool(os.getenv("WITH_RELOAD")) else "Uvicorn, Gunicorn," + process_manager = os.getenv("PROCESS_MANAGER") + if process_manager not in ["gunicorn", "uvicorn"]: + raise NameError("Process manager needs to be either uvicorn or gunicorn.") + server = "Uvicorn" if process_manager == "uvicorn" else "Uvicorn, Gunicorn," message = f"Hello World, from {server} and Python {version}!" response: Dict = {"type": "http.response.body", "body": message.encode("utf-8")} await send(response) return response -app = App +app: Callable = App diff --git a/inboard/app/utilities.py b/inboard/app/utilities.py index 54524aa..39e334e 100644 --- a/inboard/app/utilities.py +++ b/inboard/app/utilities.py @@ -1,5 +1,4 @@ import base64 -import binascii import os from secrets import compare_digest from typing import Tuple, Union @@ -27,18 +26,18 @@ async def authenticate( scheme, credentials = auth.split() decoded = base64.b64decode(credentials).decode("ascii") username, _, password = decoded.partition(":") - except (ValueError, UnicodeDecodeError, binascii.Error): - raise AuthenticationError("Unable to parse basic auth credentials") - correct_username = compare_digest( - username, str(os.getenv("BASIC_AUTH_USERNAME", "test_username")) - ) - correct_password = compare_digest( - password, - str(os.getenv("BASIC_AUTH_PASSWORD", "plunge-germane-tribal-pillar")), - ) - if not (correct_username and correct_password): - raise AuthenticationError("Invalid basic auth credentials") - return AuthCredentials(["authenticated"]), SimpleUser(username) + correct_username = compare_digest( + username, str(os.getenv("BASIC_AUTH_USERNAME", "test_username")) + ) + correct_password = compare_digest( + password, + str(os.getenv("BASIC_AUTH_PASSWORD", "plunge-germane-tribal-pillar")), + ) + if not (correct_username and correct_password): + raise AuthenticationError("Invalid basic auth credentials") + return AuthCredentials(["authenticated"]), SimpleUser(username) + except Exception: + raise def basic_auth(credentials: HTTPBasicCredentials = Depends(HTTPBasic())) -> str: diff --git a/inboard/gunicorn_conf.py b/inboard/gunicorn_conf.py index 205423f..727e09d 100644 --- a/inboard/gunicorn_conf.py +++ b/inboard/gunicorn_conf.py @@ -1,17 +1,15 @@ -import json import multiprocessing import os -from pathlib import Path from inboard.start import configure_logging -workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1") +# Gunicorn setup max_workers_str = os.getenv("MAX_WORKERS") +web_concurrency_str = os.getenv("WEB_CONCURRENCY", None) +workers_per_core_str = os.getenv("WORKERS_PER_CORE", "1") use_max_workers = None -if max_workers_str: +if max_workers_str and int(max_workers_str) > 0: use_max_workers = int(max_workers_str) -web_concurrency_str = os.getenv("WEB_CONCURRENCY", None) - host = os.getenv("HOST", "0.0.0.0") port = os.getenv("PORT", "80") bind_env = os.getenv("BIND", None) @@ -20,9 +18,8 @@ cores = multiprocessing.cpu_count() workers_per_core = float(workers_per_core_str) default_web_concurrency = workers_per_core * cores -if web_concurrency_str: +if web_concurrency_str and int(web_concurrency_str) > 0: web_concurrency = int(web_concurrency_str) - assert web_concurrency > 0 else: web_concurrency = max(int(default_web_concurrency), 2) if use_max_workers: @@ -36,14 +33,9 @@ keepalive_str = os.getenv("KEEP_ALIVE", "5") # Gunicorn config variables -try: - logconfig_dict = configure_logging( - logging_conf=os.getenv("LOGGING_CONF", "inboard.logging_conf") - ) -except Exception as e: - if use_loglevel == "debug": - msg = "Error loading logging config with Gunicorn:" - print(f"[{Path(__file__).stem}] {msg} {e}") +logconfig_dict = configure_logging( + logging_conf=os.getenv("LOGGING_CONF", "inboard.logging_conf") +) loglevel = use_loglevel workers = web_concurrency bind = use_bind @@ -53,22 +45,3 @@ graceful_timeout = int(graceful_timeout_str) timeout = int(timeout_str) keepalive = int(keepalive_str) - -log_data = { - # General - "host": host, - "port": port, - "use_max_workers": use_max_workers, - "workers_per_core": workers_per_core, - # Gunicorn - "loglevel": loglevel, - "workers": workers, - "bind": bind, - "graceful_timeout": graceful_timeout, - "timeout": timeout, - "keepalive": keepalive, - "errorlog": errorlog, - "accesslog": accesslog, -} -if loglevel == "debug": - print(f"[{Path(__file__).stem}] Custom configuration:", json.dumps(log_data)) diff --git a/inboard/start.py b/inboard/start.py index 91e9a4f..ea56705 100644 --- a/inboard/start.py +++ b/inboard/start.py @@ -18,8 +18,6 @@ def set_conf_path(module_stem: str) -> str: ) if conf_var and Path(conf_var).is_file(): conf_path = conf_var - elif Path(f"/app/inboard/{module_stem}_conf.py").is_file(): - conf_path = f"/app/inboard/{module_stem}_conf.py" else: raise FileNotFoundError(f"Unable to find {conf_var}") return conf_path @@ -40,8 +38,6 @@ def configure_logging( if spec: logging_conf_module = importlib.util.module_from_spec(spec) spec.loader.exec_module(logging_conf_module) # type: ignore[union-attr] - else: - raise ImportError(f"Unable to import {logging_conf}.") if hasattr(logging_conf_module, "LOGGING_CONFIG"): logging_conf_dict = getattr(logging_conf_module, "LOGGING_CONFIG") else: @@ -74,22 +70,18 @@ def set_app_module(logger: Logger = logging.getLogger()) -> str: def run_pre_start_script(logger: Logger = logging.getLogger()) -> str: """Run a pre-start script at the provided path.""" - try: - logger.debug("Checking for pre-start script.") - pre_start_path = os.getenv("PRE_START_PATH", "/app/inboard/app/prestart.py") - if Path(pre_start_path).is_file(): - process = "python" if Path(pre_start_path).suffix == ".py" else "sh" - run_message = f"Running pre-start script with {process} {pre_start_path}." - logger.debug(run_message) - subprocess.run([process, pre_start_path]) - message = f"Ran pre-start script with {process} {pre_start_path}." - else: - message = "No pre-start script found." - except Exception as e: - message = f"Error from pre-start script: {e}." - finally: - logger.debug(message) - return message + logger.debug("Checking for pre-start script.") + pre_start_path = os.getenv("PRE_START_PATH", "/app/inboard/app/prestart.py") + if Path(pre_start_path).is_file(): + process = "python" if Path(pre_start_path).suffix == ".py" else "sh" + run_message = f"Running pre-start script with {process} {pre_start_path}." + logger.debug(run_message) + subprocess.run([process, pre_start_path]) + message = f"Ran pre-start script with {process} {pre_start_path}." + else: + message = "No pre-start script found." + logger.debug(message) + return message def start_server( @@ -126,13 +118,11 @@ def start_server( if __name__ == "__main__": - logger = logging.getLogger() - logging_conf_dict = configure_logging(logger=logger) - app_module = set_app_module(logger=logger) - run_pre_start_script(logger=logger) - start_server( + logger = logging.getLogger() # pragma: no cover + run_pre_start_script(logger=logger) # pragma: no cover + start_server( # pragma: no cover str(os.getenv("PROCESS_MANAGER", "gunicorn")), - app_module=app_module, + app_module=set_app_module(logger=logger), logger=logger, - logging_conf_dict=logging_conf_dict, + logging_conf_dict=configure_logging(logger=logger), ) diff --git a/tests/app/test_main.py b/tests/app/test_main.py index 5bd6787..6de3e91 100644 --- a/tests/app/test_main.py +++ b/tests/app/test_main.py @@ -1,7 +1,9 @@ +import os import re from typing import Dict, List import pytest +from _pytest.monkeypatch import MonkeyPatch from fastapi import FastAPI from fastapi.testclient import TestClient from starlette.applications import Starlette @@ -76,6 +78,42 @@ class TestEndpoints: - https://docs.pytest.org/en/latest/parametrize.html """ + def test_get_asgi_uvicorn( + self, client_asgi: TestClient, monkeypatch: MonkeyPatch + ) -> None: + """Test `GET` request to base ASGI app set for Uvicorn without Gunicorn.""" + monkeypatch.setenv("PROCESS_MANAGER", "uvicorn") + monkeypatch.setenv("WITH_RELOAD", "false") + assert os.getenv("PROCESS_MANAGER") == "uvicorn" + assert os.getenv("WITH_RELOAD") == "false" + response = client_asgi.get("/") + assert response.status_code == 200 + assert response.text == "Hello World, from Uvicorn and Python 3.8!" + + def test_get_asgi_uvicorn_gunicorn( + self, client_asgi: TestClient, monkeypatch: MonkeyPatch + ) -> None: + """Test `GET` request to base ASGI app set for Uvicorn with Gunicorn.""" + monkeypatch.setenv("PROCESS_MANAGER", "gunicorn") + monkeypatch.setenv("WITH_RELOAD", "false") + assert os.getenv("PROCESS_MANAGER") == "gunicorn" + assert os.getenv("WITH_RELOAD") == "false" + response = client_asgi.get("/") + assert response.status_code == 200 + assert response.text == "Hello World, from Uvicorn, Gunicorn, and Python 3.8!" + + def test_get_asgi_incorrect_process_manager( + self, client_asgi: TestClient, monkeypatch: MonkeyPatch + ) -> None: + """Test `GET` request to base ASGI app with incorrect `PROCESS_MANAGER`.""" + monkeypatch.setenv("PROCESS_MANAGER", "incorrect") + monkeypatch.setenv("WITH_RELOAD", "false") + assert os.getenv("PROCESS_MANAGER") == "incorrect" + assert os.getenv("WITH_RELOAD") == "false" + with pytest.raises(NameError) as e: + client_asgi.get("/") + assert str(e) == "Process manager needs to be either uvicorn or gunicorn." + def test_get_root(self, clients: List[TestClient]) -> None: """Test a `GET` request to the root endpoint.""" for client in clients: @@ -115,6 +153,20 @@ def test_gets_with_basic_auth_incorrect( response = client.get(endpoint, auth=basic_auth) assert response.status_code == 200 + @pytest.mark.parametrize("endpoint", ["/health", "/status"]) + def test_gets_with_starlette_auth_exception( + self, clients: List[TestClient], endpoint: str + ) -> None: + """Test Starlette `GET` requests with incorrect Basic Auth credentials.""" + starlette_client = clients[1] + assert isinstance(starlette_client.app, Starlette) + response = starlette_client.get(endpoint, auth=("user", "pass")) + assert response.status_code in [401, 403] + assert response.json() == { + "detail": "Incorrect username or password", + "error": "Invalid basic auth credentials", + } + def test_get_status_message( self, basic_auth: tuple, diff --git a/tests/conftest.py b/tests/conftest.py index 3d75738..bee5a31 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ from inboard import gunicorn_conf as gunicorn_conf_module from inboard import logging_conf as logging_conf_module from inboard.app import prestart as pre_start_module +from inboard.app.base.main import app as base_app from inboard.app.fastapibase.main import app as fastapi_app from inboard.app.starlettebase.main import app as starlette_app @@ -39,6 +40,12 @@ def basic_auth( return username, password +@pytest.fixture(scope="session") +def client_asgi() -> TestClient: + """Instantiate test client classes.""" + return TestClient(base_app) + + @pytest.fixture(scope="session") def clients() -> List[TestClient]: """Instantiate test client classes.""" diff --git a/tests/test_start.py b/tests/test_start.py index 5334b57..88bf975 100644 --- a/tests/test_start.py +++ b/tests/test_start.py @@ -65,6 +65,17 @@ def test_configure_logging_module( f"Logging dict config loaded from {logging_conf_module_path}." ) + def test_configure_logging_module_incorrect( + self, mock_logger: logging.Logger + ) -> None: + with pytest.raises(ImportError): + start.configure_logging(logger=mock_logger, logging_conf="no.module.here") + import_error_msg = "Unable to import no.module.here." + logger_error_msg = "Error when configuring logging:" + mock_logger.debug.assert_called_once_with( # type: ignore[attr-defined] + f"{logger_error_msg} {import_error_msg}." + ) + def test_configure_logging_tmp_file( self, logging_conf_tmp_file_path: Path, mock_logger: logging.Logger ) -> None: @@ -367,22 +378,64 @@ def test_start_server_uvicorn_gunicorn( mock_logger: logging.Logger, mocker: MockerFixture, monkeypatch: MonkeyPatch, + ) -> None: + """Test `start.start_server` with Uvicorn managed by Gunicorn.""" + assert os.getenv("GUNICORN_CONF", str(gunicorn_conf_path)) + monkeypatch.setenv("LOG_LEVEL", "debug") + monkeypatch.setenv("PROCESS_MANAGER", "gunicorn") + assert os.getenv("LOG_LEVEL") == "debug" + assert os.getenv("PROCESS_MANAGER") == "gunicorn" + start.start_server( + str(os.getenv("PROCESS_MANAGER")), + app_module=app_module, + logger=mock_logger, + logging_conf_dict=logging_conf_dict, + ) + mock_logger.debug.assert_called_once_with("Running Uvicorn with Gunicorn.") # type: ignore[attr-defined] # noqa: E501 + + @pytest.mark.parametrize( + "app_module", + [ + "inboard.app.base.main:app", + "inboard.app.fastapibase.main:app", + "inboard.app.starlettebase.main:app", + ], + ) + def test_start_server_uvicorn_gunicorn_custom_config( + self, + app_module: str, + gunicorn_conf_path: Path, + logging_conf_dict: Dict[str, Any], + mock_logger: logging.Logger, + mocker: MockerFixture, + monkeypatch: MonkeyPatch, ) -> None: """Test `start.start_server` with Uvicorn managed by Gunicorn.""" assert os.getenv("GUNICORN_CONF", str(gunicorn_conf_path)) monkeypatch.setenv("LOG_FORMAT", "gunicorn") monkeypatch.setenv("LOG_LEVEL", "debug") + monkeypatch.setenv("MAX_WORKERS", "1") monkeypatch.setenv("PROCESS_MANAGER", "gunicorn") + monkeypatch.setenv("WEB_CONCURRENCY", "4") assert os.getenv("LOG_FORMAT") == "gunicorn" assert os.getenv("LOG_LEVEL") == "debug" + assert os.getenv("MAX_WORKERS") == "1" assert os.getenv("PROCESS_MANAGER") == "gunicorn" + assert os.getenv("WEB_CONCURRENCY") == "4" start.start_server( str(os.getenv("PROCESS_MANAGER")), app_module=app_module, logger=mock_logger, logging_conf_dict=logging_conf_dict, ) - mock_logger.debug.assert_called_once_with("Running Uvicorn with Gunicorn.") # type: ignore[attr-defined] # noqa: E501 + monkeypatch.delenv("WEB_CONCURRENCY") + start.start_server( + str(os.getenv("PROCESS_MANAGER")), + app_module=app_module, + logger=mock_logger, + logging_conf_dict=logging_conf_dict, + ) + mock_logger.debug.assert_called_with("Running Uvicorn with Gunicorn.") # type: ignore[attr-defined] # noqa: E501 def test_start_server_uvicorn_incorrect_module( self, diff --git a/tests/test_version.py b/tests/test_version.py index 50f2e70..4f9cc08 100644 --- a/tests/test_version.py +++ b/tests/test_version.py @@ -1,5 +1,18 @@ -from inboard import __version__ +from inboard import __version__, package_version + +current_version = "0.4.1" + + +def test_package_version() -> None: + """Test package version calculation.""" + assert package_version() == current_version + + +def test_package_version_not_found() -> None: + """Test package version calculation when package is not installed.""" + assert package_version(package="incorrect") == "Package not found." def test_version() -> None: - assert __version__ == "0.4.1" + """Test package version number.""" + assert __version__ == current_version