From d65ccb5323a0611b9e668d87210e14f54244eb5a Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 14:55:15 -0400 Subject: [PATCH 01/11] Add static build dir to gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index abd244b45b4..d28c7c8572e 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,7 @@ fidesapi/src/main/resources/application.conf docs/fides/docs/api/openapi.json docs/fides/docs/schemas/config_schema.json +fidesapi/build/static ## generic files to ignore *~ From c334f5e7ac5fd9b232912601f0e99a7cf78d4c59 Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 16:49:10 -0400 Subject: [PATCH 02/11] Proof of concept serving static files --- src/fidesapi/main.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/fidesapi/main.py b/src/fidesapi/main.py index 5afa5a8c5af..8612f01f893 100644 --- a/src/fidesapi/main.py +++ b/src/fidesapi/main.py @@ -5,9 +5,12 @@ from datetime import datetime from enum import Enum from logging import WARNING +from pathlib import Path from typing import Callable, Dict from fastapi import FastAPI, Request, Response, status +from fastapi.responses import FileResponse +from fastapi.staticfiles import StaticFiles from loguru import logger as log from uvicorn import Config, Server @@ -18,7 +21,10 @@ from fidesapi.utils.logger import setup as setup_logging from fidesctl.core.config import FidesctlConfig, get_config +WEBAPP_DIRECTORY = Path("src/fidesapi/build/static") + app = FastAPI(title="fidesctl") +app.mount("/static", StaticFiles(directory=WEBAPP_DIRECTORY), name="static") CONFIG: FidesctlConfig = get_config() @@ -114,6 +120,24 @@ async def db_action(action: DBActions) -> Dict: return {"data": {"message": f"Fidesctl database {action_text}"}} +@app.get("/") +def read_index(request: Request) -> Response: + path = WEBAPP_DIRECTORY / "index.html" + return FileResponse(path) + + +@app.get("/{catchall:path}", response_class=FileResponse) +def read_other_paths(request: Request) -> FileResponse: + # check first if requested file exists + path = request.path_params["catchall"] + file = WEBAPP_DIRECTORY / Path(path) + if file.exists(): + return FileResponse(file) + + # otherwise return the index + return FileResponse(WEBAPP_DIRECTORY / "index.html") + + def start_webserver() -> None: "Run the webserver." server = Server(Config(app, host="0.0.0.0", port=8080, log_level=WARNING)) From 64fdefc3b2fdebe3606c3fed39fbfbfe562b2edf Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 17:57:32 -0400 Subject: [PATCH 03/11] Add tests for static file --- src/fidesapi/main.py | 4 +++- tests/core/test_api.py | 7 +++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/src/fidesapi/main.py b/src/fidesapi/main.py index 8612f01f893..9703807268b 100644 --- a/src/fidesapi/main.py +++ b/src/fidesapi/main.py @@ -120,8 +120,10 @@ async def db_action(action: DBActions) -> Dict: return {"data": {"message": f"Fidesctl database {action_text}"}} +# Configure the static file sink, adapted from https://github.com/tiangolo/fastapi/issues/130 +# Do this last since otherwise it will take over all paths @app.get("/") -def read_index(request: Request) -> Response: +def read_index() -> Response: path = WEBAPP_DIRECTORY / "index.html" return FileResponse(path) diff --git a/tests/core/test_api.py b/tests/core/test_api.py index 2e40e9f7211..2748b847a26 100644 --- a/tests/core/test_api.py +++ b/tests/core/test_api.py @@ -179,3 +179,10 @@ def test_visualize(test_config: FidesctlConfig, resource_type: str) -> None: f"{test_config.cli.server_url}/{resource_type}/visualize/graphs" ) assert response.status_code == 200 + + +@pytest.mark.integration +def test_static_sink(test_config: FidesctlConfig) -> None: + """Make sure we are hosting something at / and not getting a 404""" + response = requests.get(f"{test_config.cli.server_url}") + assert response.status_code == 200 From 9f0d52e6605e68a5133756de8edc7370362ff047 Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 17:57:49 -0400 Subject: [PATCH 04/11] Add a placeholder for copying frontend files to Dockerfile --- Dockerfile | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Dockerfile b/Dockerfile index f899ec2b3a3..3698d63393d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,14 @@ FROM --platform=linux/amd64 python:3.8-slim-buster as base # Update pip in the base image since we'll use it everywhere RUN pip install -U pip +#################### +## Build frontend ## +#################### +FROM base as frontend +# Placeholder until we have a frontend app scaffolded +RUN echo "

Hello world!

" > /tmp/index.html + + ####################### ## Tool Installation ## ####################### @@ -97,3 +105,7 @@ FROM builder as prod # Install without a symlink RUN python setup.py sdist RUN pip install dist/fidesctl-*.tar.gz + +# Copy frontend build over +RUN mkdir -p src/fidesapi/build/static +COPY --from=frontend /tmp/index.html src/fidesapi/build/static/ From 463e224fefb85991464b97c4822864c5f184c282 Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 18:00:33 -0400 Subject: [PATCH 05/11] Add static file server to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1bbd67c7ae..35825a49cda 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,6 +43,7 @@ The types of changes are: * CHANGELOG.md file * On API server startup, in-use config values are logged at the DEBUG level * Send a usage analytics event upon execution of the `fidesctl init` command +* Host static files via fidesapi ### Developer Experience From 96cf93e18a19e5595c89e2e10f4191ca0c16a32e Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 18:04:50 -0400 Subject: [PATCH 06/11] Add docstrings --- src/fidesapi/main.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/fidesapi/main.py b/src/fidesapi/main.py index 9703807268b..d6c74e2b368 100644 --- a/src/fidesapi/main.py +++ b/src/fidesapi/main.py @@ -120,16 +120,21 @@ async def db_action(action: DBActions) -> Dict: return {"data": {"message": f"Fidesctl database {action_text}"}} -# Configure the static file sink, adapted from https://github.com/tiangolo/fastapi/issues/130 -# Do this last since otherwise it will take over all paths +# Configure the static file paths last since otherwise it will take over all paths @app.get("/") def read_index() -> Response: + """ + Return an index.html at the root path + """ path = WEBAPP_DIRECTORY / "index.html" return FileResponse(path) @app.get("/{catchall:path}", response_class=FileResponse) def read_other_paths(request: Request) -> FileResponse: + """ + Return related frontend files. Adapted from https://github.com/tiangolo/fastapi/issues/130 + """ # check first if requested file exists path = request.path_params["catchall"] file = WEBAPP_DIRECTORY / Path(path) From 81f1e07f53152d813ebba89250884b9c422ecae5 Mon Sep 17 00:00:00 2001 From: Allison King Date: Mon, 9 May 2022 19:24:41 -0400 Subject: [PATCH 07/11] Attempt to fix CI tests --- Dockerfile | 4 +++- tests/core/test_api.py | 7 ------- 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/Dockerfile b/Dockerfile index 3698d63393d..2c447e0b3fc 100644 --- a/Dockerfile +++ b/Dockerfile @@ -84,6 +84,9 @@ ENV PYTHONUNBUFFERED=TRUE # Enable detection of running within Docker ENV RUNNING_IN_DOCKER=TRUE +# Make a static files directory +RUN mkdir -p src/fidesapi/build/static + EXPOSE 8080 CMD ["fidesctl", "webserver"] @@ -107,5 +110,4 @@ RUN python setup.py sdist RUN pip install dist/fidesctl-*.tar.gz # Copy frontend build over -RUN mkdir -p src/fidesapi/build/static COPY --from=frontend /tmp/index.html src/fidesapi/build/static/ diff --git a/tests/core/test_api.py b/tests/core/test_api.py index 2748b847a26..2e40e9f7211 100644 --- a/tests/core/test_api.py +++ b/tests/core/test_api.py @@ -179,10 +179,3 @@ def test_visualize(test_config: FidesctlConfig, resource_type: str) -> None: f"{test_config.cli.server_url}/{resource_type}/visualize/graphs" ) assert response.status_code == 200 - - -@pytest.mark.integration -def test_static_sink(test_config: FidesctlConfig) -> None: - """Make sure we are hosting something at / and not getting a 404""" - response = requests.get(f"{test_config.cli.server_url}") - assert response.status_code == 200 From 9c97c61ccec08a19827761844c848c8acaf30301 Mon Sep 17 00:00:00 2001 From: ThomasLaPiana Date: Tue, 10 May 2022 14:37:03 -0500 Subject: [PATCH 08/11] create an index page on startup if it doesn't exist --- src/fidesapi/main.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/src/fidesapi/main.py b/src/fidesapi/main.py index d6c74e2b368..22f67133bbb 100644 --- a/src/fidesapi/main.py +++ b/src/fidesapi/main.py @@ -22,9 +22,9 @@ from fidesctl.core.config import FidesctlConfig, get_config WEBAPP_DIRECTORY = Path("src/fidesapi/build/static") +WEBAPP_INDEX = Path(WEBAPP_DIRECTORY / "index.html") app = FastAPI(title="fidesctl") -app.mount("/static", StaticFiles(directory=WEBAPP_DIRECTORY), name="static") CONFIG: FidesctlConfig = get_config() @@ -48,6 +48,18 @@ async def configure_db(database_url: str) -> None: await database.init_db(database_url) +@app.on_event("startup") +async def create_webapp_dir_if_not_exists() -> None: + """Creates the webapp directory if it doesn't exist.""" + + if not WEBAPP_INDEX.is_file(): + WEBAPP_DIRECTORY.mkdir(parents=True, exist_ok=True) + with open(WEBAPP_DIRECTORY / "index.html", "w") as index_file: + index_file.write("

Privacy is a Human Right!

") + + app.mount("/static", StaticFiles(directory=WEBAPP_DIRECTORY), name="static") + + @app.on_event("startup") async def setup_server() -> None: "Run all of the required setup steps for the webserver." From 147c7c5222f4290831e3432a6a8fce7487a28244 Mon Sep 17 00:00:00 2001 From: ThomasLaPiana Date: Tue, 10 May 2022 14:43:41 -0500 Subject: [PATCH 09/11] update the changelog --- CHANGELOG.md | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 35825a49cda..776b57d652c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,10 @@ The types of changes are: * Added dependabot to keep dependencies updated * Include a warning for any orphan datasets as part of the `apply` command. +### Changed + +* Comparing server and CLI versions ignores `.dirty` only differences, and is quiet on success when running general CLI commands + ### Developer Experience * Replaced `make` with `nox` @@ -35,6 +39,7 @@ The types of changes are: * Resolved a failure with populating applicable data subject rights to a data map * Updated `fideslog` to v1.1.5, resolving an issue where some exceptions thrown by the SDK were not handled as expected +* Host static files via fidesapi [#621](https://github.com/ethyca/fides/pull/621) ## [1.6.0](https://github.com/ethyca/fides/compare/1.5.3...1.6.0) - 2022-05-02 @@ -43,13 +48,11 @@ The types of changes are: * CHANGELOG.md file * On API server startup, in-use config values are logged at the DEBUG level * Send a usage analytics event upon execution of the `fidesctl init` command -* Host static files via fidesapi ### Developer Experience * added isort as a CI check * Include `tests/` in all static code checks (e.g. `mypy`, `pylint`) -* Comparing server and CLI versions ignores `.dirty` only differences, and is quiet on success when running general CLI commands ### Changed From 8a47f165da7d4860ad29105fbb08f1a8ebd38e6f Mon Sep 17 00:00:00 2001 From: Allison King Date: Tue, 10 May 2022 16:05:20 -0400 Subject: [PATCH 10/11] Consolidate calls to WEBAPP_INDEX --- src/fidesapi/main.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/fidesapi/main.py b/src/fidesapi/main.py index 22f67133bbb..ea9c18ad437 100644 --- a/src/fidesapi/main.py +++ b/src/fidesapi/main.py @@ -22,7 +22,7 @@ from fidesctl.core.config import FidesctlConfig, get_config WEBAPP_DIRECTORY = Path("src/fidesapi/build/static") -WEBAPP_INDEX = Path(WEBAPP_DIRECTORY / "index.html") +WEBAPP_INDEX = WEBAPP_DIRECTORY / "index.html" app = FastAPI(title="fidesctl") CONFIG: FidesctlConfig = get_config() @@ -138,8 +138,7 @@ def read_index() -> Response: """ Return an index.html at the root path """ - path = WEBAPP_DIRECTORY / "index.html" - return FileResponse(path) + return FileResponse(WEBAPP_INDEX) @app.get("/{catchall:path}", response_class=FileResponse) From 4210ee53b8196342c4ce6eb7f21f16cf0e258637 Mon Sep 17 00:00:00 2001 From: Allison King Date: Tue, 10 May 2022 16:12:08 -0400 Subject: [PATCH 11/11] Restore test --- tests/core/test_api.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tests/core/test_api.py b/tests/core/test_api.py index 2e40e9f7211..2748b847a26 100644 --- a/tests/core/test_api.py +++ b/tests/core/test_api.py @@ -179,3 +179,10 @@ def test_visualize(test_config: FidesctlConfig, resource_type: str) -> None: f"{test_config.cli.server_url}/{resource_type}/visualize/graphs" ) assert response.status_code == 200 + + +@pytest.mark.integration +def test_static_sink(test_config: FidesctlConfig) -> None: + """Make sure we are hosting something at / and not getting a 404""" + response = requests.get(f"{test_config.cli.server_url}") + assert response.status_code == 200