Fix security config race condition

.dockerignore

@@ -48,4 +48,5 @@ venv/
 
 # Dev files
 .pre-commit-config.yaml
+docker/docker-compose.minimal-config.yml
 .fides/

.fides/fides.toml

@@ -30,14 +30,14 @@ ssl = false
 ssl_cert_reqs = "required"
 
 [security]
-app_encryption_key = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
 cors_origins = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",]
 encoding = "UTF-8"
+root_username = "root_user"
+root_password = "Testpassword1!"
+app_encryption_key = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
 oauth_root_client_id = "fidesadmin"
 oauth_root_client_secret = "fidesadminsecret"
 drp_jwt_secret = "secret"
-root_username = "root_user"
-root_password = "Testpassword1!"
 
 [execution]
 masking_strict = true

.github/workflows/backend_checks.yml

@@ -71,9 +71,9 @@ jobs:
       - name: Run Static Check
         run: nox -s ${{ matrix.session_name }}
 
-  ################
-  ## Safe Tests ##
-  ################
+  #################
+  ## Safe Checks ##
+  #################
   Safe-Tests:
     needs: Build
 
@@ -87,6 +87,7 @@ jobs:
           - "check_fides_annotations"
           - "fides_db_scan"
           - "docs_check"
+          - "minimal_config_startup"
         exclude:
           # Not all jobs need to run against the Python matrix
           - python_version: "3.8.14"
@@ -104,6 +105,11 @@ jobs:
           - python_version: "3.9.14"
             test_selection: "docs_check"
 
+          - python_version: "3.8.14"
+            test_selection: "minimal_config_startup"
+          - python_version: "3.9.14"
+            test_selection: "minimal_config_startup"
+
     runs-on: ubuntu-latest
     timeout-minutes: 15
     continue-on-error: true
@@ -120,6 +126,12 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
+      - name: Set Up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: ${{ env.DEFAULT_PYTHON_VERSION }}
+          cache: "pip"
+
       - name: Install Nox
         run: pip install nox>=2022
 

CHANGELOG.md

@@ -49,6 +49,7 @@ The types of changes are:
 * Show a helpful error message if Docker daemon is not running during "fides deploy" [#1694](https://github.com/ethyca/fides/pull/1694)
 * Allow users to query their own permissions, including root user. [#1698](https://github.com/ethyca/fides/pull/1698)
 * Single-select taxonomy fields legal basis and special category can be cleared. [#1712](https://github.com/ethyca/fides/pull/1712)
+* Fixes the issue where the security config is not properly loading from environment variables. [#1718](https://github.com/ethyca/fides/pull/1718)
 * Correctly handle response from adobe jwt auth endpoint as milliseconds, rather than seconds. [#1754](https://github.com/ethyca/fides/pull/1754)
 
 ### Security

docker-compose.yml

@@ -100,8 +100,14 @@ services:
     ports:
       - "8000:8000"
     environment:
-      - FIDES__DEV_MODE=True
-      - FIDES__CLI__ANALYTICS_ID=${FIDES__CLI__ANALYTICS_ID:-}
+      FIDES__DEV_MODE: True
+      FIDES__CLI__ANALYTICS_ID: ${FIDES__CLI__ANALYTICS_ID:-}
+
+      # Required security env vars
+      FIDES__SECURITY__APP_ENCRYPTION_KEY: OLMkv91j8DHiDAULnK5Lxx3kSCov30b3
+      FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID: fidesadmin
+      FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET: fidesadminsecret
+      FIDES__SECURITY__DRP_JWT_SECRET: secret
 
   worker:
     image: ethyca/fides:local

docker/docker-compose.minimal-config.yml

@@ -0,0 +1,67 @@
+services:
+  fides:
+    image: ethyca/fides:local
+    command: uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src fides.api.main:app
+    healthcheck:
+      test: [ "CMD", "curl", "-f", "http://0.0.0.0:8080/health" ]
+      interval: 20s
+      timeout: 5s
+      retries: 10
+    ports:
+      - "8080:8080"
+    depends_on:
+      fides-db:
+        condition: service_healthy
+      redis:
+        condition: service_started
+    expose:
+      - 8080
+    environment:
+      FIDES__DATABASE__SERVER: "fides-db"
+      FIDES__DATABASE__USER: "postgres"
+      FIDES__DATABASE__PASSWORD: "fides"
+      FIDES__DATABASE__PORT: "5432"
+      FIDES__DATABASE__DB: "fides"
+      FIDES__USER__ANALYTICS_OPT_OUT: "True"
+      FIDES__SECURITY__APP_ENCRYPTION_KEY: "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
+      FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID: "fidesadmin"
+      FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET: "fidesadminsecret"
+      FIDES__SECRUITY__DRP_JWT_SECRET: "secret"
+
+  fides-db:
+    image: postgres:12
+    healthcheck:
+      test: [ "CMD-SHELL", "pg_isready -U postgres" ]
+      interval: 15s
+      timeout: 5s
+      retries: 5
+    volumes:
+      - postgres:/var/lib/postgresql/data
+    expose:
+      - 5432
+    ports:
+      - "5432:5432"
+    environment:
+      POSTGRES_USER: "postgres"
+      POSTGRES_PASSWORD: "fides"
+      POSTGRES_DB: "fides"
+    deploy:
+      placement:
+        constraints:
+          - node.labels.fides.app-db-data == true
+
+  redis:
+    image: "redis:6.2.5-alpine"
+    command: redis-server --requirepass testpassword
+    environment:
+      - REDIS_PASSWORD=testpassword
+    expose:
+      - 6379
+    ports:
+      - "0.0.0.0:6379:6379"
+
+volumes:
+  postgres: null
+
+networks:
+  fides_network:

noxfiles/ci_nox.py

@@ -103,11 +103,18 @@ def check_install(session: nox.Session) -> None:
     """Check that fides installs works correctly."""
     session.install(".")
 
+    REQUIRED_ENV_VARS = {
+        "FIDES__SECURITY__APP_ENCRYPTION_KEY": "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3",
+        "FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID": "fidesadmin",
+        "FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET": "fidesadminsecret",
+        "FIDES__SECURITY__DRP_JWT_SECRET": "secret",
+    }
+
     run_command = ("fides", "--version")
-    session.run(*run_command)
+    session.run(*run_command, env=REQUIRED_ENV_VARS)
 
     run_command = ("python", "-c", "from fides.api.main import start_webserver")
-    session.run(*run_command)
+    session.run(*run_command, env=REQUIRED_ENV_VARS)
 
 
 @nox.session()
@@ -134,6 +141,27 @@ def fides_db_scan(session: nox.Session) -> None:
     session.run(*run_command, external=True)
 
 
+@nox.session()
+def minimal_config_startup(session: nox.Session) -> None:
+    """
+    Check that the server can start successfully with a minimal
+    configuration set through environment vairables.
+    """
+    session.notify("teardown")
+    compose_file = "docker/docker-compose.minimal-config.yml"
+    start_command = (
+        "docker",
+        "compose",
+        "-f",
+        compose_file,
+        "up",
+        "--wait",
+        "-d",
+        IMAGE_NAME,
+    )
+    session.run(*start_command, external=True)
+
+
 # Pytest
 @nox.session()
 @nox.parametrize(

pyproject.toml

@@ -58,7 +58,8 @@ module = [
   "sqlalchemy.future.*",
   "sqlalchemy_utils.*",
   "twilio.*",
-  "uvicorn.*"
+  "uvicorn.*",
+  "validators.*",
 ]
 ignore_missing_imports = true
 
@@ -201,4 +202,4 @@ markers = [
     "integration_firebase_auth",
     "unit_saas"
 ]
-asyncio_mode = "auto"
+asyncio_mode = "auto"

requirements.txt

@@ -27,7 +27,7 @@ pandas==1.4.3
 passlib[bcrypt]==1.7.4
 plotly==5.4
 psycopg2-binary==2.9.3
-pydantic<1.10.0
+pydantic<1.10.2
 pydash==5.1.1
 PyJWT==2.4.0
 pymongo==3.13.0
@@ -44,4 +44,5 @@ SQLAlchemy-Utils==0.38.3
 toml>=0.10.1
 twilio==7.15.0
 Unidecode==1.3.4
+validators==0.20.0
 versioneer==0.19

src/fides/__init__.py

@@ -1,6 +1,11 @@
 """Fides CLI"""
 
+from fides.ctl.core.config import get_config
+
 from ._version import get_versions
 
+# Load the config here as a work around to the timing issue with environment variables
+get_config()
+
 __version__ = get_versions()["version"]
 del get_versions

src/fides/api/ops/api/v1/endpoints/drp_endpoints.py

@@ -48,6 +48,7 @@
 logger = logging.getLogger(__name__)
 router = APIRouter(tags=["DRP"], prefix=urls.V1_URL_PREFIX)
 CONFIG = get_config()
+
 EMBEDDED_EXECUTION_LOG_LIMIT = 50
 
 
@@ -67,7 +68,7 @@ async def create_drp_privacy_request(
     a corresponding Fidesops PrivacyRequest
     """
 
-    jwt_key: str = CONFIG.security.drp_jwt_secret
+    jwt_key = CONFIG.security.drp_jwt_secret
     if jwt_key is None:
         raise HTTPException(
             status_code=HTTP_500_INTERNAL_SERVER_ERROR,

src/fides/api/ops/api/v1/endpoints/oauth_endpoints.py

@@ -93,6 +93,7 @@ async def acquire_access_token(
         raise AuthenticationFailure(detail="Authentication Failure")
 
     logger.info("Creating access token")
+
     access_code = client_detail.create_access_code_jwe(
         CONFIG.security.app_encryption_key
     )

src/fides/api/ops/models/privacy_request.py

@@ -56,7 +56,6 @@
 from fides.api.ops.schemas.masking.masking_secrets import MaskingSecretCache
 from fides.api.ops.schemas.redis_cache import Identity
 from fides.api.ops.tasks import celery_app
-from fides.api.ops.util.identity_verification import IdentityVerificationMixin
 from fides.api.ops.util.cache import (
     FidesopsRedis,
     get_all_cache_keys_for_privacy_request,
@@ -69,6 +68,7 @@
 )
 from fides.api.ops.util.collection_util import Row
 from fides.api.ops.util.constants import API_DATE_FORMAT
+from fides.api.ops.util.identity_verification import IdentityVerificationMixin
 from fides.ctl.core.config import get_config
 
 logger = logging.getLogger(__name__)

src/fides/api/ops/schemas/encryption_request.py

@@ -17,6 +17,7 @@ class AesEncryptionRequest(BaseModel):
     @validator("key")
     def validate_key(cls, v: str) -> bytes:
         """Convert string into bytes and verify this is the correct length"""
+
         key = v.encode(CONFIG.security.encoding)
         verify_encryption_key(key)
         return key

src/fides/api/ops/tasks/storage.py

@@ -106,7 +106,6 @@ def create_presigned_url_for_s3(
     :param object_name: string
     :return: Presigned URL as string.
     """
-
     response = s3_client.generate_presigned_url(
         "get_object",
         Params={"Bucket": bucket_name, "Key": object_name},

src/fides/api/ops/util/identity_verification.py

@@ -2,14 +2,9 @@
 from typing import Optional
 
 from fides.api.ops.common_exceptions import IdentityVerificationException
-from fides.api.ops.util.cache import (
-    FidesopsRedis,
-    get_cache,
-)
-
+from fides.api.ops.util.cache import FidesopsRedis, get_cache
 from fides.ctl.core.config import get_config
 
-
 logger = logging.getLogger(__name__)
 CONFIG = get_config()