Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Set the fides dev environment to the prod security environment #2588

Merged
merged 24 commits into from
Feb 17, 2023
Merged
Show file tree
Hide file tree
Changes from 19 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
70428ed
Set the dev environment to default to the `prod` security environment
ThomasLaPiana Feb 14, 2023
5712a74
fix: update the fides_db_scan nox session to login first
ThomasLaPiana Feb 15, 2023
bc2de33
fix: add an autouse fixture that logs into the CLI
ThomasLaPiana Feb 15, 2023
f0551db
Merge branch 'main' into ThomasLaPiana-prod-env-default
ThomasLaPiana Feb 15, 2023
9c1ce4d
feat: generate the auth headers whenever the config is loaded, as opp…
ThomasLaPiana Feb 15, 2023
abe6416
fix: bad exception catch that prevented the app from starting
ThomasLaPiana Feb 15, 2023
8c66610
fix: added auth_header to test fixture that reset the db
ThomasLaPiana Feb 15, 2023
21b84e6
fix: add more auth headers to various tests
ThomasLaPiana Feb 15, 2023
070be40
fix: errant deletion causing a test regression
ThomasLaPiana Feb 15, 2023
9159ac1
docs: changelog update
ThomasLaPiana Feb 15, 2023
554f1ea
feat: proper error message for permissions check failure
ThomasLaPiana Feb 15, 2023
232363e
refactor: use 'docker exec' instead of 'docker compose run' where pos…
ThomasLaPiana Feb 15, 2023
dfb37fe
fix: static_checks
ThomasLaPiana Feb 15, 2023
380e6df
fix: TTY output for the exec commands
ThomasLaPiana Feb 15, 2023
23940a3
fix: more CI_ARGS replacement
ThomasLaPiana Feb 15, 2023
decd535
fix: add a test for coverage
ThomasLaPiana Feb 15, 2023
a9b7139
refactor: docker compose run -> exec in dev_nox
ThomasLaPiana Feb 15, 2023
abec978
refactor: add analytics opt out env var passthrough to exec and remov…
ThomasLaPiana Feb 15, 2023
661cb71
fix: update the dev shell to use an interactive exec session
ThomasLaPiana Feb 16, 2023
8a5616c
Update noxfiles/ci_nox.py
ThomasLaPiana Feb 17, 2023
aa8d050
fix: a small bug, add a bit of logging around login requirement
ThomasLaPiana Feb 17, 2023
09c3063
fix: test env regression
ThomasLaPiana Feb 17, 2023
7a0dd29
fix: test_env keep alive
ThomasLaPiana Feb 17, 2023
b01515a
docs: add a note about CLI login to the contrib docs
ThomasLaPiana Feb 17, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions .fides/fides.toml
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ app_encryption_key = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3"
oauth_root_client_id = "fidesadmin"
oauth_root_client_secret = "fidesadminsecret"
drp_jwt_secret = "secret"
env = "dev"
env = "prod"

[execution]
masking_strict = true
Expand All @@ -58,4 +58,4 @@ task_default_queue = "fides"
task_always_eager = true

[notifications]
notification_service_type = "mailgun"
notification_service_type = "mailgun"
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,7 @@ The types of changes are:
### Developer Experience

* Added new Cypress E2E smoke tests [#2241](https://github.com/ethyca/fides/pull/2241)
* Set the security environment of the fides dev setup to `prod` instead of `dev` [#2588](https://github.com/ethyca/fides/pull/2588)
* New command `nox -s e2e_test` which will spin up the test environment and run true E2E Cypress tests against it [#2417](https://github.com/ethyca/fides/pull/2417)
* Cypress E2E tests now run in CI and are reported to Cypress Cloud [#2417](https://github.com/ethyca/fides/pull/2417)

Expand Down
12 changes: 8 additions & 4 deletions noxfiles/ci_nox.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

import nox

from constants_nox import IMAGE_NAME, RUN, RUN_NO_DEPS, START_APP, WITH_TEST_CONFIG
from constants_nox import IMAGE_NAME, LOGIN, RUN_NO_DEPS, START_APP, WITH_TEST_CONFIG
from test_setup_nox import pytest_ctl, pytest_lib, pytest_ops
from utils_nox import install_requirements

Expand Down Expand Up @@ -125,16 +125,20 @@ def fides_db_scan(session: nox.Session) -> None:
"""Scan the fides application database to check for dataset discrepancies."""
session.notify("teardown")
session.run(*START_APP, external=True)
run_command = (
*RUN,
scan_command = (
"docker",
"container",
"exec",
"fides-fides-1",
ThomasLaPiana marked this conversation as resolved.
Show resolved Hide resolved
"fides",
"scan",
"dataset",
"db",
"--credentials-id",
"app_postgres",
)
session.run(*run_command, external=True)
session.run(*LOGIN, external=True)
session.run(*scan_command, external=True)


@nox.session()
Expand Down
39 changes: 27 additions & 12 deletions noxfiles/constants_nox.py
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@
# Image Names & Tags
REGISTRY = "ethyca"
IMAGE_NAME = "fides"
CONTAINER_NAME = "fides-fides-1"
COMPOSE_SERVICE_NAME = "fides"

# Image Names & Tags
Expand All @@ -38,27 +39,41 @@
# CI env variable is always set to true in Github Actions
# The else statement is required due to the way commmands are structured and is arbitrary.
CI_ARGS = "-T" if getenv("CI") else "--user=root"
CI_ARGS_EXEC = "-t" if not getenv("CI") else "--user=root"

# If FIDES__CLI__ANALYTICS_ID is set in the local environment, use its value as the analytics_id
ANALYTICS_ID_OVERRIDE = ("-e", "FIDES__CLI__ANALYTICS_ID")
ANALYTICS_OPT_OUT = ("-e", "ANALYTICS_OPT_OUT")

# Reusable Commands
RUN = (
LOGIN = (
"docker",
"compose",
"run",
"-e",
"VAULT_ADDR",
"-e",
"VAULT_NAMESPACE",
"-e",
"VAULT_TOKEN",
"--rm",
"exec",
"fides-fides-1",
"fides",
"user",
"login",
"-u",
"root_user",
ThomasLaPiana marked this conversation as resolved.
Show resolved Hide resolved
"-p",
"Testpassword1!",
)
EXEC = (
"docker",
"exec",
*ANALYTICS_OPT_OUT,
*ANALYTICS_ID_OVERRIDE,
CI_ARGS_EXEC,
CONTAINER_NAME,
)
EXEC_IT = (
"docker",
"exec",
"-it",
*ANALYTICS_OPT_OUT,
CI_ARGS,
COMPOSE_SERVICE_NAME,
*ANALYTICS_ID_OVERRIDE,
CI_ARGS_EXEC,
CONTAINER_NAME,
)
RUN_NO_DEPS = (
"docker",
Expand Down
12 changes: 6 additions & 6 deletions noxfiles/dev_nox.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,9 +7,9 @@

from constants_nox import (
COMPOSE_SERVICE_NAME,
RUN,
EXEC,
EXEC_IT,
RUN_CYPRESS_TESTS,
RUN_NO_DEPS,
START_APP,
START_APP_REMOTE_DEBUG,
START_TEST_ENV,
Expand Down Expand Up @@ -57,7 +57,7 @@ def dev(session: Session) -> None:
if not datastores:
if open_shell:
session.run(*START_APP, external=True)
session.run(*RUN, "/bin/bash", external=True)
session.run(*EXEC_IT, "/bin/bash", external=True)
else:
if remote_debug:
session.run(*START_APP_REMOTE_DEBUG, external=True)
Expand Down Expand Up @@ -152,7 +152,7 @@ def fides_env(session: Session, fides_image: Literal["test", "dev"] = "test") ->
"Running example setup scripts for DSR Automation tests... (scripts/load_examples.py)"
)
session.run(
*RUN_NO_DEPS,
*EXEC,
"python",
"/fides/scripts/load_examples.py",
external=True,
Expand All @@ -163,7 +163,7 @@ def fides_env(session: Session, fides_image: Literal["test", "dev"] = "test") ->
"Pushing example resources for Data Mapping tests... (demo_resources/*)"
)
session.run(
*RUN_NO_DEPS,
*EXEC,
"fides",
"push",
"demo_resources/",
Expand Down Expand Up @@ -204,7 +204,7 @@ def fides_env(session: Session, fides_image: Literal["test", "dev"] = "test") ->
"Example Mongo Database running at localhost:27017 (user: 'mongo_test', pass: 'mongo_pass', db: 'mongo_test')"
)
session.log("Opening Fides CLI shell... (press CTRL+D to exit)")
session.run(*RUN_NO_DEPS, shell_command, external=True, env=test_env_vars)
session.run(*EXEC, shell_command, external=True, env=test_env_vars)


@nox_session()
Expand Down
4 changes: 2 additions & 2 deletions noxfiles/docs_nox.py
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
"""Contains the nox sessions for developing docs."""
import nox

from constants_nox import CI_ARGS, RUN
from constants_nox import CI_ARGS, RUN_NO_DEPS
from docker_nox import build


Expand All @@ -14,7 +14,7 @@ def docs_build(session: nox.Session, build_type: str) -> None:
session.notify("teardown")
if build_type == "local":
build(session, "dev")
run_shell = (*RUN, "python", "scripts/generate_docs.py", "docs/fides/docs")
run_shell = (*RUN_NO_DEPS, "python", "scripts/generate_docs.py", "docs/fides/docs")
session.run(*run_shell, external=True)


Expand Down
35 changes: 17 additions & 18 deletions noxfiles/test_setup_nox.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,13 @@
from nox import Session

from constants_nox import (
CI_ARGS,
CI_ARGS_EXEC,
COMPOSE_FILE,
COMPOSE_SERVICE_NAME,
CONTAINER_NAME,
EXEC,
IMAGE_NAME,
INTEGRATION_COMPOSE_FILE,
RUN_NO_DEPS,
LOGIN,
START_APP,
START_APP_WITH_EXTERNAL_POSTGRES,
)
Expand All @@ -18,7 +19,7 @@ def pytest_lib(session: Session, coverage_arg: str) -> None:
session.notify("teardown")
session.run(*START_APP, external=True)
run_command = (
*RUN_NO_DEPS,
*EXEC,
"pytest",
coverage_arg,
"tests/lib/",
Expand All @@ -44,8 +45,7 @@ def pytest_ctl(session: Session, mark: str, coverage_arg: str) -> None:
session.run(*start_command, external=True)
run_command = (
"docker",
"compose",
"run",
"exec",
"-e",
"SNOWFLAKE_FIDESCTL_PASSWORD",
"-e",
Expand All @@ -61,8 +61,8 @@ def pytest_ctl(session: Session, mark: str, coverage_arg: str) -> None:
"-e",
"BIGQUERY_CONFIG",
"--rm",
CI_ARGS,
IMAGE_NAME,
CI_ARGS_EXEC,
CONTAINER_NAME,
"pytest",
coverage_arg,
"-m",
Expand All @@ -72,8 +72,9 @@ def pytest_ctl(session: Session, mark: str, coverage_arg: str) -> None:
session.run(*run_command, external=True)
else:
session.run(*START_APP, external=True)
session.run(*LOGIN, external=True)
run_command = (
*RUN_NO_DEPS,
*EXEC,
"pytest",
coverage_arg,
"tests/ctl/",
Expand All @@ -89,7 +90,7 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None:
if mark == "unit":
session.run(*START_APP, external=True)
run_command = (
*RUN_NO_DEPS,
*EXEC,
"pytest",
coverage_arg,
OPS_TEST_DIR,
Expand All @@ -109,8 +110,7 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None:
session.run(*START_APP, external=True)
run_command = (
"docker",
"compose",
"run",
"exec",
"-e",
"ANALYTICS_OPT_OUT",
"-e",
Expand All @@ -124,8 +124,8 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None:
"-e",
"BIGQUERY_DATASET",
"--rm",
CI_ARGS,
COMPOSE_SERVICE_NAME,
CI_ARGS_EXEC,
CONTAINER_NAME,
"pytest",
coverage_arg,
OPS_TEST_DIR,
Expand All @@ -145,8 +145,7 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None:
session.run(*START_APP_WITH_EXTERNAL_POSTGRES, external=True)
run_command = (
"docker",
"compose",
"run",
"exec",
"-e",
"ANALYTICS_OPT_OUT",
"-e",
Expand All @@ -156,8 +155,8 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None:
"-e",
"VAULT_TOKEN",
"--rm",
CI_ARGS,
COMPOSE_SERVICE_NAME,
CI_ARGS_EXEC,
CONTAINER_NAME,
"pytest",
coverage_arg,
OPS_TEST_DIR,
Expand Down
5 changes: 0 additions & 5 deletions src/fides/cli/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,6 @@
import fides
from fides.cli.utils import check_server
from fides.core.config import get_config
from fides.core.user import get_auth_header

from .commands.annotate import annotate
from .commands.core import evaluate, parse, pull, push
Expand Down Expand Up @@ -75,10 +74,6 @@ def cli(ctx: click.Context, config_path: str, local: bool) -> None:

ctx.ensure_object(dict)
config = get_config(config_path, verbose=True)
try:
config.user.auth_header = get_auth_header(verbose=False)
except SystemExit:
pass

# Dyanmically add commands to the CLI
cli.commands = LOCAL_COMMAND_DICT
Expand Down
2 changes: 2 additions & 0 deletions src/fides/cli/commands/db.py
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@ def db_init(ctx: click.Context) -> None:
handle_cli_response(
_api.db_action(
server_url=config.cli.server_url,
headers=config.user.auth_header,
action="init",
)
)
Expand All @@ -52,6 +53,7 @@ def db_reset(ctx: click.Context, yes: bool) -> None:
handle_cli_response(
_api.db_action(
server_url=config.cli.server_url,
headers=config.user.auth_header,
action="reset",
)
)
Expand Down
8 changes: 6 additions & 2 deletions src/fides/core/api.py
Original file line number Diff line number Diff line change
Expand Up @@ -119,8 +119,12 @@ def evaluate(
return requests.get(url, headers=headers, params={"tag": tag, "message": message})


def db_action(server_url: str, action: str) -> requests.Response:
def db_action(
server_url: str,
headers: Dict[str, str],
action: str,
) -> requests.Response:
"""
Tell the API to perform a database action.
"""
return requests.post(f"{server_url}{API_PREFIX}/admin/db/{action}")
return requests.post(f"{server_url}{API_PREFIX}/admin/db/{action}", headers=headers)
13 changes: 11 additions & 2 deletions src/fides/core/config/user_settings.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,16 +4,25 @@

from typing import Dict, Optional

from fides.core.utils import create_auth_header, get_auth_header

from .fides_settings import FidesSettings

ENV_PREFIX = "FIDES__USER__"


def try_get_auth_header() -> Dict[str, str]:
"""Try to get the auth header. If an error is thrown, return a default auth header instead."""
try:
return get_auth_header(verbose=False)
except SystemExit:
return create_auth_header("defaulttoken")


class UserSettings(FidesSettings):
"""Class used to store values from the 'user' section of the config."""

# Auth headers are set when the CLI is initiated.
auth_header: Optional[Dict[str, str]]
auth_header: Dict[str, str] = try_get_auth_header()
analytics_opt_out: Optional[bool]
encryption_key: str = "test_encryption_key"

Expand Down
Loading