Support remote debugging in dev environments (#1780)

* Support remote debugging in dev environments

* Add VSCode debugging guide to docs

Co-authored-by: Adam Sachs <adam@Adams-MBP.hsd1.ma.comcast.net>

.vscode/launch.json

@@ -4,6 +4,25 @@
     // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
     "version": "0.2.0",
     "configurations": [
+        // Ensure your local dev server/container is configured remote debugging.
+        // See VSCode docs for some more information about remote debugging python apps:
+        // https://code.visualstudio.com/docs/python/debugging#_debugging-by-attaching-over-a-network-connection
+        {
+            "name": "Python debugger: Remote Attach Fides",
+            "type": "python",
+            "request": "attach",
+            "connect": {
+                "host": "0.0.0.0",
+                "port": 5678
+            },
+            "pathMappings": [
+                {
+                    "localRoot": "${workspaceFolder}",
+                    "remoteRoot": "."
+                }
+            ],
+            "justMyCode": false
+        },
         {
             "name": "Launch Chrome: Admin UI",
             "request": "launch",
@@ -21,6 +40,3 @@
         }
     ]
 }
-
-
-

CHANGELOG.md

@@ -36,6 +36,7 @@ The types of changes are:
 ### Developer Experience
 
 * Admin-UI-Cypress tests that fail in CI will now upload screen recordings for debugging. [#1728](https://github.com/ethyca/fides/pull/1728/files/c23e62fea284f7910028c8483feff893903068b8#r1019491323)
+* Enable remote debugging from VSCode of live dev app (#1780)(https://github.com/ethyca/fides/pull/1780)
 
 ### Removed
 

dev-requirements.txt

@@ -1,4 +1,5 @@
 black==22.8.0
+debugpy==1.6.3
 Faker==14.1.0
 GitPython==3.1.27
 isort==5.10.1

docker-compose.remote-debug.yml

@@ -0,0 +1,6 @@
+services:
+  fides:
+    command: python -m debugpy --listen 0.0.0.0:5678 -m uvicorn --host 0.0.0.0 --port 8080 --reload --reload-dir src fides.api.main:app
+    ports:
+      - "8080:8080"
+      - "5678:5678"

docs/fides/docs/development/vscode_debugging.md

@@ -0,0 +1,43 @@
+# Debugging Fides in VSCode
+
+This is a quick guide to show how VSCode can be used to debug Fides running locally in Docker. The general approach is to allow the local `fides` Docker Compose service to allow remote debugging connections, and to start a remote debugger from a host VSCode workspace.
+
+## Setup
+
+### Run Fides with Remote Debugging Enabled
+
+In order to accept incoming remote debugging connections, the `fides` Docker Compose service must be run with slight alterations. To enable this functionality, simply add the `remote_debug` flag to a `nox` command. For example:
+
+```
+nox -s dev -- remote_debug
+```
+or 
+```
+nox -s dev -- remote_debug postgres timescale
+```
+
+With those commands, the `fides` Docker Compose service that's running the Fides server locally is able to accept incoming remote debugging connections.
+
+Note that, at this point, the `remote_debug` flag is not enabled for other `nox` sessions, e.g. `test_env`, `pytest_ops`, etc.
+
+### Attach a Remote Debugger to the Fides Server 
+
+Now that the running Fides server can accept incoming remote debugging connections, you can attach a remote debugger from a local VSCode workspace to actively debug the server application. A launch configuration is included in the `fides` repo to facilitate this step.
+
+- Open up the `fides` repo in a VSCode workspace
+- Go to the `Run and Debug` view
+- From the debugger dropdown list, select the `Python debugger: Remote Attach Fides` configuration
+- Click the `Start Debugging` play button
+- The remote debugger should now be attached to the Fides server!
+    - To confirm the debugger is attached, at least one `RUNNING` line item should appear in the `CALL STACK` window
+
+## Debug!
+
+At this point, VSCode is ready to debug the running Fides server. Try setting breakpoints and hitting them by, e.g., making certain HTTP requests against the Fides server. [This guide](https://code.visualstudio.com/docs/python/python-tutorial#_configure-and-run-the-debugger) provides more information on how to use the VSCode Python debugger. 
+
+## Links
+
+Some relevant VSCode documentation for reference:
+
+- <https://code.visualstudio.com/docs/python/debugging#_debugging-by-attaching-over-a-network-connection>
+- <https://code.visualstudio.com/docs/python/python-tutorial#_configure-and-run-the-debugger>

docs/fides/mkdocs.yml

@@ -87,6 +87,7 @@ nav:
             - Pull Requests: development/pull_requests.md
             - Releases: development/releases.md
             - Release Checklist: development/release_checklist.md
+            - VSCode Debugging: development/vscode_debugging.md
             - Jetbrains Debugging: development/jetbrains_debugging.md
             - Using Postman: development/postman/using_postman.md
             - Updating the Database Diagram: development/update_erd_diagram.md

noxfiles/constants_nox.py

@@ -5,6 +5,7 @@
 COMPOSE_FILE = "docker-compose.yml"
 INTEGRATION_COMPOSE_FILE = "docker-compose.integration-tests.yml"
 TEST_ENV_COMPOSE_FILE = "docker-compose.test-env.yml"
+REMOTE_DEBUG_COMPOSE_FILE = "docker-compose.remote-debug.yml"
 WITH_TEST_CONFIG = ("-f", "tests/ctl/test_config.toml")
 
 # Image Names & Tags
@@ -91,6 +92,16 @@
     "--wait",
     COMPOSE_SERVICE_NAME,
 )
+START_APP_REMOTE_DEBUG = (
+    "docker",
+    "compose",
+    "-f",
+    COMPOSE_FILE,
+    "-f",
+    REMOTE_DEBUG_COMPOSE_FILE,
+    "up",
+    COMPOSE_SERVICE_NAME,
+)
 START_APP_WITH_EXTERNAL_POSTGRES = (
     "docker",
     "compose",

noxfiles/dev_nox.py

@@ -8,6 +8,7 @@
     RUN,
     RUN_NO_DEPS,
     START_APP,
+    START_APP_REMOTE_DEBUG,
     START_TEST_ENV,
 )
 from docker_nox import build
@@ -38,16 +39,25 @@ def dev(session: Session) -> None:
         session.run("docker", "compose", "up", "-d", "fides-pc", external=True)
 
     open_shell = "shell" in session.posargs
+    remote_debug = "remote_debug" in session.posargs
     if not datastores:
         if open_shell:
             session.run(*START_APP, external=True)
             session.run(*RUN, "/bin/bash", external=True)
         else:
-            session.run("docker", "compose", "up", COMPOSE_SERVICE_NAME, external=True)
+            if remote_debug:
+                session.run(*START_APP_REMOTE_DEBUG, external=True)
+            else:
+                session.run(
+                    "docker", "compose", "up", COMPOSE_SERVICE_NAME, external=True
+                )
     else:
         # Run the webserver with additional datastores
         run_infrastructure(
-            open_shell=open_shell, run_application=True, datastores=datastores
+            open_shell=open_shell,
+            run_application=True,
+            datastores=datastores,
+            remote_debug=remote_debug,
         )
 
 

noxfiles/run_infrastructure.py

@@ -38,6 +38,7 @@ def run_infrastructure(
     run_tests: bool = False,  # Should we run the tests after creating the infra?
     run_create_test_data: bool = False,  # Should we run the create_test_data command?
     analytics_opt_out: bool = False,  # Should we opt out of analytics?
+    remote_debug: bool = False,  # Should remote debugging be enabled?
 ) -> None:
     """
     - Create a Docker Compose file path for all datastores specified in `datastores`.
@@ -64,7 +65,7 @@ def run_infrastructure(
     ]
 
     # Configure docker compose path
-    path: str = get_path_for_datastores(datastores)
+    path: str = get_path_for_datastores(datastores, remote_debug)
 
     _run_cmd_or_err(
         f"docker compose {path} up --wait {COMPOSE_SERVICE_NAME} {' '.join(docker_datastores)}"
@@ -120,11 +121,13 @@ def seed_initial_data(
             )
 
 
-def get_path_for_datastores(datastores: List[str]) -> str:
+def get_path_for_datastores(datastores: List[str], remote_debug: bool) -> str:
     """
     Returns the docker compose file paths for the specified datastores
     """
     path: str = "-f docker-compose.yml"
+    if remote_debug:
+        path += " -f docker-compose.remote-debug.yml"
     for datastore in datastores:
         _run_cmd_or_err(f'echo "configuring infrastructure for {datastore}"')
         if datastore in DOCKERFILE_DATASTORES: