Alter taxonomy upsert behavior

CHANGELOG.md

@@ -19,7 +19,9 @@ The types of changes are:
 
 ### Changed
 
+* Changed behavior of `load_default_taxonomy` to append instead of upsert [#1040](https://github.com/ethyca/fides/pull/1040)
 * Deleting a taxonomy field with children will now cascade delete all of its children as well. [#1042](https://github.com/ethyca/fides/pull/1042)
+
 ### Fixed
 
 * Fixed navigating directly to frontend routes loading index page instead of the correct static page for the route.

src/fidesctl/api/ctl/database/database.py

@@ -20,7 +20,7 @@
 )
 from fidesctl.ctl.core.utils import get_db_engine
 
-from .crud import create_resource, upsert_resources
+from .crud import create_resource, list_resource
 
 
 def get_alembic_config(database_url: str) -> Config:
@@ -87,18 +87,29 @@ async def load_default_taxonomy() -> None:
     upsert_resource_types = list(DEFAULT_TAXONOMY.__fields_set__)
     upsert_resource_types.remove("organization")
 
-    log.info("UPSERTING the remaining default fideslang taxonomy resources")
+    log.info("INSERTING new default fideslang taxonomy resources")
     for resource_type in upsert_resource_types:
         log.info(f"Processing {resource_type} resources...")
-        resources = list(map(dict, DEFAULT_TAXONOMY.dict()[resource_type]))
+        default_resources = DEFAULT_TAXONOMY.dict()[resource_type]
+        existing_resources = await list_resource(sql_model_map[resource_type])
+        existing_keys = [item.fides_key for item in existing_resources]
+        resources = [
+            resource
+            for resource in default_resources
+            if resource["fides_key"] not in existing_keys
+        ]
+
+        if len(resources) == 0:
+            log.info(f"No new {resource_type} resources to add from default taxonomy.")
+            continue
 
         try:
-            result = await upsert_resources(sql_model_map[resource_type], resources)
+            for resource in resources:
+                await create_resource(sql_model_map[resource_type], resource)
         except QueryError:
-            pass  # The upsert_resources function will log the error
+            pass  # The create_resource function will log the error
         else:
-            log.info(f"INSERTED {result[0]} {resource_type} resource(s)")
-            log.info(f"UPDATED {result[1]} {resource_type} resource(s)")
+            log.info(f"INSERTED {len(resources)} {resource_type} resource(s)")
 
 
 def reset_db(database_url: str) -> None:

tests/ctl/api/test_database.py

@@ -0,0 +1,109 @@
+from typing import Generator
+
+import pytest
+from fideslang import DEFAULT_TAXONOMY, DataCategory
+
+from fidesctl.api.ctl.database import database
+from fidesctl.ctl.core import api as _api
+from fidesctl.ctl.core.config import FidesctlConfig
+
+
+@pytest.fixture(scope="function", name="data_category")
+def fixture_data_category(test_config: FidesctlConfig) -> Generator:
+    """
+    Fixture that yields a data category and then deletes it for each test run.
+    """
+    fides_key = "foo"
+    yield DataCategory(fides_key=fides_key, parent_key=None)
+
+    _api.delete(
+        url=test_config.cli.server_url,
+        resource_type="data_category",
+        resource_id=fides_key,
+        headers=test_config.user.request_headers,
+    )
+
+
+@pytest.mark.integration
+class TestLoadDefaultTaxonomy:
+    """Tests related to load_default_taxonomy"""
+
+    async def test_add_to_default_taxonomy(
+        self,
+        monkeypatch: pytest.MonkeyPatch,
+        test_config: FidesctlConfig,
+        data_category: DataCategory,
+    ) -> None:
+        """Should be able to add to the existing default taxonomy"""
+        result = _api.get(
+            test_config.cli.server_url,
+            "data_category",
+            data_category.fides_key,
+            headers=test_config.user.request_headers,
+        )
+        assert result.status_code == 404
+
+        updated_default_taxonomy = DEFAULT_TAXONOMY.copy()
+        updated_default_taxonomy.data_category.append(data_category)
+
+        monkeypatch.setattr(database, "DEFAULT_TAXONOMY", updated_default_taxonomy)
+        await database.load_default_taxonomy()
+
+        result = _api.get(
+            test_config.cli.server_url,
+            "data_category",
+            data_category.fides_key,
+            headers=test_config.user.request_headers,
+        )
+        assert result.status_code == 200
+
+    async def test_does_not_override_user_changes(
+        self, test_config: FidesctlConfig
+    ) -> None:
+        """
+        Loading the default taxonomy should not override user changes
+        to their default taxonomy
+        """
+        default_category = DEFAULT_TAXONOMY.data_category[0].copy()
+        new_description = "foo description"
+        default_category.description = new_description
+        result = _api.update(
+            test_config.cli.server_url,
+            "data_category",
+            json_resource=default_category.json(),
+            headers=test_config.user.request_headers,
+        )
+        assert result.status_code == 200
+
+        await database.load_default_taxonomy()
+        result = _api.get(
+            test_config.cli.server_url,
+            "data_category",
+            default_category.fides_key,
+            headers=test_config.user.request_headers,
+        )
+        assert result.json()["description"] == new_description
+
+    async def test_does_not_remove_user_added_taxonomies(
+        self, test_config: FidesctlConfig, data_category: DataCategory
+    ) -> None:
+        """
+        Loading the default taxonomy should not delete user additions
+        to their default taxonomy
+        """
+        result = _api.create(
+            test_config.cli.server_url,
+            "data_category",
+            json_resource=data_category.json(),
+            headers=test_config.user.request_headers,
+        )
+
+        await database.load_default_taxonomy()
+
+        result = _api.get(
+            test_config.cli.server_url,
+            "data_category",
+            data_category.fides_key,
+            headers=test_config.user.request_headers,
+        )
+        assert result.status_code == 200