1184 allow delete-only saas endpoints

CHANGELOG.md

@@ -17,9 +17,12 @@ The types of changes are:
 
 ## [Unreleased](https://github.com/ethyca/fides/compare/1.9.2...main)
 
+### Added
+* Allow delete-only SaaS connector endpoints [#1200](https://github.com/ethyca/fides/pull/1200)
+
 ### Docs
 
-* Add unlinked docs and fix any remaining broken links [#1266]https://github.com/ethyca/fides/pull/1266)
+* Add unlinked docs and fix any remaining broken links [#1266](https://github.com/ethyca/fides/pull/1266)
 
 ## [1.9.2](https://github.com/ethyca/fides/compare/1.9.1...1.9.2)
 

data/saas/config/saas_example_config.yml

@@ -232,3 +232,14 @@ saas_config:
                   field: users.list_ids
                   direction: from
               unpack: True
+    - name: people
+      requests:
+        delete:
+          method: POST
+          path: /api/1/people/delete_jobs
+          query_params:
+            - name: email
+              value: <email>
+          param_values:
+            - name: email
+              identity: email

data/saas/dataset/saas_example_dataset.yml

@@ -224,3 +224,9 @@ dataset:
           - name: mailing_list_name
             fidesops_meta:
               data_type: string
+      - name: people
+        fields:
+          - name: id
+            fidesops_meta:
+              data_type: integer
+              primary_key: True

src/fides/api/ops/schemas/saas/saas_config.py

@@ -6,6 +6,7 @@
     Collection,
     CollectionAddress,
     Dataset,
+    Field,
     FieldAddress,
     ScalarField,
 )
@@ -298,21 +299,40 @@ def get_graph(self) -> Dataset:
         """Converts endpoints to a Dataset with collections and field references"""
         collections = []
         for endpoint in self.endpoints:
-            fields = []
-            for param in endpoint.requests["read"].param_values or []:
-                if param.references:
-                    references = []
-                    for reference in param.references:
-                        first, *rest = reference.field.split(".")
-                        references.append(
-                            (
-                                FieldAddress(reference.dataset, first, *rest),
-                                reference.direction,
+            fields: List[Field] = []
+            read_request = endpoint.requests.get("read")
+            delete_request = endpoint.requests.get("delete")
+            if read_request:
+                for param in read_request.param_values or []:
+                    if param.references:
+                        references = []
+                        for reference in param.references:
+                            first, *rest = reference.field.split(".")
+                            references.append(
+                                (
+                                    FieldAddress(reference.dataset, first, *rest),
+                                    reference.direction,
+                                )
                             )
+                        fields.append(
+                            ScalarField(name=param.name, references=references)
+                        )
+                    if param.identity:
+                        fields.append(
+                            ScalarField(name=param.name, identity=param.identity)
                         )
-                    fields.append(ScalarField(name=param.name, references=references))
-                if param.identity:
-                    fields.append(ScalarField(name=param.name, identity=param.identity))
+            elif delete_request:
+                # The preferred way to build the graph for a SaaS connector is to convert
+                # a read requests' param_values into identity and dataset references.
+                # If the endpoint only specifies a delete request without a read,
+                # then we must create a single placeholder field for the graph traversal
+                # to still visit this delete-only collection and call the delete request.
+                fields.append(
+                    ScalarField(
+                        name="placeholder", identity="email", primary_key="True"
+                    )
+                )
+
             if fields:
                 grouped_inputs: Optional[Set[str]] = set()
                 if endpoint.requests.get("read"):
@@ -334,6 +354,28 @@ def get_graph(self) -> Dataset:
             connection_key=super().fides_key_prop,
         )
 
+    @staticmethod
+    def _process_param_values(
+        fields: List[Field], param_values: Optional[List[ParamValue]]
+    ) -> None:
+        """
+        Converts param values to dataset fields with identity and dataset references
+        """
+        for param in param_values or []:
+            if param.references:
+                references = []
+                for reference in param.references:
+                    first, *rest = reference.field.split(".")
+                    references.append(
+                        (
+                            FieldAddress(reference.dataset, first, *rest),
+                            reference.direction,
+                        )
+                    )
+                fields.append(ScalarField(name=param.name, references=references))
+            if param.identity:
+                fields.append(ScalarField(name=param.name, identity=param.identity))
+
 
 class SaaSConfigValidationDetails(BaseSchema):
     """

src/fides/api/ops/service/connectors/saas_connector.py

@@ -116,7 +116,20 @@ def retrieve_data(
 
         query_config: SaaSQueryConfig = self.query_config(node)
         read_request: Optional[SaaSRequest] = query_config.get_request_by_action("read")
+        delete_request: Optional[SaaSRequest] = query_config.get_request_by_action(
+            "delete"
+        )
+
         if not read_request:
+            # if a delete request is specified for this endpoint without a read request
+            # then we return a single empty row to still trigger the mask_data method
+            if delete_request:
+                logger.info(
+                    "Skipping read for the '%s' collection, it is delete-only",
+                    self.collection_name,
+                )
+                return [{}]
+
             raise FidesopsException(
                 f"The 'read' action is not defined for the '{self.collection_name}' "  # type: ignore
                 f"endpoint in {self.saas_config.fides_key}"

tests/ops/api/v1/endpoints/test_saas_config_endpoints.py

@@ -244,7 +244,7 @@ def test_patch_saas_config_update(
         )
         saas_config = connection_config.saas_config
         assert saas_config is not None
-        assert len(saas_config["endpoints"]) == 9
+        assert len(saas_config["endpoints"]) == 10
 
 
 def get_saas_config_url(connection_config: Optional[ConnectionConfig] = None) -> str:
@@ -318,7 +318,7 @@ def test_get_saas_config(
             response_body["fides_key"]
             == saas_example_connection_config.get_saas_config().fides_key
         )
-        assert len(response_body["endpoints"]) == 10
+        assert len(response_body["endpoints"]) == 11
         assert response_body["type"] == "custom"
 
 

tests/ops/models/test_saasconfig.py

@@ -3,7 +3,12 @@
 import pytest
 from pydantic import ValidationError
 
-from fides.api.ops.graph.config import CollectionAddress, FieldAddress
+from fides.api.ops.graph.config import (
+    CollectionAddress,
+    FieldAddress,
+    FieldPath,
+    ScalarField,
+)
 from fides.api.ops.schemas.dataset import FidesopsDatasetReference
 from fides.api.ops.schemas.saas.saas_config import (
     ConnectorParam,
@@ -85,47 +90,65 @@ def test_saas_request_override_invalid_properties():
 @pytest.mark.unit_saas
 def test_saas_config_to_dataset(saas_example_config: Dict[str, Dict]):
     """Verify dataset generated by SaaS config"""
-    # convert endpoint references to dataset references to be able to hook SaaS connectors into the graph traversal
+
+    # convert endpoint references to dataset references to be able to hook
+    # SaaS connectors into the graph traversal
     saas_config = SaaSConfig(**saas_example_config)
     saas_dataset = saas_config.get_graph()
 
-    messages_collection = saas_dataset.collections[0]
-    member_collection = saas_dataset.collections[2]
-    query_field = member_collection.fields[0]
+    # messages
+    messages_collection = next(
+        col for col in saas_dataset.collections if col.name == "messages"
+    )
     conversation_id_field = messages_collection.fields[0]
     conversations_reference = conversation_id_field.references[0]
     field_address, direction = conversations_reference
 
-    assert messages_collection.name == "messages"
     assert conversation_id_field.name == "conversation_id"
     assert field_address == FieldAddress(saas_config.fides_key, "conversations", "id")
     assert direction == "from"
 
+    member_collection = next(
+        col for col in saas_dataset.collections if col.name == "member"
+    )
+    query_field = member_collection.fields[0]
     assert query_field.name == "email"
     assert query_field.identity == "email"
 
-    user_collection = saas_dataset.collections[5]
-    assert user_collection.after == {
+    # users
+    users_collection = next(
+        col for col in saas_dataset.collections if col.name == "users"
+    )
+    org_slug_reference, direction = users_collection.fields[0].references[0]
+    assert users_collection.after == {
         CollectionAddress("saas_connector_example", "projects")
     }
-    assert user_collection.grouped_inputs == {
+    assert users_collection.grouped_inputs == {
         "organization_slug",
         "project_slug",
         "query",
     }
-
-    org_slug_reference, direction = user_collection.fields[0].references[0]
     assert org_slug_reference == FieldAddress(
         saas_config.fides_key, "projects", "organization", "slug"
     )
     assert direction == "from"
 
-    project_slug_reference, direction = user_collection.fields[1].references[0]
+    project_slug_reference, direction = users_collection.fields[1].references[0]
     assert project_slug_reference == FieldAddress(
         saas_config.fides_key, "projects", "slug"
     )
     assert direction == "from"
 
+    # assert that delete-only endpoints generate a collection with
+    # a single primary key identity field as a placeholder
+    people_collection = next(
+        col for col in saas_dataset.collections if col.name == "people"
+    )
+    people_id_field = people_collection.field(FieldPath("placeholder"))
+    assert people_id_field == ScalarField(
+        name="placeholder", identity="email", primary_key="True"
+    )
+
 
 @pytest.mark.unit_saas
 def test_saas_config_ignore_errors_param(saas_example_config: Dict[str, Dict]):

tests/ops/service/connectors/test_saas_connector.py

@@ -5,10 +5,15 @@
 from sqlalchemy.orm import Session
 from starlette.status import HTTP_200_OK, HTTP_404_NOT_FOUND
 
+from fides.api.ops.graph.graph import Node
+from fides.api.ops.graph.traversal import TraversalNode
+from fides.api.ops.models.policy import Policy
+from fides.api.ops.models.privacy_request import PrivacyRequest
 from fides.api.ops.schemas.saas.saas_config import SaaSRequest
 from fides.api.ops.schemas.saas.shared_schemas import HTTPMethod
 from fides.api.ops.service.connectors import get_connector
 from fides.api.ops.service.connectors.saas_connector import SaaSConnector
+from src.fides.api.ops.schemas.saas.saas_config import SaaSConfig
 
 
 @pytest.mark.unit_saas
@@ -117,6 +122,29 @@ def test_unwrap_response_data_no_data_path(self):
         unwrapped = SaaSConnector._unwrap_response_data(fake_request, fake_response)
         assert response_body == unwrapped
 
+    def test_delete_only_endpoint(
+        self, saas_example_config, saas_example_connection_config
+    ):
+        """
+        Uses the SaaS example connector which contains an endpoint
+        that only contains a delete request and no read request.
+        """
+        saas_config = SaaSConfig(**saas_example_config)
+        graph = saas_config.get_graph()
+        node = Node(
+            graph,
+            next(
+                collection
+                for collection in graph.collections
+                if collection.name == "people"
+            ),
+        )
+        traversal_node = TraversalNode(node)
+        connector: SaaSConnector = get_connector(saas_example_connection_config)
+        assert connector.retrieve_data(
+            traversal_node, Policy(), PrivacyRequest(id="123"), {}
+        ) == [{}]
+
 
 @pytest.mark.integration_saas
 @pytest.mark.integration_segment