Fixing inconsistent SaaS connector integration tests (#473)

Fixing inconsistent SaaS connector integration tests

src/fidesops/task/filter_results.py

@@ -98,7 +98,7 @@ def _defaultdict_or_array(resource: Any) -> Any:
 
     elif isinstance(row, dict):
         for key in row:
-            if key == target_path.levels[0]:
+            if target_path.levels and key == target_path.levels[0]:
                 if key not in saved:
                     saved[key] = _defaultdict_or_array(row[key])
                 saved[key] = select_and_save_field(

tests/fixtures/saas/hubspot_fixtures.py

@@ -1,5 +1,11 @@
 import json
+import os
 import time
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+from sqlalchemy.orm import Session
 
 from fidesops.core.config import load_toml
 from fidesops.models.connectionconfig import (
@@ -8,18 +14,12 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
-import pytest
-import pydash
-import os
-from typing import Any, Dict, Generator
-from tests.fixtures.application_fixtures import load_dataset
-from tests.fixtures.saas_example_fixtures import load_config
-from sqlalchemy.orm import Session
-
-from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod
+from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams
 from fidesops.service.connectors import SaaSConnector
 from fidesops.util import cryptographic_util
 from fidesops.util.saas_util import format_body
+from tests.fixtures.application_fixtures import load_dataset
+from tests.fixtures.saas_example_fixtures import load_config
 
 saas_config = load_toml("saas_config.toml")
 
@@ -143,12 +143,11 @@ def hubspot_erasure_data(
     # no need to subscribe contact, since creating a contact auto-subscribes them
 
     # Allows contact to be propagated in Hubspot before calling access / erasure requests
-    remaining_tries = 5
+    retries = 10
     while _contact_exists(hubspot_erasure_identity_email, connector) is False:
-        if remaining_tries < 1:
-            raise Exception(
-                f"Contact with contact id {contact_id} could not be added to Hubspot"
-            )
+        if not retries:
+            raise Exception(f"Contact with contact id {contact_id} could not be added to Hubspot")
+        retries -= 1
         time.sleep(5)
 
     yield contact_id
@@ -161,12 +160,11 @@ def hubspot_erasure_data(
     connector.create_client().send(delete_request)
 
     # verify contact is deleted
-    remaining_tries = 5
+    retries = 10
     while _contact_exists(hubspot_erasure_identity_email, connector) is True:
-        if remaining_tries < 1:
-            raise Exception(
-                f"Contact with contact id {contact_id} could not be deleted from Hubspot"
-            )
+        if not retries:
+            raise Exception(f"Contact with contact id {contact_id} could not be deleted from Hubspot")
+        retries -= 1
         time.sleep(5)  # Ensures contact is deleted
 
 

tests/fixtures/saas/segment_fixtures.py

@@ -1,3 +1,14 @@
+import os
+import random
+import time
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+import requests
+from faker import Faker
+from sqlalchemy.orm import Session
+
 from fidesops.core.config import load_toml
 from fidesops.db import session
 from fidesops.models.connectionconfig import (
@@ -6,13 +17,8 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
-import pytest
-import pydash
-import os
-from typing import Any, Dict, Generator
 from tests.fixtures.application_fixtures import load_dataset
 from tests.fixtures.saas_example_fixtures import load_config
-from sqlalchemy.orm import Session
 
 saas_config = load_toml("saas_config.toml")
 
@@ -39,7 +45,7 @@ def segment_secrets():
     }
 
 
-@pytest.fixture(scope="function")
+@pytest.fixture(scope="session")
 def segment_identity_email():
     return pydash.get(saas_config, "segment.identity_email") or os.environ.get(
         "SEGMENT_IDENTITY_EMAIL"
@@ -96,3 +102,127 @@ def segment_dataset_config(
     )
     yield dataset
     dataset.delete(db=db)
+
+
+@pytest.fixture(scope="session")
+def segment_erasure_identity_email(segment_identity_email) -> str:
+    timestamp = int(time.time())
+    at_index: int = segment_identity_email.find("@")
+    email = f"{segment_identity_email[0:at_index]}{timestamp}{segment_identity_email[at_index:]}"
+    return email
+
+
+def _get_user_id(email: str, secrets: Dict[str, Any]) -> str:
+    personas_domain = secrets["personas_domain"]
+    namespace_id = secrets["namespace_id"]
+    access_secret = secrets["access_secret"]
+    response = requests.get(
+        f"https://{personas_domain}/v1/spaces/{namespace_id}/collections/users/profiles/user_id:{email}/metadata",
+        auth=(access_secret, None),
+    )
+    if not response.ok:
+        return None
+
+    return response.json()["segment_id"]
+
+
+def _get_track_events(segment_id: str, secrets: Dict[str, Any]) -> Dict[str, Any]:
+    personas_domain = secrets["personas_domain"]
+    namespace_id = secrets["namespace_id"]
+    access_secret = secrets["access_secret"]
+
+    response = requests.get(
+        f"https://{personas_domain}/v1/spaces/{namespace_id}/collections/users/profiles/{segment_id}/events",
+        auth=(access_secret, None),
+    )
+    if not response.ok or response.json()["data"] is None:
+        return None
+
+    return response.json()["data"][0]
+
+
+@pytest.fixture(scope="function")
+def segment_erasure_data(
+    segment_connection_config, segment_erasure_identity_email
+) -> str:
+    """Seeds a segment user and event"""
+    segment_secrets = segment_connection_config.secrets
+    if not segment_identity_email:  # Don't run unnecessarily locally
+        return
+
+    api_domain = segment_secrets["api_domain"]
+    user_token = segment_secrets["user_token"]
+
+    faker = Faker()
+
+    timestamp = int(time.time())
+    email = segment_erasure_identity_email
+    first_name = faker.first_name()
+    last_name = faker.last_name()
+
+    # Create user
+    headers = {
+        "Content-Type": "application/json",
+        "Authorization": f"Basic {user_token}",
+    }
+    body = {
+        "userId": email,
+        "traits": {
+            "subscriptionStatus": "active",
+            "address": {
+                "city": faker.city(),
+                "country": faker.country(),
+                "postalCode": faker.postcode(),
+                "state": "NY",
+            },
+            "age": random.randrange(18, 99),
+            "avatar": "",
+            "industry": "data",
+            "description": faker.job(),
+            "email": email,
+            "firstName": first_name,
+            "id": timestamp,
+            "lastName": last_name,
+            "name": f"{first_name} {last_name}",
+            "phone": faker.phone_number(),
+            "title": faker.prefix(),
+            "username": f"test_fidesops_user_{timestamp}",
+            "website": "www.example.com",
+        },
+    }
+    response = requests.post(
+        f"https://{api_domain}identify", headers=headers, json=body
+    )
+    assert response.ok
+
+    # Wait until user returns data
+    retries = 10
+    while (segment_id := _get_user_id(email, segment_secrets)) is None:
+        if not retries:
+            raise Exception(
+                "The user endpoint did not return the required data for testing during the time limit"
+            )
+        retries -= 1
+        time.sleep(5)
+
+    # Create event
+    body = {
+        "userId": email,
+        "type": "track",
+        "event": "User Registered",
+        "properties": {"plan": "Free", "accountType": faker.company()},
+        "context": {"ip": faker.ipv4()},
+    }
+
+    response = requests.post(f"https://{api_domain}track", headers=headers, json=body)
+    assert response.ok
+
+    # Wait until track_events returns data
+    retries = 10
+    while _get_track_events(segment_id, segment_secrets) is None:
+        if not retries:
+            raise Exception(
+                "The track_events endpoint did not return the required data for testing during the time limit"
+            )
+        retries -= 1
+        time.sleep(5)

tests/fixtures/saas/sentry_fixtures.py

@@ -1,3 +1,10 @@
+import os
+from typing import Any, Dict, Generator
+
+import pydash
+import pytest
+from sqlalchemy.orm import Session
+
 from fidesops.core.config import load_toml
 from fidesops.db import session
 from fidesops.models.connectionconfig import (
@@ -6,13 +13,8 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
-import pytest
-import pydash
-import os
-from typing import Any, Dict, Generator
 from tests.fixtures.application_fixtures import load_dataset
 from tests.fixtures.saas_example_fixtures import load_config
-from sqlalchemy.orm import Session
 
 saas_config = load_toml("saas_config.toml")
 

tests/fixtures/saas/stripe_fixtures.py

@@ -1,7 +1,12 @@
 import os
-from multidimensional_urlencode import urlencode as multidimensional_urlencode
 from typing import Any, Dict, Generator
 
+import pydash
+import pytest
+import requests
+from multidimensional_urlencode import urlencode as multidimensional_urlencode
+from sqlalchemy.orm import Session
+
 from fidesops.core.config import load_toml
 from fidesops.db import session
 from fidesops.models.connectionconfig import (
@@ -10,12 +15,8 @@
     ConnectionType,
 )
 from fidesops.models.datasetconfig import DatasetConfig
-import pytest
-import pydash
-import requests
 from tests.fixtures.application_fixtures import load_dataset
 from tests.fixtures.saas_example_fixtures import load_config
-from sqlalchemy.orm import Session
 
 saas_config = load_toml("saas_config.toml")