From 417e76706ac2d3beccd9c883f84aac3251f31c4e Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 10 May 2022 10:58:17 -0700 Subject: [PATCH] Fixing inconsistent SaaS connector integration tests (#473) Fixing inconsistent SaaS connector integration tests --- src/fidesops/task/filter_results.py | 2 +- tests/fixtures/saas/hubspot_fixtures.py | 36 +++-- tests/fixtures/saas/segment_fixtures.py | 142 +++++++++++++++++- tests/fixtures/saas/sentry_fixtures.py | 12 +- tests/fixtures/saas/stripe_fixtures.py | 11 +- .../saas/test_segment_task.py | 87 +---------- .../saas/test_sentry_task.py | 44 +++++- .../saas/test_stripe_task.py | 28 ++-- 8 files changed, 225 insertions(+), 137 deletions(-) diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/task/filter_results.py index bb812c41f..c242b94ac 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/task/filter_results.py @@ -98,7 +98,7 @@ def _defaultdict_or_array(resource: Any) -> Any: elif isinstance(row, dict): for key in row: - if key == target_path.levels[0]: + if target_path.levels and key == target_path.levels[0]: if key not in saved: saved[key] = _defaultdict_or_array(row[key]) saved[key] = select_and_save_field( diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 2d2de85bb..2260d2bfc 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -1,5 +1,11 @@ import json +import os import time +from typing import Any, Dict, Generator + +import pydash +import pytest +from sqlalchemy.orm import Session from fidesops.core.config import load_toml from fidesops.models.connectionconfig import ( @@ -8,18 +14,12 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import os -from typing import Any, Dict, Generator -from tests.fixtures.application_fixtures import load_dataset -from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session - -from fidesops.schemas.saas.shared_schemas import SaaSRequestParams, HTTPMethod +from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors import SaaSConnector from fidesops.util import cryptographic_util from fidesops.util.saas_util import format_body +from tests.fixtures.application_fixtures import load_dataset +from tests.fixtures.saas_example_fixtures import load_config saas_config = load_toml("saas_config.toml") @@ -143,12 +143,11 @@ def hubspot_erasure_data( # no need to subscribe contact, since creating a contact auto-subscribes them # Allows contact to be propagated in Hubspot before calling access / erasure requests - remaining_tries = 5 + retries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is False: - if remaining_tries < 1: - raise Exception( - f"Contact with contact id {contact_id} could not be added to Hubspot" - ) + if not retries: + raise Exception(f"Contact with contact id {contact_id} could not be added to Hubspot") + retries -= 1 time.sleep(5) yield contact_id @@ -161,12 +160,11 @@ def hubspot_erasure_data( connector.create_client().send(delete_request) # verify contact is deleted - remaining_tries = 5 + retries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is True: - if remaining_tries < 1: - raise Exception( - f"Contact with contact id {contact_id} could not be deleted from Hubspot" - ) + if not retries: + raise Exception(f"Contact with contact id {contact_id} could not be deleted from Hubspot") + retries -= 1 time.sleep(5) # Ensures contact is deleted diff --git a/tests/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py index a2ba5085f..ab51e932c 100644 --- a/tests/fixtures/saas/segment_fixtures.py +++ b/tests/fixtures/saas/segment_fixtures.py @@ -1,3 +1,14 @@ +import os +import random +import time +from typing import Any, Dict, Generator + +import pydash +import pytest +import requests +from faker import Faker +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -6,13 +17,8 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import os -from typing import Any, Dict, Generator from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml") @@ -39,7 +45,7 @@ def segment_secrets(): } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def segment_identity_email(): return pydash.get(saas_config, "segment.identity_email") or os.environ.get( "SEGMENT_IDENTITY_EMAIL" @@ -96,3 +102,127 @@ def segment_dataset_config( ) yield dataset dataset.delete(db=db) + + +@pytest.fixture(scope="session") +def segment_erasure_identity_email(segment_identity_email) -> str: + timestamp = int(time.time()) + at_index: int = segment_identity_email.find("@") + email = f"{segment_identity_email[0:at_index]}{timestamp}{segment_identity_email[at_index:]}" + return email + + +def _get_user_id(email: str, secrets: Dict[str, Any]) -> str: + personas_domain = secrets["personas_domain"] + namespace_id = secrets["namespace_id"] + access_secret = secrets["access_secret"] + response = requests.get( + f"https://{personas_domain}/v1/spaces/{namespace_id}/collections/users/profiles/user_id:{email}/metadata", + auth=(access_secret, None), + ) + if not response.ok: + return None + + return response.json()["segment_id"] + + +def _get_track_events(segment_id: str, secrets: Dict[str, Any]) -> Dict[str, Any]: + personas_domain = secrets["personas_domain"] + namespace_id = secrets["namespace_id"] + access_secret = secrets["access_secret"] + + response = requests.get( + f"https://{personas_domain}/v1/spaces/{namespace_id}/collections/users/profiles/{segment_id}/events", + auth=(access_secret, None), + ) + if not response.ok or response.json()["data"] is None: + return None + + return response.json()["data"][0] + + +@pytest.fixture(scope="function") +def segment_erasure_data( + segment_connection_config, segment_erasure_identity_email +) -> str: + """Seeds a segment user and event""" + segment_secrets = segment_connection_config.secrets + if not segment_identity_email: # Don't run unnecessarily locally + return + + api_domain = segment_secrets["api_domain"] + user_token = segment_secrets["user_token"] + + faker = Faker() + + timestamp = int(time.time()) + email = segment_erasure_identity_email + first_name = faker.first_name() + last_name = faker.last_name() + + # Create user + headers = { + "Content-Type": "application/json", + "Authorization": f"Basic {user_token}", + } + body = { + "userId": email, + "traits": { + "subscriptionStatus": "active", + "address": { + "city": faker.city(), + "country": faker.country(), + "postalCode": faker.postcode(), + "state": "NY", + }, + "age": random.randrange(18, 99), + "avatar": "", + "industry": "data", + "description": faker.job(), + "email": email, + "firstName": first_name, + "id": timestamp, + "lastName": last_name, + "name": f"{first_name} {last_name}", + "phone": faker.phone_number(), + "title": faker.prefix(), + "username": f"test_fidesops_user_{timestamp}", + "website": "www.example.com", + }, + } + response = requests.post( + f"https://{api_domain}identify", headers=headers, json=body + ) + assert response.ok + + # Wait until user returns data + retries = 10 + while (segment_id := _get_user_id(email, segment_secrets)) is None: + if not retries: + raise Exception( + "The user endpoint did not return the required data for testing during the time limit" + ) + retries -= 1 + time.sleep(5) + + # Create event + body = { + "userId": email, + "type": "track", + "event": "User Registered", + "properties": {"plan": "Free", "accountType": faker.company()}, + "context": {"ip": faker.ipv4()}, + } + + response = requests.post(f"https://{api_domain}track", headers=headers, json=body) + assert response.ok + + # Wait until track_events returns data + retries = 10 + while _get_track_events(segment_id, segment_secrets) is None: + if not retries: + raise Exception( + "The track_events endpoint did not return the required data for testing during the time limit" + ) + retries -= 1 + time.sleep(5) diff --git a/tests/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py index 8d4bd3410..923953b24 100644 --- a/tests/fixtures/saas/sentry_fixtures.py +++ b/tests/fixtures/saas/sentry_fixtures.py @@ -1,3 +1,10 @@ +import os +from typing import Any, Dict, Generator + +import pydash +import pytest +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -6,13 +13,8 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import os -from typing import Any, Dict, Generator from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml") diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py index 37e1a66d1..7c476a991 100644 --- a/tests/fixtures/saas/stripe_fixtures.py +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -1,7 +1,12 @@ import os -from multidimensional_urlencode import urlencode as multidimensional_urlencode from typing import Any, Dict, Generator +import pydash +import pytest +import requests +from multidimensional_urlencode import urlencode as multidimensional_urlencode +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -10,12 +15,8 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import requests from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml") diff --git a/tests/integration_tests/saas/test_segment_task.py b/tests/integration_tests/saas/test_segment_task.py index ee9cf2e91..c00a6135e 100644 --- a/tests/integration_tests/saas/test_segment_task.py +++ b/tests/integration_tests/saas/test_segment_task.py @@ -1,18 +1,13 @@ -import time import random -import requests + import pytest -from faker import Faker from fidesops.core.config import config -from fidesops.task.filter_results import filter_data_categories - - from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity - from fidesops.task import graph_task +from fidesops.task.filter_results import filter_data_categories from fidesops.task.graph_task import get_cached_data_for_erasures from tests.graph.graph_test_util import assert_rows_match @@ -140,76 +135,6 @@ def test_segment_saas_access_request_task( assert filtered_results[f"{dataset_name}:segment_user"][0]["segment_id"] -def _create_test_segment_email(base_email: str, timestamp: int) -> str: - at_index: int = base_email.find("@") - email = f"{base_email[0:at_index]}{timestamp}{base_email[at_index:]}" - return email - - -def create_segment_test_data(segment_connection_config, segment_identity_email: str): - """Seeds a segment user and event""" - segment_secrets = segment_connection_config.secrets - if not segment_identity_email: # Don't run unnecessarily locally - return - - faker = Faker() - - ts = int(time.time()) - email = _create_test_segment_email(segment_identity_email, ts) - first_name = faker.first_name() - last_name = faker.last_name() - - # Create user - headers = { - "Content-Type": "application/json", - "Authorization": f"Basic {segment_secrets['user_token']}", - } - body = { - "userId": email, - "traits": { - "subscriptionStatus": "active", - "address": { - "city": faker.city(), - "country": faker.country(), - "postalCode": faker.postcode(), - "state": "NY", - }, - "age": random.randrange(18, 99), - "avatar": "", - "industry": "data", - "description": faker.job(), - "email": email, - "firstName": first_name, - "id": ts, - "lastName": last_name, - "name": f"{first_name} {last_name}", - "phone": faker.phone_number(), - "title": faker.prefix(), - "username": f"test_fidesops_user_{ts}", - "website": "www.example.com", - }, - } - resp = requests.post( - f"https://{segment_secrets['api_domain']}identify", headers=headers, json=body - ) - assert resp.status_code == 200 - - # Create event - body = { - "userId": email, - "type": "track", - "event": "User Registered", - "properties": {"plan": "Free", "accountType": faker.company()}, - "context": {"ip": faker.ipv4()}, - } - - resp = requests.post( - f"https://{segment_secrets['api_domain']}track", headers=headers, json=body - ) - assert resp.status_code == 200 - return email - - @pytest.mark.integration_saas @pytest.mark.integration_segment def test_segment_saas_erasure_request_task( @@ -217,16 +142,14 @@ def test_segment_saas_erasure_request_task( policy, segment_connection_config, segment_dataset_config, - segment_identity_email, + segment_erasure_identity_email, + segment_erasure_data ) -> None: """Full erasure request based on the Segment SaaS config""" config.execution.MASKING_STRICT = False # Allow GDPR Delete # Create user for GDPR delete - erasure_email = create_segment_test_data( - segment_connection_config, segment_identity_email - ) - time.sleep(8) # Pause before making access/erasure requests + erasure_email = segment_erasure_identity_email privacy_request = PrivacyRequest( id=f"test_saas_access_request_task_{random.randint(0, 1000)}" ) diff --git a/tests/integration_tests/saas/test_sentry_task.py b/tests/integration_tests/saas/test_sentry_task.py index c36617b52..d71ae5e47 100644 --- a/tests/integration_tests/saas/test_sentry_task.py +++ b/tests/integration_tests/saas/test_sentry_task.py @@ -1,14 +1,15 @@ -import requests +import random +import time +from typing import Any, Dict, List, Optional -from fidesops.task.filter_results import filter_data_categories import pytest -import random +import requests from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity - from fidesops.task import graph_task +from fidesops.task.filter_results import filter_data_categories from fidesops.task.graph_task import get_cached_data_for_erasures from tests.graph.graph_test_util import assert_rows_match @@ -205,12 +206,26 @@ def test_sentry_access_request_task( ) +def _get_issues( + project: Dict[str, Any], + secrets: Dict[str, Any], + headers: Dict[str, Any], +) -> Optional[List[Dict[str, Any]]]: + response = requests.get( + f"https://{secrets['host']}/api/0/projects/{project['organization']['slug']}/{project['slug']}/issues/", + headers=headers, + ) + json = response.json() + return json if response.ok and len(json) else None + + def sentry_erasure_test_prep(sentry_connection_config, db): sentry_secrets = sentry_connection_config.secrets # Set the assignedTo field on a sentry issue to a given employee token = sentry_secrets.get("erasure_access_token") issue_url = sentry_secrets.get("issue_url") sentry_user_id = sentry_secrets.get("user_id_erasure") + host = sentry_secrets.get("host") if not token or not issue_url or not sentry_user_id: # Exit early if these haven't been set locally @@ -218,9 +233,24 @@ def sentry_erasure_test_prep(sentry_connection_config, db): headers = {"Authorization": f"Bearer {token}"} data = {"assignedTo": f"user:{sentry_user_id}"} - resp = requests.put(issue_url, json=data, headers=headers) - assert resp.status_code == 200 - assert resp.json()["assignedTo"]["id"] == sentry_user_id + response = requests.put(issue_url, json=data, headers=headers) + assert response.ok + assert response.json().get("assignedTo", {}).get("id") == sentry_user_id + + # Get projects + response = requests.get(f"https://{host}/api/0/projects/", headers=headers) + assert response.ok + project = response.json()[0] + + # Wait until issues returns data + retries = 10 + while _get_issues(project, sentry_secrets, headers) is None: + if not retries: + raise Exception( + "The issues endpoint did not return the required data for testing during the time limit" + ) + retries -= 1 + time.sleep(5) # Temporarily sets the access token to one that works for erasures sentry_connection_config.secrets["access_token"] = sentry_secrets[ diff --git a/tests/integration_tests/saas/test_stripe_task.py b/tests/integration_tests/saas/test_stripe_task.py index ee8311efc..92cd68487 100644 --- a/tests/integration_tests/saas/test_stripe_task.py +++ b/tests/integration_tests/saas/test_stripe_task.py @@ -1,14 +1,16 @@ -import pytest import random +from typing import List + +import pytest import requests + from fidesops.core.config import config from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures from fidesops.task.filter_results import filter_data_categories -from tests.fixtures.saas.stripe_fixtures import stripe_secrets +from fidesops.task.graph_task import get_cached_data_for_erasures from tests.graph.graph_test_util import assert_rows_match @@ -94,7 +96,7 @@ def test_stripe_access_request_task( assert_rows_match( v[f"{dataset_name}:charge"], - min_size=2, + min_size=3, keys=[ "amount", "amount_captured", @@ -338,7 +340,7 @@ def test_stripe_access_request_task( assert_rows_match( v[f"{dataset_name}:payment_intent"], - min_size=4, + min_size=5, keys=[ "amount", "amount_capturable", @@ -454,8 +456,6 @@ def test_stripe_access_request_task( # verify we only returned data for our identity email assert v[f"{dataset_name}:customer"][0]["email"] == stripe_identity_email customer_id: str = v[f"{dataset_name}:customer"][0]["id"] - charge_id: str = v[f"{dataset_name}:charge"][0]["id"] - payment_intent_id: str = v[f"{dataset_name}:payment_intent"][0]["id"] for bank_account in v[f"{dataset_name}:bank_account"]: assert bank_account["customer"] == customer_id @@ -463,8 +463,15 @@ def test_stripe_access_request_task( for card in v[f"{dataset_name}:card"]: assert card["customer"] == customer_id + charge_ids: List[str] = [] for charge in v[f"{dataset_name}:charge"]: assert charge["customer"] == customer_id + charge_ids.append(charge["id"]) + + payment_intent_ids: List[str] = [] + for payment_intent in v[f"{dataset_name}:payment_intent"]: + assert payment_intent["customer"] == customer_id + payment_intent_ids.append(payment_intent["id"]) for credit_note in v[f"{dataset_name}:credit_note"]: assert credit_note["customer"] == customer_id @@ -480,8 +487,8 @@ def test_stripe_access_request_task( # disputes are retrieved by charge.id or payment_intent.id for dispute in v[f"{dataset_name}:dispute"]: assert ( - dispute["charge"] == charge_id - or dispute["payment_intent_id"] == payment_intent_id + dispute["charge"] in charge_ids + or dispute["payment_intent"] in payment_intent_ids ) for invoice in v[f"{dataset_name}:invoice"]: @@ -490,9 +497,6 @@ def test_stripe_access_request_task( for invoice_item in v[f"{dataset_name}:invoice_item"]: assert invoice_item["customer"] == customer_id - for payment_intent in v[f"{dataset_name}:payment_intent"]: - assert payment_intent["customer"] == customer_id - for payment_method in v[f"{dataset_name}:payment_method"]: assert payment_method["customer"] == customer_id