From c584d7dc220c8c8df0e3bddd5afd9bafad604f6d Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 5 May 2022 15:38:43 -0700 Subject: [PATCH 1/7] Adding waits in the erasure test fixtures for Hubspot, Segment, and Sentry Updating Stripe access test --- src/fidesops/task/filter_results.py | 2 +- tests/fixtures/saas/hubspot_fixtures.py | 6 +- tests/fixtures/saas/segment_fixtures.py | 151 +++++++++++++++++- .../saas/test_segment_task.py | 79 +-------- .../saas/test_sentry_task.py | 39 ++++- .../saas/test_stripe_task.py | 22 +-- 6 files changed, 200 insertions(+), 99 deletions(-) diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/task/filter_results.py index bb812c41f..53c4bdfbf 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/task/filter_results.py @@ -98,7 +98,7 @@ def _defaultdict_or_array(resource: Any) -> Any: elif isinstance(row, dict): for key in row: - if key == target_path.levels[0]: + if len(target_path.levels) > 0 and key == target_path.levels[0]: if key not in saved: saved[key] = _defaultdict_or_array(row[key]) saved[key] = select_and_save_field( diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index e8bf05b93..78ceb4e21 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -137,8 +137,9 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema # no need to subscribe contact, since creating a contact auto-subscribes them # Allows contact to be propagated in Hubspot before calling access / erasure requests - remaining_tries = 5 + remaining_tries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is False: + remaining_tries -= 1 if remaining_tries < 1: raise Exception(f"Contact with contact id {contact_id} could not be added to Hubspot") time.sleep(5) @@ -153,8 +154,9 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema connector.create_client().send(delete_request) # verify contact is deleted - remaining_tries = 5 + remaining_tries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is True: + remaining_tries -= 1 if remaining_tries < 1: raise Exception(f"Contact with contact id {contact_id} could not be deleted from Hubspot") time.sleep(5) # Ensures contact is deleted diff --git a/tests/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py index 25fa8cb71..a95917e10 100644 --- a/tests/fixtures/saas/segment_fixtures.py +++ b/tests/fixtures/saas/segment_fixtures.py @@ -1,3 +1,7 @@ +import random +import time +import requests +from faker import Faker from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -20,19 +24,26 @@ @pytest.fixture(scope="function") def segment_secrets(): return { - "domain": pydash.get(saas_config, "segment.domain") or os.environ.get("SEGMENT_DOMAIN"), + "domain": pydash.get(saas_config, "segment.domain") + or os.environ.get("SEGMENT_DOMAIN"), "personas_domain": pydash.get(saas_config, "segment.personas_domain") or os.environ.get("SEGMENT_PERSONAS_DOMAIN"), - "workspace": pydash.get(saas_config, "segment.workspace") or os.environ.get("SEGMENT_WORKSPACE"), - "access_token": pydash.get(saas_config, "segment.access_token") or os.environ.get("SEGMENT_ACCESS_TOKEN"), - "namespace_id": pydash.get(saas_config, "segment.namespace_id") or os.environ.get("SEGMENT_NAMESPACE_ID"), - "access_secret": pydash.get(saas_config, "segment.access_secret") or os.environ.get("SEGMENT_ACCESS_SECRET"), - "api_domain": pydash.get(saas_config, "segment.api_domain") or os.environ.get("SEGMENT_API_DOMAIN"), - "user_token": pydash.get(saas_config, "segment.user_token") or os.environ.get("SEGMENT_USER_TOKEN"), + "workspace": pydash.get(saas_config, "segment.workspace") + or os.environ.get("SEGMENT_WORKSPACE"), + "access_token": pydash.get(saas_config, "segment.access_token") + or os.environ.get("SEGMENT_ACCESS_TOKEN"), + "namespace_id": pydash.get(saas_config, "segment.namespace_id") + or os.environ.get("SEGMENT_NAMESPACE_ID"), + "access_secret": pydash.get(saas_config, "segment.access_secret") + or os.environ.get("SEGMENT_ACCESS_SECRET"), + "api_domain": pydash.get(saas_config, "segment.api_domain") + or os.environ.get("SEGMENT_API_DOMAIN"), + "user_token": pydash.get(saas_config, "segment.user_token") + or os.environ.get("SEGMENT_USER_TOKEN"), } -@pytest.fixture(scope="function") +@pytest.fixture(scope="session") def segment_identity_email(): return pydash.get(saas_config, "segment.identity_email") or os.environ.get( "SEGMENT_IDENTITY_EMAIL" @@ -89,3 +100,127 @@ def segment_dataset_config( ) yield dataset dataset.delete(db=db) + + +@pytest.fixture(scope="session") +def segment_erasure_identity_email(segment_identity_email) -> str: + timestamp = int(time.time()) + at_index: int = segment_identity_email.find("@") + email = f"{segment_identity_email[0:at_index]}{timestamp}{segment_identity_email[at_index:]}" + return email + + +def _get_user_id(email: str, secrets: Dict[str, Any]) -> str: + personas_domain = secrets["personas_domain"] + namespace_id = secrets["namespace_id"] + access_secret = secrets["access_secret"] + response = requests.get( + f"https://{personas_domain}/v1/spaces/{namespace_id}/collections/users/profiles/user_id:{email}/metadata", + auth=(access_secret, None), + ) + if not response.ok: + return None + + return response.json()["segment_id"] + + +def _get_track_events(segment_id: str, secrets: Dict[str, Any]) -> Dict[str, Any]: + personas_domain = secrets["personas_domain"] + namespace_id = secrets["namespace_id"] + access_secret = secrets["access_secret"] + + response = requests.get( + f"https://{personas_domain}/v1/spaces/{namespace_id}/collections/users/profiles/{segment_id}/events", + auth=(access_secret, None), + ) + if not response.ok or response.json()["data"] is None: + return None + + return response.json()["data"][0] + + +@pytest.fixture(scope="function") +def segment_erasure_data( + segment_connection_config, segment_erasure_identity_email +) -> str: + """Seeds a segment user and event""" + segment_secrets = segment_connection_config.secrets + if not segment_identity_email: # Don't run unnecessarily locally + return + + api_domain = segment_secrets["api_domain"] + user_token = segment_secrets["user_token"] + + faker = Faker() + + timestamp = int(time.time()) + email = segment_erasure_identity_email + first_name = faker.first_name() + last_name = faker.last_name() + + # Create user + headers = { + "Content-Type": "application/json", + "Authorization": f"Basic {user_token}", + } + body = { + "userId": email, + "traits": { + "subscriptionStatus": "active", + "address": { + "city": faker.city(), + "country": faker.country(), + "postalCode": faker.postcode(), + "state": "NY", + }, + "age": random.randrange(18, 99), + "avatar": "", + "industry": "data", + "description": faker.job(), + "email": email, + "firstName": first_name, + "id": timestamp, + "lastName": last_name, + "name": f"{first_name} {last_name}", + "phone": faker.phone_number(), + "title": faker.prefix(), + "username": f"test_fidesops_user_{timestamp}", + "website": "www.example.com", + }, + } + response = requests.post( + f"https://{api_domain}identify", headers=headers, json=body + ) + assert response.ok + + # Wait until user returns data + remaining_tries = 10 + while (segment_id := _get_user_id(email, segment_secrets)) is None: + remaining_tries -= 1 + if remaining_tries < 1: + raise Exception( + "The user endpoint did not return the required data for testing during the time limit" + ) + time.sleep(5) + + # Create event + body = { + "userId": email, + "type": "track", + "event": "User Registered", + "properties": {"plan": "Free", "accountType": faker.company()}, + "context": {"ip": faker.ipv4()}, + } + + response = requests.post(f"https://{api_domain}track", headers=headers, json=body) + assert response.ok + + # Wait until track_events returns data + remaining_tries = 10 + while _get_track_events(segment_id, segment_secrets) is None: + remaining_tries -= 1 + if remaining_tries < 1: + raise Exception( + "The track_events endpoint did not return the required data for testing during the time limit" + ) + time.sleep(5) diff --git a/tests/integration_tests/saas/test_segment_task.py b/tests/integration_tests/saas/test_segment_task.py index 7ce0c5f71..15e981d47 100644 --- a/tests/integration_tests/saas/test_segment_task.py +++ b/tests/integration_tests/saas/test_segment_task.py @@ -14,6 +14,7 @@ from fidesops.task import graph_task from fidesops.task.graph_task import get_cached_data_for_erasures +from tests.fixtures.saas.segment_fixtures import segment_erasure_data from tests.graph.graph_test_util import assert_rows_match @@ -140,78 +141,6 @@ def test_segment_saas_access_request_task( assert filtered_results[f"{dataset_name}:segment_user"][0]["segment_id"] -def _create_test_segment_email(base_email: str, timestamp: int) -> str: - at_index: int = base_email.find("@") - email = f"{base_email[0:at_index]}{timestamp}{base_email[at_index:]}" - return email - - -def create_segment_test_data( - segment_connection_config, segment_identity_email: str -): - """Seeds a segment user and event""" - segment_secrets = segment_connection_config.secrets - if not segment_identity_email: # Don't run unnecessarily locally - return - - faker = Faker() - - ts = int(time.time()) - email = _create_test_segment_email(segment_identity_email, ts) - first_name = faker.first_name() - last_name = faker.last_name() - - # Create user - headers = { - "Content-Type": "application/json", - "Authorization": f"Basic {segment_secrets['user_token']}", - } - body = { - "userId": email, - "traits": { - "subscriptionStatus": "active", - "address": { - "city": faker.city(), - "country": faker.country(), - "postalCode": faker.postcode(), - "state": "NY", - }, - "age": random.randrange(18, 99), - "avatar": "", - "industry": "data", - "description": faker.job(), - "email": email, - "firstName": first_name, - "id": ts, - "lastName": last_name, - "name": f"{first_name} {last_name}", - "phone": faker.phone_number(), - "title": faker.prefix(), - "username": f"test_fidesops_user_{ts}", - "website": "www.example.com", - }, - } - resp = requests.post( - f"https://{segment_secrets['api_domain']}identify", headers=headers, json=body - ) - assert resp.status_code == 200 - - # Create event - body = { - "userId": email, - "type": "track", - "event": "User Registered", - "properties": {"plan": "Free", "accountType": faker.company()}, - "context": {"ip": faker.ipv4()}, - } - - resp = requests.post( - f"https://{segment_secrets['api_domain']}track", headers=headers, json=body - ) - assert resp.status_code == 200 - return email - - @pytest.mark.integration_saas @pytest.mark.integration_segment def test_segment_saas_erasure_request_task( @@ -219,14 +148,14 @@ def test_segment_saas_erasure_request_task( policy, segment_connection_config, segment_dataset_config, - segment_identity_email, + segment_erasure_identity_email, + segment_erasure_data ) -> None: """Full erasure request based on the Segment SaaS config""" config.execution.MASKING_STRICT = False # Allow GDPR Delete # Create user for GDPR delete - erasure_email = create_segment_test_data(segment_connection_config, segment_identity_email) - time.sleep(8) # Pause before making access/erasure requests + erasure_email = segment_erasure_identity_email privacy_request = PrivacyRequest( id=f"test_saas_access_request_task_{random.randint(0, 1000)}" ) diff --git a/tests/integration_tests/saas/test_sentry_task.py b/tests/integration_tests/saas/test_sentry_task.py index c36617b52..2ee31a82a 100644 --- a/tests/integration_tests/saas/test_sentry_task.py +++ b/tests/integration_tests/saas/test_sentry_task.py @@ -1,3 +1,5 @@ +import time +from typing import Any, Dict, Optional import requests from fidesops.task.filter_results import filter_data_categories @@ -205,12 +207,28 @@ def test_sentry_access_request_task( ) +def _get_issues( + project: Dict[str, Any], + secrets: Dict[str, Any], + headers: Dict[str, Any], +) -> Optional[Dict[str, Any]]: + response = requests.get( + f"https://{secrets['host']}/api/0/projects/{project['organization']['slug']}/{project['slug']}/issues/", + headers=headers, + ) + if not response.ok or response.json() == []: + return None + + return response.json() + + def sentry_erasure_test_prep(sentry_connection_config, db): sentry_secrets = sentry_connection_config.secrets # Set the assignedTo field on a sentry issue to a given employee token = sentry_secrets.get("erasure_access_token") issue_url = sentry_secrets.get("issue_url") sentry_user_id = sentry_secrets.get("user_id_erasure") + host = sentry_secrets.get("host") if not token or not issue_url or not sentry_user_id: # Exit early if these haven't been set locally @@ -218,9 +236,24 @@ def sentry_erasure_test_prep(sentry_connection_config, db): headers = {"Authorization": f"Bearer {token}"} data = {"assignedTo": f"user:{sentry_user_id}"} - resp = requests.put(issue_url, json=data, headers=headers) - assert resp.status_code == 200 - assert resp.json()["assignedTo"]["id"] == sentry_user_id + response = requests.put(issue_url, json=data, headers=headers) + assert response.ok + assert response.json()["assignedTo"]["id"] == sentry_user_id + + # Get projects + response = requests.get(f"https://{host}/api/0/projects/", headers=headers) + assert response.ok + project = response.json()[0] + + # Wait until issues returns data + remaining_tries = 10 + while _get_issues(project, sentry_secrets, headers) is None: + remaining_tries -= 1 + if remaining_tries < 1: + raise Exception( + "The issues endpoint did not return the required data for testing during the time limit" + ) + time.sleep(5) # Temporarily sets the access token to one that works for erasures sentry_connection_config.secrets["access_token"] = sentry_secrets[ diff --git a/tests/integration_tests/saas/test_stripe_task.py b/tests/integration_tests/saas/test_stripe_task.py index ee8311efc..d66196b19 100644 --- a/tests/integration_tests/saas/test_stripe_task.py +++ b/tests/integration_tests/saas/test_stripe_task.py @@ -1,3 +1,4 @@ +from typing import List import pytest import random import requests @@ -8,7 +9,6 @@ from fidesops.task import graph_task from fidesops.task.graph_task import get_cached_data_for_erasures from fidesops.task.filter_results import filter_data_categories -from tests.fixtures.saas.stripe_fixtures import stripe_secrets from tests.graph.graph_test_util import assert_rows_match @@ -94,7 +94,7 @@ def test_stripe_access_request_task( assert_rows_match( v[f"{dataset_name}:charge"], - min_size=2, + min_size=3, keys=[ "amount", "amount_captured", @@ -338,7 +338,7 @@ def test_stripe_access_request_task( assert_rows_match( v[f"{dataset_name}:payment_intent"], - min_size=4, + min_size=5, keys=[ "amount", "amount_capturable", @@ -454,8 +454,6 @@ def test_stripe_access_request_task( # verify we only returned data for our identity email assert v[f"{dataset_name}:customer"][0]["email"] == stripe_identity_email customer_id: str = v[f"{dataset_name}:customer"][0]["id"] - charge_id: str = v[f"{dataset_name}:charge"][0]["id"] - payment_intent_id: str = v[f"{dataset_name}:payment_intent"][0]["id"] for bank_account in v[f"{dataset_name}:bank_account"]: assert bank_account["customer"] == customer_id @@ -463,8 +461,15 @@ def test_stripe_access_request_task( for card in v[f"{dataset_name}:card"]: assert card["customer"] == customer_id + charge_ids: List[str] = [] for charge in v[f"{dataset_name}:charge"]: assert charge["customer"] == customer_id + charge_ids.append(charge["id"]) + + payment_intent_ids: List[str] = [] + for payment_intent in v[f"{dataset_name}:payment_intent"]: + assert payment_intent["customer"] == customer_id + payment_intent_ids.append(payment_intent["id"]) for credit_note in v[f"{dataset_name}:credit_note"]: assert credit_note["customer"] == customer_id @@ -480,8 +485,8 @@ def test_stripe_access_request_task( # disputes are retrieved by charge.id or payment_intent.id for dispute in v[f"{dataset_name}:dispute"]: assert ( - dispute["charge"] == charge_id - or dispute["payment_intent_id"] == payment_intent_id + dispute["charge"] in charge_ids + or dispute["payment_intent"] in payment_intent_ids ) for invoice in v[f"{dataset_name}:invoice"]: @@ -490,9 +495,6 @@ def test_stripe_access_request_task( for invoice_item in v[f"{dataset_name}:invoice_item"]: assert invoice_item["customer"] == customer_id - for payment_intent in v[f"{dataset_name}:payment_intent"]: - assert payment_intent["customer"] == customer_id - for payment_method in v[f"{dataset_name}:payment_method"]: assert payment_method["customer"] == customer_id From 04551f2c65394ac4794ca3d55a8cf3a18234959b Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 9 May 2022 16:31:00 -0700 Subject: [PATCH 2/7] Fixing import order and other misc recommendations --- src/fidesops/task/filter_results.py | 2 +- tests/fixtures/saas/hubspot_fixtures.py | 12 ++--- tests/fixtures/saas/segment_fixtures.py | 45 +++++++++---------- .../saas/test_sentry_task.py | 17 ++++--- .../saas/test_stripe_task.py | 6 ++- 5 files changed, 39 insertions(+), 43 deletions(-) diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/task/filter_results.py index 53c4bdfbf..2b7ba3d6d 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/task/filter_results.py @@ -98,7 +98,7 @@ def _defaultdict_or_array(resource: Any) -> Any: elif isinstance(row, dict): for key in row: - if len(target_path.levels) > 0 and key == target_path.levels[0]: + if len(target_path.levels) and key == target_path.levels[0]: if key not in saved: saved[key] = _defaultdict_or_array(row[key]) saved[key] = select_and_save_field( diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 78ceb4e21..fbb74c48b 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -137,11 +137,11 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema # no need to subscribe contact, since creating a contact auto-subscribes them # Allows contact to be propagated in Hubspot before calling access / erasure requests - remaining_tries = 10 + retries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is False: - remaining_tries -= 1 - if remaining_tries < 1: + if not retries: raise Exception(f"Contact with contact id {contact_id} could not be added to Hubspot") + retries -= 1 time.sleep(5) yield contact_id @@ -154,11 +154,11 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema connector.create_client().send(delete_request) # verify contact is deleted - remaining_tries = 10 + retries = 10 while _contact_exists(hubspot_erasure_identity_email, connector) is True: - remaining_tries -= 1 - if remaining_tries < 1: + if not retries: raise Exception(f"Contact with contact id {contact_id} could not be deleted from Hubspot") + retries -= 1 time.sleep(5) # Ensures contact is deleted diff --git a/tests/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py index a95917e10..aac7fe0c1 100644 --- a/tests/fixtures/saas/segment_fixtures.py +++ b/tests/fixtures/saas/segment_fixtures.py @@ -1,7 +1,14 @@ +import os import random import time +from typing import Any, Dict, Generator + +import pydash +import pytest import requests from faker import Faker +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -10,13 +17,8 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import os -from typing import Any, Dict, Generator from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml") @@ -24,22 +26,15 @@ @pytest.fixture(scope="function") def segment_secrets(): return { - "domain": pydash.get(saas_config, "segment.domain") - or os.environ.get("SEGMENT_DOMAIN"), + "domain": pydash.get(saas_config, "segment.domain") or os.environ.get("SEGMENT_DOMAIN"), "personas_domain": pydash.get(saas_config, "segment.personas_domain") or os.environ.get("SEGMENT_PERSONAS_DOMAIN"), - "workspace": pydash.get(saas_config, "segment.workspace") - or os.environ.get("SEGMENT_WORKSPACE"), - "access_token": pydash.get(saas_config, "segment.access_token") - or os.environ.get("SEGMENT_ACCESS_TOKEN"), - "namespace_id": pydash.get(saas_config, "segment.namespace_id") - or os.environ.get("SEGMENT_NAMESPACE_ID"), - "access_secret": pydash.get(saas_config, "segment.access_secret") - or os.environ.get("SEGMENT_ACCESS_SECRET"), - "api_domain": pydash.get(saas_config, "segment.api_domain") - or os.environ.get("SEGMENT_API_DOMAIN"), - "user_token": pydash.get(saas_config, "segment.user_token") - or os.environ.get("SEGMENT_USER_TOKEN"), + "workspace": pydash.get(saas_config, "segment.workspace") or os.environ.get("SEGMENT_WORKSPACE"), + "access_token": pydash.get(saas_config, "segment.access_token") or os.environ.get("SEGMENT_ACCESS_TOKEN"), + "namespace_id": pydash.get(saas_config, "segment.namespace_id") or os.environ.get("SEGMENT_NAMESPACE_ID"), + "access_secret": pydash.get(saas_config, "segment.access_secret") or os.environ.get("SEGMENT_ACCESS_SECRET"), + "api_domain": pydash.get(saas_config, "segment.api_domain") or os.environ.get("SEGMENT_API_DOMAIN"), + "user_token": pydash.get(saas_config, "segment.user_token") or os.environ.get("SEGMENT_USER_TOKEN"), } @@ -194,13 +189,13 @@ def segment_erasure_data( assert response.ok # Wait until user returns data - remaining_tries = 10 + retries = 10 while (segment_id := _get_user_id(email, segment_secrets)) is None: - remaining_tries -= 1 - if remaining_tries < 1: + if not retries: raise Exception( "The user endpoint did not return the required data for testing during the time limit" ) + retries -= 1 time.sleep(5) # Create event @@ -216,11 +211,11 @@ def segment_erasure_data( assert response.ok # Wait until track_events returns data - remaining_tries = 10 + retries = 10 while _get_track_events(segment_id, segment_secrets) is None: - remaining_tries -= 1 - if remaining_tries < 1: + if not retries: raise Exception( "The track_events endpoint did not return the required data for testing during the time limit" ) + retries -= 1 time.sleep(5) diff --git a/tests/integration_tests/saas/test_sentry_task.py b/tests/integration_tests/saas/test_sentry_task.py index 2ee31a82a..5b0506387 100644 --- a/tests/integration_tests/saas/test_sentry_task.py +++ b/tests/integration_tests/saas/test_sentry_task.py @@ -1,16 +1,15 @@ +import random import time -from typing import Any, Dict, Optional -import requests +from typing import Any, Dict, List, Optional -from fidesops.task.filter_results import filter_data_categories import pytest -import random +import requests from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity - from fidesops.task import graph_task +from fidesops.task.filter_results import filter_data_categories from fidesops.task.graph_task import get_cached_data_for_erasures from tests.graph.graph_test_util import assert_rows_match @@ -211,7 +210,7 @@ def _get_issues( project: Dict[str, Any], secrets: Dict[str, Any], headers: Dict[str, Any], -) -> Optional[Dict[str, Any]]: +) -> Optional[List[Dict[str, Any]]]: response = requests.get( f"https://{secrets['host']}/api/0/projects/{project['organization']['slug']}/{project['slug']}/issues/", headers=headers, @@ -246,13 +245,13 @@ def sentry_erasure_test_prep(sentry_connection_config, db): project = response.json()[0] # Wait until issues returns data - remaining_tries = 10 + retries = 10 while _get_issues(project, sentry_secrets, headers) is None: - remaining_tries -= 1 - if remaining_tries < 1: + if not retries: raise Exception( "The issues endpoint did not return the required data for testing during the time limit" ) + retries -= 1 time.sleep(5) # Temporarily sets the access token to one that works for erasures diff --git a/tests/integration_tests/saas/test_stripe_task.py b/tests/integration_tests/saas/test_stripe_task.py index d66196b19..92cd68487 100644 --- a/tests/integration_tests/saas/test_stripe_task.py +++ b/tests/integration_tests/saas/test_stripe_task.py @@ -1,14 +1,16 @@ +import random from typing import List + import pytest -import random import requests + from fidesops.core.config import config from fidesops.graph.graph import DatasetGraph from fidesops.models.privacy_request import PrivacyRequest from fidesops.schemas.redis_cache import PrivacyRequestIdentity from fidesops.task import graph_task -from fidesops.task.graph_task import get_cached_data_for_erasures from fidesops.task.filter_results import filter_data_categories +from fidesops.task.graph_task import get_cached_data_for_erasures from tests.graph.graph_test_util import assert_rows_match From f57f7baddabc4a428f2160b35664e69bbc0d0ae8 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 9 May 2022 16:41:10 -0700 Subject: [PATCH 3/7] Re-adding explicit check to fix linter error --- src/fidesops/task/filter_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/task/filter_results.py index 2b7ba3d6d..53c4bdfbf 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/task/filter_results.py @@ -98,7 +98,7 @@ def _defaultdict_or_array(resource: Any) -> Any: elif isinstance(row, dict): for key in row: - if len(target_path.levels) and key == target_path.levels[0]: + if len(target_path.levels) > 0 and key == target_path.levels[0]: if key not in saved: saved[key] = _defaultdict_or_array(row[key]) saved[key] = select_and_save_field( From 1c99ecb40f06249cc86a5e2f4a6e52921aa4cb33 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 9 May 2022 16:42:50 -0700 Subject: [PATCH 4/7] Simplifying tuple check --- src/fidesops/task/filter_results.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fidesops/task/filter_results.py b/src/fidesops/task/filter_results.py index 53c4bdfbf..c242b94ac 100644 --- a/src/fidesops/task/filter_results.py +++ b/src/fidesops/task/filter_results.py @@ -98,7 +98,7 @@ def _defaultdict_or_array(resource: Any) -> Any: elif isinstance(row, dict): for key in row: - if len(target_path.levels) > 0 and key == target_path.levels[0]: + if target_path.levels and key == target_path.levels[0]: if key not in saved: saved[key] = _defaultdict_or_array(row[key]) saved[key] = select_and_save_field( From 1a2cdd01cfff28478ff02ff0523adf2431e31ede Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 9 May 2022 20:27:07 -0700 Subject: [PATCH 5/7] More fixes --- tests/integration_tests/saas/test_sentry_task.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/integration_tests/saas/test_sentry_task.py b/tests/integration_tests/saas/test_sentry_task.py index 5b0506387..d71ae5e47 100644 --- a/tests/integration_tests/saas/test_sentry_task.py +++ b/tests/integration_tests/saas/test_sentry_task.py @@ -215,10 +215,8 @@ def _get_issues( f"https://{secrets['host']}/api/0/projects/{project['organization']['slug']}/{project['slug']}/issues/", headers=headers, ) - if not response.ok or response.json() == []: - return None - - return response.json() + json = response.json() + return json if response.ok and len(json) else None def sentry_erasure_test_prep(sentry_connection_config, db): @@ -237,7 +235,7 @@ def sentry_erasure_test_prep(sentry_connection_config, db): data = {"assignedTo": f"user:{sentry_user_id}"} response = requests.put(issue_url, json=data, headers=headers) assert response.ok - assert response.json()["assignedTo"]["id"] == sentry_user_id + assert response.json().get("assignedTo", {}).get("id") == sentry_user_id # Get projects response = requests.get(f"https://{host}/api/0/projects/", headers=headers) From 433cb847a7a35e251d1de18a1576ac220816ac88 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 10 May 2022 09:44:54 -0700 Subject: [PATCH 6/7] Adding import that was dropped during import cleanup --- tests/fixtures/saas/hubspot_fixtures.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 6eda75e44..de1349f8c 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -7,6 +7,7 @@ import pytest from sqlalchemy.orm import Session +from fidesops.core.config import load_toml from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors import SaaSConnector from fidesops.util import cryptographic_util From 2cded5e51c92aad5d60b77313b54b04713368351 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 10 May 2022 09:57:01 -0700 Subject: [PATCH 7/7] Fixing imports for other fixtures --- tests/fixtures/saas/hubspot_fixtures.py | 6 ++++++ tests/fixtures/saas/sentry_fixtures.py | 12 +++++++----- tests/fixtures/saas/stripe_fixtures.py | 11 ++++++----- 3 files changed, 19 insertions(+), 10 deletions(-) diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index de1349f8c..2260d2bfc 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -8,6 +8,12 @@ from sqlalchemy.orm import Session from fidesops.core.config import load_toml +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionConfig, + ConnectionType, +) +from fidesops.models.datasetconfig import DatasetConfig from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors import SaaSConnector from fidesops.util import cryptographic_util diff --git a/tests/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py index 8d4bd3410..923953b24 100644 --- a/tests/fixtures/saas/sentry_fixtures.py +++ b/tests/fixtures/saas/sentry_fixtures.py @@ -1,3 +1,10 @@ +import os +from typing import Any, Dict, Generator + +import pydash +import pytest +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -6,13 +13,8 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import os -from typing import Any, Dict, Generator from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml") diff --git a/tests/fixtures/saas/stripe_fixtures.py b/tests/fixtures/saas/stripe_fixtures.py index 37e1a66d1..7c476a991 100644 --- a/tests/fixtures/saas/stripe_fixtures.py +++ b/tests/fixtures/saas/stripe_fixtures.py @@ -1,7 +1,12 @@ import os -from multidimensional_urlencode import urlencode as multidimensional_urlencode from typing import Any, Dict, Generator +import pydash +import pytest +import requests +from multidimensional_urlencode import urlencode as multidimensional_urlencode +from sqlalchemy.orm import Session + from fidesops.core.config import load_toml from fidesops.db import session from fidesops.models.connectionconfig import ( @@ -10,12 +15,8 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -import pytest -import pydash -import requests from tests.fixtures.application_fixtures import load_dataset from tests.fixtures.saas_example_fixtures import load_config -from sqlalchemy.orm import Session saas_config = load_toml("saas_config.toml")