diff --git a/data/saas/config/recharge_config.yml b/data/saas/config/recharge_config.yml index 3494f27d557..bde298f21b5 100644 --- a/data/saas/config/recharge_config.yml +++ b/data/saas/config/recharge_config.yml @@ -36,6 +36,19 @@ saas_config: - name: email identity: email data_path: customers + update: + method: PUT + path: /customers/ + body: | + { + + } + param_values: + - name: customer_id + references: + - dataset: + field: customers.customer_id + direction: from - name: addresses requests: @@ -52,4 +65,16 @@ saas_config: field: customers.id direction: from data_path: addresses - + update: + method: PUT + path: /addresses/ + body: | + { + + } + param_values: + - name: address_id + references: + - dataset: + field: addresses.address_id + direction: from diff --git a/data/saas/dataset/recharge_dataset.yml b/data/saas/dataset/recharge_dataset.yml index bf4785d50ed..6da20f6025c 100644 --- a/data/saas/dataset/recharge_dataset.yml +++ b/data/saas/dataset/recharge_dataset.yml @@ -62,7 +62,7 @@ dataset: fidesops_meta: data_type: string - name: id - data_categories: [ user.unique_id ] + data_categories: [ system.operations ] fidesops_meta: data_type: string - name: last_name diff --git a/pyproject.toml b/pyproject.toml index a07574bca0f..23ed40337ed 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -151,7 +151,7 @@ extension-pkg-whitelist = ["pydantic", "zlib"] env = [ "TESTING=True" ] -log_cli=false +log_cli=true filterwarnings = "ignore::DeprecationWarning:aiofiles.*:" testpaths="tests" log_level = "INFO" diff --git a/tests/ops/fixtures/saas/recharge_fixtures.py b/tests/ops/fixtures/saas/recharge_fixtures.py index 62c6ae79108..1ce514ae4f8 100644 --- a/tests/ops/fixtures/saas/recharge_fixtures.py +++ b/tests/ops/fixtures/saas/recharge_fixtures.py @@ -1,8 +1,11 @@ from typing import Any, Dict, Generator - +import uuid import pydash import pytest +import requests +from faker import Faker from fideslib.db import session +from requests import Response from sqlalchemy.orm import Session from fides.api.ops.models.connectionconfig import ( @@ -15,9 +18,9 @@ load_config_with_replacement, load_dataset_with_replacement, ) +from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets - secrets = get_secrets("recharge") @@ -32,10 +35,15 @@ def recharge_secrets(saas_config): @pytest.fixture(scope="function") def recharge_identity_email(saas_config): return ( - pydash.get(saas_config, "recharge.identity_email") or secrets["identity_email"] + pydash.get(saas_config, "recharge.identity_email") or secrets["identity_email"] ) +@pytest.fixture(scope='function') +def recharge_erasure_identity_email(): + return f"{uuid.uuid4().hex}@email.com" + + @pytest.fixture def recharge_config() -> Dict[str, Any]: return load_config_with_replacement( @@ -56,7 +64,7 @@ def recharge_dataset() -> Dict[str, Any]: @pytest.fixture(scope="function") def recharge_connection_config( - db: session, recharge_config, recharge_secrets + db: session, recharge_config, recharge_secrets ) -> Generator: fides_key = recharge_config["fides_key"] connection_config = ConnectionConfig.create( @@ -76,9 +84,9 @@ def recharge_connection_config( @pytest.fixture def recharge_dataset_config( - db: Session, - recharge_connection_config: ConnectionConfig, - recharge_dataset: Dict[str, Any], + db: Session, + recharge_connection_config: ConnectionConfig, + recharge_dataset: Dict[str, Any], ) -> Generator: fides_key = recharge_dataset["fides_key"] recharge_connection_config.name = fides_key @@ -94,3 +102,164 @@ def recharge_dataset_config( ) yield dataset dataset.delete(db=db) + + +class RechargeTestClient: + """Helper to call various Recharge data management requests""" + + def __init__(self, recharge_connection_config: ConnectionConfig): + + self.recharge_secrets = recharge_connection_config.secrets + self.headers = { + "X-Recharge-Access-Token": self.recharge_secrets["api_key"], + "Content-Type": "application/json" + } + self.base_url = f"https://{self.recharge_secrets['domain']}" + self.faker = Faker() + self.first_name = self.faker.first_name() + self.last_name = self.faker.last_name() + self.street_address = self.faker.street_address() + + # 1: Creates, checks for existance and deletes customer + def create_customer(self, email) -> Response: + customer_body = { + "first_name": self.first_name, + "last_name": self.last_name, + "email": email, + "billing_address1": self.street_address, + "billing_city": "New York City", + "billing_province": "New York", + "billing_country": "United States", + "billing_first_name": self.first_name, + "billing_last_name": self.last_name, + "billing_zip": "10001", + } + + customer_response: Response = requests.post( + url=f"{self.base_url}/customers", + json=customer_body, + headers=self.headers, + ) + assert customer_response.ok + + return customer_response + + def customer_exists(self, email): + customer_response: Response = requests.get( + url=f"{self.base_url}/customers", + params={"email": email}, + headers=self.headers, + ) + assert customer_response.ok + return customer_response.json() + + def delete_customer(self, customer_id): + delete_customer_response = requests.delete( + url=f"{self.base_url}/customers/{customer_id}", + headers=self.headers, + data={} + ) + assert delete_customer_response.ok + + # 2: Creates, checks for existance and deletes address + def create_address(self, customer_id) -> Response: + address_body = { + "customer_id": customer_id, + "address1": self.street_address, + "address2": self.street_address, + "city": "Los Angeles", + "company": "Recharge", + "country_code": "US", + "country": "United States", + "first_name": self.first_name, + "last_name": self.last_name, + "order_attributes": [ + { + "name": "custom name", + "value": "custom value" + } + ], + "phone": "5551234567", + "province": "California", + "zip": "90001" + } + address_response = requests.post( + url=f"{self.base_url}/addresses", + headers=self.headers, + json=address_body, + ) + assert address_response.ok + return address_response + + def address_exists(self, customer_id): + address_response: Response = requests.get( + url=f"{self.base_url}/addresses", + params={"customer_id": customer_id}, + headers=self.headers, + ) + assert address_response.ok + return address_response.json() + + def delete_address(self, address_id): + delete_address_response = requests.delete( + url=f"{self.base_url}/addresses/{address_id}", + headers=self.headers, + data={} + ) + assert delete_address_response.ok + + # 3: Creates, checks for existance and deletes order + def create_order(self, ): pass + def order_exists(self, ): pass + def delete_order(self, ): pass + + # 4: Creates, checks for existance and deletes payment_method + def create_payment_method(self): pass + def payment_method_exists(self): pass + def delete_payment_method(self): pass + + # 5: Creates, checks for existance and deletes subscription + def create_subscription(self): pass + def subscription_exist(self): pass + def delete_subscription(self): pass + + +@pytest.fixture(scope="function") +def recharge_test_client( + recharge_connection_config: RechargeTestClient +) -> Generator: + test_client = RechargeTestClient(recharge_connection_config=recharge_connection_config) + yield test_client + + +@pytest.fixture(scope="function") +def recharge_erasure_data( + recharge_test_client: RechargeTestClient, recharge_erasure_identity_email: str +) -> Generator: + customer_response = recharge_test_client.create_customer(recharge_erasure_identity_email) + error_message = ( + f"customer with email {recharge_erasure_identity_email} could not be created in Recharge" + ) + poll_for_existence( + recharge_test_client.customer_exists, + (recharge_erasure_identity_email,), + error_message=error_message, + ) + customer_id = customer_response.json()["customer"]["id"] + + address_response = recharge_test_client.create_address(customer_id) + error_message = ( + f"address for customer '{recharge_erasure_identity_email}' could not be created in Recharge" + ) + poll_for_existence( + recharge_test_client.address_exists, + args=(customer_id,), + error_message=error_message, + ) + address_id = address_response.json()["address"]["id"] + + yield customer_response, address_response + + + recharge_test_client.delete_address(address_id) + recharge_test_client.delete_customer(customer_id) diff --git a/tests/ops/integration_tests/saas/test_recharge_tasks.py b/tests/ops/integration_tests/saas/test_recharge_tasks.py index fa8a71ca82f..f50585ef396 100644 --- a/tests/ops/integration_tests/saas/test_recharge_tasks.py +++ b/tests/ops/integration_tests/saas/test_recharge_tasks.py @@ -1,17 +1,19 @@ import logging -import random - import pytest +import pdb +import random from fides.api.ops.graph.graph import DatasetGraph from fides.api.ops.models.privacy_request import PrivacyRequest from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors import get_connector from fides.api.ops.task import graph_task +from fides.api.ops.task.graph_task import get_cached_data_for_erasures +from fides.ctl.core.config import get_config from tests.ops.graph.graph_test_util import assert_rows_match -logger = logging.getLogger(__name__) - +CONFIG = get_config() +logger = logging.getLogger() @pytest.mark.integration_saas @pytest.mark.integration_recharge @@ -19,14 +21,15 @@ def test_recharge_connection_test(recharge_connection_config) -> None: get_connector(recharge_connection_config).test_connection() +@pytest.mark.skip(reason="testing previous code") @pytest.mark.integration_recharge @pytest.mark.asyncio async def test_recharge_access_request_task( - db, - policy, - recharge_connection_config, - recharge_dataset_config, - recharge_identity_email, + db, + policy, + recharge_connection_config, + recharge_dataset_config, + recharge_identity_email, ) -> None: """Full access request based on the recharge SaaS config""" @@ -121,3 +124,125 @@ async def test_recharge_access_request_task( for item in v[key]: assert item['customer_id'] == customer_id + + +# @pytest.mark.skip(reason="testing previous code") +@pytest.mark.integration_saas +@pytest.mark.integration_recharge +@pytest.mark.asyncio +async def test_recharge_erasure_request_task( + db, + policy, + erasure_policy_string_rewrite, + recharge_connection_config, + recharge_dataset_config, + recharge_erasure_identity_email, + recharge_erasure_data, + recharge_test_client, +) -> None: + privacy_request = PrivacyRequest( + id=f"test_recharge_access_request_task_{random.randint(0, 1000)}" + ) + identity_attribute = "email" + identity_value = recharge_erasure_identity_email + identity_kwargs = {identity_attribute: identity_value} + identity = Identity(**identity_kwargs) + privacy_request.cache_identity(identity) + + dataset_name = recharge_connection_config.get_saas_config().fides_key + + merged_graph = recharge_dataset_config.get_graph() + graph = DatasetGraph(merged_graph) + v = await graph_task.run_access_request( + privacy_request, + policy, + graph, + [recharge_connection_config], + {"email": recharge_erasure_identity_email}, + db, + ) + + key = f'{dataset_name}:customers' + assert_rows_match( + v[key], + min_size=1, + keys=[ + "billing_address1", + "billing_address2", + "billing_city", + "billing_company", + "billing_country", + "billing_phone", + "billing_province", + "billing_zip", + "created_at", + "email", + "first_charge_processed_at", + "first_name", + "has_card_error_in_dunning", + "has_valid_payment_method", + "hash", + "id", + "last_name", + "number_active_subscriptions", + "number_subscriptions", + "phone", + "processor_type", + "reason_payment_method_not_valid", + "shopify_customer_id", + "status", + "tax_exempt", + "updated_at", + ] + ) + for item in v[key]: + assert item['email'] == recharge_erasure_identity_email + + customer_id = v[key][0]['id'] + + key = f'{dataset_name}:addresses' + assert_rows_match( + v[key], + min_size=1, + keys=[ + "address1", + "address2", + "cart_attributes", + "cart_note", + "city", + "company", + "country", + "created_at", + "customer_id", + "discount_id", + "first_name", + "id", + "last_name", + "note_attributes", + "original_shipping_lines", + "phone", + "presentment_currency", + "province", + "shipping_lines_override", + "updated_at", + "zip", + ] + ) + + for item in v[key]: + assert item['customer_id'] == customer_id + + temp_masking = CONFIG.execution.masking_strict + CONFIG.execution.masking_strict = True + + x = await graph_task.run_erasure( + privacy_request, + erasure_policy_string_rewrite, + graph, + [recharge_connection_config], + identity_kwargs, + get_cached_data_for_erasures(privacy_request.id), + db, + ) + logger.info(msg=f"x = {x}") + CONFIG.execution.masking_strict = temp_masking