From 23dbb25848a92035c46e11d081d3cb70ed6ccc96 Mon Sep 17 00:00:00 2001 From: Sean Preston Date: Tue, 10 May 2022 09:31:52 -0400 Subject: [PATCH] use black to format tests (#466) * use black to format tests * update command * update make command to not build docker infra twice * format more tests --- Makefile | 6 +- .../test_connection_config_endpoints.py | 36 +++-- .../v1/endpoints/test_dataset_endpoints.py | 4 +- .../api/v1/endpoints/test_policy_endpoints.py | 72 ++++++---- .../test_privacy_request_endpoints.py | 23 +-- .../endpoints/test_saas_config_endpoints.py | 31 ++-- tests/fixtures/application_fixtures.py | 6 +- tests/fixtures/bigquery_fixtures.py | 41 +++--- tests/fixtures/saas/hubspot_fixtures.py | 85 ++++++----- tests/fixtures/saas/segment_fixtures.py | 21 ++- tests/fixtures/saas/sentry_fixtures.py | 21 +-- tests/fixtures/saas_example_fixtures.py | 6 +- tests/generator/sql_data_generator.py | 6 +- tests/graph/graph_test_util.py | 15 +- tests/graph/test_traversal_node.py | 2 - .../saas/test_hubspot_task.py | 135 +++++++++--------- .../saas/test_segment_task.py | 8 +- .../test_external_database_connections.py | 3 +- .../test_integration_mongodb_example.py | 2 - tests/integration_tests/test_mongo_task.py | 14 +- tests/integration_tests/test_sql_task.py | 22 ++- tests/models/test_fidesopsuserpermissions.py | 2 +- tests/models/test_policy.py | 1 + tests/models/test_saasconfig.py | 4 +- tests/scripts/test_create_superuser.py | 4 +- .../test_mariadb_connector.py | 4 +- tests/service/connectors/test_queryconfig.py | 107 ++++++++------ .../test_pagination_strategy_cursor.py | 16 +-- .../test_pagination_strategy_factory.py | 1 + .../test_pagination_strategy_offset.py | 8 +- .../test_post_processor_strategy_factory.py | 23 ++- .../test_post_processor_strategy_filter.py | 1 + tests/task/test_filter_results.py | 7 +- tests/task/test_graph_task.py | 6 +- tests/task/test_refine_target_path.py | 3 +- tests/task/traversal_data.py | 13 +- 36 files changed, 435 insertions(+), 324 deletions(-) diff --git a/Makefile b/Makefile index bc571b133..dd0e5acba 100644 --- a/Makefile +++ b/Makefile @@ -139,9 +139,9 @@ pytest-saas: compose-build .PHONY: black black: compose-build - @echo "Running black formatting against the src/ directory..." - @docker-compose run $(IMAGE_NAME) \ - black src/ + @echo "Running black formatting against the src/ and tests/ directories..." + @docker-compose run $(IMAGE_NAME) black tests/ && black src/ + @echo "Fin" .PHONY: clean clean: diff --git a/tests/api/v1/endpoints/test_connection_config_endpoints.py b/tests/api/v1/endpoints/test_connection_config_endpoints.py index d94e56f84..3462a3835 100644 --- a/tests/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/api/v1/endpoints/test_connection_config_endpoints.py @@ -269,18 +269,14 @@ def test_patch_connections_bulk_update( mysql_connection = response_body["succeeded"][2] assert mysql_connection["access"] == "read" assert mysql_connection["updated_at"] is not None - mysql_resource = ( - db.query(ConnectionConfig).filter_by(key="my_mysql_db").first() - ) + mysql_resource = db.query(ConnectionConfig).filter_by(key="my_mysql_db").first() assert mysql_resource.access.value == "read" assert "secrets" not in mysql_connection mssql_connection = response_body["succeeded"][3] assert mssql_connection["access"] == "write" assert mssql_connection["updated_at"] is not None - mssql_resource = ( - db.query(ConnectionConfig).filter_by(key="my_mssql_db").first() - ) + mssql_resource = db.query(ConnectionConfig).filter_by(key="my_mssql_db").first() assert mssql_resource.access.value == "write" assert "secrets" not in mssql_connection @@ -687,11 +683,11 @@ def test_put_connection_config_redshift_secrets( assert redshift_connection_config.last_test_succeeded is None def test_put_connection_config_bigquery_secrets( - self, - api_client: TestClient, - db: Session, - generate_auth_header, - bigquery_connection_config_without_secrets, + self, + api_client: TestClient, + db: Session, + generate_auth_header, + bigquery_connection_config_without_secrets, ) -> None: """Note: this test does not attempt to actually connect to the db, via use of verify query param.""" auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) @@ -708,18 +704,18 @@ def test_put_connection_config_bigquery_secrets( "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/something%40project-12345.iam.gserviceaccount.com" - } + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/something%40project-12345.iam.gserviceaccount.com", + }, } resp = api_client.put( url + "?verify=False", headers=auth_header, json=payload, - ) + ) assert resp.status_code == 200 assert ( - json.loads(resp.text)["msg"] - == f"Secrets updated for ConnectionConfig with key: {bigquery_connection_config_without_secrets.key}." + json.loads(resp.text)["msg"] + == f"Secrets updated for ConnectionConfig with key: {bigquery_connection_config_without_secrets.key}." ) db.refresh(bigquery_connection_config_without_secrets) assert bigquery_connection_config_without_secrets.secrets == { @@ -735,8 +731,8 @@ def test_put_connection_config_bigquery_secrets( "auth_uri": "https://accounts.google.com/o/oauth2/auth", "token_uri": "https://oauth2.googleapis.com/token", "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", - "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/something%40project-12345.iam.gserviceaccount.com" - } + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/something%40project-12345.iam.gserviceaccount.com", + }, } assert bigquery_connection_config_without_secrets.last_test_timestamp is None assert bigquery_connection_config_without_secrets.last_test_succeeded is None @@ -824,7 +820,9 @@ def test_put_saas_example_connection_config_secrets( saas_example_secrets, ): auth_header = generate_auth_header(scopes=[CONNECTION_CREATE_OR_UPDATE]) - url = f"{V1_URL_PREFIX}{CONNECTIONS}/{saas_example_connection_config.key}/secret" + url = ( + f"{V1_URL_PREFIX}{CONNECTIONS}/{saas_example_connection_config.key}/secret" + ) payload = saas_example_secrets resp = api_client.put( diff --git a/tests/api/v1/endpoints/test_dataset_endpoints.py b/tests/api/v1/endpoints/test_dataset_endpoints.py index a28489a7d..98221bbaf 100644 --- a/tests/api/v1/endpoints/test_dataset_endpoints.py +++ b/tests/api/v1/endpoints/test_dataset_endpoints.py @@ -676,7 +676,9 @@ def test_patch_datasets_missing_saas_config( generate_auth_header, ): path = V1_URL_PREFIX + DATASETS - path_params = {"connection_key": saas_example_connection_config_without_saas_config.key} + path_params = { + "connection_key": saas_example_connection_config_without_saas_config.key + } datasets_url = path.format(**path_params) auth_header = generate_auth_header(scopes=[DATASET_CREATE_OR_UPDATE]) diff --git a/tests/api/v1/endpoints/test_policy_endpoints.py b/tests/api/v1/endpoints/test_policy_endpoints.py index b6bc48ba1..dc4d3d6a0 100644 --- a/tests/api/v1/endpoints/test_policy_endpoints.py +++ b/tests/api/v1/endpoints/test_policy_endpoints.py @@ -19,7 +19,8 @@ ActionType, Policy, Rule, - RuleTarget, DrpAction, + RuleTarget, + DrpAction, ) from fidesops.service.masking.strategy.masking_strategy_nullify import NULL_REWRITE from fidesops.util.data_category import DataCategory, generate_fides_data_categories @@ -143,7 +144,7 @@ def test_get_invalid_policy( assert resp.status_code == 404 def test_get_policy_returns_drp_action( - self, api_client: TestClient, generate_auth_header, policy_drp_action, url + self, api_client: TestClient, generate_auth_header, policy_drp_action, url ): auth_header = generate_auth_header(scopes=[scopes.POLICY_READ]) resp = api_client.get( @@ -325,18 +326,18 @@ def test_create_policy_with_duplicate_key( assert len(data["failed"]) == 2 def test_create_policy_with_duplicate_drp_action( - self, - url, - api_client: TestClient, - generate_auth_header, - policy_drp_action, - storage_config, + self, + url, + api_client: TestClient, + generate_auth_header, + policy_drp_action, + storage_config, ): data = [ { "name": "policy with pre-existing drp action", "action_type": ActionType.access.value, - "drp_action": DrpAction.access.value + "drp_action": DrpAction.access.value, } ] auth_header = generate_auth_header(scopes=[scopes.POLICY_CREATE_OR_UPDATE]) @@ -347,13 +348,13 @@ def test_create_policy_with_duplicate_drp_action( assert len(data["failed"]) == 1 def test_update_policy_with_duplicate_drp_action( - self, - db, - url, - api_client: TestClient, - generate_auth_header, - policy_drp_action, - storage_config, + self, + db, + url, + api_client: TestClient, + generate_auth_header, + policy_drp_action, + storage_config, ): # creates a new drp policy data = [ @@ -361,7 +362,7 @@ def test_update_policy_with_duplicate_drp_action( "key": "erasure_drp_policy", "name": "erasure drp policy", "action_type": ActionType.erasure.value, - "drp_action": DrpAction.deletion.value + "drp_action": DrpAction.deletion.value, } ] auth_header = generate_auth_header(scopes=[scopes.POLICY_CREATE_OR_UPDATE]) @@ -375,7 +376,7 @@ def test_update_policy_with_duplicate_drp_action( "key": "erasure_drp_policy", "name": "policy with pre-existing drp action", "action_type": ActionType.access.value, - "drp_action": DrpAction.access.value + "drp_action": DrpAction.access.value, } ] auth_header = generate_auth_header(scopes=[scopes.POLICY_CREATE_OR_UPDATE]) @@ -391,19 +392,19 @@ def test_update_policy_with_duplicate_drp_action( pol.delete(db=db) def test_update_policy_with_drp_action( - self, - url, - api_client: TestClient, - generate_auth_header, - policy, - storage_config, + self, + url, + api_client: TestClient, + generate_auth_header, + policy, + storage_config, ): data = [ { "key": policy.key, "name": "updated name", "action_type": ActionType.access.value, - "drp_action": DrpAction.access.value + "drp_action": DrpAction.access.value, } ] auth_header = generate_auth_header(scopes=[scopes.POLICY_CREATE_OR_UPDATE]) @@ -413,7 +414,7 @@ def test_update_policy_with_drp_action( assert len(response_data) == 1 def test_create_policy_invalid_drp_action( - self, url, api_client: TestClient, payload, generate_auth_header, storage_config + self, url, api_client: TestClient, payload, generate_auth_header, storage_config ): payload = [ { @@ -429,10 +430,19 @@ def test_create_policy_invalid_drp_action( assert resp.status_code == 422 response_body = json.loads(resp.text) - assert "value is not a valid enumeration member; permitted: 'access', 'deletion', 'sale:opt_out', 'sale:opt_in', 'access:categories', 'access:specific'" == response_body["detail"][0]["msg"] + assert ( + "value is not a valid enumeration member; permitted: 'access', 'deletion', 'sale:opt_out', 'sale:opt_in', 'access:categories', 'access:specific'" + == response_body["detail"][0]["msg"] + ) def test_create_policy_with_drp_action( - self, db, url, api_client: TestClient, payload, generate_auth_header, storage_config + self, + db, + url, + api_client: TestClient, + payload, + generate_auth_header, + storage_config, ): payload = [ { @@ -588,7 +598,11 @@ def test_create_rules_invalid_policy( assert resp.status_code == 404 def test_create_rules_mismatching_drp_policy( - self, api_client: TestClient, generate_auth_header, policy_drp_action, storage_config + self, + api_client: TestClient, + generate_auth_header, + policy_drp_action, + storage_config, ): data = [ { diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index d8ef29fb2..796985277 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -1383,16 +1383,18 @@ def test_deny_privacy_request_without_denial_reason( assert response_body["succeeded"][0]["id"] == privacy_request.id assert response_body["succeeded"][0]["reviewed_at"] is not None assert response_body["succeeded"][0]["reviewed_by"] == user.id - denial_audit_log: AuditLog = AuditLog.filter(db=db, conditions=( - (AuditLog.privacy_request_id == privacy_request.id) & - (AuditLog.user_id == user.id) - )).first() + denial_audit_log: AuditLog = AuditLog.filter( + db=db, + conditions=( + (AuditLog.privacy_request_id == privacy_request.id) + & (AuditLog.user_id == user.id) + ), + ).first() assert denial_audit_log.message is None assert not submit_mock.called # Shouldn't run! Privacy request was denied - @mock.patch( "fidesops.service.privacy_request.request_runner_service.PrivacyRequestRunner.submit" ) @@ -1427,10 +1429,13 @@ def test_deny_privacy_request_with_denial_reason( assert response_body["succeeded"][0]["id"] == privacy_request.id assert response_body["succeeded"][0]["reviewed_at"] is not None assert response_body["succeeded"][0]["reviewed_by"] == user.id - denial_audit_log: AuditLog = AuditLog.filter(db=db, conditions=( - (AuditLog.privacy_request_id == privacy_request.id) & - (AuditLog.user_id == user.id) - )).first() + denial_audit_log: AuditLog = AuditLog.filter( + db=db, + conditions=( + (AuditLog.privacy_request_id == privacy_request.id) + & (AuditLog.user_id == user.id) + ), + ).first() assert denial_audit_log.message == denial_reason diff --git a/tests/api/v1/endpoints/test_saas_config_endpoints.py b/tests/api/v1/endpoints/test_saas_config_endpoints.py index 86a83d76c..953412b11 100644 --- a/tests/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/api/v1/endpoints/test_saas_config_endpoints.py @@ -95,9 +95,9 @@ def test_put_validate_saas_config_reference_and_identity( ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) saas_config = saas_example_config - param_values = saas_config["endpoints"][0]["requests"]["read"][ - "param_values" - ][0] + param_values = saas_config["endpoints"][0]["requests"]["read"]["param_values"][ + 0 + ] param_values["identity"] = "email" param_values["references"] = [ { @@ -111,7 +111,10 @@ def test_put_validate_saas_config_reference_and_identity( ) assert response.status_code == 422 details = json.loads(response.text)["detail"] - assert details[0]["msg"] == "Must have exactly one of 'identity', 'references', or 'connector_param'" + assert ( + details[0]["msg"] + == "Must have exactly one of 'identity', 'references', or 'connector_param'" + ) def test_put_validate_saas_config_wrong_reference_direction( self, @@ -122,9 +125,9 @@ def test_put_validate_saas_config_wrong_reference_direction( ) -> None: auth_header = generate_auth_header(scopes=[SAAS_CONFIG_READ]) saas_config = saas_example_config - param_values = saas_config["endpoints"][0]["requests"]["read"][ - "param_values" - ][0] + param_values = saas_config["endpoints"][0]["requests"]["read"]["param_values"][ + 0 + ] param_values["references"] = [ { "dataset": "postgres_example_test_dataset", @@ -137,7 +140,11 @@ def test_put_validate_saas_config_wrong_reference_direction( ) assert response.status_code == 422 details = json.loads(response.text)["detail"] - assert details[0]["msg"] == "References can only have a direction of 'from', found 'to'" + assert ( + details[0]["msg"] + == "References can only have a direction of 'from', found 'to'" + ) + @pytest.mark.unit_saas class TestPutSaaSConfig: @@ -190,7 +197,9 @@ def test_patch_saas_config_create( generate_auth_header, ) -> None: path = V1_URL_PREFIX + SAAS_CONFIG - path_params = {"connection_key": saas_example_connection_config_without_saas_config.key} + path_params = { + "connection_key": saas_example_connection_config_without_saas_config.key + } saas_config_url = path.format(**path_params) auth_header = generate_auth_header(scopes=[SAAS_CONFIG_CREATE_OR_UPDATE]) @@ -200,7 +209,9 @@ def test_patch_saas_config_create( assert response.status_code == 200 updated_config = ConnectionConfig.get_by( - db=db, field="key", value=saas_example_connection_config_without_saas_config.key + db=db, + field="key", + value=saas_example_connection_config_without_saas_config.key, ) db.expire(updated_config) saas_config = updated_config.saas_config diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 574b44bf1..11a99f0ef 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -515,9 +515,9 @@ def policy( @pytest.fixture(scope="function") def policy_drp_action( - db: Session, - oauth_client: ClientDetail, - storage_config: StorageConfig, + db: Session, + oauth_client: ClientDetail, + storage_config: StorageConfig, ) -> Generator: access_request_policy = Policy.create( db=db, diff --git a/tests/fixtures/bigquery_fixtures.py b/tests/fixtures/bigquery_fixtures.py index 28aae4baa..cacf3a436 100644 --- a/tests/fixtures/bigquery_fixtures.py +++ b/tests/fixtures/bigquery_fixtures.py @@ -8,7 +8,11 @@ from sqlalchemy.orm import Session -from fidesops.models.connectionconfig import AccessLevel, ConnectionType, ConnectionConfig +from fidesops.models.connectionconfig import ( + AccessLevel, + ConnectionType, + ConnectionConfig, +) from fidesops.models.datasetconfig import DatasetConfig from fidesops.schemas.connection_configuration import BigQuerySchema from fidesops.service.connectors import BigQueryConnector, get_connector @@ -16,6 +20,7 @@ logger = logging.getLogger(__name__) + @pytest.fixture(scope="function") def bigquery_connection_config_without_secrets(db: Session) -> Generator: connection_config = ConnectionConfig.create( @@ -43,12 +48,12 @@ def bigquery_connection_config(db: Session) -> Generator: }, ) # Pulling from integration config file or GitHub secrets - keyfile_creds = integration_config.get("bigquery", {}).get("keyfile_creds") or ast.literal_eval(os.environ.get( - "BIGQUERY_KEYFILE_CREDS" - )) - dataset = integration_config.get("bigquery", {}).get( - "dataset" - ) or os.environ.get("BIGQUERY_DATASET") + keyfile_creds = integration_config.get("bigquery", {}).get( + "keyfile_creds" + ) or ast.literal_eval(os.environ.get("BIGQUERY_KEYFILE_CREDS")) + dataset = integration_config.get("bigquery", {}).get("dataset") or os.environ.get( + "BIGQUERY_DATASET" + ) if keyfile_creds: schema = BigQuerySchema(keyfile_creds=keyfile_creds, dataset=dataset) connection_config.secrets = schema.dict() @@ -60,9 +65,9 @@ def bigquery_connection_config(db: Session) -> Generator: @pytest.fixture def bigquery_example_test_dataset_config( - bigquery_connection_config: ConnectionConfig, - db: Session, - example_datasets: List[Dict], + bigquery_connection_config: ConnectionConfig, + db: Session, + example_datasets: List[Dict], ) -> Generator: bigquery_dataset = example_datasets[7] fides_key = bigquery_dataset["fides_key"] @@ -83,7 +88,7 @@ def bigquery_example_test_dataset_config( @pytest.fixture(scope="function") def bigquery_resources( - bigquery_example_test_dataset_config, + bigquery_example_test_dataset_config, ): bigquery_connection_config = bigquery_example_test_dataset_config.connection_config connector = BigQueryConnector(bigquery_connection_config) @@ -129,7 +134,7 @@ def bigquery_resources( stmt = f"delete from customer where email = '{customer_email}';" connection.execute(stmt) - stmt = f'delete from address where id = {address_id};' + stmt = f"delete from address where id = {address_id};" connection.execute(stmt) @@ -144,12 +149,12 @@ def bigquery_test_engine() -> Generator: ) # Pulling from integration config file or GitHub secrets - keyfile_creds = integration_config.get("bigquery", {}).get("keyfile_creds") or ast.literal_eval(os.environ.get( - "BIGQUERY_KEYFILE_CREDS" - )) - dataset = integration_config.get("bigquery", {}).get( - "dataset" - ) or os.environ.get("BIGQUERY_DATASET") + keyfile_creds = integration_config.get("bigquery", {}).get( + "keyfile_creds" + ) or ast.literal_eval(os.environ.get("BIGQUERY_KEYFILE_CREDS")) + dataset = integration_config.get("bigquery", {}).get("dataset") or os.environ.get( + "BIGQUERY_DATASET" + ) if keyfile_creds: schema = BigQuerySchema(keyfile_creds=keyfile_creds, dataset=dataset) connection_config.secrets = schema.dict() diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index e8bf05b93..2d2de85bb 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -30,9 +30,9 @@ def hubspot_secrets(): return { "domain": pydash.get(saas_config, "hubspot.domain") - or os.environ.get("HUBSPOT_DOMAIN"), + or os.environ.get("HUBSPOT_DOMAIN"), "hapikey": pydash.get(saas_config, "hubspot.hapikey") - or os.environ.get("HUBSPOT_HAPIKEY"), + or os.environ.get("HUBSPOT_HAPIKEY"), } @@ -60,7 +60,9 @@ def hubspot_dataset() -> Dict[str, Any]: @pytest.fixture(scope="function") def connection_config_hubspot( - db: Session, hubspot_config, hubspot_secrets, + db: Session, + hubspot_config, + hubspot_secrets, ) -> Generator: fides_key = hubspot_config["fides_key"] connection_config = ConnectionConfig.create( @@ -80,10 +82,10 @@ def connection_config_hubspot( @pytest.fixture def dataset_config_hubspot( - db: Session, - connection_config_hubspot: ConnectionConfig, - hubspot_dataset, - hubspot_config, + db: Session, + connection_config_hubspot: ConnectionConfig, + hubspot_dataset, + hubspot_config, ) -> Generator: fides_key = hubspot_config["fides_key"] connection_config_hubspot.name = fides_key @@ -102,7 +104,9 @@ def dataset_config_hubspot( @pytest.fixture(scope="function") -def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_email) -> Generator: +def hubspot_erasure_data( + connection_config_hubspot, hubspot_erasure_identity_email +) -> Generator: """ Gets the current value of the resource and restores it after the test is complete. Used for erasure tests. @@ -110,16 +114,18 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema connector = SaaSConnector(connection_config_hubspot) - body = json.dumps({ - "properties": { - "company": "test company", - "email": hubspot_erasure_identity_email, - "firstname": HUBSPOT_FIRSTNAME, - "lastname": "SomeoneLastname", - "phone": "(123) 123-1234", - "website": "someone.net" + body = json.dumps( + { + "properties": { + "company": "test company", + "email": hubspot_erasure_identity_email, + "firstname": HUBSPOT_FIRSTNAME, + "lastname": "SomeoneLastname", + "phone": "(123) 123-1234", + "website": "someone.net", + } } - }) + ) updated_headers, formatted_body = format_body({}, body) @@ -140,7 +146,9 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema remaining_tries = 5 while _contact_exists(hubspot_erasure_identity_email, connector) is False: if remaining_tries < 1: - raise Exception(f"Contact with contact id {contact_id} could not be added to Hubspot") + raise Exception( + f"Contact with contact id {contact_id} could not be added to Hubspot" + ) time.sleep(5) yield contact_id @@ -156,24 +164,34 @@ def hubspot_erasure_data(connection_config_hubspot, hubspot_erasure_identity_ema remaining_tries = 5 while _contact_exists(hubspot_erasure_identity_email, connector) is True: if remaining_tries < 1: - raise Exception(f"Contact with contact id {contact_id} could not be deleted from Hubspot") + raise Exception( + f"Contact with contact id {contact_id} could not be deleted from Hubspot" + ) time.sleep(5) # Ensures contact is deleted -def _contact_exists(hubspot_erasure_identity_email: str, connector: SaaSConnector) -> bool: +def _contact_exists( + hubspot_erasure_identity_email: str, connector: SaaSConnector +) -> bool: """ Confirm whether contact exists by calling search api and comparing firstname str. - """ - - body = json.dumps({ - "filterGroups": [{ - "filters": [{ - "value": hubspot_erasure_identity_email, - "propertyName": "email", - "operator": "EQ" - }] - }] - }) + """ + + body = json.dumps( + { + "filterGroups": [ + { + "filters": [ + { + "value": hubspot_erasure_identity_email, + "propertyName": "email", + "operator": "EQ", + } + ] + } + ] + } + ) updated_headers, formatted_body = format_body({}, body) contact_request: SaaSRequestParams = SaaSRequestParams( @@ -184,4 +202,7 @@ def _contact_exists(hubspot_erasure_identity_email: str, connector: SaaSConnecto ) contact_response = connector.create_client().send(contact_request) contact_body = contact_response.json() - return bool(contact_body["results"] and contact_body["results"][0]["properties"]["firstname"] == HUBSPOT_FIRSTNAME) + return bool( + contact_body["results"] + and contact_body["results"][0]["properties"]["firstname"] == HUBSPOT_FIRSTNAME + ) diff --git a/tests/fixtures/saas/segment_fixtures.py b/tests/fixtures/saas/segment_fixtures.py index 25fa8cb71..a2ba5085f 100644 --- a/tests/fixtures/saas/segment_fixtures.py +++ b/tests/fixtures/saas/segment_fixtures.py @@ -20,15 +20,22 @@ @pytest.fixture(scope="function") def segment_secrets(): return { - "domain": pydash.get(saas_config, "segment.domain") or os.environ.get("SEGMENT_DOMAIN"), + "domain": pydash.get(saas_config, "segment.domain") + or os.environ.get("SEGMENT_DOMAIN"), "personas_domain": pydash.get(saas_config, "segment.personas_domain") or os.environ.get("SEGMENT_PERSONAS_DOMAIN"), - "workspace": pydash.get(saas_config, "segment.workspace") or os.environ.get("SEGMENT_WORKSPACE"), - "access_token": pydash.get(saas_config, "segment.access_token") or os.environ.get("SEGMENT_ACCESS_TOKEN"), - "namespace_id": pydash.get(saas_config, "segment.namespace_id") or os.environ.get("SEGMENT_NAMESPACE_ID"), - "access_secret": pydash.get(saas_config, "segment.access_secret") or os.environ.get("SEGMENT_ACCESS_SECRET"), - "api_domain": pydash.get(saas_config, "segment.api_domain") or os.environ.get("SEGMENT_API_DOMAIN"), - "user_token": pydash.get(saas_config, "segment.user_token") or os.environ.get("SEGMENT_USER_TOKEN"), + "workspace": pydash.get(saas_config, "segment.workspace") + or os.environ.get("SEGMENT_WORKSPACE"), + "access_token": pydash.get(saas_config, "segment.access_token") + or os.environ.get("SEGMENT_ACCESS_TOKEN"), + "namespace_id": pydash.get(saas_config, "segment.namespace_id") + or os.environ.get("SEGMENT_NAMESPACE_ID"), + "access_secret": pydash.get(saas_config, "segment.access_secret") + or os.environ.get("SEGMENT_ACCESS_SECRET"), + "api_domain": pydash.get(saas_config, "segment.api_domain") + or os.environ.get("SEGMENT_API_DOMAIN"), + "user_token": pydash.get(saas_config, "segment.user_token") + or os.environ.get("SEGMENT_USER_TOKEN"), } diff --git a/tests/fixtures/saas/sentry_fixtures.py b/tests/fixtures/saas/sentry_fixtures.py index af3cf67f9..8d4bd3410 100644 --- a/tests/fixtures/saas/sentry_fixtures.py +++ b/tests/fixtures/saas/sentry_fixtures.py @@ -23,11 +23,16 @@ def sentry_secrets(): "host": pydash.get(saas_config, "sentry.host") or os.environ.get("SENTRY_HOST"), "access_token": pydash.get(saas_config, "sentry.access_token") or os.environ.get("SENTRY_ACCESS_TOKEN"), - "erasure_access_token": pydash.get(saas_config, "sentry.erasure_access_token") or os.environ.get("SENTRY_ERASURE_TOKEN"), - "erasure_identity_email": pydash.get(saas_config, "sentry.erasure_identity_email") or os.environ.get("SENTRY_ERASURE_IDENTITY"), - "user_id_erasure": pydash.get(saas_config, "sentry.user_id_erasure") or os.environ.get("SENTRY_USER_ID"), - "issue_url": pydash.get(saas_config, "sentry.issue_url") or os.environ.get("SENTRY_ISSUE_URL"), - + "erasure_access_token": pydash.get(saas_config, "sentry.erasure_access_token") + or os.environ.get("SENTRY_ERASURE_TOKEN"), + "erasure_identity_email": pydash.get( + saas_config, "sentry.erasure_identity_email" + ) + or os.environ.get("SENTRY_ERASURE_IDENTITY"), + "user_id_erasure": pydash.get(saas_config, "sentry.user_id_erasure") + or os.environ.get("SENTRY_USER_ID"), + "issue_url": pydash.get(saas_config, "sentry.issue_url") + or os.environ.get("SENTRY_ISSUE_URL"), } @@ -49,9 +54,7 @@ def sentry_dataset() -> Dict[str, Any]: @pytest.fixture(scope="function") -def sentry_connection_config( - db: session, sentry_config, sentry_secrets -) -> Generator: +def sentry_connection_config(db: session, sentry_config, sentry_secrets) -> Generator: fides_key = sentry_config["fides_key"] connection_config = ConnectionConfig.create( db=db, @@ -87,4 +90,4 @@ def sentry_dataset_config( }, ) yield dataset - dataset.delete(db=db) \ No newline at end of file + dataset.delete(db=db) diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index ab9c20469..33f8f0e47 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -14,13 +14,16 @@ from fidesops.models.datasetconfig import DatasetConfig from tests.fixtures.application_fixtures import load_dataset + def load_config(filename: str) -> Dict: yaml_file = load_file(filename) with open(yaml_file, "r") as file: return yaml.safe_load(file).get("saas_config", []) + saas_config = load_toml("saas_config.toml") + @pytest.fixture(scope="function") def saas_example_secrets(): return { @@ -31,6 +34,7 @@ def saas_example_secrets(): "page_limit": pydash.get(saas_config, "saas_example.page_limit"), } + @pytest.fixture def saas_example_config() -> Dict: return load_config("data/saas/config/saas_example_config.yml") @@ -123,4 +127,4 @@ def saas_example_connection_config_with_invalid_saas_config( }, ) yield connection_config - connection_config.delete(db) \ No newline at end of file + connection_config.delete(db) diff --git a/tests/generator/sql_data_generator.py b/tests/generator/sql_data_generator.py index 19d3073e1..721ea654a 100644 --- a/tests/generator/sql_data_generator.py +++ b/tests/generator/sql_data_generator.py @@ -96,9 +96,9 @@ def traversal_collection_fn( if edge.f1.collection_address() in data: collection_data = data[edge.f1.collection_address()] incoming_values[edge.f2.field_path] = ( - edge.f1.field_path in collection_data - and collection_data[edge.f1.field_path] - or [] + edge.f1.field_path in collection_data + and collection_data[edge.f1.field_path] + or [] ) # for row in collection_data] for fk, f in tn.node.collection.field_dict.items(): diff --git a/tests/graph/graph_test_util.py b/tests/graph/graph_test_util.py index 300b0e8fa..a6ca77e88 100644 --- a/tests/graph/graph_test_util.py +++ b/tests/graph/graph_test_util.py @@ -43,7 +43,11 @@ def client(self) -> Engine: raise AttributeError("Unsupported") def retrieve_data( - self, node: TraversalNode, policy: Policy, privacy_request: PrivacyRequest, input_data: Dict[str, List[Any]] + self, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + input_data: Dict[str, List[Any]], ) -> List[Row]: return [generate_collection(node.node.collection) for _ in range(3)] @@ -57,6 +61,7 @@ class MockMongoTask(GraphTask): def connector(self) -> BaseConnector: return MongoDBConnector(ConnectionConfig()) + # ------------------------------------------- # test utility functions # ------------------------------------------- @@ -132,9 +137,13 @@ def field(dataresources: List[Dataset], *address: str) -> ScalarField: try: # Assuming object field with at most one level - get ScalarField out of object field - df: ScalarField = next(df for df in ds.field_dict.values() if df.name == address[3]) + df: ScalarField = next( + df for df in ds.field_dict.values() if df.name == address[3] + ) except: - df: ScalarField = next(df for df in ds.field_dict.values() if df.name == address[2]) + df: ScalarField = next( + df for df in ds.field_dict.values() if df.name == address[2] + ) return df diff --git a/tests/graph/test_traversal_node.py b/tests/graph/test_traversal_node.py index b3aeda690..3b4ca1891 100644 --- a/tests/graph/test_traversal_node.py +++ b/tests/graph/test_traversal_node.py @@ -110,5 +110,3 @@ def test_is_root_node(self): tn = TraversalNode(generate_node("a", "b", "c", "c2")) assert not tn.is_root_node() - - diff --git a/tests/integration_tests/saas/test_hubspot_task.py b/tests/integration_tests/saas/test_hubspot_task.py index 6597e1029..5d0e841e0 100644 --- a/tests/integration_tests/saas/test_hubspot_task.py +++ b/tests/integration_tests/saas/test_hubspot_task.py @@ -20,11 +20,11 @@ @pytest.mark.integration_saas @pytest.mark.integration_hubspot def test_saas_access_request_task( - db, - policy, - connection_config_hubspot, - dataset_config_hubspot, - hubspot_identity_email, + db, + policy, + connection_config_hubspot, + dataset_config_hubspot, + hubspot_identity_email, ) -> None: """Full access request based on the Hubspot SaaS config""" @@ -52,20 +52,12 @@ def test_saas_access_request_task( assert_rows_match( v[f"{dataset_name}:contacts"], min_size=1, - keys=[ - "archived", - "createdAt", - "id", - "properties" - ], + keys=["archived", "createdAt", "id", "properties"], ) assert_rows_match( v[f"{dataset_name}:subscription_preferences"], min_size=1, - keys=[ - "recipient", - "subscriptionStatuses" - ], + keys=["recipient", "subscriptionStatuses"], ) target_categories = {"user.provided"} @@ -79,65 +71,70 @@ def test_saas_access_request_task( f"{dataset_name}:contacts", f"{dataset_name}:subscription_preferences", } - assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == { - "properties" - } + assert set(filtered_results[f"{dataset_name}:contacts"][0].keys()) == {"properties"} - assert set(filtered_results[f"{dataset_name}:contacts"][0]["properties"].keys()) == { + assert set( + filtered_results[f"{dataset_name}:contacts"][0]["properties"].keys() + ) == { "lastname", "firstname", "email", } - assert set(filtered_results[f"{dataset_name}:subscription_preferences"][0].keys()) == { - "recipient" - } - assert filtered_results[f"{dataset_name}:subscription_preferences"][0]["recipient"] == hubspot_identity_email + assert set( + filtered_results[f"{dataset_name}:subscription_preferences"][0].keys() + ) == {"recipient"} + assert ( + filtered_results[f"{dataset_name}:subscription_preferences"][0]["recipient"] + == hubspot_identity_email + ) logs = ( ExecutionLog.query(db=db) - .filter(ExecutionLog.privacy_request_id == privacy_request.id) - .all() + .filter(ExecutionLog.privacy_request_id == privacy_request.id) + .all() ) logs = [log.__dict__ for log in logs] assert ( - len( - records_matching_fields( - logs, dataset_name=dataset_name, collection_name="contacts" - ) + len( + records_matching_fields( + logs, dataset_name=dataset_name, collection_name="contacts" ) - > 0 + ) + > 0 ) assert ( - len( - records_matching_fields( - logs, - dataset_name=dataset_name, - collection_name="owners", - ) + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="owners", ) - > 0 + ) + > 0 ) assert ( - len( - records_matching_fields( - logs, dataset_name=dataset_name, collection_name="subscription_preferences" - ) + len( + records_matching_fields( + logs, + dataset_name=dataset_name, + collection_name="subscription_preferences", ) - > 0 + ) + > 0 ) @pytest.mark.integration_saas @pytest.mark.integration_hubspot def test_saas_erasure_request_task( - db, - policy, - erasure_policy_string_rewrite, - connection_config_hubspot, - dataset_config_hubspot, - hubspot_erasure_identity_email, - hubspot_erasure_data, + db, + policy, + erasure_policy_string_rewrite, + connection_config_hubspot, + dataset_config_hubspot, + hubspot_erasure_identity_email, + hubspot_erasure_data, ) -> None: """Full erasure request based on the Hubspot SaaS config""" privacy_request = PrivacyRequest( @@ -163,20 +160,12 @@ def test_saas_erasure_request_task( assert_rows_match( v[f"{dataset_name}:contacts"], min_size=1, - keys=[ - "archived", - "createdAt", - "id", - "properties" - ], + keys=["archived", "createdAt", "id", "properties"], ) assert_rows_match( v[f"{dataset_name}:subscription_preferences"], min_size=1, - keys=[ - "recipient", - "subscriptionStatuses" - ], + keys=["recipient", "subscriptionStatuses"], ) erasure = graph_task.run_erasure( @@ -192,20 +181,26 @@ def test_saas_erasure_request_task( assert erasure == { "hubspot_connector_example:contacts": 1, "hubspot_connector_example:owners": 0, - "hubspot_connector_example:subscription_preferences": 1 + "hubspot_connector_example:subscription_preferences": 1, } connector = SaaSConnector(connection_config_hubspot) - body = json.dumps({ - "filterGroups": [{ - "filters": [{ - "value": hubspot_erasure_identity_email, - "propertyName": "email", - "operator": "EQ" - }] - }] - }) + body = json.dumps( + { + "filterGroups": [ + { + "filters": [ + { + "value": hubspot_erasure_identity_email, + "propertyName": "email", + "operator": "EQ", + } + ] + } + ] + } + ) updated_headers, formatted_body = format_body({}, body) @@ -223,7 +218,7 @@ def test_saas_erasure_request_task( # verify user is unsubscribed subscription_request: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, - path=f'/communication-preferences/v3/status/email/{hubspot_erasure_identity_email}' + path=f"/communication-preferences/v3/status/email/{hubspot_erasure_identity_email}", ) subscription_response = connector.create_client().send(subscription_request) subscription_body = subscription_response.json() diff --git a/tests/integration_tests/saas/test_segment_task.py b/tests/integration_tests/saas/test_segment_task.py index 7ce0c5f71..ee9cf2e91 100644 --- a/tests/integration_tests/saas/test_segment_task.py +++ b/tests/integration_tests/saas/test_segment_task.py @@ -146,9 +146,7 @@ def _create_test_segment_email(base_email: str, timestamp: int) -> str: return email -def create_segment_test_data( - segment_connection_config, segment_identity_email: str -): +def create_segment_test_data(segment_connection_config, segment_identity_email: str): """Seeds a segment user and event""" segment_secrets = segment_connection_config.secrets if not segment_identity_email: # Don't run unnecessarily locally @@ -225,7 +223,9 @@ def test_segment_saas_erasure_request_task( config.execution.MASKING_STRICT = False # Allow GDPR Delete # Create user for GDPR delete - erasure_email = create_segment_test_data(segment_connection_config, segment_identity_email) + erasure_email = create_segment_test_data( + segment_connection_config, segment_identity_email + ) time.sleep(8) # Pause before making access/erasure requests privacy_request = PrivacyRequest( id=f"test_saas_access_request_task_{random.randint(0, 1000)}" diff --git a/tests/integration_tests/test_external_database_connections.py b/tests/integration_tests/test_external_database_connections.py index 1494aeaba..fd2d0661e 100644 --- a/tests/integration_tests/test_external_database_connections.py +++ b/tests/integration_tests/test_external_database_connections.py @@ -107,6 +107,7 @@ def test_snowflake_example_data(snowflake_test_engine): "visit", ] + @pytest.mark.integration_external @pytest.mark.integration_bigquery def test_bigquery_example_data(bigquery_test_engine): @@ -124,4 +125,4 @@ def test_bigquery_example_data(bigquery_test_engine): "report", "service_request", "visit", - ] \ No newline at end of file + ] diff --git a/tests/integration_tests/test_integration_mongodb_example.py b/tests/integration_tests/test_integration_mongodb_example.py index 5024c976b..d6d647de4 100644 --- a/tests/integration_tests/test_integration_mongodb_example.py +++ b/tests/integration_tests/test_integration_mongodb_example.py @@ -1,8 +1,6 @@ - import pytest - @pytest.mark.integration_mongodb @pytest.mark.integration def test_mongo_example_data(integration_mongodb_connector): diff --git a/tests/integration_tests/test_mongo_task.py b/tests/integration_tests/test_mongo_task.py index fd69388a4..38a24c16b 100644 --- a/tests/integration_tests/test_mongo_task.py +++ b/tests/integration_tests/test_mongo_task.py @@ -642,7 +642,7 @@ def test_return_all_elements_config_access_request( policy, integration_mongodb_config, integration_postgres_config, - integration_mongodb_connector + integration_mongodb_connector, ): """Annotating array entrypoint field with return_all_elements=true means both the entire array is returned from the queried data and used to locate data in other collections @@ -1073,7 +1073,9 @@ def test_retrieving_data_no_input( FieldAddress("mongo_test", "customer_details", "customer_id"), ) } - results = connector.retrieve_data(traversal_node, Policy(), privacy_request, {"customer_id": []}) + results = connector.retrieve_data( + traversal_node, Policy(), privacy_request, {"customer_id": []} + ) assert results == [] results = connector.retrieve_data(traversal_node, Policy(), privacy_request, {}) @@ -1084,10 +1086,14 @@ def test_retrieving_data_no_input( ) assert results == [] - results = connector.retrieve_data(traversal_node, Policy(), privacy_request, {"email": [None]}) + results = connector.retrieve_data( + traversal_node, Policy(), privacy_request, {"email": [None]} + ) assert results == [] - results = connector.retrieve_data(traversal_node, Policy(), privacy_request, {"email": None}) + results = connector.retrieve_data( + traversal_node, Policy(), privacy_request, {"email": None} + ) assert results == [] @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") diff --git a/tests/integration_tests/test_sql_task.py b/tests/integration_tests/test_sql_task.py index 3b8b20187..98fbe5126 100644 --- a/tests/integration_tests/test_sql_task.py +++ b/tests/integration_tests/test_sql_task.py @@ -860,7 +860,10 @@ def test_retrieving_data( } results = connector.retrieve_data( - traversal_node, Policy(), privacy_request, {"email": ["customer-1@example.com"]} + traversal_node, + Policy(), + privacy_request, + {"email": ["customer-1@example.com"]}, ) assert len(results) is 1 assert results == [ @@ -889,9 +892,13 @@ def test_retrieving_data_no_input( ) } - assert [] == connector.retrieve_data(traversal_node, Policy(), privacy_request, {"email": []}) + assert [] == connector.retrieve_data( + traversal_node, Policy(), privacy_request, {"email": []} + ) - assert [] == connector.retrieve_data(traversal_node, Policy(), privacy_request, {}) + assert [] == connector.retrieve_data( + traversal_node, Policy(), privacy_request, {} + ) assert [] == connector.retrieve_data( traversal_node, Policy(), privacy_request, {"bad_key": ["test"]} @@ -901,7 +908,9 @@ def test_retrieving_data_no_input( traversal_node, Policy(), privacy_request, {"email": [None]} ) - assert [] == connector.retrieve_data(traversal_node, Policy(), privacy_request, {"email": None}) + assert [] == connector.retrieve_data( + traversal_node, Policy(), privacy_request, {"email": None} + ) @mock.patch("fidesops.graph.traversal.TraversalNode.incoming_edges") def test_retrieving_data_input_not_in_table( @@ -922,7 +931,10 @@ def test_retrieving_data_input_not_in_table( ) } results = connector.retrieve_data( - traversal_node, Policy(), privacy_request, {"email": ["customer_not_in_dataset@example.com"]} + traversal_node, + Policy(), + privacy_request, + {"email": ["customer_not_in_dataset@example.com"]}, ) assert results == [] diff --git a/tests/models/test_fidesopsuserpermissions.py b/tests/models/test_fidesopsuserpermissions.py index 00ae267b2..31b43e940 100644 --- a/tests/models/test_fidesopsuserpermissions.py +++ b/tests/models/test_fidesopsuserpermissions.py @@ -15,7 +15,7 @@ def test_create_user_permissions(self, db: Session) -> None: data={"username": "user_1", "password": "test_password"}, ) - permissions:FidesopsUserPermissions = FidesopsUserPermissions.create( + permissions: FidesopsUserPermissions = FidesopsUserPermissions.create( db=db, data={"user_id": user.id, "scopes": [PRIVACY_REQUEST_READ]}, ) diff --git a/tests/models/test_policy.py b/tests/models/test_policy.py index 3ee07ed85..2921dcfae 100644 --- a/tests/models/test_policy.py +++ b/tests/models/test_policy.py @@ -19,6 +19,7 @@ from fidesops.util.text import to_snake_case from fidesops.util.data_category import DataCategory + def test_policy_sets_slug( db: Session, oauth_client: ClientDetail, diff --git a/tests/models/test_saasconfig.py b/tests/models/test_saasconfig.py index ddbee42f7..775340481 100644 --- a/tests/models/test_saasconfig.py +++ b/tests/models/test_saasconfig.py @@ -11,12 +11,14 @@ def test_saas_configs(saas_example_config) -> None: """Simple test to verify that the example config can be deserialized into SaaSConfigs""" SaaSConfig(**saas_example_config) + @pytest.mark.unit_saas def test_saas_request_without_method(): with pytest.raises(ValidationError) as exc: SaaSRequest(path="/test") assert "field required" in str(exc.value) + @pytest.mark.unit_saas def test_saas_config_to_dataset(saas_example_config: Dict[str, Dict]): """Verify dataset generated by SaaS config""" @@ -46,7 +48,7 @@ def test_saas_config_to_dataset(saas_example_config: Dict[str, Dict]): assert user_collection.grouped_inputs == { "organization_slug", "project_slug", - "query" + "query", } org_slug_reference, direction = user_collection.fields[0].references[0] diff --git a/tests/scripts/test_create_superuser.py b/tests/scripts/test_create_superuser.py index 7907e7569..5f92653e4 100644 --- a/tests/scripts/test_create_superuser.py +++ b/tests/scripts/test_create_superuser.py @@ -65,7 +65,9 @@ def test_create_user_and_client(self, mock_pass, mock_user, db): assert client_detail.fides_key == ADMIN_UI_ROOT assert CLIENT_CREATE not in client_detail.scopes - user_permissions = FidesopsUserPermissions.get_by(db=db, field="user_id", value=superuser.id) + user_permissions = FidesopsUserPermissions.get_by( + db=db, field="user_id", value=superuser.id + ) assert user_permissions is not None with pytest.raises(KeyOrNameAlreadyExists): diff --git a/tests/service/connection_config/test_mariadb_connector.py b/tests/service/connection_config/test_mariadb_connector.py index 093e8595d..11976572c 100644 --- a/tests/service/connection_config/test_mariadb_connector.py +++ b/tests/service/connection_config/test_mariadb_connector.py @@ -36,7 +36,9 @@ def test_mariadb_connector_build_uri(connection_config_mariadb, db: Session): "host": "host.docker.internal", } connection_config_mariadb.save(db) - assert connector.build_uri() == "mariadb+pymysql://mariadb_user@host.docker.internal" + assert ( + connector.build_uri() == "mariadb+pymysql://mariadb_user@host.docker.internal" + ) connection_config_mariadb.secrets = { "host": "host.docker.internal", diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index c68c5ec0c..bb3ceca4a 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -74,23 +74,29 @@ def found_query_keys(node: TraversalNode, values: Dict[str, Any]) -> Set[str]: } # values exist for all query keys - assert found_query_keys( - payment_card_node, - { - "id": ["A"], - "customer_id": ["V"], - "ignore_me": ["X"], - }, - ) == {"id", "customer_id"} + assert ( + found_query_keys( + payment_card_node, + { + "id": ["A"], + "customer_id": ["V"], + "ignore_me": ["X"], + }, + ) + == {"id", "customer_id"} + ) # with no values OR an empty set, these are omitted - assert found_query_keys( - payment_card_node, - { - "id": ["A"], - "customer_id": [], - "ignore_me": ["X"], - }, - ) == {"id"} + assert ( + found_query_keys( + payment_card_node, + { + "id": ["A"], + "customer_id": [], + "ignore_me": ["X"], + }, + ) + == {"id"} + ) assert found_query_keys( payment_card_node, {"id": ["A"], "ignore_me": ["X"]} ) == {"id"} @@ -98,21 +104,27 @@ def found_query_keys(node: TraversalNode, values: Dict[str, Any]) -> Set[str]: assert found_query_keys(payment_card_node, {}) == set() def test_typed_filtered_values(self): - assert payment_card_node.typed_filtered_values( - { - "id": ["A"], - "customer_id": ["V"], - "ignore_me": ["X"], - } - ) == {"id": ["A"], "customer_id": ["V"]} + assert ( + payment_card_node.typed_filtered_values( + { + "id": ["A"], + "customer_id": ["V"], + "ignore_me": ["X"], + } + ) + == {"id": ["A"], "customer_id": ["V"]} + ) - assert payment_card_node.typed_filtered_values( - { - "id": ["A"], - "customer_id": [], - "ignore_me": ["X"], - } - ) == {"id": ["A"]} + assert ( + payment_card_node.typed_filtered_values( + { + "id": ["A"], + "customer_id": [], + "ignore_me": ["X"], + } + ) + == {"id": ["A"]} + ) assert payment_card_node.typed_filtered_values( {"id": ["A"], "ignore_me": ["X"]} @@ -314,9 +326,12 @@ def test_generate_update_stmt_multiple_fields_same_rule( ["John Customer"], privacy_request_id=privacy_request.id )[0][0:40] ) - assert text_clause._bindparams["email"].value == HashMaskingStrategy( - HashMaskingConfiguration(algorithm="SHA-512") - ).mask(["customer-1@example.com"], privacy_request_id=privacy_request.id)[0] + assert ( + text_clause._bindparams["email"].value + == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( + ["customer-1@example.com"], privacy_request_id=privacy_request.id + )[0] + ) clear_cache_secrets(privacy_request.id) def test_generate_update_stmts_from_multiple_rules( @@ -596,9 +611,12 @@ def test_generate_update_stmt_multiple_rules( ) assert mongo_statement[0] == {"_id": 1} assert len(mongo_statement[1]["$set"]["gender"]) == 30 - assert mongo_statement[1]["$set"]["birthday"] == HashMaskingStrategy( - HashMaskingConfiguration(algorithm="SHA-512") - ).mask(["1988-01-10"], privacy_request_id=privacy_request.id)[0] + assert ( + mongo_statement[1]["$set"]["birthday"] + == HashMaskingStrategy(HashMaskingConfiguration(algorithm="SHA-512")).mask( + ["1988-01-10"], privacy_request_id=privacy_request.id + )[0] + ) @pytest.mark.unit_saas @@ -881,8 +899,9 @@ def test_generate_update_stmt_with_url_encoded_body( assert prepared_request.query_params == {} assert prepared_request.body == "name%5Bfirst%5D=MASKED&name%5Blast%5D=MASKED" - - def test_get_masking_request(self, combined_traversal, saas_example_connection_config): + def test_get_masking_request( + self, combined_traversal, saas_example_connection_config + ): saas_config: Optional[ SaaSConfig ] = saas_example_connection_config.get_saas_config() @@ -916,8 +935,7 @@ def test_get_masking_request(self, combined_traversal, saas_example_connection_c # Define delete request on conversations endpoint endpoints["conversations"].requests["delete"] = SaaSRequest( - method="DELETE", - path="/api/0///" + method="DELETE", path="/api/0///" ) # Delete endpoint not used because MASKING_STRICT is True assert config.execution.MASKING_STRICT is True @@ -935,11 +953,10 @@ def test_get_masking_request(self, combined_traversal, saas_example_connection_c assert saas_request.method == "DELETE" # Define GDPR Delete - data_protection_request = SaaSRequest( - method="PUT", - path="/api/0/gdpr_delete" + data_protection_request = SaaSRequest(method="PUT", path="/api/0/gdpr_delete") + query_config = SaaSQueryConfig( + conversations, endpoints, {}, data_protection_request ) - query_config = SaaSQueryConfig(conversations, endpoints, {}, data_protection_request) # Assert GDPR Delete takes priority over Delete saas_request: SaaSRequest = query_config.get_masking_request() @@ -948,4 +965,4 @@ def test_get_masking_request(self, combined_traversal, saas_example_connection_c # Reset config.execution.MASKING_STRICT = True - del endpoints["conversations"].requests["delete"] \ No newline at end of file + del endpoints["conversations"].requests["delete"] diff --git a/tests/service/pagination/test_pagination_strategy_cursor.py b/tests/service/pagination/test_pagination_strategy_cursor.py index 08a5576f3..14249076b 100644 --- a/tests/service/pagination/test_pagination_strategy_cursor.py +++ b/tests/service/pagination/test_pagination_strategy_cursor.py @@ -31,12 +31,9 @@ def response_with_empty_list(): def test_cursor(response_with_body): - config = CursorPaginationConfiguration( - cursor_param="after", field="id" - ) + config = CursorPaginationConfiguration(cursor_param="after", field="id") request_params: SaaSRequestParams = SaaSRequestParams( - method=HTTPMethod.GET, - path="/conversations" + method=HTTPMethod.GET, path="/conversations" ) paginator = CursorPaginationStrategy(config) next_request: Optional[SaaSRequestParams] = paginator.get_next_request( @@ -50,9 +47,7 @@ def test_cursor(response_with_body): def test_missing_cursor_value(response_with_body): - config = CursorPaginationConfiguration( - cursor_param="after", field="hash" - ) + config = CursorPaginationConfiguration(cursor_param="after", field="hash") request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", @@ -66,9 +61,7 @@ def test_missing_cursor_value(response_with_body): def test_cursor_with_empty_list(response_with_empty_list): - config = CursorPaginationConfiguration( - cursor_param="after", field="id" - ) + config = CursorPaginationConfiguration(cursor_param="after", field="id") request_params: SaaSRequestParams = SaaSRequestParams( method=HTTPMethod.GET, path="/conversations", @@ -79,4 +72,3 @@ def test_cursor_with_empty_list(response_with_empty_list): request_params, {}, response_with_empty_list, "conversations" ) assert next_request is None - diff --git a/tests/service/pagination/test_pagination_strategy_factory.py b/tests/service/pagination/test_pagination_strategy_factory.py index 0b538612c..f8bf2a99c 100644 --- a/tests/service/pagination/test_pagination_strategy_factory.py +++ b/tests/service/pagination/test_pagination_strategy_factory.py @@ -9,6 +9,7 @@ OffsetPaginationStrategy, ) + def test_get_strategy_offset(): config = { "incremental_param": "page", diff --git a/tests/service/pagination/test_pagination_strategy_offset.py b/tests/service/pagination/test_pagination_strategy_offset.py index a0581028e..26a4caf9f 100644 --- a/tests/service/pagination/test_pagination_strategy_offset.py +++ b/tests/service/pagination/test_pagination_strategy_offset.py @@ -155,7 +155,9 @@ def test_validate_request(): "limit": 10, }, } - SaaSRequest(method="GET", path="/test", query_params=query_params, pagination=pagination) + SaaSRequest( + method="GET", path="/test", query_params=query_params, pagination=pagination + ) def test_validate_request_missing_param(): @@ -169,5 +171,7 @@ def test_validate_request_missing_param(): }, } with pytest.raises(ValueError) as exc: - SaaSRequest(method="GET", path="/test", query_params=query_params, pagination=pagination) + SaaSRequest( + method="GET", path="/test", query_params=query_params, pagination=pagination + ) assert "Query param 'page' not found." in str(exc.value) diff --git a/tests/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py b/tests/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py index c58395d16..546da6c73 100644 --- a/tests/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py +++ b/tests/service/processors/post_processor_strategy/test_post_processor_strategy_factory.py @@ -1,26 +1,25 @@ import pytest from fidesops.common_exceptions import NoSuchStrategyException, ValidationError -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_factory import get_strategy -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_filter import \ - FilterPostProcessorStrategy -from fidesops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import \ - UnwrapPostProcessorStrategy +from fidesops.service.processors.post_processor_strategy.post_processor_strategy_factory import ( + get_strategy, +) +from fidesops.service.processors.post_processor_strategy.post_processor_strategy_filter import ( + FilterPostProcessorStrategy, +) +from fidesops.service.processors.post_processor_strategy.post_processor_strategy_unwrap import ( + UnwrapPostProcessorStrategy, +) def test_get_strategy_filter(): - config = { - "field": "email_contact", - "value": "somebody@email.com" - } + config = {"field": "email_contact", "value": "somebody@email.com"} strategy = get_strategy(strategy_name="filter", configuration=config) assert isinstance(strategy, FilterPostProcessorStrategy) def test_get_strategy_unwrap(): - config = { - "data_path": "exact_matches.members" - } + config = {"data_path": "exact_matches.members"} strategy = get_strategy(strategy_name="unwrap", configuration=config) assert isinstance(strategy, UnwrapPostProcessorStrategy) diff --git a/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py b/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py index ad570ff1a..62cf71e96 100644 --- a/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py +++ b/tests/service/processors/post_processor_strategy/test_post_processor_strategy_filter.py @@ -153,6 +153,7 @@ def test_filter_by_nonexistent_identity_reference(): result = processor.process(data) assert result == [] + def test_filter_by_identity_reference_with_no_identity_data(): identity_data = None config = FilterPostProcessorConfiguration( diff --git a/tests/task/test_filter_results.py b/tests/task/test_filter_results.py index f522d11bb..51ffbce94 100644 --- a/tests/task/test_filter_results.py +++ b/tests/task/test_filter_results.py @@ -6,7 +6,8 @@ from fidesops.graph.config import FieldPath, CollectionAddress from fidesops.task.filter_results import ( select_and_save_field, - remove_empty_containers, filter_data_categories, + remove_empty_containers, + filter_data_categories, ) @@ -419,9 +420,7 @@ def test_remove_empty_containers(): orig = {"A": [[{"B": "C", "D": [{"F": {}}, {"G": []}]}, {"B": "D"}, {"B": "G"}]]} results = copy.deepcopy(orig) remove_empty_containers(results) - assert results == { - "A": [[{"B": "C"}, {"B": "D"}, {"B": "G"}]] - } + assert results == {"A": [[{"B": "C"}, {"B": "D"}, {"B": "G"}]]} def test_filter_data_categories(): diff --git a/tests/task/test_graph_task.py b/tests/task/test_graph_task.py index 6f5d6b8e7..c08a814c5 100644 --- a/tests/task/test_graph_task.py +++ b/tests/task/test_graph_task.py @@ -250,13 +250,15 @@ def test_pre_process_input_conversation_collection( def test_pre_process_input_data_group_dependent_fields(self): """Test processing inputs where several reference fields and an identity field have - been marked as dependent. + been marked as dependent. """ traversal_with_grouped_inputs = traversal_paired_dependency() n = traversal_with_grouped_inputs.traversal_node_dict[ CollectionAddress("mysql", "User") ] - task = MockSqlTask(n, TaskResources(EMPTY_REQUEST, Policy(), connection_configs)) + task = MockSqlTask( + n, TaskResources(EMPTY_REQUEST, Policy(), connection_configs) + ) project_output = [ { diff --git a/tests/task/test_refine_target_path.py b/tests/task/test_refine_target_path.py index dbec7d790..f5ce00b66 100644 --- a/tests/task/test_refine_target_path.py +++ b/tests/task/test_refine_target_path.py @@ -2,7 +2,8 @@ from fidesops.task.refine_target_path import ( refine_target_path, build_refined_target_paths, - join_detailed_path, _match_found, + join_detailed_path, + _match_found, ) from fidesops.util.collection_util import FIDESOPS_DO_NOT_MASK_INDEX diff --git a/tests/task/traversal_data.py b/tests/task/traversal_data.py index e82b8df04..4815730fe 100644 --- a/tests/task/traversal_data.py +++ b/tests/task/traversal_data.py @@ -605,7 +605,7 @@ def traversal_paired_dependency() -> Traversal: ScalarField(name="project_id"), ScalarField(name="organization_id"), ScalarField(name="org_leader_email", identity="email"), - ScalarField(name="project_name") + ScalarField(name="project_name"), ], ) users = Collection( @@ -620,19 +620,18 @@ def traversal_paired_dependency() -> Traversal: ), ScalarField( name="organization", - references=[(FieldAddress("mysql", "Project", "organization_id"), "from")], + references=[ + (FieldAddress("mysql", "Project", "organization_id"), "from") + ], ), ScalarField(name="username"), ScalarField(name="email", identity="email"), ScalarField(name="position"), - ], - grouped_inputs={"project", "organization", "email"} + grouped_inputs={"project", "organization", "email"}, ) - mysql = Dataset( - name="mysql", collections=[projects, users], connection_key="mysql" - ) + mysql = Dataset(name="mysql", collections=[projects, users], connection_key="mysql") graph = DatasetGraph(mysql) identity = {"email": "email@gmail.com"}