From b550c8c564cf478a31f47c7ab7bbb8ba9730945d Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 8 Aug 2022 08:48:25 -0700 Subject: [PATCH] Removing saas_config.toml (#1043) --- saas_config.toml | 80 ------------------- .../fixtures/saas/adobe_campaign_fixtures.py | 6 +- tests/ops/fixtures/saas/auth0_fixtures.py | 16 ++-- tests/ops/fixtures/saas/hubspot_fixtures.py | 6 +- tests/ops/fixtures/saas/mailchimp_fixtures.py | 6 +- tests/ops/fixtures/saas/outreach_fixtures.py | 6 +- .../ops/fixtures/saas/salesforce_fixtures.py | 6 +- tests/ops/fixtures/saas/segment_fixtures.py | 6 +- tests/ops/fixtures/saas/sendgrid_fixtures.py | 16 ++-- tests/ops/fixtures/saas/sentry_fixtures.py | 6 +- tests/ops/fixtures/saas/stripe_fixtures.py | 6 +- tests/ops/fixtures/saas/zendesk_fixtures.py | 6 +- tests/ops/fixtures/saas_example_fixtures.py | 27 +++++-- .../integration_tests/saas/test_auth0_task.py | 2 + .../saas/test_salesforce_task.py | 3 + 15 files changed, 56 insertions(+), 142 deletions(-) delete mode 100644 saas_config.toml diff --git a/saas_config.toml b/saas_config.toml deleted file mode 100644 index f44577d0ae7..00000000000 --- a/saas_config.toml +++ /dev/null @@ -1,80 +0,0 @@ -[saas_example] -domain = "domain" -username = "username" -api_key = "api_key" -api_version = "2.0" -page_size = "10" - -[mailchimp] -domain = "" -username = "" -api_key = "" -identity_email = "" - -[hubspot] -domain = "" -hapikey = "" -identity_email = "" - -[stripe] -domain = "" -api_key = "" -payment_types = "" -page_size = "" -identity_email = "" - -[sentry] -domain = "" -access_token = "" -identity_email = "" -erasure_access_token = "" -erasure_identity_email = "" -user_id_erasure = "" -issue_url = "" - -[segment] -domain = "" -api_domain = "" -personas_domain = "" -workspace = "" -access_token = "" -namespace_id = "" -access_secret = "" -user_token = "" -identity_email = "" - -[sendgrid] -domain = "" -api_key = "" -identity_email = "" - -[auth0] -domain = "" -client_id = "" -client_secret = "" -access_token = "" -identity_email = "" - -[outreach] -domain = "" -client_id = "" -client_secret = "" -redirect_uri = "" -page_size = "" -requester_email = "" -identity_email = "" - -[salesforce] -domain = "" -client_id = "" -client_secret = "" -access_token = "" -username = "" -password = "" -identity_email = "" - -[zendesk] -domain = "" -username = "" -api_key = "" -identity_email = "" diff --git a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py index 32409ee7318..74db6198567 100644 --- a/tests/ops/fixtures/saas/adobe_campaign_fixtures.py +++ b/tests/ops/fixtures/saas/adobe_campaign_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session @@ -17,12 +16,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("adobe_campaign") @pytest.fixture(scope="function") -def adobe_campaign_secrets(): +def adobe_campaign_secrets(saas_config): return { "domain": pydash.get(saas_config, "adobe_campaign.domain") or secrets["domain"], "organization_id": pydash.get(saas_config, "adobe_campaign.organization_id") @@ -39,7 +37,7 @@ def adobe_campaign_secrets(): @pytest.fixture(scope="function") -def adobe_campaign_identity_email(): +def adobe_campaign_identity_email(saas_config): return ( pydash.get(saas_config, "adobe_campaign.identity_email") or secrets["identity_email"] diff --git a/tests/ops/fixtures/saas/auth0_fixtures.py b/tests/ops/fixtures/saas/auth0_fixtures.py index 85668343900..010f250ccd5 100644 --- a/tests/ops/fixtures/saas/auth0_fixtures.py +++ b/tests/ops/fixtures/saas/auth0_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from fideslib.db import session from sqlalchemy.orm import Session @@ -20,17 +19,11 @@ from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("auth0") -@pytest.fixture(scope="session") -def auth0_erasure_identity_email(): - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" - - @pytest.fixture(scope="function") -def auth0_secrets(): +def auth0_secrets(saas_config): return { "domain": pydash.get(saas_config, "auth0.domain") or secrets["domain"], "access_token": pydash.get(saas_config, "auth0.access_token") @@ -39,10 +32,15 @@ def auth0_secrets(): @pytest.fixture(scope="function") -def auth0_identity_email(): +def auth0_identity_email(saas_config): return pydash.get(saas_config, "auth0.identity_email") or secrets["identity_email"] +@pytest.fixture(scope="session") +def auth0_erasure_identity_email(): + return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + + @pytest.fixture def auth0_config() -> Dict[str, Any]: return load_config("data/saas/config/auth0_config.yml") diff --git a/tests/ops/fixtures/saas/hubspot_fixtures.py b/tests/ops/fixtures/saas/hubspot_fixtures.py index 6289c48b19f..9510dffc534 100644 --- a/tests/ops/fixtures/saas/hubspot_fixtures.py +++ b/tests/ops/fixtures/saas/hubspot_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session @@ -20,14 +19,13 @@ from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("hubspot") HUBSPOT_FIRSTNAME = "SomeoneFirstname" @pytest.fixture(scope="session") -def hubspot_secrets(): +def hubspot_secrets(saas_config): return { "domain": pydash.get(saas_config, "hubspot.domain") or secrets["domain"], "hapikey": pydash.get(saas_config, "hubspot.hapikey") or secrets["hapikey"], @@ -35,7 +33,7 @@ def hubspot_secrets(): @pytest.fixture(scope="function") -def hubspot_identity_email(): +def hubspot_identity_email(saas_config): return ( pydash.get(saas_config, "hubspot.identity_email") or secrets["identity_email"] ) diff --git a/tests/ops/fixtures/saas/mailchimp_fixtures.py b/tests/ops/fixtures/saas/mailchimp_fixtures.py index ad2690366c4..d455feef1d4 100644 --- a/tests/ops/fixtures/saas/mailchimp_fixtures.py +++ b/tests/ops/fixtures/saas/mailchimp_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest -from fideslib.core.config import load_toml from fideslib.db import session from sqlalchemy.orm import Session @@ -19,12 +18,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("mailchimp") @pytest.fixture(scope="session") -def mailchimp_secrets(): +def mailchimp_secrets(saas_config): return { "domain": pydash.get(saas_config, "mailchimp.domain") or secrets["domain"], "username": pydash.get(saas_config, "mailchimp.username") @@ -34,7 +32,7 @@ def mailchimp_secrets(): @pytest.fixture(scope="session") -def mailchimp_identity_email(): +def mailchimp_identity_email(saas_config): return ( pydash.get(saas_config, "mailchimp.identity_email") or secrets["identity_email"] ) diff --git a/tests/ops/fixtures/saas/outreach_fixtures.py b/tests/ops/fixtures/saas/outreach_fixtures.py index 97a01ed6e1c..14a32bbd76d 100644 --- a/tests/ops/fixtures/saas/outreach_fixtures.py +++ b/tests/ops/fixtures/saas/outreach_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from fideslib.db import session from sqlalchemy.orm import Session @@ -18,12 +17,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("outreach") @pytest.fixture(scope="session") -def outreach_secrets(): +def outreach_secrets(saas_config): return { "domain": pydash.get(saas_config, "outreach.domain") or secrets["domain"], "requester_email": pydash.get(saas_config, "outreach.requester_email") @@ -40,7 +38,7 @@ def outreach_secrets(): @pytest.fixture(scope="session") -def outreach_identity_email(): +def outreach_identity_email(saas_config): return ( pydash.get(saas_config, "outreach.identity_email") or secrets["identity_email"] ) diff --git a/tests/ops/fixtures/saas/salesforce_fixtures.py b/tests/ops/fixtures/saas/salesforce_fixtures.py index f71a2d2cc14..6e6f7a6eca1 100644 --- a/tests/ops/fixtures/saas/salesforce_fixtures.py +++ b/tests/ops/fixtures/saas/salesforce_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from fideslib.db import session from sqlalchemy.orm import Session @@ -19,12 +18,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("salesforce") @pytest.fixture(scope="session") -def salesforce_secrets(): +def salesforce_secrets(saas_config): return { "domain": pydash.get(saas_config, "salesforce.domain") or secrets["domain"], "username": pydash.get(saas_config, "salesforce.username") @@ -41,7 +39,7 @@ def salesforce_secrets(): @pytest.fixture(scope="session") -def salesforce_identity_email(): +def salesforce_identity_email(saas_config): return ( pydash.get(saas_config, "salesforce.identity_email") or secrets["identity_email"] diff --git a/tests/ops/fixtures/saas/segment_fixtures.py b/tests/ops/fixtures/saas/segment_fixtures.py index 9676e93ccfd..898b6876741 100644 --- a/tests/ops/fixtures/saas/segment_fixtures.py +++ b/tests/ops/fixtures/saas/segment_fixtures.py @@ -6,7 +6,6 @@ import pytest import requests from faker import Faker -from fideslib.core.config import load_toml from fideslib.db import session from sqlalchemy.orm import Session @@ -21,12 +20,11 @@ from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("segment") @pytest.fixture(scope="session") -def segment_secrets(): +def segment_secrets(saas_config): return { "domain": pydash.get(saas_config, "segment.domain") or secrets["domain"], "personas_domain": pydash.get(saas_config, "segment.personas_domain") @@ -47,7 +45,7 @@ def segment_secrets(): @pytest.fixture(scope="session") -def segment_identity_email(): +def segment_identity_email(saas_config): return ( pydash.get(saas_config, "segment.identity_email") or secrets["identity_email"] ) diff --git a/tests/ops/fixtures/saas/sendgrid_fixtures.py b/tests/ops/fixtures/saas/sendgrid_fixtures.py index 409787b2fd5..6d3407f08f5 100644 --- a/tests/ops/fixtures/saas/sendgrid_fixtures.py +++ b/tests/ops/fixtures/saas/sendgrid_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from fideslib.db import session from sqlalchemy.orm import Session @@ -20,19 +19,13 @@ from tests.ops.test_helpers.saas_test_utils import poll_for_existence from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("sendgrid") SENDGRID_ERASURE_FIRSTNAME = "Erasurefirstname" -@pytest.fixture(scope="session") -def sendgrid_erasure_identity_email(): - return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" - - @pytest.fixture(scope="function") -def sendgrid_secrets(): +def sendgrid_secrets(saas_config): return { "domain": pydash.get(saas_config, "sendgrid.domain") or secrets["domain"], "api_key": pydash.get(saas_config, "sendgrid.api_key") or secrets["api_key"], @@ -40,12 +33,17 @@ def sendgrid_secrets(): @pytest.fixture(scope="function") -def sendgrid_identity_email(): +def sendgrid_identity_email(saas_config): return ( pydash.get(saas_config, "sendgrid.identity_email") or secrets["identity_email"] ) +@pytest.fixture(scope="session") +def sendgrid_erasure_identity_email(): + return f"{cryptographic_util.generate_secure_random_string(13)}@email.com" + + @pytest.fixture def sendgrid_config() -> Dict[str, Any]: return load_config("data/saas/config/sendgrid_config.yml") diff --git a/tests/ops/fixtures/saas/sentry_fixtures.py b/tests/ops/fixtures/saas/sentry_fixtures.py index f75d81afad9..f4f4365717f 100644 --- a/tests/ops/fixtures/saas/sentry_fixtures.py +++ b/tests/ops/fixtures/saas/sentry_fixtures.py @@ -2,7 +2,6 @@ import pydash import pytest -from fideslib.core.config import load_toml from fideslib.db import session from sqlalchemy.orm import Session @@ -16,12 +15,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("sentry") @pytest.fixture(scope="session") -def sentry_secrets(): +def sentry_secrets(saas_config): return { "domain": pydash.get(saas_config, "sentry.domain") or secrets["sentry.domain"], "access_token": pydash.get(saas_config, "sentry.access_token") @@ -40,7 +38,7 @@ def sentry_secrets(): @pytest.fixture(scope="session") -def sentry_identity_email(): +def sentry_identity_email(saas_config): return pydash.get(saas_config, "sentry.identity_email") or secrets["identity_email"] diff --git a/tests/ops/fixtures/saas/stripe_fixtures.py b/tests/ops/fixtures/saas/stripe_fixtures.py index b0f3196d07e..ab4471ece74 100644 --- a/tests/ops/fixtures/saas/stripe_fixtures.py +++ b/tests/ops/fixtures/saas/stripe_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.db import session from multidimensional_urlencode import urlencode as multidimensional_urlencode from sqlalchemy.orm import Session @@ -18,12 +17,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("stripe") @pytest.fixture(scope="session") -def stripe_secrets(): +def stripe_secrets(saas_config): return { "domain": pydash.get(saas_config, "stripe.domain") or secrets["domain"], "api_key": pydash.get(saas_config, "stripe.api_key") or secrets["api_key"], @@ -35,7 +33,7 @@ def stripe_secrets(): @pytest.fixture(scope="session") -def stripe_identity_email(): +def stripe_identity_email(saas_config): return pydash.get(saas_config, "stripe.identity_email") or secrets["identity_email"] diff --git a/tests/ops/fixtures/saas/zendesk_fixtures.py b/tests/ops/fixtures/saas/zendesk_fixtures.py index 1be26f98aaa..d73afc6d4c1 100644 --- a/tests/ops/fixtures/saas/zendesk_fixtures.py +++ b/tests/ops/fixtures/saas/zendesk_fixtures.py @@ -3,7 +3,6 @@ import pydash import pytest import requests -from fideslib.core.config import load_toml from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session @@ -17,12 +16,11 @@ from tests.ops.fixtures.application_fixtures import load_dataset from tests.ops.test_helpers.vault_client import get_secrets -saas_config = load_toml(["saas_config.toml"]) secrets = get_secrets("zendesk") @pytest.fixture(scope="session") -def zendesk_secrets(): +def zendesk_secrets(saas_config): return { "domain": pydash.get(saas_config, "zendesk.domain") or secrets["domain"], "username": pydash.get(saas_config, "zendesk.username") or secrets["username"], @@ -31,7 +29,7 @@ def zendesk_secrets(): @pytest.fixture(scope="session") -def zendesk_identity_email(): +def zendesk_identity_email(saas_config): return ( pydash.get(saas_config, "zendesk.identity_email") or secrets["identity_email"] ) diff --git a/tests/ops/fixtures/saas_example_fixtures.py b/tests/ops/fixtures/saas_example_fixtures.py index 63ba6271b0c..e3c044e0531 100644 --- a/tests/ops/fixtures/saas_example_fixtures.py +++ b/tests/ops/fixtures/saas_example_fixtures.py @@ -1,8 +1,8 @@ +import logging from typing import Any, Dict, Generator -import pydash import pytest -from fideslib.core.config import load_file, load_toml +from fideslib.core.config import load_toml from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( @@ -14,20 +14,21 @@ from fidesops.schemas.saas.strategy_configuration import ( OAuth2AuthenticationConfiguration, ) +from fidesops.util.logger import NotPii from fidesops.util.saas_util import load_config from tests.ops.fixtures.application_fixtures import load_dataset -saas_config = load_toml(["saas_config.toml"]) +logger = logging.getLogger(__name__) @pytest.fixture(scope="function") def saas_example_secrets(): return { - "domain": pydash.get(saas_config, "saas_example.domain"), - "username": pydash.get(saas_config, "saas_example.username"), - "api_key": pydash.get(saas_config, "saas_example.api_key"), - "api_version": pydash.get(saas_config, "saas_example.api_version"), - "page_size": pydash.get(saas_config, "saas_example.page_size"), + "domain": "domain", + "username": "username", + "api_key": "api_key", + "api_version": "2.0", + "page_size": "page_size", } @@ -223,3 +224,13 @@ def oauth2_connection_config(db: Session, oauth2_configuration) -> Generator: ) yield connection_config connection_config.delete(db) + + +@pytest.fixture(scope="session") +def saas_config() -> Dict[str, Any]: + saas_config: Dict[str, Any] = {} + try: + saas_config: Dict[str, Any] = load_toml(["saas_config.toml"]) + except FileNotFoundError as e: + logger.warning("saas_config.toml could not be loaded: %s", NotPii(e)) + return saas_config diff --git a/tests/ops/integration_tests/saas/test_auth0_task.py b/tests/ops/integration_tests/saas/test_auth0_task.py index 038b7b2e0d3..6e6c824d6f7 100644 --- a/tests/ops/integration_tests/saas/test_auth0_task.py +++ b/tests/ops/integration_tests/saas/test_auth0_task.py @@ -12,6 +12,7 @@ from tests.ops.graph.graph_test_util import assert_rows_match +@pytest.mark.skip(reason="Pending development of OAuth2 JWT Bearer authentication") @pytest.mark.integration_saas @pytest.mark.integration_auth0 def test_auth0_access_request_task( @@ -83,6 +84,7 @@ def test_auth0_access_request_task( ) +@pytest.mark.skip(reason="Pending development of OAuth2 JWT Bearer authentication") @pytest.mark.integration_saas @pytest.mark.integration_auth0 def test_auth0_erasure_request_task( diff --git a/tests/ops/integration_tests/saas/test_salesforce_task.py b/tests/ops/integration_tests/saas/test_salesforce_task.py index e3d12970466..3bf37dba80a 100644 --- a/tests/ops/integration_tests/saas/test_salesforce_task.py +++ b/tests/ops/integration_tests/saas/test_salesforce_task.py @@ -13,12 +13,14 @@ from tests.ops.graph.graph_test_util import assert_rows_match +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas @pytest.mark.integration_salesforce def test_salesforce_connection_test(salesforce_connection_config) -> None: get_connector(salesforce_connection_config).test_connection() +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas @pytest.mark.integration_salesforce def test_salesforce_access_request_task( @@ -366,6 +368,7 @@ def test_salesforce_access_request_task( assert account["Id"] == account_id +@pytest.mark.skip(reason="Currently unable to test OAuth2 connectors") @pytest.mark.integration_saas @pytest.mark.integration_salesforce def test_salesforce_erasure_request_task(