From 4c6d5267672408723184188b983ae441de73d9ea Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 20 Mar 2023 11:03:11 -0700 Subject: [PATCH 01/17] Refactoring connector registry to prep for connector template uploads --- data/saas/config/adobe_campaign_config.yml | 4 +- data/saas/config/auth0_config.yml | 4 +- data/saas/config/braintree_config.yml | 4 +- data/saas/config/braze_config.yml | 2 +- data/saas/config/datadog_config.yml | 2 +- data/saas/config/delighted_config.yml | 2 +- data/saas/config/domo_config.yml | 4 +- data/saas/config/doordash_config.yml | 4 +- data/saas/config/firebase_auth_config.yml | 4 +- data/saas/config/friendbuy_config.yml | 6 +- data/saas/config/friendbuy_nextgen_config.yml | 4 +- data/saas/config/fullstory_config.yml | 2 +- data/saas/config/google_analytics_config.yml | 3 +- data/saas/config/hubspot_config.yml | 4 +- data/saas/config/jira_config.yml | 2 +- data/saas/config/kustomer_config.yml | 2 +- data/saas/config/mailchimp_config.yml | 4 +- .../config/mailchimp_transactional_config.yml | 3 +- data/saas/config/outreach_config.yml | 4 +- data/saas/config/recharge_config.yml | 2 +- data/saas/config/rollbar_config.yml | 4 +- data/saas/config/salesforce_config.yml | 2 +- data/saas/config/segment_config.yml | 4 +- data/saas/config/sendgrid_config.yml | 4 +- data/saas/config/sentry_config.yml | 4 +- data/saas/config/shopify_config.yml | 4 +- data/saas/config/slack_enterprise_config.yml | 2 +- data/saas/config/square_config.yml | 2 +- data/saas/config/stripe_config.yml | 2 +- .../config/twilio_conversations_config.yml | 4 +- .../config/universal_analytics_config.yml | 3 +- data/saas/config/vend_config.yml | 2 +- data/saas/config/wunderkind_config.yml | 2 +- data/saas/config/yotpo_loyalty_config.yml | 2 +- data/saas/config/yotpo_reviews_config.yml | 2 +- data/saas/config/zendesk_config.yml | 2 +- data/saas/dataset/adobe_campaign_dataset.yml | 2 +- data/saas/dataset/auth0_dataset.yml | 2 +- data/saas/dataset/braintree_dataset.yml | 2 +- data/saas/dataset/braze_dataset.yml | 2 +- data/saas/dataset/datadog_dataset.yml | 2 +- data/saas/dataset/domo_dataset.yml | 2 +- data/saas/dataset/doordash_dataset.yml | 2 +- data/saas/dataset/firebase_auth_dataset.yml | 2 +- data/saas/dataset/friendbuy_dataset.yml | 4 +- .../dataset/friendbuy_nextgen_dataset.yml | 2 +- data/saas/dataset/hubspot_dataset.yml | 2 +- data/saas/dataset/mailchimp_dataset.yml | 2 +- data/saas/dataset/outreach_dataset.yml | 2 +- data/saas/dataset/rollbar_dataset.yml | 2 +- data/saas/dataset/salesforce_dataset.yml | 2 +- data/saas/dataset/segment_dataset.yml | 2 +- data/saas/dataset/sendgrid_dataset.yml | 2 +- data/saas/dataset/sentry_dataset.yml | 2 +- data/saas/dataset/shopify_dataset.yml | 2 +- data/saas/dataset/square_dataset.yml | 2 +- data/saas/dataset/stripe_dataset.yml | 2 +- .../dataset/twilio_conversations_dataset.yml | 2 +- .../icon/{adobe.svg => adobe_campaign.svg} | 0 .../icon/{yotpo.svg => yotpo_loyalty.svg} | 0 data/saas/icon/yotpo_reviews.svg | 9 ++ src/fides/api/main.py | 5 +- .../api/v1/endpoints/saas_config_endpoints.py | 11 +- src/fides/api/ops/schemas/privacy_request.py | 2 +- .../ops/schemas/saas/connector_template.py | 28 ++++ .../saas/connector_registry_service.py | 122 +++++++++--------- src/fides/api/ops/util/connection_type.py | 16 +-- src/fides/api/ops/util/connection_util.py | 6 +- src/fides/api/ops/util/saas_util.py | 33 ++++- .../lib/oauth/api/routes/user_endpoints.py | 6 +- .../saas/connection_template_fixtures.py | 9 +- .../test_data}/mailchimp_override_config.yml | 4 +- .../test_data}/mailchimp_override_dataset.yml | 2 +- .../test_data}/saas_erasure_order_config.yml | 0 .../test_data}/saas_erasure_order_dataset.yml | 0 .../saas/test_data}/saas_example_config.yml | 2 +- .../saas/test_data}/saas_example_dataset.yml | 3 +- .../saas_external_example_config.yml | 4 +- tests/fixtures/saas_example_fixtures.py | 10 +- .../test_connection_template_endpoints.py | 60 +++------ .../test_connector_registry_service.py | 62 +++++---- .../test_connector_template_loaders.py | 32 +++++ tests/ops/util/test_connection_type.py | 16 +-- 83 files changed, 324 insertions(+), 275 deletions(-) rename data/saas/icon/{adobe.svg => adobe_campaign.svg} (100%) rename data/saas/icon/{yotpo.svg => yotpo_loyalty.svg} (100%) create mode 100644 data/saas/icon/yotpo_reviews.svg create mode 100644 src/fides/api/ops/schemas/saas/connector_template.py rename {data/saas/config/request_override => tests/fixtures/saas/test_data}/mailchimp_override_config.yml (92%) rename {data/saas/dataset/request_override => tests/fixtures/saas/test_data}/mailchimp_override_dataset.yml (99%) rename {data/saas/config => tests/fixtures/saas/test_data}/saas_erasure_order_config.yml (100%) rename {data/saas/dataset => tests/fixtures/saas/test_data}/saas_erasure_order_dataset.yml (100%) rename {data/saas/config => tests/fixtures/saas/test_data}/saas_example_config.yml (99%) rename {data/saas/dataset => tests/fixtures/saas/test_data}/saas_example_dataset.yml (99%) rename {data/saas/config => tests/fixtures/saas/test_data}/saas_external_example_config.yml (85%) create mode 100644 tests/ops/service/connectors/test_connector_template_loaders.py diff --git a/data/saas/config/adobe_campaign_config.yml b/data/saas/config/adobe_campaign_config.yml index 18ccb955f00..8a7b0a97d0b 100644 --- a/data/saas/config/adobe_campaign_config.yml +++ b/data/saas/config/adobe_campaign_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Adobe Campaign SaaS Config + name: Adobe Campaign type: adobe_campaign - description: A schema representing the Adobe Campaign connector for Fidesops + description: A schema representing the Adobe Campaign connector for Fides version: 0.0.2 connector_params: diff --git a/data/saas/config/auth0_config.yml b/data/saas/config/auth0_config.yml index 0d612a01ff5..f56eff9ab10 100644 --- a/data/saas/config/auth0_config.yml +++ b/data/saas/config/auth0_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Auth0 SaaS Config + name: Auth0 type: auth0 - description: A sample schema representing the Auth0 connector for Fidesops + description: A sample schema representing the Auth0 connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/braintree_config.yml b/data/saas/config/braintree_config.yml index f173122bba2..76a0b7141fb 100644 --- a/data/saas/config/braintree_config.yml +++ b/data/saas/config/braintree_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Braintree SaaS Config + name: Braintree type: braintree - description: A sample schema representing the Braintree connector for Fidesops + description: A sample schema representing the Braintree connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/braze_config.yml b/data/saas/config/braze_config.yml index 5954fea6eab..d1a2cfdaeb8 100644 --- a/data/saas/config/braze_config.yml +++ b/data/saas/config/braze_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Braze SaaS Config + name: Braze type: braze description: A sample schema representing the Braze connector for Fides version: 0.0.2 diff --git a/data/saas/config/datadog_config.yml b/data/saas/config/datadog_config.yml index 22ace9d5c0c..b5aea48a673 100644 --- a/data/saas/config/datadog_config.yml +++ b/data/saas/config/datadog_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Datadog SaaS Config + name: Datadog type: datadog description: A sample schema representing the Datadog connector for Fides version: 0.0.2 diff --git a/data/saas/config/delighted_config.yml b/data/saas/config/delighted_config.yml index 1a1282c2e12..17bebd03f7d 100644 --- a/data/saas/config/delighted_config.yml +++ b/data/saas/config/delighted_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Delighted SaaS Config + name: Delighted type: delighted description: A sample schema representing the Delighted connector for Fides version: 0.1.0 diff --git a/data/saas/config/domo_config.yml b/data/saas/config/domo_config.yml index ec85aa1904a..5b1ef6fae00 100644 --- a/data/saas/config/domo_config.yml +++ b/data/saas/config/domo_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Domo SaaS Config + name: Domo type: domo - description: A sample schema representing the Domo connector for Fidesops + description: A sample schema representing the Domo connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/doordash_config.yml b/data/saas/config/doordash_config.yml index fb3aa6018b8..dcbe8daf5ec 100644 --- a/data/saas/config/doordash_config.yml +++ b/data/saas/config/doordash_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Doordash SaaS Config + name: Doordash type: doordash - description: A sample schema representing the Doordash connector for Fidesops + description: A sample schema representing the Doordash connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/firebase_auth_config.yml b/data/saas/config/firebase_auth_config.yml index 27930f9112a..c79470e62ce 100644 --- a/data/saas/config/firebase_auth_config.yml +++ b/data/saas/config/firebase_auth_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Firebase Auth SaaS Config + name: Firebase Auth type: firebase_auth - description: A sample schema representing the Firebase Auth connector for Fidesops + description: A sample schema representing the Firebase Auth connector for Fides version: 0.0.3 connector_params: diff --git a/data/saas/config/friendbuy_config.yml b/data/saas/config/friendbuy_config.yml index b5a742a9361..c4e94259134 100644 --- a/data/saas/config/friendbuy_config.yml +++ b/data/saas/config/friendbuy_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Friendbuy SaaS Config + name: Friendbuy type: friendbuy - description: A sample schema representing the Friendbuy connector for Fidesops + description: A sample schema representing the Friendbuy connector for Fides version: 0.0.1 connector_params: @@ -55,4 +55,4 @@ saas_config: references: - dataset: field: customer.email_address - direction: from \ No newline at end of file + direction: from diff --git a/data/saas/config/friendbuy_nextgen_config.yml b/data/saas/config/friendbuy_nextgen_config.yml index a2e50cac52c..b964109ad7c 100644 --- a/data/saas/config/friendbuy_nextgen_config.yml +++ b/data/saas/config/friendbuy_nextgen_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Frienddbuy Nextgen SaaS Config + name: Friendbuy Nextgen type: friendbuy_nextgen - description: A sample schema representing the Frienddbuy Nextgen connector for Fidesops + description: A sample schema representing the Friendbuy Nextgen connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/fullstory_config.yml b/data/saas/config/fullstory_config.yml index 6fceaf958b8..1933d1aa58f 100644 --- a/data/saas/config/fullstory_config.yml +++ b/data/saas/config/fullstory_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Fullstory SaaS Config + name: Fullstory type: fullstory description: A sample schema representing the Fullstory connector for Fides version: 0.0.1 diff --git a/data/saas/config/google_analytics_config.yml b/data/saas/config/google_analytics_config.yml index d476fdc6956..a1b16107651 100644 --- a/data/saas/config/google_analytics_config.yml +++ b/data/saas/config/google_analytics_config.yml @@ -1,12 +1,11 @@ saas_config: fides_key: - name: Google Analytics 4 (GA4) SaaS Config + name: Google Analytics 4 type: google_analytics description: A schema representing the Google Analytics 4 (GA4) connector for Fides version: 0.0.1 connector_params: - - name: client_id - name: client_secret - name: redirect_uri diff --git a/data/saas/config/hubspot_config.yml b/data/saas/config/hubspot_config.yml index 5f18b00418c..70d61ac2d4e 100644 --- a/data/saas/config/hubspot_config.yml +++ b/data/saas/config/hubspot_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Hubspot SaaS Config + name: HubSpot type: hubspot - description: A sample schema representing the Hubspot connector for Fidesops + description: A sample schema representing the HubSpot connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/jira_config.yml b/data/saas/config/jira_config.yml index 90bf4e46038..0c2deabbdda 100644 --- a/data/saas/config/jira_config.yml +++ b/data/saas/config/jira_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Jira SaaS Config + name: Jira type: jira description: A sample schema representing the Jira connector for Fides version: 0.1.0 diff --git a/data/saas/config/kustomer_config.yml b/data/saas/config/kustomer_config.yml index feb6b675848..ca195d0ec2e 100644 --- a/data/saas/config/kustomer_config.yml +++ b/data/saas/config/kustomer_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Kustomer SaaS Config + name: Kustomer type: kustomer description: A sample schema representing the Kustomer connector for Fides version: 0.1.0 diff --git a/data/saas/config/mailchimp_config.yml b/data/saas/config/mailchimp_config.yml index 01baf0612c8..1aa85b3e8a4 100644 --- a/data/saas/config/mailchimp_config.yml +++ b/data/saas/config/mailchimp_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Mailchimp SaaS Config + name: Mailchimp type: mailchimp - description: A sample schema representing the Mailchimp connector for Fidesops + description: A sample schema representing the Mailchimp connector for Fides version: 0.0.2 connector_params: diff --git a/data/saas/config/mailchimp_transactional_config.yml b/data/saas/config/mailchimp_transactional_config.yml index cdf8e6b9c3e..f11fe325cb3 100644 --- a/data/saas/config/mailchimp_transactional_config.yml +++ b/data/saas/config/mailchimp_transactional_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Mailchimp Transactional SaaS Config + name: Mailchimp Transactional (Mandrill) type: mailchimp_transactional description: A sample schema representing the Mailchimp Transactional (Mandrill) connector for Fides version: 0.0.1 @@ -25,7 +25,6 @@ saas_config: method: GET path: /users/ping - consent_requests: opt_in: method: POST diff --git a/data/saas/config/outreach_config.yml b/data/saas/config/outreach_config.yml index b1570d37f6c..129888f0172 100644 --- a/data/saas/config/outreach_config.yml +++ b/data/saas/config/outreach_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Outreach Example Config + name: Outreach type: outreach - description: A sample schema representing the Outreach connector for Fidesops + description: A sample schema representing the Outreach connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/recharge_config.yml b/data/saas/config/recharge_config.yml index 9f249a06d04..789bef02aaf 100644 --- a/data/saas/config/recharge_config.yml +++ b/data/saas/config/recharge_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Recharge SaaS Config + name: Recharge type: recharge description: A sample schema representing the Recharge connector for Fides version: 0.0.1 diff --git a/data/saas/config/rollbar_config.yml b/data/saas/config/rollbar_config.yml index 5f777739b3b..6980f9fa20f 100644 --- a/data/saas/config/rollbar_config.yml +++ b/data/saas/config/rollbar_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Rollbar SaaS Config + name: Rollbar type: rollbar - description: A sample schema representing the Rollbar connector for Fidesops + description: A sample schema representing the Rollbar connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/salesforce_config.yml b/data/saas/config/salesforce_config.yml index b87a520f1b6..7651f949e53 100644 --- a/data/saas/config/salesforce_config.yml +++ b/data/saas/config/salesforce_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Salesforce Classic SaaS Config + name: Salesforce Classic type: salesforce description: A sample schema representing the Salesforce Classic connector for Fides version: 0.0.2 diff --git a/data/saas/config/segment_config.yml b/data/saas/config/segment_config.yml index f2a884a85fc..16d5dfd9ffc 100644 --- a/data/saas/config/segment_config.yml +++ b/data/saas/config/segment_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Segment SaaS Config + name: Segment type: segment - description: A sample schema representing the Segment connector for Fidesops + description: A sample schema representing the Segment connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/sendgrid_config.yml b/data/saas/config/sendgrid_config.yml index 58f33cecc5b..a7ab6806bde 100644 --- a/data/saas/config/sendgrid_config.yml +++ b/data/saas/config/sendgrid_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Sendgrid SaaS Config + name: SendGrid type: sendgrid - description: A sample schema representing the Sendgrid connector for Fidesops + description: A sample schema representing the SendGrid connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/sentry_config.yml b/data/saas/config/sentry_config.yml index 75de54823d5..582e7e312d3 100644 --- a/data/saas/config/sentry_config.yml +++ b/data/saas/config/sentry_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Sentry SaaS Config + name: Sentry type: sentry - description: A sample schema representing the Sentry connector for Fidesops + description: A sample schema representing the Sentry connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/shopify_config.yml b/data/saas/config/shopify_config.yml index 00c3f9f4971..626c2c15576 100644 --- a/data/saas/config/shopify_config.yml +++ b/data/saas/config/shopify_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Shopify SaaS Config + name: Shopify type: shopify - description: A sample schema representing the Shopify connector for Fidesops + description: A sample schema representing the Shopify connector for Fides version: 0.0.1 connector_params: diff --git a/data/saas/config/slack_enterprise_config.yml b/data/saas/config/slack_enterprise_config.yml index 46ed7deccb4..f55d4b23e24 100644 --- a/data/saas/config/slack_enterprise_config.yml +++ b/data/saas/config/slack_enterprise_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Slack Enterprise SaaS Config + name: Slack Enterprise type: slack_enterprise description: A sample schema representing the Slack Enterprise connector for Fides version: 0.0.1 diff --git a/data/saas/config/square_config.yml b/data/saas/config/square_config.yml index 901a0405206..146376659d6 100644 --- a/data/saas/config/square_config.yml +++ b/data/saas/config/square_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Square SaaS Config + name: Square type: square description: A sample schema representing the Square connector for Fides version: 0.0.2 diff --git a/data/saas/config/stripe_config.yml b/data/saas/config/stripe_config.yml index f017b09c25e..d7617ae0813 100644 --- a/data/saas/config/stripe_config.yml +++ b/data/saas/config/stripe_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Stripe SaaS Config + name: Stripe type: stripe description: A sample schema representing the Stripe connector for Fides version: 0.0.2 diff --git a/data/saas/config/twilio_conversations_config.yml b/data/saas/config/twilio_conversations_config.yml index bbdb0ec0d56..52c09be0348 100644 --- a/data/saas/config/twilio_conversations_config.yml +++ b/data/saas/config/twilio_conversations_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: - name: Twilio Conversations SaaS Config + name: Twilio Conversations type: twilio_conversations - description: A sample schema representing the Twilio Conversations connector for Fidesops + description: A sample schema representing the Twilio Conversations connector for Fides version: 0.0.2 connector_params: diff --git a/data/saas/config/universal_analytics_config.yml b/data/saas/config/universal_analytics_config.yml index a534486744d..4943ed4f981 100644 --- a/data/saas/config/universal_analytics_config.yml +++ b/data/saas/config/universal_analytics_config.yml @@ -1,12 +1,11 @@ saas_config: fides_key: - name: Universal Analytics (UA) SaaS Config + name: Universal Analytics type: universal_analytics description: A schema representing the Universal Analytics connector for Fides version: 0.0.1 connector_params: - - name: client_id - name: client_secret - name: redirect_uri diff --git a/data/saas/config/vend_config.yml b/data/saas/config/vend_config.yml index 5f24d431101..5d8635cb27d 100644 --- a/data/saas/config/vend_config.yml +++ b/data/saas/config/vend_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Vend SaaS Config + name: Vend type: vend description: A sample schema representing the Vend connector for Fides version: 0.1.0 diff --git a/data/saas/config/wunderkind_config.yml b/data/saas/config/wunderkind_config.yml index 9660858a4c2..807bec5835a 100644 --- a/data/saas/config/wunderkind_config.yml +++ b/data/saas/config/wunderkind_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Wunderkind SaaS Config + name: Wunderkind type: wunderkind description: A schema representing the Wunderkind connector for Fides version: 0.0.1 diff --git a/data/saas/config/yotpo_loyalty_config.yml b/data/saas/config/yotpo_loyalty_config.yml index 57e8bbe3c0e..517e6ae9f92 100644 --- a/data/saas/config/yotpo_loyalty_config.yml +++ b/data/saas/config/yotpo_loyalty_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Yotpo Loyalty & Referrals SaaS Config + name: Yotpo Loyalty & Referrals type: yotpo_loyalty description: A sample schema representing the Yotpo Loyalty & Referrals connector for Fides version: 0.1.0 diff --git a/data/saas/config/yotpo_reviews_config.yml b/data/saas/config/yotpo_reviews_config.yml index 198917b29a9..e60158c92e8 100644 --- a/data/saas/config/yotpo_reviews_config.yml +++ b/data/saas/config/yotpo_reviews_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Yotpo Reviews SaaS Config + name: Yotpo Reviews type: yotpo_reviews description: A sample schema representing the Yotpo Reviews connector for Fides version: 0.1.0 diff --git a/data/saas/config/zendesk_config.yml b/data/saas/config/zendesk_config.yml index 38d35a38965..ebf50e47c0f 100644 --- a/data/saas/config/zendesk_config.yml +++ b/data/saas/config/zendesk_config.yml @@ -1,6 +1,6 @@ saas_config: fides_key: - name: Zendesk SaaS Config + name: Zendesk type: zendesk description: A sample schema representing the Zendesk connector for Fides version: 0.0.1 diff --git a/data/saas/dataset/adobe_campaign_dataset.yml b/data/saas/dataset/adobe_campaign_dataset.yml index 09a729447b5..298babe3b29 100644 --- a/data/saas/dataset/adobe_campaign_dataset.yml +++ b/data/saas/dataset/adobe_campaign_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Adobe Campaign Dataset - description: A dataset representing the Adobe Campaign connector for Fidesops + description: A dataset representing the Adobe Campaign connector for Fides collections: - name: profile fields: diff --git a/data/saas/dataset/auth0_dataset.yml b/data/saas/dataset/auth0_dataset.yml index 6a812f8a8d3..af08a2a26d4 100644 --- a/data/saas/dataset/auth0_dataset.yml +++ b/data/saas/dataset/auth0_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Auth0 Dataset - description: A sample dataset representing the Auth0 connector for Fidesops + description: A sample dataset representing the Auth0 connector for Fides collections: - name: users fields: diff --git a/data/saas/dataset/braintree_dataset.yml b/data/saas/dataset/braintree_dataset.yml index 2907c93903b..67d2acbe317 100644 --- a/data/saas/dataset/braintree_dataset.yml +++ b/data/saas/dataset/braintree_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Braintree Dataset - description: A sample dataset representing the Braintree connector for Fidesops + description: A sample dataset representing the Braintree connector for Fides collections: - name: customer fields: diff --git a/data/saas/dataset/braze_dataset.yml b/data/saas/dataset/braze_dataset.yml index c8d5ca9d2a8..94908ae06b8 100644 --- a/data/saas/dataset/braze_dataset.yml +++ b/data/saas/dataset/braze_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Braze Dataset - description: A sample dataset representing the Braze connector for Fidesops + description: A sample dataset representing the Braze connector for Fides collections: - name: user fields: diff --git a/data/saas/dataset/datadog_dataset.yml b/data/saas/dataset/datadog_dataset.yml index 6e08ac53abf..d0f2eaf90a0 100644 --- a/data/saas/dataset/datadog_dataset.yml +++ b/data/saas/dataset/datadog_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Datadog Dataset - description: A sample dataset representing the Datadog connector for Fidesops + description: A sample dataset representing the Datadog connector for Fides collections: - name: events fields: diff --git a/data/saas/dataset/domo_dataset.yml b/data/saas/dataset/domo_dataset.yml index 29ee00ec7a0..83aa9a98f87 100644 --- a/data/saas/dataset/domo_dataset.yml +++ b/data/saas/dataset/domo_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Domo Dataset - description: A sample dataset representing the Domo connector for Fidesops + description: A sample dataset representing the Domo connector for Fides collections: - name: user fields: diff --git a/data/saas/dataset/doordash_dataset.yml b/data/saas/dataset/doordash_dataset.yml index e564652401a..56959fafea3 100644 --- a/data/saas/dataset/doordash_dataset.yml +++ b/data/saas/dataset/doordash_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Doordash Dataset - description: A sample dataset representing the Doordash connector for Fidesops + description: A sample dataset representing the Doordash connector for Fides collections: - name: deliveries fields: diff --git a/data/saas/dataset/firebase_auth_dataset.yml b/data/saas/dataset/firebase_auth_dataset.yml index 8af9893b4fb..a95125dc5aa 100644 --- a/data/saas/dataset/firebase_auth_dataset.yml +++ b/data/saas/dataset/firebase_auth_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Firebase Auth Dataset - description: A sample dataset representing the Firebase Auth connector for Fidesops + description: A sample dataset representing the Firebase Auth connector for Fides collections: - name: user fields: diff --git a/data/saas/dataset/friendbuy_dataset.yml b/data/saas/dataset/friendbuy_dataset.yml index 33aed13f978..7c9cd34c847 100644 --- a/data/saas/dataset/friendbuy_dataset.yml +++ b/data/saas/dataset/friendbuy_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Friendbuy Dataset - description: A sample dataset representing the Friendbuy connector for Fidesops + description: A sample dataset representing the Friendbuy connector for Fides collections: - name: customer fields: @@ -49,4 +49,4 @@ dataset: - name: last_modified_at data_categories: [system.operations] fidesops_meta: - data_type: string \ No newline at end of file + data_type: string diff --git a/data/saas/dataset/friendbuy_nextgen_dataset.yml b/data/saas/dataset/friendbuy_nextgen_dataset.yml index a6bd6ec6311..d89887a2175 100644 --- a/data/saas/dataset/friendbuy_nextgen_dataset.yml +++ b/data/saas/dataset/friendbuy_nextgen_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: friendbuy_nextgen_instance name: Friendbuy Dataset - description: A sample dataset representing the Friendbuy connector for Fidesops + description: A sample dataset representing the Friendbuy connector for Fides collections: - name: user fields: diff --git a/data/saas/dataset/hubspot_dataset.yml b/data/saas/dataset/hubspot_dataset.yml index 22e788ae610..02b7df3a3f6 100644 --- a/data/saas/dataset/hubspot_dataset.yml +++ b/data/saas/dataset/hubspot_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Hubspot Dataset - description: A sample dataset representing the Hubspot connector for Fidesops + description: A sample dataset representing the Hubspot connector for Fides collections: - name: contacts fields: diff --git a/data/saas/dataset/mailchimp_dataset.yml b/data/saas/dataset/mailchimp_dataset.yml index 9d47275437d..8dbbb427644 100644 --- a/data/saas/dataset/mailchimp_dataset.yml +++ b/data/saas/dataset/mailchimp_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Mailchimp Dataset - description: A sample dataset representing the Mailchimp connector for Fidesops + description: A sample dataset representing the Mailchimp connector for Fides collections: - name: messages fields: diff --git a/data/saas/dataset/outreach_dataset.yml b/data/saas/dataset/outreach_dataset.yml index cfbc732e1e1..85cdfd9e672 100644 --- a/data/saas/dataset/outreach_dataset.yml +++ b/data/saas/dataset/outreach_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Outreach Dataset - description: A sample dataset representing the Outreach connector for Fidesops + description: A sample dataset representing the Outreach connector for Fides collections: - name: recipients fields: diff --git a/data/saas/dataset/rollbar_dataset.yml b/data/saas/dataset/rollbar_dataset.yml index 3d30d54a169..9003e958bd0 100644 --- a/data/saas/dataset/rollbar_dataset.yml +++ b/data/saas/dataset/rollbar_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Rollbar Dataset - description: A sample dataset representing the Rollbar connector for Fidesops + description: A sample dataset representing the Rollbar connector for Fides collections: - name: projects fields: diff --git a/data/saas/dataset/salesforce_dataset.yml b/data/saas/dataset/salesforce_dataset.yml index 6302a5e44bf..3ee5f19d9e6 100644 --- a/data/saas/dataset/salesforce_dataset.yml +++ b/data/saas/dataset/salesforce_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Salesforce Example Dataset - description: A sample dataset representing the Salesforce connector for Fidesops + description: A sample dataset representing the Salesforce connector for Fides collections: - name: contact_list fields: diff --git a/data/saas/dataset/segment_dataset.yml b/data/saas/dataset/segment_dataset.yml index 473d93dd502..e85c2375b86 100644 --- a/data/saas/dataset/segment_dataset.yml +++ b/data/saas/dataset/segment_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Segment Dataset - description: A sample dataset representing the Segment connector for Fidesops + description: A sample dataset representing the Segment connector for Fides collections: - name: segment_user fields: diff --git a/data/saas/dataset/sendgrid_dataset.yml b/data/saas/dataset/sendgrid_dataset.yml index dcb625bf85b..fdc766444b6 100644 --- a/data/saas/dataset/sendgrid_dataset.yml +++ b/data/saas/dataset/sendgrid_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Sendgrid Dataset - description: A sample dataset representing the Sendgrid connector for Fidesops + description: A sample dataset representing the Sendgrid connector for Fides collections: - name: contacts fields: diff --git a/data/saas/dataset/sentry_dataset.yml b/data/saas/dataset/sentry_dataset.yml index 41617b09c6d..6dfcda5443f 100644 --- a/data/saas/dataset/sentry_dataset.yml +++ b/data/saas/dataset/sentry_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Sentry Dataset - description: A sample dataset representing the Sentry connector for Fidesops + description: A sample dataset representing the Sentry connector for Fides collections: - name: organizations fields: diff --git a/data/saas/dataset/shopify_dataset.yml b/data/saas/dataset/shopify_dataset.yml index 3ea625a994a..1cdb032ff7f 100644 --- a/data/saas/dataset/shopify_dataset.yml +++ b/data/saas/dataset/shopify_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Shopify Dataset - description: A sample dataset representing the Shopify connector for Fidesops + description: A sample dataset representing the Shopify connector for Fides collections: - name: customers fields: diff --git a/data/saas/dataset/square_dataset.yml b/data/saas/dataset/square_dataset.yml index 76482fa6a0d..d45b3d6bca3 100644 --- a/data/saas/dataset/square_dataset.yml +++ b/data/saas/dataset/square_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Square Dataset - description: A sample dataset representing the Square connector for Fidesops + description: A sample dataset representing the Square connector for Fides collections: - name: customer fields: diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml index 212e5f08efb..b89f7f7470a 100644 --- a/data/saas/dataset/stripe_dataset.yml +++ b/data/saas/dataset/stripe_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Stripe Dataset - description: A sample dataset representing the Stripe connector for Fidesops + description: A sample dataset representing the Stripe connector for Fides collections: - name: customer fields: diff --git a/data/saas/dataset/twilio_conversations_dataset.yml b/data/saas/dataset/twilio_conversations_dataset.yml index a195e81a286..9001760e454 100644 --- a/data/saas/dataset/twilio_conversations_dataset.yml +++ b/data/saas/dataset/twilio_conversations_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: name: Twilio Conversations Dataset - description: A sample dataset representing the Twilio Conversations connector for Fidesops + description: A sample dataset representing the Twilio Conversations connector for Fides collections: - name: user fields: diff --git a/data/saas/icon/adobe.svg b/data/saas/icon/adobe_campaign.svg similarity index 100% rename from data/saas/icon/adobe.svg rename to data/saas/icon/adobe_campaign.svg diff --git a/data/saas/icon/yotpo.svg b/data/saas/icon/yotpo_loyalty.svg similarity index 100% rename from data/saas/icon/yotpo.svg rename to data/saas/icon/yotpo_loyalty.svg diff --git a/data/saas/icon/yotpo_reviews.svg b/data/saas/icon/yotpo_reviews.svg new file mode 100644 index 00000000000..37e94f3f424 --- /dev/null +++ b/data/saas/icon/yotpo_reviews.svg @@ -0,0 +1,9 @@ + + + + + + + + + diff --git a/src/fides/api/main.py b/src/fides/api/main.py index e45d9385ba5..bceed0e2b3e 100644 --- a/src/fides/api/main.py +++ b/src/fides/api/main.py @@ -56,8 +56,6 @@ from fides.api.ops.models.application_config import ApplicationConfig from fides.api.ops.schemas.analytics import Event, ExtraData from fides.api.ops.service.connectors.saas.connector_registry_service import ( - load_registry, - registry_file, update_saas_configs, ) @@ -273,9 +271,8 @@ async def setup_server() -> None: logger.info("Validating SaaS connector templates...") try: - registry = load_registry(registry_file) db = get_api_session() - update_saas_configs(registry, db) + update_saas_configs(db) logger.info("Finished loading saas templates") except Exception as e: logger.error( diff --git a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py index b5009af8a14..8b0af20d02b 100644 --- a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py @@ -37,6 +37,7 @@ SaasConnectionTemplateResponse, SaasConnectionTemplateValues, ) +from fides.api.ops.schemas.saas.connector_template import ConnectorTemplate from fides.api.ops.schemas.saas.saas_config import ( SaaSConfig, SaaSConfigValidationDetails, @@ -50,10 +51,7 @@ ) from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, - ConnectorTemplate, create_connection_config_from_template_no_save, - load_registry, - registry_file, upsert_dataset_config_from_template, ) from fides.api.ops.util.api_router import APIRouter @@ -283,10 +281,9 @@ def instantiate_connection_from_template( fields are provided, persists the associated connection config and dataset to the database. """ - registry: ConnectorRegistry = load_registry(registry_file) - connector_template: Optional[ConnectorTemplate] = registry.get_connector_template( - saas_connector_type - ) + connector_template: Optional[ + ConnectorTemplate + ] = ConnectorRegistry.get_connector_template(saas_connector_type) if not connector_template: raise HTTPException( status_code=HTTP_404_NOT_FOUND, diff --git a/src/fides/api/ops/schemas/privacy_request.py b/src/fides/api/ops/schemas/privacy_request.py index 200aaf6b810..d7aaa1e68f7 100644 --- a/src/fides/api/ops/schemas/privacy_request.py +++ b/src/fides/api/ops/schemas/privacy_request.py @@ -4,8 +4,8 @@ from fideslang.validation import FidesKey from pydantic import Field, validator -from fides.api.custom_types import SafeStr +from fides.api.custom_types import SafeStr from fides.api.ops.models.policy import ActionType from fides.api.ops.models.privacy_request import ( CheckpointActionRequired, diff --git a/src/fides/api/ops/schemas/saas/connector_template.py b/src/fides/api/ops/schemas/saas/connector_template.py new file mode 100644 index 00000000000..e3b4efd0e61 --- /dev/null +++ b/src/fides/api/ops/schemas/saas/connector_template.py @@ -0,0 +1,28 @@ +import yaml +from fideslang.models import Dataset +from pydantic import BaseModel, validator + +from fides.api.ops.schemas.saas.saas_config import SaaSConfig + + +class ConnectorTemplate(BaseModel): + """ + A collection of artifacts that make up a complete SaaS connector (SaaS config, dataset, etc.) + """ + + config: str + dataset: str + icon: str + human_readable: str + + @validator("config") + def validate_config(cls, config: str) -> str: + """Validates the config at the given path""" + SaaSConfig(**yaml.safe_load(config).get("saas_config")) + return config + + @validator("dataset") + def validate_dataset(cls, dataset: str) -> str: + """Validates the dataset at the given path""" + Dataset(**yaml.safe_load(dataset).get("dataset")[0]) + return dataset diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index 0957cbfe27b..2399f24cf61 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -1,15 +1,13 @@ from __future__ import annotations -from os.path import exists +import os +from abc import ABC, abstractmethod from typing import Dict, Iterable, List, Optional, Union -from fideslang.models import Dataset from loguru import logger from packaging.version import LegacyVersion, Version from packaging.version import parse as parse_version -from pydantic import BaseModel, validator from sqlalchemy.orm import Session -from toml import load as load_toml from fides.api.ops.models.connectionconfig import ( AccessLevel, @@ -20,65 +18,71 @@ from fides.api.ops.schemas.connection_configuration.connection_config import ( SaasConnectionTemplateValues, ) +from fides.api.ops.schemas.saas.connector_template import ConnectorTemplate from fides.api.ops.schemas.saas.saas_config import SaaSConfig from fides.api.ops.util.saas_util import ( + encode_file_contents, load_config, - load_config_with_replacement, - load_dataset, - load_dataset_with_replacement, + load_config_from_string, + load_yaml_as_string, + replace_config_placeholders, + replace_dataset_placeholders, ) -_registry: Optional[ConnectorRegistry] = None -registry_file = "data/saas/saas_connector_registry.toml" +class ConnectorTemplateLoader(ABC): + @abstractmethod + def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: + """Returns a map of connection templates""" -class ConnectorTemplate(BaseModel): + +class FileConnectorTemplateLoader(ConnectorTemplateLoader): """ - A collection of paths to artifacts that make up - a complete SaaS connector (SaaS config, dataset, etc.) + Loads SaaS connector templates from the data/saas directory. """ - config: str - dataset: str - icon: str - human_readable: str - - @validator("config") - def validate_config(cls, config: str) -> str: - """Validates the config at the given path""" - SaaSConfig(**load_config(config)) - return config + def __init__(self) -> None: + self.templates: Dict[str, ConnectorTemplate] = {} + for file in os.listdir("data/saas/config"): + if file.endswith(".yml"): + config_file = os.path.join("data/saas/config", file) + config_dict = load_config(config_file) + connector_type = config_dict["type"] + human_readable = config_dict["name"] - @validator("dataset") - def validate_dataset(cls, dataset: str) -> str: - """Validates the dataset at the given path""" - Dataset(**load_dataset(dataset)[0]) - return dataset + try: + icon = encode_file_contents(f"data/saas/icon/{connector_type}.svg") + except FileNotFoundError: + icon = encode_file_contents("data/saas/icon/default.svg") - @validator("icon") - def validate_icon(cls, icon: str) -> str: - """Validates the icon at the given path""" - if not exists(icon): - raise ValueError(f"Icon file {icon} was not found") - return icon + # store connector template for retrieval + try: + self.templates[connector_type] = ConnectorTemplate( + config=load_yaml_as_string(config_file), + dataset=load_yaml_as_string( + f"data/saas/dataset/{connector_type}_dataset.yml" + ), + icon=icon, + human_readable=human_readable, + ) + except Exception: + logger.exception("Unable to load {} connector", connector_type) + def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: + return self.templates -class ConnectorRegistry(BaseModel): - """A map of SaaS connector templates""" - __root__: Dict[str, ConnectorTemplate] +class ConnectorRegistry: + loader = FileConnectorTemplateLoader() + _templates: Dict[str, ConnectorTemplate] = loader.get_connector_templates() - def connector_types(self) -> List[str]: - """List of registered SaaS connector types""" - return list(self.__root__) + @classmethod + def connector_types(cls) -> List[str]: + return list(cls._templates.keys()) - def get_connector_template( - self, connector_type: str - ) -> Optional[ConnectorTemplate]: - """ - Returns an object containing the references to the various SaaS connector artifacts - """ - return self.__root__.get(connector_type) + @classmethod + def get_connector_template(cls, connector_type: str) -> Optional[ConnectorTemplate]: + return cls._templates.get(connector_type) def create_connection_config_from_template_no_save( @@ -90,7 +94,7 @@ def create_connection_config_from_template_no_save( """Creates a SaaS connection config from a template without saving it.""" # Load saas config from template and replace every instance of "" with the fides_key # the user has chosen - config_from_template: Dict = load_config_with_replacement( + config_from_template: Dict = replace_config_placeholders( template.config, "", template_values.instance_key ) @@ -126,9 +130,9 @@ def upsert_dataset_config_from_template( """ # Load the dataset config from template and replace every instance of "" with the fides_key # the user has chosen - dataset_from_template: Dict = load_dataset_with_replacement( + dataset_from_template: Dict = replace_dataset_placeholders( template.dataset, "", template_values.instance_key - )[0] + ) data = { "connection_config_id": connection_config.id, "fides_key": template_values.instance_key, @@ -138,15 +142,7 @@ def upsert_dataset_config_from_template( return dataset_config -def load_registry(config_file: str) -> ConnectorRegistry: - """Loads a SaaS connector registry from the given config file.""" - global _registry # pylint: disable=W0603 - if _registry is None: - _registry = ConnectorRegistry.parse_obj(load_toml(config_file)) - return _registry - - -def update_saas_configs(registry: ConnectorRegistry, db: Session) -> None: +def update_saas_configs(db: Session) -> None: """ Updates SaaS config instances currently in the DB if to the corresponding template in the registry are found. @@ -154,17 +150,17 @@ def update_saas_configs(registry: ConnectorRegistry, db: Session) -> None: Effectively an "update script" for SaaS config instances, to be run on server bootstrap. """ - for connector_type in registry.connector_types(): + for connector_type in ConnectorRegistry.connector_types(): logger.debug( "Determining if any updates are needed for connectors of type {} based on templates...", connector_type, ) - template: ConnectorTemplate = registry.get_connector_template( # type: ignore + template: ConnectorTemplate = ConnectorRegistry.get_connector_template( # type: ignore connector_type ) - saas_config_template = SaaSConfig.parse_obj(load_config(template.config)) + saas_config = SaaSConfig(**load_config_from_string(template.config)) template_version: Union[LegacyVersion, Version] = parse_version( - saas_config_template.version + saas_config.version ) connection_configs: Iterable[ConnectionConfig] = ConnectionConfig.filter( @@ -215,7 +211,7 @@ def update_saas_instance( instance_key=saas_config_instance.fides_key, ) - config_from_template: Dict = load_config_with_replacement( + config_from_template: Dict = replace_config_placeholders( template.config, "", template_vals.instance_key ) diff --git a/src/fides/api/ops/util/connection_type.py b/src/fides/api/ops/util/connection_type.py index 023a16afe23..69f3ed33ad2 100644 --- a/src/fides/api/ops/util/connection_type.py +++ b/src/fides/api/ops/util/connection_type.py @@ -14,10 +14,9 @@ ) from fides.api.ops.schemas.saas.saas_config import SaaSConfig from fides.api.ops.service.connectors.saas.connector_registry_service import ( - load_registry, - registry_file, + ConnectorRegistry, ) -from fides.api.ops.util.saas_util import encode_file_contents, load_config +from fides.api.ops.util.saas_util import load_config def connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]: @@ -95,20 +94,17 @@ def is_match(elem: str) -> bool: ] ) if system_type == SystemType.saas or system_type is None: - registry = load_registry(registry_file) saas_types: list[str] = sorted( [ saas_type - for saas_type in registry.connector_types() + for saas_type in ConnectorRegistry.connector_types() if is_match(saas_type) ] ) for item in saas_types: - if registry.get_connector_template(item) is not None: - connector_template = registry.get_connector_template( # type: ignore[union-attr] - item - ) + if ConnectorRegistry.get_connector_template(item) is not None: + connector_template = ConnectorRegistry.get_connector_template(item) connection_system_types.append( ConnectionSystemTypeMap( @@ -117,7 +113,7 @@ def is_match(elem: str) -> bool: human_readable=connector_template.human_readable if connector_template is not None else "", - encoded_icon=encode_file_contents(connector_template.icon) + encoded_icon=connector_template.icon if connector_template is not None else None, ) diff --git a/src/fides/api/ops/util/connection_util.py b/src/fides/api/ops/util/connection_util.py index cfc19014210..d57a0bb6f17 100644 --- a/src/fides/api/ops/util/connection_util.py +++ b/src/fides/api/ops/util/connection_util.py @@ -33,9 +33,8 @@ validate_saas_secrets_external_references, ) from fides.api.ops.service.connectors.saas.connector_registry_service import ( + ConnectorRegistry, create_connection_config_from_template_no_save, - load_registry, - registry_file, ) from fides.api.ops.service.privacy_request.request_runner_service import ( queue_privacy_request, @@ -145,8 +144,7 @@ def patch_connection_configs( detail="saas_connector_type is missing", ) - registry = load_registry(registry_file) - connector_template = registry.get_connector_template( + connector_template = ConnectorRegistry.get_connector_template( config.saas_connector_type ) if not connector_template: diff --git a/src/fides/api/ops/util/saas_util.py b/src/fides/api/ops/util/saas_util.py index b414e45a449..62c2811a870 100644 --- a/src/fides/api/ops/util/saas_util.py +++ b/src/fides/api/ops/util/saas_util.py @@ -35,12 +35,25 @@ def load_yaml_as_string(filename: str) -> str: def load_config(filename: str) -> Dict: - """Loads the saas config from the yaml file""" + """Loads the SaaS config from provided filename""" yaml_file = load_file([filename]) with open(yaml_file, "r", encoding="utf-8") as file: return yaml.safe_load(file).get("saas_config", []) +def load_config_from_string(string: str) -> Dict: + """Loads the SaaS config dict from the yaml string""" + return yaml.safe_load(string).get("saas_config", []) + + +def replace_config_placeholders( + config: str, string_to_replace: str, replacement: str +) -> Dict: + """Loads the SaaS config from the yaml string and replaces any string with the given value""" + yaml_str: str = config.replace(string_to_replace, replacement) + return load_config_from_string(yaml_str) + + def load_config_with_replacement( filename: str, string_to_replace: str, replacement: str ) -> Dict: @@ -48,15 +61,29 @@ def load_config_with_replacement( yaml_str: str = load_yaml_as_string(filename).replace( string_to_replace, replacement ) - return yaml.safe_load(yaml_str).get("saas_config", []) + return load_config_from_string(yaml_str) -def load_dataset(filename: str) -> Dict: +def load_datasets(filename: str) -> Dict: + """Loads the datasets in the provided filename""" yaml_file = load_file([filename]) with open(yaml_file, "r", encoding="utf-8") as file: return yaml.safe_load(file).get("dataset", []) +def load_dataset_from_string(string: str) -> Dict: + """Loads the dataset dict from the yaml string""" + return yaml.safe_load(string).get("dataset", [])[0] + + +def replace_dataset_placeholders( + dataset: str, string_to_replace: str, replacement: str +) -> Dict: + """Loads the dataset from the yaml string and replaces any string with the given value""" + yaml_str: str = dataset.replace(string_to_replace, replacement) + return load_dataset_from_string(yaml_str) + + def load_dataset_with_replacement( filename: str, string_to_replace: str, replacement: str ) -> Dict: diff --git a/src/fides/lib/oauth/api/routes/user_endpoints.py b/src/fides/lib/oauth/api/routes/user_endpoints.py index d2953e6d279..31ed5620d65 100644 --- a/src/fides/lib/oauth/api/routes/user_endpoints.py +++ b/src/fides/lib/oauth/api/routes/user_endpoints.py @@ -17,11 +17,7 @@ HTTP_404_NOT_FOUND, ) -from fides.api.ops.api.v1.scope_registry import ( - USER_CREATE, - USER_DELETE, - USER_READ, -) +from fides.api.ops.api.v1.scope_registry import USER_CREATE, USER_DELETE, USER_READ from fides.api.ops.util.oauth_util import verify_oauth_client from fides.core.config import FidesConfig, get_config from fides.lib.exceptions import AuthorizationError diff --git a/tests/fixtures/saas/connection_template_fixtures.py b/tests/fixtures/saas/connection_template_fixtures.py index 4119a6bd2c4..5a1d0983237 100644 --- a/tests/fixtures/saas/connection_template_fixtures.py +++ b/tests/fixtures/saas/connection_template_fixtures.py @@ -13,8 +13,6 @@ ConnectorRegistry, ConnectorTemplate, create_connection_config_from_template_no_save, - load_registry, - registry_file, upsert_dataset_config_from_template, ) from fides.api.ops.util.connection_util import validate_secrets @@ -104,10 +102,9 @@ def instantiate_connector( """ Helper to genericize instantiation of a SaaS connector """ - registry: ConnectorRegistry = load_registry(registry_file) - connector_template: Optional[ConnectorTemplate] = registry.get_connector_template( - connector_type - ) + connector_template: Optional[ + ConnectorTemplate + ] = ConnectorRegistry.get_connector_template(connector_type) template_vals = SaasConnectionTemplateValues( name=key, key=key, diff --git a/data/saas/config/request_override/mailchimp_override_config.yml b/tests/fixtures/saas/test_data/mailchimp_override_config.yml similarity index 92% rename from data/saas/config/request_override/mailchimp_override_config.yml rename to tests/fixtures/saas/test_data/mailchimp_override_config.yml index 00440ac898d..d7fc3559878 100644 --- a/data/saas/config/request_override/mailchimp_override_config.yml +++ b/tests/fixtures/saas/test_data/mailchimp_override_config.yml @@ -1,8 +1,8 @@ saas_config: fides_key: mailchimp_override_connector_example - name: Mailchimp SaaS Override Config + name: Mailchimp type: mailchimp - description: A sample schema representing the Mailchimp connector for Fidesops that includes request function overrides + description: A sample schema representing the Mailchimp for Fides that includes request function overrides version: 0.0.1 connector_params: diff --git a/data/saas/dataset/request_override/mailchimp_override_dataset.yml b/tests/fixtures/saas/test_data/mailchimp_override_dataset.yml similarity index 99% rename from data/saas/dataset/request_override/mailchimp_override_dataset.yml rename to tests/fixtures/saas/test_data/mailchimp_override_dataset.yml index f62ebc36890..425a2c25e3a 100644 --- a/data/saas/dataset/request_override/mailchimp_override_dataset.yml +++ b/tests/fixtures/saas/test_data/mailchimp_override_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: mailchimp_override_connector_example name: Mailchimp Override Dataset - description: A sample dataset representing the Mailchimp connector for Fidesops + description: A sample dataset representing the Mailchimp connector for Fides collections: - name: messages fields: diff --git a/data/saas/config/saas_erasure_order_config.yml b/tests/fixtures/saas/test_data/saas_erasure_order_config.yml similarity index 100% rename from data/saas/config/saas_erasure_order_config.yml rename to tests/fixtures/saas/test_data/saas_erasure_order_config.yml diff --git a/data/saas/dataset/saas_erasure_order_dataset.yml b/tests/fixtures/saas/test_data/saas_erasure_order_dataset.yml similarity index 100% rename from data/saas/dataset/saas_erasure_order_dataset.yml rename to tests/fixtures/saas/test_data/saas_erasure_order_dataset.yml diff --git a/data/saas/config/saas_example_config.yml b/tests/fixtures/saas/test_data/saas_example_config.yml similarity index 99% rename from data/saas/config/saas_example_config.yml rename to tests/fixtures/saas/test_data/saas_example_config.yml index 67172375ca7..daae14db3ee 100644 --- a/data/saas/config/saas_example_config.yml +++ b/tests/fixtures/saas/test_data/saas_example_config.yml @@ -2,7 +2,7 @@ saas_config: fides_key: saas_connector_example name: SaaS Example Config type: custom - description: A sample schema representing a SaaS connector for Fidesops + description: A sample schema representing a SaaS for Fides version: 0.0.1 connector_params: diff --git a/data/saas/dataset/saas_example_dataset.yml b/tests/fixtures/saas/test_data/saas_example_dataset.yml similarity index 99% rename from data/saas/dataset/saas_example_dataset.yml rename to tests/fixtures/saas/test_data/saas_example_dataset.yml index f6d2756d6ed..6301ab69ba4 100644 --- a/data/saas/dataset/saas_example_dataset.yml +++ b/tests/fixtures/saas/test_data/saas_example_dataset.yml @@ -1,7 +1,7 @@ dataset: - fides_key: saas_connector_example name: SaaS Example Dataset - description: A sample dataset representing a SaaS connector for Fidesops + description: A sample dataset representing a SaaS connector for Fides collections: - name: messages fields: @@ -239,4 +239,3 @@ dataset: fields: - name: customer_id data_categories: [system.operations] - diff --git a/data/saas/config/saas_external_example_config.yml b/tests/fixtures/saas/test_data/saas_external_example_config.yml similarity index 85% rename from data/saas/config/saas_external_example_config.yml rename to tests/fixtures/saas/test_data/saas_external_example_config.yml index a5292f8db52..cb9170da34b 100644 --- a/data/saas/config/saas_external_example_config.yml +++ b/tests/fixtures/saas/test_data/saas_external_example_config.yml @@ -2,7 +2,7 @@ saas_config: fides_key: saas_connector_external_example name: SaaS External Example Config type: custom - description: A sample schema representing an external SaaS connector for Fidesops + description: A sample schema representing an external SaaS for Fides version: 0.0.1 connector_params: @@ -32,4 +32,4 @@ saas_config: path: /3.0/customer_id_reference_table/ param_values: - name: email - identity: email \ No newline at end of file + identity: email diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 9cabb01a7d7..e624ec2882d 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -52,22 +52,22 @@ def saas_example_secrets(): @pytest.fixture def saas_example_config() -> Dict: - return load_config("data/saas/config/saas_example_config.yml") + return load_config("tests/fixtures/saas/test_data/saas_example_config.yml") @pytest.fixture def saas_external_example_config() -> Dict: - return load_config("data/saas/config/saas_external_example_config.yml") + return load_config("tests/fixtures/saas/test_data/saas_external_example_config.yml") @pytest.fixture def saas_example_dataset() -> Dict: - return load_dataset("data/saas/dataset/saas_example_dataset.yml")[0] + return load_dataset("tests/fixtures/saas/test_data/saas_example_dataset.yml")[0] @pytest.fixture def saas_ctl_dataset(db: Session) -> Dict: - dataset = load_dataset("data/saas/dataset/saas_example_dataset.yml")[0] + dataset = load_dataset("tests/fixtures/saas/test_data/saas_example_dataset.yml")[0] ctl_dataset = CtlDataset.create_from_dataset_dict(db, dataset) yield ctl_dataset ctl_dataset.delete(db) @@ -75,7 +75,7 @@ def saas_ctl_dataset(db: Session) -> Dict: @pytest.fixture def saas_external_example_dataset() -> Dict: - return load_dataset("data/saas/dataset/saas_example_dataset.yml")[1] + return load_dataset("tests/fixtures/saas/test_data/saas_example_dataset.yml")[1] @pytest.fixture(scope="function") diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index 66e12fa31af..fc81f919347 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -23,7 +23,6 @@ from fides.api.ops.schemas.connection_configuration.connection_config import SystemType from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, - load_registry, registry_file, ) from fides.api.ops.util.saas_util import encode_file_contents @@ -35,10 +34,6 @@ class TestGetConnections: def url(self, oauth_client: ClientDetail, policy) -> str: return V1_URL_PREFIX + CONNECTION_TYPES - @pytest.fixture(scope="session") - def saas_template_registry(self): - return load_registry(registry_file) - def test_get_connection_types_not_authenticated(self, api_client, url): resp = api_client.get(url, headers={}) assert resp.status_code == 401 @@ -55,7 +50,6 @@ def test_get_connection_types( api_client: TestClient, generate_auth_header, url, - saas_template_registry: ConnectorRegistry, ) -> None: auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) resp = api_client.get(url, headers=auth_header) @@ -63,7 +57,7 @@ def test_get_connection_types( assert resp.status_code == 200 assert ( len(data) - == len(ConnectionType) + len(saas_template_registry.connector_types()) - 5 + == len(ConnectionType) + len(ConnectorRegistry.connector_types()) - 5 ) # there are 5 connection types that are not returned by the endpoint assert { @@ -72,15 +66,13 @@ def test_get_connection_types( "human_readable": "PostgreSQL", "encoded_icon": None, } in data - first_saas_type = saas_template_registry.connector_types().pop() - first_saas_template = saas_template_registry.get_connector_template( - first_saas_type - ) + first_saas_type = ConnectorRegistry.connector_types().pop() + first_saas_template = ConnectorRegistry.get_connector_template(first_saas_type) assert { "identifier": first_saas_type, "type": SystemType.saas.value, "human_readable": first_saas_template.human_readable, - "encoded_icon": encode_file_contents(first_saas_template.icon), + "encoded_icon": first_saas_template.icon, } in data assert "saas" not in [item["identifier"] for item in data] @@ -93,7 +85,6 @@ def test_search_connection_types( api_client, generate_auth_header, url, - saas_template_registry: ConnectorRegistry, ): auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) @@ -101,9 +92,9 @@ def test_search_connection_types( expected_saas_templates = [ ( connector_type, - saas_template_registry.get_connector_template(connector_type), + ConnectorRegistry.get_connector_template(connector_type), ) - for connector_type in saas_template_registry.connector_types() + for connector_type in ConnectorRegistry.connector_types() if search.lower() in connector_type.lower() ] expected_saas_data = [ @@ -111,7 +102,7 @@ def test_search_connection_types( "identifier": saas_template[0], "type": SystemType.saas.value, "human_readable": saas_template[1].human_readable, - "encoded_icon": encode_file_contents(saas_template[1].icon), + "encoded_icon": saas_template[1].icon, } for saas_template in expected_saas_templates ] @@ -127,9 +118,9 @@ def test_search_connection_types( expected_saas_templates = [ ( connector_type, - saas_template_registry.get_connector_template(connector_type), + ConnectorRegistry.get_connector_template(connector_type), ) - for connector_type in saas_template_registry.connector_types() + for connector_type in ConnectorRegistry.connector_types() if search.lower() in connector_type.lower() ] expected_saas_data = [ @@ -137,7 +128,7 @@ def test_search_connection_types( "identifier": saas_template[0], "type": SystemType.saas.value, "human_readable": saas_template[1].human_readable, - "encoded_icon": encode_file_contents(saas_template[1].icon), + "encoded_icon": saas_template[1].icon, } for saas_template in expected_saas_templates ] @@ -165,11 +156,7 @@ def test_search_connection_types( assert expected_data in data def test_search_connection_types_case_insensitive( - self, - api_client, - generate_auth_header, - url, - saas_template_registry: ConnectorRegistry, + self, api_client, generate_auth_header, url ): auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) @@ -177,9 +164,9 @@ def test_search_connection_types_case_insensitive( expected_saas_types = [ ( connector_type, - saas_template_registry.get_connector_template(connector_type), + ConnectorRegistry.get_connector_template(connector_type), ) - for connector_type in saas_template_registry.connector_types() + for connector_type in ConnectorRegistry.connector_types() if search.lower() in connector_type.lower() ] expected_saas_data = [ @@ -187,7 +174,7 @@ def test_search_connection_types_case_insensitive( "identifier": saas_template[0], "type": SystemType.saas.value, "human_readable": saas_template[1].human_readable, - "encoded_icon": encode_file_contents(saas_template[1].icon), + "encoded_icon": saas_template[1].icon, } for saas_template in expected_saas_types ] @@ -212,9 +199,9 @@ def test_search_connection_types_case_insensitive( expected_saas_types = [ ( connector_type, - saas_template_registry.get_connector_template(connector_type), + ConnectorRegistry.get_connector_template(connector_type), ) - for connector_type in saas_template_registry.connector_types() + for connector_type in ConnectorRegistry.connector_types() if search.lower() in connector_type.lower() ] expected_saas_data = [ @@ -222,7 +209,7 @@ def test_search_connection_types_case_insensitive( "identifier": saas_template[0], "type": SystemType.saas.value, "human_readable": saas_template[1].human_readable, - "encoded_icon": encode_file_contents(saas_template[1].icon), + "encoded_icon": saas_template[1].icon, } for saas_template in expected_saas_types ] @@ -248,13 +235,7 @@ def test_search_connection_types_case_insensitive( for expected_data in expected_saas_data: assert expected_data in data - def test_search_system_type( - self, - api_client, - generate_auth_header, - url, - saas_template_registry: ConnectorRegistry, - ): + def test_search_system_type(self, api_client, generate_auth_header, url): auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) resp = api_client.get(url + "?system_type=nothing", headers=auth_header) @@ -263,7 +244,7 @@ def test_search_system_type( resp = api_client.get(url + "?system_type=saas", headers=auth_header) assert resp.status_code == 200 data = resp.json()["items"] - assert len(data) == len(saas_template_registry.connector_types()) + assert len(data) == len(ConnectorRegistry.connector_types()) resp = api_client.get(url + "?system_type=database", headers=auth_header) assert resp.status_code == 200 @@ -275,7 +256,6 @@ def test_search_system_type_and_connection_type( api_client, generate_auth_header, url, - saas_template_registry: ConnectorRegistry, ): auth_header = generate_auth_header(scopes=[CONNECTION_TYPE_READ]) @@ -287,7 +267,7 @@ def test_search_system_type_and_connection_type( data = resp.json()["items"] expected_saas_types = [ connector_type - for connector_type in saas_template_registry.connector_types() + for connector_type in ConnectorRegistry.connector_types() if search.lower() in connector_type.lower() ] assert len(data) == len(expected_saas_types) diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py index c3576efc82c..8639aa61b82 100644 --- a/tests/ops/service/connectors/test_connector_registry_service.py +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -3,16 +3,20 @@ from unittest.mock import Mock import yaml -from fideslang.models import DatasetCollection, DatasetField +from fideslang.models import DatasetCollection from fides.api.ops.models.datasetconfig import DatasetConfig +from fides.api.ops.schemas.saas.connector_template import ConnectorTemplate from fides.api.ops.service.connectors.saas.connector_registry_service import ( - ConnectorTemplate, - load_registry, - registry_file, + ConnectorRegistry, update_saas_configs, ) -from fides.api.ops.util.saas_util import load_config, load_dataset, load_yaml_as_string +from fides.api.ops.util.saas_util import ( + encode_file_contents, + load_config_from_string, + load_dataset_from_string, + load_yaml_as_string, +) from fides.core.config.helpers import load_file NEW_CONFIG_DESCRIPTION = "new test config description" @@ -40,19 +44,22 @@ class TestConnectionRegistry: def test_get_connector_template(self): - registry = load_registry(registry_file) - - assert "mailchimp" in registry.connector_types() + assert "mailchimp" in ConnectorRegistry.connector_types() - assert registry.get_connector_template("bad_key") is None - mailchimp_registry = registry.get_connector_template("mailchimp") + assert ConnectorRegistry.get_connector_template("bad_key") is None + mailchimp_template = ConnectorRegistry.get_connector_template("mailchimp") + assert mailchimp_template - assert mailchimp_registry == ConnectorTemplate( - config="data/saas/config/mailchimp_config.yml", - dataset="data/saas/dataset/mailchimp_dataset.yml", - icon="data/saas/icon/mailchimp.svg", - human_readable="Mailchimp", + assert mailchimp_template.config == load_yaml_as_string( + "data/saas/config/mailchimp_config.yml" + ) + assert mailchimp_template.dataset == load_yaml_as_string( + "data/saas/dataset/mailchimp_dataset.yml" ) + assert mailchimp_template.icon == encode_file_contents( + "data/saas/icon/mailchimp.svg" + ) + assert mailchimp_template.human_readable == "Mailchimp" @mock.patch( "fides.api.ops.service.connectors.saas.connector_registry_service.load_dataset_with_replacement" @@ -151,23 +158,22 @@ def update_config( Then, confirm that the instances have been updated as expected, by invoking a plugged-in `validation_function` """ - registry = load_registry(registry_file) - assert "mailchimp" in registry.connector_types() + assert "mailchimp" in ConnectorRegistry.connector_types() - mailchimp_template_config = load_config( - registry.get_connector_template("mailchimp").config + mailchimp_template_config = load_config_from_string( + ConnectorRegistry.get_connector_template("mailchimp").config + ) + mailchimp_template_dataset = load_dataset_from_string( + ConnectorRegistry.get_connector_template("mailchimp").dataset ) - mailchimp_template_dataset = load_dataset( - registry.get_connector_template("mailchimp").dataset - )[0] mailchimp_version = mailchimp_template_config["version"] - sendgrid_template_config = load_config( - registry.get_connector_template("sendgrid").config + sendgrid_template_config = load_config_from_string( + ConnectorRegistry.get_connector_template("sendgrid").config + ) + sendgrid_template_dataset = load_dataset_from_string( + ConnectorRegistry.get_connector_template("sendgrid").dataset ) - sendgrid_template_dataset = load_dataset( - registry.get_connector_template("sendgrid").dataset - )[0] sendgrid_version = sendgrid_template_config["version"] # confirm original version of template works as expected @@ -229,7 +235,7 @@ def update_config( ) # run update "script" - update_saas_configs(registry, db) + update_saas_configs(db) # confirm updates applied successfully secondary_mailchimp_dataset: DatasetConfig = DatasetConfig.filter( diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py new file mode 100644 index 00000000000..1f6d122e7d7 --- /dev/null +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -0,0 +1,32 @@ +from fides.api.ops.service.connectors.saas.connector_registry_service import ( + FileConnectorTemplateLoader, +) +from fides.api.ops.util.saas_util import encode_file_contents, load_yaml_as_string + + +class TestFileConnectorTemplateLoader: + def test_file_connector_template_loader(self): + loader = FileConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + + assert connector_templates + + mailchimp_connector = connector_templates.get("mailchimp") + assert mailchimp_connector + + assert mailchimp_connector.config == load_yaml_as_string( + "data/saas/config/mailchimp_config.yml" + ) + assert mailchimp_connector.dataset == load_yaml_as_string( + "data/saas/dataset/mailchimp_dataset.yml" + ) + assert mailchimp_connector.icon == encode_file_contents( + "data/saas/icon/mailchimp.svg" + ) + assert mailchimp_connector.human_readable == "Mailchimp" + + def test_file_connector_template_loader_connector_not_found(self): + loader = FileConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + + assert connector_templates.get("not_found") is None diff --git a/tests/ops/util/test_connection_type.py b/tests/ops/util/test_connection_type.py index 1fd4155e7de..3d38a796d27 100644 --- a/tests/ops/util/test_connection_type.py +++ b/tests/ops/util/test_connection_type.py @@ -1,21 +1,15 @@ -import pytest - from fides.api.ops.models.connectionconfig import ConnectionType from fides.api.ops.schemas.connection_configuration.connection_config import SystemType from fides.api.ops.service.connectors.saas.connector_registry_service import ( - load_registry, - registry_file, + ConnectorRegistry, ) from fides.api.ops.util.connection_type import get_connection_types -from fides.api.ops.util.saas_util import encode_file_contents def test_get_connection_types(): - saas_template_registry = load_registry(registry_file) data = get_connection_types() assert ( - len(data) - == len(ConnectionType) + len(saas_template_registry.connector_types()) - 5 + len(data) == len(ConnectionType) + len(ConnectorRegistry.connector_types()) - 5 ) # there are 5 connection types that are not returned by the endpoint assert { @@ -24,13 +18,13 @@ def test_get_connection_types(): "human_readable": "PostgreSQL", "encoded_icon": None, } in data - first_saas_type = saas_template_registry.connector_types().pop() - first_saas_template = saas_template_registry.get_connector_template(first_saas_type) + first_saas_type = ConnectorRegistry.connector_types().pop() + first_saas_template = ConnectorRegistry.get_connector_template(first_saas_type) assert { "identifier": first_saas_type, "type": SystemType.saas.value, "human_readable": first_saas_template.human_readable, - "encoded_icon": encode_file_contents(first_saas_template.icon), + "encoded_icon": first_saas_template.icon, } in data assert "saas" not in [item.identifier for item in data] From 117c92bc10d0027137cbf9d2c72c6f585da98b6e Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 20 Mar 2023 11:40:01 -0700 Subject: [PATCH 02/17] Removing unused import --- .../ops/api/v1/endpoints/test_connection_template_endpoints.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py index fc81f919347..fd02afc8e2e 100644 --- a/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_template_endpoints.py @@ -23,9 +23,7 @@ from fides.api.ops.schemas.connection_configuration.connection_config import SystemType from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, - registry_file, ) -from fides.api.ops.util.saas_util import encode_file_contents from fides.lib.models.client import ClientDetail From 69ce2f1683f74b9920d409bdfabb41fc2b77dbd8 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 20 Mar 2023 11:55:28 -0700 Subject: [PATCH 03/17] Restoring docstrings --- .../ops/service/connectors/saas/connector_registry_service.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index 2399f24cf61..7fa9c6ec1a8 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -78,10 +78,14 @@ class ConnectorRegistry: @classmethod def connector_types(cls) -> List[str]: + """List of registered SaaS connector types""" return list(cls._templates.keys()) @classmethod def get_connector_template(cls, connector_type: str) -> Optional[ConnectorTemplate]: + """ + Returns an object containing the various SaaS connector artifacts + """ return cls._templates.get(connector_type) From d7663b4d7a3428dd8ad14bbce4956ce32819cc9b Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 23 Mar 2023 13:01:12 -0700 Subject: [PATCH 04/17] Adding custom connector template model and custom connector template loader --- requirements.txt | 2 + ...51e78b_create_custom_connector_template.py | 63 ++++++ src/fides/api/main.py | 2 +- .../ops/models/custom_connector_template.py | 19 ++ .../saas/connector_registry_service.py | 151 +++++++++++++- .../authentication_strategy_adobe_campaign.py | 4 +- ...thentication_strategy_friendbuy_nextgen.py | 4 +- .../firebase_auth_request_overrides.py | 12 +- .../friendbuy_nextgen_request_overrides.py | 3 - src/fides/api/ops/util/connection_type.py | 10 +- src/fides/api/ops/util/saas_util.py | 6 + src/fides/core/config/security_settings.py | 4 + .../fixtures/saas/test_data/custom/custom.svg | 6 + .../saas/test_data/custom/custom_config.yml | 31 +++ .../saas/test_data/custom/custom_dataset.yml | 9 + .../saas/test_data/custom/custom_functions.py | 73 +++++++ tests/fixtures/saas_example_fixtures.py | 29 ++- .../models/test_custom_connector_template.py | 38 ++++ .../test_connector_registry_service.py | 1 - .../test_connector_template_loaders.py | 190 ++++++++++++++++++ 20 files changed, 632 insertions(+), 25 deletions(-) create mode 100644 src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py create mode 100644 src/fides/api/ops/models/custom_connector_template.py create mode 100644 tests/fixtures/saas/test_data/custom/custom.svg create mode 100644 tests/fixtures/saas/test_data/custom/custom_config.yml create mode 100644 tests/fixtures/saas/test_data/custom/custom_dataset.yml create mode 100644 tests/fixtures/saas/test_data/custom/custom_functions.py create mode 100644 tests/ops/models/test_custom_connector_template.py diff --git a/requirements.txt b/requirements.txt index a7677dcdf6b..1d6d2d27c1f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,4 @@ +AccessControl==6.0 alembic==1.8.1 APScheduler==3.9.1.post1 asyncpg==0.25.0 @@ -34,6 +35,7 @@ PyMySQL==1.0.2 python-jose[cryptography]==3.3.0 pyyaml>=5,<6 redis==3.5.3 +RestrictedPython==6.0.0 rich-click==1.6.1 sendgrid==6.9.7 slowapi==0.1.7 diff --git a/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py b/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py new file mode 100644 index 00000000000..a18beea110b --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py @@ -0,0 +1,63 @@ +"""create custom_connector_template + +Revision ID: 8615ac51e78b +Revises: 50180bbbb959 +Create Date: 2023-03-20 15:11:52.634508 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "8615ac51e78b" +down_revision = "50180bbbb959" +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + "custom_connector_template", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("key", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=False), + sa.Column("config", sa.String(), nullable=False), + sa.Column("dataset", sa.String(), nullable=False), + sa.Column("icon", sa.String(), nullable=True), + sa.Column("functions", sa.String(), nullable=True), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_custom_connector_template_id"), + "custom_connector_template", + ["id"], + unique=False, + ) + op.create_index( + op.f("ix_custom_connector_template_key"), + "custom_connector_template", + ["key"], + unique=True, + ) + + +def downgrade(): + op.drop_index( + op.f("ix_custom_connector_template_key"), table_name="custom_connector_template" + ) + op.drop_index( + op.f("ix_custom_connector_template_id"), table_name="custom_connector_template" + ) + op.drop_table("custom_connector_template") diff --git a/src/fides/api/main.py b/src/fides/api/main.py index bceed0e2b3e..f015c7da8af 100644 --- a/src/fides/api/main.py +++ b/src/fides/api/main.py @@ -273,7 +273,7 @@ async def setup_server() -> None: try: db = get_api_session() update_saas_configs(db) - logger.info("Finished loading saas templates") + logger.info("Finished loading SaaS templates") except Exception as e: logger.error( "Error occurred during SaaS connector template validation: {}", diff --git a/src/fides/api/ops/models/custom_connector_template.py b/src/fides/api/ops/models/custom_connector_template.py new file mode 100644 index 00000000000..cc1669ab9c2 --- /dev/null +++ b/src/fides/api/ops/models/custom_connector_template.py @@ -0,0 +1,19 @@ +from sqlalchemy import Column, String +from sqlalchemy.ext.declarative import declared_attr + +from fides.lib.db.base_class import Base + + +class CustomConnectorTemplate(Base): + """A model representing a custom connector template""" + + @declared_attr + def __tablename__(self) -> str: + return "custom_connector_template" + + key = Column(String, index=True, unique=True, nullable=False) + name = Column(String, index=False, unique=False, nullable=False) + config = Column(String, index=False, unique=False, nullable=False) + dataset = Column(String, index=False, unique=False, nullable=False) + icon = Column(String, index=False, unique=False, nullable=True) + functions = Column(String, index=False, unique=False, nullable=True) diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index 7fa9c6ec1a8..5e1d4087ac0 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -1,19 +1,25 @@ -from __future__ import annotations - import os from abc import ABC, abstractmethod -from typing import Dict, Iterable, List, Optional, Union +from ast import AST, AnnAssign +from operator import getitem +from typing import Any, Dict, Iterable, List, Optional, Tuple, Union +from AccessControl.ZopeGuards import safe_builtins from loguru import logger from packaging.version import LegacyVersion, Version from packaging.version import parse as parse_version +from RestrictedPython import compile_restricted +from RestrictedPython.transformer import RestrictingNodeTransformer from sqlalchemy.orm import Session +from fides.api.ops.api.deps import get_api_session +from fides.api.ops.common_exceptions import FidesopsException from fides.api.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, ConnectionType, ) +from fides.api.ops.models.custom_connector_template import CustomConnectorTemplate from fides.api.ops.models.datasetconfig import DatasetConfig from fides.api.ops.schemas.connection_configuration.connection_config import ( SaasConnectionTemplateValues, @@ -28,6 +34,7 @@ replace_config_placeholders, replace_dataset_placeholders, ) +from fides.core.config import CONFIG class ConnectorTemplateLoader(ABC): @@ -42,6 +49,7 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader): """ def __init__(self) -> None: + logger.info("Loading connectors templates from the data/saas directory") self.templates: Dict[str, ConnectorTemplate] = {} for file in os.listdir("data/saas/config"): if file.endswith(".yml"): @@ -53,6 +61,9 @@ def __init__(self) -> None: try: icon = encode_file_contents(f"data/saas/icon/{connector_type}.svg") except FileNotFoundError: + logger.debug( + f"Could not find the expected {connector_type}.svg in the data/saas/icon/ directory, using default icon" + ) icon = encode_file_contents("data/saas/icon/default.svg") # store connector template for retrieval @@ -72,21 +83,78 @@ def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: return self.templates +class CustomConnectorTemplateLoader(ConnectorTemplateLoader): + """ + Loads custom connector templates defined in the custom_connector_template database table. + """ + + def __init__(self) -> None: + logger.info("Loading connectors templates from the database") + self.templates: Dict[str, ConnectorTemplate] = {} + for template in CustomConnectorTemplate.all(db=get_api_session()): + try: + connector_template = ConnectorTemplate( + config=template.config, + dataset=template.dataset, + icon=template.icon, + human_readable=template.name, + ) + + # register custom functions if available + if template.functions: + if CONFIG.security.allow_custom_connector_functions: + register_custom_functions(template.functions) + logger.info( + f"Loaded functions from the custom connector template '{template.key}'" + ) + else: + raise FidesopsException( + message="The import of connector templates with custom functions is disabled by the 'security.allow_custom_connector_functions' setting" + ) + + # only load the template if there were no issues loading the custom functions + self.templates[template.key] = connector_template + except Exception: + logger.exception("Unable to load {} connector", template.key) + + def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: + return self.templates + + +# pylint: disable=protected-access class ConnectorRegistry: - loader = FileConnectorTemplateLoader() - _templates: Dict[str, ConnectorTemplate] = loader.get_connector_templates() + + _instance = None + _templates: Dict[str, ConnectorTemplate] = {} + + @classmethod + def get_instance(cls) -> "ConnectorRegistry": + if cls._instance is None: + cls._instance = cls() + cls._instance._templates = { + **FileConnectorTemplateLoader().get_connector_templates(), + **CustomConnectorTemplateLoader().get_connector_templates(), + } + return cls._instance @classmethod def connector_types(cls) -> List[str]: """List of registered SaaS connector types""" - return list(cls._templates.keys()) + return list(cls.get_instance()._templates.keys()) @classmethod def get_connector_template(cls, connector_type: str) -> Optional[ConnectorTemplate]: """ Returns an object containing the various SaaS connector artifacts """ - return cls._templates.get(connector_type) + return cls.get_instance()._templates.get(connector_type) + + @classmethod + def register_template( + cls, connector_type: str, template: ConnectorTemplate + ) -> None: + """Used to register new connector templates during runtime""" + cls.get_instance()._templates[connector_type] = template def create_connection_config_from_template_no_save( @@ -222,3 +290,72 @@ def update_saas_instance( connection_config.update_saas_config(db, SaaSConfig(**config_from_template)) upsert_dataset_config_from_template(db, connection_config, template, template_vals) + + +def register_custom_functions(script: str) -> None: + """ + Registers custom functions by executing the given script in a restricted environment. + + The script is compiled and executed with RestrictedPython, which is designed to reduce + the risk of executing untrusted code. It provides a set of safe builtins to prevent + malicious or unintended behavior. + + Args: + script (str): The Python script containing the custom functions to be registered. + + Raises: + SyntaxError: If the script contains a syntax error or uses restricted language features. + Exception: If an exception occurs during the execution of the script. + """ + + restricted_code = compile_restricted( + script, "", "exec", policy=CustomRestrictingNodeTransformer + ) + safe_builtins["__import__"] = custom_guarded_import + safe_builtins["_getitem_"] = getitem + safe_builtins["staticmethod"] = staticmethod + exec( + restricted_code, + { + "__metaclass__": type, + "__name__": "restricted_module", + "__builtins__": safe_builtins, + }, + ) + + +class CustomRestrictingNodeTransformer(RestrictingNodeTransformer): + """ + Custom node transformer class that extends RestrictedPython's RestrictingNodeTransformer + to allow the use of type annotations (AnnAssign) in restricted code. + """ + + def visit_AnnAssign(self, node: AnnAssign) -> AST: + return self.node_contents_visit(node) + + +def custom_guarded_import( + name: str, + globals: Optional[dict] = None, + locals: Optional[dict] = None, + fromlist: Optional[Tuple[str, ...]] = None, + level: int = 0, +) -> Any: + """ + A custom import function that prevents the import of certain potentially unsafe modules. + """ + if name in [ + "os", + "sys", + "subprocess", + "shutil", + "socket", + "importlib", + "tempfile", + "glob", + ]: + # raising SyntaxError to be consistent with exceptions thrown from other guarded functions + raise SyntaxError(f"Import of '{name}' module is not allowed.") + if fromlist is None: + fromlist = () + return __import__(name, globals, locals, fromlist, level) diff --git a/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_adobe_campaign.py b/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_adobe_campaign.py index 50d032fe3cc..60eb4a6c277 100644 --- a/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_adobe_campaign.py +++ b/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_adobe_campaign.py @@ -55,7 +55,7 @@ def add_authentication( access_token: Optional[str] = secrets.get("access_token") expires_at: Optional[int] = secrets.get("expires_at") - if not access_token or self._close_to_expiration(expires_at, connection_config): + if not access_token or self.close_to_expiration(expires_at, connection_config): # generate a JWT token and sign it with the private key jwt_token = encode( { @@ -108,7 +108,7 @@ def add_authentication( return request @staticmethod - def _close_to_expiration( + def close_to_expiration( expires_at: Optional[int], connection_config: ConnectionConfig ) -> bool: """Check if the access_token will expire in the next 10 minutes.""" diff --git a/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_friendbuy_nextgen.py b/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_friendbuy_nextgen.py index 96a081f36e5..7c501a582a5 100644 --- a/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_friendbuy_nextgen.py +++ b/src/fides/api/ops/service/saas_request/override_implementations/authentication_strategy_friendbuy_nextgen.py @@ -48,7 +48,7 @@ def add_authentication( token: Optional[str] = secrets.get("token") expires_at: Optional[int] = secrets.get("expires_at") - if not token or self._close_to_expiration(expires_at, connection_config): + if not token or self.close_to_expiration(expires_at, connection_config): response = post( url=f"https://{domain}/v1/authorization", json={ @@ -87,7 +87,7 @@ def add_authentication( return request @staticmethod - def _close_to_expiration( + def close_to_expiration( expires_at: Optional[int], connection_config: ConnectionConfig ) -> bool: """Check if the access_token will expire in the next 10 minutes.""" diff --git a/src/fides/api/ops/service/saas_request/override_implementations/firebase_auth_request_overrides.py b/src/fides/api/ops/service/saas_request/override_implementations/firebase_auth_request_overrides.py index 2744bc13e85..ca4b194e328 100644 --- a/src/fides/api/ops/service/saas_request/override_implementations/firebase_auth_request_overrides.py +++ b/src/fides/api/ops/service/saas_request/override_implementations/firebase_auth_request_overrides.py @@ -44,7 +44,7 @@ def firebase_auth_user_access( # pylint: disable=R0914 for email in emails: try: user = auth.get_user_by_email(email, app=app) - processed_data.append(_user_record_to_row(user)) + processed_data.append(user_record_to_row(user)) except UserNotFoundError: logger.warning( f"Could not find user with email {Pii(email)} in firebase" @@ -54,7 +54,7 @@ def firebase_auth_user_access( # pylint: disable=R0914 for phone_number in phone_numbers: try: user = auth.get_user_by_phone_number(phone_number, app=app) - processed_data.append(_user_record_to_row(user)) + processed_data.append(user_record_to_row(user)) except UserNotFoundError: logger.warning( f"Could not find user with phone_number {Pii(phone_number)} in firebase" @@ -66,7 +66,7 @@ def firebase_auth_user_access( # pylint: disable=R0914 return processed_data -def _user_record_to_row(user: UserRecord) -> Row: +def user_record_to_row(user: UserRecord) -> Row: """ Translates a Firebase `UserRecord` to a Fides access request result `Row` """ @@ -115,7 +115,7 @@ def firebase_auth_user_update( rows_updated = 0 # each update_params dict correspond to a record that needs to be updated for row_param_values in param_values_per_row: - user: UserRecord = _retrieve_user_record(privacy_request, row_param_values, app) + user: UserRecord = retrieve_user_record(privacy_request, row_param_values, app) masked_fields = row_param_values["masked_object_fields"] email = masked_fields.get("email", user.email) display_name = masked_fields.get("display_name", user.display_name) @@ -158,13 +158,13 @@ def firebase_auth_user_delete( rows_updated = 0 # each update_params dict correspond to a record that needs to be updated for row_param_values in param_values_per_row: - user: UserRecord = _retrieve_user_record(privacy_request, row_param_values, app) + user: UserRecord = retrieve_user_record(privacy_request, row_param_values, app) auth.delete_user(user.uid, app=app) rows_updated += 1 return rows_updated -def _retrieve_user_record( +def retrieve_user_record( privacy_request: PrivacyRequest, row_param_values: Dict[str, Any], app: App ) -> UserRecord: """ diff --git a/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py b/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py index 477b9bd3e3f..34dca6effa2 100644 --- a/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py +++ b/src/fides/api/ops/service/saas_request/override_implementations/friendbuy_nextgen_request_overrides.py @@ -1,4 +1,3 @@ -import logging from typing import Any, Dict, List from fides.api.ops.models.policy import Policy @@ -12,8 +11,6 @@ register, ) -logger = logging.getLogger(__name__) - @register("friendbuy_nextgen_user_delete", [SaaSRequestType.DELETE]) def friendbuy_nextgen_user_delete( diff --git a/src/fides/api/ops/util/connection_type.py b/src/fides/api/ops/util/connection_type.py index 69f3ed33ad2..675bdb69b8d 100644 --- a/src/fides/api/ops/util/connection_type.py +++ b/src/fides/api/ops/util/connection_type.py @@ -16,7 +16,7 @@ from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, ) -from fides.api.ops.util.saas_util import load_config +from fides.api.ops.util.saas_util import load_config_from_string def connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]: @@ -34,7 +34,13 @@ def connection_type_secret_schema(*, connection_type: str) -> dict[str, Any]: if connection_type in [db_type.value for db_type in ConnectionType]: return secrets_schemas[connection_type].schema() - config = SaaSConfig(**load_config(f"data/saas/config/{connection_type}_config.yml")) + connector_template = ConnectorRegistry.get_connector_template(connection_type) + if not connector_template: + raise NoSuchConnectionTypeSecretSchemaError( + f"No SaaS connector type found with name '{connection_type}'." + ) + + config = SaaSConfig(**load_config_from_string(connector_template.config)) schema = SaaSSchemaFactory(config).get_saas_schema().schema() # rearrange the default order of the properties generated by Pydantic diff --git a/src/fides/api/ops/util/saas_util.py b/src/fides/api/ops/util/saas_util.py index 62c2811a870..a714e2a41f6 100644 --- a/src/fides/api/ops/util/saas_util.py +++ b/src/fides/api/ops/util/saas_util.py @@ -46,6 +46,12 @@ def load_config_from_string(string: str) -> Dict: return yaml.safe_load(string).get("saas_config", []) +def load_as_string(filename: str) -> str: + file_path = load_file([filename]) + with open(file_path, "r", encoding="utf-8") as file: + return file.read() + + def replace_config_placeholders( config: str, string_to_replace: str, replacement: str ) -> Dict: diff --git a/src/fides/core/config/security_settings.py b/src/fides/core/config/security_settings.py index e0a12ab52c7..eff09c144e8 100644 --- a/src/fides/core/config/security_settings.py +++ b/src/fides/core/config/security_settings.py @@ -114,6 +114,10 @@ class SecuritySettings(FidesSettings): default=432000, description="The number of seconds that a pre-signed download URL when using S3 storage will be valid. The default is equal to 5 days.", ) + allow_custom_connector_functions: Optional[bool] = Field( + default=True, + description="Enables or disables the ability to import connector templates with custom functions. When enabled, custom functions which will be loaded in a restricted environment to minimize security risks.", + ) @validator("app_encryption_key") @classmethod diff --git a/tests/fixtures/saas/test_data/custom/custom.svg b/tests/fixtures/saas/test_data/custom/custom.svg new file mode 100644 index 00000000000..15a93e49951 --- /dev/null +++ b/tests/fixtures/saas/test_data/custom/custom.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/tests/fixtures/saas/test_data/custom/custom_config.yml b/tests/fixtures/saas/test_data/custom/custom_config.yml new file mode 100644 index 00000000000..e68a3161952 --- /dev/null +++ b/tests/fixtures/saas/test_data/custom/custom_config.yml @@ -0,0 +1,31 @@ +saas_config: + fides_key: custom + name: Custom + type: custom + description: A sample schema representing a custom connector for Fides + version: 0.0.1 + + connector_params: + - name: domain + - name: api_key + + client_config: + protocol: https + host: + authentication: + strategy: bearer + configuration: + password: + + test_request: + method: GET + path: /ping + + endpoints: + - name: user + requests: + read: + request_override: custom_user_access + param_values: + - name: email + identity: email diff --git a/tests/fixtures/saas/test_data/custom/custom_dataset.yml b/tests/fixtures/saas/test_data/custom/custom_dataset.yml new file mode 100644 index 00000000000..9a4eccf781d --- /dev/null +++ b/tests/fixtures/saas/test_data/custom/custom_dataset.yml @@ -0,0 +1,9 @@ +dataset: + - fides_key: custom + name: Custom Dataset + description: A sample dataset representing a custom connector for Fides + collections: + - name: user + fields: + - name: id + data_categories: [user.unique_id] diff --git a/tests/fixtures/saas/test_data/custom/custom_functions.py b/tests/fixtures/saas/test_data/custom/custom_functions.py new file mode 100644 index 00000000000..d709871c1cd --- /dev/null +++ b/tests/fixtures/saas/test_data/custom/custom_functions.py @@ -0,0 +1,73 @@ +from typing import Any, Dict, List, cast + +from requests import PreparedRequest + +from fides.api.ops.graph.traversal import TraversalNode +from fides.api.ops.models.connectionconfig import ConnectionConfig +from fides.api.ops.models.policy import Policy +from fides.api.ops.models.privacy_request import PrivacyRequest +from fides.api.ops.schemas.saas.strategy_configuration import StrategyConfiguration +from fides.api.ops.service.authentication.authentication_strategy import ( + AuthenticationStrategy, +) +from fides.api.ops.service.connectors.saas.authenticated_client import ( + AuthenticatedClient, +) +from fides.api.ops.service.saas_request.saas_request_override_factory import ( + SaaSRequestType, + register, +) +from fides.api.ops.util.collection_util import Row +from fides.api.ops.util.saas_util import assign_placeholders + + +@register("custom_user_access", [SaaSRequestType.READ]) +def custom_user_access( + client: AuthenticatedClient, + node: TraversalNode, + policy: Policy, + privacy_request: PrivacyRequest, + input_data: Dict[str, List[Any]], + secrets: Dict[str, Any], +) -> List[Row]: + return [{"id": 1}] + + +class CustomAuthenticationConfiguration(StrategyConfiguration): + """ + Parameters for custom authentication + """ + + secret_knock: str + secret_handshake: str + + +class CustomAuthenticationStrategy(AuthenticationStrategy): + """ + Adds the secrets to the request + """ + + name = "custom" + configuration_model = CustomAuthenticationConfiguration + + def __init__(self, configuration: CustomAuthenticationConfiguration): + self.secret_knock = configuration.secret_knock + self.secret_handshake = configuration.secret_handshake + + def add_authentication( + self, request: PreparedRequest, connection_config: ConnectionConfig + ) -> PreparedRequest: + """ + Verifies you know the secret knock and secret handshake, so secure + """ + + secrets = cast(Dict, connection_config.secrets) + + secret_knock = assign_placeholders(self.secret_knock, secrets) + secret_handshake = assign_placeholders(self.secret_handshake, secrets) + + assert secret_knock + assert secret_handshake + + request.headers["Authorization"] = f"Bearer {secret_knock + secret_handshake}" + return request diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index e624ec2882d..3861b4e9067 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -28,7 +28,12 @@ StringRewriteMaskingStrategy, ) from fides.api.ops.util.data_category import DataCategory -from fides.api.ops.util.saas_util import load_config +from fides.api.ops.util.saas_util import ( + encode_file_contents, + load_as_string, + load_config, + load_yaml_as_string, +) from fides.lib.models.client import ClientDetail from tests.fixtures.application_fixtures import load_dataset @@ -621,3 +626,25 @@ def erasure_policy_complete_mask( erasure_policy.delete(db) except ObjectDeletedError: pass + + +@pytest.fixture +def custom_config() -> str: + return load_yaml_as_string("tests/fixtures/saas/test_data/custom/custom_config.yml") + + +@pytest.fixture +def custom_dataset() -> str: + return load_yaml_as_string( + "tests/fixtures/saas/test_data/custom/custom_dataset.yml" + ) + + +@pytest.fixture +def custom_icon() -> str: + return encode_file_contents("tests/fixtures/saas/test_data/custom/custom.svg") + + +@pytest.fixture +def custom_functions() -> str: + return load_as_string("tests/fixtures/saas/test_data/custom/custom_functions.py") diff --git a/tests/ops/models/test_custom_connector_template.py b/tests/ops/models/test_custom_connector_template.py new file mode 100644 index 00000000000..51a5cdac6bb --- /dev/null +++ b/tests/ops/models/test_custom_connector_template.py @@ -0,0 +1,38 @@ +from typing import Optional + +import pytest +from sqlalchemy.orm import Session + +from fides.api.ops.models.custom_connector_template import CustomConnectorTemplate +from fides.api.ops.util.saas_util import ( + encode_file_contents, + load_as_string, + load_yaml_as_string, +) + + +class TestCustomConnectorTemplate: + def test_create_custom_connector_template( + self, db: Session, custom_config, custom_dataset, custom_icon, custom_functions + ) -> None: + template = CustomConnectorTemplate( + key="custom", + name="Custom", + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + functions=custom_functions, + ) + template.save(db=db) + + # assert we can retrieve a connector template by key and + # that the values are the same as what we persisted + custom_connector: Optional[ + CustomConnectorTemplate + ] = CustomConnectorTemplate.get_by_key_or_id(db=db, data={"key": "custom"}) + assert custom_connector + assert custom_connector.name == "Custom" + assert custom_connector.config == custom_config + assert custom_connector.dataset == custom_dataset + assert custom_connector.icon == custom_icon + assert custom_connector.functions == custom_functions diff --git a/tests/ops/service/connectors/test_connector_registry_service.py b/tests/ops/service/connectors/test_connector_registry_service.py index 8639aa61b82..a0247a047a2 100644 --- a/tests/ops/service/connectors/test_connector_registry_service.py +++ b/tests/ops/service/connectors/test_connector_registry_service.py @@ -6,7 +6,6 @@ from fideslang.models import DatasetCollection from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.schemas.saas.connector_template import ConnectorTemplate from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, update_saas_configs, diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index 1f6d122e7d7..c8fb92c3bd1 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -1,7 +1,24 @@ +import os + +import pytest + +from fides.api.ops.common_exceptions import NoSuchSaaSRequestOverrideException +from fides.api.ops.models.custom_connector_template import CustomConnectorTemplate +from fides.api.ops.schemas.saas.connector_template import ConnectorTemplate +from fides.api.ops.service.authentication.authentication_strategy import ( + AuthenticationStrategy, +) from fides.api.ops.service.connectors.saas.connector_registry_service import ( + CustomConnectorTemplateLoader, FileConnectorTemplateLoader, + register_custom_functions, +) +from fides.api.ops.service.saas_request.saas_request_override_factory import ( + SaaSRequestOverrideFactory, + SaaSRequestType, ) from fides.api.ops.util.saas_util import encode_file_contents, load_yaml_as_string +from fides.core.config import CONFIG class TestFileConnectorTemplateLoader: @@ -30,3 +47,176 @@ def test_file_connector_template_loader_connector_not_found(self): connector_templates = loader.get_connector_templates() assert connector_templates.get("not_found") is None + + +class TestCustomConnectorTemplateLoader: + def test_file_connector_template_loader_no_templates(self): + loader = CustomConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + assert connector_templates == {} + + def test_file_connector_template_loader_invalid_template( + self, db, custom_dataset, custom_icon, custom_functions + ): + # save custom connector template to the database + custom_template = CustomConnectorTemplate( + key="custom", + name="Custom", + config="custom_config", + dataset=custom_dataset, + icon=custom_icon, + functions=custom_functions, + ) + custom_template.save(db=db) + + # verify the custom functions aren't loaded if the template is invalid + loader = CustomConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + assert connector_templates == {} + + with pytest.raises(NoSuchSaaSRequestOverrideException): + SaaSRequestOverrideFactory.get_override( + "custom_user_access", SaaSRequestType.READ + ) + + # assert the strategy was not registered + authentication_strategies = AuthenticationStrategy.get_strategies() + assert "custom" not in [strategy.name for strategy in authentication_strategies] + + def test_file_connector_template_loader_invalid_functions( + self, db, custom_config, custom_dataset, custom_icon + ): + # save custom connector template to the database + custom_template = CustomConnectorTemplate( + key="custom", + name="Custom", + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + functions="custom_functions", + ) + custom_template.save(db=db) + + # verify nothing is loaded if the custom functions fail to load + loader = CustomConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + assert connector_templates == {} + + def test_file_connector_template_loader_custom_connector_functions_disabled( + self, db, custom_config, custom_dataset, custom_icon, custom_functions + ): + CONFIG.security.allow_custom_connector_functions = False + + # save custom connector template to the database + custom_template = CustomConnectorTemplate( + key="custom", + name="Custom", + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + functions=custom_functions, + ) + custom_template.save(db=db) + + # load custom connector templates from the database + loader = CustomConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + assert connector_templates == {} + + with pytest.raises(NoSuchSaaSRequestOverrideException): + SaaSRequestOverrideFactory.get_override( + "custom_user_access", SaaSRequestType.READ + ) + + # assert the strategy was not registered + authentication_strategies = AuthenticationStrategy.get_strategies() + assert "custom" not in [strategy.name for strategy in authentication_strategies] + + CONFIG.security.allow_custom_connector_functions = True + + def test_file_connector_template_loader_custom_connector_functions_disabled_no_custom_functions( + self, db, custom_config, custom_dataset, custom_icon + ): + """ + A connector template with no custom functions should still be loaded + even if allow_custom_connector_functions is set to false + """ + + CONFIG.security.allow_custom_connector_functions = False + + # save custom connector template to the database + custom_template = CustomConnectorTemplate( + key="custom", + name="Custom", + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + functions=None, + ) + custom_template.save(db=db) + + # load custom connector templates from the database + loader = CustomConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + assert connector_templates == { + "custom": ConnectorTemplate( + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + human_readable="Custom", + ) + } + + CONFIG.security.allow_custom_connector_functions = True + + def test_file_connector_template_loader( + self, db, custom_config, custom_dataset, custom_icon, custom_functions + ): + # save custom connector template to the database + custom_template = CustomConnectorTemplate( + key="custom", + name="Custom", + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + functions=custom_functions, + ) + custom_template.save(db=db) + + # load custom connector templates from the database + loader = CustomConnectorTemplateLoader() + connector_templates = loader.get_connector_templates() + + # verify that the template in the registry is the same as the one in the database + assert connector_templates == { + "custom": ConnectorTemplate( + config=custom_config, + dataset=custom_dataset, + icon=custom_icon, + human_readable="Custom", + ) + } + + # assert the request override was registered + SaaSRequestOverrideFactory.get_override( + "custom_user_access", SaaSRequestType.READ + ) + + # assert the strategy was registered + authentication_strategies = AuthenticationStrategy.get_strategies() + assert "custom" in [strategy.name for strategy in authentication_strategies] + + +class TestRegisterCustomFunctions: + def test_function_loader(self): + """Verify that all override implementations can be loaded by RestrictedPython""" + + overrides_path = ( + "src/fides/api/ops/service/saas_request/override_implementations" + ) + + for filename in os.listdir(overrides_path): + if filename.endswith(".py") and filename != "__init__.py": + file_path = os.path.join(overrides_path, filename) + with open(file_path, "r") as file: + register_custom_functions(file.read()) From 5609182f18989d6be803bd3069989a812226d8be Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 23 Mar 2023 13:53:53 -0700 Subject: [PATCH 05/17] Fixing mypy and pylint issues --- pyproject.toml | 2 ++ .../service/connectors/saas/connector_registry_service.py | 8 +++++--- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index ec8fc11360b..9d8ff4d1e68 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,6 +32,7 @@ disallow_untyped_defs = false [[tool.mypy.overrides]] module = [ + "AccessControl.*", "alembic.*", "apscheduler.*", "boto3.*", @@ -52,6 +53,7 @@ module = [ "plotly.*", "pydash.*", "pymongo.*", + "RestrictedPython.*", "sendgrid.*", "snowflake.*", "sqlalchemy.ext.*", diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index 1be130783c6..f9ee63c7cb5 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -313,6 +313,8 @@ def register_custom_functions(script: str) -> None: safe_builtins["__import__"] = custom_guarded_import safe_builtins["_getitem_"] = getitem safe_builtins["staticmethod"] = staticmethod + + # pylint: disable=exec-used exec( restricted_code, { @@ -335,8 +337,8 @@ def visit_AnnAssign(self, node: AnnAssign) -> AST: def custom_guarded_import( name: str, - globals: Optional[dict] = None, - locals: Optional[dict] = None, + _globals: Optional[dict] = None, + _locals: Optional[dict] = None, fromlist: Optional[Tuple[str, ...]] = None, level: int = 0, ) -> Any: @@ -357,4 +359,4 @@ def custom_guarded_import( raise SyntaxError(f"Import of '{name}' module is not allowed.") if fromlist is None: fromlist = () - return __import__(name, globals, locals, fromlist, level) + return __import__(name, _globals, _locals, fromlist, level) From 8d954b81a91f9cc69f55d7ac3f3675e4ca1030bf Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 23 Mar 2023 15:11:52 -0700 Subject: [PATCH 06/17] Fixing down_revision for migration script --- .../versions/8615ac51e78b_create_custom_connector_template.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py b/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py index a18beea110b..d1c8972873e 100644 --- a/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py +++ b/src/fides/api/ctl/migrations/versions/8615ac51e78b_create_custom_connector_template.py @@ -1,7 +1,7 @@ """create custom_connector_template Revision ID: 8615ac51e78b -Revises: 50180bbbb959 +Revises: a0f219697fa0 Create Date: 2023-03-20 15:11:52.634508 """ @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "8615ac51e78b" -down_revision = "50180bbbb959" +down_revision = "a0f219697fa0" branch_labels = None depends_on = None From ab7b35d5ddfd679845cc425a01e1971154ea3bcc Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 23 Mar 2023 16:54:04 -0700 Subject: [PATCH 07/17] Updating db_dataset to include new table --- .fides/db_dataset.yml | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 9b17830f933..4f05c59cdf9 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -1094,6 +1094,47 @@ dataset: description: 'An optional link to the privacy request if one was created to propagate request preferences' data_categories: [ system.operations ] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: custom_connector_template + description: 'A table used to hold custom connector templates which include a SaaS config, dataset, and an optional icon and functions' + data_categories: [] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + fields: + - name: config + description: 'Fides Generated Description for Column: config' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: created_at + description: 'Fides Generated Description for Column: created_at' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: dataset + description: 'Fides Generated Description for Column: dataset' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: functions + description: 'Fides Generated Description for Column: functions' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: icon + description: 'Fides Generated Description for Column: icon' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: id + description: 'Fides Generated Description for Column: id' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: key + description: 'Fides Generated Description for Column: key' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: name + description: 'Fides Generated Description for Column: name' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: updated_at + description: 'Fides Generated Description for Column: updated_at' + data_categories: [system.operations] + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: datasetconfig data_categories: [] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified From e239b8560efa0135659a53b23262600c01a50268 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Mon, 27 Mar 2023 14:04:52 -0700 Subject: [PATCH 08/17] Updating sample connector name to prevent confusion --- src/fides/core/config/security_settings.py | 2 +- .../fixtures/saas/test_data/custom/custom.svg | 6 - .../saas/test_data/custom/custom_dataset.yml | 9 - .../planet_express/planet_express.svg | 189 ++++++++++++++++++ .../planet_express_config.yml} | 10 +- .../planet_express/planet_express_dataset.yml | 9 + .../planet_express_functions.py} | 14 +- tests/fixtures/saas_example_fixtures.py | 22 +- .../models/test_custom_connector_template.py | 39 ++-- .../test_connector_template_loaders.py | 118 ++++++----- 10 files changed, 312 insertions(+), 106 deletions(-) delete mode 100644 tests/fixtures/saas/test_data/custom/custom.svg delete mode 100644 tests/fixtures/saas/test_data/custom/custom_dataset.yml create mode 100644 tests/fixtures/saas/test_data/planet_express/planet_express.svg rename tests/fixtures/saas/test_data/{custom/custom_config.yml => planet_express/planet_express_config.yml} (66%) create mode 100644 tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml rename tests/fixtures/saas/test_data/{custom/custom_functions.py => planet_express/planet_express_functions.py} (82%) diff --git a/src/fides/core/config/security_settings.py b/src/fides/core/config/security_settings.py index eff09c144e8..2a786380416 100644 --- a/src/fides/core/config/security_settings.py +++ b/src/fides/core/config/security_settings.py @@ -115,7 +115,7 @@ class SecuritySettings(FidesSettings): description="The number of seconds that a pre-signed download URL when using S3 storage will be valid. The default is equal to 5 days.", ) allow_custom_connector_functions: Optional[bool] = Field( - default=True, + default=False, description="Enables or disables the ability to import connector templates with custom functions. When enabled, custom functions which will be loaded in a restricted environment to minimize security risks.", ) diff --git a/tests/fixtures/saas/test_data/custom/custom.svg b/tests/fixtures/saas/test_data/custom/custom.svg deleted file mode 100644 index 15a93e49951..00000000000 --- a/tests/fixtures/saas/test_data/custom/custom.svg +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - diff --git a/tests/fixtures/saas/test_data/custom/custom_dataset.yml b/tests/fixtures/saas/test_data/custom/custom_dataset.yml deleted file mode 100644 index 9a4eccf781d..00000000000 --- a/tests/fixtures/saas/test_data/custom/custom_dataset.yml +++ /dev/null @@ -1,9 +0,0 @@ -dataset: - - fides_key: custom - name: Custom Dataset - description: A sample dataset representing a custom connector for Fides - collections: - - name: user - fields: - - name: id - data_categories: [user.unique_id] diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express.svg b/tests/fixtures/saas/test_data/planet_express/planet_express.svg new file mode 100644 index 00000000000..af02207300f --- /dev/null +++ b/tests/fixtures/saas/test_data/planet_express/planet_express.svg @@ -0,0 +1,189 @@ + + + +image/svg+xml + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/tests/fixtures/saas/test_data/custom/custom_config.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml similarity index 66% rename from tests/fixtures/saas/test_data/custom/custom_config.yml rename to tests/fixtures/saas/test_data/planet_express/planet_express_config.yml index e68a3161952..ac8a4242c04 100644 --- a/tests/fixtures/saas/test_data/custom/custom_config.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml @@ -1,8 +1,8 @@ saas_config: - fides_key: custom - name: Custom - type: custom - description: A sample schema representing a custom connector for Fides + fides_key: planet_express + name: Planet Express + type: planet_express + description: A sample schema representing the Planet Express connector for Fides version: 0.0.1 connector_params: @@ -25,7 +25,7 @@ saas_config: - name: user requests: read: - request_override: custom_user_access + request_override: planet_express_user_access param_values: - name: email identity: email diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml new file mode 100644 index 00000000000..0c6619bbbb5 --- /dev/null +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml @@ -0,0 +1,9 @@ +dataset: + - fides_key: planet_express + name: Planet Express Dataset + description: A sample dataset representing the Planet Express connector for Fides + collections: + - name: user + fields: + - name: id + data_categories: [user.unique_id] diff --git a/tests/fixtures/saas/test_data/custom/custom_functions.py b/tests/fixtures/saas/test_data/planet_express/planet_express_functions.py similarity index 82% rename from tests/fixtures/saas/test_data/custom/custom_functions.py rename to tests/fixtures/saas/test_data/planet_express/planet_express_functions.py index d709871c1cd..78929f2d89f 100644 --- a/tests/fixtures/saas/test_data/custom/custom_functions.py +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_functions.py @@ -21,8 +21,8 @@ from fides.api.ops.util.saas_util import assign_placeholders -@register("custom_user_access", [SaaSRequestType.READ]) -def custom_user_access( +@register("planet_express_user_access", [SaaSRequestType.READ]) +def planet_express_user_access( client: AuthenticatedClient, node: TraversalNode, policy: Policy, @@ -33,7 +33,7 @@ def custom_user_access( return [{"id": 1}] -class CustomAuthenticationConfiguration(StrategyConfiguration): +class PlanetExpressAuthenticationConfiguration(StrategyConfiguration): """ Parameters for custom authentication """ @@ -42,15 +42,15 @@ class CustomAuthenticationConfiguration(StrategyConfiguration): secret_handshake: str -class CustomAuthenticationStrategy(AuthenticationStrategy): +class PlanetExpressAuthenticationStrategy(AuthenticationStrategy): """ Adds the secrets to the request """ - name = "custom" - configuration_model = CustomAuthenticationConfiguration + name = "planet_express" + configuration_model = PlanetExpressAuthenticationConfiguration - def __init__(self, configuration: CustomAuthenticationConfiguration): + def __init__(self, configuration: PlanetExpressAuthenticationConfiguration): self.secret_knock = configuration.secret_knock self.secret_handshake = configuration.secret_handshake diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 3861b4e9067..c689520990d 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -629,22 +629,28 @@ def erasure_policy_complete_mask( @pytest.fixture -def custom_config() -> str: - return load_yaml_as_string("tests/fixtures/saas/test_data/custom/custom_config.yml") +def planet_express_config() -> str: + return load_yaml_as_string( + "tests/fixtures/saas/test_data/planet_express/planet_express_config.yml" + ) @pytest.fixture -def custom_dataset() -> str: +def planet_express_dataset() -> str: return load_yaml_as_string( - "tests/fixtures/saas/test_data/custom/custom_dataset.yml" + "tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml" ) @pytest.fixture -def custom_icon() -> str: - return encode_file_contents("tests/fixtures/saas/test_data/custom/custom.svg") +def planet_express_icon() -> str: + return encode_file_contents( + "tests/fixtures/saas/test_data/planet_express/planet_express.svg" + ) @pytest.fixture -def custom_functions() -> str: - return load_as_string("tests/fixtures/saas/test_data/custom/custom_functions.py") +def planet_express_functions() -> str: + return load_as_string( + "tests/fixtures/saas/test_data/planet_express/planet_express_functions.py" + ) diff --git a/tests/ops/models/test_custom_connector_template.py b/tests/ops/models/test_custom_connector_template.py index 51a5cdac6bb..ef12f751e90 100644 --- a/tests/ops/models/test_custom_connector_template.py +++ b/tests/ops/models/test_custom_connector_template.py @@ -1,27 +1,26 @@ from typing import Optional -import pytest from sqlalchemy.orm import Session from fides.api.ops.models.custom_connector_template import CustomConnectorTemplate -from fides.api.ops.util.saas_util import ( - encode_file_contents, - load_as_string, - load_yaml_as_string, -) class TestCustomConnectorTemplate: def test_create_custom_connector_template( - self, db: Session, custom_config, custom_dataset, custom_icon, custom_functions + self, + db: Session, + planet_express_config, + planet_express_dataset, + planet_express_icon, + planet_express_functions, ) -> None: template = CustomConnectorTemplate( - key="custom", - name="Custom", - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, - functions=custom_functions, + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, ) template.save(db=db) @@ -29,10 +28,12 @@ def test_create_custom_connector_template( # that the values are the same as what we persisted custom_connector: Optional[ CustomConnectorTemplate - ] = CustomConnectorTemplate.get_by_key_or_id(db=db, data={"key": "custom"}) + ] = CustomConnectorTemplate.get_by_key_or_id( + db=db, data={"key": "planet_express"} + ) assert custom_connector - assert custom_connector.name == "Custom" - assert custom_connector.config == custom_config - assert custom_connector.dataset == custom_dataset - assert custom_connector.icon == custom_icon - assert custom_connector.functions == custom_functions + assert custom_connector.name == "Planet Express" + assert custom_connector.config == planet_express_config + assert custom_connector.dataset == planet_express_dataset + assert custom_connector.icon == planet_express_icon + assert custom_connector.functions == planet_express_functions diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index c8fb92c3bd1..c74ca7e343e 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -56,16 +56,16 @@ def test_file_connector_template_loader_no_templates(self): assert connector_templates == {} def test_file_connector_template_loader_invalid_template( - self, db, custom_dataset, custom_icon, custom_functions + self, db, planet_express_dataset, planet_express_icon, planet_express_functions ): # save custom connector template to the database custom_template = CustomConnectorTemplate( - key="custom", - name="Custom", - config="custom_config", - dataset=custom_dataset, - icon=custom_icon, - functions=custom_functions, + key="planet_express", + name="Planet Express", + config="planet_express_config", + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, ) custom_template.save(db=db) @@ -76,24 +76,26 @@ def test_file_connector_template_loader_invalid_template( with pytest.raises(NoSuchSaaSRequestOverrideException): SaaSRequestOverrideFactory.get_override( - "custom_user_access", SaaSRequestType.READ + "planet_express_user_access", SaaSRequestType.READ ) # assert the strategy was not registered authentication_strategies = AuthenticationStrategy.get_strategies() - assert "custom" not in [strategy.name for strategy in authentication_strategies] + assert "planet_express" not in [ + strategy.name for strategy in authentication_strategies + ] def test_file_connector_template_loader_invalid_functions( - self, db, custom_config, custom_dataset, custom_icon + self, db, planet_express_config, planet_express_dataset, planet_express_icon ): # save custom connector template to the database custom_template = CustomConnectorTemplate( - key="custom", - name="Custom", - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, - functions="custom_functions", + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions="planet_express_functions", ) custom_template.save(db=db) @@ -103,18 +105,23 @@ def test_file_connector_template_loader_invalid_functions( assert connector_templates == {} def test_file_connector_template_loader_custom_connector_functions_disabled( - self, db, custom_config, custom_dataset, custom_icon, custom_functions + self, + db, + planet_express_config, + planet_express_dataset, + planet_express_icon, + planet_express_functions, ): CONFIG.security.allow_custom_connector_functions = False # save custom connector template to the database custom_template = CustomConnectorTemplate( - key="custom", - name="Custom", - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, - functions=custom_functions, + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, ) custom_template.save(db=db) @@ -125,17 +132,19 @@ def test_file_connector_template_loader_custom_connector_functions_disabled( with pytest.raises(NoSuchSaaSRequestOverrideException): SaaSRequestOverrideFactory.get_override( - "custom_user_access", SaaSRequestType.READ + "planet_express_user_access", SaaSRequestType.READ ) # assert the strategy was not registered authentication_strategies = AuthenticationStrategy.get_strategies() - assert "custom" not in [strategy.name for strategy in authentication_strategies] + assert "planet_express" not in [ + strategy.name for strategy in authentication_strategies + ] CONFIG.security.allow_custom_connector_functions = True - def test_file_connector_template_loader_custom_connector_functions_disabled_no_custom_functions( - self, db, custom_config, custom_dataset, custom_icon + def test_file_connector_template_loader_custom_connector_functions_disabled_no_planet_express_functions( + self, db, planet_express_config, planet_express_dataset, planet_express_icon ): """ A connector template with no custom functions should still be loaded @@ -146,11 +155,11 @@ def test_file_connector_template_loader_custom_connector_functions_disabled_no_c # save custom connector template to the database custom_template = CustomConnectorTemplate( - key="custom", - name="Custom", - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, functions=None, ) custom_template.save(db=db) @@ -159,27 +168,32 @@ def test_file_connector_template_loader_custom_connector_functions_disabled_no_c loader = CustomConnectorTemplateLoader() connector_templates = loader.get_connector_templates() assert connector_templates == { - "custom": ConnectorTemplate( - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, - human_readable="Custom", + "planet_express": ConnectorTemplate( + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + human_readable="Planet Express", ) } CONFIG.security.allow_custom_connector_functions = True def test_file_connector_template_loader( - self, db, custom_config, custom_dataset, custom_icon, custom_functions + self, + db, + planet_express_config, + planet_express_dataset, + planet_express_icon, + planet_express_functions, ): # save custom connector template to the database custom_template = CustomConnectorTemplate( - key="custom", - name="Custom", - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, - functions=custom_functions, + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, ) custom_template.save(db=db) @@ -189,22 +203,24 @@ def test_file_connector_template_loader( # verify that the template in the registry is the same as the one in the database assert connector_templates == { - "custom": ConnectorTemplate( - config=custom_config, - dataset=custom_dataset, - icon=custom_icon, - human_readable="Custom", + "planet_express": ConnectorTemplate( + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + human_readable="Planet Express", ) } # assert the request override was registered SaaSRequestOverrideFactory.get_override( - "custom_user_access", SaaSRequestType.READ + "planet_express_user_access", SaaSRequestType.READ ) # assert the strategy was registered authentication_strategies = AuthenticationStrategy.get_strategies() - assert "custom" in [strategy.name for strategy in authentication_strategies] + assert "planet_express" in [ + strategy.name for strategy in authentication_strategies + ] class TestRegisterCustomFunctions: From 1ec11d5ebb6a79193249f1571467dcfd0352213e Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Tue, 28 Mar 2023 15:09:34 -0700 Subject: [PATCH 09/17] Adding endpoint for connector template registration --- .../src/types/api/models/ScopeRegistryEnum.ts | 1 + .../ops/api/v1/endpoints/masking_endpoints.py | 2 +- .../api/v1/endpoints/saas_config_endpoints.py | 40 +- src/fides/api/ops/api/v1/scope_registry.py | 10 +- src/fides/api/ops/api/v1/urn_registry.py | 1 + .../ops/schemas/saas/connector_template.py | 17 +- src/fides/api/ops/schemas/saas/saas_config.py | 2 +- .../saas/connector_registry_service.py | 285 +++++++++---- src/fides/api/ops/util/saas_util.py | 16 +- .../planet_express_invalid_config.yml | 27 ++ .../planet_express_invalid_dataset.yml | 9 + tests/fixtures/saas_example_fixtures.py | 14 + .../endpoints/test_saas_config_endpoints.py | 392 +++++++++++++++++- .../test_connector_template_loaders.py | 61 ++- 14 files changed, 748 insertions(+), 129 deletions(-) create mode 100644 tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml create mode 100644 tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml diff --git a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts index eef31bd2bce..eac0e8c5509 100644 --- a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts +++ b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts @@ -21,6 +21,7 @@ export enum ScopeRegistryEnum { CONNECTION_READ = "connection:read", CONNECTION_AUTHORIZE = "connection:authorize", CONNECTION_TYPE_READ = "connection_type:read", + CONNECTOR_TEMPLATE_REGISTER = "connector_template:register", CONSENT_READ = "consent:read", CTL_DATASET_CREATE = "ctl_dataset:create", CTL_DATASET_READ = "ctl_dataset:read", diff --git a/src/fides/api/ops/api/v1/endpoints/masking_endpoints.py b/src/fides/api/ops/api/v1/endpoints/masking_endpoints.py index fc975f43776..5dd4dee4913 100644 --- a/src/fides/api/ops/api/v1/endpoints/masking_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/masking_endpoints.py @@ -4,6 +4,7 @@ from loguru import logger from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND +from fides.api.ops.api.v1.scope_registry import MASKING_EXEC, MASKING_READ from fides.api.ops.api.v1.urn_registry import MASKING, MASKING_STRATEGY, V1_URL_PREFIX from fides.api.ops.common_exceptions import NoSuchStrategyException, ValidationError from fides.api.ops.schemas.masking.masking_api import ( @@ -17,7 +18,6 @@ from fides.api.ops.service.masking.strategy.masking_strategy import MaskingStrategy from fides.api.ops.util.api_router import APIRouter from fides.api.ops.util.oauth_util import verify_oauth_client_prod -from fides.api.ops.api.v1.scope_registry import MASKING_EXEC, MASKING_READ router = APIRouter(tags=["Masking"], prefix=V1_URL_PREFIX) diff --git a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py index 326f394de39..ef2f6b25174 100644 --- a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py @@ -1,6 +1,8 @@ +from io import BytesIO from typing import Optional +from zipfile import BadZipFile, ZipFile -from fastapi import Depends, HTTPException +from fastapi import Depends, HTTPException, UploadFile from fastapi.params import Security from fideslang.validation import FidesKey from loguru import logger @@ -17,6 +19,7 @@ from fides.api.ops.api import deps from fides.api.ops.api.v1.scope_registry import ( CONNECTION_AUTHORIZE, + CONNECTOR_TEMPLATE_REGISTER, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, @@ -25,6 +28,7 @@ from fides.api.ops.api.v1.urn_registry import ( AUTHORIZE, CONNECTION_TYPES, + REGISTER_CONNECTOR_TEMPLATE, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, SAAS_CONNECTOR_FROM_TEMPLATE, @@ -51,6 +55,7 @@ ) from fides.api.ops.service.connectors.saas.connector_registry_service import ( ConnectorRegistry, + CustomConnectorTemplateLoader, create_connection_config_from_template_no_save, upsert_dataset_config_from_template, ) @@ -337,3 +342,36 @@ def instantiate_connection_from_template( return SaasConnectionTemplateResponse( connection=connection_config, dataset=dataset_config.ctl_dataset ) + + +@router.post( + REGISTER_CONNECTOR_TEMPLATE, + dependencies=[Security(verify_oauth_client, scopes=[CONNECTOR_TEMPLATE_REGISTER])], +) +async def register_custom_connector_template( + connector_template: UploadFile, + db: Session = Depends(deps.get_db), +) -> None: + """ + Registers a custom connector template from a zip file uploaded by the user. + The endpoint performs the following steps: + + 1. Validates the uploaded file is a proper zip file. + 2. Uses the CustomConnectorTemplateLoader to validate, register, and save the template to the database. + + If the uploaded file is not a valid zip file or there are any validation errors + when creating the ConnectorTemplates an HTTP 400 status code is returned. + """ + + contents = await connector_template.read() + + try: + zip_file = ZipFile(BytesIO(contents), "r") + except BadZipFile: + raise HTTPException(status_code=400, detail="Invalid zip file") + + try: + CustomConnectorTemplateLoader.save_template(db=db, zip_file=zip_file) + except Exception as exc: + logger.exception("Error loading connector template from zip file.") + raise HTTPException(status_code=400, detail=str(exc)) diff --git a/src/fides/api/ops/api/v1/scope_registry.py b/src/fides/api/ops/api/v1/scope_registry.py index 4b9672f034d..6b56ff52a8f 100644 --- a/src/fides/api/ops/api/v1/scope_registry.py +++ b/src/fides/api/ops/api/v1/scope_registry.py @@ -16,6 +16,7 @@ CONFIG = "config" CONNECTION = "connection" CONNECTION_TYPE = "connection_type" +CONNECTOR_TEMPLATE = "connector_template" CONSENT = "consent" CREATE = "create" CREATE_OR_UPDATE = "create_or_update" @@ -43,6 +44,7 @@ PRIVACY_REQUEST = "privacy-request" PRIVACY_REQUEST_NOTIFICATIONS = "privacy-request-notifications" READ = "read" +REGISTER = "register" REGISTRY = "registry" RESET = "reset" RESUME = "resume" @@ -179,6 +181,7 @@ SAAS_CONFIG_READ = f"{SAAS_CONFIG}:{READ}" SAAS_CONNECTION_INSTANTIATE = f"{CONNECTION}:{INSTANTIATE}" +CONNECTOR_TEMPLATE_REGISTER = f"{CONNECTOR_TEMPLATE}:{REGISTER}" SCOPE_READ = f"{SCOPE}:{READ}" @@ -233,6 +236,7 @@ CONNECTION_READ: "View connections", CONNECTION_AUTHORIZE: "OAuth2 Authorization", CONNECTION_TYPE_READ: "View types of connections", + CONNECTOR_TEMPLATE_REGISTER: "Register a connector template", CONSENT_READ: "Read consent preferences", CTL_DATASET_CREATE: "Create a ctl dataset", CTL_DATASET_READ: "Read ctl datasets", @@ -299,9 +303,9 @@ RULE_CREATE_OR_UPDATE: "Create or update rules", RULE_DELETE: "Remove rules", RULE_READ: "View rules", - SAAS_CONFIG_CREATE_OR_UPDATE: "Create or update SAAS configurations", - SAAS_CONFIG_DELETE: "Remove SAAS configurations", - SAAS_CONFIG_READ: "View SAAS configurations", + SAAS_CONFIG_CREATE_OR_UPDATE: "Create or update SaaS configurations", + SAAS_CONFIG_DELETE: "Remove SaaS configurations", + SAAS_CONFIG_READ: "View SaaS configurations", SAAS_CONNECTION_INSTANTIATE: "", SCOPE_READ: "View authorization scopes", STORAGE_CREATE_OR_UPDATE: "Create or update storage", diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py index d65d9bd83f8..692790012f2 100644 --- a/src/fides/api/ops/api/v1/urn_registry.py +++ b/src/fides/api/ops/api/v1/urn_registry.py @@ -134,6 +134,7 @@ SAAS_CONFIG_VALIDATE = CONNECTION_BY_KEY + "/validate_saas_config" SAAS_CONFIG = CONNECTION_BY_KEY + "/saas_config" SAAS_CONNECTOR_FROM_TEMPLATE = "/connection/instantiate/{saas_connector_type}" +REGISTER_CONNECTOR_TEMPLATE = "/connector_template/register" SYSTEM_CONNECTIONS = "/system/{fides_key}/connection" diff --git a/src/fides/api/ops/schemas/saas/connector_template.py b/src/fides/api/ops/schemas/saas/connector_template.py index e3b4efd0e61..a6cd7f602bf 100644 --- a/src/fides/api/ops/schemas/saas/connector_template.py +++ b/src/fides/api/ops/schemas/saas/connector_template.py @@ -1,28 +1,35 @@ -import yaml +from typing import Optional + from fideslang.models import Dataset from pydantic import BaseModel, validator from fides.api.ops.schemas.saas.saas_config import SaaSConfig +from fides.api.ops.util.saas_util import ( + load_config_from_string, + load_dataset_from_string, +) class ConnectorTemplate(BaseModel): """ - A collection of artifacts that make up a complete SaaS connector (SaaS config, dataset, etc.) + A collection of artifacts that make up a complete + SaaS connector (SaaS config, dataset, icon, functions, etc.) """ config: str dataset: str - icon: str + icon: Optional[str] + functions: Optional[str] human_readable: str @validator("config") def validate_config(cls, config: str) -> str: """Validates the config at the given path""" - SaaSConfig(**yaml.safe_load(config).get("saas_config")) + SaaSConfig(**load_config_from_string(config)) return config @validator("dataset") def validate_dataset(cls, dataset: str) -> str: """Validates the dataset at the given path""" - Dataset(**yaml.safe_load(dataset).get("dataset")[0]) + Dataset(**load_dataset_from_string(dataset)) return dataset diff --git a/src/fides/api/ops/schemas/saas/saas_config.py b/src/fides/api/ops/schemas/saas/saas_config.py index 05b36fd8934..fada9c9c508 100644 --- a/src/fides/api/ops/schemas/saas/saas_config.py +++ b/src/fides/api/ops/schemas/saas/saas_config.py @@ -316,7 +316,7 @@ class ExternalDatasetReference(BaseModel): class SaaSConfigBase(BaseModel): """ - Used to store base info for a saas config + Used to store base info for a SaaS config """ fides_key: FidesKey diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index f6a430ea5bc..ca2df8d3980 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -1,8 +1,10 @@ +# pylint: disable=protected-access import os from abc import ABC, abstractmethod from ast import AST, AnnAssign from operator import getitem from typing import Any, Dict, Iterable, List, Optional, Tuple +from zipfile import ZipFile from AccessControl.ZopeGuards import safe_builtins from loguru import logger @@ -13,7 +15,7 @@ from sqlalchemy.orm import Session from fides.api.ops.api.deps import get_api_session -from fides.api.ops.common_exceptions import FidesopsException +from fides.api.ops.common_exceptions import FidesopsException, ValidationError from fides.api.ops.models.connectionconfig import ( AccessLevel, ConnectionConfig, @@ -38,8 +40,9 @@ class ConnectorTemplateLoader(ABC): + @classmethod @abstractmethod - def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: + def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: """Returns a map of connection templates""" @@ -48,39 +51,49 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader): Loads SaaS connector templates from the data/saas directory. """ - def __init__(self) -> None: - logger.info("Loading connectors templates from the data/saas directory") - self.templates: Dict[str, ConnectorTemplate] = {} - for file in os.listdir("data/saas/config"): - if file.endswith(".yml"): - config_file = os.path.join("data/saas/config", file) - config_dict = load_config(config_file) - connector_type = config_dict["type"] - human_readable = config_dict["name"] - - try: - icon = encode_file_contents(f"data/saas/icon/{connector_type}.svg") - except FileNotFoundError: - logger.debug( - f"Could not find the expected {connector_type}.svg in the data/saas/icon/ directory, using default icon" - ) - icon = encode_file_contents("data/saas/icon/default.svg") + _instance = None + _templates: Dict[str, ConnectorTemplate] = {} - # store connector template for retrieval - try: - self.templates[connector_type] = ConnectorTemplate( - config=load_yaml_as_string(config_file), - dataset=load_yaml_as_string( - f"data/saas/dataset/{connector_type}_dataset.yml" - ), - icon=icon, - human_readable=human_readable, - ) - except Exception: - logger.exception("Unable to load {} connector", connector_type) + @classmethod + def get_instance(cls) -> "FileConnectorTemplateLoader": + if cls._instance is None: + logger.info("Loading connectors templates from the data/saas directory") + cls._instance = cls() + for file in os.listdir("data/saas/config"): + if file.endswith(".yml"): + config_file = os.path.join("data/saas/config", file) + config_dict = load_config(config_file) + connector_type = config_dict["type"] + human_readable = config_dict["name"] + + try: + icon = encode_file_contents( + f"data/saas/icon/{connector_type}.svg" + ) + except FileNotFoundError: + logger.debug( + f"Could not find the expected {connector_type}.svg in the data/saas/icon/ directory, using default icon" + ) + icon = encode_file_contents("data/saas/icon/default.svg") + + # store connector template for retrieval + try: + cls._templates[connector_type] = ConnectorTemplate( + config=load_yaml_as_string(config_file), + dataset=load_yaml_as_string( + f"data/saas/dataset/{connector_type}_dataset.yml" + ), + icon=icon, + functions=None, + human_readable=human_readable, + ) + except Exception: + logger.exception("Unable to load {} connector", connector_type) + return cls._instance - def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: - return self.templates + @classmethod + def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: + return cls.get_instance()._templates class CustomConnectorTemplateLoader(ConnectorTemplateLoader): @@ -88,72 +101,154 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader): Loads custom connector templates defined in the custom_connector_template database table. """ - def __init__(self) -> None: - logger.info("Loading connectors templates from the database") - self.templates: Dict[str, ConnectorTemplate] = {} - for template in CustomConnectorTemplate.all(db=get_api_session()): - try: - connector_template = ConnectorTemplate( - config=template.config, - dataset=template.dataset, - icon=template.icon, - human_readable=template.name, - ) + _instance = None + _templates: Dict[str, ConnectorTemplate] = {} - # register custom functions if available - if template.functions: - if CONFIG.security.allow_custom_connector_functions: - register_custom_functions(template.functions) - logger.info( - f"Loaded functions from the custom connector template '{template.key}'" - ) - else: - raise FidesopsException( - message="The import of connector templates with custom functions is disabled by the 'security.allow_custom_connector_functions' setting" - ) + @classmethod + def get_instance(cls) -> "CustomConnectorTemplateLoader": + if cls._instance is None: + logger.info("Loading connectors templates from the database") + cls._instance = cls() + for template in CustomConnectorTemplate.all(db=get_api_session()): + try: + cls._register_template(template) + except Exception: + logger.exception("Unable to load {} connector", template.key) + return cls._instance - # only load the template if there were no issues loading the custom functions - self.templates[template.key] = connector_template - except Exception: - logger.exception("Unable to load {} connector", template.key) + @classmethod + def _register_template( + cls, + template: CustomConnectorTemplate, + ) -> None: + """ + Registers a custom connector template by converting it to a ConnectorTemplate, + registering any custom functions, and adding it to the loader's template dictionary. + """ + connector_template = ConnectorTemplate( + config=template.config, + dataset=template.dataset, + icon=template.icon, + functions=template.functions, + human_readable=template.name, + ) - def get_connector_templates(self) -> Dict[str, ConnectorTemplate]: - return self.templates + # register custom functions if available + if template.functions: + register_custom_functions(template.functions) + logger.info( + f"Loaded functions from the custom connector template '{template.key}'" + ) + + # register the template in the loader's template dictionary + cls._instance._templates[template.key] = connector_template # type: ignore + + @classmethod + def save_template(cls, db: Session, zip_file: ZipFile) -> None: + """ + Extracts and validates the contents of a zip file containing a + custom connector template, registers the template, and saves it to the database. + """ + + config_contents = None + dataset_contents = None + icon_contents = None + function_contents = None + + for info in zip_file.infolist(): + file_contents = zip_file.read(info).decode() + if info.filename.endswith("config.yml"): + if not config_contents: + config_contents = file_contents + else: + raise ValidationError( + "Multiple files ending with config.yml found, only one is allowed." + ) + elif info.filename.endswith("dataset.yml"): + if not dataset_contents: + dataset_contents = file_contents + else: + raise ValidationError( + "Multiple files ending with dataset.yml found, only one is allowed." + ) + elif info.filename.endswith(".svg"): + if not icon_contents: + icon_contents = file_contents + else: + raise ValidationError( + "Multiple svg files found, only one is allowed." + ) + elif info.filename.endswith(".py"): + if not function_contents: + function_contents = file_contents + else: + raise ValidationError( + "Multiple Python (.py) files found, only one is allowed." + ) + + if not config_contents: + raise ValidationError("Zip file does not contain a config.yml file.") + + if not dataset_contents: + raise ValidationError("Zip file does not contain a dataset.yml file.") + + # extract connector_type and human_readable values from the SaaS config + config = load_config_from_string(config_contents) + connector_type = config["type"] + human_readable = config["name"] + + custom_connector_template = CustomConnectorTemplate( + key=connector_type, + name=human_readable, + config=config_contents, + dataset=dataset_contents, + icon=icon_contents, + functions=function_contents, + ) + + # attempt to register the template, raises an exception if validation fails + cls._register_template(custom_connector_template) + + # save the custom connector to the database if it passed validation + custom_connector_template.save(db=db) + + @classmethod + def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: + return cls.get_instance()._templates -# pylint: disable=protected-access class ConnectorRegistry: _instance = None - _templates: Dict[str, ConnectorTemplate] = {} @classmethod def get_instance(cls) -> "ConnectorRegistry": if cls._instance is None: cls._instance = cls() - cls._instance._templates = { - **FileConnectorTemplateLoader().get_connector_templates(), - **CustomConnectorTemplateLoader().get_connector_templates(), - } return cls._instance + @classmethod + def _get_combined_templates(cls) -> Dict[str, ConnectorTemplate]: + """ + Returns a combined map of connector templates from all registered loaders. + The resulting map is an aggregation of templates from the file loader and the custom loader, + with custom loader templates taking precedence in case of conflicts. + """ + return { + **FileConnectorTemplateLoader.get_connector_templates(), # type: ignore + **CustomConnectorTemplateLoader.get_connector_templates(), # type: ignore + } + @classmethod def connector_types(cls) -> List[str]: """List of registered SaaS connector types""" - return list(cls.get_instance()._templates.keys()) + return list(cls._get_combined_templates().keys()) @classmethod def get_connector_template(cls, connector_type: str) -> Optional[ConnectorTemplate]: """ Returns an object containing the various SaaS connector artifacts """ - return cls.get_instance()._templates.get(connector_type) - - @classmethod - def register_template( - cls, connector_type: str, template: ConnectorTemplate - ) -> None: - """Used to register new connector templates during runtime""" - cls.get_instance()._templates[connector_type] = template + return cls._get_combined_templates().get(connector_type) def create_connection_config_from_template_no_save( @@ -300,26 +395,32 @@ def register_custom_functions(script: str) -> None: script (str): The Python script containing the custom functions to be registered. Raises: + FidesopsException: If allow_custom_connector_functions is disabled. SyntaxError: If the script contains a syntax error or uses restricted language features. Exception: If an exception occurs during the execution of the script. """ - restricted_code = compile_restricted( - script, "", "exec", policy=CustomRestrictingNodeTransformer - ) - safe_builtins["__import__"] = custom_guarded_import - safe_builtins["_getitem_"] = getitem - safe_builtins["staticmethod"] = staticmethod - - # pylint: disable=exec-used - exec( - restricted_code, - { - "__metaclass__": type, - "__name__": "restricted_module", - "__builtins__": safe_builtins, - }, - ) + if CONFIG.security.allow_custom_connector_functions: + restricted_code = compile_restricted( + script, "", "exec", policy=CustomRestrictingNodeTransformer + ) + safe_builtins["__import__"] = custom_guarded_import + safe_builtins["_getitem_"] = getitem + safe_builtins["staticmethod"] = staticmethod + + # pylint: disable=exec-used + exec( + restricted_code, + { + "__metaclass__": type, + "__name__": "restricted_module", + "__builtins__": safe_builtins, + }, + ) + else: + raise FidesopsException( + message="The import of connector templates with custom functions is disabled by the 'security.allow_custom_connector_functions' setting." + ) class CustomRestrictingNodeTransformer(RestrictingNodeTransformer): diff --git a/src/fides/api/ops/util/saas_util.py b/src/fides/api/ops/util/saas_util.py index a714e2a41f6..086eb9a2d7b 100644 --- a/src/fides/api/ops/util/saas_util.py +++ b/src/fides/api/ops/util/saas_util.py @@ -9,7 +9,7 @@ import yaml from multidimensional_urlencode import urlencode as multidimensional_urlencode -from fides.api.ops.common_exceptions import FidesopsException +from fides.api.ops.common_exceptions import FidesopsException, ValidationError from fides.api.ops.graph.config import ( Collection, CollectionAddress, @@ -43,7 +43,12 @@ def load_config(filename: str) -> Dict: def load_config_from_string(string: str) -> Dict: """Loads the SaaS config dict from the yaml string""" - return yaml.safe_load(string).get("saas_config", []) + try: + return yaml.safe_load(string)["saas_config"] + except: + raise ValidationError( + "Config contents do not contain a 'saas_config' key at the root level." + ) def load_as_string(filename: str) -> str: @@ -79,7 +84,12 @@ def load_datasets(filename: str) -> Dict: def load_dataset_from_string(string: str) -> Dict: """Loads the dataset dict from the yaml string""" - return yaml.safe_load(string).get("dataset", [])[0] + try: + return yaml.safe_load(string)["dataset"][0] + except: + raise ValidationError( + "Dataset contents do not contain a 'dataset' key at the root level." + ) def replace_dataset_placeholders( diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml new file mode 100644 index 00000000000..cb6b39c65f5 --- /dev/null +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml @@ -0,0 +1,27 @@ +saas_config: + fides_key: planet_express + name: Planet Express + type: planet_express + description: A sample schema representing the Planet Express connector for Fides + version: 0.0.1 + + connector_params: + - name: domain + - name: api_key + + client_config: + protocol: https + host: + authentication: + strategy: bearer + configuration: + password: + + endpoints: + - name: user + requests: + read: + request_override: planet_express_user_access + param_values: + - name: email + identity: email diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml new file mode 100644 index 00000000000..c4b0556894b --- /dev/null +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml @@ -0,0 +1,9 @@ +dataset: + - fides_key: planet_express + name: Planet Express Dataset + description: A sample dataset representing the Planet Express connector for Fides + collections: + - names: user + fields: + - name: id + data_categories: [user.unique_id] diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index c689520990d..229f56c1ddd 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -635,6 +635,13 @@ def planet_express_config() -> str: ) +@pytest.fixture +def planet_express_invalid_config() -> str: + return load_yaml_as_string( + "tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml" + ) + + @pytest.fixture def planet_express_dataset() -> str: return load_yaml_as_string( @@ -642,6 +649,13 @@ def planet_express_dataset() -> str: ) +@pytest.fixture +def planet_express_invalid_dataset() -> str: + return load_yaml_as_string( + "tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml" + ) + + @pytest.fixture def planet_express_icon() -> str: return encode_file_contents( diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index 06a4b785ca6..e16e2ece0d6 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -1,7 +1,9 @@ import json -from typing import Optional +from io import BytesIO +from typing import Dict, Optional from unittest import mock from unittest.mock import Mock +from zipfile import ZipFile import pytest from sqlalchemy.orm import Session @@ -10,12 +12,14 @@ from fides.api.ops.api.v1.scope_registry import ( CLIENT_READ, CONNECTION_AUTHORIZE, + CONNECTOR_TEMPLATE_REGISTER, SAAS_CONFIG_CREATE_OR_UPDATE, SAAS_CONFIG_DELETE, SAAS_CONFIG_READ, ) from fides.api.ops.api.v1.urn_registry import ( AUTHORIZE, + REGISTER_CONNECTOR_TEMPLATE, SAAS_CONFIG, SAAS_CONFIG_VALIDATE, V1_URL_PREFIX, @@ -25,6 +29,7 @@ ConnectionConfig, ConnectionType, ) +from fides.core.config import CONFIG from tests.ops.api.v1.endpoints.test_dataset_endpoints import _reject_key @@ -452,3 +457,388 @@ def test_get_authorize_url( response = api_client.get(authorize_url, headers=auth_header) response.raise_for_status() assert response.text == f'"{authorization_url}"' + + +class TestRegisterConnectorTemplate: + def create_zip_file(self, file_data: Dict[str, str]) -> BytesIO: + """ + Create a zip file in memory with the given files. + + Args: + files (Dict[str, str]): A mapping of filenames to their contents + + Returns: + io.BytesIO: An in-memory zip file with the specified files. + """ + zip_buffer = BytesIO() + + with ZipFile(zip_buffer, "w") as zip_file: + for filename, file_content in file_data.items(): + zip_file.writestr(filename, file_content) + + # resetting the file position pointer to the beginning of the stream + # so the tests can read the zip file from the beginning + zip_buffer.seek(0) + return zip_buffer + + @pytest.fixture + def register_connector_template_url(self) -> str: + return V1_URL_PREFIX + REGISTER_CONNECTOR_TEMPLATE + + @pytest.fixture + def complete_connector_template( + self, + planet_express_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_missing_config( + self, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_wrong_contents_config( + self, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": "planet_express_config", + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_invalid_config( + self, + planet_express_invalid_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_invalid_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_missing_dataset( + self, + planet_express_config, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_wrong_contents_dataset( + self, + planet_express_config, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": "planet_express_dataset", + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_invalid_dataset( + self, + planet_express_config, + planet_express_invalid_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_invalid_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_no_functions( + self, + planet_express_config, + planet_express_dataset, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_no_icon( + self, + planet_express_config, + planet_express_dataset, + planet_express_functions, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + } + ) + + @pytest.fixture + def connector_template_duplicate_configs( + self, + planet_express_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "1_config.yml": planet_express_config, + "2_config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_duplicate_datasets( + self, + planet_express_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "1_dataset.yml": planet_express_dataset, + "2_dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_duplicate_functions( + self, + planet_express_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "1_functions.py": planet_express_functions, + "2_functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_duplicate_icons( + self, + planet_express_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ): + return self.create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "1_icon.svg": planet_express_icon, + "2_icon.svg": planet_express_icon, + } + ) + + def test_register_connector_template_wrong_scope( + self, + api_client: TestClient, + register_connector_template_url, + generate_auth_header, + complete_connector_template, + ): + CONFIG.security.allow_custom_connector_functions = True + auth_header = generate_auth_header(scopes=[CLIENT_READ]) + response = api_client.post( + register_connector_template_url, + headers=auth_header, + files={"connector_template": complete_connector_template}, + ) + assert response.status_code == 403 + + @pytest.mark.parametrize( + "zip_file, status_code, details", + [ + ("complete_connector_template", 200, None), + ( + "connector_template_missing_config", + 400, + {"detail": "Zip file does not contain a config.yml file."}, + ), + ( + "connector_template_wrong_contents_config", + 400, + { + "detail": "Config contents do not contain a 'saas_config' key at the root level." + }, + ), + ( + "connector_template_invalid_config", + 400, + { + "detail": "1 validation error for ConnectorTemplate\nconfig -> test_request\n field required (type=value_error.missing)" + }, + ), + ( + "connector_template_missing_dataset", + 400, + {"detail": "Zip file does not contain a dataset.yml file."}, + ), + ( + "connector_template_wrong_contents_dataset", + 400, + { + "detail": "Dataset contents do not contain a 'dataset' key at the root level." + }, + ), + ( + "connector_template_invalid_dataset", + 400, + { + "detail": "1 validation error for ConnectorTemplate\ndataset -> collections -> 0 -> name\n field required (type=value_error.missing)" + }, + ), + ("connector_template_no_functions", 200, None), + ("connector_template_no_icon", 200, None), + ( + "connector_template_duplicate_configs", + 400, + { + "detail": "Multiple files ending with config.yml found, only one is allowed." + }, + ), + ( + "connector_template_duplicate_datasets", + 400, + { + "detail": "Multiple files ending with dataset.yml found, only one is allowed." + }, + ), + ( + "connector_template_duplicate_functions", + 400, + {"detail": "Multiple Python (.py) files found, only one is allowed."}, + ), + ( + "connector_template_duplicate_icons", + 400, + {"detail": "Multiple svg files found, only one is allowed."}, + ), + ], + ) + def test_register_connector_template_allow_custom_connector_functions( + self, + api_client: TestClient, + register_connector_template_url, + generate_auth_header, + zip_file, + status_code, + details, + request, + ): + CONFIG.security.allow_custom_connector_functions = True + auth_header = generate_auth_header(scopes=[CONNECTOR_TEMPLATE_REGISTER]) + response = api_client.post( + register_connector_template_url, + headers=auth_header, + files={"connector_template": request.getfixturevalue(zip_file)}, + ) + assert response.status_code == status_code + assert response.json() == details + + @pytest.mark.parametrize( + "zip_file, status_code, details", + [ + ( + "complete_connector_template", + 400, + { + "detail": "The import of connector templates with custom functions is disabled by the 'security.allow_custom_connector_functions' setting." + }, + ), + ( + "connector_template_no_functions", + 200, + None, + ), + ], + ) + def test_register_connector_template_disallow_custom_connector_functions( + self, + api_client: TestClient, + register_connector_template_url, + generate_auth_header, + zip_file, + status_code, + details, + request, + ): + CONFIG.security.allow_custom_connector_functions = False + auth_header = generate_auth_header(scopes=[CONNECTOR_TEMPLATE_REGISTER]) + response = api_client.post( + register_connector_template_url, + headers=auth_header, + files={"connector_template": request.getfixturevalue(zip_file)}, + ) + assert response.status_code == status_code + assert response.json() == details diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index c74ca7e343e..03db996abf4 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -43,21 +43,34 @@ def test_file_connector_template_loader(self): assert mailchimp_connector.human_readable == "Mailchimp" def test_file_connector_template_loader_connector_not_found(self): - loader = FileConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + connector_templates = FileConnectorTemplateLoader.get_connector_templates() assert connector_templates.get("not_found") is None class TestCustomConnectorTemplateLoader: + @pytest.fixture(autouse=True) + def reset_custom_connector_template_loader(self): + """ + Resets the CustomConnectorTemplateLoader singleton instance before each test. + """ + CustomConnectorTemplateLoader._instance = None + def test_file_connector_template_loader_no_templates(self): - loader = CustomConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + CONFIG.security.allow_custom_connector_functions = True + + connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} def test_file_connector_template_loader_invalid_template( - self, db, planet_express_dataset, planet_express_icon, planet_express_functions + self, + db, + planet_express_dataset, + planet_express_icon, + planet_express_functions, ): + CONFIG.security.allow_custom_connector_functions = True + # save custom connector template to the database custom_template = CustomConnectorTemplate( key="planet_express", @@ -70,8 +83,7 @@ def test_file_connector_template_loader_invalid_template( custom_template.save(db=db) # verify the custom functions aren't loaded if the template is invalid - loader = CustomConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} with pytest.raises(NoSuchSaaSRequestOverrideException): @@ -86,8 +98,14 @@ def test_file_connector_template_loader_invalid_template( ] def test_file_connector_template_loader_invalid_functions( - self, db, planet_express_config, planet_express_dataset, planet_express_icon + self, + db, + planet_express_config, + planet_express_dataset, + planet_express_icon, ): + CONFIG.security.allow_custom_connector_functions = True + # save custom connector template to the database custom_template = CustomConnectorTemplate( key="planet_express", @@ -100,8 +118,7 @@ def test_file_connector_template_loader_invalid_functions( custom_template.save(db=db) # verify nothing is loaded if the custom functions fail to load - loader = CustomConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} def test_file_connector_template_loader_custom_connector_functions_disabled( @@ -126,8 +143,7 @@ def test_file_connector_template_loader_custom_connector_functions_disabled( custom_template.save(db=db) # load custom connector templates from the database - loader = CustomConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} with pytest.raises(NoSuchSaaSRequestOverrideException): @@ -141,10 +157,12 @@ def test_file_connector_template_loader_custom_connector_functions_disabled( strategy.name for strategy in authentication_strategies ] - CONFIG.security.allow_custom_connector_functions = True - - def test_file_connector_template_loader_custom_connector_functions_disabled_no_planet_express_functions( - self, db, planet_express_config, planet_express_dataset, planet_express_icon + def test_file_connector_template_loader_custom_connector_functions_disabled_custom_functions( + self, + db, + planet_express_config, + planet_express_dataset, + planet_express_icon, ): """ A connector template with no custom functions should still be loaded @@ -165,8 +183,7 @@ def test_file_connector_template_loader_custom_connector_functions_disabled_no_p custom_template.save(db=db) # load custom connector templates from the database - loader = CustomConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == { "planet_express": ConnectorTemplate( config=planet_express_config, @@ -176,8 +193,6 @@ def test_file_connector_template_loader_custom_connector_functions_disabled_no_p ) } - CONFIG.security.allow_custom_connector_functions = True - def test_file_connector_template_loader( self, db, @@ -186,6 +201,8 @@ def test_file_connector_template_loader( planet_express_icon, planet_express_functions, ): + CONFIG.security.allow_custom_connector_functions = True + # save custom connector template to the database custom_template = CustomConnectorTemplate( key="planet_express", @@ -198,8 +215,7 @@ def test_file_connector_template_loader( custom_template.save(db=db) # load custom connector templates from the database - loader = CustomConnectorTemplateLoader() - connector_templates = loader.get_connector_templates() + connector_templates = CustomConnectorTemplateLoader.get_connector_templates() # verify that the template in the registry is the same as the one in the database assert connector_templates == { @@ -207,6 +223,7 @@ def test_file_connector_template_loader( config=planet_express_config, dataset=planet_express_dataset, icon=planet_express_icon, + functions=planet_express_functions, human_readable="Planet Express", ) } From fcabb3c05a1c48b968ca34cd8f71190270997df4 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 29 Mar 2023 13:56:22 -0700 Subject: [PATCH 10/17] Fixing misc issues --- docker-compose.yml | 1 + .../api/v1/endpoints/saas_config_endpoints.py | 7 +- .../saas/connector_registry_service.py | 29 ++- .../planet_express/planet_express.svg | 218 +++--------------- .../planet_express/planet_express_config.yml | 1 + .../planet_express_invalid_config.yml | 1 + .../endpoints/test_saas_config_endpoints.py | 93 ++++---- .../test_connector_template_loaders.py | 74 +++++- tests/ops/test_helpers/saas_test_utils.py | 24 ++ 9 files changed, 207 insertions(+), 241 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7d3e0523072..22869abb198 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -27,6 +27,7 @@ services: FIDES__DEV_MODE: "True" FIDES__REDIS__ENABLED: "True" FIDES__USER__ANALYTICS_OPT_OUT: "True" + FIDES__SECURITY__ALLOW_CUSTOM_CONNECTOR_FUNCTIONS: "True" VAULT_ADDR: ${VAULT_ADDR-} VAULT_NAMESPACE: ${VAULT_NAMESPACE-} VAULT_TOKEN: ${VAULT_TOKEN-} diff --git a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py index ef2f6b25174..6265d1acfc2 100644 --- a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py @@ -4,6 +4,7 @@ from fastapi import Depends, HTTPException, UploadFile from fastapi.params import Security +from fastapi.responses import JSONResponse from fideslang.validation import FidesKey from loguru import logger from sqlalchemy.orm import Session @@ -351,7 +352,7 @@ def instantiate_connection_from_template( async def register_custom_connector_template( connector_template: UploadFile, db: Session = Depends(deps.get_db), -) -> None: +) -> JSONResponse: """ Registers a custom connector template from a zip file uploaded by the user. The endpoint performs the following steps: @@ -375,3 +376,7 @@ async def register_custom_connector_template( except Exception as exc: logger.exception("Error loading connector template from zip file.") raise HTTPException(status_code=400, detail=str(exc)) + + return JSONResponse( + content={"message": "Connector template successfully registered."} + ) diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index ca2df8d3980..aadb3070a7b 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -37,6 +37,7 @@ replace_dataset_placeholders, ) from fides.core.config import CONFIG +from fides.lib.cryptography.cryptographic_util import bytes_to_b64_str, str_to_b64_str class ConnectorTemplateLoader(ABC): @@ -141,7 +142,7 @@ def _register_template( ) # register the template in the loader's template dictionary - cls._instance._templates[template.key] = connector_template # type: ignore + cls.get_instance()._templates[template.key] = connector_template # type: ignore @classmethod def save_template(cls, db: Session, zip_file: ZipFile) -> None: @@ -156,7 +157,13 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: function_contents = None for info in zip_file.infolist(): - file_contents = zip_file.read(info).decode() + try: + file_contents = zip_file.read(info).decode() + except UnicodeDecodeError: + # skip any hidden metadata files that can't be decoded with UTF-8 + logger.debug(f"Unable to decode the file: {info.filename}") + continue + if info.filename.endswith("config.yml"): if not config_contents: config_contents = file_contents @@ -173,7 +180,7 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: ) elif info.filename.endswith(".svg"): if not icon_contents: - icon_contents = file_contents + icon_contents = str_to_b64_str(file_contents) else: raise ValidationError( "Multiple svg files found, only one is allowed." @@ -197,7 +204,7 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: connector_type = config["type"] human_readable = config["name"] - custom_connector_template = CustomConnectorTemplate( + template = CustomConnectorTemplate( key=connector_type, name=human_readable, config=config_contents, @@ -207,10 +214,20 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: ) # attempt to register the template, raises an exception if validation fails - cls._register_template(custom_connector_template) + cls._register_template(template) # save the custom connector to the database if it passed validation - custom_connector_template.save(db=db) + CustomConnectorTemplate.create_or_update( + db=db, + data={ + "key": connector_type, + "name": human_readable, + "config": config_contents, + "dataset": dataset_contents, + "icon": icon_contents, + "functions": function_contents, + }, + ) @classmethod def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express.svg b/tests/fixtures/saas/test_data/planet_express/planet_express.svg index af02207300f..92b189fe952 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express.svg +++ b/tests/fixtures/saas/test_data/planet_express/planet_express.svg @@ -1,189 +1,31 @@ - - - -image/svg+xml - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + - - - - \ No newline at end of file + + + + + + diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml index ac8a4242c04..da360f094e4 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml @@ -8,6 +8,7 @@ saas_config: connector_params: - name: domain - name: api_key + label: API Key client_config: protocol: https diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml index cb6b39c65f5..b2690442272 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml @@ -8,6 +8,7 @@ saas_config: connector_params: - name: domain - name: api_key + label: API Key client_config: protocol: https diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index e16e2ece0d6..183a8066414 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -1,9 +1,7 @@ import json -from io import BytesIO -from typing import Dict, Optional +from typing import Optional from unittest import mock from unittest.mock import Mock -from zipfile import ZipFile import pytest from sqlalchemy.orm import Session @@ -31,6 +29,7 @@ ) from fides.core.config import CONFIG from tests.ops.api.v1.endpoints.test_dataset_endpoints import _reject_key +from tests.ops.test_helpers.saas_test_utils import create_zip_file @pytest.mark.unit_saas @@ -460,27 +459,6 @@ def test_get_authorize_url( class TestRegisterConnectorTemplate: - def create_zip_file(self, file_data: Dict[str, str]) -> BytesIO: - """ - Create a zip file in memory with the given files. - - Args: - files (Dict[str, str]): A mapping of filenames to their contents - - Returns: - io.BytesIO: An in-memory zip file with the specified files. - """ - zip_buffer = BytesIO() - - with ZipFile(zip_buffer, "w") as zip_file: - for filename, file_content in file_data.items(): - zip_file.writestr(filename, file_content) - - # resetting the file position pointer to the beginning of the stream - # so the tests can read the zip file from the beginning - zip_buffer.seek(0) - return zip_buffer - @pytest.fixture def register_connector_template_url(self) -> str: return V1_URL_PREFIX + REGISTER_CONNECTOR_TEMPLATE @@ -493,7 +471,7 @@ def complete_connector_template( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": planet_express_dataset, @@ -509,7 +487,7 @@ def connector_template_missing_config( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "dataset.yml": planet_express_dataset, "functions.py": planet_express_functions, @@ -524,7 +502,7 @@ def connector_template_wrong_contents_config( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": "planet_express_config", "dataset.yml": planet_express_dataset, @@ -541,7 +519,7 @@ def connector_template_invalid_config( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_invalid_config, "dataset.yml": planet_express_dataset, @@ -557,7 +535,7 @@ def connector_template_missing_dataset( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "functions.py": planet_express_functions, @@ -572,7 +550,7 @@ def connector_template_wrong_contents_dataset( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": "planet_express_dataset", @@ -589,7 +567,7 @@ def connector_template_invalid_dataset( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": planet_express_invalid_dataset, @@ -605,10 +583,26 @@ def connector_template_no_functions( planet_express_dataset, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "icon.svg": planet_express_icon, + } + ) + + @pytest.fixture + def connector_template_invalid_functions( + self, + planet_express_config, + planet_express_dataset, + planet_express_icon, + ): + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": planet_express_dataset, + "functions.py": "import os", "icon.svg": planet_express_icon, } ) @@ -620,7 +614,7 @@ def connector_template_no_icon( planet_express_dataset, planet_express_functions, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": planet_express_dataset, @@ -636,7 +630,7 @@ def connector_template_duplicate_configs( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "1_config.yml": planet_express_config, "2_config.yml": planet_express_config, @@ -654,7 +648,7 @@ def connector_template_duplicate_datasets( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "1_dataset.yml": planet_express_dataset, @@ -672,7 +666,7 @@ def connector_template_duplicate_functions( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": planet_express_dataset, @@ -690,7 +684,7 @@ def connector_template_duplicate_icons( planet_express_functions, planet_express_icon, ): - return self.create_zip_file( + return create_zip_file( { "config.yml": planet_express_config, "dataset.yml": planet_express_dataset, @@ -719,7 +713,11 @@ def test_register_connector_template_wrong_scope( @pytest.mark.parametrize( "zip_file, status_code, details", [ - ("complete_connector_template", 200, None), + ( + "complete_connector_template", + 200, + {"message": "Connector template successfully registered."}, + ), ( "connector_template_missing_config", 400, @@ -758,8 +756,16 @@ def test_register_connector_template_wrong_scope( "detail": "1 validation error for ConnectorTemplate\ndataset -> collections -> 0 -> name\n field required (type=value_error.missing)" }, ), - ("connector_template_no_functions", 200, None), - ("connector_template_no_icon", 200, None), + ( + "connector_template_no_functions", + 200, + {"message": "Connector template successfully registered."}, + ), + ( + "connector_template_no_icon", + 200, + {"message": "Connector template successfully registered."}, + ), ( "connector_template_duplicate_configs", 400, @@ -784,6 +790,11 @@ def test_register_connector_template_wrong_scope( 400, {"detail": "Multiple svg files found, only one is allowed."}, ), + ( + "connector_template_invalid_functions", + 400, + {"detail": "Import of 'os' module is not allowed."}, + ), ], ) def test_register_connector_template_allow_custom_connector_functions( @@ -819,7 +830,7 @@ def test_register_connector_template_allow_custom_connector_functions( ( "connector_template_no_functions", 200, - None, + {"message": "Connector template successfully registered."}, ), ], ) diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index 03db996abf4..cadd163ace7 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -1,4 +1,5 @@ import os +from zipfile import ZipFile import pytest @@ -19,6 +20,7 @@ ) from fides.api.ops.util.saas_util import encode_file_contents, load_yaml_as_string from fides.core.config import CONFIG +from tests.ops.test_helpers.saas_test_utils import create_zip_file class TestFileConnectorTemplateLoader: @@ -62,7 +64,7 @@ def test_file_connector_template_loader_no_templates(self): connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} - def test_file_connector_template_loader_invalid_template( + def test_custom_connector_template_loader_invalid_template( self, db, planet_express_dataset, @@ -97,7 +99,7 @@ def test_file_connector_template_loader_invalid_template( strategy.name for strategy in authentication_strategies ] - def test_file_connector_template_loader_invalid_functions( + def test_custom_connector_template_loader_invalid_functions( self, db, planet_express_config, @@ -121,7 +123,7 @@ def test_file_connector_template_loader_invalid_functions( connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} - def test_file_connector_template_loader_custom_connector_functions_disabled( + def test_custom_connector_template_loader_custom_connector_functions_disabled( self, db, planet_express_config, @@ -157,7 +159,7 @@ def test_file_connector_template_loader_custom_connector_functions_disabled( strategy.name for strategy in authentication_strategies ] - def test_file_connector_template_loader_custom_connector_functions_disabled_custom_functions( + def test_custom_connector_template_loader_custom_connector_functions_disabled_custom_functions( self, db, planet_express_config, @@ -193,7 +195,7 @@ def test_file_connector_template_loader_custom_connector_functions_disabled_cust ) } - def test_file_connector_template_loader( + def test_custom_connector_template_loader( self, db, planet_express_config, @@ -239,6 +241,68 @@ def test_file_connector_template_loader( strategy.name for strategy in authentication_strategies ] + def test_custom_connector_save_template( + self, + db, + planet_express_config, + planet_express_dataset, + planet_express_icon, + planet_express_functions, + ): + CustomConnectorTemplateLoader.save_template( + db, + ZipFile( + create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + ), + ) + + # verify that a connector template can updated with no issue + CustomConnectorTemplateLoader.save_template( + db, + ZipFile( + create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": planet_express_functions, + "icon.svg": planet_express_icon, + } + ) + ), + ) + + def test_custom_connector_template_loader_disallowed_modules( + self, + db, + planet_express_config, + planet_express_dataset, + planet_express_icon, + ): + CONFIG.security.allow_custom_connector_functions = True + + with pytest.raises(SyntaxError) as exc: + CustomConnectorTemplateLoader.save_template( + db, + ZipFile( + create_zip_file( + { + "config.yml": planet_express_config, + "dataset.yml": planet_express_dataset, + "functions.py": "import os", + "icon.svg": planet_express_icon, + } + ) + ), + ) + assert "Import of 'os' module is not allowed." == str(exc.value) + class TestRegisterCustomFunctions: def test_function_loader(self): diff --git a/tests/ops/test_helpers/saas_test_utils.py b/tests/ops/test_helpers/saas_test_utils.py index e617cf0d14e..bde76c062a2 100644 --- a/tests/ops/test_helpers/saas_test_utils.py +++ b/tests/ops/test_helpers/saas_test_utils.py @@ -1,5 +1,7 @@ import time +from io import BytesIO from typing import Any, Callable, Dict, List +from zipfile import ZipFile DEFAULT_POLLING_ERROR_MESSAGE = ( "The endpoint did not return the required data for testing during the time limit" @@ -26,3 +28,25 @@ def poll_for_existence( time.sleep(interval) retries = original_retries return return_val + + +def create_zip_file(file_data: Dict[str, str]) -> BytesIO: + """ + Create a zip file in memory with the given files. + + Args: + files (Dict[str, str]): A mapping of filenames to their contents + + Returns: + io.BytesIO: An in-memory zip file with the specified files. + """ + zip_buffer = BytesIO() + + with ZipFile(zip_buffer, "w") as zip_file: + for filename, file_content in file_data.items(): + zip_file.writestr(filename, file_content) + + # resetting the file position pointer to the beginning of the stream + # so the tests can read the zip file from the beginning + zip_buffer.seek(0) + return zip_buffer From 8d109f2394bdab0397401e7d51d7695b034423e5 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 29 Mar 2023 14:32:03 -0700 Subject: [PATCH 11/17] Fixing static issues --- .../service/connectors/saas/connector_registry_service.py | 4 +++- src/fides/api/ops/util/saas_util.py | 6 ------ .../service/connectors/test_connector_template_loaders.py | 2 +- 3 files changed, 4 insertions(+), 8 deletions(-) diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index 7f4afa12183..a7c861479b2 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -37,7 +37,7 @@ replace_dataset_placeholders, ) from fides.core.config import CONFIG -from fides.lib.cryptography.cryptographic_util import bytes_to_b64_str, str_to_b64_str +from fides.lib.cryptography.cryptographic_util import str_to_b64_str class ConnectorTemplateLoader(ABC): @@ -144,6 +144,7 @@ def _register_template( # register the template in the loader's template dictionary cls.get_instance()._templates[template.key] = connector_template # type: ignore + # pylint: disable=too-many-branches @classmethod def save_template(cls, db: Session, zip_file: ZipFile) -> None: """ @@ -236,6 +237,7 @@ def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: class ConnectorRegistry: _instance = None + _templates: Dict[str, ConnectorTemplate] = {} @classmethod def get_instance(cls) -> "ConnectorRegistry": diff --git a/src/fides/api/ops/util/saas_util.py b/src/fides/api/ops/util/saas_util.py index 23a2af11443..086eb9a2d7b 100644 --- a/src/fides/api/ops/util/saas_util.py +++ b/src/fides/api/ops/util/saas_util.py @@ -57,12 +57,6 @@ def load_as_string(filename: str) -> str: return file.read() -def load_as_string(filename: str) -> str: - file_path = load_file([filename]) - with open(file_path, "r", encoding="utf-8") as file: - return file.read() - - def replace_config_placeholders( config: str, string_to_replace: str, replacement: str ) -> Dict: diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index cadd163ace7..dd92b106190 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -58,7 +58,7 @@ def reset_custom_connector_template_loader(self): """ CustomConnectorTemplateLoader._instance = None - def test_file_connector_template_loader_no_templates(self): + def test_custom_connector_template_loader_no_templates(self): CONFIG.security.allow_custom_connector_functions = True connector_templates = CustomConnectorTemplateLoader.get_connector_templates() From 18cb40837639d5f90580bdaec68a458c468f791c Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Thu, 30 Mar 2023 12:41:01 -0700 Subject: [PATCH 12/17] Adding replaceable flag to connector templates plus additional tests --- .../dataset/friendbuy_nextgen_dataset.yml | 2 +- ...able_field_to_custom_connector_template.py | 28 ++ .../api/v1/endpoints/saas_config_endpoints.py | 16 +- .../ops/models/custom_connector_template.py | 7 +- .../ops/schemas/saas/connector_template.py | 12 +- src/fides/api/ops/schemas/saas/saas_config.py | 1 + .../saas/connector_registry_service.py | 103 +++-- src/fides/api/ops/util/saas_util.py | 11 + .../planet_express/planet_express_config.yml | 2 +- .../planet_express/planet_express_dataset.yml | 2 +- .../planet_express_invalid_config.yml | 2 +- .../planet_express_invalid_dataset.yml | 2 +- .../replaceable_planet_express_config.yml | 33 ++ .../test_data/replaceable_zendesk_config.yml | 120 ++++++ .../endpoints/test_saas_config_endpoints.py | 16 +- .../test_connector_template_loaders.py | 399 +++++++++++++++--- tests/ops/util/test_saas_util.py | 37 +- 17 files changed, 675 insertions(+), 118 deletions(-) create mode 100644 src/fides/api/ctl/migrations/versions/ff782b0dc07e_add_replaceable_field_to_custom_connector_template.py create mode 100644 tests/fixtures/saas/test_data/planet_express/replaceable_planet_express_config.yml create mode 100644 tests/fixtures/saas/test_data/replaceable_zendesk_config.yml diff --git a/data/saas/dataset/friendbuy_nextgen_dataset.yml b/data/saas/dataset/friendbuy_nextgen_dataset.yml index d89887a2175..31837af937f 100644 --- a/data/saas/dataset/friendbuy_nextgen_dataset.yml +++ b/data/saas/dataset/friendbuy_nextgen_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: friendbuy_nextgen_instance + - fides_key: name: Friendbuy Dataset description: A sample dataset representing the Friendbuy connector for Fides collections: diff --git a/src/fides/api/ctl/migrations/versions/ff782b0dc07e_add_replaceable_field_to_custom_connector_template.py b/src/fides/api/ctl/migrations/versions/ff782b0dc07e_add_replaceable_field_to_custom_connector_template.py new file mode 100644 index 00000000000..b28f62ee902 --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/ff782b0dc07e_add_replaceable_field_to_custom_connector_template.py @@ -0,0 +1,28 @@ +"""add replaceable field to custom_connector_template + +Revision ID: ff782b0dc07e +Revises: 6d6b0b7cbb36 +Create Date: 2023-03-29 23:41:26.164600 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "ff782b0dc07e" +down_revision = "6d6b0b7cbb36" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "custom_connector_template", + sa.Column( + "replaceable", sa.Boolean(), unique=False, nullable=False, default=False + ), + ) + + +def downgrade(): + op.drop_column("custom_connector_template", "replaceable") diff --git a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py index 6265d1acfc2..2c1cae2dd7e 100644 --- a/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/saas_config_endpoints.py @@ -2,7 +2,7 @@ from typing import Optional from zipfile import BadZipFile, ZipFile -from fastapi import Depends, HTTPException, UploadFile +from fastapi import Body, Depends, HTTPException from fastapi.params import Security from fastapi.responses import JSONResponse from fideslang.validation import FidesKey @@ -349,8 +349,8 @@ def instantiate_connection_from_template( REGISTER_CONNECTOR_TEMPLATE, dependencies=[Security(verify_oauth_client, scopes=[CONNECTOR_TEMPLATE_REGISTER])], ) -async def register_custom_connector_template( - connector_template: UploadFile, +def register_custom_connector_template( + file: bytes = Body(..., media_type="application/zip"), db: Session = Depends(deps.get_db), ) -> JSONResponse: """ @@ -361,18 +361,14 @@ async def register_custom_connector_template( 2. Uses the CustomConnectorTemplateLoader to validate, register, and save the template to the database. If the uploaded file is not a valid zip file or there are any validation errors - when creating the ConnectorTemplates an HTTP 400 status code is returned. + when creating the ConnectorTemplates an HTTP 400 status code with error details is returned. """ - contents = await connector_template.read() - try: - zip_file = ZipFile(BytesIO(contents), "r") + with ZipFile(BytesIO(file), "r") as zip_file: + CustomConnectorTemplateLoader.save_template(db=db, zip_file=zip_file) except BadZipFile: raise HTTPException(status_code=400, detail="Invalid zip file") - - try: - CustomConnectorTemplateLoader.save_template(db=db, zip_file=zip_file) except Exception as exc: logger.exception("Error loading connector template from zip file.") raise HTTPException(status_code=400, detail=str(exc)) diff --git a/src/fides/api/ops/models/custom_connector_template.py b/src/fides/api/ops/models/custom_connector_template.py index cc1669ab9c2..f8e2fb57d32 100644 --- a/src/fides/api/ops/models/custom_connector_template.py +++ b/src/fides/api/ops/models/custom_connector_template.py @@ -1,4 +1,4 @@ -from sqlalchemy import Column, String +from sqlalchemy import Boolean, Column, String from sqlalchemy.ext.declarative import declared_attr from fides.lib.db.base_class import Base @@ -17,3 +17,8 @@ def __tablename__(self) -> str: dataset = Column(String, index=False, unique=False, nullable=False) icon = Column(String, index=False, unique=False, nullable=True) functions = Column(String, index=False, unique=False, nullable=True) + # indicates that this custom connector template should be replaced + # if a newer version is available from other sources + replaceable = Column( + Boolean, index=False, unique=False, nullable=False, default=False + ) diff --git a/src/fides/api/ops/schemas/saas/connector_template.py b/src/fides/api/ops/schemas/saas/connector_template.py index a6cd7f602bf..15d662a27fd 100644 --- a/src/fides/api/ops/schemas/saas/connector_template.py +++ b/src/fides/api/ops/schemas/saas/connector_template.py @@ -25,11 +25,19 @@ class ConnectorTemplate(BaseModel): @validator("config") def validate_config(cls, config: str) -> str: """Validates the config at the given path""" - SaaSConfig(**load_config_from_string(config)) + saas_config = SaaSConfig(**load_config_from_string(config)) + if saas_config.fides_key != "": + raise ValueError( + "Hard-coded fides_key detected in the config, replace all instances of it with " + ) return config @validator("dataset") def validate_dataset(cls, dataset: str) -> str: """Validates the dataset at the given path""" - Dataset(**load_dataset_from_string(dataset)) + saas_dataset = Dataset(**load_dataset_from_string(dataset)) + if saas_dataset.fides_key != "": + raise ValueError( + "Hard-coded fides_key detected in the dataset, replace all instances of it with " + ) return dataset diff --git a/src/fides/api/ops/schemas/saas/saas_config.py b/src/fides/api/ops/schemas/saas/saas_config.py index fada9c9c508..26c53753c50 100644 --- a/src/fides/api/ops/schemas/saas/saas_config.py +++ b/src/fides/api/ops/schemas/saas/saas_config.py @@ -355,6 +355,7 @@ class SaaSConfig(SaaSConfigBase): description: str version: str + replaceable: bool = False connector_params: List[ConnectorParam] external_references: Optional[List[ExternalDatasetReference]] client_config: ClientConfig diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index a7c861479b2..5ce38912deb 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -35,6 +35,7 @@ load_yaml_as_string, replace_config_placeholders, replace_dataset_placeholders, + replace_version, ) from fides.core.config import CONFIG from fides.lib.cryptography.cryptographic_util import str_to_b64_str @@ -53,7 +54,12 @@ class FileConnectorTemplateLoader(ConnectorTemplateLoader): """ _instance = None - _templates: Dict[str, ConnectorTemplate] = {} + + def __new__(cls: Any, *args: Any, **kwargs: Any) -> "FileConnectorTemplateLoader": + if cls._instance is None: + cls._instance = super().__new__(cls, *args, **kwargs) + cls._instance._templates = {} + return cls._instance @classmethod def get_instance(cls) -> "FileConnectorTemplateLoader": @@ -79,7 +85,9 @@ def get_instance(cls) -> "FileConnectorTemplateLoader": # store connector template for retrieval try: - cls._templates[connector_type] = ConnectorTemplate( + cls.get_instance()._templates[ # type: ignore[attr-defined] + connector_type + ] = ConnectorTemplate( config=load_yaml_as_string(config_file), dataset=load_yaml_as_string( f"data/saas/dataset/{connector_type}_dataset.yml" @@ -94,7 +102,7 @@ def get_instance(cls) -> "FileConnectorTemplateLoader": @classmethod def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: - return cls.get_instance()._templates + return cls.get_instance()._templates # type: ignore[attr-defined] class CustomConnectorTemplateLoader(ConnectorTemplateLoader): @@ -103,20 +111,51 @@ class CustomConnectorTemplateLoader(ConnectorTemplateLoader): """ _instance = None - _templates: Dict[str, ConnectorTemplate] = {} + + def __new__(cls: Any, *args: Any, **kwargs: Any) -> "CustomConnectorTemplateLoader": + if cls._instance is None: + cls._instance = super().__new__(cls, *args, **kwargs) + cls._instance._templates = {} + return cls._instance @classmethod def get_instance(cls) -> "CustomConnectorTemplateLoader": if cls._instance is None: - logger.info("Loading connectors templates from the database") + logger.info("Loading connectors templates from the database.") cls._instance = cls() - for template in CustomConnectorTemplate.all(db=get_api_session()): + db = get_api_session() + for template in CustomConnectorTemplate.all(db=db): + if template.replaceable and cls._replacement_available(template): + logger.info( + f"Replacing {template.key} connector template with newer version." + ) + template.delete(db=db) + continue try: cls._register_template(template) except Exception: logger.exception("Unable to load {} connector", template.key) return cls._instance + @staticmethod + def _replacement_available(template: CustomConnectorTemplate) -> bool: + """ + Check the connector templates in the FileConnectorTemplateLoader and return if a newer version is available. + """ + replacement_connector = ( + FileConnectorTemplateLoader.get_connector_templates().get(template.key) + ) + if not replacement_connector: + return False + + custom_saas_config = SaaSConfig(**load_config_from_string(template.config)) + replacement_saas_config = SaaSConfig( + **load_config_from_string(replacement_connector.config) + ) + return parse_version(replacement_saas_config.version) > parse_version( + custom_saas_config.version + ) + @classmethod def _register_template( cls, @@ -200,10 +239,28 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: if not dataset_contents: raise ValidationError("Zip file does not contain a dataset.yml file.") - # extract connector_type and human_readable values from the SaaS config - config = load_config_from_string(config_contents) - connector_type = config["type"] - human_readable = config["name"] + # extract connector_type, human_readable, and replaceable values from the SaaS config + saas_config = SaaSConfig(**load_config_from_string(config_contents)) + connector_type = saas_config.type + human_readable = saas_config.name + replaceable = saas_config.replaceable + + # if the incoming connector is flagged as replaceable we will update the version to match + # that of the existing connector template this way the custom connector template can be + # removed once a newer version is bundled with Fides + if replaceable: + existing_connector = ( + FileConnectorTemplateLoader.get_connector_templates().get( + connector_type + ) + ) + if existing_connector: + existing_config = SaaSConfig( + **load_config_from_string(existing_connector.config) + ) + config_contents = replace_version( + config_contents, existing_config.version + ) template = CustomConnectorTemplate( key=connector_type, @@ -212,6 +269,7 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: dataset=dataset_contents, icon=icon_contents, functions=function_contents, + replaceable=replaceable, ) # attempt to register the template, raises an exception if validation fails @@ -227,28 +285,16 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: "dataset": dataset_contents, "icon": icon_contents, "functions": function_contents, + "replaceable": replaceable, }, ) @classmethod def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: - return cls.get_instance()._templates + return cls.get_instance()._templates # type: ignore[attr-defined] class ConnectorRegistry: - _instance = None - _templates: Dict[str, ConnectorTemplate] = {} - - @classmethod - def get_instance(cls) -> "ConnectorRegistry": - if cls._instance is None: - cls._instance = cls() - cls._instance._templates = { - **FileConnectorTemplateLoader().get_connector_templates(), - **CustomConnectorTemplateLoader().get_connector_templates(), - } - return cls._instance - @classmethod def _get_combined_templates(cls) -> Dict[str, ConnectorTemplate]: """ @@ -273,13 +319,6 @@ def get_connector_template(cls, connector_type: str) -> Optional[ConnectorTempla """ return cls._get_combined_templates().get(connector_type) - @classmethod - def register_template( - cls, connector_type: str, template: ConnectorTemplate - ) -> None: - """Used to register new connector templates during runtime""" - cls.get_instance()._templates[connector_type] = template - def create_connection_config_from_template_no_save( db: Session, @@ -288,7 +327,7 @@ def create_connection_config_from_template_no_save( system_id: Optional[str] = None, ) -> ConnectionConfig: """Creates a SaaS connection config from a template without saving it.""" - # Load saas config from template and replace every instance of "" with the fides_key + # Load SaaS config from template and replace every instance of "" with the fides_key # the user has chosen config_from_template: Dict = replace_config_placeholders( template.config, "", template_values.instance_key diff --git a/src/fides/api/ops/util/saas_util.py b/src/fides/api/ops/util/saas_util.py index 086eb9a2d7b..e269d674425 100644 --- a/src/fides/api/ops/util/saas_util.py +++ b/src/fides/api/ops/util/saas_util.py @@ -385,3 +385,14 @@ def to_pascal_case(s: str) -> str: s = s.title() s = s.replace("_", "") return s + + +def replace_version(saas_config: str, new_version: str) -> str: + """ + Replace the version number in the given saas_config string with the provided new_version. + """ + version_pattern = r"version:\s*[\d\.]+" + updated_config = re.sub( + version_pattern, f"version: {new_version}", saas_config, count=1 + ) + return updated_config diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml index da360f094e4..b6fa6a9385d 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: planet_express + fides_key: name: Planet Express type: planet_express description: A sample schema representing the Planet Express connector for Fides diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml index 0c6619bbbb5..298e8c38df2 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: planet_express + - fides_key: name: Planet Express Dataset description: A sample dataset representing the Planet Express connector for Fides collections: diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml index b2690442272..610fd8f6b3e 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_config.yml @@ -1,5 +1,5 @@ saas_config: - fides_key: planet_express + fides_key: name: Planet Express type: planet_express description: A sample schema representing the Planet Express connector for Fides diff --git a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml index c4b0556894b..6101af37223 100644 --- a/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml +++ b/tests/fixtures/saas/test_data/planet_express/planet_express_invalid_dataset.yml @@ -1,5 +1,5 @@ dataset: - - fides_key: planet_express + - fides_key: name: Planet Express Dataset description: A sample dataset representing the Planet Express connector for Fides collections: diff --git a/tests/fixtures/saas/test_data/planet_express/replaceable_planet_express_config.yml b/tests/fixtures/saas/test_data/planet_express/replaceable_planet_express_config.yml new file mode 100644 index 00000000000..b2dc3368da1 --- /dev/null +++ b/tests/fixtures/saas/test_data/planet_express/replaceable_planet_express_config.yml @@ -0,0 +1,33 @@ +saas_config: + fides_key: + name: Planet Express + type: planet_express + description: A sample schema representing the Planet Express connector for Fides + version: 0.0.1 + replaceable: True + + connector_params: + - name: domain + - name: api_key + label: API Key + + client_config: + protocol: https + host: + authentication: + strategy: bearer + configuration: + password: + + test_request: + method: GET + path: /ping + + endpoints: + - name: user + requests: + read: + request_override: planet_express_user_access + param_values: + - name: email + identity: email diff --git a/tests/fixtures/saas/test_data/replaceable_zendesk_config.yml b/tests/fixtures/saas/test_data/replaceable_zendesk_config.yml new file mode 100644 index 00000000000..d7290bb6b3b --- /dev/null +++ b/tests/fixtures/saas/test_data/replaceable_zendesk_config.yml @@ -0,0 +1,120 @@ +saas_config: + fides_key: + name: Zendesk + type: zendesk + description: A sample schema representing the Zendesk connector for Fides + version: 0.0.1 + replaceable: True + + connector_params: + - name: domain + - name: username + - name: api_key + + client_config: + protocol: https + host: + authentication: + strategy: basic + configuration: + username: + password: + + test_request: + method: GET + path: /api/v2/users/search.json + query_params: + - name: query + value: test@ethyca + + endpoints: + - name: user + requests: + read: + method: GET + path: /api/v2/users/search.json + query_params: + - name: query + value: + param_values: + - name: email + identity: email + data_path: users + delete: + method: DELETE + path: /api/v2/users/.json + param_values: + - name: user_id + references: + - dataset: + field: user.id + direction: from + - name: user_identities + requests: + read: + method: GET + path: /api/v2/users//identities.json + query_params: + - name: page[size] + value: 100 + param_values: + - name: user_id + references: + - dataset: + field: user.id + direction: from + data_path: identities + pagination: + strategy: link + configuration: + source: body + path: links.next + - name: tickets + requests: + read: + method: GET + path: /api/v2/users//tickets/requested.json + query_params: + - name: page[size] + value: 100 + param_values: + - name: user_id + references: + - dataset: + field: user.id + direction: from + data_path: tickets + pagination: + strategy: link + configuration: + source: body + path: links.next + delete: + method: DELETE + path: /api/v2/tickets/.json + param_values: + - name: ticket_id + references: + - dataset: + field: tickets.id + direction: from + - name: ticket_comments + requests: + read: + method: GET + path: /api/v2/tickets//comments.json + query_params: + - name: page[size] + value: 100 + param_values: + - name: ticket_id + references: + - dataset: + field: tickets.id + direction: from + data_path: comments + pagination: + strategy: link + configuration: + source: body + path: links.next diff --git a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py index 183a8066414..a971ed860c2 100644 --- a/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_saas_config_endpoints.py @@ -812,7 +812,13 @@ def test_register_connector_template_allow_custom_connector_functions( response = api_client.post( register_connector_template_url, headers=auth_header, - files={"connector_template": request.getfixturevalue(zip_file)}, + files={ + "file": ( + "template.zip", + request.getfixturevalue(zip_file).read(), + "application/zip", + ) + }, ) assert response.status_code == status_code assert response.json() == details @@ -849,7 +855,13 @@ def test_register_connector_template_disallow_custom_connector_functions( response = api_client.post( register_connector_template_url, headers=auth_header, - files={"connector_template": request.getfixturevalue(zip_file)}, + files={ + "file": ( + "template.zip", + request.getfixturevalue(zip_file).read(), + "application/zip", + ) + }, ) assert response.status_code == status_code assert response.json() == details diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index dd92b106190..6953907c476 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -1,7 +1,11 @@ import os +from io import BytesIO +from unittest import mock +from unittest.mock import MagicMock from zipfile import ZipFile import pytest +from loguru import logger from fides.api.ops.common_exceptions import NoSuchSaaSRequestOverrideException from fides.api.ops.models.custom_connector_template import CustomConnectorTemplate @@ -10,6 +14,7 @@ AuthenticationStrategy, ) from fides.api.ops.service.connectors.saas.connector_registry_service import ( + ConnectorRegistry, CustomConnectorTemplateLoader, FileConnectorTemplateLoader, register_custom_functions, @@ -18,7 +23,12 @@ SaaSRequestOverrideFactory, SaaSRequestType, ) -from fides.api.ops.util.saas_util import encode_file_contents, load_yaml_as_string +from fides.api.ops.util.saas_util import ( + encode_file_contents, + load_config_from_string, + load_yaml_as_string, + replace_version, +) from fides.core.config import CONFIG from tests.ops.test_helpers.saas_test_utils import create_zip_file @@ -52,37 +62,107 @@ def test_file_connector_template_loader_connector_not_found(self): class TestCustomConnectorTemplateLoader: @pytest.fixture(autouse=True) - def reset_custom_connector_template_loader(self): + def reset_custom_connector_template_data(self): """ - Resets the CustomConnectorTemplateLoader singleton instance before each test. + Resets the loader singleton instances before each test """ + FileConnectorTemplateLoader._instance = None CustomConnectorTemplateLoader._instance = None + @pytest.fixture + def zendesk_config(self) -> str: + return load_yaml_as_string("data/saas/config/zendesk_config.yml") + + @pytest.fixture + def zendesk_dataset(self) -> str: + return load_yaml_as_string("data/saas/dataset/zendesk_dataset.yml") + + @pytest.fixture + def replaceable_zendesk_config(self) -> str: + return load_yaml_as_string( + "tests/fixtures/saas/test_data/replaceable_zendesk_config.yml" + ) + + @pytest.fixture + def replaceable_planet_express_config(self) -> str: + return load_yaml_as_string( + "tests/fixtures/saas/test_data/planet_express/replaceable_planet_express_config.yml" + ) + + @pytest.fixture + def replaceable_zendesk_zip( + self, replaceable_zendesk_config, zendesk_dataset + ) -> BytesIO: + return create_zip_file( + { + "config.yml": replace_version(replaceable_zendesk_config, "0.0.0"), + "dataset.yml": zendesk_dataset, + } + ) + + @pytest.fixture + def non_replaceable_zendesk_zip(self, zendesk_config, zendesk_dataset) -> BytesIO: + return create_zip_file( + { + "config.yml": replace_version(zendesk_config, "0.0.0"), + "dataset.yml": zendesk_dataset, + } + ) + + @pytest.fixture + def replaceable_planet_express_zip( + self, + replaceable_planet_express_config, + planet_express_dataset, + planet_express_functions, + planet_express_icon, + ) -> BytesIO: + return create_zip_file( + { + "config.yml": replaceable_planet_express_config, + "dataset.yml": planet_express_dataset, + "icon.svg": planet_express_icon, + "functions.py": planet_express_functions, + } + ) + + @pytest.fixture + def non_replaceable_zendesk_zip(self, zendesk_config, zendesk_dataset) -> BytesIO: + return create_zip_file( + { + "config.yml": replace_version(zendesk_config, "0.0.0"), + "dataset.yml": zendesk_dataset, + } + ) + def test_custom_connector_template_loader_no_templates(self): CONFIG.security.allow_custom_connector_functions = True connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) def test_custom_connector_template_loader_invalid_template( self, - db, + mock_all: MagicMock, planet_express_dataset, planet_express_icon, planet_express_functions, ): CONFIG.security.allow_custom_connector_functions = True - # save custom connector template to the database - custom_template = CustomConnectorTemplate( - key="planet_express", - name="Planet Express", - config="planet_express_config", - dataset=planet_express_dataset, - icon=planet_express_icon, - functions=planet_express_functions, - ) - custom_template.save(db=db) + mock_all.return_value = [ + CustomConnectorTemplate( + key="planet_express", + name="Planet Express", + config="planet_express_config", + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, + ) + ] # verify the custom functions aren't loaded if the template is invalid connector_templates = CustomConnectorTemplateLoader.get_connector_templates() @@ -99,9 +179,12 @@ def test_custom_connector_template_loader_invalid_template( strategy.name for strategy in authentication_strategies ] + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) def test_custom_connector_template_loader_invalid_functions( self, - db, + mock_all: MagicMock, planet_express_config, planet_express_dataset, planet_express_icon, @@ -109,23 +192,27 @@ def test_custom_connector_template_loader_invalid_functions( CONFIG.security.allow_custom_connector_functions = True # save custom connector template to the database - custom_template = CustomConnectorTemplate( - key="planet_express", - name="Planet Express", - config=planet_express_config, - dataset=planet_express_dataset, - icon=planet_express_icon, - functions="planet_express_functions", - ) - custom_template.save(db=db) + mock_all.return_value = [ + CustomConnectorTemplate( + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions="planet_express_functions", + ) + ] # verify nothing is loaded if the custom functions fail to load connector_templates = CustomConnectorTemplateLoader.get_connector_templates() assert connector_templates == {} + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) def test_custom_connector_template_loader_custom_connector_functions_disabled( self, - db, + mock_all: MagicMock, planet_express_config, planet_express_dataset, planet_express_icon, @@ -133,16 +220,16 @@ def test_custom_connector_template_loader_custom_connector_functions_disabled( ): CONFIG.security.allow_custom_connector_functions = False - # save custom connector template to the database - custom_template = CustomConnectorTemplate( - key="planet_express", - name="Planet Express", - config=planet_express_config, - dataset=planet_express_dataset, - icon=planet_express_icon, - functions=planet_express_functions, - ) - custom_template.save(db=db) + mock_all.return_value = [ + CustomConnectorTemplate( + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, + ) + ] # load custom connector templates from the database connector_templates = CustomConnectorTemplateLoader.get_connector_templates() @@ -159,9 +246,12 @@ def test_custom_connector_template_loader_custom_connector_functions_disabled( strategy.name for strategy in authentication_strategies ] + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) def test_custom_connector_template_loader_custom_connector_functions_disabled_custom_functions( self, - db, + mock_all: MagicMock, planet_express_config, planet_express_dataset, planet_express_icon, @@ -174,15 +264,16 @@ def test_custom_connector_template_loader_custom_connector_functions_disabled_cu CONFIG.security.allow_custom_connector_functions = False # save custom connector template to the database - custom_template = CustomConnectorTemplate( - key="planet_express", - name="Planet Express", - config=planet_express_config, - dataset=planet_express_dataset, - icon=planet_express_icon, - functions=None, - ) - custom_template.save(db=db) + mock_all.return_value = [ + CustomConnectorTemplate( + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=None, + ) + ] # load custom connector templates from the database connector_templates = CustomConnectorTemplateLoader.get_connector_templates() @@ -195,9 +286,12 @@ def test_custom_connector_template_loader_custom_connector_functions_disabled_cu ) } + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) def test_custom_connector_template_loader( self, - db, + mock_all: MagicMock, planet_express_config, planet_express_dataset, planet_express_icon, @@ -205,16 +299,16 @@ def test_custom_connector_template_loader( ): CONFIG.security.allow_custom_connector_functions = True - # save custom connector template to the database - custom_template = CustomConnectorTemplate( - key="planet_express", - name="Planet Express", - config=planet_express_config, - dataset=planet_express_dataset, - icon=planet_express_icon, - functions=planet_express_functions, - ) - custom_template.save(db=db) + mock_all.return_value = [ + CustomConnectorTemplate( + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + icon=planet_express_icon, + functions=planet_express_functions, + ) + ] # load custom connector templates from the database connector_templates = CustomConnectorTemplateLoader.get_connector_templates() @@ -241,14 +335,19 @@ def test_custom_connector_template_loader( strategy.name for strategy in authentication_strategies ] + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.create_or_update" + ) def test_custom_connector_save_template( self, - db, + mock_create_or_update: MagicMock, planet_express_config, planet_express_dataset, planet_express_icon, planet_express_functions, ): + db = MagicMock() + CustomConnectorTemplateLoader.save_template( db, ZipFile( @@ -277,10 +376,10 @@ def test_custom_connector_save_template( ) ), ) + assert mock_create_or_update.call_count == 2 def test_custom_connector_template_loader_disallowed_modules( self, - db, planet_express_config, planet_express_dataset, planet_express_icon, @@ -289,7 +388,7 @@ def test_custom_connector_template_loader_disallowed_modules( with pytest.raises(SyntaxError) as exc: CustomConnectorTemplateLoader.save_template( - db, + MagicMock(), ZipFile( create_zip_file( { @@ -303,6 +402,188 @@ def test_custom_connector_template_loader_disallowed_modules( ) assert "Import of 'os' module is not allowed." == str(exc.value) + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.delete" + ) + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) + def test_custom_connector_replacement_replaceable_with_update_available( + self, + mock_all: MagicMock, + mock_delete: MagicMock, + zendesk_config, + zendesk_dataset, + ): + """ + Verify that an existing connector template flagged as replaceable is + deleted when a newer version of the connector template is found in + the FileConnectorTemplateLoader. + """ + + mock_all.return_value = [ + CustomConnectorTemplate( + key="zendesk", + name="Zendesk", + config=replace_version(zendesk_config, "0.0.0"), + dataset=zendesk_dataset, + replaceable=True, + ) + ] + + template = ConnectorRegistry.get_connector_template("zendesk") + assert template + saas_config = load_config_from_string(template.config) + assert ( + saas_config["version"] == load_config_from_string(zendesk_config)["version"] + ) + assert CustomConnectorTemplateLoader.get_connector_templates() == {} + mock_delete.assert_called_once() + + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.delete" + ) + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) + def test_custom_connector_replacement_replaceable_with_update_not_available( + self, + mock_all: MagicMock, + mock_delete: MagicMock, + planet_express_config, + planet_express_dataset, + ): + """ + Verify that an existing connector template flagged as replaceable is + not deleted if a newer version of the connector template is not found + in the FileConnectorTemplateLoader. + """ + planet_express_config = replace_version(planet_express_config, "0.0.0") + + mock_all.return_value = [ + CustomConnectorTemplate( + key="planet_express", + name="Planet Express", + config=planet_express_config, + dataset=planet_express_dataset, + replaceable=True, + ) + ] + + template = ConnectorRegistry.get_connector_template("planet_express") + assert template + saas_config = load_config_from_string(template.config) + assert saas_config["version"] == "0.0.0" + assert CustomConnectorTemplateLoader.get_connector_templates() == { + "planet_express": ConnectorTemplate( + config=planet_express_config, + dataset=planet_express_dataset, + human_readable="Planet Express", + ) + } + mock_delete.assert_not_called() + + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.delete" + ) + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.all" + ) + def test_custom_connector_replacement_not_replaceable( + self, + mock_all: MagicMock, + mock_delete: MagicMock, + zendesk_config, + zendesk_dataset, + ): + """ + Verify that an existing custom connector template flagged as not replaceable is + not deleted even if a newer version of the connector template is found + in the FileConnectorTemplateLoader. + """ + zendesk_config = replace_version(zendesk_config, "0.0.0") + + mock_all.return_value = [ + CustomConnectorTemplate( + key="zendesk", + name="Zendesk", + config=zendesk_config, + dataset=zendesk_dataset, + replaceable=False, + ) + ] + + template = ConnectorRegistry.get_connector_template("zendesk") + assert template + saas_config = load_config_from_string(template.config) + assert saas_config["version"] == "0.0.0" + assert CustomConnectorTemplateLoader.get_connector_templates() == { + "zendesk": ConnectorTemplate( + config=zendesk_config, + dataset=zendesk_dataset, + human_readable="Zendesk", + ) + } + mock_delete.assert_not_called() + + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.create_or_update" + ) + def test_replaceable_template_for_existing_template( + self, mock_create_or_update: MagicMock, zendesk_config, replaceable_zendesk_zip + ): + """ + Verify that a replaceable custom connector template takes on the version of the existing connector template. + """ + CustomConnectorTemplateLoader.save_template( + db=MagicMock(), zip_file=ZipFile(replaceable_zendesk_zip) + ) + + assert mock_create_or_update.call_args.kwargs["data"]["replaceable"] + + config_contents = mock_create_or_update.call_args.kwargs["data"]["config"] + custom_config = load_config_from_string(config_contents) + existing_config = load_config_from_string(zendesk_config) + assert custom_config["version"] == existing_config["version"] + + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.create_or_update" + ) + def test_replaceable_template_for_new_template( + self, mock_create_or_update: MagicMock, replaceable_planet_express_zip + ): + """ + Verify that a replaceable custom connector template keeps its version if there is no existing connector template. + """ + CustomConnectorTemplateLoader.save_template( + db=MagicMock(), zip_file=ZipFile(replaceable_planet_express_zip) + ) + + assert mock_create_or_update.call_args.kwargs["data"]["replaceable"] + + config_contents = mock_create_or_update.call_args.kwargs["data"]["config"] + custom_config = load_config_from_string(config_contents) + assert custom_config["version"] == "0.0.1" + + @mock.patch( + "fides.api.ops.models.custom_connector_template.CustomConnectorTemplate.create_or_update" + ) + def test_non_replaceable_template( + self, + mock_create_or_update: MagicMock, + non_replaceable_zendesk_zip, + ): + """ + Verify that a non replaceable connector template keeps its version even if there is an existing connector template. + """ + CustomConnectorTemplateLoader.save_template( + db=MagicMock(), zip_file=ZipFile(non_replaceable_zendesk_zip) + ) + assert not mock_create_or_update.call_args.kwargs["data"]["replaceable"] + config_contents = mock_create_or_update.call_args.kwargs["data"]["config"] + custom_config = load_config_from_string(config_contents) + assert custom_config["version"] == "0.0.0" + class TestRegisterCustomFunctions: def test_function_loader(self): diff --git a/tests/ops/util/test_saas_util.py b/tests/ops/util/test_saas_util.py index 97707160556..88cddb12eb3 100644 --- a/tests/ops/util/test_saas_util.py +++ b/tests/ops/util/test_saas_util.py @@ -12,6 +12,7 @@ from fides.api.ops.util.saas_util import ( assign_placeholders, merge_datasets, + replace_version, unflatten_dict, ) @@ -342,13 +343,7 @@ def test_unflatten_dict(): assert unflatten_dict({"A.B": "1", "A.C": "2"}) == {"A": {"B": "1", "C": "2"}} # mixed levels - assert unflatten_dict( - { - "A": "1", - "B.C": "2", - "B.D": "3", - } - ) == { + assert unflatten_dict({"A": "1", "B.C": "2", "B.D": "3",}) == { "A": "1", "B": {"C": "2", "D": "3"}, } @@ -372,3 +367,31 @@ def test_unflatten_dict(): # unflatten_dict shouldn't be called with a None separator with pytest.raises(IndexError): unflatten_dict({"": "1"}, separator=None) + + +def test_replace_version(): + # base case + assert ( + replace_version("saas_config:\n version: 0.0.1\n key: example", "0.0.2") + == "saas_config:\n version: 0.0.2\n key: example" + ) + + # ignore extra spaces + assert ( + replace_version("saas_config:\n version: 0.0.1\n key: example", "0.0.2") + == "saas_config:\n version: 0.0.2\n key: example" + ) + + # version not found + replace_version( + "saas_config:\n key: example", "1.0.0" + ) == "saas_config:\n key: example" + + # occurrences of *version: in the rest of the config + assert ( + replace_version( + "saas_config:\n version: 0.0.1\n key: example\n other_version: 0.0.2", + "0.0.3", + ) + == "saas_config:\n version: 0.0.3\n key: example\n other_version: 0.0.2" + ) From 087794571cfa07086f1342d52bf58bd2bbcc7511 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 5 Apr 2023 12:18:40 -0700 Subject: [PATCH 13/17] First draft of the custom connector front-end --- clients/admin-ui/package.json | 1 + clients/admin-ui/src/constants.ts | 1 + .../ConnectorTemplateUploadModal.tsx | 117 ++++++++++++++++++ .../connector-template.slice.ts | 44 +++++++ .../src/features/connector-templates/types.ts | 4 + .../add-connection/ChooseConnection.tsx | 30 ++++- .../sass/ConnectorParameters.tsx | 10 +- .../datastore-connection.slice.ts | 8 +- .../features/datastore-connections/types.ts | 4 +- .../src/types/api/models/ScopeRegistryEnum.ts | 1 - 10 files changed, 206 insertions(+), 14 deletions(-) create mode 100644 clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx create mode 100644 clients/admin-ui/src/features/connector-templates/connector-template.slice.ts create mode 100644 clients/admin-ui/src/features/connector-templates/types.ts diff --git a/clients/admin-ui/package.json b/clients/admin-ui/package.json index fe77252cf67..52476b3bad1 100644 --- a/clients/admin-ui/package.json +++ b/clients/admin-ui/package.json @@ -45,6 +45,7 @@ "next": "^12.3.4", "react": "^17.0.2", "react-dom": "^17.0.2", + "react-dropzone": "^14.2.3", "react-redux": "^8.0.5", "redux-persist": "^6.0.0", "whatwg-fetch": "^3.6.2", diff --git a/clients/admin-ui/src/constants.ts b/clients/admin-ui/src/constants.ts index 6f5400567c9..ba568de5268 100644 --- a/clients/admin-ui/src/constants.ts +++ b/clients/admin-ui/src/constants.ts @@ -137,6 +137,7 @@ export const LOGIN_ROUTE = "/login"; export const USER_MANAGEMENT_ROUTE = "/user-management"; export const CONNECTION_ROUTE = "/connection"; export const CONNECTION_TYPE_ROUTE = "/connection_type"; +export const CONNECTOR_TEMPLATE = "/connector_template"; // UI ROUTES export const ADD_SYSTEMS_ROUTE = "/add-systems"; diff --git a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx new file mode 100644 index 00000000000..11c67ac9fc6 --- /dev/null +++ b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx @@ -0,0 +1,117 @@ +import { + Box, + Button, + Modal, + ModalContent, + ModalOverlay, + Text, + useToast, +} from "@fidesui/react"; +import { FetchBaseQueryError } from "@reduxjs/toolkit/dist/query/fetchBaseQuery"; +import React, { useState } from "react"; +import { useDropzone } from "react-dropzone"; +import { getErrorMessage } from "../common/helpers"; +import { errorToastParams, successToastParams } from "../common/toast"; +import { useRegisterConnectorTemplateMutation } from "./connector-template.slice"; + +type RequestModalProps = { + isOpen: boolean; + onClose: () => void; +}; + +const ConnectorTemplateUploadModal: React.FC = ({ + isOpen, + onClose, +}) => { + const [uploadedFile, setUploadedFile] = useState(null); + const toast = useToast(); + const { getRootProps, getInputProps, isDragActive } = useDropzone({ + onDrop: (acceptedFiles: File[]) => { + const file = acceptedFiles[0]; + const fileExtension = file.name.split(".").pop()?.toLowerCase(); + + if (fileExtension !== "zip") { + toast(errorToastParams("Only zip files are allowed.")); + return; + } + + setUploadedFile(acceptedFiles[0]); + }, + }); + + const [registerConnectorTemplate, { isLoading }] = + useRegisterConnectorTemplateMutation(); + + const handleSubmit = async () => { + if (uploadedFile) { + try { + await registerConnectorTemplate(uploadedFile).unwrap(); + toast(successToastParams("Connector template uploaded successfully.")); + onClose(); + } catch (error) { + toast(errorToastParams(getErrorMessage(error as FetchBaseQueryError))); + } finally { + setUploadedFile(null); + } + } + }; + + const renderFileText = () => { + if (uploadedFile) { + return {uploadedFile.name}; + } + if (isDragActive) { + return Drop the file here...; + } + return Click or drag and drop your file here.; + }; + + return ( + + + + + Upload Connector Template + + + Drag and drop your connector template zip file here, or click to + browse your files. + + + + {renderFileText()} + + + A connector template zip file must include a SaaS config and dataset, + but may also contain an icon (.svg) and custom functions (.py) as + optional files. + + + + + ); +}; + +export default ConnectorTemplateUploadModal; diff --git a/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts b/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts new file mode 100644 index 00000000000..808d6b9f672 --- /dev/null +++ b/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts @@ -0,0 +1,44 @@ +import { createSlice, PayloadAction } from "@reduxjs/toolkit"; +import { CONNECTOR_TEMPLATE } from "~/constants"; +import { baseApi } from "~/features/common/api.slice"; +import { ConnectorTemplateState } from "./types"; + +const initialState: ConnectorTemplateState = { + loading: false, + error: null, +}; + +export const connectorTemplateSlice = createSlice({ + name: "connectorTemplate", + initialState, + reducers: { + setLoading: (draftState, action: PayloadAction) => { + draftState.loading = action.payload; + }, + setError: (draftState, action: PayloadAction) => { + draftState.error = action.payload; + }, + }, +}); + +export const { setLoading, setError } = connectorTemplateSlice.actions; +export const { reducer } = connectorTemplateSlice; + +export const connectorTemplateApi = baseApi.injectEndpoints({ + endpoints: (build) => ({ + registerConnectorTemplate: build.mutation({ + query: (file) => { + const formData = new FormData(); + formData.append("file", file); + + return { + url: `${CONNECTOR_TEMPLATE}/register`, + method: "POST", + body: formData, + }; + }, + }), + }), +}); + +export const { useRegisterConnectorTemplateMutation } = connectorTemplateApi; diff --git a/clients/admin-ui/src/features/connector-templates/types.ts b/clients/admin-ui/src/features/connector-templates/types.ts new file mode 100644 index 00000000000..f88effae126 --- /dev/null +++ b/clients/admin-ui/src/features/connector-templates/types.ts @@ -0,0 +1,4 @@ +export interface ConnectorTemplateState { + loading: boolean; + error: string | null; +} diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx index 36b793dd925..3936b4f4cec 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx @@ -1,5 +1,6 @@ import { Box, + Button, Center, Flex, Input, @@ -15,10 +16,16 @@ import { setSearch, useGetAllConnectionTypesQuery, } from "connection-type/connection-type.slice"; -import React, { useCallback, useEffect, useMemo, useRef } from "react"; +import React, { + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; import { useDispatch } from "react-redux"; - import { useAppSelector } from "~/app/hooks"; +import ConnectorTemplateUploadModal from "../../connector-templates/ConnectorTemplateUploadModal"; import Breadcrumb from "./Breadcrumb"; import ConnectionTypeFilter from "./ConnectionTypeFilter"; @@ -32,6 +39,7 @@ const ChooseConnection: React.FC = () => { const filters = useAppSelector(selectConnectionTypeFilters); const { data, isFetching, isLoading, isSuccess } = useGetAllConnectionTypesQuery(filters); + const [isModalOpen, setIsModalOpen] = useState(false); const handleSearchChange = useCallback( (event: React.ChangeEvent) => { @@ -56,6 +64,10 @@ const ChooseConnection: React.FC = () => { [data] ); + const handleUploadButtonClick = () => { + setIsModalOpen(true); + }; + useEffect(() => { mounted.current = true; return () => { @@ -95,7 +107,21 @@ const ChooseConnection: React.FC = () => { type="search" /> + + setIsModalOpen(false)} + /> {(isFetching || isLoading) && (
diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/sass/ConnectorParameters.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/sass/ConnectorParameters.tsx index 54502612cdc..ed24ea8e6d6 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/sass/ConnectorParameters.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/sass/ConnectorParameters.tsx @@ -3,19 +3,19 @@ import { useAPIHelper } from "common/hooks"; import { useAlert } from "common/hooks/useAlert"; import { selectConnectionTypeState, - setConnection, + setConnection } from "connection-type/connection-type.slice"; import { ConnectionTypeSecretSchemaReponse } from "connection-type/types"; import { SaasType } from "datastore-connections/constants"; import { useCreateSassConnectionConfigMutation, usePatchDatastoreConnectionMutation, - useUpdateDatastoreConnectionSecretsMutation, + useUpdateDatastoreConnectionSecretsMutation } from "datastore-connections/datastore-connection.slice"; import { - CreateSassConnectionConfigRequest, + CreateSaasConnectionConfigRequest, DatastoreConnectionRequest, - DatastoreConnectionSecretsRequest, + DatastoreConnectionSecretsRequest } from "datastore-connections/types"; import React, { useState } from "react"; import { useDispatch } from "react-redux"; @@ -105,7 +105,7 @@ export const ConnectorParameters: React.FC = ({ } } else { // Create new Sass connector - const params: CreateSassConnectionConfigRequest = { + const params: CreateSaasConnectionConfigRequest = { description: values.description, name: values.name, instance_key: formatKey(values.instance_key as string), diff --git a/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts b/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts index 3a2e67888f8..ef2ff11b90a 100644 --- a/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts +++ b/clients/admin-ui/src/features/datastore-connections/datastore-connection.slice.ts @@ -13,8 +13,8 @@ import { DisabledStatus, TestingStatus } from "./constants"; import { CreateAccessManualWebhookRequest, CreateAccessManualWebhookResponse, - CreateSassConnectionConfigRequest, - CreateSassConnectionConfigResponse, + CreateSaasConnectionConfigRequest, + CreateSaasConnectionConfigResponse, DatastoreConnection, DatastoreConnectionParams, DatastoreConnectionRequest, @@ -164,8 +164,8 @@ export const datastoreConnectionApi = baseApi.injectEndpoints({ invalidatesTags: () => ["DatastoreConnection"], }), createSassConnectionConfig: build.mutation< - CreateSassConnectionConfigResponse, - CreateSassConnectionConfigRequest + CreateSaasConnectionConfigResponse, + CreateSaasConnectionConfigRequest >({ query: (params) => ({ url: `${CONNECTION_ROUTE}/instantiate/${params.saas_connector_type}`, diff --git a/clients/admin-ui/src/features/datastore-connections/types.ts b/clients/admin-ui/src/features/datastore-connections/types.ts index 6309b2ba2d9..e6c700ea668 100644 --- a/clients/admin-ui/src/features/datastore-connections/types.ts +++ b/clients/admin-ui/src/features/datastore-connections/types.ts @@ -208,7 +208,7 @@ export type SaasConfig = { type: SaasType; }; -export type CreateSassConnectionConfigRequest = { +export type CreateSaasConnectionConfigRequest = { name: string; description: string; instance_key: string; @@ -218,7 +218,7 @@ export type CreateSassConnectionConfigRequest = { }; }; -export type CreateSassConnectionConfigResponse = { +export type CreateSaasConnectionConfigResponse = { connection: DatastoreConnection; dataset: { fides_key: string; diff --git a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts index eac0e8c5509..eef31bd2bce 100644 --- a/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts +++ b/clients/admin-ui/src/types/api/models/ScopeRegistryEnum.ts @@ -21,7 +21,6 @@ export enum ScopeRegistryEnum { CONNECTION_READ = "connection:read", CONNECTION_AUTHORIZE = "connection:authorize", CONNECTION_TYPE_READ = "connection_type:read", - CONNECTOR_TEMPLATE_REGISTER = "connector_template:register", CONSENT_READ = "consent:read", CTL_DATASET_CREATE = "ctl_dataset:create", CTL_DATASET_READ = "ctl_dataset:read", From 1619d953623c838e1338c9e4e88f2c0dde53f532 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 5 Apr 2023 14:34:47 -0700 Subject: [PATCH 14/17] Fixing eslint issues and adding Restrict around upload connector button --- .../ConnectorTemplateUploadModal.tsx | 111 ++++++----- .../connector-template.slice.ts | 2 + .../add-connection/ChooseConnection.tsx | 27 +-- .../src/types/api/models/ScopeRegistryEnum.ts | 1 + .../saas/connector_registry_service.py | 184 +----------------- 5 files changed, 94 insertions(+), 231 deletions(-) diff --git a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx index 11c67ac9fc6..b4704347b28 100644 --- a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx +++ b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx @@ -1,8 +1,12 @@ import { Box, Button, + ButtonGroup, Modal, + ModalBody, ModalContent, + ModalFooter, + ModalHeader, ModalOverlay, Text, useToast, @@ -10,6 +14,7 @@ import { import { FetchBaseQueryError } from "@reduxjs/toolkit/dist/query/fetchBaseQuery"; import React, { useState } from "react"; import { useDropzone } from "react-dropzone"; + import { getErrorMessage } from "../common/helpers"; import { errorToastParams, successToastParams } from "../common/toast"; import { useRegisterConnectorTemplateMutation } from "./connector-template.slice"; @@ -17,11 +22,13 @@ import { useRegisterConnectorTemplateMutation } from "./connector-template.slice type RequestModalProps = { isOpen: boolean; onClose: () => void; + testId?: String; }; const ConnectorTemplateUploadModal: React.FC = ({ isOpen, onClose, + testId = "connector-template-modal", }) => { const [uploadedFile, setUploadedFile] = useState(null); const toast = useToast(); @@ -58,57 +65,73 @@ const ConnectorTemplateUploadModal: React.FC = ({ const renderFileText = () => { if (uploadedFile) { - return {uploadedFile.name}; + return {uploadedFile.name}; } if (isDragActive) { - return Drop the file here...; + return Drop the file here...; } - return Click or drag and drop your file here.; + return Click or drag and drop your file here.; }; return ( - + - - - Upload Connector Template - - - Drag and drop your connector template zip file here, or click to - browse your files. - - - - {renderFileText()} - - - A connector template zip file must include a SaaS config and dataset, - but may also contain an icon (.svg) and custom functions (.py) as - optional files. - - + + Upload connector template + + + Drag and drop your connector template zip file here, or click to + browse your files. + + + + {renderFileText()} + + + A connector template zip file must include a SaaS config and + dataset, but may also contain an icon (.svg) and custom functions + (.py) as optional files. + + + + + + + + ); diff --git a/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts b/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts index 808d6b9f672..381117c1ee3 100644 --- a/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts +++ b/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts @@ -1,6 +1,8 @@ import { createSlice, PayloadAction } from "@reduxjs/toolkit"; + import { CONNECTOR_TEMPLATE } from "~/constants"; import { baseApi } from "~/features/common/api.slice"; + import { ConnectorTemplateState } from "./types"; const initialState: ConnectorTemplateState = { diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx index 3936b4f4cec..554316fbf78 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx @@ -24,9 +24,12 @@ import React, { useState, } from "react"; import { useDispatch } from "react-redux"; + import { useAppSelector } from "~/app/hooks"; -import ConnectorTemplateUploadModal from "../../connector-templates/ConnectorTemplateUploadModal"; +import Restrict from "~/features/common/Restrict"; +import { ScopeRegistryEnum } from "~/types/api"; +import ConnectorTemplateUploadModal from "../../connector-templates/ConnectorTemplateUploadModal"; import Breadcrumb from "./Breadcrumb"; import ConnectionTypeFilter from "./ConnectionTypeFilter"; import ConnectionTypeList from "./ConnectionTypeList"; @@ -107,16 +110,18 @@ const ChooseConnection: React.FC = () => { type="search" /> - + + + None: connector_type = config_dict["type"] human_readable = config_dict["name"] - def __new__(cls: Any, *args: Any, **kwargs: Any) -> "FileConnectorTemplateLoader": - if cls._instance is None: - cls._instance = super().__new__(cls, *args, **kwargs) - cls._instance._templates = {} - return cls._instance + try: + icon = encode_file_contents(f"data/saas/icon/{connector_type}.svg") + except FileNotFoundError: + logger.debug( + f"Could not find the expected {connector_type}.svg in the data/saas/icon/ directory, using default icon" + ) + icon = encode_file_contents("data/saas/icon/default.svg") # store connector template for retrieval try: @@ -281,176 +283,6 @@ def save_template(cls, db: Session, zip_file: ZipFile) -> None: ) -class ConnectorRegistry: - @classmethod - def _get_combined_templates(cls) -> Dict[str, ConnectorTemplate]: - """ - Returns a combined map of connector templates from all registered loaders. - The resulting map is an aggregation of templates from the file loader and the custom loader, - with custom loader templates taking precedence in case of conflicts. - """ - return { - **FileConnectorTemplateLoader.get_connector_templates(), # type: ignore - **CustomConnectorTemplateLoader.get_connector_templates(), # type: ignore - } - - @staticmethod - def _replacement_available(template: CustomConnectorTemplate) -> bool: - """ - Check the connector templates in the FileConnectorTemplateLoader and return if a newer version is available. - """ - replacement_connector = ( - FileConnectorTemplateLoader.get_connector_templates().get(template.key) - ) - if not replacement_connector: - return False - - custom_saas_config = SaaSConfig(**load_config_from_string(template.config)) - replacement_saas_config = SaaSConfig( - **load_config_from_string(replacement_connector.config) - ) - return parse_version(replacement_saas_config.version) > parse_version( - custom_saas_config.version - ) - - @classmethod - def _register_template( - cls, - template: CustomConnectorTemplate, - ) -> None: - """ - Registers a custom connector template by converting it to a ConnectorTemplate, - registering any custom functions, and adding it to the loader's template dictionary. - """ - connector_template = ConnectorTemplate( - config=template.config, - dataset=template.dataset, - icon=template.icon, - functions=template.functions, - human_readable=template.name, - ) - - # register custom functions if available - if template.functions: - register_custom_functions(template.functions) - logger.info( - f"Loaded functions from the custom connector template '{template.key}'" - ) - - # register the template in the loader's template dictionary - cls.get_instance()._templates[template.key] = connector_template # type: ignore - - # pylint: disable=too-many-branches - @classmethod - def save_template(cls, db: Session, zip_file: ZipFile) -> None: - """ - Extracts and validates the contents of a zip file containing a - custom connector template, registers the template, and saves it to the database. - """ - - config_contents = None - dataset_contents = None - icon_contents = None - function_contents = None - - for info in zip_file.infolist(): - try: - file_contents = zip_file.read(info).decode() - except UnicodeDecodeError: - # skip any hidden metadata files that can't be decoded with UTF-8 - logger.debug(f"Unable to decode the file: {info.filename}") - continue - - if info.filename.endswith("config.yml"): - if not config_contents: - config_contents = file_contents - else: - raise ValidationError( - "Multiple files ending with config.yml found, only one is allowed." - ) - elif info.filename.endswith("dataset.yml"): - if not dataset_contents: - dataset_contents = file_contents - else: - raise ValidationError( - "Multiple files ending with dataset.yml found, only one is allowed." - ) - elif info.filename.endswith(".svg"): - if not icon_contents: - icon_contents = str_to_b64_str(file_contents) - else: - raise ValidationError( - "Multiple svg files found, only one is allowed." - ) - elif info.filename.endswith(".py"): - if not function_contents: - function_contents = file_contents - else: - raise ValidationError( - "Multiple Python (.py) files found, only one is allowed." - ) - - if not config_contents: - raise ValidationError("Zip file does not contain a config.yml file.") - - if not dataset_contents: - raise ValidationError("Zip file does not contain a dataset.yml file.") - - # extract connector_type, human_readable, and replaceable values from the SaaS config - saas_config = SaaSConfig(**load_config_from_string(config_contents)) - connector_type = saas_config.type - human_readable = saas_config.name - replaceable = saas_config.replaceable - - # if the incoming connector is flagged as replaceable we will update the version to match - # that of the existing connector template this way the custom connector template can be - # removed once a newer version is bundled with Fides - if replaceable: - existing_connector = ( - FileConnectorTemplateLoader.get_connector_templates().get( - connector_type - ) - ) - if existing_connector: - existing_config = SaaSConfig( - **load_config_from_string(existing_connector.config) - ) - config_contents = replace_version( - config_contents, existing_config.version - ) - - template = CustomConnectorTemplate( - key=connector_type, - name=human_readable, - config=config_contents, - dataset=dataset_contents, - icon=icon_contents, - functions=function_contents, - replaceable=replaceable, - ) - - # attempt to register the template, raises an exception if validation fails - cls._register_template(template) - - # save the custom connector to the database if it passed validation - CustomConnectorTemplate.create_or_update( - db=db, - data={ - "key": connector_type, - "name": human_readable, - "config": config_contents, - "dataset": dataset_contents, - "icon": icon_contents, - "functions": function_contents, - "replaceable": replaceable, - }, - ) - - @classmethod - def get_connector_templates(cls) -> Dict[str, ConnectorTemplate]: - return cls.get_instance()._templates # type: ignore[attr-defined] - - class ConnectorRegistry: @classmethod def _get_combined_templates(cls) -> Dict[str, ConnectorTemplate]: @@ -683,4 +515,4 @@ def custom_guarded_import( raise SyntaxError(f"Import of '{name}' module is not allowed.") if fromlist is None: fromlist = () - return __import__(name, _globals, _locals, fromlist, level) + return __import__(name, _globals, _locals, fromlist, level) \ No newline at end of file From f9e2638f0e03dc28542ce95279fe4ee2b4705307 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Wed, 5 Apr 2023 14:38:46 -0700 Subject: [PATCH 15/17] Reverting accidental changes --- .../ops/service/connectors/saas/connector_registry_service.py | 2 +- tests/ops/service/connectors/test_connector_template_loaders.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py index 11666bb60fe..5047f8af1e4 100644 --- a/src/fides/api/ops/service/connectors/saas/connector_registry_service.py +++ b/src/fides/api/ops/service/connectors/saas/connector_registry_service.py @@ -515,4 +515,4 @@ def custom_guarded_import( raise SyntaxError(f"Import of '{name}' module is not allowed.") if fromlist is None: fromlist = () - return __import__(name, _globals, _locals, fromlist, level) \ No newline at end of file + return __import__(name, _globals, _locals, fromlist, level) diff --git a/tests/ops/service/connectors/test_connector_template_loaders.py b/tests/ops/service/connectors/test_connector_template_loaders.py index d63355bb417..5d279cd3593 100644 --- a/tests/ops/service/connectors/test_connector_template_loaders.py +++ b/tests/ops/service/connectors/test_connector_template_loaders.py @@ -5,7 +5,6 @@ from zipfile import ZipFile import pytest -from loguru import logger from fides.api.ops.common_exceptions import NoSuchSaaSRequestOverrideException from fides.api.ops.models.custom_connector_template import CustomConnectorTemplate From 9ebab466250024f113011145c75da19b9a2f0754 Mon Sep 17 00:00:00 2001 From: Adrian Galvan Date: Fri, 7 Apr 2023 09:49:23 -0700 Subject: [PATCH 16/17] Fixing imports and removing unnecessary code --- .../ConnectorTemplateUploadModal.tsx | 5 +++-- .../connector-template.slice.ts | 20 ++++--------------- .../src/features/connector-templates/types.ts | 4 ---- .../add-connection/ChooseConnection.tsx | 4 ++-- .../sass/ConnectorParameters.tsx | 6 +++--- 5 files changed, 12 insertions(+), 27 deletions(-) delete mode 100644 clients/admin-ui/src/features/connector-templates/types.ts diff --git a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx index b4704347b28..74d2154ae49 100644 --- a/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx +++ b/clients/admin-ui/src/features/connector-templates/ConnectorTemplateUploadModal.tsx @@ -15,8 +15,9 @@ import { FetchBaseQueryError } from "@reduxjs/toolkit/dist/query/fetchBaseQuery" import React, { useState } from "react"; import { useDropzone } from "react-dropzone"; -import { getErrorMessage } from "../common/helpers"; -import { errorToastParams, successToastParams } from "../common/toast"; +import { getErrorMessage } from "~/features/common/helpers"; +import { errorToastParams, successToastParams } from "~/features/common/toast"; + import { useRegisterConnectorTemplateMutation } from "./connector-template.slice"; type RequestModalProps = { diff --git a/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts b/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts index 381117c1ee3..44bb0f62b89 100644 --- a/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts +++ b/clients/admin-ui/src/features/connector-templates/connector-template.slice.ts @@ -1,29 +1,17 @@ -import { createSlice, PayloadAction } from "@reduxjs/toolkit"; +import { createSlice } from "@reduxjs/toolkit"; import { CONNECTOR_TEMPLATE } from "~/constants"; import { baseApi } from "~/features/common/api.slice"; -import { ConnectorTemplateState } from "./types"; - -const initialState: ConnectorTemplateState = { - loading: false, - error: null, -}; +export interface State {} +const initialState: State = {}; export const connectorTemplateSlice = createSlice({ name: "connectorTemplate", initialState, - reducers: { - setLoading: (draftState, action: PayloadAction) => { - draftState.loading = action.payload; - }, - setError: (draftState, action: PayloadAction) => { - draftState.error = action.payload; - }, - }, + reducers: {}, }); -export const { setLoading, setError } = connectorTemplateSlice.actions; export const { reducer } = connectorTemplateSlice; export const connectorTemplateApi = baseApi.injectEndpoints({ diff --git a/clients/admin-ui/src/features/connector-templates/types.ts b/clients/admin-ui/src/features/connector-templates/types.ts deleted file mode 100644 index f88effae126..00000000000 --- a/clients/admin-ui/src/features/connector-templates/types.ts +++ /dev/null @@ -1,4 +0,0 @@ -export interface ConnectorTemplateState { - loading: boolean; - error: string | null; -} diff --git a/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx b/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx index 554316fbf78..31eb855bdcb 100644 --- a/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx +++ b/clients/admin-ui/src/features/datastore-connections/add-connection/ChooseConnection.tsx @@ -27,9 +27,9 @@ import { useDispatch } from "react-redux"; import { useAppSelector } from "~/app/hooks"; import Restrict from "~/features/common/Restrict"; +import ConnectorTemplateUploadModal from "~/features/connector-templates/ConnectorTemplateUploadModal"; import { ScopeRegistryEnum } from "~/types/api"; -import ConnectorTemplateUploadModal from "../../connector-templates/ConnectorTemplateUploadModal"; import Breadcrumb from "./Breadcrumb"; import ConnectionTypeFilter from "./ConnectionTypeFilter"; import ConnectionTypeList from "./ConnectionTypeList"; @@ -114,7 +114,7 @@ const ChooseConnection: React.FC = () => {