From fc08df87eb77c6aa78daf7ebd6f77201b24e60b3 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 17 Aug 2023 14:46:45 +0800 Subject: [PATCH 01/44] Update Fideslang to version 2.0.0 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 578e9fee423..1dcd701fa83 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang==1.4.4 +fideslang==2.0.0 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From c180cd52399419d2574a913813897dedb8ff55fb Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 17 Aug 2023 18:17:26 +0800 Subject: [PATCH 02/44] feat: get cod in place for the find/replace --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 126 ++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100644 src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py new file mode 100644 index 00000000000..f194c2d39d7 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -0,0 +1,126 @@ +"""migrate to fideslang 2.0 uses and categories + +Revision ID: 1e750936a141 +Revises: fd52d5f08c17 +Create Date: 2023-08-17 08:51:47.226612 + +""" +import json +from typing import Dict, List, Optional + +from alembic import op +from loguru import logger +from sqlalchemy import text +from sqlalchemy.engine import Connection, ResultProxy +from sqlalchemy.sql.elements import TextClause + +############### +## Data Uses ## +############### +# The `key` is the old value, the `value` is the new value +data_use_upgrades: Dict[str, str] = { + "improve.system": "functional.service.improve", # Verified in 2.0 + "improve": "functional", # Posted a question in the Confluence doc to verify this + "essential.service.operations.support.optimizations": "essential.service.operations.improve", # Verified in 2.0 +} +data_use_downgrades: Dict[str, str] = { + value: key for key, value in data_use_upgrades.items() +} + +##################### +## Data Categories ## +##################### +# The `key` is the old value, the `value` is the new value +data_category_upgrades: Dict[str, str] = { + "user.credentials": "user.authorization.credentials", # Verified in 2.0 + "user.observed": "user.behavior", # Verified in 2.0 + "user.browsing_history": "user.behavior.browsing_history", # Verified in 2.0 + "user.media_consumption": "user.behavior.media_consumption", # Verified in 2.0 + "user.search_history": "user.behavior.search_history", # Verified in 2.0 + "user.organization": "user.contact.organization", # Verified in 2.0 + "user.non_specific_age": "user.demographic.age_range", # Verified in 2.0 + "user.date_of_birth": "user.demographic.date_of_birth", # Verified in 2.0 + "user.gender": "user.demographic.gender", # Verified in 2.0 + "user.political_opinion": "user.demographic.political_opinion", # Verified in 2.0 + "user.profiling": "user.demographic.profile", # Verified in 2.0 + "user.race": "user.demographic.race_ethnicity", # Verified in 2.0 + "user.religious_belief": "user.demographic.religious_belief", # Verified in 2.0 + "user.sexual_orientation": "user.demographic.sexual_orientation", # Verified in 2.0 + "user.financial.account_number": "user.financial.bank_account", # Verified in 2.0 + "user.genetic": "user.health_and_medical.genetic", # Verified in 2.0 +} +data_category_downgrades: Dict[str, str] = { + value: key for key, value in data_use_upgrades.items() +} + +def update_privacy_declaration_data_uses( + bind: Connection, data_use_map: Dict[str, str] +) -> None: + """Upgrade or downgrade data uses from fideslang 1.4 for privacy declarations""" + existing_ctl_policies: ResultProxy = bind.execute( + text("SELECT id, data_use FROM privacydeclaration;") + ) + for row in existing_ctl_policies: + data_use: str = row["data_use"] + updated_data_use: Optional[str] = data_use_map.get(data_use, None) + + if updated_data_use: + update_data_use_query: TextClause = text( + "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" + ) + bind.execute( + update_data_use_query, + {"declaration_id": row["id"], "updated_use": updated_data_use}, + ) + + +def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) -> None: + """Upgrade or downgrade data uses from fideslang 1.4 for ctl policies""" + existing_ctl_policies: ResultProxy = bind.execute( + text("SELECT id, rules FROM ctl_policies;") + ) + for row in existing_ctl_policies: + needs_update: bool = False + rules: List[Dict] = row["rules"] + for i, rule in enumerate(rules or []): + data_uses: Dict = rule.get("data_uses", {}) + for j, val in enumerate(data_uses.get("values", [])): + new_data_use: Optional[str] = data_use_map.get(val, None) + if new_data_use: + rules[i]["data_uses"]["values"][j] = new_data_use + needs_update = True + + if needs_update: + update_data_use_query: TextClause = text( + "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" + ) + bind.execute( + update_data_use_query, + {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, + ) + + +# revision identifiers, used by Alembic. +revision = "1e750936a141" +down_revision = "fd52d5f08c17" +branch_labels = None +depends_on = None + +def upgrade() -> None: + bind: Connection = op.get_bind() + + logger.info("Upgrading data use on privacy declaration") + update_privacy_declaration_data_uses(bind, data_use_upgrades) + + logger.info("Upgrading ctl policy rule data uses") + update_ctl_policy_data_uses(bind, data_use_upgrades) + + +def downgrade() -> None: + bind: Connection = op.get_bind() + + logger.info("Downgrading data use on privacy declaration") + update_privacy_declaration_data_uses(bind, data_use_downgrades) + + logger.info("Downgrading ctl policy rule data uses") + update_ctl_policy_data_uses(bind, data_use_downgrades) From 674769f734f4f2fa8a370a5aa8aeca703539c346 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 17 Aug 2023 22:39:58 +0800 Subject: [PATCH 03/44] feat: checkpoint --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 15 ++-- ...1ba_add_version_fields_to_default_types.py | 70 +++++++++++++++++++ src/fides/api/db/seed.py | 2 +- src/fides/api/models/sql_models.py | 28 ++++++-- src/fides/api/util/data_category.py | 2 +- 5 files changed, 104 insertions(+), 13 deletions(-) create mode 100644 src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index f194c2d39d7..9503f5ad505 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -14,6 +14,12 @@ from sqlalchemy.engine import Connection, ResultProxy from sqlalchemy.sql.elements import TextClause +# revision identifiers, used by Alembic. +revision = "1e750936a141" +down_revision = "708a780b01ba" +branch_labels = None +depends_on = None + ############### ## Data Uses ## ############### @@ -21,7 +27,7 @@ data_use_upgrades: Dict[str, str] = { "improve.system": "functional.service.improve", # Verified in 2.0 "improve": "functional", # Posted a question in the Confluence doc to verify this - "essential.service.operations.support.optimizations": "essential.service.operations.improve", # Verified in 2.0 + "essential.service.operations.support.optimizations": "essential.service.operations.improve", # Verified in 2.0 } data_use_downgrades: Dict[str, str] = { value: key for key, value in data_use_upgrades.items() @@ -53,6 +59,7 @@ value: key for key, value in data_use_upgrades.items() } + def update_privacy_declaration_data_uses( bind: Connection, data_use_map: Dict[str, str] ) -> None: @@ -100,12 +107,6 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) ) -# revision identifiers, used by Alembic. -revision = "1e750936a141" -down_revision = "fd52d5f08c17" -branch_labels = None -depends_on = None - def upgrade() -> None: bind: Connection = op.get_bind() diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py new file mode 100644 index 00000000000..22ba06547d4 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -0,0 +1,70 @@ +"""add version fields to default types + +Revision ID: 708a780b01ba +Revises: 1e750936a141 +Create Date: 2023-08-17 12:29:04.855626 + +""" +from alembic import op +import sqlalchemy as sa + + +# revision identifiers, used by Alembic. +revision = "708a780b01ba" +down_revision = "fd52d5f08c17" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.add_column( + "ctl_data_categories", sa.Column("version_added", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_categories", sa.Column("version_deprecated", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_categories", sa.Column("replaced_by", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_qualifiers", sa.Column("version_added", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_qualifiers", sa.Column("version_deprecated", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_qualifiers", sa.Column("replaced_by", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_subjects", sa.Column("version_added", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_subjects", sa.Column("version_deprecated", sa.Text(), nullable=True) + ) + op.add_column( + "ctl_data_subjects", sa.Column("replaced_by", sa.Text(), nullable=True) + ) + op.add_column("ctl_data_uses", sa.Column("version_added", sa.Text(), nullable=True)) + op.add_column( + "ctl_data_uses", sa.Column("version_deprecated", sa.Text(), nullable=True) + ) + op.add_column("ctl_data_uses", sa.Column("replaced_by", sa.Text(), nullable=True)) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.drop_column("ctl_data_uses", "replaced_by") + op.drop_column("ctl_data_uses", "version_deprecated") + op.drop_column("ctl_data_uses", "version_added") + op.drop_column("ctl_data_subjects", "replaced_by") + op.drop_column("ctl_data_subjects", "version_deprecated") + op.drop_column("ctl_data_subjects", "version_added") + op.drop_column("ctl_data_qualifiers", "replaced_by") + op.drop_column("ctl_data_qualifiers", "version_deprecated") + op.drop_column("ctl_data_qualifiers", "version_added") + op.drop_column("ctl_data_categories", "replaced_by") + op.drop_column("ctl_data_categories", "version_deprecated") + op.drop_column("ctl_data_categories", "version_added") + # ### end Alembic commands ### diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index 7345e453a36..eb7dd879c11 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -401,7 +401,7 @@ async def load_default_taxonomy(async_session: AsyncSession) -> None: await upsert_resources( sql_model_map[resource_type], resources, async_session ) - except QueryError: # pragma: no cover + except QueryError as e: # pragma: no cover pass # The create_resource function will log the error else: log.debug(f"UPSERTED {len(resources)} {resource_type} resource(s)") diff --git a/src/fides/api/models/sql_models.py b/src/fides/api/models/sql_models.py index 1df132dad05..c492273333c 100644 --- a/src/fides/api/models/sql_models.py +++ b/src/fides/api/models/sql_models.py @@ -151,9 +151,14 @@ class DataCategory(Base, FidesBase): __tablename__ = "ctl_data_categories" parent_key = Column(Text) - is_default = Column(BOOLEAN, default=False) active = Column(BOOLEAN, default=True, nullable=False) + # Default Fields + is_default = Column(BOOLEAN, default=False) + version_added = Column(Text) + version_deprecated = Column(Text) + replaced_by = Column(Text) + @classmethod def from_fideslang_obj( cls, data_category: FideslangDataCategory @@ -177,9 +182,14 @@ class DataQualifier(Base, FidesBase): __tablename__ = "ctl_data_qualifiers" parent_key = Column(Text) - is_default = Column(BOOLEAN, default=False) active = Column(BOOLEAN, default=True, nullable=False) + # Default Fields + is_default = Column(BOOLEAN, default=False) + version_added = Column(Text) + version_deprecated = Column(Text) + replaced_by = Column(Text) + class DataSubject(Base, FidesBase): """ @@ -189,9 +199,14 @@ class DataSubject(Base, FidesBase): __tablename__ = "ctl_data_subjects" rights = Column(JSON, nullable=True) automated_decisions_or_profiling = Column(BOOLEAN, nullable=True) - is_default = Column(BOOLEAN, default=False) active = Column(BOOLEAN, default=True, nullable=False) + # Default Fields + is_default = Column(BOOLEAN, default=False) + version_added = Column(Text) + version_deprecated = Column(Text) + replaced_by = Column(Text) + class DataUse(Base, FidesBase): """ @@ -214,9 +229,14 @@ class DataUse(Base, FidesBase): legitimate_interest_impact_assessment = Column( String, nullable=True ) # Deprecated in favor of PrivacyDeclaration.legal_basis_for_processing - is_default = Column(BOOLEAN, default=False) active = Column(BOOLEAN, default=True, nullable=False) + # Default Fields + is_default = Column(BOOLEAN, default=False) + version_added = Column(Text) + version_deprecated = Column(Text) + replaced_by = Column(Text) + @staticmethod def get_parent_uses_from_key(data_use_key: str) -> Set[str]: """ diff --git a/src/fides/api/util/data_category.py b/src/fides/api/util/data_category.py index 472cc71d704..f77fb4bf30b 100644 --- a/src/fides/api/util/data_category.py +++ b/src/fides/api/util/data_category.py @@ -37,6 +37,6 @@ def _validate_data_category( valid_categories = get_data_categories_from_db(db=db) if data_category not in valid_categories: raise common_exceptions.DataCategoryNotSupported( - f"The data category {data_category} is not supported." + f"The data category '{data_category}' was not found in the database, and is therefore not valid for use here." ) return data_category From 0e28ef151ab7b3b8346cf5bb8b2a15377ec8046c Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 17 Aug 2023 23:38:46 +0800 Subject: [PATCH 04/44] feat: pin a new fideslang version to get migrations working --- requirements.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 1dcd701fa83..24ae4a909b8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,8 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang==2.0.0 +# fideslang==2.0.0 +fideslang @ git+https://github.com/ethyca/fideslang.git@008bb46b3ed92a7b9359c9ab7007aade3a45609d#egg=fideslang fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From fd5dcbeb972ba14ee883a20ee9cac94e0ff97d38 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 18 Aug 2023 00:16:07 +0800 Subject: [PATCH 05/44] feat: find/replace new categories and uses --- clients/admin-ui/cypress/e2e/systems.cy.ts | 37 +- .../fixtures/connectors/datasetconfig.json | 2 +- .../fixtures/privacy-notices/list.json | 2 +- .../cypress/fixtures/systems/system.json | 2 +- .../cypress/fixtures/systems/systems.json | 4 +- .../systems/systems_with_data_uses.json | 4 +- .../fixtures/taxonomy/data_categories.json | 30 +- .../cypress/fixtures/taxonomy/data_uses.json | 2 +- clients/fides-js/__tests__/lib/cookie.test.ts | 2 +- .../dataset/bigquery_example_test_dataset.yml | 2 +- data/dataset/email_dataset.yml | 2 +- data/dataset/example_test_dataset.invalid | 2 +- data/dataset/example_test_datasets.yml | 4 +- data/dataset/mariadb_example_test_dataset.yml | 2 +- data/dataset/mongo_example_test_dataset.yml | 8 +- data/dataset/mssql_example_test_dataset.yml | 2 +- data/dataset/mysql_example_test_dataset.yml | 2 +- .../dataset/postgres_example_test_dataset.yml | 2 +- .../dataset/redshift_example_test_dataset.yml | 2 +- .../snowflake_example_test_dataset.yml | 2 +- .../dataset/timebase_example_test_dataset.yml | 2 +- data/saas/dataset/adobe_campaign_dataset.yml | 12 +- data/saas/dataset/amplitude_dataset.yml | 4 +- data/saas/dataset/braze_dataset.yml | 4 +- data/saas/dataset/mailchimp_dataset.yml | 2 +- data/saas/dataset/outreach_dataset.yml | 4 +- data/saas/dataset/salesforce_dataset.yml | 2 +- data/saas/dataset/segment_dataset.yml | 4 +- data/saas/dataset/sentry_dataset.yml | 4 +- data/saas/dataset/square_dataset.yml | 2 +- data/saas/dataset/stripe_dataset.yml | 16 +- data/saas/dataset/vend_dataset.yml | 4 +- data/saas/dataset/yotpo_reviews_dataset.yml | 2 +- demo_resources/demo_system.yml | 2 +- .../postman/Fides.postman_collection.json | 11664 +++++++--------- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 2 +- .../api/api/v1/endpoints/user_endpoints.py | 2 +- src/fides/api/db/seed.py | 6 +- .../connectors/consent_email_connector.py | 2 +- .../mongo_example_test_dataset.yml | 8 +- .../postgres_example_test_dataset.yml | 2 +- .../sample_resources/sample_systems.yml | 2 +- tests/conftest.py | 2 +- tests/ctl/api/test_seed.py | 29 +- tests/ctl/core/test_evaluate.py | 6 +- .../failing_dataset_collection_taxonomy.yml | 4 +- .../data/failing_dataset_field_taxonomy.yml | 4 +- tests/ctl/data/failing_dataset_taxonomy.yml | 4 +- .../ctl/data/failing_declaration_taxonomy.yml | 2 +- tests/ctl/data/failing_nested_dataset.yml | 2 +- .../ctl/data/passing_declaration_taxonomy.yml | 2 +- .../test_data/mailchimp_override_dataset.yml | 2 +- .../saas/test_data/saas_example_dataset.yml | 2 +- tests/fixtures/saas_example_fixtures.py | 4 +- tests/lib/test_fides_user.py | 10 +- .../test_privacy_notice_endpoints.py | 12 +- .../api/v1/endpoints/test_user_endpoints.py | 8 +- .../ops/integration_tests/test_mongo_task.py | 12 +- tests/ops/integration_tests/test_sql_task.py | 2 +- tests/ops/models/test_consent_request.py | 4 +- tests/ops/models/test_fidesopsuser.py | 10 +- .../test_consent_email_connector.py | 2 +- .../service/connectors/test_queryconfig.py | 4 +- .../message_dispatch_service_test.py | 2 +- tests/ops/task/test_filter_results.py | 6 +- 65 files changed, 5607 insertions(+), 6395 deletions(-) diff --git a/clients/admin-ui/cypress/e2e/systems.cy.ts b/clients/admin-ui/cypress/e2e/systems.cy.ts index 8f9a63258d6..185925617d9 100644 --- a/clients/admin-ui/cypress/e2e/systems.cy.ts +++ b/clients/admin-ui/cypress/e2e/systems.cy.ts @@ -315,8 +315,8 @@ describe("System management page", () => { }); // edit the existing declaration - cy.getByTestId("accordion-header-improve.system").click(); - cy.getByTestId("improve.system-form").within(() => { + cy.getByTestId("accordion-header-functional.service.improve").click(); + cy.getByTestId("functional.service.improve-form").within(() => { cy.getByTestId("input-data_subjects").type(`customer{enter}`); cy.getByTestId("save-btn").click(); }); @@ -433,12 +433,14 @@ describe("System management page", () => { cy.getByTestId("more-btn").click(); cy.getByTestId("edit-btn").click(); }); - // "improve.system" and "Store system data." are already being used + // "functional.service.improve" and "Store system data." are already being used cy.getByTestId("tab-Data uses").click(); cy.getByTestId("add-btn").click(); cy.wait(["@getDataCategories", "@getDataSubjects", "@getDataUses"]); cy.getByTestId("new-declaration-form").within(() => { - cy.getByTestId("input-data_use").type(`improve.system{enter}`); + cy.getByTestId("input-data_use").type( + `functional.service.improve{enter}` + ); cy.getByTestId("input-name").type(`Store system data.{enter}`); cy.getByTestId("input-data_categories").type(`user.biometric{enter}`); cy.getByTestId("input-data_subjects").type(`anonymous{enter}`); @@ -468,12 +470,14 @@ describe("System management page", () => { cy.getByTestId("add-btn").click(); cy.wait(["@getDataCategories", "@getDataSubjects", "@getDataUses"]); - cy.getByTestId(`accordion-header-improve.system`); + cy.getByTestId(`accordion-header-functional.service.improve`); cy.getByTestId(`accordion-header-advertising`).click(); - // try to change 'advertising' to 'improve.system' and make their names match + // try to change 'advertising' to 'functional.service.improve' and make their names match cy.getByTestId("advertising-form").within(() => { - cy.getByTestId("input-data_use").type(`improve.system{enter}`); + cy.getByTestId("input-data_use").type( + `functional.service.improve{enter}` + ); cy.getByTestId("input-name").clear().type(`Store system data.{enter}`); cy.getByTestId("save-btn").click(); }); @@ -486,13 +490,15 @@ describe("System management page", () => { cy.getByTestId("more-btn").click(); cy.getByTestId("edit-btn").click(); }); - // "improve.system" and "Store system data." are already being used - // use "improve.system" again but a different name + // "functional.service.improve" and "Store system data." are already being used + // use "functional.service.improve" again but a different name cy.getByTestId("tab-Data uses").click(); cy.getByTestId("add-btn").click(); cy.wait(["@getDataCategories", "@getDataSubjects", "@getDataUses"]); cy.getByTestId("new-declaration-form").within(() => { - cy.getByTestId("input-data_use").type(`improve.system{enter}`); + cy.getByTestId("input-data_use").type( + `functional.service.improve{enter}` + ); cy.getByTestId("input-name").type(`A different description.{enter}`); cy.getByTestId("input-data_categories").type(`user.biometric{enter}`); cy.getByTestId("input-data_subjects").type(`anonymous{enter}`); @@ -557,7 +563,7 @@ describe("System management page", () => { // Edit the existing data use cy.getByTestId("privacy-declaration-accordion").within(() => { - cy.getByTestId("accordion-header-improve.system").click(); + cy.getByTestId("accordion-header-functional.service.improve").click(); // Add a data subject cy.getByTestId("input-data_subjects").type(`citizen{enter}`); cy.getByTestId("save-btn").click(); @@ -608,15 +614,18 @@ describe("System management page", () => { }); it("deletes an accordion privacy declaration", () => { - cy.getByTestId("accordion-header-improve.system").click(); - cy.getByTestId("improve.system-form").within(() => { + cy.getByTestId("accordion-header-functional.service.improve").click(); + cy.getByTestId("functional.service.improve-form").within(() => { cy.getByTestId("delete-btn").click(); }); cy.getByTestId("continue-btn").click(); cy.wait("@putFidesSystem").then((interception) => { const { body } = interception.request; expect(body.privacy_declarations.length).to.eql(1); - expect(body.privacy_declarations[0].data_use !== "improve.system"); + expect( + body.privacy_declarations[0].data_use !== + "functional.service.improve" + ); }); cy.getByTestId("toast-success-msg").contains("Data use deleted"); }); diff --git a/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json b/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json index a7fd4bbe5ca..9ee88417925 100644 --- a/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json +++ b/clients/admin-ui/cypress/fixtures/connectors/datasetconfig.json @@ -495,7 +495,7 @@ { "name": "ccn", "description": null, - "data_categories": ["user.financial.account_number"], + "data_categories": ["user.financial.bank_account"], "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "retention": null, "fides_meta": null, diff --git a/clients/admin-ui/cypress/fixtures/privacy-notices/list.json b/clients/admin-ui/cypress/fixtures/privacy-notices/list.json index 934666fa558..33880a70de6 100644 --- a/clients/admin-ui/cypress/fixtures/privacy-notices/list.json +++ b/clients/admin-ui/cypress/fixtures/privacy-notices/list.json @@ -71,7 +71,7 @@ "origin": null, "regions": ["fr", "ie"], "consent_mechanism": "opt_in", - "data_uses": ["improve.system"], + "data_uses": ["functional.service.improve"], "enforcement_level": "system_wide", "disabled": false, "has_gpc_flag": false, diff --git a/clients/admin-ui/cypress/fixtures/systems/system.json b/clients/admin-ui/cypress/fixtures/systems/system.json index 09e930e287b..3f5409d6720 100644 --- a/clients/admin-ui/cypress/fixtures/systems/system.json +++ b/clients/admin-ui/cypress/fixtures/systems/system.json @@ -13,7 +13,7 @@ { "name": "Analyze customer behaviour for improvements.", "data_categories": ["user.contact", "user.device.cookie_id"], - "data_use": "improve.system", + "data_use": "functional.service.improve", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], "dataset_references": ["demo_users_dataset"], diff --git a/clients/admin-ui/cypress/fixtures/systems/systems.json b/clients/admin-ui/cypress/fixtures/systems/systems.json index 6e9e6aa2649..289d74ce6e9 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems.json @@ -14,7 +14,7 @@ { "name": "Store system data.", "data_categories": ["system.operations", "user.contact"], - "data_use": "improve.system", + "data_use": "functional.service.improve", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], "dataset_references": ["public"], @@ -56,7 +56,7 @@ { "name": "Analyze customer behaviour for improvements.", "data_categories": ["user.contact", "user.device.cookie_id"], - "data_use": "improve.system", + "data_use": "functional.service.improve", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], "dataset_references": ["demo_users_dataset"], diff --git a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json index 37c89489c34..a69a12807a7 100644 --- a/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json +++ b/clients/admin-ui/cypress/fixtures/systems/systems_with_data_uses.json @@ -14,7 +14,7 @@ { "name": "Store system data.", "data_categories": ["system.operations", "user.contact"], - "data_use": "improve.system", + "data_use": "functional.service.improve", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["anonymous_user"], "dataset_references": ["public"], @@ -68,7 +68,7 @@ { "name": "Analyze customer behaviour for improvements.", "data_categories": ["user.contact", "user.device.cookie_id"], - "data_use": "improve.system", + "data_use": "functional.service.improve", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], "dataset_references": ["demo_users_dataset"], diff --git a/clients/admin-ui/cypress/fixtures/taxonomy/data_categories.json b/clients/admin-ui/cypress/fixtures/taxonomy/data_categories.json index e407134871e..9915ced2540 100644 --- a/clients/admin-ui/cypress/fixtures/taxonomy/data_categories.json +++ b/clients/admin-ui/cypress/fixtures/taxonomy/data_categories.json @@ -80,7 +80,7 @@ "active": true }, { - "fides_key": "user.browsing_history", + "fides_key": "user.behavior.browsing_history", "organization_fides_key": "default_organization", "tags": null, "name": "Browsing History", @@ -160,7 +160,7 @@ "active": true }, { - "fides_key": "user.gender", + "fides_key": "user.demographic.gender", "organization_fides_key": "default_organization", "tags": null, "name": "Gender", @@ -180,7 +180,7 @@ "active": true }, { - "fides_key": "user.media_consumption", + "fides_key": "user.behavior.media_consumption", "organization_fides_key": "default_organization", "tags": null, "name": "Media Consumption Data", @@ -190,7 +190,7 @@ "active": true }, { - "fides_key": "user.non_specific_age", + "fides_key": "user.demographic.age_range", "organization_fides_key": "default_organization", "tags": null, "name": "Non-Specific Age", @@ -200,7 +200,7 @@ "active": true }, { - "fides_key": "user.observed", + "fides_key": "user.behavior", "organization_fides_key": "default_organization", "tags": null, "name": "Observed Data", @@ -210,7 +210,7 @@ "active": true }, { - "fides_key": "user.profiling", + "fides_key": "user.demographic.profile", "organization_fides_key": "default_organization", "tags": null, "name": "Profiling Data", @@ -220,7 +220,7 @@ "active": true }, { - "fides_key": "user.race", + "fides_key": "user.demographic.race_ethnicity", "organization_fides_key": "default_organization", "tags": null, "name": "Race", @@ -230,7 +230,7 @@ "active": true }, { - "fides_key": "user.religious_belief", + "fides_key": "user.demographic.religious_belief", "organization_fides_key": "default_organization", "tags": null, "name": "Religious Belief", @@ -240,7 +240,7 @@ "active": true }, { - "fides_key": "user.search_history", + "fides_key": "user.behavior.search_history", "organization_fides_key": "default_organization", "tags": null, "name": "Search History", @@ -250,7 +250,7 @@ "active": true }, { - "fides_key": "user.sexual_orientation", + "fides_key": "user.demographic.sexual_orientation", "organization_fides_key": "default_organization", "tags": null, "name": "Sexual Orientation", @@ -300,7 +300,7 @@ "active": true }, { - "fides_key": "user.organization", + "fides_key": "user.contact.organization", "organization_fides_key": "default_organization", "tags": null, "name": "Organization Identifiable Data", @@ -440,7 +440,7 @@ "active": true }, { - "fides_key": "user.date_of_birth", + "fides_key": "user.demographic.date_of_birth", "organization_fides_key": "default_organization", "tags": null, "name": "Date of Birth", @@ -460,7 +460,7 @@ "active": true }, { - "fides_key": "user.financial.account_number", + "fides_key": "user.financial.bank_account", "organization_fides_key": "default_organization", "tags": null, "name": "User Financial Account Number", @@ -470,7 +470,7 @@ "active": true }, { - "fides_key": "user.genetic", + "fides_key": "user.health_and_medical.genetic", "organization_fides_key": "default_organization", "tags": null, "name": "Genetic Data", @@ -550,7 +550,7 @@ "active": true }, { - "fides_key": "user.political_opinion", + "fides_key": "user.demographic.political_opinion", "organization_fides_key": "default_organization", "tags": null, "name": "Political Opinion", diff --git a/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json b/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json index 9823eb7d7f4..e8ef7a17be3 100644 --- a/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json +++ b/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json @@ -105,7 +105,7 @@ "active": true }, { - "fides_key": "improve.system", + "fides_key": "functional.service.improve", "organization_fides_key": "default_organization", "tags": null, "name": "System", diff --git a/clients/fides-js/__tests__/lib/cookie.test.ts b/clients/fides-js/__tests__/lib/cookie.test.ts index 407aec55e3d..cd5e9195163 100644 --- a/clients/fides-js/__tests__/lib/cookie.test.ts +++ b/clients/fides-js/__tests__/lib/cookie.test.ts @@ -206,7 +206,7 @@ describe("makeConsentDefaultsLegacy", () => { { cookieKeys: ["default_true"], default: true, - fidesDataUseKey: "improve.system", + fidesDataUseKey: "functional.service.improve", }, { cookieKeys: ["default_false"], diff --git a/data/dataset/bigquery_example_test_dataset.yml b/data/dataset/bigquery_example_test_dataset.yml index bd33b94d432..f9fc7eeee8f 100644 --- a/data/dataset/bigquery_example_test_dataset.yml +++ b/data/dataset/bigquery_example_test_dataset.yml @@ -136,7 +136,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/email_dataset.yml b/data/dataset/email_dataset.yml index 52152068c1f..c829e8a4ea7 100644 --- a/data/dataset/email_dataset.yml +++ b/data/dataset/email_dataset.yml @@ -61,4 +61,4 @@ dataset: fides_meta: identity: email - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] diff --git a/data/dataset/example_test_dataset.invalid b/data/dataset/example_test_dataset.invalid index deff6212e12..46e5235876c 100644 --- a/data/dataset/example_test_dataset.invalid +++ b/data/dataset/example_test_dataset.invalid @@ -136,7 +136,7 @@ dataset: field: address.id direction: to * name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] * name: code data_categories: [user.financial] * name: customer_id diff --git a/data/dataset/example_test_datasets.yml b/data/dataset/example_test_datasets.yml index 446dd20f7e8..898d61bc710 100644 --- a/data/dataset/example_test_datasets.yml +++ b/data/dataset/example_test_datasets.yml @@ -129,7 +129,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id @@ -338,7 +338,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/mariadb_example_test_dataset.yml b/data/dataset/mariadb_example_test_dataset.yml index 109a7c22998..5e3c90f08f4 100644 --- a/data/dataset/mariadb_example_test_dataset.yml +++ b/data/dataset/mariadb_example_test_dataset.yml @@ -129,7 +129,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/mongo_example_test_dataset.yml b/data/dataset/mongo_example_test_dataset.yml index f3234cb4765..3971b6481dd 100644 --- a/data/dataset/mongo_example_test_dataset.yml +++ b/data/dataset/mongo_example_test_dataset.yml @@ -17,11 +17,11 @@ dataset: field: customer.id direction: from - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fides_meta: data_type: string - name: birthday - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fides_meta: data_type: string - name: workplace_info @@ -189,7 +189,7 @@ dataset: fides_meta: data_type: string - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fides_meta: data_type: string - name: employee @@ -245,7 +245,7 @@ dataset: - name: billing_address_id data_categories: [system.operations] - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fides_meta: references: - dataset: mongo_test diff --git a/data/dataset/mssql_example_test_dataset.yml b/data/dataset/mssql_example_test_dataset.yml index 81d7e48d9ed..661c6007270 100644 --- a/data/dataset/mssql_example_test_dataset.yml +++ b/data/dataset/mssql_example_test_dataset.yml @@ -129,7 +129,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/mysql_example_test_dataset.yml b/data/dataset/mysql_example_test_dataset.yml index 17ff0db8983..dfd8ae090ab 100644 --- a/data/dataset/mysql_example_test_dataset.yml +++ b/data/dataset/mysql_example_test_dataset.yml @@ -129,7 +129,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/postgres_example_test_dataset.yml b/data/dataset/postgres_example_test_dataset.yml index 9978c88ff86..e519a75008b 100644 --- a/data/dataset/postgres_example_test_dataset.yml +++ b/data/dataset/postgres_example_test_dataset.yml @@ -136,7 +136,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/redshift_example_test_dataset.yml b/data/dataset/redshift_example_test_dataset.yml index 1994219c34f..9794f86bb3b 100644 --- a/data/dataset/redshift_example_test_dataset.yml +++ b/data/dataset/redshift_example_test_dataset.yml @@ -136,7 +136,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/snowflake_example_test_dataset.yml b/data/dataset/snowflake_example_test_dataset.yml index e44ad966884..da137236938 100644 --- a/data/dataset/snowflake_example_test_dataset.yml +++ b/data/dataset/snowflake_example_test_dataset.yml @@ -140,7 +140,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/dataset/timebase_example_test_dataset.yml b/data/dataset/timebase_example_test_dataset.yml index ac0f95594d5..ffd57a7c676 100644 --- a/data/dataset/timebase_example_test_dataset.yml +++ b/data/dataset/timebase_example_test_dataset.yml @@ -136,7 +136,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/data/saas/dataset/adobe_campaign_dataset.yml b/data/saas/dataset/adobe_campaign_dataset.yml index 298babe3b29..9a7fc42b5eb 100644 --- a/data/saas/dataset/adobe_campaign_dataset.yml +++ b/data/saas/dataset/adobe_campaign_dataset.yml @@ -11,11 +11,11 @@ dataset: data_type: string primary_key: True - name: age - data_categories: [user.non_specific_age] + data_categories: [user.demographic.age_range] fidesops_meta: data_type: integer - name: birthDate - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] - name: blackList data_categories: [system.operations] fidesops_meta: @@ -93,7 +93,7 @@ dataset: fidesops_meta: data_type: string - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: href @@ -207,11 +207,11 @@ dataset: fidesops_meta: data_type: string - name: age - data_categories: [user.non_specific_age] + data_categories: [user.demographic.age_range] fidesops_meta: data_type: integer - name: birthDate - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] - name: blackList data_categories: [system.operations] fidesops_meta: @@ -295,7 +295,7 @@ dataset: fidesops_meta: data_type: string - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: href diff --git a/data/saas/dataset/amplitude_dataset.yml b/data/saas/dataset/amplitude_dataset.yml index 9efee217b57..d5d58f8d902 100644 --- a/data/saas/dataset/amplitude_dataset.yml +++ b/data/saas/dataset/amplitude_dataset.yml @@ -108,7 +108,7 @@ dataset: data_type: object fields: - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: interests @@ -246,7 +246,7 @@ dataset: data_type: object fields: - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: interests diff --git a/data/saas/dataset/braze_dataset.yml b/data/saas/dataset/braze_dataset.yml index 94908ae06b8..0078bdbe84d 100644 --- a/data/saas/dataset/braze_dataset.yml +++ b/data/saas/dataset/braze_dataset.yml @@ -36,7 +36,7 @@ dataset: fidesops_meta: data_type: string - name: dob - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: country @@ -52,7 +52,7 @@ dataset: fidesops_meta: data_type: string - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: phone diff --git a/data/saas/dataset/mailchimp_dataset.yml b/data/saas/dataset/mailchimp_dataset.yml index 8dbbb427644..05e3e45a2e7 100644 --- a/data/saas/dataset/mailchimp_dataset.yml +++ b/data/saas/dataset/mailchimp_dataset.yml @@ -96,7 +96,7 @@ dataset: fidesops_meta: data_type: string - name: BIRTHDAY - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: ip_signup diff --git a/data/saas/dataset/outreach_dataset.yml b/data/saas/dataset/outreach_dataset.yml index 85cdfd9e672..90f6318779a 100644 --- a/data/saas/dataset/outreach_dataset.yml +++ b/data/saas/dataset/outreach_dataset.yml @@ -78,7 +78,7 @@ dataset: - name: createdAt data_categories: [system.operations] - name: dateOfBirth - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: degree @@ -108,7 +108,7 @@ dataset: fidesops_meta: data_type: string - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: githubUrl diff --git a/data/saas/dataset/salesforce_dataset.yml b/data/saas/dataset/salesforce_dataset.yml index 3ee5f19d9e6..44e0489f5da 100644 --- a/data/saas/dataset/salesforce_dataset.yml +++ b/data/saas/dataset/salesforce_dataset.yml @@ -186,7 +186,7 @@ dataset: fidesops_meta: data_type: string - name: Birthdate - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] - name: Description data_categories: [system.operations] fidesops_meta: diff --git a/data/saas/dataset/segment_dataset.yml b/data/saas/dataset/segment_dataset.yml index e85c2375b86..9fea86e4911 100644 --- a/data/saas/dataset/segment_dataset.yml +++ b/data/saas/dataset/segment_dataset.yml @@ -72,7 +72,7 @@ dataset: fidesops_meta: data_type: string - name: age - data_categories: [user.non_specific_age] + data_categories: [user.demographic.age_range] - name: avatar data_categories: [system.operations] - name: createdAt @@ -88,7 +88,7 @@ dataset: fidesops_meta: data_type: string - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: id diff --git a/data/saas/dataset/sentry_dataset.yml b/data/saas/dataset/sentry_dataset.yml index 6dfcda5443f..172bb08333f 100644 --- a/data/saas/dataset/sentry_dataset.yml +++ b/data/saas/dataset/sentry_dataset.yml @@ -88,7 +88,7 @@ dataset: fidesops_meta: data_type: string - name: username - data_categories: [user.credentials] + data_categories: [user.authorization.credentials] fidesops_meta: data_type: string - name: email @@ -548,7 +548,7 @@ dataset: fidesops_meta: data_type: string - name: username - data_categories: [user.credentials] + data_categories: [user.authorization.credentials] fidesops_meta: data_type: string - name: email diff --git a/data/saas/dataset/square_dataset.yml b/data/saas/dataset/square_dataset.yml index d45b3d6bca3..4cf16a4285d 100644 --- a/data/saas/dataset/square_dataset.yml +++ b/data/saas/dataset/square_dataset.yml @@ -75,7 +75,7 @@ dataset: fidesops_meta: data_type: string - name: birthday - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: segment_ids diff --git a/data/saas/dataset/stripe_dataset.yml b/data/saas/dataset/stripe_dataset.yml index b89f7f7470a..5fd4707b320 100644 --- a/data/saas/dataset/stripe_dataset.yml +++ b/data/saas/dataset/stripe_dataset.yml @@ -272,7 +272,7 @@ dataset: - name: installments data_categories: [system.operations] - name: last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: network @@ -342,7 +342,7 @@ dataset: - name: cvc_check data_categories: [system.operations] - name: dynamic_last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: exp_month @@ -356,7 +356,7 @@ dataset: - name: id data_categories: [system.operations] - name: last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: name @@ -687,7 +687,7 @@ dataset: - name: generated_from data_categories: [system.operations] - name: last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: networks @@ -741,11 +741,11 @@ dataset: - name: fingerprint data_categories: [system.operations] - name: last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: routing_number - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: status @@ -798,7 +798,7 @@ dataset: - name: cvc_check data_categories: [system.operations] - name: dynamic_last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: exp_month @@ -810,7 +810,7 @@ dataset: - name: funding data_categories: [system.operations] - name: last4 - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fidesops_meta: data_type: string - name: name diff --git a/data/saas/dataset/vend_dataset.yml b/data/saas/dataset/vend_dataset.yml index 56f6c583d29..9fe9c3e3cf2 100644 --- a/data/saas/dataset/vend_dataset.yml +++ b/data/saas/dataset/vend_dataset.yml @@ -49,11 +49,11 @@ dataset: data_type: integer - name: note - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: date_of_birth - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: company_name diff --git a/data/saas/dataset/yotpo_reviews_dataset.yml b/data/saas/dataset/yotpo_reviews_dataset.yml index 661e33c57d7..6cdc4676b05 100644 --- a/data/saas/dataset/yotpo_reviews_dataset.yml +++ b/data/saas/dataset/yotpo_reviews_dataset.yml @@ -27,7 +27,7 @@ dataset: fidesops_meta: data_type: string - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fidesops_meta: data_type: string - name: account_created_at diff --git a/demo_resources/demo_system.yml b/demo_resources/demo_system.yml index 39a2308ee2b..df41eda9b5c 100644 --- a/demo_resources/demo_system.yml +++ b/demo_resources/demo_system.yml @@ -20,7 +20,7 @@ system: data_categories: - user.contact - user.device.cookie_id - data_use: improve.system + data_use: functional.service.improve data_subjects: - customer data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 823acc508e8..660c0b4a580 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -1,6234 +1,5432 @@ { - "info": { - "_postman_id": "90f366bb-e733-4cca-93b7-75eb599871c2", - "name": "Fides", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" - }, - "item": [ - { - "name": "Minimum API calls to create an Access Privacy Request", - "item": [ - { - "name": "Get Root Client Token", - "event": [ - { - "listen": "test", - "script": { - "exec": [ - "pm.collectionVariables.set(\"client_token\", pm.response.json()[\"access_token\"])" - ], - "type": "text/javascript" - } - } - ], - "request": { - "auth": { - "type": "noauth" - }, - "method": "POST", - "header": [], - "body": { - "mode": "formdata", - "formdata": [ - { - "key": "client_id", - "value": "{{OAUTH_ROOT_CLIENT_ID}}", - "type": "text" - }, - { - "key": "client_secret", - "value": "{{OAUTH_ROOT_CLIENT_SECRET}}", - "type": "text" - } - ] - }, - "url": { - "raw": "{{host}}/oauth/token", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "token" - ] - } - }, - "response": [] - }, - { - "name": "Create Client", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{root_client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"consent:read\",\n \"current-privacy-preference:read\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"messaging:create_or_update\",\n \"messaging:read\",\n \"messaging:delete\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-experience:create\",\n \"privacy-experience:read\",\n \"privacy-experience:update\",\n \"privacy-notice:create\",\n \"privacy-notice:read\",\n \"privacy-notice:update\",\n \"privacy-preference-history:read\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"privacy-request-notifications:create_or_update\",\n \"privacy-request-notifications:read\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"system_manager:delete\",\n \"system_manager:update\",\n \"system_manager:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\",\n \"user-permission:create\",\n \"user-permission:update\"\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/oauth/client", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "client" - ] - } - }, - "response": [] - }, - { - "name": "Get Client Token", - "request": { - "auth": { - "type": "noauth" - }, - "method": "POST", - "header": [], - "body": { - "mode": "formdata", - "formdata": [ - { - "key": "client_id", - "value": "{{client_id}}", - "type": "text" - }, - { - "key": "client_secret", - "value": "{{client_secret}}", - "type": "text" - } - ] - }, - "url": { - "raw": "{{host}}/oauth/token", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "token" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Storage Local Storage Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"key\": \"{{storage_key}}\",\n \"name\": \"My Local Storage Location\",\n \"type\": \"local\",\n \"details\": {\n \"naming\": \"request_id\"\n }\n }\n]\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/storage/config/", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "config", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Policies", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [ - { - "key": "Authorization", - "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..uqJeGbx19RfR_K44pRaB3A.4c8dNV4oFwfn6lmyJ33s8ldlO0n2jn4XXKrfSoOjmBAlrvSzydoUdvEiLvA_JnQ6Aym89I8hC-ncx-QAtVeuLgUsxAYoUFPJhIAxEr2A3M3r.bfUXQcTP6vrShKLCnOHBWw", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": " [\n {\n \"name\": \"Primary Policy\",\n \"key\": \"{{policy_key}}\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Access Rule", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [ - { - "warning": "This is a duplicate header and will be overridden by the Authorization header generated by Postman.", - "key": "Authorization", - "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..4Qwlk9cjkxFs7Fiql7FIKA.kl0Z7k6Ucuv17PP4nf4PatMXRNHkZNmgsjOSpl-_6w7A60c_IetPOXt-Q6anc219kx2pkhHScdbwK-cv7Nyg5oHTLsVnKssFtkuUFFdd0MhviFGQDoOkPspF524nk1UKC8ZF09nhdN2pJw-OlH9pVaUQWQRO4-CSwTKnLgJN9bqnp_b2US9qhSPGavCHYs63nkh0b_j3sImf_-zGQns4RP702zicDPXJfgRVOtjCUpnB6N_gtaqwoGSOvcQYPC7uwei0KPhYJcNui3YA3enIT4Oxo4wsIXj56XIn9t8HmVmsB3fxVcb-xG36jUMzmXlJmnkzoWyaGvJMwURAB9oJEzH_Z7Vk_vYEQtpOpm4rcc02g8f3N7WI-X8Cm0609vQ1nAGeGNpFym-P8_QgRfs0bSWCLP99ZmUscaGkFu0M-NHw7OGT0uuYgcnLlQrPfnPTG-ZQgP4HubNLV79G1A0UWXDPBGrybz658H6mhJzudFe02rI1M94-UD5J8ysdkvnkvpaDwTKEMToVBFwo2k5r8PWCd4u2fZxo01OIkqfOg-ZgpXXWCYJ6tSbS1rvskyIBsjW_aCiiqMGWH3A0zghTvvr52BukQOpI2A--OFP_7LvTt5DzmspvrgZWhYbRdXiZfN0uulAbPyWWzjZ0pltYPeIfLURjzKOBdt5a7NnN95B1p0Z_siirDukzOCrWcWEZZlzHqx30z6-fF0mjDfOq6iklDXc9raEjaknzO1CiZdFDmHr3ldoxBR3vqVnkH0rSHth2hg.RhVYCotD220myIOGLF2NyA", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "[{\n \"storage_destination_key\": \"{{storage_key}}\",\n \"name\": \"My User Data Access\",\n \"action_type\": \"access\",\n \"key\": \"{{rule_key}}\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/rule/", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "rule", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Rule Targets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [ - { - "key": "Authorization", - "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..uqJeGbx19RfR_K44pRaB3A.4c8dNV4oFwfn6lmyJ33s8ldlO0n2jn4XXKrfSoOjmBAlrvSzydoUdvEiLvA_JnQ6Aym89I8hC-ncx-QAtVeuLgUsxAYoUFPJhIAxEr2A3M3r.bfUXQcTP6vrShKLCnOHBWw", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "[{\n \"data_category\": \"user\",\n \"key\": \"{{target_key}}\",\n \"name\": \"Provided User Info\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/rule/{{rule_key}}/target", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "rule", - "{{rule_key}}", - "target" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Connection Configs: Postgres", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Application PostgreSQL DB\",\n \"key\": \"{{postgres_key}}\",\n \"connection_type\": \"postgres\",\n \"access\": \"write\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Connection Configs: Mongo", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"name\": \"My Mongo DB\",\n \"key\": \"{{mongo_key}}\",\n \"connection_type\": \"mongodb\",\n \"access\": \"write\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: Postgres", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"host\": \"host.docker.internal\",\n \"port\": 6432,\n \"dbname\": \"postgres_example\",\n \"username\": \"postgres\",\n \"password\": \"postgres\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{postgres_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{postgres_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: Mongo", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"host\": \"mongodb_example\",\n \"defaultauthdb\": \"mongo_test\",\n \"username\": \"mongo_user\",\n \"password\": \"mongo_pass\",\n \"port\": \"27017\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mongo_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mongo_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Postgres CTL Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"name\": \"Postgres Example Test Dataset\",\n \"organization_fides_key\": \"default_organization\",\n \"data_qualifier\": \"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"description\": \"Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ],\n \"primary_key\": true\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"primary_key\": true\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dataset/upsert", - "host": [ - "{{host}}" - ], - "path": [ - "dataset", - "upsert" - ] - } - }, - "response": [] - }, - { - "name": "Create Dataset Config with Postgres CTL Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"ctl_dataset_fides_key\": \"postgres_example\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{postgres_key}}/datasetconfig/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{postgres_key}}", - "datasetconfig", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Mongo CTL Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"fides_key\":\"mongo_test\",\n \"name\":\"Mongo Example Test Dataset\",\n \"description\":\"Example of a Mongo dataset that contains 'details' about customers defined in the 'postgres_example'\",\n \"organization_fides_key\": \"default_organization\",\n \"data_qualifier\": \"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"collections\":[\n {\n \"name\":\"customer_details\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"gender\",\n \"data_categories\":[\n \"user.gender\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.date_of_birth\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"workplace_info\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"employer\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"position\",\n \"data_categories\":[\n \"user.job_title\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"direct_reports\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"emergency_contacts\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"relationship\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"travel_identifiers\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n },\n {\n \"name\":\"comments\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.comment\",\n \"direction\":\"to\"\n }\n ]\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"internal_customer_profile\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_identifiers\",\n \"fields\":[\n {\n \"name\":\"internal_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_feedback.customer_information.internal_customer_id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"derived_emails\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"identity\":\"email\"\n }\n },\n {\n \"name\":\"derived_phone\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"return_all_elements\":true,\n \"identity\":\"phone_number\"\n }\n }\n ]\n },\n {\n \"name\":\"derived_interests\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"customer_feedback\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_information\",\n \"fields\":[\n {\n \"name\":\"email\",\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"internal_customer_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"rating\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"date\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flights\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"passenger_information\",\n \"fields\":[\n {\n \"name\":\"passenger_ids\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_details.travel_identifiers\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"full_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flight_no\"\n },\n {\n \"name\":\"date\"\n },\n {\n \"name\":\"pilots\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"plane\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n }\n ]\n },\n {\n \"name\":\"conversations\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"thread\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"chat_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.account_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"employee\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.pilots\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"aircraft\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"planes\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.plane\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"model\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"payment_card\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"billing_address_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.account_number\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.ccn\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"code\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"preferred\",\n \"data_categories\":[\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\":\"rewards\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"owner\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\",\n \"return_all_elements\":true\n },\n \"fields\":[\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"internal_customer_profile.customer_identifiers.derived_phone\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"shopper_name\"\n }\n ]\n },\n {\n \"name\":\"points\",\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"expiration_date\"\n }\n ]\n }\n ]\n}]\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dataset/upsert", - "host": [ - "{{host}}" - ], - "path": [ - "dataset", - "upsert" - ] - } - }, - "response": [] - }, - { - "name": "Create Dataset Config with Mongo CTL Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"fides_key\": \"mongo_test\",\n \"ctl_dataset_fides_key\": \"mongo_test\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mongo_key}}/datasetconfig/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mongo_key}}", - "datasetconfig", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create Access Privacy Requests", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [ - { - "key": "Authorization", - "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..a3XF2oIZ6zDlSSg9Oj8cvw.30l5gg7yd0vyadLKDu_BV30i9lqJevKDigTtiCXHLaz11BWgTKE_juHqvGChllgyaVrXR-VxWxC2zEauatKrnJFChae4vOeXsmM6ojWu1ICKXiBGR8j51htqFA_w5IBEUvd27oMAC108-DRinsjAL52lFbW47z2oDff5lnvJay_cvBbRxIaCr38obiXuuzFbV4Gbgka9BVUp0gre78Io79k.qMdtGCySVHC2U1vpQp3GLg", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "[\n {\n \"requested_at\": \"2021-08-30T16:09:37.359Z\",\n \"identity\": {\"email\": \"customer-1@example.com\"},\n \"policy_key\": \"{{policy_key}}\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request" - ] - } - }, - "response": [] - } - ], - "description": "These API calls step you through how to set up all the resources in Fidesops to execute access privacy requests in your system." - }, - { - "name": "Calls to create an Erasure Request (Assume Basic Configs already set up from Access Request section)", - "item": [ - { - "name": "Create/Update separate Policy", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": " [\n {\n \"name\": \"Separate Policy\",\n \"key\": \"{{separate_policy_key}}\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update an Erasure Rule", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"name\": \"My User Data Erasure\",\n \"action_type\": \"erasure\",\n \"key\": \"{{erasure_rule_key}}\",\n \"masking_strategy\": {\n \"strategy\": \"null_rewrite\",\n \"configuration\": {}\n }\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{separate_policy_key}}/rule/", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{separate_policy_key}}", - "rule", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Erasure Rule Targets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"data_category\": \"user\",\n \"key\": \"{{mask_target_key}}\",\n \"name\": \"Derived User Info\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{separate_policy_key}}/rule/{{erasure_rule_key}}/target", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{separate_policy_key}}", - "rule", - "{{erasure_rule_key}}", - "target" - ] - } - }, - "response": [] - }, - { - "name": "Create Erasure Privacy Request", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"requested_at\": \"2021-08-30T16:09:37.359Z\",\n \"identity\": {\"email\": \"customer-1@example.com\"},\n \"policy_key\": \"{{separate_policy_key}}\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request" - ] - } - }, - "response": [] - } - ], - "description": "These API calls step you through how to set up all the resources in Fidesops to execute erasure privacy requests in your system.\n\nIt assumes you've already run through the API requests in the \"Access\" section so database connections, storage, and annotated datasets are already configured." - }, - { - "name": "Privacy Request Management", - "item": [ - { - "name": "Preview a Privacy Request", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\"postgres_example\"]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/preview/", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "preview", - "" - ] - } - }, - "response": [] - }, - { - "name": "Check the Status of my Privacy Request", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/privacy-request/?id={{privacy_request_id}}&verbose=True", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "" - ], - "query": [ - { - "key": "id", - "value": "{{privacy_request_id}}" - }, - { - "key": "verbose", - "value": "True" - } - ] - } - }, - "response": [] - }, - { - "name": "Get all Privacy Request Execution Logs", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/log", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "log" - ] - } - }, - "response": [] - }, - { - "name": "Approve a Privacy Request", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\"request_ids\": [\"{{privacy_request_id}}\", \"bad_id\"]}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/administrate/approve", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "administrate", - "approve" - ] - } - }, - "response": [] - }, - { - "name": "Deny a Privacy Request", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\"request_ids\": [\"{{privacy_request_id}}\", \"bad_id\"], \"reason\": \"{{denial_reason}}\"}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/administrate/deny", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "administrate", - "deny" - ] - } - }, - "response": [] - }, - { - "name": "Verify Identity", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\"code\": \"123456\"}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/verify", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "verify" - ] - } - }, - "response": [] - }, - { - "name": "Get Privacy Request Notification Info", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/privacy-request/notification/", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "notification", - "" - ] - } - }, - "response": [] - }, - { - "name": "Insert/Update Privacy Request Notification Info", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"email_addresses\": [\"test@email.com\", \"another@email.com\"],\n \"notify_after_failures\": 5\n}" - }, - "url": { - "raw": "{{host}}/privacy-request/notification/", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "notification", - "" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Other Useful API calls", - "item": [ - { - "name": "Get Allowed Scopes", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/oauth/scope", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "scope" - ] - } - }, - "response": [] - }, - { - "name": "Get Client Scopes", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{root_client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/oauth/client/{{client_id}}/scope", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "client", - "{{client_id}}", - "scope" - ] - } - }, - "response": [] - }, - { - "name": "Set Client Scopes", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{root_client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\"\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/oauth/client/{{client_id}}/scope", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "client", - "{{client_id}}", - "scope" - ] - } - }, - "response": [] - }, - { - "name": "Mask Value", - "request": { - "auth": { - "type": "noauth" - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": " {\n \"values\": [\"test@example.com\"],\n \"masking_strategy\": {\n \"strategy\": \"random_string_rewrite\",\n \"configuration\": {\n \"length\": 20,\n \"format_preservation\": {\n \"suffix\": \"@masked.com\"\n }\n \n }\n }\n}\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/masking/mask", - "host": [ - "{{host}}" - ], - "path": [ - "masking", - "mask" - ] - } - }, - "response": [] - }, - { - "name": "Get all Available Masking Strategies", - "request": { - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/masking/strategy", - "host": [ - "{{host}}" - ], - "path": [ - "masking", - "strategy" - ] - } - }, - "response": [] - }, - { - "name": "Get Policy Details", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [ - { - "key": "Authorization", - "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..uqJeGbx19RfR_K44pRaB3A.4c8dNV4oFwfn6lmyJ33s8ldlO0n2jn4XXKrfSoOjmBAlrvSzydoUdvEiLvA_JnQ6Aym89I8hC-ncx-QAtVeuLgUsxAYoUFPJhIAxEr2A3M3r.bfUXQcTP6vrShKLCnOHBWw", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "" - ] - } - }, - "response": [] - }, - { - "name": "Get all Policies", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [ - { - "key": "Authorization", - "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..SSKjokoVH8kO1zcqYNL8Qw.jd3RcnzftWVOT5nK2LmeMryGNQGdiFiduJiL679XetrPf0U2ppH51w9MLI3MVUKtDFnoqYXzEARxBoXXESmH2WkHZhZ99m3-vARKDisbqKqC5LGmdWmkxPu8AGdF2Nm6P9jC4kvtpK0NE40qiyixzdOz9A2ZQ5ta-9COT1Fhe8QRHN-rHrYcaPy3Kw2B6aFsroblMuVzRtzoic8RMg513Pc.PtuBEFEfAiq-ALL4SfWD0A", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy" - ] - } - }, - "response": [] - }, - { - "name": "Get all Connections", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection", - "host": [ - "{{host}}" - ], - "path": [ - "connection" - ] - } - }, - "response": [] - }, - { - "name": "Get Connection Detail", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [ - { - "key": "Content-Type", - "value": "application/json", - "type": "text" - } - ], - "body": { - "mode": "file", - "file": {} - }, - "url": { - "raw": "{{host}}/connection/{{postgres_key}}", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{postgres_key}}" - ] - } - }, - "response": [] - }, - { - "name": "Get Storage Configs", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/storage/config", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "config" - ] - } - }, - "response": [] - }, - { - "name": "Get Storage Config Status", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/storage/default/status", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "default", - "status" - ] - } - }, - "response": [] - }, - { - "name": "Get Storage Config Detail", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/storage/config/{{storage_key}}", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "config", - "{{storage_key}}" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update S3 Storage Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"name\": \"My Access Request Upload Bucket\",\n \"key\": \"{{S3_BUCKET_KEY}}\",\n \"type\": \"s3\",\n \"format\": \"csv\",\n \"details\": {\n \"bucket\": \"test_bucket\"\n }\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/storage/config/", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "config", - "" - ] - } - }, - "response": [] - }, - { - "name": "Add S3 Storage Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\"aws_access_key_id\": \"{{AWS_ACCESS_KEY_ID}}\", \"aws_secret_access_key\":\"{{AWS_SECRET_ACCESS_KEY}}\"}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/storage/config/{{S3_BUCKET_KEY}}/secret/", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "config", - "{{S3_BUCKET_KEY}}", - "secret", - "" - ] - } - }, - "response": [] - }, - { - "name": "Test a Postgres Database Connection", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{postgres_key}}/test", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{postgres_key}}", - "test" - ] - } - }, - "response": [] - }, - { - "name": "Test Uploading to my Storage Destination", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"storage_key\": \"{{storage_key}}\",\n \"data\": {\n \"email\": \"ashley@example.com\",\n \"address\": \"123 main ST, Asheville NC\",\n \"zip codes\": [12345, 54321]\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/storage/{{privacy_request_id}}", - "host": [ - "{{host}}" - ], - "path": [ - "storage", - "{{privacy_request_id}}" - ] - } - }, - "response": [] - }, - { - "name": "Validate Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"postgres_example\",\n \"name\": \"Postgres Example Test Dataset\",\n \"description\": \"Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{postgres_key}}/validate_dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{postgres_key}}", - "validate_dataset" - ] - } - }, - "response": [] - }, - { - "name": "Get all Connection Datasets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{postgres_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{postgres_key}}", - "dataset" - ] - } - }, - "response": [] - }, - { - "name": "Get Dataset Detail", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{mongo_key}}/dataset/mongo_test", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mongo_key}}", - "dataset", - "mongo_test" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update HTTPS Connection Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"My Webhook Connection Configuration\",\n \"key\": \"{{http_connection_key}}\",\n \"connection_type\": \"https\",\n \"access\": \"read\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update HTTPS Connection Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"url\": \"example.com\",\n \"authorization\": \"test_authorization\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{http_connection_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{http_connection_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Set Pre-Execution Webhooks", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"one_way\",\n \"key\": \"{{pre_webhook_one}}\",\n \"name\": \"Wake up Snowflake DB Webhook\"\n },\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"two_way\",\n \"key\": \"{{pre_webhook_two}}\",\n \"name\": \"Prep Systems Webhook\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "pre_execution" - ] - } - }, - "response": [] - }, - { - "name": "Get Pre-Execution Webhook Detail", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution/{{pre_webhook_one}}", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "pre_execution", - "{{pre_webhook_one}}" - ] - } - }, - "response": [] - }, - { - "name": "Update Pre-Execution Webhook", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"direction\": \"two_way\",\n \"order\": 1\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution/{{pre_webhook_one}}", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "pre_execution", - "{{pre_webhook_one}}" - ] - } - }, - "response": [] - }, - { - "name": "Delete Pre-Execution Webhook", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "DELETE", - "header": [], - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution/{{pre_webhook_one}}", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "pre_execution", - "{{pre_webhook_one}}" - ] - } - }, - "response": [] - }, - { - "name": "Set Post-Execution Webhooks", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"one_way\",\n \"key\": \"{{post_webhook_one}}\",\n \"name\": \"Cache Busting Webhook\"\n },\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"one_way\",\n \"key\": \"{{post_webhook_two}}\",\n \"name\": \"Finalize Privacy Request Webhook\"\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "post_execution" - ] - } - }, - "response": [] - }, - { - "name": "Get Post-Execution Webhook Detail", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution/{{post_webhook_one}}", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "post_execution", - "{{post_webhook_one}}" - ] - } - }, - "response": [] - }, - { - "name": "Update Post-Execution Webhook", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"direction\": \"two_way\",\n \"order\": 1\n\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution/{{post_webhook_one}}", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "post_execution", - "{{post_webhook_one}}" - ] - } - }, - "response": [] - }, - { - "name": "Delete Post-Execution Webhook", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "DELETE", - "header": [], - "url": { - "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution/{{post_webhook_two}}", - "host": [ - "{{host}}" - ], - "path": [ - "dsr", - "policy", - "{{policy_key}}", - "webhook", - "post_execution", - "{{post_webhook_two}}" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Postgres Dataset YAML", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [ - { - "key": "Content-Type", - "value": "application/x-yaml", - "type": "text" - } - ], - "body": { - "mode": "raw", - "raw": "dataset:\n - fides_key: bigquery_example_test_dataset\n name: BigQuery Example Test Dataset\n description: Example of a BigQuery dataset containing a variety of related tables like customers, products, addresses, etc.\n collections:\n - name: address\n fields:\n - name: city\n data_categories: [user.contact.address.city]\n - name: house\n data_categories: [user.contact.address.street]\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: state\n data_categories: [user.contact.address.state]\n - name: street\n data_categories: [user.contact.address.street]\n - name: zip\n data_categories: [user.contact.address.postal_code]\n\n - name: customer\n fields:\n - name: address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n - name: created\n data_categories: [system.operations]\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: id\n data_categories: [user.unique_id]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [user.name]\n fidesops_meta:\n data_type: string\n length: 40\n\n - name: employee\n fields:\n - name: address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: id\n data_categories: [user.unique_id]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [user.name]\n fidesops_meta:\n data_type: string\n\n - name: login\n fields:\n - name: customer_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: customer.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: time\n data_categories: [user.sensor]\n\n - name: order\n fields:\n - name: customer_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: customer.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: shipping_address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n\n # order_item\n - name: order_item\n fields:\n - name: order_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: order.id\n direction: from\n - name: product_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: product.id\n direction: to\n - name: quantity\n data_categories: [system.operations]\n\n - name: payment_card\n fields:\n - name: billing_address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n - name: ccn\n data_categories: [user.financial.account_number]\n - name: code\n data_categories: [user.financial]\n - name: customer_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: customer.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [user.financial]\n - name: preferred\n data_categories: [user]\n\n - name: product\n fields:\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [system.operations]\n - name: price\n data_categories: [system.operations]\n\n - name: report\n fields:\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: month\n data_categories: [system.operations]\n - name: name\n data_categories: [system.operations]\n - name: total_visits\n data_categories: [system.operations]\n - name: year\n data_categories: [system.operations]\n\n - name: service_request\n fields:\n - name: alt_email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: closed\n data_categories: [system.operations]\n - name: email\n data_categories: [system.operations]\n fidesops_meta:\n identity: email\n data_type: string\n - name: employee_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: employee.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: opened\n data_categories: [system.operations]\n\n - name: visit\n fields:\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: last_visit\n data_categories: [system.operations]\n", - "options": { - "raw": { - "language": "text" - } - } - }, - "url": { - "raw": "{{host}}/yml/connection/{{postgres_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "yml", - "connection", - "{{postgres_key}}", - "dataset" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "MsSQL", - "item": [ - { - "name": "Create/Update Connection Configs: MsSQL", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Application MsSQL DB\",\n \"key\": \"{{mssql_key}}\",\n \"connection_type\": \"mssql\",\n \"access\": \"read\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: MsSQL", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"host\": \"mssql_example\",\n \"port\": 1433,\n \"dbname\": \"mssql_example\",\n \"username\": \"sa\",\n \"password\": \"Ms_sql1234\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mssql_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mssql_key}}", - "secret" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "MySQL", - "item": [ - { - "name": "Create/Update Connection Configs: MySQL", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Application MySQL DB\",\n \"key\": \"{{mysql_key}}\",\n \"connection_type\": \"mysql\",\n \"access\": \"read\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: MySQL", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"host\": \"mysql_example\",\n \"port\": 3306,\n \"dbname\": \"mysql_example\",\n \"username\": \"mysql_user\",\n \"password\": \"mysql_pw\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mysql_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mysql_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update MySQL Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"fides_key\": \"mysql_example_test_dataset\",\n \"name\": \"MySQL Example Test Dataset\",\n \"description\": \"Example of a MySQL dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mysql_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mysql_key}}", - "dataset" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "MariaDB", - "item": [ - { - "name": "Create/Update Connection Configs: MariaDB", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Application Maria DB\",\n \"key\": \"{{mariadb_key}}\",\n \"connection_type\": \"mariadb\",\n \"access\": \"write\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: MariaDB", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"host\": \"mariadb_example\",\n \"port\": 3306,\n \"dbname\": \"mariadb_example\",\n \"username\": \"mariadb_user\",\n \"password\": \"mariadb_pw\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mariadb_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mariadb_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update MariaDB Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"fides_key\": \"mariadb_example_test_dataset\",\n \"name\": \"MariaDB Example Test Dataset\",\n \"description\": \"Example of a MariaDB dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n }\n ]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{mariadb_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{mariadb_key}}", - "dataset" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "BigQuery", - "item": [ - { - "name": "Create/Update Connection Configs: BigQuery", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Application BigQuery DB\",\n \"key\": \"{{bigquery_key}}\",\n \"connection_type\": \"bigquery\",\n \"access\": \"read\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: BigQuery", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"dataset\": \"\",\n \"keyfile_creds\": {\n \"type\": \"\",\n \"project_id\": \"\",\n \"private_key_id\": \"\",\n \"private_key\": \"\",\n \"client_email\": \"\",\n \"client_id\": \"\",\n \"auth_uri\": \"\",\n \"token_uri\": \"\",\n \"auth_provider_x509_cert_url\": \"\",\n \"client_x509_cert_url\": \"\"\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{bigquery_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{bigquery_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update BigQuery Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"fides_key\": \"bigquery_example_test_dataset\",\n \"name\": \"BigQuery Example Test Dataset\",\n \"description\": \"Example of a BigQuery dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\",\n \"length\": 40\n }\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.account_number\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n }\n ]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{bigquery_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{bigquery_key}}", - "dataset" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Email ConnectionConfig", - "item": [ - { - "name": "Create/Update Connection Configs: BigQuery Copy", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Email ConnectionConfig\",\n \"key\": \"{{email_connection_config_key}}\",\n \"connection_type\": \"email\",\n \"access\": \"read\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: BigQuery Copy", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"to_email\": \"customer-1@example.com\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{email_connection_config_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{email_connection_config_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Email Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"fides_key\":\"email_dataset\",\n \"name\":\"An example of a dataset not automatically accessible\",\n \"description\":\"Example of a email dataset with a collection waiting on postgres input\",\n \"collections\":[\n {\n \"name\":\"daycare_customer\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"first_name\",\n \"data_categories\":[\n \"user.childrens\"\n ]\n },\n {\n \"name\":\"last_name\",\n \"data_categories\":[\n \"user.childrens\"\n ]\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"parent_id\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"email_dataset\",\n \"field\":\"daycare_customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{email_connection_config_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{email_connection_config_key}}", - "dataset" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "SaaS", - "item": [ - { - "name": "Sentry", - "item": [ - { - "name": "Create/Update Sentry Connection Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Sentry Connection Config\",\n \"key\": \"{{sentry_saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"write\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Validate Sentry Saas Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"sentry_connector\",\n \"name\": \"Sentry SaaS Config\",\n \"type\": \"sentry\",\n \"description\": \"A sample schema representing the Sentry connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"host\"\n },\n {\n \"name\": \"access_token\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\"\n },\n \"endpoints\": [\n {\n \"name\": \"organizations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"employees\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations//users/\",\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"organizations.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"projects\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"issues\",\n \"requests\": {\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/api/0/issues//\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"issue_id\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"issues.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{\\\"assignedTo\\\": \\\"\\\"}\"\n },\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///issues/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"assigned:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"user_feedback\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///user-feedback/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\"\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"person\",\n \"after\": [\n \"sentry_connector.projects\"\n ],\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"ignore_errors\": true,\n \"path\": \"/api/0/projects///users/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"email:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/validate_saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "validate_saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Sentry Saas Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"sentry_connector\",\n \"name\": \"Sentry SaaS Config\",\n \"type\": \"sentry\",\n \"description\": \"A sample schema representing the Sentry connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"host\"\n },\n {\n \"name\": \"access_token\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\"\n },\n \"endpoints\": [\n {\n \"name\": \"organizations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"employees\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations//users/\",\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"organizations.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"projects\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"issues\",\n \"requests\": {\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/api/0/issues//\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"issue_id\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"issues.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{\\\"assignedTo\\\": \\\"\\\"}\"\n },\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///issues/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"assigned:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"user_feedback\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///user-feedback/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\"\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"person\",\n \"after\": [\n \"sentry_connector.projects\"\n ],\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"ignore_errors\": true,\n \"path\": \"/api/0/projects///users/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"email:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Update Sentry Saas Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"host\": \"{{sentry_host}}\",\n \"access_token\": \"{{sentry_access_token}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Sentry Saas Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"fides_key\":\"sentry_connector\",\n \"name\":\"Sentry Dataset\",\n \"description\":\"A sample dataset representing the Sentry connector for Fidesops\",\n \"collections\":[\n {\n \"name\":\"organizations\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isEarlyAdopter\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"require2FA\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"requireEmailVerification\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"avatar\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"features\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"employees\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"user\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"username\",\n \"data_categories\":[\n \"user.credentials\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUrl\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isActive\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"hasPasswordAuth\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isManaged\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"dateJoined\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastLogin\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"has2fa\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastActive\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isSuperuser\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isStaff\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"emails\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"is_verified\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n }\n ]\n },\n {\n \"name\":\"avatar\",\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"role\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"roleName\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"pending\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"expired\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"flags\",\n \"fields\":[\n {\n \"name\":\"sso:linked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"sso_invalid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"member-limit:restricted\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n }\n ]\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"inviteStatus\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"inviterName\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"projects\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"projects\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isPublic\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isBookmarked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"color\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"firstEvent\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"firstTransactionEvent\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"hasSessions\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"features\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isInternal\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isMember\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"hasAccess\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"avatar\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"organization\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isEarlyAdopter\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"require2FA\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"requireEmailVerification\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"avatar\",\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"features\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"issues\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"primary_key\":true\n }\n },\n {\n \"name\":\"shareId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"shortId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"title\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"culprit\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"permalink\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"logger\"\n },\n {\n \"name\":\"level\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"statusDetails\"\n },\n {\n \"name\":\"isPublic\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"project\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"metadata\",\n \"fields\":[\n {\n \"name\":\"value\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"filename\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"function\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"display_title_with_tree_label\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n }\n ]\n },\n {\n \"name\":\"numComments\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"assignedTo\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"isBookmarked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isSubscribed\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"subscriptionDetails\",\n \"fields\":[\n {\n \"name\":\"reason\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"hasSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"annotations\"\n },\n {\n \"name\":\"isUnhandled\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"count\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"userCount\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"firstSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"stats\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"24h\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer[]\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"user_feedback\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"eventID\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"comments\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"user\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"hash\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"tagValue\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"identifier\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"username\",\n \"data_categories\":[\n \"user.credentials\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ipAddress\",\n \"data_categories\":[\n \"user.device.ip_address\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUrl\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"event\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"eventID\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"issue\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"shareId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"shortId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"title\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"culprit\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"permalink\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"logger\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"level\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"statusDetails\"\n },\n {\n \"name\":\"isPublic\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"project\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"metadata\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"value\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"type\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"filename\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"function\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"numComments\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"assignedTo\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isBookmarked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isSubscribed\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"subscriptionDetails\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"hasSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"annotations\"\n },\n {\n \"name\":\"isUnhandled\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"count\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"userCount\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"firstSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"person\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"hash\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"tagValue\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"identifier\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"username\",\n \"data_categories\":[\n \"user.credentials\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ipAddress\",\n \"data_categories\":[\n \"user.device.ip_address\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUrl\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "dataset" - ] - } - }, - "response": [] - }, - { - "name": "Sentry Saas Connection Test", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/test", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "test" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Hubspot", - "item": [ - { - "name": "Create/Update Hubspot Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Hubspot Connection Config\",\n \"key\": \"{{hubspot_saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"write\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Validate Hubspot Saas Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"hubspot_connector_example\",\n \"name\": \"Hubspot SaaS Config\",\n \"type\": \"hubspot\",\n \"description\": \"A sample schema representing the Hubspot connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"hapikey\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"query_param\",\n \"configuration\": {\n \"name\": \"hapikey\",\n \"value\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/companies/v2/companies/paged\"\n },\n \"endpoints\": [\n {\n \"name\": \"contacts\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/objects/contacts/search\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"filterGroups\\\": [{ \\\"filters\\\": [{ \\\"value\\\": \\\"\\\", \\\"propertyName\\\": \\\"email\\\", \\\"operator\\\": \\\"EQ\\\" }] }] }\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n },\n \"update\": {\n \"path\": \"/crm/v3/objects/contacts/\",\n \"method\": \"PATCH\",\n \"body\": \"{}\",\n \"param_values\": [\n {\n \"name\": \"contactId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"contacts.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n },\n {\n \"name\": \"owners\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/owners\",\n \"method\": \"GET\",\n \"query_params\": [\n {\n \"name\": \"email\",\n \"value\": \"\"\n },\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n }\n }\n },\n {\n \"name\": \"subscription_preferences\",\n \"requests\": {\n \"read\": {\n \"path\": \"/communication-preferences/v3/status/email/\",\n \"method\": \"GET\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ]\n },\n \"update\": {\n \"path\": \"/communication-preferences/v3/unsubscribe\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"emailAddress\\\": \\\"\\\", \\\"subscriptionId\\\": \\\"\\\", \\\"legalBasis\\\": \\\"LEGITIMATE_INTEREST_CLIENT\\\", \\\"legalBasisExplanation\\\": \\\"At users request, we opted them out\\\" }\",\n \"data_path\": \"subscriptionStatuses\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n },\n {\n \"name\": \"subscriptionId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"subscription_preferences.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"status\",\n \"value\": \"SUBSCRIBED\"\n }\n }\n ]\n }\n }\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{hubspot_saas_key}}/validate_saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{hubspot_saas_key}}", - "validate_saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Hubspot Saas Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"hubspot_connector_example\",\n \"name\": \"Hubspot SaaS Config\",\n \"type\": \"hubspot\",\n \"description\": \"A sample schema representing the Hubspot connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"hapikey\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"query_param\",\n \"configuration\": {\n \"name\": \"hapikey\",\n \"value\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/companies/v2/companies/paged\"\n },\n \"endpoints\": [\n {\n \"name\": \"contacts\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/objects/contacts/search\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"filterGroups\\\": [{ \\\"filters\\\": [{ \\\"value\\\": \\\"\\\", \\\"propertyName\\\": \\\"email\\\", \\\"operator\\\": \\\"EQ\\\" }] }] }\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n },\n \"update\": {\n \"path\": \"/crm/v3/objects/contacts/\",\n \"method\": \"PATCH\",\n \"body\": \"{}\",\n \"param_values\": [\n {\n \"name\": \"contactId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"contacts.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n },\n {\n \"name\": \"owners\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/owners\",\n \"method\": \"GET\",\n \"query_params\": [\n {\n \"name\": \"email\",\n \"value\": \"\"\n },\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n }\n }\n },\n {\n \"name\": \"subscription_preferences\",\n \"requests\": {\n \"read\": {\n \"path\": \"/communication-preferences/v3/status/email/\",\n \"method\": \"GET\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ]\n },\n \"update\": {\n \"path\": \"/communication-preferences/v3/unsubscribe\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"emailAddress\\\": \\\"\\\", \\\"subscriptionId\\\": \\\"\\\", \\\"legalBasis\\\": \\\"LEGITIMATE_INTEREST_CLIENT\\\", \\\"legalBasisExplanation\\\": \\\"At users request, we opted them out\\\" }\",\n \"data_path\": \"subscriptionStatuses\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n },\n {\n \"name\": \"subscriptionId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"subscription_preferences.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"status\",\n \"value\": \"SUBSCRIBED\"\n }\n }\n ]\n }\n }\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{hubspot_saas_key}}/saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{hubspot_saas_key}}", - "saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Update Hubspot Saas Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"domain\": \"{{hubspot_domain}}\",\n \"hapikey\": \"{{hubspot_api_key}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{hubspot_saas_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{hubspot_saas_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Hubspot Saas Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n{\n \"fides_key\": \"hubspot_connector_example\",\n \"name\": \"Hubspot Dataset\",\n \"description\": \"A sample dataset representing the Hubspot connector for Fidesops\",\n \"collections\": [\n {\n \"name\": \"contacts\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true,\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"properties\",\n \"fidesops_meta\": {\n \"data_type\": \"object\"\n },\n \"fields\": [\n {\n \"name\": \"company\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"createdate\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"firstname\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"lastmodifieddate\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"lastname\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"phone\",\n \"data_categories\": [\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"website\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"createdAt\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"updatedAt\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"archived\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"boolean\"\n }\n }\n ]\n },\n {\n \"name\": \"owners\",\n \"fields\": [\n {\n \"name\": \"firstName\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"lastName\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"createdAt\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"archived\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"boolean\"\n }\n },\n {\n \"name\": \"teams\",\n \"fidesops_meta\": {\n \"data_type\": \"object[]\"\n },\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true,\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"userId\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"integer\"\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"updatedAt\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"subscription_preferences\",\n \"fields\": [\n {\n \"name\": \"recipient\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"subscriptionStatuses\",\n \"fidesops_meta\": {\n \"data_type\": \"object[]\"\n },\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true,\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"description\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"status\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"sourceOfStatus\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"preferenceGroupName\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"legalBasis\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"legalBasisExplanation\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n }\n ]\n }\n ]\n}\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{hubspot_saas_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{hubspot_saas_key}}", - "dataset" - ] - } - }, - "response": [] - }, - { - "name": "Hubspot Saas Connection Test", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{hubspot_saas_key}}/test", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{hubspot_saas_key}}", - "test" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Segment", - "item": [ - { - "name": "Create/Update Segment Connection Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Segment Connection Config\",\n \"key\": \"{{segment_saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"write\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Validate Segment Saas Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"segment_connector_example\",\n \"name\": \"Segment SaaS Config\",\n \"type\": \"segment\",\n \"description\": \"A sample schema representing the Segment connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"personas_domain\"\n },\n {\n \"name\": \"workspace\"\n },\n {\n \"name\": \"access_token\"\n },\n {\n \"name\": \"namespace_id\"\n },\n {\n \"name\": \"access_secret\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/v1beta/workspaces/\"\n },\n \"endpoints\": [\n {\n \"name\": \"segment_user\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles/user_id:/metadata\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"track_events\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//events\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"traits\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//traits\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 17\n }\n ],\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"traits\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"external_ids\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//external_ids\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n }\n ],\n \"data_protection_request\": {\n \"method\": \"POST\",\n \"path\": \"/v1beta/workspaces//regulations\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"workspace_name\",\n \"connector_param\": \"workspace\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"body\": \"{\\\"regulation_type\\\": \\\"Suppress_With_Delete\\\", \\\"attributes\\\": {\\\"name\\\": \\\"userId\\\", \\\"values\\\": [\\\"\\\"]}}\",\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/validate_saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "validate_saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Segment Saas Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"segment_connector_example\",\n \"name\": \"Segment SaaS Config\",\n \"type\": \"segment\",\n \"description\": \"A sample schema representing the Segment connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"personas_domain\"\n },\n {\n \"name\": \"workspace\"\n },\n {\n \"name\": \"access_token\"\n },\n {\n \"name\": \"namespace_id\"\n },\n {\n \"name\": \"access_secret\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/v1beta/workspaces/\"\n },\n \"endpoints\": [\n {\n \"name\": \"segment_user\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles/user_id:/metadata\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"track_events\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//events\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"traits\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//traits\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 17\n }\n ],\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"traits\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"external_ids\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//external_ids\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n }\n ],\n \"data_protection_request\": {\n \"method\": \"POST\",\n \"path\": \"/v1beta/workspaces//regulations\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"workspace_name\",\n \"connector_param\": \"workspace\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"body\": \"{\\\"regulation_type\\\": \\\"Suppress_With_Delete\\\", \\\"attributes\\\": {\\\"name\\\": \\\"userId\\\", \\\"values\\\": [\\\"\\\"]}}\",\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n }\n }\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{segment_saas_key}}/saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{segment_saas_key}}", - "saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Update Segment Saas Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"domain\": \"{{segment_domain}}\",\n \"personas_domain\": \"{{segment_personas_domain}}\",\n \"workspace\": \"{{segment_workspace}}\",\n \"access_token\": \"{{segment_access_token}}\",\n \"namespace_id\": \"{{segment_namespace_id}}\",\n \"access_secret\": \"{{segment_access_secret}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{segment_saas_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{segment_saas_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update Segment Saas Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n{\n \"fides_key\": \"segment_connector_example\",\n \"name\": \"Segment Dataset\",\n \"description\": \"A sample dataset representing the Segment connector for Fidesops\",\n \"collections\": [\n {\n \"name\": \"segment_user\",\n \"fields\": [\n {\n \"name\": \"segment_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n }\n ]\n },\n {\n \"name\": \"track_events\",\n \"fields\": [\n {\n \"name\": \"external_ids\",\n \"fields\": [\n {\n \"name\": \"collection\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"type\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n }\n ]\n },\n {\n \"name\": \"context\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"library\",\n \"fields\": [\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"version\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"type\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"source_id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"message_id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"timestamp\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"properties\",\n \"fields\": [\n {\n \"name\": \"accountType\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"plan\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"event\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"related\",\n \"fields\": [\n {\n \"name\": \"users\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"cursor\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"traits\",\n \"fields\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"country\",\n \"data_categories\": [\n \"user.contact.address.country\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"postalCode\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"age\",\n \"data_categories\": [\n \"user.non_specific_age\"\n ]\n },\n {\n \"name\": \"avatar\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"createdAt\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"description\",\n \"data_categories\": [\n \"user\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"firstName\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"gender\",\n \"data_categories\": [\n \"user.gender\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"industry\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"lastName\"\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"phone\",\n \"data_categories\": [\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"subscriptionStatus\"\n },\n {\n \"name\": \"title\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"username\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"website\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"external_ids\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"type\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"source_id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"collection\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"created_at\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"encoding\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{segment_saas_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{segment_saas_key}}", - "dataset" - ] - } - }, - "response": [] - }, - { - "name": "Sentry Saas Connection Test", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{segment_saas_key}}/test", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{segment_saas_key}}", - "test" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Mailchimp", - "item": [ - { - "name": "Create/Update Connection Configs: SaaS", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"SaaS Application\",\n \"key\": \"{{saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"read\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Validate SaaS Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"type\": \"mailchimp\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\",\n \"password\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"param_values\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"param_values\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{}\"\n }\n }\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{saas_key}}/validate_saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{saas_key}}", - "validate_saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update SaaS Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"type\": \"mailchimp\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\",\n \"password\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"param_values\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"param_values\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{}\"\n }\n }\n }\n ]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{saas_key}}/saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{saas_key}}", - "saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Update Connection Secrets: SaaS", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"domain\": \"{{mailchimp_domain}}\",\n \"username\": \"{{mailchimp_username}}\",\n \"api_key\": \"{{mailchimp_api_key}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{saas_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{saas_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Create/Update SaaS Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"fides_key\":\"mailchimp_connector_example\",\n \"name\":\"Mailchimp Dataset\",\n \"description\":\"A sample dataset representing the Mailchimp connector for Fidesops\",\n \"collections\":[\n {\n \"name\":\"messages\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"conversation_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"from_label\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"from_email\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"subject\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"message\",\n \"data_categories\":[\n \"user\"\n ]\n },\n {\n \"name\":\"read\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"timestamp\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\":\"conversations\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"campaign_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"list_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"from_email\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"from_label\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"subject\",\n \"data_categories\":[\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\":\"member\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"list_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"email_address\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"unique_email_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"web_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"email_type\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"merge_fields\",\n \"fields\":[\n {\n \"name\":\"FNAME\",\n \"data_categories\":[\n \"user.name\"\n ]\n },\n {\n \"name\":\"LNAME\",\n \"data_categories\":[\n \"user.name\"\n ]\n },\n {\n \"name\":\"ADDRESS\",\n \"fields\":[\n {\n \"name\":\"addr1\",\n \"data_categories\":[\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\":\"addr2\",\n \"data_categories\":[\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\":\"city\",\n \"data_categories\":[\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\":\"state\",\n \"data_categories\":[\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\":\"zip\",\n \"data_categories\":[\n \"user.contact.address.postal_code\"\n ]\n },\n {\n \"name\":\"country\",\n \"data_categories\":[\n \"user.contact.address.country\"\n ]\n }\n ]\n },\n {\n \"name\":\"PHONE\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ]\n },\n {\n \"name\":\"BIRTHDAY\",\n \"data_categories\":[\n \"user.date_of_birth\"\n ]\n }\n ]\n },\n {\n \"name\":\"ip_signup\",\n \"data_categories\":[\n \"user.device.ip_address\"\n ]\n },\n {\n \"name\":\"timestamp_signup\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"ip_opt\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"timestamp_opt\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"language\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"email_client\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"location\",\n \"fields\":[\n {\n \"name\":\"latitude\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"longitude\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"gmtoff\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"dstoff\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"country_code\",\n \"data_categories\":[\n \"user.contact.address.country\"\n ]\n },\n {\n \"name\":\"timezone\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\":\"source\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"tags\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n }\n ]\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{saas_key}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{saas_key}}", - "dataset" - ] - } - }, - "response": [] - }, - { - "name": "Connection Test: SaaS", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{saas_key}}/test", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{saas_key}}", - "test" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Get SaaS Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection/{{sentry_saas_key}}/saas_config", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{sentry_saas_key}}", - "saas_config" - ] - } - }, - "response": [] - }, - { - "name": "Create From Template", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"name\": \"{{saas_connector_type}} connector\",\n \"instance_key\": \"primary_{{saas_connector_type}}\",\n \"secrets\": {\n \"domain\": \"{{mailchimp_domain}}\",\n \"api_key\": \"{{mailchimp_api_key}}\",\n \"username\": \"{{mailchimp_username}}\"\n }\n\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/instantiate/{{saas_connector_type}}", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "instantiate", - "{{saas_connector_type}}" - ] - } - }, - "response": [] - }, - { - "name": "Authorize Oauth Connector", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "http://localhost:8080/api/v1/connection/{{oauth_connector_key}}/authorize", - "protocol": "http", - "host": [ - "localhost" - ], - "port": "8080", - "path": [ - "api", - "v1", - "connection", - "{{oauth_connector_key}}", - "authorize" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Users", - "item": [ - { - "name": "Create User", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"username\": \"{{username}}\",\n \"password\": \"{{password}}\"\n\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user", - "host": [ - "{{host}}" - ], - "path": [ - "user" - ] - } - }, - "response": [] - }, - { - "name": "Get Users", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user", - "host": [ - "{{host}}" - ], - "path": [ - "user" - ] - } - }, - "response": [] - }, - { - "name": "Delete User", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "DELETE", - "header": [], - "url": { - "raw": "{{host}}/user/{{user_id}}", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}" - ] - } - }, - "response": [] - }, - { - "name": "User login", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"username\": \"{{username}}\",\n \"password\": \"{{password}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/login", - "host": [ - "{{host}}" - ], - "path": [ - "login" - ] - } - }, - "response": [] - }, - { - "name": "User logout", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{user_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "url": { - "raw": "{{host}}/logout", - "host": [ - "{{host}}" - ], - "path": [ - "logout" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "DRP", - "item": [ - { - "name": "Exercise", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"meta\": {\"version\": \"0.5\"},\n \"regime\": \"ccpa\",\n \"exercise\": [\n \"access\"\n ],\n \"identity\": \"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20ifQ.4I8XLWnTYp8oMHjN2ypP3Hpg45DIaGNAEmj1QCYONUI\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/drp/exercise", - "host": [ - "{{host}}" - ], - "path": [ - "drp", - "exercise" - ] - } - }, - "response": [] - }, - { - "name": "Status", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/drp/status?request_id={{privacy_request_id}}", - "host": [ - "{{host}}" - ], - "path": [ - "drp", - "status" - ], - "query": [ - { - "key": "request_id", - "value": "{{privacy_request_id}}" - } - ] - } - }, - "response": [] - }, - { - "name": "Data Rights", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/drp/data-rights", - "host": [ - "{{host}}" - ], - "path": [ - "drp", - "data-rights" - ] - } - }, - "response": [] - }, - { - "name": "Revoke request", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"request_id\": \"{{privacy_request_id}}\", \n \"reason\": \"Accidentally submitted\"\n\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/drp/revoke", - "host": [ - "{{host}}" - ], - "path": [ - "drp", - "revoke" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Manual Graph Nodes", - "item": [ - { - "name": "Add Manual Connection Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\"name\": \"Manual connector\",\n \"key\": \"{{manual_connector}}\",\n \"connection_type\": \"manual\",\n \"access\": \"write\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "" - ] - } - }, - "response": [] - }, - { - "name": "Add Manual Dataset", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"fides_key\": \"manual_key\",\n \"name\": \"Manual Dataset\",\n \"description\": \"Example of a Manual dataset with a node waiting on postgres input\",\n \"collections\": [\n {\n \"name\": \"filing_cabinet\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"authorized_user\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"payment_card_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"payment_card.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"storage_unit\",\n \"fields\": [\n {\n \"name\": \"box_id\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\",\n \"identity\": \"email\"\n }\n }\n ]\n }\n ]\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/connection/{{manual_connector}}/dataset", - "host": [ - "{{host}}" - ], - "path": [ - "connection", - "{{manual_connector}}", - "dataset" - ] - } - }, - "response": [] - }, - { - "name": "Check Status", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/privacy-request/?request_id={{privacy_request_id}}&verbose=True", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "" - ], - "query": [ - { - "key": "request_id", - "value": "{{privacy_request_id}}" - }, - { - "key": "verbose", - "value": "True" - } - ] - } - }, - "response": [] - }, - { - "name": "Resume Privacy Request with Manual Filing Cabinet Input", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"id\": 2,\n \"authorized_user\": \"Jiminy Customer\",\n \"customer_id\": 1,\n \"payment_card_id\": \"pay_bbb-bbb\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/manual_input", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "manual_input" - ] - } - }, - "response": [] - }, - { - "name": "Resume Privacy Request with Manual Storage Unit Input", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[{\n \"box_id\": 5,\n \"email\": \"customer-1@example.com\"\n}]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/manual_input", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "manual_input" - ] - } - }, - "response": [] - }, - { - "name": "Resume Erasure Request with Confirmed Masked Row Count", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"row_count\": 5\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/erasure_confirm", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "erasure_confirm" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Restart from Failure", - "item": [ - { - "name": "Restart failed node", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "url": { - "raw": "{{host}}/privacy-request/{{privacy_request_id}}/retry", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-request", - "{{privacy_request_id}}", - "retry" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "ConnectionType", - "item": [ - { - "name": "Get available connectors", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection_type", - "host": [ - "{{host}}" - ], - "path": [ - "connection_type" - ] - } - }, - "response": [] - }, - { - "name": "Get available connectors - consent", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection_type?consent=true", - "host": [ - "{{host}}" - ], - "path": [ - "connection_type" - ], - "query": [ - { - "key": "consent", - "value": "true" - } - ] - } - }, - "response": [] - }, - { - "name": "Get connection secrets schema", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/connection_type/outreach/secret", - "host": [ - "{{host}}" - ], - "path": [ - "connection_type", - "outreach", - "secret" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Messaging Config - Email", - "item": [ - { - "name": "Post Messaging Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "" - ] - } - }, - "response": [] - }, - { - "name": "Patch Messaging Config By Key", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "{{mailgun_config_key}}" - ] - } - }, - "response": [] - }, - { - "name": "Get Messaging Configs", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/messaging/config/", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "" - ] - } - }, - "response": [] - }, - { - "name": "Get Messaging Config By Key", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "{{mailgun_config_key}}" - ] - } - }, - "response": [] - }, - { - "name": "Delete Messaging Config By Key", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "DELETE", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n }\n]\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "{{mailgun_config_key}}" - ] - } - }, - "response": [] - }, - { - "name": "Messaging Config Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"mailgun_api_key\": \"{{mailgun_api_key}}\"\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/{{mailgun_config_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "{{mailgun_config_key}}", - "secret" - ] - } - }, - "response": [] - }, - { - "name": "Test Message", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"email\": \"{{test_email}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/test", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "test" - ] - } - }, - "response": [] - }, - { - "name": "Get Messaging Config Status", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "" - }, - "url": { - "raw": "{{host}}/messaging/default/status", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "default", - "status" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Messaging Config - SMS", - "item": [ - { - "name": "Post Messaging Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "" - ] - } - }, - "response": [] - }, - { - "name": "Patch Messaging Config By Key", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/{{twilio_config_key}}", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "{{twilio_config_key}}" - ] - } - }, - "response": [] - }, - { - "name": "Messaging Config Secrets", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"twilio_account_sid\": \"{{twilio_account_sid}}\",\n \"twilio_auth_token\": \"{{twilio_auth_token}}\",\n \"twilio_messaging_service_sid\": \"{{twilio_messaging_service_id}}\"\n}\n\n", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/messaging/config/{{twilio_config_key}}/secret", - "host": [ - "{{host}}" - ], - "path": [ - "messaging", - "config", - "{{twilio_config_key}}", - "secret" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Identity Verification", - "item": [ - { - "name": "Get Identity Verification Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/id-verification/config/", - "host": [ - "{{host}}" - ], - "path": [ - "id-verification", - "config", - "" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Consent Request", - "item": [ - { - "name": "Create Verification Code for Consent Request", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/consent-request", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request" - ] - } - }, - "response": [] - }, - { - "name": "Verify Code and Return Current Preferences", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"code\": \"{{verification_code}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/consent-request/{{consent_request_id}}/verify", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request", - "{{consent_request_id}}", - "verify" - ] - } - }, - "response": [] - }, - { - "name": "Authenticated Get Consent Preferences", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}" - }, - "url": { - "raw": "{{host}}/consent-request/preferences", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request", - "preferences" - ] - } - }, - "response": [] - }, - { - "name": "Verify Code and Save Preferences", - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"identity\": {\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n },\n \"consent\": [\n {\n \"data_use\": \"{{data_use}}\",\n \"data_use_description\": \"{{data_use_description}}\",\n \"opt_in\": {{opt_in}}\n }\n ],\n \"code\": \"{{verification_code}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/consent-request/{{consent_request_id}}/preferences", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request", - "{{consent_request_id}}", - "preferences" - ] - } - }, - "response": [] - } - ], - "auth": { - "type": "noauth" - }, - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ] - }, - { - "name": "Systems", - "item": [ - { - "name": "Get System", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/system/", - "host": [ - "{{host}}" - ], - "path": [ - "system", - "" - ] - } - }, - "response": [] - }, - { - "name": "Create System", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"personalize\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"test_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/system", - "host": [ - "{{host}}" - ], - "path": [ - "system" - ] - } - }, - "response": [] - }, - { - "name": "Update System", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"marketing.advertising\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"another_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/system?", - "host": [ - "{{host}}" - ], - "path": [ - "system" - ], - "query": [ - { - "key": "", - "value": null - } - ] - } - }, - "response": [] - }, - { - "name": "Delete System", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "DELETE", - "header": [], - "url": { - "raw": "{{host}}/system/test_system", - "host": [ - "{{host}}" - ], - "path": [ - "system", - "test_system" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Roles", - "item": [ - { - "name": "Add role (Create User Permissions - Usually you'll use Update Permissions instead)", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\"roles\": [\"viewer\"]}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user/{{user_id}}/permission", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}", - "permission" - ] - } - }, - "response": [] - }, - { - "name": "Update role (Update User Permissions)", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\"demo_analytics_system\", \"demo_marketing_system\"]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user/{{user_id}}/system-manager", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}", - "system-manager" - ] - } - }, - "response": [] - }, - { - "name": "Get Allowed Roles", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/oauth/role", - "host": [ - "{{host}}" - ], - "path": [ - "oauth", - "role" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "System Managers", - "item": [ - { - "name": "Update Managed Systems", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PUT", - "header": [], - "body": { - "mode": "raw", - "raw": "[\"demo_analytics_system\", \"demo_marketing_system\"]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user/{{user_id}}/system-manager", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}", - "system-manager" - ] - } - }, - "response": [] - }, - { - "name": "Get Systems Managed By User", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user/{{user_id}}/system-manager", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}", - "system-manager" - ] - } - }, - "response": [] - }, - { - "name": "Get Single System Managed By User", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user/{{user_id}}/system-manager/demo_marketing_system", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}", - "system-manager", - "demo_marketing_system" - ] - } - }, - "response": [] - }, - { - "name": "Remove User as System Manager", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "DELETE", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/user/{{user_id}}/system-manager/demo_marketing_system", - "host": [ - "{{host}}" - ], - "path": [ - "user", - "{{user_id}}", - "system-manager", - "demo_marketing_system" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Privacy Notices", - "item": [ - { - "name": "Create Privacy Notices", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"name\":\"Profiling\",\n \"notice_key\": \"profiling\",\n \"regions\":[\n \"us_wv\",\n \"us_mt\"\n ],\n \"description\":\"Making a decision solely by automated means.\",\n \"consent_mechanism\":\"opt_in\",\n \"data_uses\":[\n \"personalize\"\n ],\n \"enforcement_level\":\"system_wide\",\n \"has_gpc_flag\":false,\n \"displayed_in_overlay\":true\n },\n {\n \"name\":\"Essential\",\n \"notice_key\": \"essential\",\n \"regions\":[\n \"us_ak\"\n ],\n \"description\":\"Notify the user about data processing activities that are essential to your services' functionality. Typically consent is not required for this.\",\n \"consent_mechanism\":\"notice_only\",\n \"data_uses\":[\n \"essential.service\"\n ],\n \"enforcement_level\":\"system_wide\",\n \"displayed_in_overlay\":true\n },\n {\n \"name\":\"Advertising\",\n \"notice_key\": \"advertising\",\n \"regions\":[\n \"us_nc\"\n ],\n \"description\":\"Sample advertising notice\",\n \"consent_mechanism\":\"opt_out\",\n \"data_uses\":[\n \"marketing.advertising\"\n ],\n \"enforcement_level\":\"system_wide\",\n \"displayed_in_privacy_center\":true\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-notice", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-notice" - ] - } - }, - "response": [] - }, - { - "name": "Update Privacy Notice", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "[\n {\n \"id\": \"{{privacy_notice_id}}\",\n \"data_uses\": [\n \"third_party_sharing\"\n ]\n }\n]", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-notice", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-notice" - ] - } - }, - "response": [] - }, - { - "name": "Get Privacy Notices", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-notice?systems_applicable=false&show_disabled=true", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-notice" - ], - "query": [ - { - "key": "systems_applicable", - "value": "false" - }, - { - "key": "show_disabled", - "value": "true" - } - ] - } - }, - "response": [] - }, - { - "name": "Get Privacy Notices By Data Use", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-notice-by-data-use", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-notice-by-data-use" - ] - } - }, - "response": [] - }, - { - "name": "Get Single Privacy Notice", - "protocolProfileBehavior": { - "disableBodyPruning": true - }, - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "body": { - "mode": "raw", - "raw": "", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-notice/{{privacy_notice_id}}", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-notice", - "{{privacy_notice_id}}" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "Privacy Experiences and Configs", - "item": [ - { - "name": "Privacy Experience List", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/privacy-experience/?region=us_ca", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-experience", - "" - ], - "query": [ - { - "key": "region", - "value": "us_ca" - } - ] - } - }, - "response": [] - }, - { - "name": "Experience Config", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"accept_button_label\": \"Accept all\",\n \"acknowledge_button_label\": \"Acknowledge\",\n \"banner_enabled\": \"enabled_where_required\",\n \"component\": \"overlay\",\n \"description\": \"On this page you can opt in and out of these data uses cases\",\n \"disabled\": false,\n \"privacy_preferences_link_label\": \"Manage preferences\",\n \"privacy_policy_link_label\": \"View our privacy policy\",\n \"privacy_policy_url\": \"example.com/privacy\",\n \"regions\": [\"us_nc\"],\n \"reject_button_label\": \"Reject all\",\n \"save_button_label\": \"Save\",\n \"title\": \"Manage your consent\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/experience-config", - "host": [ - "{{host}}" - ], - "path": [ - "experience-config" - ] - } - }, - "response": [] - }, - { - "name": "Experience Config Update", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"accept_button_label\": \"Accept\",\n \"reject_button_label\": \"Reject\",\n \"save_button_label\": \"Save all\",\n \"title\": \"Here is where you manage your consent!\",\n \"regions\": [\"us_ca\", \"de\"]\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/experience-config/{{experience-config-id}}", - "host": [ - "{{host}}" - ], - "path": [ - "experience-config", - "{{experience-config-id}}" - ] - } - }, - "response": [] - }, - { - "name": "Get Experience Configs", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/experience-config?component=overlay&show_disabled=False", - "host": [ - "{{host}}" - ], - "path": [ - "experience-config" - ], - "query": [ - { - "key": "component", - "value": "overlay" - }, - { - "key": "show_disabled", - "value": "False" - } - ] - } - }, - "response": [] - }, - { - "name": "Get Experience Config Detail", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/experience-config/{{experience-config-id}}", - "host": [ - "{{host}}" - ], - "path": [ - "experience-config", - "{{experience-config-id}}" - ] - } - }, - "response": [] - } - ] - }, - { - "name": "PrivacyPreferences", - "item": [ - { - "name": "Create Verification Code for Consent Request", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/consent-request", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request" - ] - } - }, - "response": [] - }, - { - "name": "Verify Code and Return Current Privacy Preferences", - "request": { - "method": "POST", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"code\": \"{{verification_code}}\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/consent-request/{{consent_request_id}}/verify-for-privacy-preferences", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request", - "{{consent_request_id}}", - "verify-for-privacy-preferences" - ] - } - }, - "response": [] - }, - { - "name": "Verify Code and Save Privacy Preferences", - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"browser_identity\": {\n \"ga_client_id\": \"UA-XXXXXXXXX\",\n \"ljt_readerID\": \"test_sovrn_id\"\n },\n \"code\": \"{{verification_code}}\",\n \"preferences\": [{\n \"privacy_notice_history_id\": \"{{privacy_notice_history_id}}\",\n \"preference\": \"opt_out\"\n }],\n \"request_origin\": \"privacy_center\",\n \"url_recorded\": \"example.com/privacy_center\",\n \"user_agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24\",\n \"user_geography\": \"us_ca\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/consent-request/{{consent_request_id}}/privacy-preferences", - "host": [ - "{{host}}" - ], - "path": [ - "consent-request", - "{{consent_request_id}}", - "privacy-preferences" - ] - } - }, - "response": [] - }, - { - "name": "Save Privacy Preferences for Device Id", - "request": { - "method": "PATCH", - "header": [], - "body": { - "mode": "raw", - "raw": "{\n \"browser_identity\": {\n \"ga_client_id\": \"UA-XXXXXXXXX\",\n \"ljt_readerID\": \"test_sovrn_id\",\n \"fides_user_device_id\": \"{{fides_user_device_id}}\"\n },\n \"preferences\": [{\n \"privacy_notice_history_id\": \"{{privacy_notice_history_id}}\",\n \"preference\": \"opt_out\"\n }],\n \"user_geography\": \"us_ca\",\n \"privacy_experience_id\": \"{{privacy_experience_id}}\",\n \"method\": \"button\"\n}", - "options": { - "raw": { - "language": "json" - } - } - }, - "url": { - "raw": "{{host}}/privacy-preferences", - "host": [ - "{{host}}" - ], - "path": [ - "privacy-preferences" - ] - } - }, - "response": [] - }, - { - "name": "Get Historical Preferences", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/historical-privacy-preferences", - "host": [ - "{{host}}" - ], - "path": [ - "historical-privacy-preferences" - ] - } - }, - "response": [] - }, - { - "name": "Get Current Preferences", - "request": { - "auth": { - "type": "bearer", - "bearer": [ - { - "key": "token", - "value": "{{client_token}}", - "type": "string" - } - ] - }, - "method": "GET", - "header": [], - "url": { - "raw": "{{host}}/current-privacy-preferences", - "host": [ - "{{host}}" - ], - "path": [ - "current-privacy-preferences" - ] - } - }, - "response": [] - } - ] - } - ], - "event": [ - { - "listen": "prerequest", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - }, - { - "listen": "test", - "script": { - "type": "text/javascript", - "exec": [ - "" - ] - } - } - ], - "variable": [ - { - "key": "host", - "value": "http://0.0.0.0:8080/api/v1" - }, - { - "key": "OAUTH_ROOT_CLIENT_ID", - "value": "" - }, - { - "key": "OAUTH_ROOT_CLIENT_SECRET", - "value": "" - }, - { - "key": "root_client_token", - "value": "" - }, - { - "key": "client_id", - "value": "" - }, - { - "key": "client_secret", - "value": "" - }, - { - "key": "client_token", - "value": "" - }, - { - "key": "policy_key", - "value": "my_primary_policy" - }, - { - "key": "rule_key", - "value": "my_access_rule" - }, - { - "key": "target_key", - "value": "my_user_info" - }, - { - "key": "postgres_key", - "value": "app_postgres_db" - }, - { - "key": "mssql_key", - "value": "app_mssql_db" - }, - { - "key": "mysql_key", - "value": "app_mysql_db" - }, - { - "key": "mongo_key", - "value": "app_mongo_db" - }, - { - "key": "mariadb_key", - "value": "app_mariadb_db", - "type": "string" - }, - { - "key": "bigquery_key", - "value": "app_bigquery_db", - "type": "string" - }, - { - "key": "privacy_request_id", - "value": "" - }, - { - "key": "denial_reason", - "value": "" - }, - { - "key": "storage_key", - "value": "my_local_storage" - }, - { - "key": "AWS_ACCESS_KEY_ID", - "value": "" - }, - { - "key": "AWS_SECRET_ACCESS_KEY", - "value": "" - }, - { - "key": "S3_BUCKET_KEY", - "value": "my_access_request_upload_bucket" - }, - { - "key": "separate_policy_key", - "value": "my_separate_policy" - }, - { - "key": "erasure_rule_key", - "value": "my_erasure_rule" - }, - { - "key": "mask_target_key", - "value": "my_mask_target" - }, - { - "key": "http_connection_key", - "value": "my_http_connection" - }, - { - "key": "pre_webhook_one", - "value": "wake_up_snowflake_db" - }, - { - "key": "pre_webhook_two", - "value": "prep_systems_webhook" - }, - { - "key": "post_webhook_one", - "value": "cache_busting_webhook" - }, - { - "key": "post_webhook_two", - "value": "finalize_privacy_request" - }, - { - "key": "saas_key", - "value": "saas_key", - "type": "string" - }, - { - "key": "mailchimp_domain", - "value": "", - "type": "string" - }, - { - "key": "mailchimp_username", - "value": "", - "type": "string" - }, - { - "key": "mailchimp_api_key", - "value": "", - "type": "string" - }, - { - "key": "username", - "value": "" - }, - { - "key": "password", - "value": "" - }, - { - "key": "user_id", - "value": "" - }, - { - "key": "user_token", - "value": "" - }, - { - "key": "sentry_saas_key", - "value": "sentry_saas_key" - }, - { - "key": "sentry_host", - "value": "" - }, - { - "key": "sentry_access_token", - "value": "" - }, - { - "key": "hubspot_domain", - "value": "" - }, - { - "key": "hubspot_api_key", - "value": "" - }, - { - "key": "hubspot_saas_key", - "value": "hubspot_saas_key" - }, - { - "key": "segment_saas_key", - "value": "segment_saas_key" - }, - { - "key": "segment_domain", - "value": "" - }, - { - "key": "segment_personas_domain", - "value": "" - }, - { - "key": "segment_workspace", - "value": "" - }, - { - "key": "segment_access_token", - "value": "" - }, - { - "key": "segment_namespace_id", - "value": "" - }, - { - "key": "segment_access_secret", - "value": "" - }, - { - "key": "manual_connector", - "value": "manual_key" - }, - { - "key": "mailgun_config_key", - "value": "my_mailgun_config", - "type": "string" - }, - { - "key": "saas_connector_type", - "value": "mailchimp", - "type": "string" - }, - { - "key": "email_connection_config_key", - "value": "email_connection_config_key", - "type": "string" - }, - { - "key": "mailgun_domain", - "value": "", - "type": "string" - }, - { - "key": "mailgun_api_key", - "value": "", - "type": "string" - }, - { - "key": "manual_webhook_key", - "value": "manual_webhook_key", - "type": "string" - }, - { - "key": "consent_request_id", - "value": "", - "type": "default" - }, - { - "key": "timescale_key", - "value": "", - "type": "string" - }, - { - "key": "phone_number", - "value": "", - "type": "string" - }, - { - "key": "email", - "value": "", - "type": "string" - }, - { - "key": "verification_code", - "value": "", - "type": "string" - }, - { - "key": "twilio_config_key", - "value": "my_twilio_config", - "type": "string" - }, - { - "key": "twilio_account_sid", - "value": "", - "type": "string" - }, - { - "key": "twilio_auth_token", - "value": "", - "type": "string" - }, - { - "key": "twilio_messaging_service_id", - "value": "", - "type": "string" - }, - { - "key": "privacy_notice_id", - "value": "", - "type": "string" - }, - { - "key": "privacy_notice_history_id", - "value": "", - "type": "string" - }, - { - "key": "test_email", - "value": "", - "type": "string" - }, - { - "key": "oauth_connector_key", - "value": "", - "type": "string" - }, - { - "key": "fides_user_device_id", - "value": "", - "type": "string" - }, - { - "key": "privacy_experience_id", - "value": "", - "type": "string" - }, - { - "key": "experience-config-id", - "value": "", - "type": "string" - } - ] -} \ No newline at end of file + "info": { + "_postman_id": "90f366bb-e733-4cca-93b7-75eb599871c2", + "name": "Fides", + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + }, + "item": [ + { + "name": "Minimum API calls to create an Access Privacy Request", + "item": [ + { + "name": "Get Root Client Token", + "event": [ + { + "listen": "test", + "script": { + "exec": [ + "pm.collectionVariables.set(\"client_token\", pm.response.json()[\"access_token\"])" + ], + "type": "text/javascript" + } + } + ], + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "client_id", + "value": "{{OAUTH_ROOT_CLIENT_ID}}", + "type": "text" + }, + { + "key": "client_secret", + "value": "{{OAUTH_ROOT_CLIENT_SECRET}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{host}}/oauth/token", + "host": ["{{host}}"], + "path": ["oauth", "token"] + } + }, + "response": [] + }, + { + "name": "Create Client", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{root_client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"consent:read\",\n \"current-privacy-preference:read\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"messaging:create_or_update\",\n \"messaging:read\",\n \"messaging:delete\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-experience:create\",\n \"privacy-experience:read\",\n \"privacy-experience:update\",\n \"privacy-notice:create\",\n \"privacy-notice:read\",\n \"privacy-notice:update\",\n \"privacy-preference-history:read\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"privacy-request-notifications:create_or_update\",\n \"privacy-request-notifications:read\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"system_manager:delete\",\n \"system_manager:update\",\n \"system_manager:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\",\n \"user-permission:create\",\n \"user-permission:update\"\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/oauth/client", + "host": ["{{host}}"], + "path": ["oauth", "client"] + } + }, + "response": [] + }, + { + "name": "Get Client Token", + "request": { + "auth": { + "type": "noauth" + }, + "method": "POST", + "header": [], + "body": { + "mode": "formdata", + "formdata": [ + { + "key": "client_id", + "value": "{{client_id}}", + "type": "text" + }, + { + "key": "client_secret", + "value": "{{client_secret}}", + "type": "text" + } + ] + }, + "url": { + "raw": "{{host}}/oauth/token", + "host": ["{{host}}"], + "path": ["oauth", "token"] + } + }, + "response": [] + }, + { + "name": "Create/Update Storage Local Storage Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"key\": \"{{storage_key}}\",\n \"name\": \"My Local Storage Location\",\n \"type\": \"local\",\n \"details\": {\n \"naming\": \"request_id\"\n }\n }\n]\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/storage/config/", + "host": ["{{host}}"], + "path": ["storage", "config", ""] + } + }, + "response": [] + }, + { + "name": "Create/Update Policies", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [ + { + "key": "Authorization", + "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..uqJeGbx19RfR_K44pRaB3A.4c8dNV4oFwfn6lmyJ33s8ldlO0n2jn4XXKrfSoOjmBAlrvSzydoUdvEiLvA_JnQ6Aym89I8hC-ncx-QAtVeuLgUsxAYoUFPJhIAxEr2A3M3r.bfUXQcTP6vrShKLCnOHBWw", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": " [\n {\n \"name\": \"Primary Policy\",\n \"key\": \"{{policy_key}}\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy", + "host": ["{{host}}"], + "path": ["dsr", "policy"] + } + }, + "response": [] + }, + { + "name": "Create/Update Access Rule", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [ + { + "warning": "This is a duplicate header and will be overridden by the Authorization header generated by Postman.", + "key": "Authorization", + "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..4Qwlk9cjkxFs7Fiql7FIKA.kl0Z7k6Ucuv17PP4nf4PatMXRNHkZNmgsjOSpl-_6w7A60c_IetPOXt-Q6anc219kx2pkhHScdbwK-cv7Nyg5oHTLsVnKssFtkuUFFdd0MhviFGQDoOkPspF524nk1UKC8ZF09nhdN2pJw-OlH9pVaUQWQRO4-CSwTKnLgJN9bqnp_b2US9qhSPGavCHYs63nkh0b_j3sImf_-zGQns4RP702zicDPXJfgRVOtjCUpnB6N_gtaqwoGSOvcQYPC7uwei0KPhYJcNui3YA3enIT4Oxo4wsIXj56XIn9t8HmVmsB3fxVcb-xG36jUMzmXlJmnkzoWyaGvJMwURAB9oJEzH_Z7Vk_vYEQtpOpm4rcc02g8f3N7WI-X8Cm0609vQ1nAGeGNpFym-P8_QgRfs0bSWCLP99ZmUscaGkFu0M-NHw7OGT0uuYgcnLlQrPfnPTG-ZQgP4HubNLV79G1A0UWXDPBGrybz658H6mhJzudFe02rI1M94-UD5J8ysdkvnkvpaDwTKEMToVBFwo2k5r8PWCd4u2fZxo01OIkqfOg-ZgpXXWCYJ6tSbS1rvskyIBsjW_aCiiqMGWH3A0zghTvvr52BukQOpI2A--OFP_7LvTt5DzmspvrgZWhYbRdXiZfN0uulAbPyWWzjZ0pltYPeIfLURjzKOBdt5a7NnN95B1p0Z_siirDukzOCrWcWEZZlzHqx30z6-fF0mjDfOq6iklDXc9raEjaknzO1CiZdFDmHr3ldoxBR3vqVnkH0rSHth2hg.RhVYCotD220myIOGLF2NyA", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "[{\n \"storage_destination_key\": \"{{storage_key}}\",\n \"name\": \"My User Data Access\",\n \"action_type\": \"access\",\n \"key\": \"{{rule_key}}\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/rule/", + "host": ["{{host}}"], + "path": ["dsr", "policy", "{{policy_key}}", "rule", ""] + } + }, + "response": [] + }, + { + "name": "Create/Update Rule Targets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [ + { + "key": "Authorization", + "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..uqJeGbx19RfR_K44pRaB3A.4c8dNV4oFwfn6lmyJ33s8ldlO0n2jn4XXKrfSoOjmBAlrvSzydoUdvEiLvA_JnQ6Aym89I8hC-ncx-QAtVeuLgUsxAYoUFPJhIAxEr2A3M3r.bfUXQcTP6vrShKLCnOHBWw", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "[{\n \"data_category\": \"user\",\n \"key\": \"{{target_key}}\",\n \"name\": \"Provided User Info\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/rule/{{rule_key}}/target", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "rule", + "{{rule_key}}", + "target" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Connection Configs: Postgres", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application PostgreSQL DB\",\n \"key\": \"{{postgres_key}}\",\n \"connection_type\": \"postgres\",\n \"access\": \"write\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Create/Update Connection Configs: Mongo", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"name\": \"My Mongo DB\",\n \"key\": \"{{mongo_key}}\",\n \"connection_type\": \"mongodb\",\n \"access\": \"write\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: Postgres", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"host.docker.internal\",\n \"port\": 6432,\n \"dbname\": \"postgres_example\",\n \"username\": \"postgres\",\n \"password\": \"postgres\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{postgres_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{postgres_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: Mongo", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"mongodb_example\",\n \"defaultauthdb\": \"mongo_test\",\n \"username\": \"mongo_user\",\n \"password\": \"mongo_pass\",\n \"port\": \"27017\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mongo_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{mongo_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update Postgres CTL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"name\": \"Postgres Example Test Dataset\",\n \"organization_fides_key\": \"default_organization\",\n \"data_qualifier\": \"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"description\": \"Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ],\n \"primary_key\": true\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.bank_account\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"primary_key\": true\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dataset/upsert", + "host": ["{{host}}"], + "path": ["dataset", "upsert"] + } + }, + "response": [] + }, + { + "name": "Create Dataset Config with Postgres CTL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"postgres_example\",\n \"ctl_dataset_fides_key\": \"postgres_example\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{postgres_key}}/datasetconfig/", + "host": ["{{host}}"], + "path": ["connection", "{{postgres_key}}", "datasetconfig", ""] + } + }, + "response": [] + }, + { + "name": "Create/Update Mongo CTL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\":\"mongo_test\",\n \"name\":\"Mongo Example Test Dataset\",\n \"description\":\"Example of a Mongo dataset that contains 'details' about customers defined in the 'postgres_example'\",\n \"organization_fides_key\": \"default_organization\",\n \"data_qualifier\": \"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"collections\":[\n {\n \"name\":\"customer_details\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"gender\",\n \"data_categories\":[\n \"user.demographic.gender\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.demographic.date_of_birth\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"workplace_info\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"employer\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"position\",\n \"data_categories\":[\n \"user.job_title\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"direct_reports\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"emergency_contacts\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"relationship\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"travel_identifiers\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n },\n {\n \"name\":\"comments\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.comment\",\n \"direction\":\"to\"\n }\n ]\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"internal_customer_profile\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_identifiers\",\n \"fields\":[\n {\n \"name\":\"internal_id\",\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_feedback.customer_information.internal_customer_id\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"derived_emails\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"identity\":\"email\"\n }\n },\n {\n \"name\":\"derived_phone\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"return_all_elements\":true,\n \"identity\":\"phone_number\"\n }\n }\n ]\n },\n {\n \"name\":\"derived_interests\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"customer_feedback\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"customer_information\",\n \"fields\":[\n {\n \"name\":\"email\",\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"internal_customer_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"rating\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"date\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flights\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"passenger_information\",\n \"fields\":[\n {\n \"name\":\"passenger_ids\",\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"customer_details.travel_identifiers\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"full_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"flight_no\"\n },\n {\n \"name\":\"date\"\n },\n {\n \"name\":\"pilots\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"plane\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n }\n ]\n },\n {\n \"name\":\"conversations\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"thread\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"comment\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"message\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"chat_name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.bank_account\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"employee\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"identity\":\"email\",\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"user.unique_id\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.pilots\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"aircraft\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"planes\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"flights.plane\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"model\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"payment_card\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"billing_address_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"ccn\",\n \"data_categories\":[\n \"user.financial.bank_account\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"conversations.thread.ccn\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"code\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.financial\"\n ]\n },\n {\n \"name\":\"preferred\",\n \"data_categories\":[\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\":\"rewards\",\n \"fields\":[\n {\n \"name\":\"_id\",\n \"fidesops_meta\":{\n \"primary_key\":true,\n \"data_type\":\"object_id\"\n }\n },\n {\n \"name\":\"owner\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\",\n \"return_all_elements\":true\n },\n \"fields\":[\n {\n \"name\":\"phone\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"references\":[\n {\n \"dataset\":\"mongo_test\",\n \"field\":\"internal_customer_profile.customer_identifiers.derived_phone\",\n \"direction\":\"from\"\n }\n ]\n }\n },\n {\n \"name\":\"shopper_name\"\n }\n ]\n },\n {\n \"name\":\"points\",\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"expiration_date\"\n }\n ]\n }\n ]\n}]\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dataset/upsert", + "host": ["{{host}}"], + "path": ["dataset", "upsert"] + } + }, + "response": [] + }, + { + "name": "Create Dataset Config with Mongo CTL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"mongo_test\",\n \"ctl_dataset_fides_key\": \"mongo_test\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mongo_key}}/datasetconfig/", + "host": ["{{host}}"], + "path": ["connection", "{{mongo_key}}", "datasetconfig", ""] + } + }, + "response": [] + }, + { + "name": "Create Access Privacy Requests", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [ + { + "key": "Authorization", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..a3XF2oIZ6zDlSSg9Oj8cvw.30l5gg7yd0vyadLKDu_BV30i9lqJevKDigTtiCXHLaz11BWgTKE_juHqvGChllgyaVrXR-VxWxC2zEauatKrnJFChae4vOeXsmM6ojWu1ICKXiBGR8j51htqFA_w5IBEUvd27oMAC108-DRinsjAL52lFbW47z2oDff5lnvJay_cvBbRxIaCr38obiXuuzFbV4Gbgka9BVUp0gre78Io79k.qMdtGCySVHC2U1vpQp3GLg", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "[\n {\n \"requested_at\": \"2021-08-30T16:09:37.359Z\",\n \"identity\": {\"email\": \"customer-1@example.com\"},\n \"policy_key\": \"{{policy_key}}\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request", + "host": ["{{host}}"], + "path": ["privacy-request"] + } + }, + "response": [] + } + ], + "description": "These API calls step you through how to set up all the resources in Fidesops to execute access privacy requests in your system." + }, + { + "name": "Calls to create an Erasure Request (Assume Basic Configs already set up from Access Request section)", + "item": [ + { + "name": "Create/Update separate Policy", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": " [\n {\n \"name\": \"Separate Policy\",\n \"key\": \"{{separate_policy_key}}\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy", + "host": ["{{host}}"], + "path": ["dsr", "policy"] + } + }, + "response": [] + }, + { + "name": "Create/Update an Erasure Rule", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"name\": \"My User Data Erasure\",\n \"action_type\": \"erasure\",\n \"key\": \"{{erasure_rule_key}}\",\n \"masking_strategy\": {\n \"strategy\": \"null_rewrite\",\n \"configuration\": {}\n }\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{separate_policy_key}}/rule/", + "host": ["{{host}}"], + "path": ["dsr", "policy", "{{separate_policy_key}}", "rule", ""] + } + }, + "response": [] + }, + { + "name": "Create/Update Erasure Rule Targets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"data_category\": \"user\",\n \"key\": \"{{mask_target_key}}\",\n \"name\": \"Derived User Info\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{separate_policy_key}}/rule/{{erasure_rule_key}}/target", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{separate_policy_key}}", + "rule", + "{{erasure_rule_key}}", + "target" + ] + } + }, + "response": [] + }, + { + "name": "Create Erasure Privacy Request", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"requested_at\": \"2021-08-30T16:09:37.359Z\",\n \"identity\": {\"email\": \"customer-1@example.com\"},\n \"policy_key\": \"{{separate_policy_key}}\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request", + "host": ["{{host}}"], + "path": ["privacy-request"] + } + }, + "response": [] + } + ], + "description": "These API calls step you through how to set up all the resources in Fidesops to execute erasure privacy requests in your system.\n\nIt assumes you've already run through the API requests in the \"Access\" section so database connections, storage, and annotated datasets are already configured." + }, + { + "name": "Privacy Request Management", + "item": [ + { + "name": "Preview a Privacy Request", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "[\"postgres_example\"]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/preview/", + "host": ["{{host}}"], + "path": ["privacy-request", "preview", ""] + } + }, + "response": [] + }, + { + "name": "Check the Status of my Privacy Request", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/privacy-request/?id={{privacy_request_id}}&verbose=True", + "host": ["{{host}}"], + "path": ["privacy-request", ""], + "query": [ + { + "key": "id", + "value": "{{privacy_request_id}}" + }, + { + "key": "verbose", + "value": "True" + } + ] + } + }, + "response": [] + }, + { + "name": "Get all Privacy Request Execution Logs", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/log", + "host": ["{{host}}"], + "path": ["privacy-request", "{{privacy_request_id}}", "log"] + } + }, + "response": [] + }, + { + "name": "Approve a Privacy Request", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\"request_ids\": [\"{{privacy_request_id}}\", \"bad_id\"]}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/administrate/approve", + "host": ["{{host}}"], + "path": ["privacy-request", "administrate", "approve"] + } + }, + "response": [] + }, + { + "name": "Deny a Privacy Request", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\"request_ids\": [\"{{privacy_request_id}}\", \"bad_id\"], \"reason\": \"{{denial_reason}}\"}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/administrate/deny", + "host": ["{{host}}"], + "path": ["privacy-request", "administrate", "deny"] + } + }, + "response": [] + }, + { + "name": "Verify Identity", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\"code\": \"123456\"}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/verify", + "host": ["{{host}}"], + "path": ["privacy-request", "{{privacy_request_id}}", "verify"] + } + }, + "response": [] + }, + { + "name": "Get Privacy Request Notification Info", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/privacy-request/notification/", + "host": ["{{host}}"], + "path": ["privacy-request", "notification", ""] + } + }, + "response": [] + }, + { + "name": "Insert/Update Privacy Request Notification Info", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"email_addresses\": [\"test@email.com\", \"another@email.com\"],\n \"notify_after_failures\": 5\n}" + }, + "url": { + "raw": "{{host}}/privacy-request/notification/", + "host": ["{{host}}"], + "path": ["privacy-request", "notification", ""] + } + }, + "response": [] + } + ] + }, + { + "name": "Other Useful API calls", + "item": [ + { + "name": "Get Allowed Scopes", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/oauth/scope", + "host": ["{{host}}"], + "path": ["oauth", "scope"] + } + }, + "response": [] + }, + { + "name": "Get Client Scopes", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{root_client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/oauth/client/{{client_id}}/scope", + "host": ["{{host}}"], + "path": ["oauth", "client", "{{client_id}}", "scope"] + } + }, + "response": [] + }, + { + "name": "Set Client Scopes", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{root_client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\"\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/oauth/client/{{client_id}}/scope", + "host": ["{{host}}"], + "path": ["oauth", "client", "{{client_id}}", "scope"] + } + }, + "response": [] + }, + { + "name": "Mask Value", + "request": { + "auth": { + "type": "noauth" + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": " {\n \"values\": [\"test@example.com\"],\n \"masking_strategy\": {\n \"strategy\": \"random_string_rewrite\",\n \"configuration\": {\n \"length\": 20,\n \"format_preservation\": {\n \"suffix\": \"@masked.com\"\n }\n \n }\n }\n}\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/masking/mask", + "host": ["{{host}}"], + "path": ["masking", "mask"] + } + }, + "response": [] + }, + { + "name": "Get all Available Masking Strategies", + "request": { + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/masking/strategy", + "host": ["{{host}}"], + "path": ["masking", "strategy"] + } + }, + "response": [] + }, + { + "name": "Get Policy Details", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "Bearer eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..uqJeGbx19RfR_K44pRaB3A.4c8dNV4oFwfn6lmyJ33s8ldlO0n2jn4XXKrfSoOjmBAlrvSzydoUdvEiLvA_JnQ6Aym89I8hC-ncx-QAtVeuLgUsxAYoUFPJhIAxEr2A3M3r.bfUXQcTP6vrShKLCnOHBWw", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/", + "host": ["{{host}}"], + "path": ["dsr", "policy", "{{policy_key}}", ""] + } + }, + "response": [] + }, + { + "name": "Get all Policies", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [ + { + "key": "Authorization", + "value": "eyJhbGciOiJkaXIiLCJlbmMiOiJBMjU2R0NNIn0..SSKjokoVH8kO1zcqYNL8Qw.jd3RcnzftWVOT5nK2LmeMryGNQGdiFiduJiL679XetrPf0U2ppH51w9MLI3MVUKtDFnoqYXzEARxBoXXESmH2WkHZhZ99m3-vARKDisbqKqC5LGmdWmkxPu8AGdF2Nm6P9jC4kvtpK0NE40qiyixzdOz9A2ZQ5ta-9COT1Fhe8QRHN-rHrYcaPy3Kw2B6aFsroblMuVzRtzoic8RMg513Pc.PtuBEFEfAiq-ALL4SfWD0A", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy", + "host": ["{{host}}"], + "path": ["dsr", "policy"] + } + }, + "response": [] + }, + { + "name": "Get all Connections", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection", + "host": ["{{host}}"], + "path": ["connection"] + } + }, + "response": [] + }, + { + "name": "Get Connection Detail", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [ + { + "key": "Content-Type", + "value": "application/json", + "type": "text" + } + ], + "body": { + "mode": "file", + "file": {} + }, + "url": { + "raw": "{{host}}/connection/{{postgres_key}}", + "host": ["{{host}}"], + "path": ["connection", "{{postgres_key}}"] + } + }, + "response": [] + }, + { + "name": "Get Storage Configs", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/storage/config", + "host": ["{{host}}"], + "path": ["storage", "config"] + } + }, + "response": [] + }, + { + "name": "Get Storage Config Status", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/storage/default/status", + "host": ["{{host}}"], + "path": ["storage", "default", "status"] + } + }, + "response": [] + }, + { + "name": "Get Storage Config Detail", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/storage/config/{{storage_key}}", + "host": ["{{host}}"], + "path": ["storage", "config", "{{storage_key}}"] + } + }, + "response": [] + }, + { + "name": "Create/Update S3 Storage Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"name\": \"My Access Request Upload Bucket\",\n \"key\": \"{{S3_BUCKET_KEY}}\",\n \"type\": \"s3\",\n \"format\": \"csv\",\n \"details\": {\n \"bucket\": \"test_bucket\"\n }\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/storage/config/", + "host": ["{{host}}"], + "path": ["storage", "config", ""] + } + }, + "response": [] + }, + { + "name": "Add S3 Storage Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\"aws_access_key_id\": \"{{AWS_ACCESS_KEY_ID}}\", \"aws_secret_access_key\":\"{{AWS_SECRET_ACCESS_KEY}}\"}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/storage/config/{{S3_BUCKET_KEY}}/secret/", + "host": ["{{host}}"], + "path": ["storage", "config", "{{S3_BUCKET_KEY}}", "secret", ""] + } + }, + "response": [] + }, + { + "name": "Test a Postgres Database Connection", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{postgres_key}}/test", + "host": ["{{host}}"], + "path": ["connection", "{{postgres_key}}", "test"] + } + }, + "response": [] + }, + { + "name": "Test Uploading to my Storage Destination", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"storage_key\": \"{{storage_key}}\",\n \"data\": {\n \"email\": \"ashley@example.com\",\n \"address\": \"123 main ST, Asheville NC\",\n \"zip codes\": [12345, 54321]\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/storage/{{privacy_request_id}}", + "host": ["{{host}}"], + "path": ["storage", "{{privacy_request_id}}"] + } + }, + "response": [] + }, + { + "name": "Validate Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"postgres_example\",\n \"name\": \"Postgres Example Test Dataset\",\n \"description\": \"Example of a Postgres dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.bank_account\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{postgres_key}}/validate_dataset", + "host": ["{{host}}"], + "path": ["connection", "{{postgres_key}}", "validate_dataset"] + } + }, + "response": [] + }, + { + "name": "Get all Connection Datasets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{postgres_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{postgres_key}}", "dataset"] + } + }, + "response": [] + }, + { + "name": "Get Dataset Detail", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{mongo_key}}/dataset/mongo_test", + "host": ["{{host}}"], + "path": ["connection", "{{mongo_key}}", "dataset", "mongo_test"] + } + }, + "response": [] + }, + { + "name": "Create/Update HTTPS Connection Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"My Webhook Connection Configuration\",\n \"key\": \"{{http_connection_key}}\",\n \"connection_type\": \"https\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update HTTPS Connection Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"url\": \"example.com\",\n \"authorization\": \"test_authorization\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{http_connection_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{http_connection_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Set Pre-Execution Webhooks", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"one_way\",\n \"key\": \"{{pre_webhook_one}}\",\n \"name\": \"Wake up Snowflake DB Webhook\"\n },\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"two_way\",\n \"key\": \"{{pre_webhook_two}}\",\n \"name\": \"Prep Systems Webhook\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "pre_execution" + ] + } + }, + "response": [] + }, + { + "name": "Get Pre-Execution Webhook Detail", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution/{{pre_webhook_one}}", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "pre_execution", + "{{pre_webhook_one}}" + ] + } + }, + "response": [] + }, + { + "name": "Update Pre-Execution Webhook", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"direction\": \"two_way\",\n \"order\": 1\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution/{{pre_webhook_one}}", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "pre_execution", + "{{pre_webhook_one}}" + ] + } + }, + "response": [] + }, + { + "name": "Delete Pre-Execution Webhook", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/pre_execution/{{pre_webhook_one}}", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "pre_execution", + "{{pre_webhook_one}}" + ] + } + }, + "response": [] + }, + { + "name": "Set Post-Execution Webhooks", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"one_way\",\n \"key\": \"{{post_webhook_one}}\",\n \"name\": \"Cache Busting Webhook\"\n },\n {\n \"connection_config_key\": \"{{http_connection_key}}\",\n \"direction\": \"one_way\",\n \"key\": \"{{post_webhook_two}}\",\n \"name\": \"Finalize Privacy Request Webhook\"\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "post_execution" + ] + } + }, + "response": [] + }, + { + "name": "Get Post-Execution Webhook Detail", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution/{{post_webhook_one}}", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "post_execution", + "{{post_webhook_one}}" + ] + } + }, + "response": [] + }, + { + "name": "Update Post-Execution Webhook", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"direction\": \"two_way\",\n \"order\": 1\n\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution/{{post_webhook_one}}", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "post_execution", + "{{post_webhook_one}}" + ] + } + }, + "response": [] + }, + { + "name": "Delete Post-Execution Webhook", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "{{host}}/dsr/policy/{{policy_key}}/webhook/post_execution/{{post_webhook_two}}", + "host": ["{{host}}"], + "path": [ + "dsr", + "policy", + "{{policy_key}}", + "webhook", + "post_execution", + "{{post_webhook_two}}" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Postgres Dataset YAML", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [ + { + "key": "Content-Type", + "value": "application/x-yaml", + "type": "text" + } + ], + "body": { + "mode": "raw", + "raw": "dataset:\n - fides_key: bigquery_example_test_dataset\n name: BigQuery Example Test Dataset\n description: Example of a BigQuery dataset containing a variety of related tables like customers, products, addresses, etc.\n collections:\n - name: address\n fields:\n - name: city\n data_categories: [user.contact.address.city]\n - name: house\n data_categories: [user.contact.address.street]\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: state\n data_categories: [user.contact.address.state]\n - name: street\n data_categories: [user.contact.address.street]\n - name: zip\n data_categories: [user.contact.address.postal_code]\n\n - name: customer\n fields:\n - name: address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n - name: created\n data_categories: [system.operations]\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: id\n data_categories: [user.unique_id]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [user.name]\n fidesops_meta:\n data_type: string\n length: 40\n\n - name: employee\n fields:\n - name: address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: id\n data_categories: [user.unique_id]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [user.name]\n fidesops_meta:\n data_type: string\n\n - name: login\n fields:\n - name: customer_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: customer.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: time\n data_categories: [user.sensor]\n\n - name: order\n fields:\n - name: customer_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: customer.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: shipping_address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n\n # order_item\n - name: order_item\n fields:\n - name: order_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: order.id\n direction: from\n - name: product_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: product.id\n direction: to\n - name: quantity\n data_categories: [system.operations]\n\n - name: payment_card\n fields:\n - name: billing_address_id\n data_categories: [system.operations]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: address.id\n direction: to\n - name: ccn\n data_categories: [user.financial.bank_account]\n - name: code\n data_categories: [user.financial]\n - name: customer_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: customer.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [user.financial]\n - name: preferred\n data_categories: [user]\n\n - name: product\n fields:\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: name\n data_categories: [system.operations]\n - name: price\n data_categories: [system.operations]\n\n - name: report\n fields:\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: month\n data_categories: [system.operations]\n - name: name\n data_categories: [system.operations]\n - name: total_visits\n data_categories: [system.operations]\n - name: year\n data_categories: [system.operations]\n\n - name: service_request\n fields:\n - name: alt_email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: closed\n data_categories: [system.operations]\n - name: email\n data_categories: [system.operations]\n fidesops_meta:\n identity: email\n data_type: string\n - name: employee_id\n data_categories: [user.unique_id]\n fidesops_meta:\n references:\n - dataset: bigquery_example_test_dataset\n field: employee.id\n direction: from\n - name: id\n data_categories: [system.operations]\n fidesops_meta:\n primary_key: True\n - name: opened\n data_categories: [system.operations]\n\n - name: visit\n fields:\n - name: email\n data_categories: [user.contact.email]\n fidesops_meta:\n identity: email\n data_type: string\n - name: last_visit\n data_categories: [system.operations]\n", + "options": { + "raw": { + "language": "text" + } + } + }, + "url": { + "raw": "{{host}}/yml/connection/{{postgres_key}}/dataset", + "host": ["{{host}}"], + "path": ["yml", "connection", "{{postgres_key}}", "dataset"] + } + }, + "response": [] + } + ] + }, + { + "name": "MsSQL", + "item": [ + { + "name": "Create/Update Connection Configs: MsSQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application MsSQL DB\",\n \"key\": \"{{mssql_key}}\",\n \"connection_type\": \"mssql\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: MsSQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"mssql_example\",\n \"port\": 1433,\n \"dbname\": \"mssql_example\",\n \"username\": \"sa\",\n \"password\": \"Ms_sql1234\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mssql_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{mssql_key}}", "secret"] + } + }, + "response": [] + } + ] + }, + { + "name": "MySQL", + "item": [ + { + "name": "Create/Update Connection Configs: MySQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application MySQL DB\",\n \"key\": \"{{mysql_key}}\",\n \"connection_type\": \"mysql\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: MySQL", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"mysql_example\",\n \"port\": 3306,\n \"dbname\": \"mysql_example\",\n \"username\": \"mysql_user\",\n \"password\": \"mysql_pw\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mysql_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{mysql_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update MySQL Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"mysql_example_test_dataset\",\n \"name\": \"MySQL Example Test Dataset\",\n \"description\": \"Example of a MySQL dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.bank_account\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mysql_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mysql_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{mysql_key}}", "dataset"] + } + }, + "response": [] + } + ] + }, + { + "name": "MariaDB", + "item": [ + { + "name": "Create/Update Connection Configs: MariaDB", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application Maria DB\",\n \"key\": \"{{mariadb_key}}\",\n \"connection_type\": \"mariadb\",\n \"access\": \"write\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: MariaDB", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"mariadb_example\",\n \"port\": 3306,\n \"dbname\": \"mariadb_example\",\n \"username\": \"mariadb_user\",\n \"password\": \"mariadb_pw\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mariadb_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{mariadb_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update MariaDB Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"fides_key\": \"mariadb_example_test_dataset\",\n \"name\": \"MariaDB Example Test Dataset\",\n \"description\": \"Example of a MariaDB dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ]\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.bank_account\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"mariadb_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n }\n ]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{mariadb_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{mariadb_key}}", "dataset"] + } + }, + "response": [] + } + ] + }, + { + "name": "BigQuery", + "item": [ + { + "name": "Create/Update Connection Configs: BigQuery", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Application BigQuery DB\",\n \"key\": \"{{bigquery_key}}\",\n \"connection_type\": \"bigquery\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: BigQuery", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"dataset\": \"\",\n \"keyfile_creds\": {\n \"type\": \"\",\n \"project_id\": \"\",\n \"private_key_id\": \"\",\n \"private_key\": \"\",\n \"client_email\": \"\",\n \"client_id\": \"\",\n \"auth_uri\": \"\",\n \"token_uri\": \"\",\n \"auth_provider_x509_cert_url\": \"\",\n \"client_x509_cert_url\": \"\"\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{bigquery_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{bigquery_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update BigQuery Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"fides_key\": \"bigquery_example_test_dataset\",\n \"name\": \"BigQuery Example Test Dataset\",\n \"description\": \"Example of a BigQuery dataset containing a variety of related tables like customers, products, addresses, etc.\",\n \"collections\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\": \"house\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\": \"street\",\n \"data_categories\": [\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\": \"zip\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ]\n }\n ]\n },\n {\n \"name\": \"customer\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"created\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\",\n \"length\": 40\n }\n }\n ]\n },\n {\n \"name\": \"employee\",\n \"fields\": [\n {\n \"name\": \"address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"login\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"time\",\n \"data_categories\": [\n \"user.sensor\"\n ]\n }\n ]\n },\n {\n \"name\": \"orders\",\n \"fields\": [\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"shipping_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"order_item\",\n \"fields\": [\n {\n \"name\": \"order_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"orders.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"product_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"product.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"quantity\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"payment_card\",\n \"fields\": [\n {\n \"name\": \"billing_address_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"address.id\",\n \"direction\": \"to\"\n }\n ]\n }\n },\n {\n \"name\": \"ccn\",\n \"data_categories\": [\n \"user.financial.bank_account\"\n ]\n },\n {\n \"name\": \"code\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.financial\"\n ]\n },\n {\n \"name\": \"preferred\",\n \"data_categories\": [\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\": \"product\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"price\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"report\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"month\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"total_visits\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"year\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"service_request\",\n \"fields\": [\n {\n \"name\": \"alt_email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"closed\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"employee_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"bigquery_example_test_dataset\",\n \"field\": \"employee.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"opened\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"visit\",\n \"fields\": [\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"identity\": \"email\",\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"last_visit\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n }\n ]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{bigquery_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{bigquery_key}}", "dataset"] + } + }, + "response": [] + } + ] + }, + { + "name": "Email ConnectionConfig", + "item": [ + { + "name": "Create/Update Connection Configs: BigQuery Copy", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Email ConnectionConfig\",\n \"key\": \"{{email_connection_config_key}}\",\n \"connection_type\": \"email\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: BigQuery Copy", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"to_email\": \"customer-1@example.com\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{email_connection_config_key}}/secret", + "host": ["{{host}}"], + "path": [ + "connection", + "{{email_connection_config_key}}", + "secret" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Email Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"fides_key\":\"email_dataset\",\n \"name\":\"An example of a dataset not automatically accessible\",\n \"description\":\"Example of a email dataset with a collection waiting on postgres input\",\n \"collections\":[\n {\n \"name\":\"daycare_customer\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"customer_id\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"postgres_example\",\n \"field\":\"customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\":\"children\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"primary_key\":true\n }\n },\n {\n \"name\":\"first_name\",\n \"data_categories\":[\n \"user.childrens\"\n ]\n },\n {\n \"name\":\"last_name\",\n \"data_categories\":[\n \"user.childrens\"\n ]\n },\n {\n \"name\":\"birthday\",\n \"data_categories\":[\n \"user.childrens\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"parent_id\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"references\":[\n {\n \"dataset\":\"email_dataset\",\n \"field\":\"daycare_customer.id\",\n \"direction\":\"from\"\n }\n ]\n }\n }\n ]\n }\n ]\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{email_connection_config_key}}/dataset", + "host": ["{{host}}"], + "path": [ + "connection", + "{{email_connection_config_key}}", + "dataset" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "SaaS", + "item": [ + { + "name": "Sentry", + "item": [ + { + "name": "Create/Update Sentry Connection Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Sentry Connection Config\",\n \"key\": \"{{sentry_saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"write\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Validate Sentry Saas Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"sentry_connector\",\n \"name\": \"Sentry SaaS Config\",\n \"type\": \"sentry\",\n \"description\": \"A sample schema representing the Sentry connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"host\"\n },\n {\n \"name\": \"access_token\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\"\n },\n \"endpoints\": [\n {\n \"name\": \"organizations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"employees\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations//users/\",\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"organizations.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"projects\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"issues\",\n \"requests\": {\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/api/0/issues//\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"issue_id\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"issues.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{\\\"assignedTo\\\": \\\"\\\"}\"\n },\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///issues/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"assigned:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"user_feedback\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///user-feedback/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\"\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"person\",\n \"after\": [\n \"sentry_connector.projects\"\n ],\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"ignore_errors\": true,\n \"path\": \"/api/0/projects///users/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"email:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/validate_saas_config", + "host": ["{{host}}"], + "path": [ + "connection", + "{{sentry_saas_key}}", + "validate_saas_config" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Sentry Saas Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"sentry_connector\",\n \"name\": \"Sentry SaaS Config\",\n \"type\": \"sentry\",\n \"description\": \"A sample schema representing the Sentry connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"host\"\n },\n {\n \"name\": \"access_token\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\"\n },\n \"endpoints\": [\n {\n \"name\": \"organizations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"employees\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/organizations//users/\",\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"organizations.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"projects\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects/\",\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"issues\",\n \"requests\": {\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/api/0/issues//\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"issue_id\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"issues.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{\\\"assignedTo\\\": \\\"\\\"}\"\n },\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///issues/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"assigned:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"user_feedback\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/api/0/projects///user-feedback/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\"\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n },\n {\n \"name\": \"person\",\n \"after\": [\n \"sentry_connector.projects\"\n ],\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"ignore_errors\": true,\n \"path\": \"/api/0/projects///users/\",\n \"grouped_inputs\": [\n \"organization_slug\",\n \"project_slug\",\n \"query\"\n ],\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"email:\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"organization_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.organization.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"project_slug\",\n \"references\": [\n {\n \"dataset\": \"sentry_connector\",\n \"field\": \"projects.slug\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"query\",\n \"identity\": \"email\"\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"headers\",\n \"rel\": \"next\"\n }\n }\n }\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/saas_config", + "host": ["{{host}}"], + "path": ["connection", "{{sentry_saas_key}}", "saas_config"] + } + }, + "response": [] + }, + { + "name": "Update Sentry Saas Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"host\": \"{{sentry_host}}\",\n \"access_token\": \"{{sentry_access_token}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{sentry_saas_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update Sentry Saas Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"fides_key\":\"sentry_connector\",\n \"name\":\"Sentry Dataset\",\n \"description\":\"A sample dataset representing the Sentry connector for Fidesops\",\n \"collections\":[\n {\n \"name\":\"organizations\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isEarlyAdopter\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"require2FA\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"requireEmailVerification\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"avatar\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"features\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n },\n {\n \"name\":\"employees\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"user\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"username\",\n \"data_categories\":[\n \"user.authorization.credentials\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUrl\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isActive\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"hasPasswordAuth\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isManaged\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"dateJoined\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastLogin\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"has2fa\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastActive\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isSuperuser\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isStaff\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"emails\",\n \"fidesops_meta\":{\n \"data_type\":\"object[]\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"is_verified\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n }\n ]\n },\n {\n \"name\":\"avatar\",\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"role\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"roleName\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"pending\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"expired\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"flags\",\n \"fields\":[\n {\n \"name\":\"sso:linked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"sso_invalid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"member-limit:restricted\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n }\n ]\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"inviteStatus\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"inviterName\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"projects\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"projects\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isPublic\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isBookmarked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"color\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"firstEvent\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"firstTransactionEvent\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"hasSessions\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"features\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isInternal\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isMember\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"hasAccess\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"avatar\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"organization\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isEarlyAdopter\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"require2FA\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"requireEmailVerification\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"avatar\",\n \"fields\":[\n {\n \"name\":\"avatarType\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUuid\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"features\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string[]\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"issues\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\",\n \"primary_key\":true\n }\n },\n {\n \"name\":\"shareId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"shortId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"title\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"culprit\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"permalink\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"logger\"\n },\n {\n \"name\":\"level\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"statusDetails\"\n },\n {\n \"name\":\"isPublic\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"project\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"metadata\",\n \"fields\":[\n {\n \"name\":\"value\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"filename\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"function\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"display_title_with_tree_label\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n }\n ]\n },\n {\n \"name\":\"numComments\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"assignedTo\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"isBookmarked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isSubscribed\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"subscriptionDetails\",\n \"fields\":[\n {\n \"name\":\"reason\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"hasSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"annotations\"\n },\n {\n \"name\":\"isUnhandled\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"count\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"userCount\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"firstSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"stats\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"24h\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer[]\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"user_feedback\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"eventID\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"comments\",\n \"data_categories\":[\n \"user\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"user\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"hash\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"tagValue\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"identifier\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"username\",\n \"data_categories\":[\n \"user.authorization.credentials\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ipAddress\",\n \"data_categories\":[\n \"user.device.ip_address\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUrl\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"event\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"eventID\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"issue\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"shareId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"shortId\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"title\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"culprit\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"permalink\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"logger\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"level\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"statusDetails\"\n },\n {\n \"name\":\"isPublic\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"project\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"slug\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"platform\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"type\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"metadata\",\n \"fidesops_meta\":{\n \"data_type\":\"object\"\n },\n \"fields\":[\n {\n \"name\":\"value\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"type\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"filename\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"function\",\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n },\n {\n \"name\":\"numComments\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"assignedTo\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"isBookmarked\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"isSubscribed\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"subscriptionDetails\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"hasSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"annotations\"\n },\n {\n \"name\":\"isUnhandled\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"boolean\"\n }\n },\n {\n \"name\":\"count\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"userCount\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"integer\"\n }\n },\n {\n \"name\":\"firstSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"lastSeen\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n },\n {\n \"name\":\"person\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"hash\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"tagValue\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"identifier\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"username\",\n \"data_categories\":[\n \"user.authorization.credentials\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"email\",\n \"data_categories\":[\n \"user.contact.email\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"user.name\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"ipAddress\",\n \"data_categories\":[\n \"user.device.ip_address\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"dateCreated\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n },\n {\n \"name\":\"avatarUrl\",\n \"data_categories\":[\n \"system.operations\"\n ],\n \"fidesops_meta\":{\n \"data_type\":\"string\"\n }\n }\n ]\n }\n ]\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{sentry_saas_key}}", "dataset"] + } + }, + "response": [] + }, + { + "name": "Sentry Saas Connection Test", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/test", + "host": ["{{host}}"], + "path": ["connection", "{{sentry_saas_key}}", "test"] + } + }, + "response": [] + } + ] + }, + { + "name": "Hubspot", + "item": [ + { + "name": "Create/Update Hubspot Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Hubspot Connection Config\",\n \"key\": \"{{hubspot_saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"write\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Validate Hubspot Saas Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"hubspot_connector_example\",\n \"name\": \"Hubspot SaaS Config\",\n \"type\": \"hubspot\",\n \"description\": \"A sample schema representing the Hubspot connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"hapikey\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"query_param\",\n \"configuration\": {\n \"name\": \"hapikey\",\n \"value\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/companies/v2/companies/paged\"\n },\n \"endpoints\": [\n {\n \"name\": \"contacts\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/objects/contacts/search\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"filterGroups\\\": [{ \\\"filters\\\": [{ \\\"value\\\": \\\"\\\", \\\"propertyName\\\": \\\"email\\\", \\\"operator\\\": \\\"EQ\\\" }] }] }\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n },\n \"update\": {\n \"path\": \"/crm/v3/objects/contacts/\",\n \"method\": \"PATCH\",\n \"body\": \"{}\",\n \"param_values\": [\n {\n \"name\": \"contactId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"contacts.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n },\n {\n \"name\": \"owners\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/owners\",\n \"method\": \"GET\",\n \"query_params\": [\n {\n \"name\": \"email\",\n \"value\": \"\"\n },\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n }\n }\n },\n {\n \"name\": \"subscription_preferences\",\n \"requests\": {\n \"read\": {\n \"path\": \"/communication-preferences/v3/status/email/\",\n \"method\": \"GET\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ]\n },\n \"update\": {\n \"path\": \"/communication-preferences/v3/unsubscribe\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"emailAddress\\\": \\\"\\\", \\\"subscriptionId\\\": \\\"\\\", \\\"legalBasis\\\": \\\"LEGITIMATE_INTEREST_CLIENT\\\", \\\"legalBasisExplanation\\\": \\\"At users request, we opted them out\\\" }\",\n \"data_path\": \"subscriptionStatuses\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n },\n {\n \"name\": \"subscriptionId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"subscription_preferences.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"status\",\n \"value\": \"SUBSCRIBED\"\n }\n }\n ]\n }\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{hubspot_saas_key}}/validate_saas_config", + "host": ["{{host}}"], + "path": [ + "connection", + "{{hubspot_saas_key}}", + "validate_saas_config" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Hubspot Saas Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"hubspot_connector_example\",\n \"name\": \"Hubspot SaaS Config\",\n \"type\": \"hubspot\",\n \"description\": \"A sample schema representing the Hubspot connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"hapikey\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"query_param\",\n \"configuration\": {\n \"name\": \"hapikey\",\n \"value\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/companies/v2/companies/paged\"\n },\n \"endpoints\": [\n {\n \"name\": \"contacts\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/objects/contacts/search\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"filterGroups\\\": [{ \\\"filters\\\": [{ \\\"value\\\": \\\"\\\", \\\"propertyName\\\": \\\"email\\\", \\\"operator\\\": \\\"EQ\\\" }] }] }\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n },\n \"update\": {\n \"path\": \"/crm/v3/objects/contacts/\",\n \"method\": \"PATCH\",\n \"body\": \"{}\",\n \"param_values\": [\n {\n \"name\": \"contactId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"contacts.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ]\n }\n }\n },\n {\n \"name\": \"owners\",\n \"requests\": {\n \"read\": {\n \"path\": \"/crm/v3/owners\",\n \"method\": \"GET\",\n \"query_params\": [\n {\n \"name\": \"email\",\n \"value\": \"\"\n },\n {\n \"name\": \"limit\",\n \"value\": 100\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"unwrap\",\n \"configuration\": {\n \"data_path\": \"results\"\n }\n }\n ],\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"paging.next.link\"\n }\n }\n }\n }\n },\n {\n \"name\": \"subscription_preferences\",\n \"requests\": {\n \"read\": {\n \"path\": \"/communication-preferences/v3/status/email/\",\n \"method\": \"GET\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ]\n },\n \"update\": {\n \"path\": \"/communication-preferences/v3/unsubscribe\",\n \"method\": \"POST\",\n \"body\": \"{ \\\"emailAddress\\\": \\\"\\\", \\\"subscriptionId\\\": \\\"\\\", \\\"legalBasis\\\": \\\"LEGITIMATE_INTEREST_CLIENT\\\", \\\"legalBasisExplanation\\\": \\\"At users request, we opted them out\\\" }\",\n \"data_path\": \"subscriptionStatuses\",\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n },\n {\n \"name\": \"subscriptionId\",\n \"references\": [\n {\n \"dataset\": \"hubspot_connector_example\",\n \"field\": \"subscription_preferences.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"status\",\n \"value\": \"SUBSCRIBED\"\n }\n }\n ]\n }\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{hubspot_saas_key}}/saas_config", + "host": ["{{host}}"], + "path": ["connection", "{{hubspot_saas_key}}", "saas_config"] + } + }, + "response": [] + }, + { + "name": "Update Hubspot Saas Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"domain\": \"{{hubspot_domain}}\",\n \"hapikey\": \"{{hubspot_api_key}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{hubspot_saas_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{hubspot_saas_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update Hubspot Saas Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n{\n \"fides_key\": \"hubspot_connector_example\",\n \"name\": \"Hubspot Dataset\",\n \"description\": \"A sample dataset representing the Hubspot connector for Fidesops\",\n \"collections\": [\n {\n \"name\": \"contacts\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true,\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"properties\",\n \"fidesops_meta\": {\n \"data_type\": \"object\"\n },\n \"fields\": [\n {\n \"name\": \"company\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"createdate\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"firstname\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"lastmodifieddate\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"lastname\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"phone\",\n \"data_categories\": [\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"website\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"createdAt\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"updatedAt\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"archived\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"boolean\"\n }\n }\n ]\n },\n {\n \"name\": \"owners\",\n \"fields\": [\n {\n \"name\": \"firstName\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"lastName\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"createdAt\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"archived\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"boolean\"\n }\n },\n {\n \"name\": \"teams\",\n \"fidesops_meta\": {\n \"data_type\": \"object[]\"\n },\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true,\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"userId\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"integer\"\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"updatedAt\",\n \"data_categories\": [\n \"user.sensor\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"subscription_preferences\",\n \"fields\": [\n {\n \"name\": \"recipient\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"subscriptionStatuses\",\n \"fidesops_meta\": {\n \"data_type\": \"object[]\"\n },\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true,\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"description\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"status\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"sourceOfStatus\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"preferenceGroupName\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"legalBasis\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"legalBasisExplanation\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n }\n ]\n }\n ]\n}\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{hubspot_saas_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{hubspot_saas_key}}", "dataset"] + } + }, + "response": [] + }, + { + "name": "Hubspot Saas Connection Test", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{hubspot_saas_key}}/test", + "host": ["{{host}}"], + "path": ["connection", "{{hubspot_saas_key}}", "test"] + } + }, + "response": [] + } + ] + }, + { + "name": "Segment", + "item": [ + { + "name": "Create/Update Segment Connection Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Segment Connection Config\",\n \"key\": \"{{segment_saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"write\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Validate Segment Saas Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"segment_connector_example\",\n \"name\": \"Segment SaaS Config\",\n \"type\": \"segment\",\n \"description\": \"A sample schema representing the Segment connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"personas_domain\"\n },\n {\n \"name\": \"workspace\"\n },\n {\n \"name\": \"access_token\"\n },\n {\n \"name\": \"namespace_id\"\n },\n {\n \"name\": \"access_secret\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/v1beta/workspaces/\"\n },\n \"endpoints\": [\n {\n \"name\": \"segment_user\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles/user_id:/metadata\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"track_events\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//events\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"traits\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//traits\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 17\n }\n ],\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"traits\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"external_ids\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//external_ids\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n }\n ],\n \"data_protection_request\": {\n \"method\": \"POST\",\n \"path\": \"/v1beta/workspaces//regulations\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"workspace_name\",\n \"connector_param\": \"workspace\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"body\": \"{\\\"regulation_type\\\": \\\"Suppress_With_Delete\\\", \\\"attributes\\\": {\\\"name\\\": \\\"userId\\\", \\\"values\\\": [\\\"\\\"]}}\",\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/validate_saas_config", + "host": ["{{host}}"], + "path": [ + "connection", + "{{sentry_saas_key}}", + "validate_saas_config" + ] + } + }, + "response": [] + }, + { + "name": "Create/Update Segment Saas Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"segment_connector_example\",\n \"name\": \"Segment SaaS Config\",\n \"type\": \"segment\",\n \"description\": \"A sample schema representing the Segment connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"personas_domain\"\n },\n {\n \"name\": \"workspace\"\n },\n {\n \"name\": \"access_token\"\n },\n {\n \"name\": \"namespace_id\"\n },\n {\n \"name\": \"access_secret\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/v1beta/workspaces/\"\n },\n \"endpoints\": [\n {\n \"name\": \"segment_user\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles/user_id:/metadata\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"track_events\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//events\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"traits\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//traits\",\n \"query_params\": [\n {\n \"name\": \"limit\",\n \"value\": 17\n }\n ],\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"traits\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n },\n {\n \"name\": \"external_ids\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/v1/spaces//collections/users/profiles//external_ids\",\n \"param_values\": [\n {\n \"name\": \"namespace_id\",\n \"connector_param\": \"namespace_id\"\n },\n {\n \"name\": \"segment_id\",\n \"references\": [\n {\n \"dataset\": \"segment_connector_example\",\n \"field\": \"segment_user.segment_id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"data\",\n \"pagination\": {\n \"strategy\": \"link\",\n \"configuration\": {\n \"source\": \"body\",\n \"path\": \"cursor.url\"\n }\n },\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\"\n }\n }\n }\n }\n }\n }\n ],\n \"data_protection_request\": {\n \"method\": \"POST\",\n \"path\": \"/v1beta/workspaces//regulations\",\n \"headers\": [\n {\n \"name\": \"Content-Type\",\n \"value\": \"application/json\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"workspace_name\",\n \"connector_param\": \"workspace\"\n },\n {\n \"name\": \"user_id\",\n \"identity\": \"email\"\n }\n ],\n \"body\": \"{\\\"regulation_type\\\": \\\"Suppress_With_Delete\\\", \\\"attributes\\\": {\\\"name\\\": \\\"userId\\\", \\\"values\\\": [\\\"\\\"]}}\",\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"bearer\",\n \"configuration\": {\n \"token\": \"\"\n }\n }\n }\n }\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{segment_saas_key}}/saas_config", + "host": ["{{host}}"], + "path": ["connection", "{{segment_saas_key}}", "saas_config"] + } + }, + "response": [] + }, + { + "name": "Update Segment Saas Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"domain\": \"{{segment_domain}}\",\n \"personas_domain\": \"{{segment_personas_domain}}\",\n \"workspace\": \"{{segment_workspace}}\",\n \"access_token\": \"{{segment_access_token}}\",\n \"namespace_id\": \"{{segment_namespace_id}}\",\n \"access_secret\": \"{{segment_access_secret}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{segment_saas_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{segment_saas_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update Segment Saas Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n{\n \"fides_key\": \"segment_connector_example\",\n \"name\": \"Segment Dataset\",\n \"description\": \"A sample dataset representing the Segment connector for Fidesops\",\n \"collections\": [\n {\n \"name\": \"segment_user\",\n \"fields\": [\n {\n \"name\": \"segment_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n }\n ]\n },\n {\n \"name\": \"track_events\",\n \"fields\": [\n {\n \"name\": \"external_ids\",\n \"fields\": [\n {\n \"name\": \"collection\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"type\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n }\n ]\n },\n {\n \"name\": \"context\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"library\",\n \"fields\": [\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"version\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"type\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"source_id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"message_id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"timestamp\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"properties\",\n \"fields\": [\n {\n \"name\": \"accountType\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"plan\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"event\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"related\",\n \"fields\": [\n {\n \"name\": \"users\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"cursor\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\": \"traits\",\n \"fields\": [\n {\n \"name\": \"address\",\n \"fields\": [\n {\n \"name\": \"city\",\n \"data_categories\": [\n \"user.contact.address.city\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"country\",\n \"data_categories\": [\n \"user.contact.address.country\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"postalCode\",\n \"data_categories\": [\n \"user.contact.address.postal_code\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"state\",\n \"data_categories\": [\n \"user.contact.address.state\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"age\",\n \"data_categories\": [\n \"user.demographic.age_range\"\n ]\n },\n {\n \"name\": \"avatar\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"createdAt\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"description\",\n \"data_categories\": [\n \"user\"\n ]\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"firstName\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"gender\",\n \"data_categories\": [\n \"user.demographic.gender\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"industry\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"lastName\"\n },\n {\n \"name\": \"name\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"phone\",\n \"data_categories\": [\n \"user.contact.phone_number\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"subscriptionStatus\"\n },\n {\n \"name\": \"title\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"username\",\n \"data_categories\": [\n \"user.name\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"website\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n }\n ]\n },\n {\n \"name\": \"external_ids\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"user.unique_id\"\n ]\n },\n {\n \"name\": \"type\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"source_id\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"collection\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"created_at\",\n \"data_categories\": [\n \"system.operations\"\n ]\n },\n {\n \"name\": \"encoding\",\n \"data_categories\": [\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n}\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{segment_saas_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{segment_saas_key}}", "dataset"] + } + }, + "response": [] + }, + { + "name": "Sentry Saas Connection Test", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{segment_saas_key}}/test", + "host": ["{{host}}"], + "path": ["connection", "{{segment_saas_key}}", "test"] + } + }, + "response": [] + } + ] + }, + { + "name": "Mailchimp", + "item": [ + { + "name": "Create/Update Connection Configs: SaaS", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"SaaS Application\",\n \"key\": \"{{saas_key}}\",\n \"connection_type\": \"saas\",\n \"access\": \"read\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Validate SaaS Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"type\": \"mailchimp\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\",\n \"password\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"param_values\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"param_values\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{}\"\n }\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{saas_key}}/validate_saas_config", + "host": ["{{host}}"], + "path": ["connection", "{{saas_key}}", "validate_saas_config"] + } + }, + "response": [] + }, + { + "name": "Create/Update SaaS Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"fides_key\": \"mailchimp_connector_example\",\n \"name\": \"Mailchimp SaaS Config\",\n \"type\": \"mailchimp\",\n \"description\": \"A sample schema representing the Mailchimp connector for Fidesops\",\n \"version\": \"0.0.1\",\n \"connector_params\": [\n {\n \"name\": \"domain\"\n },\n {\n \"name\": \"username\"\n },\n {\n \"name\": \"api_key\"\n }\n ],\n \"client_config\": {\n \"protocol\": \"https\",\n \"host\": \"\",\n \"authentication\": {\n \"strategy\": \"basic\",\n \"configuration\": {\n \"username\": \"\",\n \"password\": \"\"\n }\n }\n },\n \"test_request\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/lists\"\n },\n \"endpoints\": [\n {\n \"name\": \"messages\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations//messages\",\n \"param_values\": [\n {\n \"name\": \"conversation_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"conversations.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"data_path\": \"conversation_messages\",\n \"postprocessors\": [\n {\n \"strategy\": \"filter\",\n \"configuration\": {\n \"field\": \"from_email\",\n \"value\": {\n \"identity\": \"email\"\n }\n }\n }\n ]\n }\n }\n },\n {\n \"name\": \"conversations\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/conversations\",\n \"query_params\": [\n {\n \"name\": \"count\",\n \"value\": 1000\n },\n {\n \"name\": \"offset\",\n \"value\": 0\n }\n ],\n \"param_values\": [\n {\n \"name\": \"placeholder\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"conversations\",\n \"pagination\": {\n \"strategy\": \"offset\",\n \"configuration\": {\n \"incremental_param\": \"offset\",\n \"increment_by\": 1000,\n \"limit\": 10000\n }\n }\n }\n }\n },\n {\n \"name\": \"member\",\n \"requests\": {\n \"read\": {\n \"method\": \"GET\",\n \"path\": \"/3.0/search-members\",\n \"query_params\": [\n {\n \"name\": \"query\",\n \"value\": \"\"\n }\n ],\n \"param_values\": [\n {\n \"name\": \"email\",\n \"identity\": \"email\"\n }\n ],\n \"data_path\": \"exact_matches.members\"\n },\n \"update\": {\n \"method\": \"PUT\",\n \"path\": \"/3.0/lists//members/\",\n \"param_values\": [\n {\n \"name\": \"list_id\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.list_id\",\n \"direction\": \"from\"\n }\n ]\n },\n {\n \"name\": \"subscriber_hash\",\n \"references\": [\n {\n \"dataset\": \"mailchimp_connector_example\",\n \"field\": \"member.id\",\n \"direction\": \"from\"\n }\n ]\n }\n ],\n \"body\": \"{}\"\n }\n }\n }\n ]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{saas_key}}/saas_config", + "host": ["{{host}}"], + "path": ["connection", "{{saas_key}}", "saas_config"] + } + }, + "response": [] + }, + { + "name": "Update Connection Secrets: SaaS", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"domain\": \"{{mailchimp_domain}}\",\n \"username\": \"{{mailchimp_username}}\",\n \"api_key\": \"{{mailchimp_api_key}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{saas_key}}/secret", + "host": ["{{host}}"], + "path": ["connection", "{{saas_key}}", "secret"] + } + }, + "response": [] + }, + { + "name": "Create/Update SaaS Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"fides_key\":\"mailchimp_connector_example\",\n \"name\":\"Mailchimp Dataset\",\n \"description\":\"A sample dataset representing the Mailchimp connector for Fidesops\",\n \"collections\":[\n {\n \"name\":\"messages\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"conversation_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"from_label\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"from_email\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"subject\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"message\",\n \"data_categories\":[\n \"user\"\n ]\n },\n {\n \"name\":\"read\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"timestamp\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\":\"conversations\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"campaign_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"list_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"from_email\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"from_label\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"subject\",\n \"data_categories\":[\n \"user\"\n ]\n }\n ]\n },\n {\n \"name\":\"member\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"list_id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"email_address\",\n \"data_categories\":[\n \"user.contact.email\"\n ]\n },\n {\n \"name\":\"unique_email_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"web_id\",\n \"data_categories\":[\n \"user.unique_id\"\n ]\n },\n {\n \"name\":\"email_type\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"status\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"merge_fields\",\n \"fields\":[\n {\n \"name\":\"FNAME\",\n \"data_categories\":[\n \"user.name\"\n ]\n },\n {\n \"name\":\"LNAME\",\n \"data_categories\":[\n \"user.name\"\n ]\n },\n {\n \"name\":\"ADDRESS\",\n \"fields\":[\n {\n \"name\":\"addr1\",\n \"data_categories\":[\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\":\"addr2\",\n \"data_categories\":[\n \"user.contact.address.street\"\n ]\n },\n {\n \"name\":\"city\",\n \"data_categories\":[\n \"user.contact.address.city\"\n ]\n },\n {\n \"name\":\"state\",\n \"data_categories\":[\n \"user.contact.address.state\"\n ]\n },\n {\n \"name\":\"zip\",\n \"data_categories\":[\n \"user.contact.address.postal_code\"\n ]\n },\n {\n \"name\":\"country\",\n \"data_categories\":[\n \"user.contact.address.country\"\n ]\n }\n ]\n },\n {\n \"name\":\"PHONE\",\n \"data_categories\":[\n \"user.contact.phone_number\"\n ]\n },\n {\n \"name\":\"BIRTHDAY\",\n \"data_categories\":[\n \"user.demographic.date_of_birth\"\n ]\n }\n ]\n },\n {\n \"name\":\"ip_signup\",\n \"data_categories\":[\n \"user.device.ip_address\"\n ]\n },\n {\n \"name\":\"timestamp_signup\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"ip_opt\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"timestamp_opt\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"language\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"email_client\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"location\",\n \"fields\":[\n {\n \"name\":\"latitude\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"longitude\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"gmtoff\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"dstoff\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"country_code\",\n \"data_categories\":[\n \"user.contact.address.country\"\n ]\n },\n {\n \"name\":\"timezone\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n ]\n },\n {\n \"name\":\"source\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"tags\",\n \"fields\":[\n {\n \"name\":\"id\",\n \"data_categories\":[\n \"system.operations\"\n ]\n },\n {\n \"name\":\"name\",\n \"data_categories\":[\n \"system.operations\"\n ]\n }\n ]\n }\n ]\n }\n ]\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{saas_key}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{saas_key}}", "dataset"] + } + }, + "response": [] + }, + { + "name": "Connection Test: SaaS", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{saas_key}}/test", + "host": ["{{host}}"], + "path": ["connection", "{{saas_key}}", "test"] + } + }, + "response": [] + } + ] + }, + { + "name": "Get SaaS Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection/{{sentry_saas_key}}/saas_config", + "host": ["{{host}}"], + "path": ["connection", "{{sentry_saas_key}}", "saas_config"] + } + }, + "response": [] + }, + { + "name": "Create From Template", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"name\": \"{{saas_connector_type}} connector\",\n \"instance_key\": \"primary_{{saas_connector_type}}\",\n \"secrets\": {\n \"domain\": \"{{mailchimp_domain}}\",\n \"api_key\": \"{{mailchimp_api_key}}\",\n \"username\": \"{{mailchimp_username}}\"\n }\n\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/instantiate/{{saas_connector_type}}", + "host": ["{{host}}"], + "path": ["connection", "instantiate", "{{saas_connector_type}}"] + } + }, + "response": [] + }, + { + "name": "Authorize Oauth Connector", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "http://localhost:8080/api/v1/connection/{{oauth_connector_key}}/authorize", + "protocol": "http", + "host": ["localhost"], + "port": "8080", + "path": [ + "api", + "v1", + "connection", + "{{oauth_connector_key}}", + "authorize" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Users", + "item": [ + { + "name": "Create User", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"username\": \"{{username}}\",\n \"password\": \"{{password}}\"\n\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user", + "host": ["{{host}}"], + "path": ["user"] + } + }, + "response": [] + }, + { + "name": "Get Users", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user", + "host": ["{{host}}"], + "path": ["user"] + } + }, + "response": [] + }, + { + "name": "Delete User", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "{{host}}/user/{{user_id}}", + "host": ["{{host}}"], + "path": ["user", "{{user_id}}"] + } + }, + "response": [] + }, + { + "name": "User login", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"username\": \"{{username}}\",\n \"password\": \"{{password}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/login", + "host": ["{{host}}"], + "path": ["login"] + } + }, + "response": [] + }, + { + "name": "User logout", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{user_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "url": { + "raw": "{{host}}/logout", + "host": ["{{host}}"], + "path": ["logout"] + } + }, + "response": [] + } + ] + }, + { + "name": "DRP", + "item": [ + { + "name": "Exercise", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"meta\": {\"version\": \"0.5\"},\n \"regime\": \"ccpa\",\n \"exercise\": [\n \"access\"\n ],\n \"identity\": \"eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzI1NiJ9.eyJlbWFpbCI6InRlc3RAZXhhbXBsZS5jb20ifQ.4I8XLWnTYp8oMHjN2ypP3Hpg45DIaGNAEmj1QCYONUI\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/drp/exercise", + "host": ["{{host}}"], + "path": ["drp", "exercise"] + } + }, + "response": [] + }, + { + "name": "Status", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/drp/status?request_id={{privacy_request_id}}", + "host": ["{{host}}"], + "path": ["drp", "status"], + "query": [ + { + "key": "request_id", + "value": "{{privacy_request_id}}" + } + ] + } + }, + "response": [] + }, + { + "name": "Data Rights", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/drp/data-rights", + "host": ["{{host}}"], + "path": ["drp", "data-rights"] + } + }, + "response": [] + }, + { + "name": "Revoke request", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"request_id\": \"{{privacy_request_id}}\", \n \"reason\": \"Accidentally submitted\"\n\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/drp/revoke", + "host": ["{{host}}"], + "path": ["drp", "revoke"] + } + }, + "response": [] + } + ] + }, + { + "name": "Manual Graph Nodes", + "item": [ + { + "name": "Add Manual Connection Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\"name\": \"Manual connector\",\n \"key\": \"{{manual_connector}}\",\n \"connection_type\": \"manual\",\n \"access\": \"write\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/", + "host": ["{{host}}"], + "path": ["connection", ""] + } + }, + "response": [] + }, + { + "name": "Add Manual Dataset", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"fides_key\": \"manual_key\",\n \"name\": \"Manual Dataset\",\n \"description\": \"Example of a Manual dataset with a node waiting on postgres input\",\n \"collections\": [\n {\n \"name\": \"filing_cabinet\",\n \"fields\": [\n {\n \"name\": \"id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"authorized_user\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\"\n }\n },\n {\n \"name\": \"customer_id\",\n \"data_categories\": [\n \"user.unique_id\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"customer.id\",\n \"direction\": \"from\"\n }\n ]\n }\n },\n {\n \"name\": \"payment_card_id\",\n \"data_categories\": [\n \"system.operations\"\n ],\n \"fidesops_meta\": {\n \"references\": [\n {\n \"dataset\": \"postgres_example\",\n \"field\": \"payment_card.id\",\n \"direction\": \"to\"\n }\n ]\n }\n }\n ]\n },\n {\n \"name\": \"storage_unit\",\n \"fields\": [\n {\n \"name\": \"box_id\",\n \"data_categories\": [\n \"user\"\n ],\n \"fidesops_meta\": {\n \"primary_key\": true\n }\n },\n {\n \"name\": \"email\",\n \"data_categories\": [\n \"user.contact.email\"\n ],\n \"fidesops_meta\": {\n \"data_type\": \"string\",\n \"identity\": \"email\"\n }\n }\n ]\n }\n ]\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/connection/{{manual_connector}}/dataset", + "host": ["{{host}}"], + "path": ["connection", "{{manual_connector}}", "dataset"] + } + }, + "response": [] + }, + { + "name": "Check Status", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/privacy-request/?request_id={{privacy_request_id}}&verbose=True", + "host": ["{{host}}"], + "path": ["privacy-request", ""], + "query": [ + { + "key": "request_id", + "value": "{{privacy_request_id}}" + }, + { + "key": "verbose", + "value": "True" + } + ] + } + }, + "response": [] + }, + { + "name": "Resume Privacy Request with Manual Filing Cabinet Input", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"id\": 2,\n \"authorized_user\": \"Jiminy Customer\",\n \"customer_id\": 1,\n \"payment_card_id\": \"pay_bbb-bbb\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/manual_input", + "host": ["{{host}}"], + "path": [ + "privacy-request", + "{{privacy_request_id}}", + "manual_input" + ] + } + }, + "response": [] + }, + { + "name": "Resume Privacy Request with Manual Storage Unit Input", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[{\n \"box_id\": 5,\n \"email\": \"customer-1@example.com\"\n}]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/manual_input", + "host": ["{{host}}"], + "path": [ + "privacy-request", + "{{privacy_request_id}}", + "manual_input" + ] + } + }, + "response": [] + }, + { + "name": "Resume Erasure Request with Confirmed Masked Row Count", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"row_count\": 5\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/erasure_confirm", + "host": ["{{host}}"], + "path": [ + "privacy-request", + "{{privacy_request_id}}", + "erasure_confirm" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Restart from Failure", + "item": [ + { + "name": "Restart failed node", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "url": { + "raw": "{{host}}/privacy-request/{{privacy_request_id}}/retry", + "host": ["{{host}}"], + "path": ["privacy-request", "{{privacy_request_id}}", "retry"] + } + }, + "response": [] + } + ] + }, + { + "name": "ConnectionType", + "item": [ + { + "name": "Get available connectors", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection_type", + "host": ["{{host}}"], + "path": ["connection_type"] + } + }, + "response": [] + }, + { + "name": "Get available connectors - consent", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection_type?consent=true", + "host": ["{{host}}"], + "path": ["connection_type"], + "query": [ + { + "key": "consent", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Get connection secrets schema", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/connection_type/outreach/secret", + "host": ["{{host}}"], + "path": ["connection_type", "outreach", "secret"] + } + }, + "response": [] + } + ] + }, + { + "name": "Messaging Config - Email", + "item": [ + { + "name": "Post Messaging Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/", + "host": ["{{host}}"], + "path": ["messaging", "config", ""] + } + }, + "response": [] + }, + { + "name": "Patch Messaging Config By Key", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", + "host": ["{{host}}"], + "path": ["messaging", "config", "{{mailgun_config_key}}"] + } + }, + "response": [] + }, + { + "name": "Get Messaging Configs", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/messaging/config/", + "host": ["{{host}}"], + "path": ["messaging", "config", ""] + } + }, + "response": [] + }, + { + "name": "Get Messaging Config By Key", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", + "host": ["{{host}}"], + "path": ["messaging", "config", "{{mailgun_config_key}}"] + } + }, + "response": [] + }, + { + "name": "Delete Messaging Config By Key", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n }\n]\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", + "host": ["{{host}}"], + "path": ["messaging", "config", "{{mailgun_config_key}}"] + } + }, + "response": [] + }, + { + "name": "Messaging Config Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"mailgun_api_key\": \"{{mailgun_api_key}}\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}/secret", + "host": ["{{host}}"], + "path": [ + "messaging", + "config", + "{{mailgun_config_key}}", + "secret" + ] + } + }, + "response": [] + }, + { + "name": "Test Message", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"email\": \"{{test_email}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/test", + "host": ["{{host}}"], + "path": ["messaging", "config", "test"] + } + }, + "response": [] + }, + { + "name": "Get Messaging Config Status", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "" + }, + "url": { + "raw": "{{host}}/messaging/default/status", + "host": ["{{host}}"], + "path": ["messaging", "default", "status"] + } + }, + "response": [] + } + ] + }, + { + "name": "Messaging Config - SMS", + "item": [ + { + "name": "Post Messaging Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/", + "host": ["{{host}}"], + "path": ["messaging", "config", ""] + } + }, + "response": [] + }, + { + "name": "Patch Messaging Config By Key", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{twilio_config_key}}", + "host": ["{{host}}"], + "path": ["messaging", "config", "{{twilio_config_key}}"] + } + }, + "response": [] + }, + { + "name": "Messaging Config Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"twilio_account_sid\": \"{{twilio_account_sid}}\",\n \"twilio_auth_token\": \"{{twilio_auth_token}}\",\n \"twilio_messaging_service_sid\": \"{{twilio_messaging_service_id}}\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{twilio_config_key}}/secret", + "host": ["{{host}}"], + "path": ["messaging", "config", "{{twilio_config_key}}", "secret"] + } + }, + "response": [] + } + ] + }, + { + "name": "Identity Verification", + "item": [ + { + "name": "Get Identity Verification Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/id-verification/config/", + "host": ["{{host}}"], + "path": ["id-verification", "config", ""] + } + }, + "response": [] + } + ] + }, + { + "name": "Consent Request", + "item": [ + { + "name": "Create Verification Code for Consent Request", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request", + "host": ["{{host}}"], + "path": ["consent-request"] + } + }, + "response": [] + }, + { + "name": "Verify Code and Return Current Preferences", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"code\": \"{{verification_code}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request/{{consent_request_id}}/verify", + "host": ["{{host}}"], + "path": ["consent-request", "{{consent_request_id}}", "verify"] + } + }, + "response": [] + }, + { + "name": "Authenticated Get Consent Preferences", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}" + }, + "url": { + "raw": "{{host}}/consent-request/preferences", + "host": ["{{host}}"], + "path": ["consent-request", "preferences"] + } + }, + "response": [] + }, + { + "name": "Verify Code and Save Preferences", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"identity\": {\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n },\n \"consent\": [\n {\n \"data_use\": \"{{data_use}}\",\n \"data_use_description\": \"{{data_use_description}}\",\n \"opt_in\": {{opt_in}}\n }\n ],\n \"code\": \"{{verification_code}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request/{{consent_request_id}}/preferences", + "host": ["{{host}}"], + "path": [ + "consent-request", + "{{consent_request_id}}", + "preferences" + ] + } + }, + "response": [] + } + ], + "auth": { + "type": "noauth" + }, + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [""] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [""] + } + } + ] + }, + { + "name": "Systems", + "item": [ + { + "name": "Get System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/system/", + "host": ["{{host}}"], + "path": ["system", ""] + } + }, + "response": [] + }, + { + "name": "Create System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"personalize\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"test_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/system", + "host": ["{{host}}"], + "path": ["system"] + } + }, + "response": [] + }, + { + "name": "Update System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"data_responsibility_title\":\"Processor\",\n \"description\":\"Collect data about our users for marketing.\",\n \"egress\":[\n {\n \"fides_key\":\"demo_analytics_system\",\n \"type\":\"system\",\n \"data_categories\":null\n }\n ],\n \"fides_key\":\"test_system\",\n \"ingress\":null,\n \"name\":\"Test system\",\n \"organization_fides_key\":\"default_organization\",\n \"privacy_declarations\":[\n {\n \"name\":\"Collect data for marketing\",\n \"data_categories\":[\n \"user.device.cookie_id\"\n ],\n \"data_use\":\"marketing.advertising\",\n \"data_qualifier\":\"aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified\",\n \"data_subjects\":[\n \"customer\"\n ],\n \"dataset_references\":null,\n \"egress\":null,\n \"ingress\":null,\n \"cookies\":[\n {\n \"name\":\"another_cookie\",\n \"path\":\"/\"\n }\n ]\n }\n ],\n \"system_dependencies\":[\n \"demo_analytics_system\"\n ],\n \"system_type\":\"Service\",\n \"tags\":null,\n \"third_country_transfers\":null,\n \"administrating_department\":\"Marketing\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/system?", + "host": ["{{host}}"], + "path": ["system"], + "query": [ + { + "key": "", + "value": null + } + ] + } + }, + "response": [] + }, + { + "name": "Delete System", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "url": { + "raw": "{{host}}/system/test_system", + "host": ["{{host}}"], + "path": ["system", "test_system"] + } + }, + "response": [] + } + ] + }, + { + "name": "Roles", + "item": [ + { + "name": "Add role (Create User Permissions - Usually you'll use Update Permissions instead)", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\"roles\": [\"viewer\"]}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user/{{user_id}}/permission", + "host": ["{{host}}"], + "path": ["user", "{{user_id}}", "permission"] + } + }, + "response": [] + }, + { + "name": "Update role (Update User Permissions)", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "[\"demo_analytics_system\", \"demo_marketing_system\"]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user/{{user_id}}/system-manager", + "host": ["{{host}}"], + "path": ["user", "{{user_id}}", "system-manager"] + } + }, + "response": [] + }, + { + "name": "Get Allowed Roles", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/oauth/role", + "host": ["{{host}}"], + "path": ["oauth", "role"] + } + }, + "response": [] + } + ] + }, + { + "name": "System Managers", + "item": [ + { + "name": "Update Managed Systems", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "[\"demo_analytics_system\", \"demo_marketing_system\"]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user/{{user_id}}/system-manager", + "host": ["{{host}}"], + "path": ["user", "{{user_id}}", "system-manager"] + } + }, + "response": [] + }, + { + "name": "Get Systems Managed By User", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user/{{user_id}}/system-manager", + "host": ["{{host}}"], + "path": ["user", "{{user_id}}", "system-manager"] + } + }, + "response": [] + }, + { + "name": "Get Single System Managed By User", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user/{{user_id}}/system-manager/demo_marketing_system", + "host": ["{{host}}"], + "path": [ + "user", + "{{user_id}}", + "system-manager", + "demo_marketing_system" + ] + } + }, + "response": [] + }, + { + "name": "Remove User as System Manager", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "DELETE", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/user/{{user_id}}/system-manager/demo_marketing_system", + "host": ["{{host}}"], + "path": [ + "user", + "{{user_id}}", + "system-manager", + "demo_marketing_system" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Privacy Notices", + "item": [ + { + "name": "Create Privacy Notices", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"name\":\"Profiling\",\n \"notice_key\": \"profiling\",\n \"regions\":[\n \"us_wv\",\n \"us_mt\"\n ],\n \"description\":\"Making a decision solely by automated means.\",\n \"consent_mechanism\":\"opt_in\",\n \"data_uses\":[\n \"personalize\"\n ],\n \"enforcement_level\":\"system_wide\",\n \"has_gpc_flag\":false,\n \"displayed_in_overlay\":true\n },\n {\n \"name\":\"Essential\",\n \"notice_key\": \"essential\",\n \"regions\":[\n \"us_ak\"\n ],\n \"description\":\"Notify the user about data processing activities that are essential to your services' functionality. Typically consent is not required for this.\",\n \"consent_mechanism\":\"notice_only\",\n \"data_uses\":[\n \"essential.service\"\n ],\n \"enforcement_level\":\"system_wide\",\n \"displayed_in_overlay\":true\n },\n {\n \"name\":\"Advertising\",\n \"notice_key\": \"advertising\",\n \"regions\":[\n \"us_nc\"\n ],\n \"description\":\"Sample advertising notice\",\n \"consent_mechanism\":\"opt_out\",\n \"data_uses\":[\n \"marketing.advertising\"\n ],\n \"enforcement_level\":\"system_wide\",\n \"displayed_in_privacy_center\":true\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-notice", + "host": ["{{host}}"], + "path": ["privacy-notice"] + } + }, + "response": [] + }, + { + "name": "Update Privacy Notice", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "[\n {\n \"id\": \"{{privacy_notice_id}}\",\n \"data_uses\": [\n \"third_party_sharing\"\n ]\n }\n]", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-notice", + "host": ["{{host}}"], + "path": ["privacy-notice"] + } + }, + "response": [] + }, + { + "name": "Get Privacy Notices", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-notice?systems_applicable=false&show_disabled=true", + "host": ["{{host}}"], + "path": ["privacy-notice"], + "query": [ + { + "key": "systems_applicable", + "value": "false" + }, + { + "key": "show_disabled", + "value": "true" + } + ] + } + }, + "response": [] + }, + { + "name": "Get Privacy Notices By Data Use", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-notice-by-data-use", + "host": ["{{host}}"], + "path": ["privacy-notice-by-data-use"] + } + }, + "response": [] + }, + { + "name": "Get Single Privacy Notice", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-notice/{{privacy_notice_id}}", + "host": ["{{host}}"], + "path": ["privacy-notice", "{{privacy_notice_id}}"] + } + }, + "response": [] + } + ] + }, + { + "name": "Privacy Experiences and Configs", + "item": [ + { + "name": "Privacy Experience List", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/privacy-experience/?region=us_ca", + "host": ["{{host}}"], + "path": ["privacy-experience", ""], + "query": [ + { + "key": "region", + "value": "us_ca" + } + ] + } + }, + "response": [] + }, + { + "name": "Experience Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"accept_button_label\": \"Accept all\",\n \"acknowledge_button_label\": \"Acknowledge\",\n \"banner_enabled\": \"enabled_where_required\",\n \"component\": \"overlay\",\n \"description\": \"On this page you can opt in and out of these data uses cases\",\n \"disabled\": false,\n \"privacy_preferences_link_label\": \"Manage preferences\",\n \"privacy_policy_link_label\": \"View our privacy policy\",\n \"privacy_policy_url\": \"example.com/privacy\",\n \"regions\": [\"us_nc\"],\n \"reject_button_label\": \"Reject all\",\n \"save_button_label\": \"Save\",\n \"title\": \"Manage your consent\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/experience-config", + "host": ["{{host}}"], + "path": ["experience-config"] + } + }, + "response": [] + }, + { + "name": "Experience Config Update", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"accept_button_label\": \"Accept\",\n \"reject_button_label\": \"Reject\",\n \"save_button_label\": \"Save all\",\n \"title\": \"Here is where you manage your consent!\",\n \"regions\": [\"us_ca\", \"de\"]\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/experience-config/{{experience-config-id}}", + "host": ["{{host}}"], + "path": ["experience-config", "{{experience-config-id}}"] + } + }, + "response": [] + }, + { + "name": "Get Experience Configs", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/experience-config?component=overlay&show_disabled=False", + "host": ["{{host}}"], + "path": ["experience-config"], + "query": [ + { + "key": "component", + "value": "overlay" + }, + { + "key": "show_disabled", + "value": "False" + } + ] + } + }, + "response": [] + }, + { + "name": "Get Experience Config Detail", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/experience-config/{{experience-config-id}}", + "host": ["{{host}}"], + "path": ["experience-config", "{{experience-config-id}}"] + } + }, + "response": [] + } + ] + }, + { + "name": "PrivacyPreferences", + "item": [ + { + "name": "Create Verification Code for Consent Request", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"phone_number\": \"{{phone_number}}\",\n \"email\": \"{{email}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request", + "host": ["{{host}}"], + "path": ["consent-request"] + } + }, + "response": [] + }, + { + "name": "Verify Code and Return Current Privacy Preferences", + "request": { + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"code\": \"{{verification_code}}\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request/{{consent_request_id}}/verify-for-privacy-preferences", + "host": ["{{host}}"], + "path": [ + "consent-request", + "{{consent_request_id}}", + "verify-for-privacy-preferences" + ] + } + }, + "response": [] + }, + { + "name": "Verify Code and Save Privacy Preferences", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"browser_identity\": {\n \"ga_client_id\": \"UA-XXXXXXXXX\",\n \"ljt_readerID\": \"test_sovrn_id\"\n },\n \"code\": \"{{verification_code}}\",\n \"preferences\": [{\n \"privacy_notice_history_id\": \"{{privacy_notice_history_id}}\",\n \"preference\": \"opt_out\"\n }],\n \"request_origin\": \"privacy_center\",\n \"url_recorded\": \"example.com/privacy_center\",\n \"user_agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24\",\n \"user_geography\": \"us_ca\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/consent-request/{{consent_request_id}}/privacy-preferences", + "host": ["{{host}}"], + "path": [ + "consent-request", + "{{consent_request_id}}", + "privacy-preferences" + ] + } + }, + "response": [] + }, + { + "name": "Save Privacy Preferences for Device Id", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"browser_identity\": {\n \"ga_client_id\": \"UA-XXXXXXXXX\",\n \"ljt_readerID\": \"test_sovrn_id\",\n \"fides_user_device_id\": \"{{fides_user_device_id}}\"\n },\n \"preferences\": [{\n \"privacy_notice_history_id\": \"{{privacy_notice_history_id}}\",\n \"preference\": \"opt_out\"\n }],\n \"user_geography\": \"us_ca\",\n \"privacy_experience_id\": \"{{privacy_experience_id}}\",\n \"method\": \"button\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-preferences", + "host": ["{{host}}"], + "path": ["privacy-preferences"] + } + }, + "response": [] + }, + { + "name": "Get Historical Preferences", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/historical-privacy-preferences", + "host": ["{{host}}"], + "path": ["historical-privacy-preferences"] + } + }, + "response": [] + }, + { + "name": "Get Current Preferences", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "GET", + "header": [], + "url": { + "raw": "{{host}}/current-privacy-preferences", + "host": ["{{host}}"], + "path": ["current-privacy-preferences"] + } + }, + "response": [] + } + ] + } + ], + "event": [ + { + "listen": "prerequest", + "script": { + "type": "text/javascript", + "exec": [""] + } + }, + { + "listen": "test", + "script": { + "type": "text/javascript", + "exec": [""] + } + } + ], + "variable": [ + { + "key": "host", + "value": "http://0.0.0.0:8080/api/v1" + }, + { + "key": "OAUTH_ROOT_CLIENT_ID", + "value": "" + }, + { + "key": "OAUTH_ROOT_CLIENT_SECRET", + "value": "" + }, + { + "key": "root_client_token", + "value": "" + }, + { + "key": "client_id", + "value": "" + }, + { + "key": "client_secret", + "value": "" + }, + { + "key": "client_token", + "value": "" + }, + { + "key": "policy_key", + "value": "my_primary_policy" + }, + { + "key": "rule_key", + "value": "my_access_rule" + }, + { + "key": "target_key", + "value": "my_user_info" + }, + { + "key": "postgres_key", + "value": "app_postgres_db" + }, + { + "key": "mssql_key", + "value": "app_mssql_db" + }, + { + "key": "mysql_key", + "value": "app_mysql_db" + }, + { + "key": "mongo_key", + "value": "app_mongo_db" + }, + { + "key": "mariadb_key", + "value": "app_mariadb_db", + "type": "string" + }, + { + "key": "bigquery_key", + "value": "app_bigquery_db", + "type": "string" + }, + { + "key": "privacy_request_id", + "value": "" + }, + { + "key": "denial_reason", + "value": "" + }, + { + "key": "storage_key", + "value": "my_local_storage" + }, + { + "key": "AWS_ACCESS_KEY_ID", + "value": "" + }, + { + "key": "AWS_SECRET_ACCESS_KEY", + "value": "" + }, + { + "key": "S3_BUCKET_KEY", + "value": "my_access_request_upload_bucket" + }, + { + "key": "separate_policy_key", + "value": "my_separate_policy" + }, + { + "key": "erasure_rule_key", + "value": "my_erasure_rule" + }, + { + "key": "mask_target_key", + "value": "my_mask_target" + }, + { + "key": "http_connection_key", + "value": "my_http_connection" + }, + { + "key": "pre_webhook_one", + "value": "wake_up_snowflake_db" + }, + { + "key": "pre_webhook_two", + "value": "prep_systems_webhook" + }, + { + "key": "post_webhook_one", + "value": "cache_busting_webhook" + }, + { + "key": "post_webhook_two", + "value": "finalize_privacy_request" + }, + { + "key": "saas_key", + "value": "saas_key", + "type": "string" + }, + { + "key": "mailchimp_domain", + "value": "", + "type": "string" + }, + { + "key": "mailchimp_username", + "value": "", + "type": "string" + }, + { + "key": "mailchimp_api_key", + "value": "", + "type": "string" + }, + { + "key": "username", + "value": "" + }, + { + "key": "password", + "value": "" + }, + { + "key": "user_id", + "value": "" + }, + { + "key": "user_token", + "value": "" + }, + { + "key": "sentry_saas_key", + "value": "sentry_saas_key" + }, + { + "key": "sentry_host", + "value": "" + }, + { + "key": "sentry_access_token", + "value": "" + }, + { + "key": "hubspot_domain", + "value": "" + }, + { + "key": "hubspot_api_key", + "value": "" + }, + { + "key": "hubspot_saas_key", + "value": "hubspot_saas_key" + }, + { + "key": "segment_saas_key", + "value": "segment_saas_key" + }, + { + "key": "segment_domain", + "value": "" + }, + { + "key": "segment_personas_domain", + "value": "" + }, + { + "key": "segment_workspace", + "value": "" + }, + { + "key": "segment_access_token", + "value": "" + }, + { + "key": "segment_namespace_id", + "value": "" + }, + { + "key": "segment_access_secret", + "value": "" + }, + { + "key": "manual_connector", + "value": "manual_key" + }, + { + "key": "mailgun_config_key", + "value": "my_mailgun_config", + "type": "string" + }, + { + "key": "saas_connector_type", + "value": "mailchimp", + "type": "string" + }, + { + "key": "email_connection_config_key", + "value": "email_connection_config_key", + "type": "string" + }, + { + "key": "mailgun_domain", + "value": "", + "type": "string" + }, + { + "key": "mailgun_api_key", + "value": "", + "type": "string" + }, + { + "key": "manual_webhook_key", + "value": "manual_webhook_key", + "type": "string" + }, + { + "key": "consent_request_id", + "value": "", + "type": "default" + }, + { + "key": "timescale_key", + "value": "", + "type": "string" + }, + { + "key": "phone_number", + "value": "", + "type": "string" + }, + { + "key": "email", + "value": "", + "type": "string" + }, + { + "key": "verification_code", + "value": "", + "type": "string" + }, + { + "key": "twilio_config_key", + "value": "my_twilio_config", + "type": "string" + }, + { + "key": "twilio_account_sid", + "value": "", + "type": "string" + }, + { + "key": "twilio_auth_token", + "value": "", + "type": "string" + }, + { + "key": "twilio_messaging_service_id", + "value": "", + "type": "string" + }, + { + "key": "privacy_notice_id", + "value": "", + "type": "string" + }, + { + "key": "privacy_notice_history_id", + "value": "", + "type": "string" + }, + { + "key": "test_email", + "value": "", + "type": "string" + }, + { + "key": "oauth_connector_key", + "value": "", + "type": "string" + }, + { + "key": "fides_user_device_id", + "value": "", + "type": "string" + }, + { + "key": "privacy_experience_id", + "value": "", + "type": "string" + }, + { + "key": "experience-config-id", + "value": "", + "type": "string" + } + ] +} diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 9503f5ad505..6c9e5b097fe 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -27,7 +27,7 @@ data_use_upgrades: Dict[str, str] = { "improve.system": "functional.service.improve", # Verified in 2.0 "improve": "functional", # Posted a question in the Confluence doc to verify this - "essential.service.operations.support.optimizations": "essential.service.operations.improve", # Verified in 2.0 + "essential.service.operations.support.optimization": "essential.service.operations.improve", # Verified in 2.0 } data_use_downgrades: Dict[str, str] = { value: key for key, value in data_use_upgrades.items() diff --git a/src/fides/api/api/v1/endpoints/user_endpoints.py b/src/fides/api/api/v1/endpoints/user_endpoints.py index cb4bc33b62c..3e3ee2dc073 100644 --- a/src/fides/api/api/v1/endpoints/user_endpoints.py +++ b/src/fides/api/api/v1/endpoints/user_endpoints.py @@ -151,7 +151,7 @@ def update_user_password( """ _validate_current_user(user_id, current_user) - if not current_user.credentials_valid( + if not current_user.authorization.credentials_valid( b64_str_to_str(data.old_password), CONFIG.security.encoding ): raise HTTPException( diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index eb7dd879c11..93ebc78a819 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -92,7 +92,9 @@ def create_or_update_parent_user() -> None: ) if user and CONFIG.security.parent_server_password: - if not user.credentials_valid(CONFIG.security.parent_server_password): + if not user.authorization.credentials_valid( + CONFIG.security.parent_server_password + ): log.debug("Updating Fides parent user credentials") user.update_password(db_session, CONFIG.security.parent_server_password) return @@ -327,7 +329,7 @@ def load_default_dsr_policies() -> None: excluded_data_categories = [ "user.financial", "user.payment", - "user.credentials", + "user.authorization.credentials", ] all_data_categories = [ str(category.fides_key) for category in DEFAULT_TAXONOMY.data_category diff --git a/src/fides/api/service/connectors/consent_email_connector.py b/src/fides/api/service/connectors/consent_email_connector.py index 0ec904ecd24..4b6c03c0bef 100644 --- a/src/fides/api/service/connectors/consent_email_connector.py +++ b/src/fides/api/service/connectors/consent_email_connector.py @@ -128,7 +128,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: id="test_2", privacy_notice_id="67890", consent_mechanism=ConsentMechanism.opt_out, - data_uses=["improve.system"], + data_uses=["functional.service.improve"], enforcement_level=EnforcementLevel.system_wide, version=1.0, displayed_in_overlay=True, diff --git a/src/fides/data/sample_project/sample_resources/mongo_example_test_dataset.yml b/src/fides/data/sample_project/sample_resources/mongo_example_test_dataset.yml index f3234cb4765..3971b6481dd 100644 --- a/src/fides/data/sample_project/sample_resources/mongo_example_test_dataset.yml +++ b/src/fides/data/sample_project/sample_resources/mongo_example_test_dataset.yml @@ -17,11 +17,11 @@ dataset: field: customer.id direction: from - name: gender - data_categories: [user.gender] + data_categories: [user.demographic.gender] fides_meta: data_type: string - name: birthday - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fides_meta: data_type: string - name: workplace_info @@ -189,7 +189,7 @@ dataset: fides_meta: data_type: string - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fides_meta: data_type: string - name: employee @@ -245,7 +245,7 @@ dataset: - name: billing_address_id data_categories: [system.operations] - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] fides_meta: references: - dataset: mongo_test diff --git a/src/fides/data/sample_project/sample_resources/postgres_example_test_dataset.yml b/src/fides/data/sample_project/sample_resources/postgres_example_test_dataset.yml index 9978c88ff86..e519a75008b 100644 --- a/src/fides/data/sample_project/sample_resources/postgres_example_test_dataset.yml +++ b/src/fides/data/sample_project/sample_resources/postgres_example_test_dataset.yml @@ -136,7 +136,7 @@ dataset: field: address.id direction: to - name: ccn - data_categories: [user.financial.account_number] + data_categories: [user.financial.bank_account] - name: code data_categories: [user.financial] - name: customer_id diff --git a/src/fides/data/sample_project/sample_resources/sample_systems.yml b/src/fides/data/sample_project/sample_resources/sample_systems.yml index 5de34267c1f..12b1fc1e237 100644 --- a/src/fides/data/sample_project/sample_resources/sample_systems.yml +++ b/src/fides/data/sample_project/sample_resources/sample_systems.yml @@ -16,7 +16,7 @@ system: - data_categories: - user.contact - user.device.cookie_id - data_use: improve.system + data_use: functional.service.improve data_subjects: - customer data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified diff --git a/tests/conftest.py b/tests/conftest.py index ebb44ffb27b..283ae23979d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1174,7 +1174,7 @@ def system_provide_service_operations_support_optimization(db: Session) -> Syste "name": "Optimize and improve support operations in order to provide the service", "system_id": system_provide_service_operations_support_optimization.id, "data_categories": ["user.device.cookie_id"], - "data_use": "essential.service.operations.support.optimization", + "data_use": "essential.service.operations.improve", "data_qualifier": "aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified", "data_subjects": ["customer"], "dataset_references": None, diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index d19eef41d61..ad49c12788a 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -86,7 +86,7 @@ def test_filter_data_categories_excluded(self) -> None: excluded_data_categories = [ "user.financial", "user.payment", - "user.credentials", + "user.authorization.credentials", ] all_data_categories = [ "user.name", @@ -94,9 +94,9 @@ def test_filter_data_categories_excluded(self) -> None: # These should be excluded "user.payment", "user.payment.financial_account_number", - "user.credentials", - "user.credentials.biometric_credentials", - "user.financial.account_number", + "user.authorization.credentials", + "user.authorization.credentials.biometric_credentials", + "user.financial.bank_account", "user.financial", ] expected_result = [ @@ -112,7 +112,7 @@ def test_filter_data_categories_no_third_level(self) -> None: excluded_data_categories = [ "user.financial", "user.payment", - "user.credentials", + "user.authorization.credentials", ] all_data_categories = [ "user.name", @@ -120,9 +120,9 @@ def test_filter_data_categories_no_third_level(self) -> None: # These should be excluded "user.payment", "user.payment.financial_account_number", - "user.credentials", - "user.credentials.biometric_credentials", - "user.financial.account_number", + "user.authorization.credentials", + "user.authorization.credentials.biometric_credentials", + "user.financial.bank_account", "user.financial", ] expected_result = [ @@ -151,7 +151,7 @@ def test_filter_data_categories_empty_excluded(self) -> None: all_data_categories = [ "user.name", "user.payment", - "user.credentials", + "user.authorization.credentials", "user.financial", ] assert seed.filter_data_categories(all_data_categories, []) == sorted( @@ -163,7 +163,7 @@ def test_filter_data_categories_no_exclusions(self) -> None: excluded_data_categories = ["user.payment"] all_data_categories = [ "user.name", - "user.credentials", + "user.authorization.credentials", "user.financial", ] assert seed.filter_data_categories( @@ -174,7 +174,7 @@ def test_filter_data_categories_only_return_users(self) -> None: """Test that the filter method works as intended""" all_data_categories = [ "user.name", - "user.credentials", + "user.authorization.credentials", "user.financial", # These are excluded "nonuser.foo", @@ -182,7 +182,7 @@ def test_filter_data_categories_only_return_users(self) -> None: ] expected_categories = [ "user.name", - "user.credentials", + "user.authorization.credentials", "user.financial", ] assert seed.filter_data_categories(all_data_categories, []) == sorted( @@ -326,7 +326,10 @@ def test_create_or_update_parent_user_change_password(db): db.refresh(user) assert user.password_reset_at is not None - assert user.credentials_valid(CONFIG.security.parent_server_password) is True + assert ( + user.authorization.credentials_valid(CONFIG.security.parent_server_password) + is True + ) user.delete(db) diff --git a/tests/ctl/core/test_evaluate.py b/tests/ctl/core/test_evaluate.py index acbbeac88d5..ebdaabd3881 100644 --- a/tests/ctl/core/test_evaluate.py +++ b/tests/ctl/core/test_evaluate.py @@ -207,7 +207,7 @@ def test_hydrate_missing_resources(test_config: FidesConfig) -> None: headers=test_config.user.auth_header, dehydrated_taxonomy=dehydrated_taxonomy, missing_resource_keys={ - "user.credentials", + "user.authorization.credentials", "user", }, ) @@ -576,13 +576,13 @@ def test_failed_evaluation_error_message( 'system (customer_data_sharing_system) failed ' 'rule (reject_targeted_marketing) from policy ' '(primary_privacy_policy). Violated usage of ' - 'data categories (user.political_opinion) with ' + 'data categories (user.demographic.political_opinion) with ' 'qualifier ' '(aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified) ' 'for data uses ' '(marketing.advertising.third_party) and ' 'subjects (customer)', - 'violating_attributes': { 'data_categories': [ 'user.political_opinion'], + 'violating_attributes': { 'data_categories': [ 'user.demographic.political_opinion'], 'data_qualifier': 'aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified', 'data_subjects': ['customer'], 'data_uses': [ 'marketing.advertising.third_party']}}]} diff --git a/tests/ctl/data/failing_dataset_collection_taxonomy.yml b/tests/ctl/data/failing_dataset_collection_taxonomy.yml index dd9fdba8ebb..93e03aef6d1 100644 --- a/tests/ctl/data/failing_dataset_collection_taxonomy.yml +++ b/tests/ctl/data/failing_dataset_collection_taxonomy.yml @@ -6,7 +6,7 @@ dataset: - name: users description: User's information data_categories: - - user.political_opinion + - user.demographic.political_opinion data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized fields: - name: First_Name @@ -41,7 +41,7 @@ policy: data_categories: matches: ANY values: - - user.political_opinion + - user.demographic.political_opinion data_uses: matches: ANY values: diff --git a/tests/ctl/data/failing_dataset_field_taxonomy.yml b/tests/ctl/data/failing_dataset_field_taxonomy.yml index 63a5527c6df..57fca03f0b6 100644 --- a/tests/ctl/data/failing_dataset_field_taxonomy.yml +++ b/tests/ctl/data/failing_dataset_field_taxonomy.yml @@ -14,7 +14,7 @@ dataset: - name: political_opinion description: User's political opinion data_categories: - - user.political_opinion + - user.demographic.political_opinion data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized system: - fides_key: customer_data_sharing_system @@ -42,7 +42,7 @@ policy: data_categories: matches: ANY values: - - user.political_opinion + - user.demographic.political_opinion data_uses: matches: ANY values: diff --git a/tests/ctl/data/failing_dataset_taxonomy.yml b/tests/ctl/data/failing_dataset_taxonomy.yml index 845f89ee456..39800612715 100644 --- a/tests/ctl/data/failing_dataset_taxonomy.yml +++ b/tests/ctl/data/failing_dataset_taxonomy.yml @@ -3,7 +3,7 @@ dataset: name: Sample DB Dataset description: This is a Sample Database Dataset data_categories: - - user.political_opinion + - user.demographic.political_opinion data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized collections: - name: users @@ -41,7 +41,7 @@ policy: data_categories: matches: ANY values: - - user.political_opinion + - user.demographic.political_opinion data_uses: matches: ANY values: diff --git a/tests/ctl/data/failing_declaration_taxonomy.yml b/tests/ctl/data/failing_declaration_taxonomy.yml index 91e802d668f..e0372d70dc9 100644 --- a/tests/ctl/data/failing_declaration_taxonomy.yml +++ b/tests/ctl/data/failing_declaration_taxonomy.yml @@ -6,7 +6,7 @@ system: privacy_declarations: - name: Share Political Opinions data_categories: - - user.political_opinion + - user.demographic.political_opinion data_use: marketing.advertising.third_party data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified data_subjects: diff --git a/tests/ctl/data/failing_nested_dataset.yml b/tests/ctl/data/failing_nested_dataset.yml index 7a4b1b075d0..49750d4f963 100644 --- a/tests/ctl/data/failing_nested_dataset.yml +++ b/tests/ctl/data/failing_nested_dataset.yml @@ -28,7 +28,7 @@ system: - name: Mesaure usage of users data_categories: - user - data_use: improve.system + data_use: functional.service.improve data_subjects: - customer data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified diff --git a/tests/ctl/data/passing_declaration_taxonomy.yml b/tests/ctl/data/passing_declaration_taxonomy.yml index bd0235fc59d..6b3f7bed5f8 100644 --- a/tests/ctl/data/passing_declaration_taxonomy.yml +++ b/tests/ctl/data/passing_declaration_taxonomy.yml @@ -6,7 +6,7 @@ system: privacy_declarations: - name: Share Political Opinions data_categories: - - user.political_opinion + - user.demographic.political_opinion data_use: essential.service.payment_processing data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified data_subjects: diff --git a/tests/fixtures/saas/test_data/mailchimp_override_dataset.yml b/tests/fixtures/saas/test_data/mailchimp_override_dataset.yml index 425a2c25e3a..b0d39058e38 100644 --- a/tests/fixtures/saas/test_data/mailchimp_override_dataset.yml +++ b/tests/fixtures/saas/test_data/mailchimp_override_dataset.yml @@ -102,7 +102,7 @@ dataset: fidesops_meta: data_type: string - name: BIRTHDAY - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: ip_signup diff --git a/tests/fixtures/saas/test_data/saas_example_dataset.yml b/tests/fixtures/saas/test_data/saas_example_dataset.yml index 6301ab69ba4..cb547b01dec 100644 --- a/tests/fixtures/saas/test_data/saas_example_dataset.yml +++ b/tests/fixtures/saas/test_data/saas_example_dataset.yml @@ -102,7 +102,7 @@ dataset: fidesops_meta: data_type: string - name: BIRTHDAY - data_categories: [user.date_of_birth] + data_categories: [user.demographic.date_of_birth] fidesops_meta: data_type: string - name: ip_signup diff --git a/tests/fixtures/saas_example_fixtures.py b/tests/fixtures/saas_example_fixtures.py index 58c714f32b9..873d8ecc19c 100644 --- a/tests/fixtures/saas_example_fixtures.py +++ b/tests/fixtures/saas_example_fixtures.py @@ -395,7 +395,7 @@ def erasure_policy_complete_mask( db=db, data={ "client_id": oauth_client.id, - "data_category": DataCategory("user.gender").value, + "data_category": DataCategory("user.demographic.gender").value, "rule_id": user_gender_rule.id, }, ) @@ -567,7 +567,7 @@ def erasure_policy_complete_mask( db=db, data={ "client_id": oauth_client.id, - "data_category": DataCategory("user.date_of_birth").value, + "data_category": DataCategory("user.demographic.date_of_birth").value, "rule_id": user_date_of_birth_rule.id, }, ) diff --git a/tests/lib/test_fides_user.py b/tests/lib/test_fides_user.py index 5752562f52a..63eb72d1dab 100644 --- a/tests/lib/test_fides_user.py +++ b/tests/lib/test_fides_user.py @@ -16,8 +16,8 @@ def test_create_user(): assert user.username == "user_1" assert user.hashed_password != "test_password" - assert not user.credentials_valid("bad_password") - assert user.credentials_valid(password) + assert not user.authorization.credentials_valid("bad_password") + assert user.authorization.credentials_valid(password) def test_create_user_bad_payload(): @@ -37,13 +37,13 @@ def test_update_user_password(): assert user.username == "user_1" assert user.password_reset_at is None - assert user.credentials_valid(password) + assert user.authorization.credentials_valid(password) new_password = "new_test_password" user.update_password(MagicMock(), new_password) assert user.username == "user_1" assert user.password_reset_at is not None - assert user.credentials_valid(new_password) + assert user.authorization.credentials_valid(new_password) assert user.hashed_password != new_password - assert not user.credentials_valid(password) + assert not user.authorization.credentials_valid(password) diff --git a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py index c7471723649..6b3bcad44b5 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py @@ -1144,7 +1144,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( PrivacyNoticeRegion.us_va, ], consent_mechanism=ConsentMechanism.opt_in, - data_uses=["essential.service.operations.support.optimization"], + data_uses=["essential.service.operations.improve"], enforcement_level=EnforcementLevel.system_wide, created_at=NOW, updated_at=NOW, @@ -1181,7 +1181,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( ], ) ], - "essential.service.operations.support.optimization": [ + "essential.service.operations.improve": [ PrivacyNoticeResponse( id=f"{PRIVACY_NOTICE_NAME}-4", name=f"{PRIVACY_NOTICE_NAME}-4", @@ -1192,9 +1192,7 @@ def test_get_privacy_notice_by_data_use_wrong_scope( PrivacyNoticeRegion.us_va, ], consent_mechanism=ConsentMechanism.opt_in, - data_uses=[ - "essential.service.operations.support.optimization" - ], + data_uses=["essential.service.operations.improve"], enforcement_level=EnforcementLevel.system_wide, created_at=NOW, updated_at=NOW, @@ -2302,7 +2300,9 @@ def test_patch_invalid_privacy_notice_data_use_conflicts_within_request( # conflict with parent/child data uses within region # we make the two patch payload items have parent/child data uses, they should fail - patch_privacy_notice_us_ca_updated_data_use["data_uses"] = ["improve.system"] + patch_privacy_notice_us_ca_updated_data_use["data_uses"] = [ + "functional.service.improve" + ] resp = api_client.patch( url, diff --git a/tests/ops/api/v1/endpoints/test_user_endpoints.py b/tests/ops/api/v1/endpoints/test_user_endpoints.py index 2a057c400f6..030933ffbf1 100644 --- a/tests/ops/api/v1/endpoints/test_user_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_endpoints.py @@ -741,7 +741,7 @@ def test_update_different_user_password( db.expunge(application_user) application_user = application_user.refresh_from_db(db=db) - assert application_user.credentials_valid(password=OLD_PASSWORD) + assert application_user.authorization.credentials_valid(password=OLD_PASSWORD) def test_update_user_password_invalid( self, @@ -768,7 +768,7 @@ def test_update_user_password_invalid( db.expunge(application_user) application_user = application_user.refresh_from_db(db=db) - assert application_user.credentials_valid(password=OLD_PASSWORD) + assert application_user.authorization.credentials_valid(password=OLD_PASSWORD) def test_update_user_password( self, @@ -792,7 +792,7 @@ def test_update_user_password( assert resp.status_code == HTTP_200_OK db.expunge(application_user) application_user = application_user.refresh_from_db(db=db) - assert application_user.credentials_valid(password=NEW_PASSWORD) + assert application_user.authorization.credentials_valid(password=NEW_PASSWORD) def test_force_update_different_user_password_without_scope( self, @@ -849,7 +849,7 @@ def test_force_update_different_user_password( assert resp.status_code == HTTP_200_OK db.expunge(user) user = user.refresh_from_db(db=db) - assert user.credentials_valid(password=NEW_PASSWORD) + assert user.authorization.credentials_valid(password=NEW_PASSWORD) def test_force_update_non_existent_user( self, diff --git a/tests/ops/integration_tests/test_mongo_task.py b/tests/ops/integration_tests/test_mongo_task.py index 0ebcf5448a0..2fd53fb928e 100644 --- a/tests/ops/integration_tests/test_mongo_task.py +++ b/tests/ops/integration_tests/test_mongo_task.py @@ -531,8 +531,8 @@ async def test_object_querying_mongo( ) target_categories = { - "user.gender", - "user.date_of_birth", + "user.demographic.gender", + "user.demographic.date_of_birth", } filtered_results = filter_data_categories( access_request_results, @@ -848,7 +848,7 @@ async def test_array_querying_mongo( { "path": "mongo_test:customer_details:birthday", "field_name": "birthday", - "data_categories": ["user.date_of_birth"], + "data_categories": ["user.demographic.date_of_birth"], }, { "path": "mongo_test:customer_details:children", @@ -873,7 +873,7 @@ async def test_array_querying_mongo( { "path": "mongo_test:customer_details:gender", "field_name": "gender", - "data_categories": ["user.gender"], + "data_categories": ["user.demographic.gender"], }, { "path": "mongo_test:customer_details:workplace_info.direct_reports", @@ -930,7 +930,7 @@ async def test_array_querying_mongo( { "path": "mongo_test:conversations:thread.ccn", "field_name": "thread.ccn", - "data_categories": ["user.financial.account_number"], + "data_categories": ["user.financial.bank_account"], }, ] @@ -1015,7 +1015,7 @@ async def test_array_querying_mongo( { "path": "mongo_test:payment_card:ccn", "field_name": "ccn", - "data_categories": ["user.financial.account_number"], + "data_categories": ["user.financial.bank_account"], }, { "path": "mongo_test:payment_card:code", diff --git a/tests/ops/integration_tests/test_sql_task.py b/tests/ops/integration_tests/test_sql_task.py index d0935690c01..6506df4e482 100644 --- a/tests/ops/integration_tests/test_sql_task.py +++ b/tests/ops/integration_tests/test_sql_task.py @@ -49,7 +49,7 @@ "user.contact.address.postal_code", "user.contact.address.state", "user.contact.address.street", - "user.financial.account_number", + "user.financial.bank_account", "user.financial", "user.name", "user", diff --git a/tests/ops/models/test_consent_request.py b/tests/ops/models/test_consent_request.py index f8570a14fd8..5976d8c5722 100644 --- a/tests/ops/models/test_consent_request.py +++ b/tests/ops/models/test_consent_request.py @@ -40,7 +40,7 @@ def test_consent(db): consent_data_2 = { "provided_identity_id": provided_identity.id, - "data_use": "user.browsing_history", + "data_use": "user.behavior.browsing_history", "opt_in": False, } consent_2 = Consent.create(db, data=consent_data_2) @@ -74,7 +74,7 @@ def test_consent_with_gpc(db): consent_data_2 = { "provided_identity_id": provided_identity.id, - "data_use": "user.browsing_history", + "data_use": "user.behavior.browsing_history", "opt_in": False, } consent_2 = Consent.create(db, data=consent_data_2) diff --git a/tests/ops/models/test_fidesopsuser.py b/tests/ops/models/test_fidesopsuser.py index 1cecf48e063..65d9ebf7b5e 100644 --- a/tests/ops/models/test_fidesopsuser.py +++ b/tests/ops/models/test_fidesopsuser.py @@ -18,8 +18,8 @@ def test_create_user(self, db: Session) -> None: assert user.last_login_at is None assert user.password_reset_at is None - assert not user.credentials_valid("bad_password") - assert user.credentials_valid("test_password") + assert not user.authorization.credentials_valid("bad_password") + assert user.authorization.credentials_valid("test_password") user.delete(db) @@ -38,14 +38,14 @@ def test_update_user_password(self, db: Session) -> None: assert user.username == "user_1" assert user.password_reset_at is None - assert user.credentials_valid("test_password") + assert user.authorization.credentials_valid("test_password") user.update_password(db, "new_test_password") assert user.username == "user_1" assert user.password_reset_at is not None - assert user.credentials_valid("new_test_password") + assert user.authorization.credentials_valid("new_test_password") assert user.hashed_password != "new_test_password" - assert not user.credentials_valid("test_password") + assert not user.authorization.credentials_valid("test_password") user.delete(db) diff --git a/tests/ops/service/connectors/test_consent_email_connector.py b/tests/ops/service/connectors/test_consent_email_connector.py index aa823f37168..c35280f7b4b 100644 --- a/tests/ops/service/connectors/test_consent_email_connector.py +++ b/tests/ops/service/connectors/test_consent_email_connector.py @@ -288,7 +288,7 @@ def test_send_single_consent_email_preferences_by_privacy_notice( id="test_2", privacy_notice_id="67890", consent_mechanism=ConsentMechanism.opt_out, - data_uses=["improve.system"], + data_uses=["functional.service.improve"], enforcement_level=EnforcementLevel.system_wide, version=1.0, displayed_in_overlay=True, diff --git a/tests/ops/service/connectors/test_queryconfig.py b/tests/ops/service/connectors/test_queryconfig.py index 555ae5abffa..a73de222b1a 100644 --- a/tests/ops/service/connectors/test_queryconfig.py +++ b/tests/ops/service/connectors/test_queryconfig.py @@ -579,7 +579,7 @@ def test_generate_update_stmt_multiple_rules( "configuration": {"algorithm": "SHA-512"}, } target = rule.targets[0] - target.data_category = DataCategory("user.date_of_birth").value + target.data_category = DataCategory("user.demographic.date_of_birth").value rule_two = erasure_policy_two_rules.rules[1] rule_two.masking_strategy = { @@ -587,7 +587,7 @@ def test_generate_update_stmt_multiple_rules( "configuration": {"length": 30}, } target = rule_two.targets[0] - target.data_category = DataCategory("user.gender").value + target.data_category = DataCategory("user.demographic.gender").value # cache secrets for hash strategy secret = MaskingSecretCache[str]( secret="adobo", diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index 42778f01bfa..13f55f7217c 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -505,7 +505,7 @@ def test_email_dispatch_consent_request_email_fulfillment_for_sovrn_new_workflow id="test_3", privacy_notice_id="39391", consent_mechanism=ConsentMechanism.opt_in, - data_uses=["improve.system"], + data_uses=["functional.service.improve"], enforcement_level=EnforcementLevel.system_wide, version=1.0, displayed_in_overlay=True, diff --git a/tests/ops/task/test_filter_results.py b/tests/ops/task/test_filter_results.py index 227e1da5d87..9e9a015b56b 100644 --- a/tests/ops/task/test_filter_results.py +++ b/tests/ops/task/test_filter_results.py @@ -830,7 +830,7 @@ def test_filter_data_categories_limited_results(self): "id", ), ], - "user.financial.account_number": [ + "user.financial.bank_account": [ FieldPath( "ccn", ) @@ -935,7 +935,7 @@ def test_filter_data_categories_limited_results(self): "_id", ) ], - "user.date_of_birth": [ + "user.demographic.date_of_birth": [ FieldPath( "birthday", ) @@ -945,7 +945,7 @@ def test_filter_data_categories_limited_results(self): "customer_id", ) ], - "user.gender": [ + "user.demographic.gender": [ FieldPath( "gender", ) From 78ab451d1d953efdc6b4b43944ea4bf61e3bcc65 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 18 Aug 2023 01:22:08 +0800 Subject: [PATCH 06/44] feat: bump fideslang version again for more fixes --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 24ae4a909b8..ae9a6465cec 100644 --- a/requirements.txt +++ b/requirements.txt @@ -14,7 +14,7 @@ fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 # fideslang==2.0.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@008bb46b3ed92a7b9359c9ab7007aade3a45609d#egg=fideslang +fideslang @ git+https://github.com/ethyca/fideslang.git@ae2031a213a630a1b5e08f01155efb3dde5e642b#egg=fideslang fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From 655fcbf227939738c7d863f919333ea91707b9d1 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 18 Aug 2023 01:58:43 +0800 Subject: [PATCH 07/44] fix: ctl api tests --- tests/ctl/core/test_api.py | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 8b0bc075dad..cf9aeb06e68 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -2163,7 +2163,11 @@ def test_api_cannot_create_default_taxonomy( self, test_config: FidesConfig, resources_dict: Dict, endpoint: str ) -> None: manifest = resources_dict[endpoint] + + # Set fields for default labels manifest.is_default = True + manifest.version_added = "2.0.0" + result = _api.create( url=test_config.cli.server_url, resource_type=endpoint, @@ -2184,7 +2188,11 @@ def test_api_cannot_upsert_default_taxonomy( self, test_config: FidesConfig, resources_dict: Dict, endpoint: str ) -> None: manifest = resources_dict[endpoint] + + # Set fields for default labels manifest.is_default = True + manifest.version_added = "2.0.0" + result = _api.upsert( url=test_config.cli.server_url, headers=test_config.user.auth_header, @@ -2212,7 +2220,10 @@ def test_api_cannot_update_is_default( headers=test_config.user.auth_header, ) + # Set fields for default labels manifest.is_default = True + manifest.version_added = "2.0.0" + result = _api.update( url=test_config.cli.server_url, headers=test_config.user.auth_header, @@ -2226,8 +2237,12 @@ def test_api_cannot_upsert_is_default( self, test_config: FidesConfig, resources_dict: Dict, endpoint: str ) -> None: manifest = resources_dict[endpoint] - manifest.is_default = True second_item = manifest.copy() + + # Set fields for default labels + manifest.is_default = True + manifest.version_added = "2.0.0" + second_item.is_default = False _api.create( @@ -2329,6 +2344,7 @@ def test_api_can_create_with_active_property( ) # cast resource to extended model resource.fides_key = resource.fides_key + "_test_create_active_false" resource.is_default = False + resource.version_added = None resource.active = False json_resource = resource.json(exclude_none=True) token_scopes: List[str] = [f"{CLI_SCOPE_PREFIX_MAPPING[endpoint]}:{CREATE}"] From 18c493c4c4764f59b39fd785544ae95ca27996db Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Thu, 17 Aug 2023 17:02:08 -0400 Subject: [PATCH 08/44] fix: revert mistaken user.authorization.credentials_valid --> user.credentials_valid function --- src/fides/api/api/v1/endpoints/user_endpoints.py | 2 +- src/fides/api/db/seed.py | 4 +--- tests/ctl/api/test_seed.py | 5 +---- tests/lib/test_fides_user.py | 10 +++++----- tests/ops/api/v1/endpoints/test_user_endpoints.py | 8 ++++---- tests/ops/models/test_fidesopsuser.py | 10 +++++----- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/user_endpoints.py b/src/fides/api/api/v1/endpoints/user_endpoints.py index 3e3ee2dc073..cb4bc33b62c 100644 --- a/src/fides/api/api/v1/endpoints/user_endpoints.py +++ b/src/fides/api/api/v1/endpoints/user_endpoints.py @@ -151,7 +151,7 @@ def update_user_password( """ _validate_current_user(user_id, current_user) - if not current_user.authorization.credentials_valid( + if not current_user.credentials_valid( b64_str_to_str(data.old_password), CONFIG.security.encoding ): raise HTTPException( diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index 93ebc78a819..e3415d5423b 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -92,9 +92,7 @@ def create_or_update_parent_user() -> None: ) if user and CONFIG.security.parent_server_password: - if not user.authorization.credentials_valid( - CONFIG.security.parent_server_password - ): + if not user.credentials_valid(CONFIG.security.parent_server_password): log.debug("Updating Fides parent user credentials") user.update_password(db_session, CONFIG.security.parent_server_password) return diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index ad49c12788a..25aa7dd1cf5 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -326,10 +326,7 @@ def test_create_or_update_parent_user_change_password(db): db.refresh(user) assert user.password_reset_at is not None - assert ( - user.authorization.credentials_valid(CONFIG.security.parent_server_password) - is True - ) + assert user.credentials_valid(CONFIG.security.parent_server_password) is True user.delete(db) diff --git a/tests/lib/test_fides_user.py b/tests/lib/test_fides_user.py index 63eb72d1dab..5752562f52a 100644 --- a/tests/lib/test_fides_user.py +++ b/tests/lib/test_fides_user.py @@ -16,8 +16,8 @@ def test_create_user(): assert user.username == "user_1" assert user.hashed_password != "test_password" - assert not user.authorization.credentials_valid("bad_password") - assert user.authorization.credentials_valid(password) + assert not user.credentials_valid("bad_password") + assert user.credentials_valid(password) def test_create_user_bad_payload(): @@ -37,13 +37,13 @@ def test_update_user_password(): assert user.username == "user_1" assert user.password_reset_at is None - assert user.authorization.credentials_valid(password) + assert user.credentials_valid(password) new_password = "new_test_password" user.update_password(MagicMock(), new_password) assert user.username == "user_1" assert user.password_reset_at is not None - assert user.authorization.credentials_valid(new_password) + assert user.credentials_valid(new_password) assert user.hashed_password != new_password - assert not user.authorization.credentials_valid(password) + assert not user.credentials_valid(password) diff --git a/tests/ops/api/v1/endpoints/test_user_endpoints.py b/tests/ops/api/v1/endpoints/test_user_endpoints.py index 030933ffbf1..2a057c400f6 100644 --- a/tests/ops/api/v1/endpoints/test_user_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_user_endpoints.py @@ -741,7 +741,7 @@ def test_update_different_user_password( db.expunge(application_user) application_user = application_user.refresh_from_db(db=db) - assert application_user.authorization.credentials_valid(password=OLD_PASSWORD) + assert application_user.credentials_valid(password=OLD_PASSWORD) def test_update_user_password_invalid( self, @@ -768,7 +768,7 @@ def test_update_user_password_invalid( db.expunge(application_user) application_user = application_user.refresh_from_db(db=db) - assert application_user.authorization.credentials_valid(password=OLD_PASSWORD) + assert application_user.credentials_valid(password=OLD_PASSWORD) def test_update_user_password( self, @@ -792,7 +792,7 @@ def test_update_user_password( assert resp.status_code == HTTP_200_OK db.expunge(application_user) application_user = application_user.refresh_from_db(db=db) - assert application_user.authorization.credentials_valid(password=NEW_PASSWORD) + assert application_user.credentials_valid(password=NEW_PASSWORD) def test_force_update_different_user_password_without_scope( self, @@ -849,7 +849,7 @@ def test_force_update_different_user_password( assert resp.status_code == HTTP_200_OK db.expunge(user) user = user.refresh_from_db(db=db) - assert user.authorization.credentials_valid(password=NEW_PASSWORD) + assert user.credentials_valid(password=NEW_PASSWORD) def test_force_update_non_existent_user( self, diff --git a/tests/ops/models/test_fidesopsuser.py b/tests/ops/models/test_fidesopsuser.py index 65d9ebf7b5e..1cecf48e063 100644 --- a/tests/ops/models/test_fidesopsuser.py +++ b/tests/ops/models/test_fidesopsuser.py @@ -18,8 +18,8 @@ def test_create_user(self, db: Session) -> None: assert user.last_login_at is None assert user.password_reset_at is None - assert not user.authorization.credentials_valid("bad_password") - assert user.authorization.credentials_valid("test_password") + assert not user.credentials_valid("bad_password") + assert user.credentials_valid("test_password") user.delete(db) @@ -38,14 +38,14 @@ def test_update_user_password(self, db: Session) -> None: assert user.username == "user_1" assert user.password_reset_at is None - assert user.authorization.credentials_valid("test_password") + assert user.credentials_valid("test_password") user.update_password(db, "new_test_password") assert user.username == "user_1" assert user.password_reset_at is not None - assert user.authorization.credentials_valid("new_test_password") + assert user.credentials_valid("new_test_password") assert user.hashed_password != "new_test_password" - assert not user.authorization.credentials_valid("test_password") + assert not user.credentials_valid("test_password") user.delete(db) From 9c4fe119b49793f9919a5e1d1361bc74ce8618e1 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Thu, 17 Aug 2023 17:13:16 -0400 Subject: [PATCH 09/44] remove unused exception var --- src/fides/api/db/seed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index e3415d5423b..0bf45c6d3a1 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -401,7 +401,7 @@ async def load_default_taxonomy(async_session: AsyncSession) -> None: await upsert_resources( sql_model_map[resource_type], resources, async_session ) - except QueryError as e: # pragma: no cover + except QueryError: # pragma: no cover pass # The create_resource function will log the error else: log.debug(f"UPSERTED {len(resources)} {resource_type} resource(s)") From a887ec2f7cee1829ebba62d8dd3873d3f47a8c46 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Thu, 17 Aug 2023 17:20:48 -0400 Subject: [PATCH 10/44] feat: add new taxonomy model columns to db dataset annotation --- .fides/db_dataset.yml | 48 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index b741671f295..f37cf9714b0 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -402,6 +402,18 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_added + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_deprecated + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: replaced_by + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_data_qualifiers data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -449,6 +461,18 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_added + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_deprecated + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: replaced_by + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_data_subjects data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -504,6 +528,18 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_added + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_deprecated + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: replaced_by + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_data_uses data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: @@ -574,6 +610,18 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_added + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: version_deprecated + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: replaced_by + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: ctl_datasets data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: From b26a26f7c20ef256d5aacf0be8ccd3755193af1d Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Thu, 17 Aug 2023 17:25:07 -0400 Subject: [PATCH 11/44] feat: update to use new data_use in systems.yml --- .fides/systems.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.fides/systems.yml b/.fides/systems.yml index 8943068a960..37952df903f 100644 --- a/.fides/systems.yml +++ b/.fides/systems.yml @@ -20,7 +20,7 @@ system: data_categories: - system.operations - user.contact - data_use: improve.system + data_use: functional.service.improve data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified data_subjects: - anonymous_user From e2409496fbc4e2cb4ad35eb52377f6ac9d885423 Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 18 Aug 2023 13:09:48 +0800 Subject: [PATCH 12/44] fix: more ctl tests --- tests/ctl/api/test_seed.py | 16 ++++++++-------- tests/ctl/core/test_evaluate.py | 3 ++- tests/ctl/core/test_system.py | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/tests/ctl/api/test_seed.py b/tests/ctl/api/test_seed.py index 25aa7dd1cf5..9c746c2f532 100644 --- a/tests/ctl/api/test_seed.py +++ b/tests/ctl/api/test_seed.py @@ -86,7 +86,7 @@ def test_filter_data_categories_excluded(self) -> None: excluded_data_categories = [ "user.financial", "user.payment", - "user.authorization.credentials", + "user.authorization", ] all_data_categories = [ "user.name", @@ -95,7 +95,7 @@ def test_filter_data_categories_excluded(self) -> None: "user.payment", "user.payment.financial_account_number", "user.authorization.credentials", - "user.authorization.credentials.biometric_credentials", + "user.authorization.biometric", "user.financial.bank_account", "user.financial", ] @@ -112,7 +112,7 @@ def test_filter_data_categories_no_third_level(self) -> None: excluded_data_categories = [ "user.financial", "user.payment", - "user.authorization.credentials", + "user.authorization", ] all_data_categories = [ "user.name", @@ -121,7 +121,7 @@ def test_filter_data_categories_no_third_level(self) -> None: "user.payment", "user.payment.financial_account_number", "user.authorization.credentials", - "user.authorization.credentials.biometric_credentials", + "user.authorization.biometric", "user.financial.bank_account", "user.financial", ] @@ -151,7 +151,7 @@ def test_filter_data_categories_empty_excluded(self) -> None: all_data_categories = [ "user.name", "user.payment", - "user.authorization.credentials", + "user.authorization", "user.financial", ] assert seed.filter_data_categories(all_data_categories, []) == sorted( @@ -163,7 +163,7 @@ def test_filter_data_categories_no_exclusions(self) -> None: excluded_data_categories = ["user.payment"] all_data_categories = [ "user.name", - "user.authorization.credentials", + "user.authorization", "user.financial", ] assert seed.filter_data_categories( @@ -174,7 +174,7 @@ def test_filter_data_categories_only_return_users(self) -> None: """Test that the filter method works as intended""" all_data_categories = [ "user.name", - "user.authorization.credentials", + "user.authorization", "user.financial", # These are excluded "nonuser.foo", @@ -182,7 +182,7 @@ def test_filter_data_categories_only_return_users(self) -> None: ] expected_categories = [ "user.name", - "user.authorization.credentials", + "user.authorization", "user.financial", ] assert seed.filter_data_categories(all_data_categories, []) == sorted( diff --git a/tests/ctl/core/test_evaluate.py b/tests/ctl/core/test_evaluate.py index ebdaabd3881..faf580ed503 100644 --- a/tests/ctl/core/test_evaluate.py +++ b/tests/ctl/core/test_evaluate.py @@ -576,7 +576,8 @@ def test_failed_evaluation_error_message( 'system (customer_data_sharing_system) failed ' 'rule (reject_targeted_marketing) from policy ' '(primary_privacy_policy). Violated usage of ' - 'data categories (user.demographic.political_opinion) with ' + 'data categories ' + '(user.demographic.political_opinion) with ' 'qualifier ' '(aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified) ' 'for data uses ' diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index c67f768891b..b353a0120ea 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -371,7 +371,7 @@ async def test_cookie_system( PrivacyDeclarationSchema( name="declaration-name-2", data_categories=[], - data_use="improve", + data_use="functional.service.improve", data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], From 4ac028b3f155919366bad0a9c4661ad8a3e3af7b Mon Sep 17 00:00:00 2001 From: Thomas Date: Fri, 18 Aug 2023 18:43:35 +0800 Subject: [PATCH 13/44] feat: add number labels for tests with param --- tests/ctl/core/test_api.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index cf9aeb06e68..0f3a5a01f24 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -1715,7 +1715,7 @@ def test_system_update_updates_declarations( "update_declarations", [ ( - [ # update a dec matching one existing dec + [ # Check 1: update a dec matching one existing dec models.PrivacyDeclaration( name="Collect data for marketing", data_categories=[], @@ -1727,7 +1727,7 @@ def test_system_update_updates_declarations( ] ), ( - [ # add a new single dec with same data use + [ # Check 2: add a new single dec with same data use models.PrivacyDeclaration( name="declaration-name-1", data_categories=[], @@ -1739,7 +1739,7 @@ def test_system_update_updates_declarations( ] ), ( - [ # add a new single dec with same data use, no name + [ # Check 3: add a new single dec with same data use, no name models.PrivacyDeclaration( name="", data_categories=[], @@ -1751,7 +1751,7 @@ def test_system_update_updates_declarations( ] ), ( - # update 2 privacy declarations both matching existing decs + # Check 4: update 2 privacy declarations both matching existing decs [ models.PrivacyDeclaration( name="Collect data for marketing", @@ -1772,7 +1772,7 @@ def test_system_update_updates_declarations( ] ), ( - # update 2 privacy declarations, one with matching name and data use, other only data use + # Check 5: update 2 privacy declarations, one with matching name and data use, other only data use [ models.PrivacyDeclaration( name="Collect data for marketing", @@ -1793,7 +1793,7 @@ def test_system_update_updates_declarations( ] ), ( - # update 2 privacy declarations, one with matching name and data use, other only data use but same data use + # Check 6: update 2 privacy declarations, one with matching name and data use, other only data use but same data use [ models.PrivacyDeclaration( name="Collect data for marketing", @@ -1814,7 +1814,7 @@ def test_system_update_updates_declarations( ] ), ( - # update 2 privacy declarations, one with only matching data use, other totally new + # Check 7: update 2 privacy declarations, one with only matching data use, other totally new [ models.PrivacyDeclaration( name="declaration-name-1", @@ -1835,7 +1835,7 @@ def test_system_update_updates_declarations( ] ), ( - # add 2 new privacy declarations + # Check 8: add 2 new privacy declarations [ models.PrivacyDeclaration( name="declaration-name", @@ -1856,7 +1856,7 @@ def test_system_update_updates_declarations( ] ), ( - # add 2 new privacy declarations, same data uses as existing decs but no names + # Check 9: add 2 new privacy declarations, same data uses as existing decs but no names [ models.PrivacyDeclaration( name="", @@ -1877,7 +1877,7 @@ def test_system_update_updates_declarations( ] ), ( - # specify no declarations, declarations should be cleared off the system + # Check 10: specify no declarations, declarations should be cleared off the system [] ), ], From 8992d135d6060583bab516e8a451c01c5b78975c Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Fri, 18 Aug 2023 08:42:13 -0400 Subject: [PATCH 14/44] fix: ctl test using outdated 'improve' data_use --- tests/ctl/core/test_api.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/ctl/core/test_api.py b/tests/ctl/core/test_api.py index 0f3a5a01f24..b50e22e8896 100644 --- a/tests/ctl/core/test_api.py +++ b/tests/ctl/core/test_api.py @@ -1848,7 +1848,7 @@ def test_system_update_updates_declarations( models.PrivacyDeclaration( name="declaration-name-2", data_categories=[], - data_use="improve", + data_use="functional", data_subjects=[], data_qualifier="aggregated_data", dataset_references=[], From 8ec6a87331fde0b328cab0045950ec361340c6ce Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Fri, 18 Aug 2023 09:46:26 -0400 Subject: [PATCH 15/44] fix: clean up ops tests based on updates --- .../api/service/connectors/consent_email_connector.py | 2 +- .../api/v1/endpoints/test_privacy_notice_endpoints.py | 6 +++--- tests/ops/models/test_policy.py | 5 ++++- .../service/connectors/test_consent_email_connector.py | 5 +---- tests/ops/util/test_consent_util.py | 9 ++++----- 5 files changed, 13 insertions(+), 14 deletions(-) diff --git a/src/fides/api/service/connectors/consent_email_connector.py b/src/fides/api/service/connectors/consent_email_connector.py index 4b6c03c0bef..af7cd8192bd 100644 --- a/src/fides/api/service/connectors/consent_email_connector.py +++ b/src/fides/api/service/connectors/consent_email_connector.py @@ -99,7 +99,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: identities=self.identities_for_test_email, consent_preferences=[ # TODO slated for deprecation Consent(data_use="marketing.advertising", opt_in=False), - Consent(data_use="improve", opt_in=True), + Consent(data_use="functional", opt_in=True), ], privacy_preferences=[ MinimalPrivacyPreferenceHistorySchema( diff --git a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py index 6b3bcad44b5..fa2cf17fa2a 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_notice_endpoints.py @@ -2275,14 +2275,14 @@ def test_patch_invalid_privacy_notice_data_use_conflicts_within_request( patch_privacy_notice_payload_updated_data_use = ( patch_privacy_notice_payload.copy() ) - patch_privacy_notice_payload_updated_data_use["data_uses"] = ["improve"] + patch_privacy_notice_payload_updated_data_use["data_uses"] = ["functional"] # ensure we are not disabling privacy notice, because that will bypass validation patch_privacy_notice_payload_updated_data_use["disabled"] = False patch_privacy_notice_us_ca_updated_data_use = ( patch_privacy_notice_payload_us_ca_provide.copy() ) - patch_privacy_notice_us_ca_updated_data_use["data_uses"] = ["improve"] + patch_privacy_notice_us_ca_updated_data_use["data_uses"] = ["functional"] resp = api_client.patch( url, @@ -2295,7 +2295,7 @@ def test_patch_invalid_privacy_notice_data_use_conflicts_within_request( assert resp.status_code == 422 assert ( resp.json()["detail"] - == "Privacy Notice 'my notice's name' has already assigned data use 'improve' to region 'PrivacyNoticeRegion.us_ca'" + == "Privacy Notice 'my notice's name' has already assigned data use 'functional' to region 'PrivacyNoticeRegion.us_ca'" ) # conflict with parent/child data uses within region diff --git a/tests/ops/models/test_policy.py b/tests/ops/models/test_policy.py index a91a4013a14..bfef5f0af69 100644 --- a/tests/ops/models/test_policy.py +++ b/tests/ops/models/test_policy.py @@ -216,7 +216,10 @@ def test_create_rule_target_invalid_data_category( ) assert exc.value.__class__ == DataCategoryNotSupported - assert exc.value.args[0] == "The data category a_fake_category is not supported." + assert ( + exc.value.args[0] + == "The data category 'a_fake_category' was not found in the database, and is therefore not valid for use here." + ) def test_create_rule_target_valid_data_category( diff --git a/tests/ops/service/connectors/test_consent_email_connector.py b/tests/ops/service/connectors/test_consent_email_connector.py index c35280f7b4b..8c341582585 100644 --- a/tests/ops/service/connectors/test_consent_email_connector.py +++ b/tests/ops/service/connectors/test_consent_email_connector.py @@ -551,10 +551,7 @@ def test_test_connection_call( ) assert preferences[0]["consent_preferences"][0]["opt_in"] is False - assert ( - preferences[0]["consent_preferences"][1]["data_use"] - == "Improves the product, service, application or system." - ) + assert preferences[0]["consent_preferences"][1]["data_use"] == "Functional" assert preferences[0]["consent_preferences"][1]["opt_in"] is True assert ( diff --git a/tests/ops/util/test_consent_util.py b/tests/ops/util/test_consent_util.py index 069f43d2546..dd818312700 100644 --- a/tests/ops/util/test_consent_util.py +++ b/tests/ops/util/test_consent_util.py @@ -931,7 +931,7 @@ def test_create_two_templates_then_update_second(self, db, load_default_data_use name="B", regions=["it"], consent_mechanism=ConsentMechanism.opt_in, - data_uses=["improve"], + data_uses=["functional"], enforcement_level=EnforcementLevel.frontend, disabled=True, displayed_in_overlay=True, @@ -955,7 +955,7 @@ def test_create_two_templates_then_update_second(self, db, load_default_data_use assert second_template.name == "B" assert second_template.regions == [PrivacyNoticeRegion.it] assert second_template.consent_mechanism == ConsentMechanism.opt_in - assert second_template.data_uses == ["improve"] + assert second_template.data_uses == ["functional"] assert second_template.enforcement_level == EnforcementLevel.frontend assert second_template.disabled @@ -979,7 +979,7 @@ def test_create_two_templates_then_update_second(self, db, load_default_data_use name="C", regions=["it"], consent_mechanism=ConsentMechanism.opt_out, - data_uses=["improve"], + data_uses=["functional"], enforcement_level=EnforcementLevel.system_wide, disabled=False, displayed_in_overlay=True, @@ -1018,7 +1018,7 @@ def test_create_two_templates_then_update_second(self, db, load_default_data_use assert third_template.name == "C" assert third_template.regions == [PrivacyNoticeRegion.it] assert third_template.consent_mechanism == ConsentMechanism.opt_out - assert third_template.data_uses == ["improve"] + assert third_template.data_uses == ["functional"] assert third_template.enforcement_level == EnforcementLevel.system_wide assert not third_template.disabled @@ -1196,7 +1196,6 @@ def custom_data_use(self, db): name="New data use", description="A test data use", parent_key=None, - is_default=True, ).dict(), ) From b4e266ded99c87a799d52e52246740132200247f Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 22 Aug 2023 03:01:09 +0800 Subject: [PATCH 16/44] feat: start adding migrations for data categories --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 92 +++++++++++++++---- ...1ba_add_version_fields_to_default_types.py | 3 +- .../post_processor_strategy_filter.py | 3 +- 3 files changed, 77 insertions(+), 21 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 6c9e5b097fe..705a8289bfb 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -33,6 +33,54 @@ value: key for key, value in data_use_upgrades.items() } + +def update_privacy_declaration_data_uses( + bind: Connection, data_use_map: Dict[str, str] +) -> None: + """Upgrade or downgrade data uses from fideslang 1.4 for privacy declarations""" + existing_ctl_policies: ResultProxy = bind.execute( + text("SELECT id, data_use FROM privacydeclaration;") + ) + for row in existing_ctl_policies: + data_use: str = row["data_use"] + updated_data_use: Optional[str] = data_use_map.get(data_use, None) + + if updated_data_use: + update_data_use_query: TextClause = text( + "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" + ) + bind.execute( + update_data_use_query, + {"declaration_id": row["id"], "updated_use": updated_data_use}, + ) + + +def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) -> None: + """Upgrade or downgrade data uses from fideslang 1.4 for ctl policies""" + existing_ctl_policies: ResultProxy = bind.execute( + text("SELECT id, rules FROM ctl_policies;") + ) + for row in existing_ctl_policies: + needs_update: bool = False + rules: List[Dict] = row["rules"] + for i, rule in enumerate(rules or []): + data_uses: Dict = rule.get("data_uses", {}) + for j, val in enumerate(data_uses.get("values", [])): + new_data_use: Optional[str] = data_use_map.get(val, None) + if new_data_use: + rules[i]["data_uses"]["values"][j] = new_data_use + needs_update = True + + if needs_update: + update_data_use_query: TextClause = text( + "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" + ) + bind.execute( + update_data_use_query, + {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, + ) + + ##################### ## Data Categories ## ##################### @@ -59,29 +107,37 @@ value: key for key, value in data_use_upgrades.items() } +# Data Category fides keys are referenced in the following places: +# - Datasets (fields, collections and dataset-level) +# - Policy Rule - DONE +# - Privacy Declaration - DONE +# - Data Flow -def update_privacy_declaration_data_uses( - bind: Connection, data_use_map: Dict[str, str] + +def update_privacy_declaration_data_categories( + bind: Connection, data_label_map: Dict[str, str] ) -> None: """Upgrade or downgrade data uses from fideslang 1.4 for privacy declarations""" existing_ctl_policies: ResultProxy = bind.execute( - text("SELECT id, data_use FROM privacydeclaration;") + text("SELECT id, data_category FROM privacydeclaration;") ) for row in existing_ctl_policies: - data_use: str = row["data_use"] - updated_data_use: Optional[str] = data_use_map.get(data_use, None) + label: str = row["data_category"] + updated_label: Optional[str] = data_label_map.get(label, None) - if updated_data_use: - update_data_use_query: TextClause = text( - "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" + if updated_label: + update_label_query: TextClause = text( + "UPDATE privacydeclaration SET data_category = :updated_label WHERE id= :declaration_id" ) bind.execute( - update_data_use_query, - {"declaration_id": row["id"], "updated_use": updated_data_use}, + update_label_query, + {"declaration_id": row["id"], "updated_label": updated_label}, ) -def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) -> None: +def update_ctl_policy_data_categories( + bind: Connection, data_label_map: Dict[str, str] +) -> None: """Upgrade or downgrade data uses from fideslang 1.4 for ctl policies""" existing_ctl_policies: ResultProxy = bind.execute( text("SELECT id, rules FROM ctl_policies;") @@ -90,19 +146,19 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) needs_update: bool = False rules: List[Dict] = row["rules"] for i, rule in enumerate(rules or []): - data_uses: Dict = rule.get("data_uses", {}) - for j, val in enumerate(data_uses.get("values", [])): - new_data_use: Optional[str] = data_use_map.get(val, None) - if new_data_use: - rules[i]["data_uses"]["values"][j] = new_data_use + data_labels: Dict = rule.get("data_categories", {}) + for j, val in enumerate(data_labels.get("values", [])): + new_data_label: Optional[str] = data_label_map.get(val, None) + if new_data_label: + rules[i]["data_categories"]["values"][j] = new_data_label needs_update = True if needs_update: - update_data_use_query: TextClause = text( + update_data_label_query: TextClause = text( "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" ) bind.execute( - update_data_use_query, + update_data_label_query, {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, ) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index 22ba06547d4..3e88445a792 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -5,9 +5,8 @@ Create Date: 2023-08-17 12:29:04.855626 """ -from alembic import op import sqlalchemy as sa - +from alembic import op # revision identifiers, used by Alembic. revision = "708a780b01ba" diff --git a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py index 3027e3e018a..8f71b7f5064 100644 --- a/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py +++ b/src/fides/api/service/processors/post_processor_strategy/post_processor_strategy_filter.py @@ -183,7 +183,8 @@ def _get_nested_values(self, data: Dict[str, Any], path: str) -> Any: if len(components) > 1: if isinstance(value, dict): return self._get_nested_values(value, ".".join(components[1:])) - elif isinstance(value, list): + + if isinstance(value, list): return pydash.flatten( [ self._get_nested_values(item, ".".join(components[1:])) From 96cc06748c315db7b0dbafd0fae1c8d303df7fd9 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 22 Aug 2023 06:39:11 +0800 Subject: [PATCH 17/44] feat: add a data migration for the datasets --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 58 +++++++++++++++++++ 1 file changed, 58 insertions(+) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 705a8289bfb..a244712ec34 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -114,6 +114,46 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) # - Data Flow +def update_datasets_data_categories( + bind: Connection, data_label_map: Dict[str, str] +) -> None: + """Upgrade the datasets in the database to use the new data categories.""" + + # Get all datasets out of the database + existing_datasets: ResultProxy = bind.execute( + text("SELECT id, data_categories, collections FROM ctl_datasets;") + ) + + for row in existing_datasets: + # Update data categories at the top level + labels: List[str] = row["data_categories"] + updated_labels: List[str] = [ + data_label_map.get(label, label) for label in labels + ] + + update_label_query: TextClause = text( + "UPDATE datasets SET data_categories = :updated_labels WHERE id= :dataset_id" + ) + bind.execute( + update_label_query, + {"dataset_id": row["id"], "updated_labels": updated_labels}, + ) + + # Update the collections objects + collections = row["collections"] + + for key, value in data_label_map.items(): + collections.replace(key, value) + + update_collections_query: TextClause = text( + "UPDATE datasets SET collections = :updated_labels WHERE id= :dataset_id" + ) + bind.execute( + update_collections_query, + {"dataset_id": row["id"], "updated_labels": collections}, + ) + + def update_privacy_declaration_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: @@ -172,6 +212,15 @@ def upgrade() -> None: logger.info("Upgrading ctl policy rule data uses") update_ctl_policy_data_uses(bind, data_use_upgrades) + logger.info("Upgrading data category on privacy declaration") + update_privacy_declaration_data_uses(bind, data_category_upgrades) + + logger.info("Upgrading ctl policy rule data categories") + update_ctl_policy_data_uses(bind, data_category_upgrades) + + logger.info("Upgrading data categories in datasets") + update_datasets_data_categories(bind, data_category_upgrades) + def downgrade() -> None: bind: Connection = op.get_bind() @@ -181,3 +230,12 @@ def downgrade() -> None: logger.info("Downgrading ctl policy rule data uses") update_ctl_policy_data_uses(bind, data_use_downgrades) + + logger.info("Downgrading data category on privacy declaration") + update_privacy_declaration_data_uses(bind, data_category_downgrades) + + logger.info("Downgrading ctl policy rule data categories") + update_ctl_policy_data_uses(bind, data_category_downgrades) + + logger.info("Downgrading data categories in datasets") + update_datasets_data_categories(bind, data_category_downgrades) From 6c97d88fbef93641e6533e888855609a8778d364 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 22 Aug 2023 07:04:17 +0800 Subject: [PATCH 18/44] feat: update to the new pinned fideslang version --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ae9a6465cec..5253ff3a7f9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,8 +13,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -# fideslang==2.0.0 -fideslang @ git+https://github.com/ethyca/fideslang.git@ae2031a213a630a1b5e08f01155efb3dde5e642b#egg=fideslang +fideslang==2.0.1 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From c0aabac623cdf8511d505409c63a7e06a4c98847 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 22 Aug 2023 08:06:39 +0800 Subject: [PATCH 19/44] feat: add system updates --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 48 ++++++++++++++++--- 1 file changed, 42 insertions(+), 6 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index a244712ec34..e4d48f4062a 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -107,12 +107,6 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) value: key for key, value in data_use_upgrades.items() } -# Data Category fides keys are referenced in the following places: -# - Datasets (fields, collections and dataset-level) -# - Policy Rule - DONE -# - Privacy Declaration - DONE -# - Data Flow - def update_datasets_data_categories( bind: Connection, data_label_map: Dict[str, str] @@ -154,6 +148,42 @@ def update_datasets_data_categories( ) +def update_system_ingress_egress_data_categories( + bind: Connection, data_label_map: Dict[str, str] +) -> None: + """Upgrade or downgrade system DataFlow objects""" + existing_systems: ResultProxy = bind.execute( + text("SELECT id, egress, ingress FROM ctl_systems;") + ) + + for row in existing_systems: + ingress = row["ingress"] + egress = row["egress"] + + # Do a blunt find/replace + for key, value in data_label_map.items(): + ingress.replace(key, value) + egress.replace(key, value) + + # Insert ingress changes + update_ingress_query: TextClause = text( + "UPDATE ctl_systems SET ingress = :updated_ingress WHERE id= :system_id" + ) + bind.execute( + update_ingress_query, + {"system_id": row["id"], "updated_ingress": ingress}, + ) + + # Insert egress changes + update_egress_query: TextClause = text( + "UPDATE ctl_systems SET egress = :updated_egress WHERE id= :system_id" + ) + bind.execute( + update_egress_query, + {"system_id": row["id"], "updated_egress": egress}, + ) + + def update_privacy_declaration_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: @@ -221,6 +251,9 @@ def upgrade() -> None: logger.info("Upgrading data categories in datasets") update_datasets_data_categories(bind, data_category_upgrades) + logger.info("Upgrading the System egress/ingress data cateogries") + update_system_ingress_egress_data_categories(bind, data_category_upgrades) + def downgrade() -> None: bind: Connection = op.get_bind() @@ -239,3 +272,6 @@ def downgrade() -> None: logger.info("Downgrading data categories in datasets") update_datasets_data_categories(bind, data_category_downgrades) + + logger.info("Downgrading the System egress/ingress data cateogries") + update_system_ingress_egress_data_categories(bind, data_category_downgrades) From 526b023a2bc43be97d51619feb4597480edb8f2f Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 23 Aug 2023 05:12:22 +0800 Subject: [PATCH 20/44] feat: small naming nit --- .../1e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index e4d48f4062a..4cb880c9c92 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -140,11 +140,11 @@ def update_datasets_data_categories( collections.replace(key, value) update_collections_query: TextClause = text( - "UPDATE datasets SET collections = :updated_labels WHERE id= :dataset_id" + "UPDATE datasets SET collections = :updated_collections WHERE id= :dataset_id" ) bind.execute( update_collections_query, - {"dataset_id": row["id"], "updated_labels": collections}, + {"dataset_id": row["id"], "updated_collections": collections}, ) From 7b22f0e1e790f3bb770efe9aa5a94e5f1daede01 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 23 Aug 2023 07:12:04 +0800 Subject: [PATCH 21/44] fix: migrations --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 89 ++++++++++--------- ...1ba_add_version_fields_to_default_types.py | 2 +- 2 files changed, 48 insertions(+), 43 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 4cb880c9c92..e45a8b9558b 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -121,26 +121,28 @@ def update_datasets_data_categories( for row in existing_datasets: # Update data categories at the top level labels: List[str] = row["data_categories"] - updated_labels: List[str] = [ - data_label_map.get(label, label) for label in labels - ] - update_label_query: TextClause = text( - "UPDATE datasets SET data_categories = :updated_labels WHERE id= :dataset_id" - ) - bind.execute( - update_label_query, - {"dataset_id": row["id"], "updated_labels": updated_labels}, - ) + if labels: + updated_labels: List[str] = [ + data_label_map.get(label, label) for label in labels + ] + + update_label_query: TextClause = text( + "UPDATE ctl_datasets SET data_categories = :updated_labels WHERE id= :dataset_id" + ) + bind.execute( + update_label_query, + {"dataset_id": row["id"], "updated_labels": updated_labels}, + ) # Update the collections objects - collections = row["collections"] + collections = json.dumps(row["collections"]) for key, value in data_label_map.items(): collections.replace(key, value) update_collections_query: TextClause = text( - "UPDATE datasets SET collections = :updated_collections WHERE id= :dataset_id" + "UPDATE ctl_datasets SET collections = :updated_collections WHERE id= :dataset_id" ) bind.execute( update_collections_query, @@ -162,26 +164,29 @@ def update_system_ingress_egress_data_categories( # Do a blunt find/replace for key, value in data_label_map.items(): - ingress.replace(key, value) - egress.replace(key, value) + if ingress: + ingress.replace(key, value) - # Insert ingress changes - update_ingress_query: TextClause = text( - "UPDATE ctl_systems SET ingress = :updated_ingress WHERE id= :system_id" - ) - bind.execute( - update_ingress_query, - {"system_id": row["id"], "updated_ingress": ingress}, - ) + if egress: + egress.replace(key, value) - # Insert egress changes - update_egress_query: TextClause = text( - "UPDATE ctl_systems SET egress = :updated_egress WHERE id= :system_id" - ) - bind.execute( - update_egress_query, - {"system_id": row["id"], "updated_egress": egress}, - ) + if ingress: + update_ingress_query: TextClause = text( + "UPDATE ctl_systems SET ingress = :updated_ingress WHERE id= :system_id" + ) + bind.execute( + update_ingress_query, + {"system_id": row["id"], "updated_ingress": ingress}, + ) + + if egress: + update_egress_query: TextClause = text( + "UPDATE ctl_systems SET egress = :updated_egress WHERE id= :system_id" + ) + bind.execute( + update_egress_query, + {"system_id": row["id"], "updated_egress": egress}, + ) def update_privacy_declaration_data_categories( @@ -189,20 +194,20 @@ def update_privacy_declaration_data_categories( ) -> None: """Upgrade or downgrade data uses from fideslang 1.4 for privacy declarations""" existing_ctl_policies: ResultProxy = bind.execute( - text("SELECT id, data_category FROM privacydeclaration;") + text("SELECT id, data_categories FROM privacydeclaration;") ) for row in existing_ctl_policies: - label: str = row["data_category"] - updated_label: Optional[str] = data_label_map.get(label, None) + labels: List[str] = [ + data_label_map.get(label, label) for label in row["data_categories"] + ] - if updated_label: - update_label_query: TextClause = text( - "UPDATE privacydeclaration SET data_category = :updated_label WHERE id= :declaration_id" - ) - bind.execute( - update_label_query, - {"declaration_id": row["id"], "updated_label": updated_label}, - ) + update_label_query: TextClause = text( + "UPDATE privacydeclaration SET data_categories = :updated_label WHERE id= :declaration_id" + ) + bind.execute( + update_label_query, + {"declaration_id": row["id"], "updated_label": labels}, + ) def update_ctl_policy_data_categories( @@ -243,7 +248,7 @@ def upgrade() -> None: update_ctl_policy_data_uses(bind, data_use_upgrades) logger.info("Upgrading data category on privacy declaration") - update_privacy_declaration_data_uses(bind, data_category_upgrades) + update_privacy_declaration_data_categories(bind, data_category_upgrades) logger.info("Upgrading ctl policy rule data categories") update_ctl_policy_data_uses(bind, data_category_upgrades) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index 3e88445a792..e872e63c1be 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "708a780b01ba" -down_revision = "fd52d5f08c17" +down_revision = "507563f6f8d4" branch_labels = None depends_on = None From 72c8797d18ae56e7b89cc699c40b48158c7bbe42 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 24 Aug 2023 00:24:09 +0800 Subject: [PATCH 22/44] checkin --- .../1e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index e45a8b9558b..438c12a3625 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -120,7 +120,7 @@ def update_datasets_data_categories( for row in existing_datasets: # Update data categories at the top level - labels: List[str] = row["data_categories"] + labels: Optional[List[str]] = row["data_categories"] if labels: updated_labels: List[str] = [ @@ -136,7 +136,7 @@ def update_datasets_data_categories( ) # Update the collections objects - collections = json.dumps(row["collections"]) + collections: str = json.dumps(row["collections"]) for key, value in data_label_map.items(): collections.replace(key, value) From 2f2eb0f447295bd0698410b6dede1736121c6931 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 24 Aug 2023 07:22:53 +0800 Subject: [PATCH 23/44] fix: migrations for egress/ingress on systems --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 23 ++++++++++++------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 438c12a3625..65c238892bf 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -139,7 +139,7 @@ def update_datasets_data_categories( collections: str = json.dumps(row["collections"]) for key, value in data_label_map.items(): - collections.replace(key, value) + collections = collections.replace(key, value) update_collections_query: TextClause = text( "UPDATE ctl_datasets SET collections = :updated_collections WHERE id= :dataset_id" @@ -163,12 +163,19 @@ def update_system_ingress_egress_data_categories( egress = row["egress"] # Do a blunt find/replace - for key, value in data_label_map.items(): - if ingress: - ingress.replace(key, value) + if ingress: + for item in ingress: + item["data_categories"] = [ + data_label_map.get(label, label) + for label in item.get("data_categories", []) + ] - if egress: - egress.replace(key, value) + if egress: + for item in egress: + item["data_categories"] = [ + data_label_map.get(label, label) + for label in item.get("data_categories", []) + ] if ingress: update_ingress_query: TextClause = text( @@ -176,7 +183,7 @@ def update_system_ingress_egress_data_categories( ) bind.execute( update_ingress_query, - {"system_id": row["id"], "updated_ingress": ingress}, + {"system_id": row["id"], "updated_ingress": json.dumps(ingress)}, ) if egress: @@ -185,7 +192,7 @@ def update_system_ingress_egress_data_categories( ) bind.execute( update_egress_query, - {"system_id": row["id"], "updated_egress": egress}, + {"system_id": row["id"], "updated_egress": json.dumps(egress)}, ) From 24c6c17387ebfce4a18f77abb075d18a5fe72c5e Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 26 Aug 2023 02:41:18 +0800 Subject: [PATCH 24/44] fix: incorrect function calls --- .../1e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 65c238892bf..aebda77c12e 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -258,7 +258,7 @@ def upgrade() -> None: update_privacy_declaration_data_categories(bind, data_category_upgrades) logger.info("Upgrading ctl policy rule data categories") - update_ctl_policy_data_uses(bind, data_category_upgrades) + update_ctl_policy_data_categories(bind, data_category_upgrades) logger.info("Upgrading data categories in datasets") update_datasets_data_categories(bind, data_category_upgrades) @@ -277,10 +277,10 @@ def downgrade() -> None: update_ctl_policy_data_uses(bind, data_use_downgrades) logger.info("Downgrading data category on privacy declaration") - update_privacy_declaration_data_uses(bind, data_category_downgrades) + update_privacy_declaration_data_categories(bind, data_category_downgrades) logger.info("Downgrading ctl policy rule data categories") - update_ctl_policy_data_uses(bind, data_category_downgrades) + update_ctl_policy_data_categories(bind, data_category_downgrades) logger.info("Downgrading data categories in datasets") update_datasets_data_categories(bind, data_category_downgrades) From c79cbe88d376a799463da9111885ac8117c95b86 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 26 Aug 2023 14:57:21 +0800 Subject: [PATCH 25/44] Apply suggestions from code review Co-authored-by: Dawn Pattison --- ...e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index aebda77c12e..b0951024aef 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -37,7 +37,7 @@ def update_privacy_declaration_data_uses( bind: Connection, data_use_map: Dict[str, str] ) -> None: - """Upgrade or downgrade data uses from fideslang 1.4 for privacy declarations""" + """Upgrade or downgrade data uses from fideslang 2.0 for privacy declarations""" existing_ctl_policies: ResultProxy = bind.execute( text("SELECT id, data_use FROM privacydeclaration;") ) @@ -56,7 +56,7 @@ def update_privacy_declaration_data_uses( def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) -> None: - """Upgrade or downgrade data uses from fideslang 1.4 for ctl policies""" + """Upgrade or downgrade data uses from fideslang 2.0 for ctl policies""" existing_ctl_policies: ResultProxy = bind.execute( text("SELECT id, rules FROM ctl_policies;") ) @@ -199,7 +199,7 @@ def update_system_ingress_egress_data_categories( def update_privacy_declaration_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: - """Upgrade or downgrade data uses from fideslang 1.4 for privacy declarations""" + """Upgrade or downgrade data uses from fideslang 2.0 for data categories""" existing_ctl_policies: ResultProxy = bind.execute( text("SELECT id, data_categories FROM privacydeclaration;") ) @@ -220,7 +220,7 @@ def update_privacy_declaration_data_categories( def update_ctl_policy_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: - """Upgrade or downgrade data uses from fideslang 1.4 for ctl policies""" + """Upgrade or downgrade data uses from fideslang 2.0 for ctl policies""" existing_ctl_policies: ResultProxy = bind.execute( text("SELECT id, rules FROM ctl_policies;") ) @@ -263,7 +263,7 @@ def upgrade() -> None: logger.info("Upgrading data categories in datasets") update_datasets_data_categories(bind, data_category_upgrades) - logger.info("Upgrading the System egress/ingress data cateogries") + logger.info("Upgrading the System egress/ingress data categories") update_system_ingress_egress_data_categories(bind, data_category_upgrades) @@ -285,5 +285,5 @@ def downgrade() -> None: logger.info("Downgrading data categories in datasets") update_datasets_data_categories(bind, data_category_downgrades) - logger.info("Downgrading the System egress/ingress data cateogries") + logger.info("Downgrading the System egress/ingress data categories") update_system_ingress_egress_data_categories(bind, data_category_downgrades) From 166ac042a2e7caf7813c72faffc1f8b772725bd0 Mon Sep 17 00:00:00 2001 From: Thomas Date: Sat, 26 Aug 2023 14:58:54 +0800 Subject: [PATCH 26/44] fix: misname --- .../versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 2 +- .../api/service/saas_request/saas_request_override_factory.py | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index aebda77c12e..c3e7381093e 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -104,7 +104,7 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) "user.genetic": "user.health_and_medical.genetic", # Verified in 2.0 } data_category_downgrades: Dict[str, str] = { - value: key for key, value in data_use_upgrades.items() + value: key for key, value in data_category_upgrades.items() } diff --git a/src/fides/api/service/saas_request/saas_request_override_factory.py b/src/fides/api/service/saas_request/saas_request_override_factory.py index 10a5c2c77c5..476221a7ff2 100644 --- a/src/fides/api/service/saas_request/saas_request_override_factory.py +++ b/src/fides/api/service/saas_request/saas_request_override_factory.py @@ -166,4 +166,5 @@ def validate_update_override_function(f: Callable) -> None: ) +# TODO: Avoid running this on import? register = SaaSRequestOverrideFactory.register From 887e7688048399298db9af93a3ca768b899d5c23 Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 28 Aug 2023 10:41:37 +0800 Subject: [PATCH 27/44] feat: update the data migration to do broader replacement in order to catch child items --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 132 ++++++++++-------- 1 file changed, 75 insertions(+), 57 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index e75b9ee1666..9d86fc7ba08 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -23,11 +23,14 @@ ############### ## Data Uses ## ############### -# The `key` is the old value, the `value` is the new value +""" +The `key` is the old value, the `value` is the new value +These are ordered specifically so that string replacement works on both parent and child items +""" data_use_upgrades: Dict[str, str] = { - "improve.system": "functional.service.improve", # Verified in 2.0 - "improve": "functional", # Posted a question in the Confluence doc to verify this - "essential.service.operations.support.optimization": "essential.service.operations.improve", # Verified in 2.0 + "essential.service.operations.support.optimization": "essential.service.operations.improve", + "improve.system": "functional.service.improve", + "improve": "functional", } data_use_downgrades: Dict[str, str] = { value: key for key, value in data_use_upgrades.items() @@ -38,21 +41,22 @@ def update_privacy_declaration_data_uses( bind: Connection, data_use_map: Dict[str, str] ) -> None: """Upgrade or downgrade data uses from fideslang 2.0 for privacy declarations""" - existing_ctl_policies: ResultProxy = bind.execute( + existing_privacy_declarations: ResultProxy = bind.execute( text("SELECT id, data_use FROM privacydeclaration;") ) - for row in existing_ctl_policies: + for row in existing_privacy_declarations: data_use: str = row["data_use"] - updated_data_use: Optional[str] = data_use_map.get(data_use, None) + for key, value in data_use_map.items(): + if key in data_use: + data_use = data_use.replace(key, value) - if updated_data_use: - update_data_use_query: TextClause = text( - "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" - ) - bind.execute( - update_data_use_query, - {"declaration_id": row["id"], "updated_use": updated_data_use}, - ) + update_data_use_query: TextClause = text( + "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" + ) + bind.execute( + update_data_use_query, + {"declaration_id": row["id"], "updated_use": data_use}, + ) def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) -> None: @@ -61,47 +65,49 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) text("SELECT id, rules FROM ctl_policies;") ) for row in existing_ctl_policies: - needs_update: bool = False rules: List[Dict] = row["rules"] for i, rule in enumerate(rules or []): data_uses: Dict = rule.get("data_uses", {}) for j, val in enumerate(data_uses.get("values", [])): - new_data_use: Optional[str] = data_use_map.get(val, None) - if new_data_use: - rules[i]["data_uses"]["values"][j] = new_data_use - needs_update = True - - if needs_update: - update_data_use_query: TextClause = text( - "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" - ) - bind.execute( - update_data_use_query, - {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, - ) + data_use: str = val + for key, value in data_use_map.items(): + if key in data_use: + data_use = data_use.replace(key, value) + rules[i]["data_uses"]["values"][j] = data_use + + update_data_use_query: TextClause = text( + "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" + ) + bind.execute( + update_data_use_query, + {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, + ) ##################### ## Data Categories ## ##################### -# The `key` is the old value, the `value` is the new value +""" +The `key` is the old value, the `value` is the new value +These are ordered specifically so that string replacement works on both parent and child items +""" data_category_upgrades: Dict[str, str] = { - "user.credentials": "user.authorization.credentials", # Verified in 2.0 - "user.observed": "user.behavior", # Verified in 2.0 - "user.browsing_history": "user.behavior.browsing_history", # Verified in 2.0 - "user.media_consumption": "user.behavior.media_consumption", # Verified in 2.0 - "user.search_history": "user.behavior.search_history", # Verified in 2.0 - "user.organization": "user.contact.organization", # Verified in 2.0 - "user.non_specific_age": "user.demographic.age_range", # Verified in 2.0 - "user.date_of_birth": "user.demographic.date_of_birth", # Verified in 2.0 - "user.gender": "user.demographic.gender", # Verified in 2.0 - "user.political_opinion": "user.demographic.political_opinion", # Verified in 2.0 - "user.profiling": "user.demographic.profile", # Verified in 2.0 - "user.race": "user.demographic.race_ethnicity", # Verified in 2.0 - "user.religious_belief": "user.demographic.religious_belief", # Verified in 2.0 - "user.sexual_orientation": "user.demographic.sexual_orientation", # Verified in 2.0 - "user.financial.account_number": "user.financial.bank_account", # Verified in 2.0 - "user.genetic": "user.health_and_medical.genetic", # Verified in 2.0 + "user.financial.account_number": "user.financial.bank_account", + "user.credentials": "user.authorization.credentials", + "user.observed": "user.behavior", + "user.browsing_history": "user.behavior.browsing_history", + "user.media_consumption": "user.behavior.media_consumption", + "user.search_history": "user.behavior.search_history", + "user.organization": "user.contact.organization", + "user.non_specific_age": "user.demographic.age_range", + "user.date_of_birth": "user.demographic.date_of_birth", + "user.gender": "user.demographic.gender", + "user.political_opinion": "user.demographic.political_opinion", + "user.profiling": "user.demographic.profile", + "user.race": "user.demographic.race_ethnicity", + "user.religious_belief": "user.demographic.religious_belief", + "user.sexual_orientation": "user.demographic.sexual_orientation", + "user.genetic": "user.health_and_medical.genetic", } data_category_downgrades: Dict[str, str] = { value: key for key, value in data_category_upgrades.items() @@ -123,8 +129,12 @@ def update_datasets_data_categories( labels: Optional[List[str]] = row["data_categories"] if labels: + # Do a string replace here to catch child items updated_labels: List[str] = [ - data_label_map.get(label, label) for label in labels + label.replace(key, value) + for key, value in data_label_map.items() + for label in labels + if key in label ] update_label_query: TextClause = text( @@ -166,15 +176,19 @@ def update_system_ingress_egress_data_categories( if ingress: for item in ingress: item["data_categories"] = [ - data_label_map.get(label, label) + label.replace(key, value) + for key, value in data_label_map.items() for label in item.get("data_categories", []) + if key in label ] if egress: for item in egress: item["data_categories"] = [ - data_label_map.get(label, label) + label.replace(key, value) + for key, value in data_label_map.items() for label in item.get("data_categories", []) + if key in label ] if ingress: @@ -200,12 +214,15 @@ def update_privacy_declaration_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: """Upgrade or downgrade data uses from fideslang 2.0 for data categories""" - existing_ctl_policies: ResultProxy = bind.execute( + existing_privacy_declarations: ResultProxy = bind.execute( text("SELECT id, data_categories FROM privacydeclaration;") ) - for row in existing_ctl_policies: - labels: List[str] = [ - data_label_map.get(label, label) for label in row["data_categories"] + for row in existing_privacy_declarations: + labels = [ + label.replace(key, value) + for key, value in data_label_map.items() + for label in row["data_categories"] + if key in label ] update_label_query: TextClause = text( @@ -230,10 +247,11 @@ def update_ctl_policy_data_categories( for i, rule in enumerate(rules or []): data_labels: Dict = rule.get("data_categories", {}) for j, val in enumerate(data_labels.get("values", [])): - new_data_label: Optional[str] = data_label_map.get(val, None) - if new_data_label: - rules[i]["data_categories"]["values"][j] = new_data_label - needs_update = True + data_category: str = val + for key, value in data_label_map.items(): + if key in data_category: + data_category = data_category.replace(key, value) + rules[i]["data_uses"]["values"][j] = data_category if needs_update: update_data_label_query: TextClause = text( From 791f67febfd63149b1b0ec506bc86e37418ba34a Mon Sep 17 00:00:00 2001 From: Thomas Date: Mon, 28 Aug 2023 11:03:28 +0800 Subject: [PATCH 28/44] fix: use 'startswith' instead of 'in' to avoid overwriting changes --- .../1e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 9d86fc7ba08..8cb41a0a8ad 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -47,7 +47,7 @@ def update_privacy_declaration_data_uses( for row in existing_privacy_declarations: data_use: str = row["data_use"] for key, value in data_use_map.items(): - if key in data_use: + if data_use.startswith(key): data_use = data_use.replace(key, value) update_data_use_query: TextClause = text( @@ -71,7 +71,7 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) for j, val in enumerate(data_uses.get("values", [])): data_use: str = val for key, value in data_use_map.items(): - if key in data_use: + if data_use.startswith(key): data_use = data_use.replace(key, value) rules[i]["data_uses"]["values"][j] = data_use @@ -134,7 +134,7 @@ def update_datasets_data_categories( label.replace(key, value) for key, value in data_label_map.items() for label in labels - if key in label + if label.startswith(key) ] update_label_query: TextClause = text( From bb67fa12f4e55f2492131e611fe23984d8d99a01 Mon Sep 17 00:00:00 2001 From: Thomas Date: Tue, 29 Aug 2023 11:31:40 +0800 Subject: [PATCH 29/44] fix: don't filter out categories that aren't updated --- ...50936a141_migrate_to_fideslang_2_0_uses_and_.py | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 8cb41a0a8ad..74b31f9d70d 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -94,7 +94,6 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) data_category_upgrades: Dict[str, str] = { "user.financial.account_number": "user.financial.bank_account", "user.credentials": "user.authorization.credentials", - "user.observed": "user.behavior", "user.browsing_history": "user.behavior.browsing_history", "user.media_consumption": "user.behavior.media_consumption", "user.search_history": "user.behavior.search_history", @@ -108,6 +107,7 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) "user.religious_belief": "user.demographic.religious_belief", "user.sexual_orientation": "user.demographic.sexual_orientation", "user.genetic": "user.health_and_medical.genetic", + "user.observed": "user.behavior", } data_category_downgrades: Dict[str, str] = { value: key for key, value in data_category_upgrades.items() @@ -131,10 +131,9 @@ def update_datasets_data_categories( if labels: # Do a string replace here to catch child items updated_labels: List[str] = [ - label.replace(key, value) + label.replace(key, value) if label.startswith(key) else label for key, value in data_label_map.items() for label in labels - if label.startswith(key) ] update_label_query: TextClause = text( @@ -176,19 +175,17 @@ def update_system_ingress_egress_data_categories( if ingress: for item in ingress: item["data_categories"] = [ - label.replace(key, value) + label.replace(key, value) if key in label else label for key, value in data_label_map.items() for label in item.get("data_categories", []) - if key in label ] if egress: for item in egress: item["data_categories"] = [ - label.replace(key, value) + label.replace(key, value) if key in label else label for key, value in data_label_map.items() for label in item.get("data_categories", []) - if key in label ] if ingress: @@ -219,10 +216,9 @@ def update_privacy_declaration_data_categories( ) for row in existing_privacy_declarations: labels = [ - label.replace(key, value) + label.replace(key, value) if key in label else label for key, value in data_label_map.items() for label in row["data_categories"] - if key in label ] update_label_query: TextClause = text( From 26e120390e1399450eb56949bbc366815cb3cc12 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Tue, 29 Aug 2023 14:54:10 -0400 Subject: [PATCH 30/44] bump to new fideslang release to keep up to date --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5253ff3a7f9..62010d2b1db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]~= 0.10.0 -fideslang==2.0.1 +fideslang==2.0.2 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 From 4249d8b03f00bc338cba913bb9e6e62906271b26 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Tue, 29 Aug 2023 14:59:59 -0400 Subject: [PATCH 31/44] update down rev on migration --- .../708a780b01ba_add_version_fields_to_default_types.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index e872e63c1be..d7a80116b39 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -1,7 +1,7 @@ """add version fields to default types Revision ID: 708a780b01ba -Revises: 1e750936a141 +Revises: 3038667ba898 Create Date: 2023-08-17 12:29:04.855626 """ @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "708a780b01ba" -down_revision = "507563f6f8d4" +down_revision = "3038667ba898" branch_labels = None depends_on = None From 3792c71d6531165395657981c7e8d0c7a447af70 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 31 Aug 2023 15:29:09 +0800 Subject: [PATCH 32/44] fix: review comments --- data/saas/dataset/sentry_dataset.yml | 2 +- ...936a141_migrate_to_fideslang_2_0_uses_and_.py | 16 +++++++--------- 2 files changed, 8 insertions(+), 10 deletions(-) diff --git a/data/saas/dataset/sentry_dataset.yml b/data/saas/dataset/sentry_dataset.yml index 172bb08333f..a3bfa79aa61 100644 --- a/data/saas/dataset/sentry_dataset.yml +++ b/data/saas/dataset/sentry_dataset.yml @@ -734,7 +734,7 @@ dataset: fidesops_meta: data_type: string - name: username - data_categories: [user.credentials] + data_categories: [user.authorization.credentials] fidesops_meta: data_type: string - name: email diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index 74b31f9d70d..ff03b975640 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -238,7 +238,6 @@ def update_ctl_policy_data_categories( text("SELECT id, rules FROM ctl_policies;") ) for row in existing_ctl_policies: - needs_update: bool = False rules: List[Dict] = row["rules"] for i, rule in enumerate(rules or []): data_labels: Dict = rule.get("data_categories", {}) @@ -249,14 +248,13 @@ def update_ctl_policy_data_categories( data_category = data_category.replace(key, value) rules[i]["data_uses"]["values"][j] = data_category - if needs_update: - update_data_label_query: TextClause = text( - "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" - ) - bind.execute( - update_data_label_query, - {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, - ) + update_data_label_query: TextClause = text( + "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" + ) + bind.execute( + update_data_label_query, + {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, + ) def upgrade() -> None: From ccf118f379ab5d89a52b71ce0152c58212a44681 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 31 Aug 2023 15:54:26 +0800 Subject: [PATCH 33/44] feat: add a migration step that removes all "old" default categories and uses --- .../1e750936a141_migrate_to_fideslang_2_0_uses_and_.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index ff03b975640..d18f1fd0ce3 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -260,6 +260,13 @@ def update_ctl_policy_data_categories( def upgrade() -> None: bind: Connection = op.get_bind() + # Given that our advice is to turn off auto-migrations and make a db copy, + # there is no "downgrade" version of this. It also wouldn't be feasible given + # it would require an older version of fideslang. + logger.info("Removing old default data categories and data uses") + bind.execute(text("DELETE FROM ctl_data_uses WHERE is_default = TRUE;")) + bind.execute(text("DELETE FROM ctl_data_categories WHERE is_default = TRUE;")) + logger.info("Upgrading data use on privacy declaration") update_privacy_declaration_data_uses(bind, data_use_upgrades) From 9308569ad9e9342d9e43256f65a3f9bd266c20cd Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Thu, 31 Aug 2023 17:30:19 -0500 Subject: [PATCH 34/44] Fix some bugs, consolidate a bit: - Double for loops within list comprehensions not correctly constructed for data category upgrades, causing migration to crash, and large data category lists to be added - Data uses in ctl policies were getting replaced with data categories w/ copy/paste - Some category replacements were using "startswith" others were using "in" - Copy paste issues in docstrings --- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 130 ++++++++++-------- 1 file changed, 72 insertions(+), 58 deletions(-) diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py index d18f1fd0ce3..0c84efb63a1 100644 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py @@ -37,6 +37,19 @@ } +def _replace_matching_data_uses( + data_use: Optional[str], data_use_map: Dict[str, str] +) -> Optional[str]: + """Helper method to loop through all data uses in the data use map and iteratively replace any overlaps + in the current data use""" + if not data_use: + return + for key, value in data_use_map.items(): + if data_use.startswith(key): + data_use = data_use.replace(key, value) + return data_use + + def update_privacy_declaration_data_uses( bind: Connection, data_use_map: Dict[str, str] ) -> None: @@ -45,10 +58,7 @@ def update_privacy_declaration_data_uses( text("SELECT id, data_use FROM privacydeclaration;") ) for row in existing_privacy_declarations: - data_use: str = row["data_use"] - for key, value in data_use_map.items(): - if data_use.startswith(key): - data_use = data_use.replace(key, value) + data_use: str = _replace_matching_data_uses(row["data_use"], data_use_map) update_data_use_query: TextClause = text( "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" @@ -68,12 +78,10 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) rules: List[Dict] = row["rules"] for i, rule in enumerate(rules or []): data_uses: Dict = rule.get("data_uses", {}) - for j, val in enumerate(data_uses.get("values", [])): - data_use: str = val - for key, value in data_use_map.items(): - if data_use.startswith(key): - data_use = data_use.replace(key, value) - rules[i]["data_uses"]["values"][j] = data_use + rules[i]["data_uses"]["values"] = [ + _replace_matching_data_uses(use, data_use_map) + for use in data_uses.get("values", []) + ] update_data_use_query: TextClause = text( "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" @@ -114,10 +122,28 @@ def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) } +def _replace_matching_data_categories( + data_categories: List[str], data_label_map: Dict[str, str] +) -> List[str]: + """ + For every data category in the list, loop through every data category upgrade and replace with a match if applicable. + This also picks up category upgrades for child categories + """ + updated_data_categories: List[str] = [] + for category in data_categories or []: + for old_cat, new_cat in data_label_map.items(): + if category.startswith(old_cat): + # Do a string replace to catch child items + category = category.replace(old_cat, new_cat) + updated_data_categories.append(category) + + return updated_data_categories + + def update_datasets_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: - """Upgrade the datasets in the database to use the new data categories.""" + """Upgrade the datasets and their collections/fields in the database to use the new data categories.""" # Get all datasets out of the database existing_datasets: ResultProxy = bind.execute( @@ -126,43 +152,40 @@ def update_datasets_data_categories( for row in existing_datasets: # Update data categories at the top level - labels: Optional[List[str]] = row["data_categories"] - - if labels: - # Do a string replace here to catch child items - updated_labels: List[str] = [ - label.replace(key, value) if label.startswith(key) else label - for key, value in data_label_map.items() - for label in labels - ] + dataset_data_categories: Optional[List[str]] = row["data_categories"] + + if dataset_data_categories: + updated_categories: List[str] = _replace_matching_data_categories( + dataset_data_categories, data_label_map + ) update_label_query: TextClause = text( "UPDATE ctl_datasets SET data_categories = :updated_labels WHERE id= :dataset_id" ) bind.execute( update_label_query, - {"dataset_id": row["id"], "updated_labels": updated_labels}, + {"dataset_id": row["id"], "updated_labels": updated_categories}, ) # Update the collections objects collections: str = json.dumps(row["collections"]) + if collections: + for key, value in data_label_map.items(): + collections = collections.replace(key, value) - for key, value in data_label_map.items(): - collections = collections.replace(key, value) - - update_collections_query: TextClause = text( - "UPDATE ctl_datasets SET collections = :updated_collections WHERE id= :dataset_id" - ) - bind.execute( - update_collections_query, - {"dataset_id": row["id"], "updated_collections": collections}, - ) + update_collections_query: TextClause = text( + "UPDATE ctl_datasets SET collections = :updated_collections WHERE id= :dataset_id" + ) + bind.execute( + update_collections_query, + {"dataset_id": row["id"], "updated_collections": collections}, + ) def update_system_ingress_egress_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: - """Upgrade or downgrade system DataFlow objects""" + """Upgrade or downgrade data categories on system DataFlow objects (egress/ingress)""" existing_systems: ResultProxy = bind.execute( text("SELECT id, egress, ingress FROM ctl_systems;") ) @@ -174,21 +197,10 @@ def update_system_ingress_egress_data_categories( # Do a blunt find/replace if ingress: for item in ingress: - item["data_categories"] = [ - label.replace(key, value) if key in label else label - for key, value in data_label_map.items() - for label in item.get("data_categories", []) - ] - - if egress: - for item in egress: - item["data_categories"] = [ - label.replace(key, value) if key in label else label - for key, value in data_label_map.items() - for label in item.get("data_categories", []) - ] + item["data_categories"] = _replace_matching_data_categories( + item.get("data_categories"), data_label_map + ) - if ingress: update_ingress_query: TextClause = text( "UPDATE ctl_systems SET ingress = :updated_ingress WHERE id= :system_id" ) @@ -198,6 +210,11 @@ def update_system_ingress_egress_data_categories( ) if egress: + for item in egress: + item["data_categories"] = _replace_matching_data_categories( + item.get("data_categories"), data_label_map + ) + update_egress_query: TextClause = text( "UPDATE ctl_systems SET egress = :updated_egress WHERE id= :system_id" ) @@ -215,11 +232,9 @@ def update_privacy_declaration_data_categories( text("SELECT id, data_categories FROM privacydeclaration;") ) for row in existing_privacy_declarations: - labels = [ - label.replace(key, value) if key in label else label - for key, value in data_label_map.items() - for label in row["data_categories"] - ] + labels: Optional[List[str]] = _replace_matching_data_categories( + row["data_categories"], data_label_map + ) update_label_query: TextClause = text( "UPDATE privacydeclaration SET data_categories = :updated_label WHERE id= :declaration_id" @@ -233,7 +248,7 @@ def update_privacy_declaration_data_categories( def update_ctl_policy_data_categories( bind: Connection, data_label_map: Dict[str, str] ) -> None: - """Upgrade or downgrade data uses from fideslang 2.0 for ctl policies""" + """Upgrade or downgrade data categories from fideslang 2.0 for ctl policies""" existing_ctl_policies: ResultProxy = bind.execute( text("SELECT id, rules FROM ctl_policies;") ) @@ -241,12 +256,11 @@ def update_ctl_policy_data_categories( rules: List[Dict] = row["rules"] for i, rule in enumerate(rules or []): data_labels: Dict = rule.get("data_categories", {}) - for j, val in enumerate(data_labels.get("values", [])): - data_category: str = val - for key, value in data_label_map.items(): - if key in data_category: - data_category = data_category.replace(key, value) - rules[i]["data_uses"]["values"][j] = data_category + rule_data_categories: List[str] = data_labels.get("values", []) + updated_data_categories: List[str] = _replace_matching_data_categories( + rule_data_categories, data_label_map + ) + rules[i]["data_categories"]["values"] = updated_data_categories update_data_label_query: TextClause = text( "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" From 08cb5c698156fa133f2b98cd2c0054041ce5acd8 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 6 Sep 2023 11:56:49 +0800 Subject: [PATCH 35/44] feat: pull out the data migration and get the pure upgrade working --- requirements.txt | 2 +- ...a141_migrate_to_fideslang_2_0_uses_and_.py | 322 ------------------ ...1ba_add_version_fields_to_default_types.py | 2 +- 3 files changed, 2 insertions(+), 324 deletions(-) delete mode 100644 src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py diff --git a/requirements.txt b/requirements.txt index 29c15205096..f09d8e70dbf 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,7 +12,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]==0.11.4 -fideslang==1.4.5 +fideslang==2.0.2 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.31 diff --git a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py b/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py deleted file mode 100644 index 0c84efb63a1..00000000000 --- a/src/fides/api/alembic/migrations/versions/1e750936a141_migrate_to_fideslang_2_0_uses_and_.py +++ /dev/null @@ -1,322 +0,0 @@ -"""migrate to fideslang 2.0 uses and categories - -Revision ID: 1e750936a141 -Revises: fd52d5f08c17 -Create Date: 2023-08-17 08:51:47.226612 - -""" -import json -from typing import Dict, List, Optional - -from alembic import op -from loguru import logger -from sqlalchemy import text -from sqlalchemy.engine import Connection, ResultProxy -from sqlalchemy.sql.elements import TextClause - -# revision identifiers, used by Alembic. -revision = "1e750936a141" -down_revision = "708a780b01ba" -branch_labels = None -depends_on = None - -############### -## Data Uses ## -############### -""" -The `key` is the old value, the `value` is the new value -These are ordered specifically so that string replacement works on both parent and child items -""" -data_use_upgrades: Dict[str, str] = { - "essential.service.operations.support.optimization": "essential.service.operations.improve", - "improve.system": "functional.service.improve", - "improve": "functional", -} -data_use_downgrades: Dict[str, str] = { - value: key for key, value in data_use_upgrades.items() -} - - -def _replace_matching_data_uses( - data_use: Optional[str], data_use_map: Dict[str, str] -) -> Optional[str]: - """Helper method to loop through all data uses in the data use map and iteratively replace any overlaps - in the current data use""" - if not data_use: - return - for key, value in data_use_map.items(): - if data_use.startswith(key): - data_use = data_use.replace(key, value) - return data_use - - -def update_privacy_declaration_data_uses( - bind: Connection, data_use_map: Dict[str, str] -) -> None: - """Upgrade or downgrade data uses from fideslang 2.0 for privacy declarations""" - existing_privacy_declarations: ResultProxy = bind.execute( - text("SELECT id, data_use FROM privacydeclaration;") - ) - for row in existing_privacy_declarations: - data_use: str = _replace_matching_data_uses(row["data_use"], data_use_map) - - update_data_use_query: TextClause = text( - "UPDATE privacydeclaration SET data_use = :updated_use WHERE id= :declaration_id" - ) - bind.execute( - update_data_use_query, - {"declaration_id": row["id"], "updated_use": data_use}, - ) - - -def update_ctl_policy_data_uses(bind: Connection, data_use_map: Dict[str, str]) -> None: - """Upgrade or downgrade data uses from fideslang 2.0 for ctl policies""" - existing_ctl_policies: ResultProxy = bind.execute( - text("SELECT id, rules FROM ctl_policies;") - ) - for row in existing_ctl_policies: - rules: List[Dict] = row["rules"] - for i, rule in enumerate(rules or []): - data_uses: Dict = rule.get("data_uses", {}) - rules[i]["data_uses"]["values"] = [ - _replace_matching_data_uses(use, data_use_map) - for use in data_uses.get("values", []) - ] - - update_data_use_query: TextClause = text( - "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" - ) - bind.execute( - update_data_use_query, - {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, - ) - - -##################### -## Data Categories ## -##################### -""" -The `key` is the old value, the `value` is the new value -These are ordered specifically so that string replacement works on both parent and child items -""" -data_category_upgrades: Dict[str, str] = { - "user.financial.account_number": "user.financial.bank_account", - "user.credentials": "user.authorization.credentials", - "user.browsing_history": "user.behavior.browsing_history", - "user.media_consumption": "user.behavior.media_consumption", - "user.search_history": "user.behavior.search_history", - "user.organization": "user.contact.organization", - "user.non_specific_age": "user.demographic.age_range", - "user.date_of_birth": "user.demographic.date_of_birth", - "user.gender": "user.demographic.gender", - "user.political_opinion": "user.demographic.political_opinion", - "user.profiling": "user.demographic.profile", - "user.race": "user.demographic.race_ethnicity", - "user.religious_belief": "user.demographic.religious_belief", - "user.sexual_orientation": "user.demographic.sexual_orientation", - "user.genetic": "user.health_and_medical.genetic", - "user.observed": "user.behavior", -} -data_category_downgrades: Dict[str, str] = { - value: key for key, value in data_category_upgrades.items() -} - - -def _replace_matching_data_categories( - data_categories: List[str], data_label_map: Dict[str, str] -) -> List[str]: - """ - For every data category in the list, loop through every data category upgrade and replace with a match if applicable. - This also picks up category upgrades for child categories - """ - updated_data_categories: List[str] = [] - for category in data_categories or []: - for old_cat, new_cat in data_label_map.items(): - if category.startswith(old_cat): - # Do a string replace to catch child items - category = category.replace(old_cat, new_cat) - updated_data_categories.append(category) - - return updated_data_categories - - -def update_datasets_data_categories( - bind: Connection, data_label_map: Dict[str, str] -) -> None: - """Upgrade the datasets and their collections/fields in the database to use the new data categories.""" - - # Get all datasets out of the database - existing_datasets: ResultProxy = bind.execute( - text("SELECT id, data_categories, collections FROM ctl_datasets;") - ) - - for row in existing_datasets: - # Update data categories at the top level - dataset_data_categories: Optional[List[str]] = row["data_categories"] - - if dataset_data_categories: - updated_categories: List[str] = _replace_matching_data_categories( - dataset_data_categories, data_label_map - ) - - update_label_query: TextClause = text( - "UPDATE ctl_datasets SET data_categories = :updated_labels WHERE id= :dataset_id" - ) - bind.execute( - update_label_query, - {"dataset_id": row["id"], "updated_labels": updated_categories}, - ) - - # Update the collections objects - collections: str = json.dumps(row["collections"]) - if collections: - for key, value in data_label_map.items(): - collections = collections.replace(key, value) - - update_collections_query: TextClause = text( - "UPDATE ctl_datasets SET collections = :updated_collections WHERE id= :dataset_id" - ) - bind.execute( - update_collections_query, - {"dataset_id": row["id"], "updated_collections": collections}, - ) - - -def update_system_ingress_egress_data_categories( - bind: Connection, data_label_map: Dict[str, str] -) -> None: - """Upgrade or downgrade data categories on system DataFlow objects (egress/ingress)""" - existing_systems: ResultProxy = bind.execute( - text("SELECT id, egress, ingress FROM ctl_systems;") - ) - - for row in existing_systems: - ingress = row["ingress"] - egress = row["egress"] - - # Do a blunt find/replace - if ingress: - for item in ingress: - item["data_categories"] = _replace_matching_data_categories( - item.get("data_categories"), data_label_map - ) - - update_ingress_query: TextClause = text( - "UPDATE ctl_systems SET ingress = :updated_ingress WHERE id= :system_id" - ) - bind.execute( - update_ingress_query, - {"system_id": row["id"], "updated_ingress": json.dumps(ingress)}, - ) - - if egress: - for item in egress: - item["data_categories"] = _replace_matching_data_categories( - item.get("data_categories"), data_label_map - ) - - update_egress_query: TextClause = text( - "UPDATE ctl_systems SET egress = :updated_egress WHERE id= :system_id" - ) - bind.execute( - update_egress_query, - {"system_id": row["id"], "updated_egress": json.dumps(egress)}, - ) - - -def update_privacy_declaration_data_categories( - bind: Connection, data_label_map: Dict[str, str] -) -> None: - """Upgrade or downgrade data uses from fideslang 2.0 for data categories""" - existing_privacy_declarations: ResultProxy = bind.execute( - text("SELECT id, data_categories FROM privacydeclaration;") - ) - for row in existing_privacy_declarations: - labels: Optional[List[str]] = _replace_matching_data_categories( - row["data_categories"], data_label_map - ) - - update_label_query: TextClause = text( - "UPDATE privacydeclaration SET data_categories = :updated_label WHERE id= :declaration_id" - ) - bind.execute( - update_label_query, - {"declaration_id": row["id"], "updated_label": labels}, - ) - - -def update_ctl_policy_data_categories( - bind: Connection, data_label_map: Dict[str, str] -) -> None: - """Upgrade or downgrade data categories from fideslang 2.0 for ctl policies""" - existing_ctl_policies: ResultProxy = bind.execute( - text("SELECT id, rules FROM ctl_policies;") - ) - for row in existing_ctl_policies: - rules: List[Dict] = row["rules"] - for i, rule in enumerate(rules or []): - data_labels: Dict = rule.get("data_categories", {}) - rule_data_categories: List[str] = data_labels.get("values", []) - updated_data_categories: List[str] = _replace_matching_data_categories( - rule_data_categories, data_label_map - ) - rules[i]["data_categories"]["values"] = updated_data_categories - - update_data_label_query: TextClause = text( - "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" - ) - bind.execute( - update_data_label_query, - {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, - ) - - -def upgrade() -> None: - bind: Connection = op.get_bind() - - # Given that our advice is to turn off auto-migrations and make a db copy, - # there is no "downgrade" version of this. It also wouldn't be feasible given - # it would require an older version of fideslang. - logger.info("Removing old default data categories and data uses") - bind.execute(text("DELETE FROM ctl_data_uses WHERE is_default = TRUE;")) - bind.execute(text("DELETE FROM ctl_data_categories WHERE is_default = TRUE;")) - - logger.info("Upgrading data use on privacy declaration") - update_privacy_declaration_data_uses(bind, data_use_upgrades) - - logger.info("Upgrading ctl policy rule data uses") - update_ctl_policy_data_uses(bind, data_use_upgrades) - - logger.info("Upgrading data category on privacy declaration") - update_privacy_declaration_data_categories(bind, data_category_upgrades) - - logger.info("Upgrading ctl policy rule data categories") - update_ctl_policy_data_categories(bind, data_category_upgrades) - - logger.info("Upgrading data categories in datasets") - update_datasets_data_categories(bind, data_category_upgrades) - - logger.info("Upgrading the System egress/ingress data categories") - update_system_ingress_egress_data_categories(bind, data_category_upgrades) - - -def downgrade() -> None: - bind: Connection = op.get_bind() - - logger.info("Downgrading data use on privacy declaration") - update_privacy_declaration_data_uses(bind, data_use_downgrades) - - logger.info("Downgrading ctl policy rule data uses") - update_ctl_policy_data_uses(bind, data_use_downgrades) - - logger.info("Downgrading data category on privacy declaration") - update_privacy_declaration_data_categories(bind, data_category_downgrades) - - logger.info("Downgrading ctl policy rule data categories") - update_ctl_policy_data_categories(bind, data_category_downgrades) - - logger.info("Downgrading data categories in datasets") - update_datasets_data_categories(bind, data_category_downgrades) - - logger.info("Downgrading the System egress/ingress data categories") - update_system_ingress_egress_data_categories(bind, data_category_downgrades) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index d7a80116b39..88d4d372216 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -10,7 +10,7 @@ # revision identifiers, used by Alembic. revision = "708a780b01ba" -down_revision = "3038667ba898" +down_revision = "093bb28a8270" branch_labels = None depends_on = None From 289c16c163009fce7fe6dec5c71ab50a4dc0140f Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 6 Sep 2023 18:00:17 +0800 Subject: [PATCH 36/44] fix: loguru-related test error --- src/fides/api/api/v1/endpoints/messaging_endpoints.py | 8 +++++++- tests/ops/api/v1/endpoints/test_messaging_endpoints.py | 7 ++++--- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/fides/api/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/api/v1/endpoints/messaging_endpoints.py index 245e2ce507b..2efb394aaef 100644 --- a/src/fides/api/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/api/v1/endpoints/messaging_endpoints.py @@ -170,7 +170,13 @@ def get_active_default_config(*, db: Session = Depends(deps.get_db)) -> Messagin Retrieves the active default messaging config. """ logger.info("Finding active default messaging config") - messaging_config = MessagingConfig.get_active_default(db) + try: + messaging_config = MessagingConfig.get_active_default(db) + except ValueError: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail="Invalid notification_service_type configured.", + ) if not messaging_config: raise HTTPException( status_code=HTTP_404_NOT_FOUND, diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index bb10a47e3d3..8d0879d9e25 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -1473,14 +1473,15 @@ def test_get_active_default_app_setting_invalid( """ error_message = "Unknown notification_service_type" + response_error = "Invalid notification_service_type configured." auth_header = generate_auth_header([MESSAGING_READ]) - api_client.get( + response = api_client.get( url, headers=auth_header, ) - assert "ERROR" in loguru_caplog.text - assert error_message in loguru_caplog.text + assert response.status_code == 400 + assert response.json().get("detail") == response_error @pytest.mark.usefixtures("notification_service_type_mailgun") def test_get_active_default_config( From 325dd6ef86da1f6a6a878f353f5f2cdafe54c45d Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 6 Sep 2023 20:46:59 +0800 Subject: [PATCH 37/44] fix: drop default uses and categories as part of the 2.0 upgrade --- .../708a780b01ba_add_version_fields_to_default_types.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index 88d4d372216..cf08b070833 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -7,6 +7,7 @@ """ import sqlalchemy as sa from alembic import op +from sqlalchemy.engine import Connection # revision identifiers, used by Alembic. revision = "708a780b01ba" @@ -17,6 +18,10 @@ def upgrade(): # ### commands auto generated by Alembic - please adjust! ### + bind: Connection = op.get_bind() + bind.execute(sa.text("DELETE FROM ctl_data_uses WHERE is_default = TRUE;")) + bind.execute(sa.text("DELETE FROM ctl_data_categories WHERE is_default = TRUE;")) + op.add_column( "ctl_data_categories", sa.Column("version_added", sa.Text(), nullable=True) ) From 87bc2091ba6a1815a80d980bc3a5d5903ca1d448 Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Fri, 8 Sep 2023 09:18:35 -0400 Subject: [PATCH 38/44] fix: drop default data_qualifiers and data_subjects as part of migration --- .../708a780b01ba_add_version_fields_to_default_types.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index cf08b070833..f9f61efe839 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -21,6 +21,8 @@ def upgrade(): bind: Connection = op.get_bind() bind.execute(sa.text("DELETE FROM ctl_data_uses WHERE is_default = TRUE;")) bind.execute(sa.text("DELETE FROM ctl_data_categories WHERE is_default = TRUE;")) + bind.execute(sa.text("DELETE FROM ctl_data_qualifiers WHERE is_default = TRUE;")) + bind.execute(sa.text("DELETE FROM ctl_data_subjects WHERE is_default = TRUE;")) op.add_column( "ctl_data_categories", sa.Column("version_added", sa.Text(), nullable=True) From b6aecaa570d2243fd3c20b43801deb87bb6cf8b2 Mon Sep 17 00:00:00 2001 From: Dawn Pattison Date: Tue, 12 Sep 2023 21:21:47 -0500 Subject: [PATCH 39/44] Update config.json Improve -> Functional (#4076) Co-authored-by: Allison King --- clients/admin-ui/cypress/e2e/taxonomy.cy.ts | 2 +- .../cypress/fixtures/taxonomy/data_uses.json | 22 +++++++++++++++++-- .../cypress-e2e/cypress/e2e/smoke_test.cy.ts | 2 +- clients/privacy-center/config/config.json | 2 +- .../privacy-center/config/examples/full.json | 2 +- .../config/examples/v2Consent.json | 2 +- .../privacy-center/cypress/e2e/consent.cy.ts | 4 ++-- .../cypress/fixtures/config/config_all.json | 2 +- .../fixtures/config/config_consent.json | 2 +- .../cypress/fixtures/consent/verify.json | 2 +- .../privacy_center/config/config.json | 2 +- 11 files changed, 31 insertions(+), 13 deletions(-) diff --git a/clients/admin-ui/cypress/e2e/taxonomy.cy.ts b/clients/admin-ui/cypress/e2e/taxonomy.cy.ts index 159d415460a..bb4bed0917c 100644 --- a/clients/admin-ui/cypress/e2e/taxonomy.cy.ts +++ b/clients/admin-ui/cypress/e2e/taxonomy.cy.ts @@ -104,7 +104,7 @@ describe("Taxonomy management page", () => { { tab: "Data Uses", name: "Improve the capability", - key: "improve", + key: "functional", description: "Improve the product, service, application or system.", parentKey: "", isParent: true, diff --git a/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json b/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json index e8ef7a17be3..b70bd378a3b 100644 --- a/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json +++ b/clients/admin-ui/cypress/fixtures/taxonomy/data_uses.json @@ -90,7 +90,7 @@ "active": true }, { - "fides_key": "improve", + "fides_key": "functional", "organization_fides_key": "default_organization", "tags": null, "name": "Improve the capability", @@ -104,13 +104,31 @@ "is_default": true, "active": true }, + { + "version_added": "2.0.0", + "version_deprecated": null, + "replaced_by": null, + "is_default": true, + "fides_key": "functional.service", + "organization_fides_key": "default_organization", + "tags": null, + "name": "Service", + "description": "Functions relating to provided services, products, applications or systems.", + "parent_key": "functional", + "legal_basis": null, + "special_category": null, + "recipients": null, + "legitimate_interest": null, + "legitimate_interest_impact_assessment": null, + "active": true + }, { "fides_key": "functional.service.improve", "organization_fides_key": "default_organization", "tags": null, "name": "System", "description": "The source system, product, service or application being improved.", - "parent_key": "improve", + "parent_key": "functional.service", "legal_basis": null, "special_category": null, "recipients": null, diff --git a/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts b/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts index 95fc6f51c0c..b4a4ac8f44e 100644 --- a/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts +++ b/clients/cypress-e2e/cypress/e2e/smoke_test.cy.ts @@ -102,7 +102,7 @@ describe("Smoke test", () => { cy.getRadio("false").should("not.be.checked"); } ); - cy.getByTestId(`consent-item-improve`).within(() => { + cy.getByTestId(`consent-item-functional`).within(() => { cy.contains("Product Analytics"); cy.getRadio("true").should("be.checked"); cy.getRadio("false").should("not.be.checked"); diff --git a/clients/privacy-center/config/config.json b/clients/privacy-center/config/config.json index c11e68f965f..fccb1807448 100644 --- a/clients/privacy-center/config/config.json +++ b/clients/privacy-center/config/config.json @@ -76,7 +76,7 @@ "executable": false }, { - "fidesDataUseKey": "improve", + "fidesDataUseKey": "functional", "name": "Product Analytics", "description": "We may use some of your personal information to collect analytics about how you use our products & services.", "url": "https://example.com/privacy#analytics", diff --git a/clients/privacy-center/config/examples/full.json b/clients/privacy-center/config/examples/full.json index 36a7ad46a7b..c27706ed19f 100644 --- a/clients/privacy-center/config/examples/full.json +++ b/clients/privacy-center/config/examples/full.json @@ -66,7 +66,7 @@ "executable": false }, { - "fidesDataUseKey": "improve", + "fidesDataUseKey": "functional", "name": "Product Analytics", "description": "We may use some of your personal information to collect analytics about how you use our products & services.", "url": "https://example.com/privacy#analytics", diff --git a/clients/privacy-center/config/examples/v2Consent.json b/clients/privacy-center/config/examples/v2Consent.json index 12b09b1d97f..327c2d6f287 100644 --- a/clients/privacy-center/config/examples/v2Consent.json +++ b/clients/privacy-center/config/examples/v2Consent.json @@ -70,7 +70,7 @@ "executable": false }, { - "fidesDataUseKey": "improve", + "fidesDataUseKey": "functional", "name": "Product Analytics", "description": "We may use some of your personal information to collect analytics about how you use our products & services.", "url": "https://example.com/privacy#analytics", diff --git a/clients/privacy-center/cypress/e2e/consent.cy.ts b/clients/privacy-center/cypress/e2e/consent.cy.ts index e2d06e7e149..82d1a47dbef 100644 --- a/clients/privacy-center/cypress/e2e/consent.cy.ts +++ b/clients/privacy-center/cypress/e2e/consent.cy.ts @@ -222,7 +222,7 @@ describe("Consent settings", () => { cy.contains("Test advertising.first_party"); cy.getRadio().should("not.be.checked"); }); - cy.getByTestId(`consent-item-improve`).within(() => { + cy.getByTestId(`consent-item-functional`).within(() => { cy.getRadio().should("be.checked"); }); @@ -288,7 +288,7 @@ describe("Consent settings", () => { cy.getByTestId(`consent-item-advertising`).within(() => { cy.getRadio("false").check({ force: true }); }); - cy.getByTestId(`consent-item-improve`).within(() => { + cy.getByTestId(`consent-item-functional`).within(() => { cy.getRadio("false").check({ force: true }); }); cy.getByTestId("save-btn").click(); diff --git a/clients/privacy-center/cypress/fixtures/config/config_all.json b/clients/privacy-center/cypress/fixtures/config/config_all.json index 81f7f1d7df2..bb7679da145 100644 --- a/clients/privacy-center/cypress/fixtures/config/config_all.json +++ b/clients/privacy-center/cypress/fixtures/config/config_all.json @@ -66,7 +66,7 @@ "executable": false }, { - "fidesDataUseKey": "improve", + "fidesDataUseKey": "functional", "name": "Product Analytics", "description": "We may use some of your personal information to collect analytics about how you use our products & services.", "url": "https://example.com/privacy#analytics", diff --git a/clients/privacy-center/cypress/fixtures/config/config_consent.json b/clients/privacy-center/cypress/fixtures/config/config_consent.json index 55cb29b2c9d..5d94a9eccea 100644 --- a/clients/privacy-center/cypress/fixtures/config/config_consent.json +++ b/clients/privacy-center/cypress/fixtures/config/config_consent.json @@ -34,7 +34,7 @@ "cookieKeys": ["tracking"] }, { - "fidesDataUseKey": "improve", + "fidesDataUseKey": "functional", "name": "Test improve", "description": "", "url": "https://example.com/privacy#analytics", diff --git a/clients/privacy-center/cypress/fixtures/consent/verify.json b/clients/privacy-center/cypress/fixtures/consent/verify.json index 9cc20d8f6bf..6028b240a22 100644 --- a/clients/privacy-center/cypress/fixtures/consent/verify.json +++ b/clients/privacy-center/cypress/fixtures/consent/verify.json @@ -11,7 +11,7 @@ "opt_in": false }, { - "data_use": "improve", + "data_use": "functional", "description": "Product Analytics", "opt_in": true } diff --git a/src/fides/data/sample_project/privacy_center/config/config.json b/src/fides/data/sample_project/privacy_center/config/config.json index 2e3e9e2b35c..1e2543531df 100644 --- a/src/fides/data/sample_project/privacy_center/config/config.json +++ b/src/fides/data/sample_project/privacy_center/config/config.json @@ -68,7 +68,7 @@ "executable": false }, { - "fidesDataUseKey": "improve", + "fidesDataUseKey": "functional", "name": "Product Analytics", "description": "We may use some of your personal information to collect analytics about how you use our products & services.", "url": "https://example.com/privacy#analytics", From 702f3491e1c441e1f0ddb1f80798ec121928f281 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 13 Sep 2023 15:51:55 +0800 Subject: [PATCH 40/44] fix: migration order --- .../708a780b01ba_add_version_fields_to_default_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index f9f61efe839..130f67e5ce2 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -11,7 +11,7 @@ # revision identifiers, used by Alembic. revision = "708a780b01ba" -down_revision = "093bb28a8270" +down_revision = "192f23f4c968" branch_labels = None depends_on = None From 3d3a67e9b1ff91d0c5bad756df263121e3ac3df6 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 13 Sep 2023 16:22:45 +0800 Subject: [PATCH 41/44] Data Migration for Fideslang 2.0 (#4030) Co-authored-by: Dawn Pattison --- scripts/verify_fideslang_2_data_migration.py | 384 +++++++++++++++ ...ea164cee8bc_fideslang_2_data_migrations.py | 452 ++++++++++++++++++ .../v1/endpoints/consent_request_endpoints.py | 2 +- .../v1/endpoints/privacy_request_endpoints.py | 2 + src/fides/api/models/privacy_request.py | 9 +- src/fides/api/schemas/privacy_request.py | 8 +- .../test_consent_request_endpoints.py | 8 +- tests/ops/models/test_privacy_experience.py | 2 +- tests/ops/models/test_privacy_notice.py | 8 +- tests/ops/util/test_consent_util.py | 2 +- 10 files changed, 860 insertions(+), 17 deletions(-) create mode 100644 scripts/verify_fideslang_2_data_migration.py create mode 100644 src/fides/api/alembic/migrations/versions/1ea164cee8bc_fideslang_2_data_migrations.py diff --git a/scripts/verify_fideslang_2_data_migration.py b/scripts/verify_fideslang_2_data_migration.py new file mode 100644 index 00000000000..965cc80ce9a --- /dev/null +++ b/scripts/verify_fideslang_2_data_migration.py @@ -0,0 +1,384 @@ +""" +This script is designed to help with testing the Data Migration script +for moving users to Fideslang 2.0, found in Fides release 2.20 + +The steps to run the script are as follows: +1. In a terminal, run `nox -s teardown -- volumes ; nox -s dev` to get the server running +2. In a separate terminal, run `nox -s shell`, and then `fides user login ; fides push` +3. You can run `python scripts/verify_fideslang_2_data_migration.py -h` to understand how to invoke script +""" +import argparse +import json +from functools import partial +from typing import Dict + +import fideslang +import requests +from alembic import command + +from fides.api.db.database import get_alembic_config +from fides.api.schemas.privacy_notice import PrivacyNoticeCreation, PrivacyNoticeRegion +from fides.config import CONFIG +from fides.core.api_helpers import get_server_resource +from fides.core.push import push +from fides.core.utils import get_db_engine + +DATABASE_URL = CONFIG.database.sync_database_uri +AUTH_HEADER = CONFIG.user.auth_header +SERVER_URL = CONFIG.cli.server_url +DOWN_REVISION = "708a780b01ba" +PRIVACY_NOTICE_ID = "" # Guido forgive me, this gets mutated later +CONSENT_REQUEST_ID = "" # Guido forgive me, this gets mutated later +CONSENT_CODE = "1234" + +data_category_upgrades: Dict[str, str] = { + "user.financial.account_number": "user.financial.bank_account", + "user.credentials": "user.authorization.credentials", + "user.browsing_history": "user.behavior.browsing_history", + "user.media_consumption": "user.behavior.media_consumption", + "user.search_history": "user.behavior.search_history", + "user.organization": "user.contact.organization", + "user.non_specific_age": "user.demographic.age_range", + "user.date_of_birth": "user.demographic.date_of_birth", + "user.gender": "user.demographic.gender", + "user.political_opinion": "user.demographic.political_opinion", + "user.profiling": "user.demographic.profile", + "user.race": "user.demographic.race_ethnicity", + "user.religious_belief": "user.demographic.religious_belief", + "user.sexual_orientation": "user.demographic.sexual_orientation", + "user.genetic": "user.health_and_medical.genetic", + "user.observed": "user.behavior", +} + +print(f"Using Server URL: {SERVER_URL}") + +assert ( + fideslang.__version__.split(".")[0] == "2" +), "Must have at least Fideslang version 2!" + +# Test updating Datasets, including deeply nested ones +old_dataset = fideslang.models.Dataset( + fides_key="old_dataset", + data_categories=["user.observed"], # new key = user.behavior + collections=[ + fideslang.models.DatasetCollection( + name="old_dataset_collection", + data_categories=["user.observed"], # new key = user.behavior + fields=[ + fideslang.models.DatasetField( + name="old_dataset_field", + data_categories=["user.observed"], # new key = user.behavior + retention=None, + fields=None, + ) + ], + ) + ], +) + +# Test Privacy Declarations and Egress/Ingress updates +old_system = fideslang.models.System( + fides_key="old_system", + system_type="service", + privacy_declarations=[ + fideslang.models.PrivacyDeclaration( + name="old_privacy_declaration_1", + data_categories=["user.observed"], # new key = user.behavior + data_subjects=["employee"], + data_use="improve.system", # new key = functional.service.improve + shared_categories=["user.observed"], # new key = user.behavior + ), + ], + ingress=[ + fideslang.models.DataFlow( + fides_key="privacy_annotations", + type="system", + data_categories=["user.observed"], # new key = user.behavior + ) + ], + egress=[ + fideslang.models.DataFlow( + type="system", + fides_key="privacy_annotations", + data_categories=["user.observed"], # new key = user.behavior + ) + ], + dataset_references=["old_dataset"], +) + +# This needs to exist for testing, since we can't switch back and +# forth between fideslang versions on the fly in a test script +fideslang_1_category = fideslang.models.DataCategory( + fides_key="user.observed", parent_key="user" +) +# This needs to exist for testing, since we can't switch back and +# forth between fideslang versions on the fly in a test script +fideslang_1_use = fideslang.models.DataUse( + fides_key="improve.system", parent_key="improve" +) + +# This is used to test what happens to Taxonomy items that extended off of now-updated default items +orphaned_data_category = fideslang.models.DataCategory( + fides_key="user.observed.custom", parent_key="user.observed" +) + +# This is used to test Privacy Notices +old_notice = PrivacyNoticeCreation( + name="old_notice", + data_uses=["improve.system"], + regions=[PrivacyNoticeRegion("lu")], + consent_mechanism="opt_in", + displayed_in_overlay=True, + enforcement_level="system_wide", +) + +# This is used to test updating Policy Rules +old_policy = fideslang.models.Policy( + fides_key="old_policy", + rules=[ + fideslang.models.PolicyRule( + name="old_policy_rule", + data_subjects=fideslang.models.PrivacyRule( + matches=fideslang.models.MatchesEnum.ALL, values=["employee"] + ), + data_uses=fideslang.models.PrivacyRule( + matches=fideslang.models.MatchesEnum.ALL, + values=["improve.system"], # new key = function.service.improve + ), + data_categories=fideslang.models.PrivacyRule( + matches=fideslang.models.MatchesEnum.ALL, + values=["user.observed"], # new key = user.behavior + ), + ) + ], +) + + +def reload_objects() -> None: + """ + Good luck :D + + Downgrades the database to the previous migration, + loads the "outdated" objects into the database, and then + migrates back up to 'head' + """ + alembic_config = get_alembic_config(DATABASE_URL) + + print(f"> Rolling back one migration to: {DOWN_REVISION}") + command.downgrade(alembic_config, DOWN_REVISION) + + print("> Seeding the database with 'outdated' Taxonomy objects") + create_outdated_objects() + + print("Upgrading database to migration revision: head") + command.upgrade(alembic_config, "head") + + +def create_outdated_objects() -> None: + """ + Make a smattering of requests to get our DB into the state + we want for testing the migration. + """ + + # The push order is deliberate here! + push( + url=SERVER_URL, + headers=AUTH_HEADER, + taxonomy=fideslang.models.Taxonomy( + data_category=[orphaned_data_category, fideslang_1_category], + data_use=[fideslang_1_use], + ), + ) + push( + url=SERVER_URL, + headers=AUTH_HEADER, + taxonomy=fideslang.models.Taxonomy( + system=[old_system], + ), + ) + push( + url=SERVER_URL, + headers=AUTH_HEADER, + taxonomy=fideslang.models.Taxonomy( + dataset=[old_dataset], + policy=[old_policy], + ), + ) + + # Create Privacy Notice + response = requests.post( + url=f"{SERVER_URL}/api/v1/privacy-notice", + headers=AUTH_HEADER, + allow_redirects=True, + data=json.dumps([old_notice.dict()]), + ) + assert response.ok, f"Failed to Create Privacy Notice: {response.text}" + global PRIVACY_NOTICE_ID # I'm so sorry + PRIVACY_NOTICE_ID = response.json()[0]["id"] # Please forgive me + + # Create a Consent Request + response = requests.post( + url=f"{SERVER_URL}/api/v1/consent-request", + allow_redirects=True, + data=json.dumps({"email": "user@example.com"}), + ) + global CONSENT_REQUEST_ID + CONSENT_REQUEST_ID = response.json()["consent_request_id"] + + # Create a Privacy Request from a Consent request? + response = requests.patch( + url=f"{SERVER_URL}/api/v1/consent-request/{CONSENT_REQUEST_ID}/preferences", + allow_redirects=True, + data=json.dumps( + { + "code": CONSENT_CODE, + "consent": [ + { + "data_use": "improve.system", + "opt_in": True, + "has_gpc_flag": False, + "conflicts_with_gpc": False, + } + ], + "executable_options": [ + {"data_use": "improve.system", "executable": True} + ], + } + ), + ) + assert response.ok, response.text + + +def verify_migration(server_url: str, auth_header: Dict[str, str]) -> None: + """ + Run a battery of assertions to verify the data migration worked as intended. + """ + partial_get = partial(get_server_resource, url=server_url, headers=auth_header) + + # Verify Dataset + server_old_dataset: fideslang.models.Dataset = partial_get( + resource_key="old_dataset", resource_type="dataset" + ) + assert server_old_dataset.data_categories == [ + "user.behavior" + ], server_old_dataset.data_categories + assert server_old_dataset.collections[0].data_categories == ["user.behavior"] + assert server_old_dataset.collections[0].fields[0].data_categories == [ + "user.behavior" + ] + print("> Verified Datasets.") + + # Verify Systems + server_old_system: fideslang.models.System = partial_get( + resource_key="old_system", resource_type="system" + ) + assert server_old_system.privacy_declarations[0].data_categories == [ + "user.behavior" + ] + assert ( + server_old_system.privacy_declarations[0].data_use + == "functional.service.improve" + ) + assert server_old_system.privacy_declarations[0].shared_categories == [ + "user.behavior" + ] + assert server_old_system.ingress[0].data_categories == ["user.behavior"] + assert server_old_system.egress[0].data_categories == ["user.behavior"] + print("> Verified Systems.") + + # Verify Policies + server_old_policy: fideslang.models.Policy = partial_get( + resource_key="old_policy", resource_type="policy" + ) + assert server_old_policy.rules[0].data_categories.values == ["user.behavior"] + assert server_old_policy.rules[0].data_uses.values == ["functional.service.improve"] + print("> Verified Policy Rules.") + + # Verify Data Category + server_orphaned_category: fideslang.models.DataCategory = partial_get( + resource_key="user.behavior.custom", resource_type="data_category" + ) + assert server_orphaned_category, server_orphaned_category + print("> Verified Data Categories.") + + # Verify Privacy Notices + # NOTE: This only tests the `privacynotice` table explicitly because the other + # tables appear to be carbon copies (inheritance) + privacy_notice_response = requests.get( + url=f"{SERVER_URL}/api/v1/privacy-notice/{PRIVACY_NOTICE_ID}", + headers=AUTH_HEADER, + allow_redirects=True, + ).json() + assert privacy_notice_response["data_uses"] == ["functional.service.improve"] + print("> Verified Privacy Notices.") + + # Verify Consent + consent_response = requests.post( + url=f"{SERVER_URL}/api/v1/consent-request/{CONSENT_REQUEST_ID}/verify", + allow_redirects=True, + data=json.dumps( + { + "code": CONSENT_CODE, + } + ), + ) + assert ( + consent_response.json()["consent"][0]["data_use"] + == "functional.service.improve" + ) + + consent_result = get_db_engine(DATABASE_URL).execute( + "select id, data_use, opt_in from consent;" + ) + for r in consent_result: + result = r["data_use"] + assert result == "functional.service.improve", result + + privacy_request_result = get_db_engine(DATABASE_URL).execute( + "select id, status, consent_preferences from privacyrequest;" + ) + for r in privacy_request_result: + result = r["consent_preferences"][0]["data_use"] + assert result == "functional.service.improve", result + + consent_request_result = get_db_engine(DATABASE_URL).execute( + "select id, preferences, privacy_request_id from consentrequest;" + ) + for r in consent_request_result: + result = r["preferences"][0]["data_use"] + assert result == "functional.service.improve", result + print("> Verified Consent.") + + # Verify Rule Target + rule_target_result = get_db_engine(DATABASE_URL).execute( + "select data_category from ruletarget;" + ) + for res in rule_target_result: + data_category = res["data_category"] + assert data_category not in data_category_upgrades.keys() + print("> Verified Rule Targets.") + + +if __name__ == "__main__": + print("> Running Fideslang 2.0 Data Migration Test Script...") + + parser = argparse.ArgumentParser( + description="Verify the Fideslang 2.0 Data Migrations" + ) + parser.add_argument( + "--reload", + dest="reload", + action="store_const", + const=True, + default=False, + help="whether or not to redo the migrations and reload the objects", + ) + + args = parser.parse_args() + if args.reload: + reload_objects() + + print("> Verifying Data Migration Updates...") + verify_migration(server_url=SERVER_URL, auth_header=AUTH_HEADER) + + print("> Data Verification Complete!") diff --git a/src/fides/api/alembic/migrations/versions/1ea164cee8bc_fideslang_2_data_migrations.py b/src/fides/api/alembic/migrations/versions/1ea164cee8bc_fideslang_2_data_migrations.py new file mode 100644 index 00000000000..3819565c943 --- /dev/null +++ b/src/fides/api/alembic/migrations/versions/1ea164cee8bc_fideslang_2_data_migrations.py @@ -0,0 +1,452 @@ +"""fideslang 2 data migrations + +Revision ID: 1ea164cee8bc +Revises: 708a780b01ba +Create Date: 2023-09-06 04:09:06.212144 + +""" +import json +from typing import Dict, List, Optional + +from alembic import op +from loguru import logger +from sqlalchemy import text +from sqlalchemy.engine import Connection, ResultProxy +from sqlalchemy.sql.elements import TextClause + +# revision identifiers, used by Alembic. +revision = "1ea164cee8bc" +down_revision = "708a780b01ba" +branch_labels = None +depends_on = None + +############### +## Data Uses ## +############### +""" +The `key` is the old value, the `value` is the new value +These are ordered specifically so that string replacement works on both parent and child items +""" +data_use_upgrades: Dict[str, str] = { + "essential.service.operations.support.optimization": "essential.service.operations.improve", + "improve.system": "functional.service.improve", + "improve": "functional", +} +data_use_downgrades: Dict[str, str] = { + value: key for key, value in data_use_upgrades.items() +} +##################### +## Data Categories ## +##################### +""" +The `key` is the old value, the `value` is the new value +These are ordered specifically so that string replacement works on both parent and child items +""" +data_category_upgrades: Dict[str, str] = { + "user.financial.account_number": "user.financial.bank_account", + "user.credentials": "user.authorization.credentials", + "user.browsing_history": "user.behavior.browsing_history", + "user.media_consumption": "user.behavior.media_consumption", + "user.search_history": "user.behavior.search_history", + "user.organization": "user.contact.organization", + "user.non_specific_age": "user.demographic.age_range", + "user.date_of_birth": "user.demographic.date_of_birth", + "user.gender": "user.demographic.gender", + "user.political_opinion": "user.demographic.political_opinion", + "user.profiling": "user.demographic.profile", + "user.race": "user.demographic.race_ethnicity", + "user.religious_belief": "user.demographic.religious_belief", + "user.sexual_orientation": "user.demographic.sexual_orientation", + "user.genetic": "user.health_and_medical.genetic", + "user.observed": "user.behavior", +} +data_category_downgrades: Dict[str, str] = { + value: key for key, value in data_category_upgrades.items() +} + + +def _replace_matching_data_label( + data_label: str, data_label_map: Dict[str, str] +) -> str: + """ + Helper function to do string replacement for updated fides_keys. + """ + for old, new in data_label_map.items(): + if data_label.startswith(old): + return data_label.replace(old, new) + + return data_label + + +def update_privacy_declarations( + bind: Connection, data_use_map: Dict[str, str], data_category_map: Dict[str, str] +) -> None: + """ + Upgrade or downgrade Privacy Declarations for Fideslang 2.0 + + This updates: + - data uses + - data categories + - shared categories + """ + existing_privacy_declarations: ResultProxy = bind.execute( + text( + "SELECT id, data_use, data_categories, shared_categories FROM privacydeclaration;" + ) + ) + for row in existing_privacy_declarations: + data_use: str = _replace_matching_data_label(row["data_use"], data_use_map) + data_categories: List[str] = [ + _replace_matching_data_label(data_category, data_category_map) + for data_category in row["data_categories"] + ] + shared_categories: List[str] = [ + _replace_matching_data_label(data_category, data_category_map) + for data_category in row["shared_categories"] + ] + + update_query: TextClause = text( + "UPDATE privacydeclaration SET data_use = :updated_use, data_categories = :updated_categories, shared_categories = :updated_shared WHERE id= :declaration_id" + ) + bind.execute( + update_query, + { + "declaration_id": row["id"], + "updated_use": data_use, + "updated_categories": data_categories, + "updated_shared": shared_categories, + }, + ) + + +def update_ctl_policies( + bind: Connection, data_use_map: Dict[str, str], data_category_map: Dict[str, str] +) -> None: + """ + Upgrade or downgrade Policy Rules for Fideslang 2.0 + + This updates: + - data uses + - data categories + """ + existing_ctl_policies: ResultProxy = bind.execute( + text("SELECT id, rules FROM ctl_policies;") + ) + + for row in existing_ctl_policies: + rules: List[Dict] = row["rules"] + + for i, rule in enumerate(rules or []): + data_uses: List = rule.get("data_uses", {}).get("values", []) + rules[i]["data_uses"]["values"] = [ + _replace_matching_data_label(use, data_use_map) for use in data_uses + ] + + data_categories: List = rule.get("data_categories", {}).get("values", []) + rules[i]["data_categories"]["values"] = [ + _replace_matching_data_label(category, data_category_map) + for category in data_categories + ] + + update_data_use_query: TextClause = text( + "UPDATE ctl_policies SET rules = :updated_rules WHERE id= :policy_id" + ) + bind.execute( + update_data_use_query, + {"policy_id": row["id"], "updated_rules": json.dumps(rules)}, + ) + + +def update_data_label_tables( + bind: Connection, update_map: Dict[str, str], table_name: str +) -> None: + """ + Upgrade or downgrade Data Labels for Fideslang 2.0 + """ + existing_labels: ResultProxy = bind.execute( + text(f"SELECT fides_key, parent_key FROM {table_name};") + ) + for row in existing_labels: + old_key = row["fides_key"] + new_key = _replace_matching_data_label(old_key, update_map) + + old_parent = row["parent_key"] + new_parent = _replace_matching_data_label(old_parent, update_map) + + update_query: TextClause = text( + f"UPDATE {table_name} SET fides_key = :updated_key, parent_key = :updated_parent WHERE fides_key = :old_key" + ) + bind.execute( + update_query, + { + "updated_key": new_key, + "old_key": old_key, + "updated_parent": new_parent, + }, + ) + + +def update_rule_targets(bind: Connection, data_label_map: Dict[str, str]) -> None: + """Upgrade ruletargets to use the new data categories.""" + + existing_rule_targets: ResultProxy = bind.execute( + text("SELECT id, data_category FROM ruletarget;") + ) + + for row in existing_rule_targets: + data_category = row["data_category"] + + if not data_category: + continue + + updated_category: str = _replace_matching_data_label( + data_category, data_label_map + ) + + update_data_category_query: TextClause = text( + "UPDATE ruletarget SET data_category = :updated_category WHERE id= :target_id" + ) + bind.execute( + update_data_category_query, + {"target_id": row["id"], "updated_category": updated_category}, + ) + + +def update_datasets_data_categories( + bind: Connection, data_label_map: Dict[str, str] +) -> None: + """Upgrade the datasets and their collections/fields in the database to use the new data categories.""" + + # Get all datasets out of the database + existing_datasets: ResultProxy = bind.execute( + text("SELECT id, data_categories, collections FROM ctl_datasets;") + ) + + for row in existing_datasets: + # Update data categories at the top level + dataset_data_categories: Optional[List[str]] = row["data_categories"] + + if dataset_data_categories: + updated_categories: List[str] = [ + _replace_matching_data_label(category, data_label_map) + for category in dataset_data_categories + ] + + update_label_query: TextClause = text( + "UPDATE ctl_datasets SET data_categories = :updated_labels WHERE id= :dataset_id" + ) + bind.execute( + update_label_query, + {"dataset_id": row["id"], "updated_labels": updated_categories}, + ) + + # Update the collections objects + collections: str = json.dumps(row["collections"]) + if collections: + for key, value in data_label_map.items(): + collections = collections.replace(key, value) + + update_collections_query: TextClause = text( + "UPDATE ctl_datasets SET collections = :updated_collections WHERE id= :dataset_id" + ) + bind.execute( + update_collections_query, + {"dataset_id": row["id"], "updated_collections": collections}, + ) + + +def update_system_ingress_egress_data_categories( + bind: Connection, data_label_map: Dict[str, str] +) -> None: + """Upgrade or downgrade data categories on system DataFlow objects (egress/ingress)""" + existing_systems: ResultProxy = bind.execute( + text("SELECT id, egress, ingress FROM ctl_systems;") + ) + + for row in existing_systems: + ingress = row["ingress"] + egress = row["egress"] + + # Do a blunt find/replace + if ingress: + for item in ingress: + item["data_categories"] = [ + _replace_matching_data_label(category, data_label_map) + for category in item.get("data_categories") + ] + + update_ingress_query: TextClause = text( + "UPDATE ctl_systems SET ingress = :updated_ingress WHERE id= :system_id" + ) + bind.execute( + update_ingress_query, + {"system_id": row["id"], "updated_ingress": json.dumps(ingress)}, + ) + + if egress: + for item in egress: + item["data_categories"] = [ + _replace_matching_data_label(category, data_label_map) + for category in item.get("data_categories") + ] + + update_egress_query: TextClause = text( + "UPDATE ctl_systems SET egress = :updated_egress WHERE id= :system_id" + ) + bind.execute( + update_egress_query, + {"system_id": row["id"], "updated_egress": json.dumps(egress)}, + ) + + +def update_privacy_notices(bind: Connection, data_use_map: Dict[str, str]) -> None: + """ + Update the Privacy Notice Models. + + This includes the following models: + - PrivacyNotice + - PrivacyNoticeHistory + - PrivacyNoticeTemplate + """ + privacy_notice_tables = [ + "privacynotice", + "privacynoticetemplate", + "privacynoticehistory", + ] + for table in privacy_notice_tables: + existing_notices: ResultProxy = bind.execute( + text(f"SELECT id, data_uses FROM {table};") + ) + + for row in existing_notices: + data_uses = row["data_uses"] + + # Do a blunt find/replace + updated_data_uses = [ + _replace_matching_data_label(use, data_use_map) for use in data_uses + ] + + update_query: TextClause = text( + f"UPDATE {table} SET data_uses= :updated_uses WHERE id= :notice_id" + ) + bind.execute( + update_query, + {"notice_id": row["id"], "updated_uses": updated_data_uses}, + ) + + +def update_consent(bind: Connection, data_use_map: Dict[str, str]) -> None: + """ + Update Consent objects in the database. + """ + + # Update the Consent table + existing_consents: ResultProxy = bind.execute( + text("SELECT provided_identity_id, data_use FROM consent;") + ) + + for row in existing_consents: + updated_use: str = _replace_matching_data_label(row["data_use"], data_use_map) + + update_label_query: TextClause = text( + "UPDATE consent SET data_use= :updated_label WHERE provided_identity_id= :key AND data_use = :old_use" + ) + bind.execute( + update_label_query, + { + "key": row["provided_identity_id"], + "old_use": row["data_use"], + "updated_label": updated_use, + }, + ) + + # Update the Privacy Request Table + existing_privacy_requests: ResultProxy = bind.execute( + text("select id, consent_preferences from privacyrequest;") + ) + + for row in existing_privacy_requests: + preferences: List[Dict] = row["consent_preferences"] + + if preferences: + for index, preference in enumerate(preferences): + preferences[index]["data_use"] = _replace_matching_data_label( + data_label=preference["data_use"], data_label_map=data_use_map + ) + + update_pr_query: TextClause = text( + "UPDATE privacyrequest SET consent_preferences= :updated_preferences WHERE id= :id" + ) + bind.execute( + update_pr_query, + {"id": row["id"], "updated_preferences": json.dumps(preferences)}, + ) + + # Update the Consent Request Table + existing_consent_requests: ResultProxy = bind.execute( + text("select id, preferences from consentrequest;") + ) + + for row in existing_consent_requests: + preferences: List[Dict] = row["preferences"] + + if preferences: + for index, preference in enumerate(preferences): + preferences[index]["data_use"] = _replace_matching_data_label( + data_label=preference["data_use"], data_label_map=data_use_map + ) + + update_cr_query: TextClause = text( + "UPDATE consentrequest SET preferences= :updated_preferences WHERE id= :id" + ) + bind.execute( + update_cr_query, + {"id": row["id"], "updated_preferences": json.dumps(preferences)}, + ) + + +def upgrade() -> None: + """ + Given that our advice is to turn off auto-migrations and make a db copy, + there is no "downgrade" version of this. It also wouldn't be feasible given + it would require an older version of fideslang. + """ + bind: Connection = op.get_bind() + + logger.info("Removing old default data categories and data uses") + bind.execute(text("DELETE FROM ctl_data_uses WHERE is_default = TRUE;")) + bind.execute(text("DELETE FROM ctl_data_categories WHERE is_default = TRUE;")) + + logger.info("Upgrading Privacy Declarations for Fideslang 2.0") + update_privacy_declarations(bind, data_use_upgrades, data_category_upgrades) + + logger.info("Upgrading Policy Rules for Fideslang 2.0") + update_ctl_policies(bind, data_use_upgrades, data_category_upgrades) + + logger.info("Upgrading Data Categories in Datasets") + update_datasets_data_categories(bind, data_category_upgrades) + + logger.info("Upgrading Data Categories in System egress/ingress") + update_system_ingress_egress_data_categories(bind, data_category_upgrades) + + logger.info("Updating Privacy Notices") + update_privacy_notices(bind, data_use_upgrades) + + logger.info("Updating Consent") + update_consent(bind, data_use_upgrades) + + logger.info("Updating Rule Targets") + update_rule_targets(bind, data_category_upgrades) + + logger.info("Upgrading Taxonomy Items for Fideslang 2.0") + update_data_label_tables(bind, data_use_upgrades, "ctl_data_uses") + update_data_label_tables(bind, data_category_upgrades, "ctl_data_categories") + + +def downgrade() -> None: + """ + This migration does not support downgrades. + """ + logger.info( + "Data migrations from Fideslang 2.0 to Fideslang 1.0 are not supported." + ) diff --git a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py index 8feae125c2d..32ff05191e3 100644 --- a/src/fides/api/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/consent_request_endpoints.py @@ -264,7 +264,7 @@ def consent_request_verify( "highlight": False, }, { - "data_use": "improve", + "data_use": "functional", "data_use_description": "We may use some of your personal information to collect analytics about " "how you use our products & services, in order to improve our service.", "opt_in": False, diff --git a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py index 5a355d93fa0..7868a2756e8 100644 --- a/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/api/v1/endpoints/privacy_request_endpoints.py @@ -1728,6 +1728,8 @@ def create_privacy_request_func( else: created.append(privacy_request) + # TODO: Don't return a 200 if there are failed requests, or at least not + # if there are zero successful ones return BulkPostPrivacyRequests( succeeded=created, failed=failed, diff --git a/src/fides/api/models/privacy_request.py b/src/fides/api/models/privacy_request.py index adeebe12256..c21617bc650 100644 --- a/src/fides/api/models/privacy_request.py +++ b/src/fides/api/models/privacy_request.py @@ -86,11 +86,12 @@ class ManualAction(FidesSchema): - """Surface how to retrieve or mask data in a database-agnostic way + """ + Surface how to retrieve or mask data in a database-agnostic way - "locators" are similar to the SQL "WHERE" information. - "get" contains a list of fields that should be retrieved from the source - "update" is a dictionary of fields and the replacement value/masking strategy + - 'locators' are similar to the SQL "WHERE" information. + - 'get' contains a list of fields that should be retrieved from the source + - 'update' is a dictionary of fields and the replacement value/masking strategy """ locators: Dict[str, Any] diff --git a/src/fides/api/schemas/privacy_request.py b/src/fides/api/schemas/privacy_request.py index 5034c330122..0f4d0ae53c2 100644 --- a/src/fides/api/schemas/privacy_request.py +++ b/src/fides/api/schemas/privacy_request.py @@ -52,7 +52,9 @@ class Config: class Consent(FidesSchema): - """Schema for consent.""" + """ + Deprecated: This used to be populated and sent to the server by a `config.json` in the UI + """ data_use: str data_use_description: Optional[str] = None @@ -62,7 +64,9 @@ class Consent(FidesSchema): class ConsentReport(Consent): - """Schema for reporting Consent requests.""" + """ + Keeps record of each of the preferences that have been recorded via ConsentReporting endpoints. + """ id: str identity: Identity diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index 3235e385ea0..672b597eaae 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -954,7 +954,7 @@ def test_set_consent_consent_preferences( "conflicts_with_gpc": False, }, { - "data_use": "improve", + "data_use": "functional", "data_use_description": None, "opt_in": True, }, @@ -973,7 +973,7 @@ def test_set_consent_consent_preferences( "policy_key": consent_policy.key, # Optional policy_key supplied, "executable_options": [ {"data_use": "marketing.advertising", "executable": True}, - {"data_use": "improve", "executable": False}, + {"data_use": "functional", "executable": False}, ], "browser_identity": {"ga_client_id": "test_ga_client_id"}, } @@ -992,7 +992,7 @@ def test_set_consent_consent_preferences( "conflicts_with_gpc": False, }, { - "data_use": "improve", + "data_use": "functional", "data_use_description": None, "opt_in": False, "has_gpc_flag": False, @@ -1071,7 +1071,7 @@ def test_set_consent_preferences_privacy_request_pending_when_id_verification_re "policy_key": consent_policy.key, # Optional policy_key supplied, "executable_options": [ {"data_use": "marketing.advertising", "executable": True}, - {"data_use": "improve", "executable": False}, + {"data_use": "functional", "executable": False}, ], "browser_identity": {"ga_client_id": "test_ga_client_id"}, } diff --git a/tests/ops/models/test_privacy_experience.py b/tests/ops/models/test_privacy_experience.py index 58c27903bf8..f2d57d1762b 100644 --- a/tests/ops/models/test_privacy_experience.py +++ b/tests/ops/models/test_privacy_experience.py @@ -551,7 +551,7 @@ def test_get_related_privacy_notices_with_fides_user_device_id_preferences( privacy_notice_us_ca_provide.update( db=db, data={ - "data_uses": ["improve"], + "data_uses": ["functional"], "enforcement_level": EnforcementLevel.frontend, }, ) diff --git a/tests/ops/models/test_privacy_notice.py b/tests/ops/models/test_privacy_notice.py index f3c2460260a..7101ca40889 100644 --- a/tests/ops/models/test_privacy_notice.py +++ b/tests/ops/models/test_privacy_notice.py @@ -717,7 +717,7 @@ def test_conflicting_data_uses( PrivacyNotice( name="pn_1", notice_key="pn_1", - data_uses=["improve"], + data_uses=["functional"], regions=[PrivacyNoticeRegion.us_ca], ) ], @@ -728,7 +728,7 @@ def test_conflicting_data_uses( PrivacyNotice( name="pn_2", notice_key="pn_2", - data_uses=["improve"], + data_uses=["functional"], regions=[PrivacyNoticeRegion.us_ca], ), PrivacyNotice( @@ -761,7 +761,7 @@ def test_conflicting_data_uses( PrivacyNotice( name="pn_2", notice_key="pn_2", - data_uses=["improve"], + data_uses=["functional"], regions=[PrivacyNoticeRegion.us_ca], ) ], @@ -780,7 +780,7 @@ def test_conflicting_data_uses( PrivacyNotice( name="pn_1", notice_key="pn_1", - data_uses=["improve"], + data_uses=["functional"], regions=[PrivacyNoticeRegion.us_va], ) ], diff --git a/tests/ops/util/test_consent_util.py b/tests/ops/util/test_consent_util.py index dd818312700..097741b1bfe 100644 --- a/tests/ops/util/test_consent_util.py +++ b/tests/ops/util/test_consent_util.py @@ -305,7 +305,7 @@ def test_old_workflow_preferences_saved_with_respect_to_data_use( privacy_request_with_consent_policy.consent_preferences = [ {"data_use": "marketing.advertising", "opt_in": True}, - {"data_use": "improve", "opt_in": False}, + {"data_use": "functional", "opt_in": False}, ] collapsed_opt_in_preference, filtered_preferences = should_opt_in_to_service( system, privacy_request_with_consent_policy From 699074f45395cfe8508dee195b2de3d8d76e7ae7 Mon Sep 17 00:00:00 2001 From: Thomas Date: Wed, 13 Sep 2023 18:14:06 +0800 Subject: [PATCH 42/44] feat: bump fideslang version --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9d04867618f..137902403c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -13,7 +13,7 @@ expandvars==0.9.0 fastapi[all]==0.89.1 fastapi-caching[redis]==0.3.0 fastapi-pagination[sqlalchemy]==0.11.4 -fideslang==2.0.2 +fideslang==2.0.3 fideslog==1.2.10 firebase-admin==5.3.0 GitPython==3.1.35 From e7ecbb4a143bd00454c087a5b4156ef7c575fa39 Mon Sep 17 00:00:00 2001 From: Thomas Date: Thu, 14 Sep 2023 00:32:08 +0800 Subject: [PATCH 43/44] fix: seed.py updated to use user.authorization --- src/fides/api/db/seed.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/db/seed.py b/src/fides/api/db/seed.py index 0bf45c6d3a1..cb6706ca2ac 100644 --- a/src/fides/api/db/seed.py +++ b/src/fides/api/db/seed.py @@ -327,7 +327,7 @@ def load_default_dsr_policies() -> None: excluded_data_categories = [ "user.financial", "user.payment", - "user.authorization.credentials", + "user.authorization", ] all_data_categories = [ str(category.fides_key) for category in DEFAULT_TAXONOMY.data_category From 4a271f9293f9b40a88f7cc845915d7de88eb1b1c Mon Sep 17 00:00:00 2001 From: Adam Sachs Date: Wed, 13 Sep 2023 14:32:39 -0400 Subject: [PATCH 44/44] reference correct down_rev in migration comment --- .../708a780b01ba_add_version_fields_to_default_types.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py index 130f67e5ce2..c1be1bd3e41 100644 --- a/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py +++ b/src/fides/api/alembic/migrations/versions/708a780b01ba_add_version_fields_to_default_types.py @@ -1,7 +1,7 @@ """add version fields to default types Revision ID: 708a780b01ba -Revises: 3038667ba898 +Revises: 192f23f4c968 Create Date: 2023-08-17 12:29:04.855626 """