diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 0b5678ef35e..4d6dd204fd4 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -871,8 +871,8 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - - name: emailconfig - description: 'Fides Generated Description for Table: emailconfig' + - name: messagingconfig + description: 'Fides Generated Description for Table: messagingconfig' data_categories: [] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: diff --git a/CHANGELOG.md b/CHANGELOG.md index 948a957960a..1d4e51b4114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The types of changes are: * Config Wizard: Added a column selector to the scan results page of the config wizard [#1590](https://github.com/ethyca/fides/pull/1590) * Config Wizard: Flow for runtime scanner option [#1640](https://github.com/ethyca/fides/pull/1640) * Access support for Twilio Conversations API [#1520](https://github.com/ethyca/fides/pull/1520) +* Message Config: Adds Twilio Email/SMS support [#1519](https://github.com/ethyca/fides/pull/1519) ### Changed diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 148e16553bd..6e1ea6d23e3 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -1,8 +1,9 @@ { "info": { - "_postman_id": "002813f4-12b7-4467-9377-a57706b6dbc8", + "_postman_id": "652de86a-b40e-4711-b488-4b26252683ab", "name": "Fidesops", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "13396647" }, "item": [ { @@ -72,7 +73,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"consent:read\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"email:create_or_update\",\n \"email:read\",\n \"email:delete\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", + "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"consent:read\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"messaging:create_or_update\",\n \"messaging:read\",\n \"messaging:delete\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", "options": { "raw": { "language": "json" @@ -4104,10 +4105,10 @@ ] }, { - "name": "Primary Email Config", + "name": "Messaging Config - Email", "item": [ { - "name": "Post Email Config", + "name": "Post Messaging Config", "request": { "auth": { "type": "bearer", @@ -4123,7 +4124,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"key\": \"{{email_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", + "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", "options": { "raw": { "language": "json" @@ -4131,12 +4132,12 @@ } }, "url": { - "raw": "{{host}}/email/config/", + "raw": "{{host}}/messaging/config/", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", "" ] @@ -4145,7 +4146,7 @@ "response": [] }, { - "name": "Patch Email Config By Key", + "name": "Patch Messaging Config By Key", "request": { "auth": { "type": "bearer", @@ -4161,7 +4162,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"key\": \"{{email_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", + "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", "options": { "raw": { "language": "json" @@ -4169,21 +4170,21 @@ } }, "url": { - "raw": "{{host}}/email/config/{{email_config_key}}", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}" + "{{mailgun_config_key}}" ] } }, "response": [] }, { - "name": "Get Email Configs", + "name": "Get Messaging Configs", "request": { "auth": { "type": "bearer", @@ -4198,12 +4199,12 @@ "method": "GET", "header": [], "url": { - "raw": "{{host}}/email/config/", + "raw": "{{host}}/messaging/config/", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", "" ] @@ -4212,7 +4213,7 @@ "response": [] }, { - "name": "Get Email Config By Key", + "name": "Get Messaging Config By Key", "request": { "auth": { "type": "bearer", @@ -4227,21 +4228,21 @@ "method": "GET", "header": [], "url": { - "raw": "{{host}}/email/config/{{email_config_key}}", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}" + "{{mailgun_config_key}}" ] } }, "response": [] }, { - "name": "Delete Email Config By Key", + "name": "Delete Messaging Config By Key", "request": { "auth": { "type": "bearer", @@ -4257,7 +4258,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n {\n \"key\": \"{{email_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n }\n]\n", + "raw": "[\n {\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n }\n]\n", "options": { "raw": { "language": "json" @@ -4265,21 +4266,21 @@ } }, "url": { - "raw": "{{host}}/email/config/{{email_config_key}}", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}" + "{{mailgun_config_key}}" ] } }, "response": [] }, { - "name": "Email Config Secrets", + "name": "Messaging Config Secrets", "request": { "auth": { "type": "bearer", @@ -4303,14 +4304,134 @@ } }, "url": { - "raw": "{{host}}/email/config/{{email_config_key}}/secret", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}/secret", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}", + "{{mailgun_config_key}}", + "secret" + ] + } + }, + "response": [] + } + ] + }, + { + "name": "Messaging Config - SMS", + "item": [ + { + "name": "Post Messaging Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/", + "host": [ + "{{host}}" + ], + "path": [ + "messaging", + "config", + "" + ] + } + }, + "response": [] + }, + { + "name": "Patch Messaging Config By Key", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{twilio_config_key}}", + "host": [ + "{{host}}" + ], + "path": [ + "messaging", + "config", + "{{twilio_config_key}}" + ] + } + }, + "response": [] + }, + { + "name": "Messaging Config Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"twilio_account_sid\": \"{{twilio_account_sid}}\",\n \"twilio_auth_token\": \"{{twilio_auth_token}}\",\n \"twilio_messaging_service_sid\": \"{{twilio_messaging_service_id}}\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{twilio_config_key}}/secret", + "host": [ + "{{host}}" + ], + "path": [ + "messaging", + "config", + "{{twilio_config_key}}", "secret" ] } @@ -4732,8 +4853,8 @@ "value": "manual_key" }, { - "key": "email_config_key", - "value": "my_email_config", + "key": "mailgun_config_key", + "value": "my_mailgun_config", "type": "string" }, { @@ -4785,6 +4906,26 @@ "key": "verification_code", "value": "", "type": "string" + }, + { + "key": "twilio_config_key", + "value": "my_twilio_config", + "type": "string" + }, + { + "key": "twilio_account_sid", + "value": "", + "type": "string" + }, + { + "key": "twilio_auth_token", + "value": "", + "type": "string" + }, + { + "key": "twilio_messaging_service_id", + "value": "", + "type": "string" } ] -} +} \ No newline at end of file diff --git a/docs/fides/docs/guides/email_communications.md b/docs/fides/docs/guides/email_communications.md index 54350f11695..97fda6d052e 100644 --- a/docs/fides/docs/guides/email_communications.md +++ b/docs/fides/docs/guides/email_communications.md @@ -21,11 +21,11 @@ Fides currently supports Mailgun for email integrations. Ensure you register or ## Configuration -### Create the email configuration +### Create the messaging configuration -```json title="POST api/v1/email/config" +```json title="POST api/v1/messaging/config" { - "key": "{{email_config_key}}", + "key": "{{messaging_config_key}}", "name": "mailgun", "service_type": "mailgun", "details": { @@ -36,8 +36,8 @@ Fides currently supports Mailgun for email integrations. Ensure you register or | Field | Description | |----|----| -| `key` | *Optional.* A unique key used to manage your email config. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`. | -| `name` | A unique user-friendly name for your email config. | +| `key` | *Optional.* A unique key used to manage your messaging config. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`. | +| `name` | A unique user-friendly name for your messaging config. | | `service_type` | The email service to configure. Currently, Fides supports `mailgun`. | | `details` | A dict of key/val config vars specific to Mailgun. | | `domain` | Your unique Mailgun domain. | @@ -45,9 +45,9 @@ Fides currently supports Mailgun for email integrations. Ensure you register or | `api_version` | *Optional.* A string that denotes the API version. Defaults to `v3`. | -### Add the email configuration secrets +### Add the messaging configuration secrets -```json title="POST api/v1/email/config/{{email_config_key}}/secret" +```json title="POST api/v1/messaging/config/{{messaging_config_key}}/secret" { "mailgun_api_key": "nc123849ycnpq98fnu" } diff --git a/docs/fides/docs/guides/messaging.md b/docs/fides/docs/guides/messaging.md new file mode 100644 index 00000000000..21471cb6fb4 --- /dev/null +++ b/docs/fides/docs/guides/messaging.md @@ -0,0 +1,59 @@ +# Configure Email/SMS Messaging +## What is email/SMS used for? + +Fides supports email and SMS server configurations for sending processing notices to privacy request subjects. Future updates will support outbound email communications with data processors. + +Supported modes of use: + +- Subject Identity Verification - for more information on identity verification in subject requests, see the [Privacy Requests](../getting-started/privacy_requests.md#subject-identity-verification) guide. + + +## Prerequisites + +Fides currently supports Mailgun for email integrations. Ensure you register or use an existing Mailgun account in order to get up and running with email communications. + +1. Generate a Mailgun Domain Sending Key + + Follow the [Mailgun documentation](https://documentation.mailgun.com/en/latest/api-intro.html#authentication-1) to create a new Domain Sending Key for Fides. + + !!! Note + Mailgun automatically generates a **primary account API key** when you sign up for an account. This key allows you to perform all CRUD operations via Mailgun's API endpoints, and for any of your sending domains. For security purposes, using a new **domain sending key** is recommended over your primary API key. + +## Configuration + +### Create the messaging configuration + +```json title="POST api/v1/messaging/config" +{ + "key": "{{messaging_config_key}}", + "name": "mailgun", + "service_type": "mailgun", + "details": { + "domain": "your.mailgun.domain" + } +} +``` + +| Field | Description | +|----|----| +| `key` | *Optional.* A unique key used to manage your messaging config. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`. | +| `name` | A unique user-friendly name for your messaging config. | +| `service_type` | The email service to configure. Currently, Fides supports `mailgun`. | +| `details` | A dict of key/val config vars specific to Mailgun. | +| `domain` | Your unique Mailgun domain. | +| `is_eu_domain` | *Optional.* A boolean that denotes whether your Mailgun domain was created in the EU region. Defaults to `False`. | +| `api_version` | *Optional.* A string that denotes the API version. Defaults to `v3`. | + + +### Add the messaging configuration secrets + +```json title="POST api/v1/messaging/config/{{messaging_config_key}}/secret" +{ + "mailgun_api_key": "nc123849ycnpq98fnu" +} + +``` + +| Field | Description | +|---|----| +| `mailgun_api_key` | Your Mailgun Domain Sending Key. | diff --git a/pyproject.toml b/pyproject.toml index 1c9a6300520..2d382c793e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ module = [ "sqlalchemy.ext.*", "sqlalchemy.future.*", "sqlalchemy_utils.*", + "twilio.*", "uvicorn.*" ] ignore_missing_imports = true diff --git a/requirements.txt b/requirements.txt index cc72a5342b1..3caa6ca150b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,5 +33,6 @@ sqlalchemy-stubs==0.4 SQLAlchemy-Utils==0.38.3 sqlalchemy[asyncio]==1.4.42 toml>=0.10.1 +twilio==7.15.0 Unidecode==1.3.4 versioneer==0.19 diff --git a/scripts/setup/email.py b/scripts/setup/email.py index 99e3df9b938..f15f538db68 100644 --- a/scripts/setup/email.py +++ b/scripts/setup/email.py @@ -17,7 +17,7 @@ def create_email_integration( key: str = "fides_email", ): response = requests.post( - f"{constants.BASE_URL}{urls.EMAIL_CONFIG}", + f"{constants.BASE_URL}{urls.MESSAGING_CONFIG}", headers=auth_header, json={ "name": "fides Emails", @@ -32,20 +32,12 @@ def create_email_integration( ) if not response.ok: - if ( - response.json()["detail"] - != f"Only one email config is supported at a time. Config with key {key} is already configured." - ): - raise RuntimeError( - f"fides email config creation failed! response.status_code={response.status_code}, response.json()={response.json()}" - ) - logger.info( - f"fides email config is already created. Using the existing config." + raise RuntimeError( + f"fides messaging config creation failed! response.status_code={response.status_code}, response.json()={response.json()}" ) - return # Now add secrets - email_secrets_path = urls.EMAIL_SECRETS.format(config_key=key) + email_secrets_path = urls.MESSAGING_SECRETS.format(config_key=key) response = requests.put( f"{constants.BASE_URL}{email_secrets_path}", headers=auth_header, @@ -56,7 +48,7 @@ def create_email_integration( if not response.ok: raise RuntimeError( - f"fides email config secrets update failed! response.status_code={response.status_code}, response.json()={response.json()}" + f"fides messaging config secrets update failed! response.status_code={response.status_code}, response.json()={response.json()}" ) logger.info(response.json()["msg"]) diff --git a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py new file mode 100644 index 00000000000..30911f27480 --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py @@ -0,0 +1,119 @@ +"""Update table for twilio + +Revision ID: 179f2bb623ae +Revises: 8f1a19465239 +Create Date: 2022-10-31 18:19:26.845723 + +""" +import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = "179f2bb623ae" +down_revision = "8f1a19465239" +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "messagingconfig", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("key", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.Column( + "service_type", + sa.Enum( + "MAILGUN", "TWILIO_TEXT", "TWILIO_EMAIL", name="messagingservicetype" + ), + nullable=False, + ), + sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column( + "secrets", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_messagingconfig_id"), "messagingconfig", ["id"], unique=False + ) + op.create_index( + op.f("ix_messagingconfig_key"), "messagingconfig", ["key"], unique=True + ) + op.create_index( + op.f("ix_messagingconfig_name"), "messagingconfig", ["name"], unique=True + ) + op.create_index( + op.f("ix_messagingconfig_service_type"), + "messagingconfig", + ["service_type"], + unique=True, + ) + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table( + "emailconfig", + sa.Column("id", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=True, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=True, + ), + sa.Column("key", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "service_type", + postgresql.ENUM("MAILGUN", name="emailservicetype"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "details", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + sa.Column("secrets", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint("id", name="emailconfig_pkey"), + ) + op.create_index( + "ix_emailconfig_service_type", "emailconfig", ["service_type"], unique=False + ) + op.create_index("ix_emailconfig_name", "emailconfig", ["name"], unique=False) + op.create_index("ix_emailconfig_key", "emailconfig", ["key"], unique=False) + op.create_index("ix_emailconfig_id", "emailconfig", ["id"], unique=False) + op.drop_index(op.f("ix_messagingconfig_service_type"), table_name="messagingconfig") + op.drop_index(op.f("ix_messagingconfig_name"), table_name="messagingconfig") + op.drop_index(op.f("ix_messagingconfig_key"), table_name="messagingconfig") + op.drop_index(op.f("ix_messagingconfig_id"), table_name="messagingconfig") + op.drop_table("messagingconfig") + # ### end Alembic commands ### diff --git a/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py b/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py index 3aa4320c3b0..bbddb080fcf 100644 --- a/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py +++ b/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py @@ -6,12 +6,10 @@ Create Date: 2022-08-10 19:29:12.119401 """ -import sqlalchemy as sa -import sqlalchemy_utils from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. + revision = "c61fd9d4f73e" down_revision = "7abe778b7082" branch_labels = None @@ -19,45 +17,8 @@ def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "emailconfig", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - ), - sa.Column("key", sa.String(), nullable=False), - sa.Column("name", sa.String(), nullable=True), - sa.Column( - "service_type", sa.Enum("MAILGUN", name="emailservicetype"), nullable=False - ), - sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column( - "secrets", - sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), - nullable=True, - ), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_emailconfig_id"), "emailconfig", ["id"], unique=False) - op.create_index(op.f("ix_emailconfig_key"), "emailconfig", ["key"], unique=True) - op.create_index(op.f("ix_emailconfig_name"), "emailconfig", ["name"], unique=True) - op.create_index( - op.f("ix_emailconfig_service_type"), - "emailconfig", - ["service_type"], - unique=False, - ) - # ### end Alembic commands ### + # Removing due to the following error: type "emailservicetype" already exists + pass def downgrade(): diff --git a/src/fides/api/ops/api/v1/api.py b/src/fides/api/ops/api/v1/api.py index 7a3a83dcdac..ccdf32fd6b2 100644 --- a/src/fides/api/ops/api/v1/api.py +++ b/src/fides/api/ops/api/v1/api.py @@ -5,11 +5,11 @@ consent_request_endpoints, dataset_endpoints, drp_endpoints, - email_endpoints, encryption_endpoints, identity_verification_endpoints, manual_webhook_endpoints, masking_endpoints, + messaging_endpoints, oauth_endpoints, policy_endpoints, policy_webhook_endpoints, @@ -37,7 +37,7 @@ api_router.include_router(privacy_request_endpoints.router) api_router.include_router(identity_verification_endpoints.router) api_router.include_router(storage_endpoints.router) -api_router.include_router(email_endpoints.router) +api_router.include_router(messaging_endpoints.router) api_router.include_router(saas_config_endpoints.router) api_router.include_router(user_endpoints.router) api_router.include_router(user_permission_endpoints.router) diff --git a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py index 1946eabba9e..ccf5a2a8a7a 100644 --- a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py @@ -24,9 +24,9 @@ V1_URL_PREFIX, ) from fides.api.ops.common_exceptions import ( - EmailDispatchException, FunctionalityNotConfigured, IdentityVerificationException, + MessageDispatchException, ) from fides.api.ops.models.privacy_request import ( Consent, @@ -98,12 +98,12 @@ def create_consent_request( if CONFIG.execution.subject_identity_verification_required: try: - send_verification_code_to_user(db, consent_request, data.email) - except EmailDispatchException as exc: - logger.error("Error sending the verification code email: %s", str(exc)) + send_verification_code_to_user(db, consent_request, data) + except MessageDispatchException as exc: + logger.error("Error sending the verification code message: %s", str(exc)) raise HTTPException( status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error sending the verification code email: {str(exc)}", + detail=f"Error sending the verification code message: {str(exc)}", ) return ConsentRequestResponse( identity=data, diff --git a/src/fides/api/ops/api/v1/endpoints/email_endpoints.py b/src/fides/api/ops/api/v1/endpoints/email_endpoints.py deleted file mode 100644 index e375c6ad294..00000000000 --- a/src/fides/api/ops/api/v1/endpoints/email_endpoints.py +++ /dev/null @@ -1,230 +0,0 @@ -import logging -from typing import Optional - -from fastapi import Depends, Security -from fastapi_pagination import Page, Params -from fastapi_pagination.bases import AbstractPage -from fastapi_pagination.ext.sqlalchemy import paginate -from sqlalchemy.orm import Session -from starlette.exceptions import HTTPException -from starlette.status import ( - HTTP_200_OK, - HTTP_204_NO_CONTENT, - HTTP_400_BAD_REQUEST, - HTTP_404_NOT_FOUND, - HTTP_422_UNPROCESSABLE_ENTITY, - HTTP_500_INTERNAL_SERVER_ERROR, -) - -from fides.api.ops.api import deps -from fides.api.ops.api.v1.scope_registry import ( - EMAIL_CREATE_OR_UPDATE, - EMAIL_DELETE, - EMAIL_READ, -) -from fides.api.ops.api.v1.urn_registry import ( - EMAIL_BY_KEY, - EMAIL_CONFIG, - EMAIL_SECRETS, - V1_URL_PREFIX, -) -from fides.api.ops.common_exceptions import ( - EmailConfigAlreadyExistsException, - EmailConfigNotFoundException, -) -from fides.api.ops.models.email import EmailConfig, get_schema_for_secrets -from fides.api.ops.schemas.email.email import ( - EmailConfigRequest, - EmailConfigResponse, - TestEmailStatusMessage, -) -from fides.api.ops.schemas.email.email_secrets_docs_only import possible_email_secrets -from fides.api.ops.schemas.shared_schemas import FidesOpsKey -from fides.api.ops.service.email.email_crud_service import ( - create_email_config, - delete_email_config, - get_email_config_by_key, - update_email_config, -) -from fides.api.ops.util.api_router import APIRouter -from fides.api.ops.util.logger import Pii -from fides.api.ops.util.oauth_util import verify_oauth_client - -router = APIRouter(tags=["email"], prefix=V1_URL_PREFIX) -logger = logging.getLogger(__name__) - - -@router.post( - EMAIL_CONFIG, - status_code=HTTP_200_OK, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_CREATE_OR_UPDATE])], - response_model=EmailConfigResponse, -) -def post_config( - *, - db: Session = Depends(deps.get_db), - email_config: EmailConfigRequest, -) -> EmailConfigResponse: - """ - Given an email config, create corresponding EmailConfig object, provided no config already exists - """ - - try: - return create_email_config(db=db, config=email_config) - except EmailConfigAlreadyExistsException as e: - logger.warning(e.message) - raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=e.message) - - except Exception as exc: - logger.warning( - "Create failed for email config %s: %s", email_config.key, Pii(str(exc)) - ) - raise HTTPException( - status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Config with key {email_config.key} failed to be added", - ) - - -@router.patch( - EMAIL_BY_KEY, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_CREATE_OR_UPDATE])], - response_model=EmailConfigResponse, -) -def patch_config_by_key( - config_key: FidesOpsKey, - *, - db: Session = Depends(deps.get_db), - email_config: EmailConfigRequest, -) -> Optional[EmailConfigResponse]: - """ - Updates config for email by key, provided config with key can be found. - """ - try: - return update_email_config(db=db, key=config_key, config=email_config) - except EmailConfigNotFoundException: - logger.warning("No email config found with key %s", config_key) - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No email config found with key {config_key}", - ) - - except Exception as exc: - logger.warning( - "Patch failed for email config %s: %s", email_config.key, Pii(str(exc)) - ) - raise HTTPException( - status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Config with key {email_config.key} failed to be added", - ) - - -@router.put( - EMAIL_SECRETS, - status_code=HTTP_200_OK, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_CREATE_OR_UPDATE])], - response_model=TestEmailStatusMessage, -) -def put_config_secrets( - config_key: FidesOpsKey, - *, - db: Session = Depends(deps.get_db), - unvalidated_email_secrets: possible_email_secrets, -) -> TestEmailStatusMessage: - """ - Add or update secrets for email config. - """ - logger.info("Finding email config with key '%s'", config_key) - email_config = EmailConfig.get_by(db=db, field="key", value=config_key) - if not email_config: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No email configuration with key {config_key}.", - ) - - try: - secrets_schema = get_schema_for_secrets( - service_type=email_config.service_type, - secrets=unvalidated_email_secrets, - ) - except KeyError as exc: - raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - detail=exc.args[0], - ) - except ValueError as exc: - raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, - detail=exc.args[0], - ) - - logger.info("Updating email config secrets for config with key '%s'", config_key) - try: - email_config.set_secrets(db=db, email_secrets=secrets_schema.dict()) - except ValueError as exc: - raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, - detail=exc.args[0], - ) - - msg = f"Secrets updated for EmailConfig with key: {config_key}." - # todo- implement test status for email service - return TestEmailStatusMessage(msg=msg, test_status=None) - - -@router.get( - EMAIL_CONFIG, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_READ])], - response_model=Page[EmailConfigResponse], -) -def get_configs( - *, db: Session = Depends(deps.get_db), params: Params = Depends() -) -> AbstractPage[EmailConfig]: - """ - Retrieves configs for email. - """ - logger.info("Finding all email configurations with pagination params %s", params) - return paginate( - EmailConfig.query(db=db).order_by(EmailConfig.created_at.desc()), params=params - ) - - -@router.get( - EMAIL_BY_KEY, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_READ])], - response_model=EmailConfigResponse, -) -def get_config_by_key( - config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) -) -> EmailConfigResponse: - """ - Retrieves configs for email by key. - """ - logger.info("Finding email config with key '%s'", config_key) - - try: - return get_email_config_by_key(db=db, key=config_key) - except EmailConfigNotFoundException as e: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=e.message, - ) - - -@router.delete( - EMAIL_BY_KEY, - status_code=HTTP_204_NO_CONTENT, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_DELETE])], -) -def delete_config_by_key( - config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) -) -> None: - """ - Deletes email configs by key. - """ - try: - delete_email_config(db=db, key=config_key) - except EmailConfigNotFoundException as e: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=e.message, - ) diff --git a/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py b/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py index 56fa8a67412..be6aa16d5fb 100644 --- a/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py @@ -6,7 +6,7 @@ from fides.api.ops.api import deps from fides.api.ops.api.v1 import urn_registry as urls -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.schemas.identity_verification import ( IdentityVerificationConfigResponse, ) @@ -27,8 +27,8 @@ def get_id_verification_config( ) -> IdentityVerificationConfigResponse: """Returns id verification config.""" config = get_config() - email_config: Optional[EmailConfig] = db.query(EmailConfig).first() + messaging_config: Optional[MessagingConfig] = db.query(MessagingConfig).first() return IdentityVerificationConfigResponse( identity_verification_required=config.execution.subject_identity_verification_required, - valid_email_config_exists=bool(email_config and email_config.secrets), + valid_email_config_exists=bool(messaging_config and messaging_config.secrets), ) diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py new file mode 100644 index 00000000000..009b8a5dfd3 --- /dev/null +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -0,0 +1,242 @@ +import logging +from typing import Optional + +from fastapi import Depends, Security +from fastapi_pagination import Page, Params +from fastapi_pagination.bases import AbstractPage +from fastapi_pagination.ext.sqlalchemy import paginate +from sqlalchemy.orm import Session +from starlette.exceptions import HTTPException +from starlette.status import ( + HTTP_200_OK, + HTTP_204_NO_CONTENT, + HTTP_400_BAD_REQUEST, + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_500_INTERNAL_SERVER_ERROR, +) + +from fides.api.ops.api import deps +from fides.api.ops.api.v1.scope_registry import ( + MESSAGING_CREATE_OR_UPDATE, + MESSAGING_DELETE, + MESSAGING_READ, +) +from fides.api.ops.api.v1.urn_registry import ( + MESSAGING_BY_KEY, + MESSAGING_CONFIG, + MESSAGING_SECRETS, + V1_URL_PREFIX, +) +from fides.api.ops.common_exceptions import MessagingConfigNotFoundException +from fides.api.ops.models.messaging import MessagingConfig, get_schema_for_secrets +from fides.api.ops.schemas.messaging.messaging import ( + MessagingConfigRequest, + MessagingConfigResponse, + TestMessagingStatusMessage, +) +from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import ( + possible_messaging_secrets, +) +from fides.api.ops.schemas.shared_schemas import FidesOpsKey +from fides.api.ops.service.messaging.messaging_crud_service import ( + create_or_update_messaging_config, + delete_messaging_config, + get_messaging_config_by_key, + update_messaging_config, +) +from fides.api.ops.util.api_router import APIRouter +from fides.api.ops.util.logger import Pii +from fides.api.ops.util.oauth_util import verify_oauth_client + +router = APIRouter(tags=["messaging"], prefix=V1_URL_PREFIX) +logger = logging.getLogger(__name__) + + +@router.post( + MESSAGING_CONFIG, + status_code=HTTP_200_OK, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])], + response_model=MessagingConfigResponse, +) +def post_config( + *, + db: Session = Depends(deps.get_db), + messaging_config: MessagingConfigRequest, +) -> MessagingConfigResponse: + """ + Given a messaging config, create corresponding MessagingConfig object, provided no config already exists + """ + + try: + return create_or_update_messaging_config(db=db, config=messaging_config) + except ValueError as e: + logger.warning( + "Create failed for messaging config %s: %s", + messaging_config.key, + Pii(str(e)), + ) + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"Config with key {messaging_config.key} failed to be added: {e}", + ) + except Exception as exc: + logger.warning( + "Create failed for messaging config %s: %s", + messaging_config.key, + Pii(str(exc)), + ) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Config with key {messaging_config.key} failed to be added: {exc}", + ) + + +@router.patch( + MESSAGING_BY_KEY, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])], + response_model=MessagingConfigResponse, +) +def patch_config_by_key( + config_key: FidesOpsKey, + *, + db: Session = Depends(deps.get_db), + messaging_config: MessagingConfigRequest, +) -> Optional[MessagingConfigResponse]: + """ + Updates config for messaging by key, provided config with key can be found. + """ + try: + return update_messaging_config(db=db, key=config_key, config=messaging_config) + except MessagingConfigNotFoundException: + logger.warning("No messaging config found with key %s", config_key) + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No messaging config found with key {config_key}", + ) + except Exception as exc: + logger.warning( + "Patch failed for messaging config %s: %s", + messaging_config.key, + Pii(str(exc)), + ) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Config with key {messaging_config.key} failed to be added", + ) + + +@router.put( + MESSAGING_SECRETS, + status_code=HTTP_200_OK, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])], + response_model=TestMessagingStatusMessage, +) +def put_config_secrets( + config_key: FidesOpsKey, + *, + db: Session = Depends(deps.get_db), + unvalidated_messaging_secrets: possible_messaging_secrets, +) -> TestMessagingStatusMessage: + """ + Add or update secrets for messaging config. + """ + logger.info("Finding messaging config with key '%s'", config_key) + messaging_config = MessagingConfig.get_by(db=db, field="key", value=config_key) + if not messaging_config: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No messaging configuration with key {config_key}.", + ) + + try: + secrets_schema = get_schema_for_secrets( + service_type=messaging_config.service_type, + secrets=unvalidated_messaging_secrets, + ) + except KeyError as exc: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=exc.args[0], + ) + except ValueError as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=exc.args[0], + ) + + logger.info( + "Updating messaging config secrets for config with key '%s'", config_key + ) + try: + messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.dict()) + except ValueError as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=exc.args[0], + ) + msg = f"Secrets updated for MessagingConfig with key: {config_key}." + # todo- implement test status for messaging service + return TestMessagingStatusMessage(msg=msg, test_status=None) + + +@router.get( + MESSAGING_CONFIG, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_READ])], + response_model=Page[MessagingConfigResponse], +) +def get_configs( + *, db: Session = Depends(deps.get_db), params: Params = Depends() +) -> AbstractPage[MessagingConfig]: + """ + Retrieves configs for messaging. + """ + logger.info( + "Finding all messaging configurations with pagination params %s", params + ) + return paginate( + MessagingConfig.query(db=db).order_by(MessagingConfig.created_at.desc()), + params=params, + ) + + +@router.get( + MESSAGING_BY_KEY, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_READ])], + response_model=MessagingConfigResponse, +) +def get_config_by_key( + config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) +) -> MessagingConfigResponse: + """ + Retrieves configs for messaging service by key. + """ + logger.info("Finding messaging config with key '%s'", config_key) + + try: + return get_messaging_config_by_key(db=db, key=config_key) + except MessagingConfigNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=e.message, + ) + + +@router.delete( + MESSAGING_BY_KEY, + status_code=HTTP_204_NO_CONTENT, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_DELETE])], +) +def delete_config_by_key( + config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) +) -> None: + """ + Deletes messaging configs by key. + """ + try: + delete_messaging_config(db=db, key=config_key) + except MessagingConfigNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=e.message, + ) diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index b178a3229bb..70a966fa7ff 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -59,11 +59,11 @@ REQUEST_PREVIEW, ) from fides.api.ops.common_exceptions import ( - EmailDispatchException, FunctionalityNotConfigured, IdentityNotFoundException, IdentityVerificationException, ManualWebhookFieldsUnset, + MessageDispatchException, NoCachedManualWebhookEntry, PolicyNotFoundException, TraversalError, @@ -75,6 +75,7 @@ from fides.api.ops.models.connectionconfig import ConnectionConfig from fides.api.ops.models.datasetconfig import DatasetConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook +from fides.api.ops.models.messaging import get_messaging_method from fides.api.ops.models.policy import ( ActionType, CurrentStep, @@ -93,13 +94,13 @@ CollectionAddressResponse, DryRunDatasetResponse, ) -from fides.api.ops.schemas.email.email import ( - EmailActionType, - FidesopsEmail, +from fides.api.ops.schemas.external_https import PrivacyRequestResumeFormat +from fides.api.ops.schemas.messaging.messaging import ( + FidesopsMessage, + MessagingActionType, RequestReceiptBodyParams, RequestReviewDenyBodyParams, ) -from fides.api.ops.schemas.external_https import PrivacyRequestResumeFormat from fides.api.ops.schemas.privacy_request import ( BulkPostPrivacyRequests, BulkReviewResponse, @@ -113,8 +114,11 @@ RowCountRequest, VerificationCode, ) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service._verification import send_verification_code_to_user -from fides.api.ops.service.email.email_dispatch_service import dispatch_email_task +from fides.api.ops.service.messaging.message_dispatch_service import ( + dispatch_message_task, +) from fides.api.ops.service.privacy_request.request_runner_service import ( queue_privacy_request, ) @@ -124,7 +128,7 @@ ) from fides.api.ops.task.graph_task import EMPTY_REQUEST, collect_queries from fides.api.ops.task.task_resources import TaskResources -from fides.api.ops.tasks import EMAIL_QUEUE_NAME +from fides.api.ops.tasks import MESSAGING_QUEUE_NAME from fides.api.ops.util.api_router import APIRouter from fides.api.ops.util.cache import FidesopsRedis from fides.api.ops.util.collection_util import Row @@ -240,13 +244,13 @@ async def create_privacy_request( if CONFIG.execution.subject_identity_verification_required: send_verification_code_to_user( - db, privacy_request, privacy_request_data.identity.email + db, privacy_request, privacy_request_data.identity ) created.append(privacy_request) continue # Skip further processing for this privacy request if CONFIG.notifications.send_request_receipt_notification: - _send_privacy_request_receipt_email_to_user( - policy, privacy_request_data.identity.email + _send_privacy_request_receipt_message_to_user( + policy, privacy_request_data.identity ) if not CONFIG.execution.require_manual_request_approval: AuditLog.create( @@ -259,7 +263,7 @@ async def create_privacy_request( }, ) queue_privacy_request(privacy_request.id) - except EmailDispatchException as exc: + except MessageDispatchException as exc: kwargs["privacy_request_id"] = privacy_request.id logger.error("EmailDispatchException: %s", exc) failure = { @@ -290,21 +294,21 @@ async def create_privacy_request( ) -def _send_privacy_request_receipt_email_to_user( - policy: Optional[Policy], email: Optional[str] +def _send_privacy_request_receipt_message_to_user( + policy: Optional[Policy], to_identity: Optional[Identity] ) -> None: - """Helper function to send request receipt email to the user""" - if not email: + """Helper function to send request receipt message to the user""" + if not to_identity: logger.error( IdentityNotFoundException( - "Identity email was not found, so request receipt email could not be sent." + "Identity was not found, so request receipt message could not be sent." ) ) return if not policy: logger.error( PolicyNotFoundException( - "Policy was not found, so request receipt email could not be sent." + "Policy was not found, so request receipt message could not be sent." ) ) return @@ -312,14 +316,17 @@ def _send_privacy_request_receipt_email_to_user( for action_type in ActionType: if policy.get_rules_for_action(action_type=ActionType(action_type)): request_types.add(action_type) - dispatch_email_task.apply_async( - queue=EMAIL_QUEUE_NAME, + dispatch_message_task.apply_async( + queue=MESSAGING_QUEUE_NAME, kwargs={ - "email_meta": FidesopsEmail( - action_type=EmailActionType.PRIVACY_REQUEST_RECEIPT, + "message_meta": FidesopsMessage( + action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT, body_params=RequestReceiptBodyParams(request_types=request_types), ).dict(), - "to_email": email, + "messaging_method": get_messaging_method( + CONFIG.notifications.notification_service_type + ), + "to_identity": to_identity, }, ) @@ -1143,30 +1150,37 @@ def review_privacy_request( ) -def _send_privacy_request_review_email_to_user( - action_type: EmailActionType, - email: Optional[str], +def _send_privacy_request_review_message_to_user( + action_type: MessagingActionType, + identity_data: Dict[str, Any], rejection_reason: Optional[str], ) -> None: - """Helper method to send review notification email to user, shared between approve and deny""" - if not email: + """Helper method to send review notification message to user, shared between approve and deny""" + if not identity_data: logger.error( IdentityNotFoundException( - "Identity email was not found, so request review email could not be sent." + "Identity was not found, so request review message could not be sent." ) ) - dispatch_email_task.apply_async( - queue=EMAIL_QUEUE_NAME, + to_identity: Identity = Identity( + email=identity_data.get(ProvidedIdentityType.email.value), + phone_number=identity_data.get(ProvidedIdentityType.phone_number.value), + ) + dispatch_message_task.apply_async( + queue=MESSAGING_QUEUE_NAME, kwargs={ - "email_meta": FidesopsEmail( + "message_meta": FidesopsMessage( action_type=action_type, body_params=RequestReviewDenyBodyParams( rejection_reason=rejection_reason ) - if action_type is EmailActionType.PRIVACY_REQUEST_REVIEW_DENY + if action_type is MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY else None, ).dict(), - "to_email": email, + "messaging_method": get_messaging_method( + CONFIG.notifications.notification_service_type + ), + "to_identity": to_identity, }, ) @@ -1197,8 +1211,8 @@ async def verify_identification_code( db=db, object_id=privacy_request.policy_id ) if CONFIG.notifications.send_request_receipt_notification: - _send_privacy_request_receipt_email_to_user( - policy, privacy_request.get_persisted_identity().email + _send_privacy_request_receipt_message_to_user( + policy, privacy_request.get_persisted_identity() ) except IdentityVerificationException as exc: raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=exc.message) @@ -1256,11 +1270,9 @@ def _approve_request(privacy_request: PrivacyRequest) -> None: }, ) if CONFIG.notifications.send_request_review_notification: - _send_privacy_request_review_email_to_user( - action_type=EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE, - email=privacy_request.get_cached_identity_data().get( - ProvidedIdentityType.email.value - ), + _send_privacy_request_review_message_to_user( + action_type=MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE, + identity_data=privacy_request.get_cached_identity_data(), rejection_reason=None, ) @@ -1308,11 +1320,9 @@ def _deny_request( }, ) if CONFIG.notifications.send_request_review_notification: - _send_privacy_request_review_email_to_user( - action_type=EmailActionType.PRIVACY_REQUEST_REVIEW_DENY, - email=privacy_request.get_cached_identity_data().get( - ProvidedIdentityType.email.value - ), + _send_privacy_request_review_message_to_user( + action_type=MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY, + identity_data=privacy_request.get_cached_identity_data(), rejection_reason=privacy_requests.reason, ) diff --git a/src/fides/api/ops/api/v1/scope_registry.py b/src/fides/api/ops/api/v1/scope_registry.py index 9b9e8d24e32..c13199c27a4 100644 --- a/src/fides/api/ops/api/v1/scope_registry.py +++ b/src/fides/api/ops/api/v1/scope_registry.py @@ -40,9 +40,9 @@ STORAGE_READ = "storage:read" STORAGE_DELETE = "storage:delete" -EMAIL_CREATE_OR_UPDATE = "email:create_or_update" -EMAIL_READ = "email:read" -EMAIL_DELETE = "email:delete" +MESSAGING_CREATE_OR_UPDATE = "messaging:create_or_update" +MESSAGING_READ = "messaging:read" +MESSAGING_DELETE = "messaging:delete" SCOPE_READ = "scope:read" @@ -99,9 +99,9 @@ STORAGE_CREATE_OR_UPDATE, STORAGE_DELETE, STORAGE_READ, - EMAIL_CREATE_OR_UPDATE, - EMAIL_DELETE, - EMAIL_READ, + MESSAGING_CREATE_OR_UPDATE, + MESSAGING_DELETE, + MESSAGING_READ, WEBHOOK_CREATE_OR_UPDATE, WEBHOOK_READ, WEBHOOK_DELETE, diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py index bff93e7d6b0..f3624da52a9 100644 --- a/src/fides/api/ops/api/v1/urn_registry.py +++ b/src/fides/api/ops/api/v1/urn_registry.py @@ -38,9 +38,9 @@ STORAGE_UPLOAD = "/storage/{request_id}" # Email URLs -EMAIL_CONFIG = "/email/config" -EMAIL_SECRETS = "/email/config/{config_key}/secret" -EMAIL_BY_KEY = "/email/config/{config_key}" +MESSAGING_CONFIG = "/messaging/config" +MESSAGING_SECRETS = "/messaging/config/{config_key}/secret" +MESSAGING_BY_KEY = "/messaging/config/{config_key}" # Policy URLs POLICY_LIST = "/dsr/policy" diff --git a/src/fides/api/ops/common_exceptions.py b/src/fides/api/ops/common_exceptions.py index 749bc596b14..587e3d05541 100644 --- a/src/fides/api/ops/common_exceptions.py +++ b/src/fides/api/ops/common_exceptions.py @@ -121,16 +121,12 @@ class SaaSConfigNotFoundException(FidesopsException): """Custom Exception - SaaS Config Not Found""" -class EmailConfigAlreadyExistsException(FidesopsException): - """Custom Exception - Email Config already exists""" +class MessagingConfigNotFoundException(FidesopsException): + """Custom Exception - Messaging Config Not Found""" -class EmailConfigNotFoundException(FidesopsException): - """Custom Exception - Email Config Not Found""" - - -class EmailDispatchException(FidesopsException): - """Custom Exception - Email Dispatch Error""" +class MessageDispatchException(FidesopsException): + """Custom Exception - Message Dispatch Error""" class EmailTemplateUnhandledActionType(FidesopsException): diff --git a/src/fides/api/ops/db/base.py b/src/fides/api/ops/db/base.py index 668e84d436a..b8aee77edc0 100644 --- a/src/fides/api/ops/db/base.py +++ b/src/fides/api/ops/db/base.py @@ -10,8 +10,8 @@ from fides.api.ops.models.authentication_request import AuthenticationRequest from fides.api.ops.models.connectionconfig import ConnectionConfig from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.models.email import EmailConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import Policy, Rule, RuleTarget from fides.api.ops.models.privacy_request import PrivacyRequest from fides.api.ops.models.registration import UserRegistration diff --git a/src/fides/api/ops/email_templates/get_email_template.py b/src/fides/api/ops/email_templates/get_email_template.py index 5ed51f506ea..a128fa878eb 100644 --- a/src/fides/api/ops/email_templates/get_email_template.py +++ b/src/fides/api/ops/email_templates/get_email_template.py @@ -14,7 +14,7 @@ PRIVACY_REQUEST_REVIEW_DENY_TEMPLATE, SUBJECT_IDENTITY_VERIFICATION_TEMPLATE, ) -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import MessagingActionType pathlib.Path(__file__).parent.resolve() logger = logging.getLogger(__name__) @@ -27,23 +27,23 @@ def get_email_template( # pylint: disable=too-many-return-statements - action_type: EmailActionType, + action_type: MessagingActionType, ) -> Template: - if action_type == EmailActionType.CONSENT_REQUEST: + if action_type == MessagingActionType.CONSENT_REQUEST: return template_env.get_template(CONSENT_REQUEST_VERIFICATION_TEMPLATE) - if action_type == EmailActionType.SUBJECT_IDENTITY_VERIFICATION: + if action_type == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION: return template_env.get_template(SUBJECT_IDENTITY_VERIFICATION_TEMPLATE) - if action_type == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT: + if action_type == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT: return template_env.get_template(EMAIL_ERASURE_REQUEST_FULFILLMENT) - if action_type == EmailActionType.PRIVACY_REQUEST_RECEIPT: + if action_type == MessagingActionType.PRIVACY_REQUEST_RECEIPT: return template_env.get_template(PRIVACY_REQUEST_RECEIPT_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: return template_env.get_template(PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION: + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION: return template_env.get_template(PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY: + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY: return template_env.get_template(PRIVACY_REQUEST_REVIEW_DENY_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE: + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE: return template_env.get_template(PRIVACY_REQUEST_REVIEW_APPROVE_TEMPLATE) logger.error("No corresponding template linked to the %s", action_type) diff --git a/src/fides/api/ops/email_templates/template_names.py b/src/fides/api/ops/email_templates/template_names.py index 8ceb5cff023..857053cb716 100644 --- a/src/fides/api/ops/email_templates/template_names.py +++ b/src/fides/api/ops/email_templates/template_names.py @@ -1,6 +1,6 @@ CONSENT_REQUEST_VERIFICATION_TEMPLATE = "consent_request_verification.html" SUBJECT_IDENTITY_VERIFICATION_TEMPLATE = "subject_identity_verification.html" -EMAIL_ERASURE_REQUEST_FULFILLMENT = "erasure_request_email_fulfillment.html" +EMAIL_ERASURE_REQUEST_FULFILLMENT = "message_request_email_fulfillment.html" PRIVACY_REQUEST_RECEIPT_TEMPLATE = "privacy_request_receipt.html" PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE = "privacy_request_complete_deletion.html" PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE = "privacy_request_complete_access.html" diff --git a/src/fides/api/ops/email_templates/templates/erasure_request_email_fulfillment.html b/src/fides/api/ops/email_templates/templates/message_request_email_fulfillment.html similarity index 100% rename from src/fides/api/ops/email_templates/templates/erasure_request_email_fulfillment.html rename to src/fides/api/ops/email_templates/templates/message_request_email_fulfillment.html diff --git a/src/fides/api/ops/models/email.py b/src/fides/api/ops/models/messaging.py similarity index 57% rename from src/fides/api/ops/models/email.py rename to src/fides/api/ops/models/messaging.py index 82dc66d2c7e..985a62615f1 100644 --- a/src/fides/api/ops/models/email.py +++ b/src/fides/api/ops/models/messaging.py @@ -12,14 +12,21 @@ StringEncryptedType, ) -from fides.api.ops.common_exceptions import EmailDispatchException +from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.db.base_class import JSONTypeOverride -from fides.api.ops.schemas.email.email import ( - SUPPORTED_EMAIL_SERVICE_SECRETS, - EmailServiceSecretsMailgun, - EmailServiceType, +from fides.api.ops.schemas.messaging.messaging import ( + EMAIL_MESSAGING_SERVICES, + SMS_MESSAGING_SERVICES, + SUPPORTED_MESSAGING_SERVICE_SECRETS, + MessagingMethod, + MessagingServiceSecretsMailgun, + MessagingServiceSecretsTwilioEmail, + MessagingServiceSecretsTwilioSMS, + MessagingServiceType, +) +from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import ( + possible_messaging_secrets, ) -from fides.api.ops.schemas.email.email_secrets_docs_only import possible_email_secrets from fides.api.ops.util.logger import Pii from fides.ctl.core.config import get_config @@ -27,17 +34,30 @@ logger = logging.getLogger(__name__) +def get_messaging_method( + service_type: Optional[str], +) -> Optional[MessagingMethod]: + """returns messaging method based on configured service type""" + if service_type in EMAIL_MESSAGING_SERVICES: + return MessagingMethod.EMAIL + if service_type in SMS_MESSAGING_SERVICES: + return MessagingMethod.SMS + return None + + def get_schema_for_secrets( - service_type: EmailServiceType, - secrets: possible_email_secrets, -) -> SUPPORTED_EMAIL_SERVICE_SECRETS: + service_type: MessagingServiceType, + secrets: possible_messaging_secrets, +) -> SUPPORTED_MESSAGING_SERVICE_SECRETS: """ Returns the secrets that pertain to `service_type` represented as a Pydantic schema for validation purposes. """ try: schema = { - EmailServiceType.MAILGUN: EmailServiceSecretsMailgun, + MessagingServiceType.MAILGUN: MessagingServiceSecretsMailgun, + MessagingServiceType.TWILIO_TEXT: MessagingServiceSecretsTwilioSMS, + MessagingServiceType.TWILIO_EMAIL: MessagingServiceSecretsTwilioEmail, }[service_type] except KeyError: raise ValueError( @@ -53,13 +73,15 @@ def get_schema_for_secrets( raise ValueError(errors) -class EmailConfig(Base): - """The DB ORM model for EmailConfig""" +class MessagingConfig(Base): + """The DB ORM model for MessagingConfig""" key = Column(String, index=True, unique=True, nullable=False) name = Column(String, unique=True, index=True) - service_type = Column(Enum(EmailServiceType), index=True, nullable=False) - details = Column(MutableDict.as_mutable(JSONB), nullable=False) + service_type = Column( + Enum(MessagingServiceType), index=True, unique=True, nullable=False + ) + details = Column(MutableDict.as_mutable(JSONB), nullable=True) secrets = Column( MutableDict.as_mutable( StringEncryptedType( @@ -75,19 +97,18 @@ class EmailConfig(Base): @classmethod def get_configuration(cls, db: Session) -> Base: """ - Fetches the first configured EmailConfig record. As of v1.7.3 Fidesops does not support - multiple configured email connectors. Once fetched this function validates that - the EmailConfig is configured with secrets. + Fetches the first configured MessagingConfig record. Once fetched this function validates that + the MessagingConfig is configured with secrets. """ instance: Optional[Base] = cls.query(db=db).first() if not instance: - raise EmailDispatchException("No email config found.") + raise MessageDispatchException("No messaging config found.") if not instance.secrets: logger.warning( - "Email secrets not found for config with key: %s", instance.key + "Messaging secrets not found for config with key: %s", instance.key ) - raise EmailDispatchException( - f"Email secrets not found for config with key: {instance.key}" + raise MessageDispatchException( + f"Messaging secrets not found for config with key: {instance.key}" ) return instance @@ -95,7 +116,7 @@ def set_secrets( self, *, db: Session, - email_secrets: possible_email_secrets, + messaging_secrets: possible_messaging_secrets, ) -> None: """Creates or updates secrets associated with a config id""" @@ -108,7 +129,7 @@ def set_secrets( try: get_schema_for_secrets( service_type=service_type, # type: ignore - secrets=email_secrets, + secrets=messaging_secrets, ) except ( KeyError, @@ -118,5 +139,5 @@ def set_secrets( # We don't want to handle these explicitly here, only in the API view raise - self.secrets = email_secrets + self.secrets = messaging_secrets self.save(db=db) diff --git a/src/fides/api/ops/schemas/email/email.py b/src/fides/api/ops/schemas/email/email.py deleted file mode 100644 index c90d61acae0..00000000000 --- a/src/fides/api/ops/schemas/email/email.py +++ /dev/null @@ -1,178 +0,0 @@ -from enum import Enum -from typing import Any, Dict, List, Optional, Union - -from pydantic import BaseModel, Extra - -from fides.api.ops.models.privacy_request import CheckpointActionRequired -from fides.api.ops.schemas import Msg -from fides.api.ops.schemas.shared_schemas import FidesOpsKey - - -class EmailServiceType(Enum): - """Enum for email service type""" - - # may support twilio or google in the future - MAILGUN = "mailgun" - - -class EmailActionType(str, Enum): - """Enum for email action type""" - - # verify email upon acct creation - CONSENT_REQUEST = "consent_request" - SUBJECT_IDENTITY_VERIFICATION = "subject_identity_verification" - EMAIL_ERASURE_REQUEST_FULFILLMENT = "email_erasure_fulfillment" - PRIVACY_REQUEST_RECEIPT = "privacy_request_receipt" - PRIVACY_REQUEST_COMPLETE_ACCESS = "privacy_request_complete_access" - PRIVACY_REQUEST_COMPLETE_DELETION = "privacy_request_complete_deletion" - PRIVACY_REQUEST_REVIEW_DENY = "privacy_request_review_deny" - PRIVACY_REQUEST_REVIEW_APPROVE = "privacy_request_review_approve" - - -class EmailTemplateBodyParams(Enum): - """Enum for all possible email template body params""" - - VERIFICATION_CODE = "verification_code" - - -class SubjectIdentityVerificationBodyParams(BaseModel): - """Body params required for subject identity verification email template""" - - verification_code: str - verification_code_ttl_seconds: int - - def get_verification_code_ttl_minutes(self) -> int: - """returns verification_code_ttl_seconds in minutes""" - if self.verification_code_ttl_seconds < 60: - return 0 - return self.verification_code_ttl_seconds // 60 - - -class RequestReceiptBodyParams(BaseModel): - """Body params required for privacy request receipt email template""" - - request_types: List[str] - - -class AccessRequestCompleteBodyParams(BaseModel): - """Body params required for privacy request completion access email template""" - - download_links: List[str] - - -class RequestReviewDenyBodyParams(BaseModel): - """Body params required for privacy request review deny email template""" - - rejection_reason: Optional[str] - - -class FidesopsEmail( - BaseModel, - smart_union=True, - arbitrary_types_allowed=True, -): - """A mapping of action_type to body_params""" - - action_type: EmailActionType - body_params: Optional[ - Union[ - SubjectIdentityVerificationBodyParams, - RequestReceiptBodyParams, - RequestReviewDenyBodyParams, - AccessRequestCompleteBodyParams, - List[CheckpointActionRequired], - ] - ] - - -class EmailForActionType(BaseModel): - """Email details that depend on action type""" - - subject: str - body: str - - -class EmailServiceDetails(Enum): - """Enum for email service details""" - - # mailgun-specific - IS_EU_DOMAIN = "is_eu_domain" - API_VERSION = "api_version" - DOMAIN = "domain" - - -class EmailServiceDetailsMailgun(BaseModel): - """The details required to represent a Mailgun email configuration.""" - - is_eu_domain: Optional[bool] = False - api_version: Optional[str] = "v3" - domain: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid - - -class EmailServiceSecrets(Enum): - """Enum for email service secrets""" - - # mailgun-specific - MAILGUN_API_KEY = "mailgun_api_key" - - -class EmailServiceSecretsMailgun(BaseModel): - """The secrets required to connect to mailgun.""" - - mailgun_api_key: str - - class Config: - """Restrict adding other fields through this schema.""" - - extra = Extra.forbid - - -class EmailConfigRequest(BaseModel): - """Email Config Request Schema""" - - name: str - key: Optional[FidesOpsKey] - service_type: EmailServiceType - details: Union[ - EmailServiceDetailsMailgun, - ] - - class Config: - use_enum_values = False - orm_mode = True - - -class EmailConfigResponse(BaseModel): - """Email Config Response Schema""" - - name: str - key: FidesOpsKey - service_type: EmailServiceType - details: Dict[EmailServiceDetails, Any] - - class Config: - orm_mode = True - use_enum_values = True - - -SUPPORTED_EMAIL_SERVICE_SECRETS = Union[EmailServiceSecretsMailgun] - - -class EmailConnectionTestStatus(Enum): - """Enum for supplying statuses of validating credentials for an Email Config""" - - succeeded = "succeeded" - failed = "failed" - skipped = "skipped" - - -class TestEmailStatusMessage(Msg): - """A schema for checking status of email config.""" - - test_status: Optional[EmailConnectionTestStatus] = None - failure_reason: Optional[str] = None diff --git a/src/fides/api/ops/schemas/email/email_secrets_docs_only.py b/src/fides/api/ops/schemas/email/email_secrets_docs_only.py deleted file mode 100644 index 6620939b01f..00000000000 --- a/src/fides/api/ops/schemas/email/email_secrets_docs_only.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Union - -from fides.api.ops.schemas.base_class import NoValidationSchema -from fides.api.ops.schemas.email.email import EmailServiceSecretsMailgun - - -class EmailSecretsMailgunDocs(EmailServiceSecretsMailgun, NoValidationSchema): - """The secrets required to connect to Mailgun, for documentation""" - - -possible_email_secrets = Union[EmailSecretsMailgunDocs] diff --git a/src/fides/api/ops/schemas/email/__init__.py b/src/fides/api/ops/schemas/messaging/__init__.py similarity index 100% rename from src/fides/api/ops/schemas/email/__init__.py rename to src/fides/api/ops/schemas/messaging/__init__.py diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py new file mode 100644 index 00000000000..3a4d9ae3851 --- /dev/null +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -0,0 +1,248 @@ +from enum import Enum +from re import compile as regex +from typing import Any, Dict, List, Optional, Tuple, Union + +from pydantic import BaseModel, Extra, root_validator + +from fides.api.ops.models.privacy_request import CheckpointActionRequired +from fides.api.ops.schemas import Msg +from fides.api.ops.schemas.shared_schemas import FidesOpsKey + + +class MessagingMethod(Enum): + """Enum for messaging method""" + + EMAIL = "email" + SMS = "sms" + + +class MessagingServiceType(Enum): + """Enum for messaging service type""" + + MAILGUN = "mailgun" + + TWILIO_TEXT = "twilio_text" + TWILIO_EMAIL = "twilio_email" + + +EMAIL_MESSAGING_SERVICES: Tuple[str, ...] = ( + MessagingServiceType.MAILGUN.value, + MessagingServiceType.TWILIO_EMAIL.value, +) +SMS_MESSAGING_SERVICES: Tuple[str, ...] = tuple(MessagingServiceType.TWILIO_TEXT.value) + + +class MessagingActionType(str, Enum): + """Enum for messaging action type""" + + # verify email upon acct creation + CONSENT_REQUEST = "consent_request" + SUBJECT_IDENTITY_VERIFICATION = "subject_identity_verification" + MESSAGE_ERASURE_REQUEST_FULFILLMENT = "message_erasure_fulfillment" + PRIVACY_REQUEST_RECEIPT = "privacy_request_receipt" + PRIVACY_REQUEST_COMPLETE_ACCESS = "privacy_request_complete_access" + PRIVACY_REQUEST_COMPLETE_DELETION = "privacy_request_complete_deletion" + PRIVACY_REQUEST_REVIEW_DENY = "privacy_request_review_deny" + PRIVACY_REQUEST_REVIEW_APPROVE = "privacy_request_review_approve" + + +class SubjectIdentityVerificationBodyParams(BaseModel): + """Body params required for subject identity verification email/sms template""" + + verification_code: str + verification_code_ttl_seconds: int + + def get_verification_code_ttl_minutes(self) -> int: + """returns verification_code_ttl_seconds in minutes""" + if self.verification_code_ttl_seconds < 60: + return 0 + return self.verification_code_ttl_seconds // 60 + + +class RequestReceiptBodyParams(BaseModel): + """Body params required for privacy request receipt template""" + + request_types: List[str] + + +class AccessRequestCompleteBodyParams(BaseModel): + """Body params required for privacy request completion access template""" + + download_links: List[str] + + +class RequestReviewDenyBodyParams(BaseModel): + """Body params required for privacy request review deny template""" + + rejection_reason: Optional[str] + + +class FidesopsMessage( + BaseModel, + smart_union=True, + arbitrary_types_allowed=True, +): + """A mapping of action_type to body_params""" + + action_type: MessagingActionType + body_params: Optional[ + Union[ + SubjectIdentityVerificationBodyParams, + RequestReceiptBodyParams, + RequestReviewDenyBodyParams, + AccessRequestCompleteBodyParams, + List[CheckpointActionRequired], + ] + ] + + +class EmailForActionType(BaseModel): + """Email details that depend on action type""" + + subject: str + body: str + + +class MessagingServiceDetails(Enum): + """Enum for messaging service details""" + + # Mailgun + IS_EU_DOMAIN = "is_eu_domain" + API_VERSION = "api_version" + DOMAIN = "domain" + + +class MessagingServiceDetailsMailgun(BaseModel): + """The details required to represent a Mailgun email configuration.""" + + is_eu_domain: Optional[bool] = False + api_version: Optional[str] = "v3" + domain: str + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + +class MessagingServiceSecrets(Enum): + """Enum for message service secrets""" + + # Mailgun + MAILGUN_API_KEY = "mailgun_api_key" + + # Twilio SMS + TWILIO_ACCOUNT_SID = "twilio_account_sid" + TWILIO_AUTH_TOKEN = "twilio_auth_token" + TWILIO_MESSAGING_SERVICE_SID = "twilio_messaging_service_sid" + TWILIO_SENDER_PHONE_NUMBER = "twilio_sender_phone_number" + + # Twilio Sendgrid/Email + TWILIO_API_KEY = "twilio_api_key" + + +class MessagingServiceSecretsMailgun(BaseModel): + """The secrets required to connect to mailgun.""" + + mailgun_api_key: str + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + +class MessagingServiceSecretsTwilioSMS(BaseModel): + """The secrets required to connect to twilio SMS.""" + + twilio_account_sid: str + twilio_auth_token: str + twilio_messaging_service_sid: Optional[str] + twilio_sender_phone_number: Optional[str] + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + @root_validator + def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + sender_phone = values.get("twilio_sender_phone_number") + if not values.get("twilio_messaging_service_sid") and not sender_phone: + raise ValueError( + "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." + ) + if sender_phone: + pattern = regex(r"^\+\d+$") + if not pattern.search(sender_phone): + raise ValueError( + "Sender phone number must include country code, formatted like +15558675309" + ) + return values + + +class MessagingServiceSecretsTwilioEmail(BaseModel): + """The secrets required to connect to twilio email.""" + + twilio_api_key: str + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + +class MessagingConfigRequest(BaseModel): + """Messaging Config Request Schema""" + + name: str + key: Optional[FidesOpsKey] + service_type: MessagingServiceType + details: Optional[MessagingServiceDetailsMailgun] + + class Config: + use_enum_values = False + orm_mode = True + + @root_validator + def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + service_type: MessagingServiceType = values.get("service_type") # type: ignore + if service_type == MessagingServiceType.MAILGUN: + if not values.get("details"): + raise ValueError("Mailgun messaging config must include details") + return values + + +class MessagingConfigResponse(BaseModel): + """Messaging Config Response Schema""" + + name: str + key: FidesOpsKey + service_type: MessagingServiceType + details: Optional[Dict[MessagingServiceDetails, Any]] + + class Config: + orm_mode = True + use_enum_values = True + + +SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[ + MessagingServiceSecretsMailgun, + MessagingServiceSecretsTwilioSMS, + MessagingServiceSecretsTwilioEmail, +] + + +class MessagingConnectionTestStatus(Enum): + """Enum for supplying statuses of validating credentials for a messaging Config""" + + succeeded = "succeeded" + failed = "failed" + skipped = "skipped" + + +class TestMessagingStatusMessage(Msg): + """A schema for checking status of message config.""" + + test_status: Optional[MessagingConnectionTestStatus] = None + failure_reason: Optional[str] = None diff --git a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py new file mode 100644 index 00000000000..18e24a52f46 --- /dev/null +++ b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py @@ -0,0 +1,31 @@ +from typing import Union + +from fides.api.ops.schemas.base_class import NoValidationSchema +from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceSecretsMailgun, + MessagingServiceSecretsTwilioEmail, + MessagingServiceSecretsTwilioSMS, +) + + +class MessagingSecretsMailgunDocs(MessagingServiceSecretsMailgun, NoValidationSchema): + """The secrets required to connect to Mailgun, for documentation""" + + +class MessagingSecretsTwilioSMSDocs( + MessagingServiceSecretsTwilioSMS, NoValidationSchema +): + """The secrets required to connect to Twilio sms, for documentation""" + + +class MessagingSecretsTwilioEmailDocs( + MessagingServiceSecretsTwilioEmail, NoValidationSchema +): + """The secrets required to connect to Twilio email, for documentation""" + + +possible_messaging_secrets = Union[ + MessagingSecretsMailgunDocs, + MessagingSecretsTwilioSMSDocs, + MessagingSecretsTwilioEmailDocs, +] diff --git a/src/fides/api/ops/service/_verification.py b/src/fides/api/ops/service/_verification.py index 84d0cb873ae..36979da6562 100644 --- a/src/fides/api/ops/service/_verification.py +++ b/src/fides/api/ops/service/_verification.py @@ -2,13 +2,14 @@ from sqlalchemy.orm import Session -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig, get_messaging_method from fides.api.ops.models.privacy_request import ConsentRequest, PrivacyRequest -from fides.api.ops.schemas.email.email import ( - EmailActionType, +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, SubjectIdentityVerificationBodyParams, ) -from fides.api.ops.service.email.email_dispatch_service import dispatch_email +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message from fides.api.ops.service.privacy_request.request_runner_service import ( generate_id_verification_code, ) @@ -18,24 +19,27 @@ def send_verification_code_to_user( - db: Session, request: ConsentRequest | PrivacyRequest, email: str | None + db: Session, request: ConsentRequest | PrivacyRequest, to_identity: Identity | None ) -> str: - """Generate and cache a verification code, and then email to the user""" - EmailConfig.get_configuration( + """Generate and cache a verification code, and then message the user""" + MessagingConfig.get_configuration( db=db - ) # Validates Fidesops is currently configured to send emails + ) # Validates Fidesops is currently configured to send messages verification_code = generate_id_verification_code() request.cache_identity_verification_code(verification_code) - email_action_type = ( - EmailActionType.CONSENT_REQUEST + messaging_action_type = ( + MessagingActionType.CONSENT_REQUEST if isinstance(request, ConsentRequest) - else EmailActionType.SUBJECT_IDENTITY_VERIFICATION + else MessagingActionType.SUBJECT_IDENTITY_VERIFICATION ) - dispatch_email( + dispatch_message( db, - action_type=email_action_type, - to_email=email, - email_body_params=SubjectIdentityVerificationBodyParams( + action_type=messaging_action_type, + to_identity=to_identity, + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), + message_body_params=SubjectIdentityVerificationBodyParams( verification_code=verification_code, verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, ), diff --git a/src/fides/api/ops/service/connectors/email_connector.py b/src/fides/api/ops/service/connectors/email_connector.py index 57eef852648..c85447ea152 100644 --- a/src/fides/api/ops/service/connectors/email_connector.py +++ b/src/fides/api/ops/service/connectors/email_connector.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import Session from fides.api.ops.common_exceptions import ( - EmailDispatchException, + MessageDispatchException, PrivacyRequestErasureEmailSendRequired, ) from fides.api.ops.graph.config import CollectionAddress, FieldPath @@ -23,10 +23,14 @@ PrivacyRequest, ) from fides.api.ops.schemas.connection_configuration import EmailSchema -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, + MessagingMethod, +) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.base_connector import BaseConnector from fides.api.ops.service.connectors.query_config import ManualQueryConfig -from fides.api.ops.service.email.email_dispatch_service import dispatch_email +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message from fides.api.ops.util.collection_util import Row, append logger = logging.getLogger(__name__) @@ -56,11 +60,12 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: try: # synchronous for now since failure to send is considered a connection test failure - dispatch_email( + dispatch_message( db=db, - action_type=EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT, - to_email=config.test_email, - email_body_params=[ + action_type=MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, + to_identity=Identity(email=config.test_email), + messaging_method=MessagingMethod.EMAIL, + message_body_params=[ CheckpointActionRequired( step=CurrentStep.erasure, collection=CollectionAddress("test_dataset", "test_collection"), @@ -76,7 +81,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: ) ], ) - except EmailDispatchException as exc: + except MessageDispatchException as exc: logger.info("Email connector test failed with exception %s", exc) return ConnectionTestStatus.failed return ConnectionTestStatus.succeeded @@ -194,11 +199,12 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - ) return - dispatch_email( + dispatch_message( db, - action_type=EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT, - to_email=cc.secrets.get("to_email"), - email_body_params=template_values, + action_type=MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, + to_identity=Identity(email=cc.secrets.get("to_email")), + messaging_method=MessagingMethod.EMAIL, + message_body_params=template_values, ) logger.info( diff --git a/src/fides/api/ops/service/email/email_crud_service.py b/src/fides/api/ops/service/email/email_crud_service.py deleted file mode 100644 index 5497682fd72..00000000000 --- a/src/fides/api/ops/service/email/email_crud_service.py +++ /dev/null @@ -1,74 +0,0 @@ -import logging - -from fideslang.validation import FidesKey -from sqlalchemy.orm import Session - -from fides.api.ops.common_exceptions import ( - EmailConfigAlreadyExistsException, - EmailConfigNotFoundException, -) -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.schemas.email.email import EmailConfigRequest, EmailConfigResponse - -logger = logging.getLogger(__name__) - - -def create_email_config(db: Session, config: EmailConfigRequest) -> EmailConfigResponse: - existing_config = db.query(EmailConfig).first() - if existing_config: - raise EmailConfigAlreadyExistsException( - f"Only one email config is supported at a time. Config with key {config.key} is already configured." - ) - return _create_or_update_email_config(db=db, config=config) - - -def update_email_config( - db: Session, key: FidesKey, config: EmailConfigRequest -) -> EmailConfigResponse: - existing_config_with_key: EmailConfig = EmailConfig.get_by( - db=db, field="key", value=key - ) - if not existing_config_with_key: - raise EmailConfigNotFoundException(f"No email config found with key {key}") - return _create_or_update_email_config(db=db, config=config) - - -def _create_or_update_email_config( - db: Session, config: EmailConfigRequest -) -> EmailConfigResponse: - email_config: EmailConfig = EmailConfig.create_or_update( - db=db, - data={ - "key": config.key, - "name": config.name, - "service_type": config.service_type, - "details": config.details.__dict__, - }, - ) - return EmailConfigResponse( - name=email_config.name, - key=email_config.key, - service_type=email_config.service_type, - details=email_config.details, - ) - - -def delete_email_config(db: Session, key: FidesKey) -> None: - logger.info("Finding email config with key '%s'", key) - email_config: EmailConfig = EmailConfig.get_by(db, field="key", value=key) - if not email_config: - raise EmailConfigNotFoundException(f"No email config found with key {key}") - logger.info("Deleting email config with key '%s'", key) - email_config.delete(db) - - -def get_email_config_by_key(db: Session, key: FidesKey) -> EmailConfigResponse: - config = EmailConfig.get_by(db=db, field="key", value=key) - if not config: - raise EmailConfigNotFoundException(f"No email config found with key {key}") - return EmailConfigResponse( - name=config.name, - key=config.key, - service_type=config.service_type, - details=config.details, - ) diff --git a/src/fides/api/ops/service/email/email_dispatch_service.py b/src/fides/api/ops/service/email/email_dispatch_service.py deleted file mode 100644 index ebe6e885a20..00000000000 --- a/src/fides/api/ops/service/email/email_dispatch_service.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Any, Dict, List, Optional, Union - -import requests -from sqlalchemy.orm import Session - -from fides.api.ops.common_exceptions import EmailDispatchException -from fides.api.ops.email_templates import get_email_template -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.models.privacy_request import CheckpointActionRequired -from fides.api.ops.schemas.email.email import ( - AccessRequestCompleteBodyParams, - EmailActionType, - EmailForActionType, - EmailServiceDetails, - EmailServiceSecrets, - EmailServiceType, - FidesopsEmail, - RequestReceiptBodyParams, - RequestReviewDenyBodyParams, - SubjectIdentityVerificationBodyParams, -) -from fides.api.ops.tasks import DatabaseTask, celery_app -from fides.api.ops.util.logger import Pii - -logger = logging.getLogger(__name__) - - -@celery_app.task(base=DatabaseTask, bind=True) -def dispatch_email_task( - self: DatabaseTask, - email_meta: Dict[str, Any], - to_email: str, -) -> None: - """ - A wrapper function to dispatch an email task into the Celery queues - """ - schema = FidesopsEmail.parse_obj(email_meta) - with self.session as db: - dispatch_email( - db, - schema.action_type, - to_email, - schema.body_params, - ) - - -def dispatch_email( - db: Session, - action_type: EmailActionType, - to_email: Optional[str], - email_body_params: Optional[ - Union[ - AccessRequestCompleteBodyParams, - SubjectIdentityVerificationBodyParams, - RequestReceiptBodyParams, - RequestReviewDenyBodyParams, - List[CheckpointActionRequired], - ] - ] = None, -) -> None: - """ - Sends an email to `to_email` with content supplied in `email_body_params` - """ - if not to_email: - logger.error("Email failed to send. No email supplied.") - raise EmailDispatchException("No email supplied.") - - logger.info("Retrieving email config") - email_config: EmailConfig = EmailConfig.get_configuration(db=db) - logger.info("Building appropriate email template for action type: %s", action_type) - email: EmailForActionType = _build_email( - action_type=action_type, - body_params=email_body_params, - ) - email_service: EmailServiceType = email_config.service_type # type: ignore - logger.info( - "Retrieving appropriate dispatcher for email service: %s", email_service - ) - dispatcher: Any = _get_dispatcher_from_config_type(email_service_type=email_service) - logger.info( - "Starting email dispatch for email service with action type: %s", action_type - ) - dispatcher( - email_config=email_config, - email=email, - to_email=to_email, - ) - - -def _build_email( # pylint: disable=too-many-return-statements - action_type: EmailActionType, - body_params: Any, -) -> EmailForActionType: - if action_type == EmailActionType.CONSENT_REQUEST: - template = get_email_template(action_type) - return EmailForActionType( - subject="Your one-time code", - body=template.render( - { - "code": body_params.verification_code, - "minutes": body_params.get_verification_code_ttl_minutes(), - } - ), - ) - if action_type == EmailActionType.SUBJECT_IDENTITY_VERIFICATION: - template = get_email_template(action_type) - return EmailForActionType( - subject="Your one-time code", - body=template.render( - { - "code": body_params.verification_code, - "minutes": body_params.get_verification_code_ttl_minutes(), - } - ), - ) - if action_type == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Data erasure request", - body=base_template.render( - {"dataset_collection_action_required": body_params} - ), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_RECEIPT: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your request has been received", - body=base_template.render({"request_types": body_params.request_types}), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your data is ready to be downloaded", - body=base_template.render( - { - "download_links": body_params.download_links, - } - ), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your data has been deleted", - body=base_template.render(), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your request has been approved", - body=base_template.render(), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your request has been denied", - body=base_template.render( - {"rejection_reason": body_params.rejection_reason} - ), - ) - logger.error("Email action type %s is not implemented", action_type) - raise EmailDispatchException(f"Email action type {action_type} is not implemented") - - -def _get_dispatcher_from_config_type(email_service_type: EmailServiceType) -> Any: - """Determines which dispatcher to use based on email service type""" - return { - EmailServiceType.MAILGUN.value: _mailgun_dispatcher, - }[email_service_type.value] - - -def _mailgun_dispatcher( - email_config: EmailConfig, email: EmailForActionType, to_email: str -) -> None: - """Dispatches email using mailgun""" - base_url = ( - "https://api.mailgun.net" - if email_config.details[EmailServiceDetails.IS_EU_DOMAIN.value] is False - else "https://api.eu.mailgun.net" - ) - domain = email_config.details[EmailServiceDetails.DOMAIN.value] - data = { - "from": f"", - "to": [to_email], - "subject": email.subject, - "html": email.body, - } - try: - response: requests.Response = requests.post( - f"{base_url}/{email_config.details[EmailServiceDetails.API_VERSION.value]}/{domain}/messages", - auth=( - "api", - email_config.secrets[EmailServiceSecrets.MAILGUN_API_KEY.value], # type: ignore - ), - data=data, - ) - if not response.ok: - logger.error( - "Email failed to send with status code: %s", response.status_code - ) - raise EmailDispatchException( - f"Email failed to send with status code {response.status_code}" - ) - except Exception as e: - logger.error("Email failed to send: %s", Pii(str(e))) - raise EmailDispatchException(f"Email failed to send due to: {Pii(e)}") diff --git a/src/fides/api/ops/service/email/__init__.py b/src/fides/api/ops/service/messaging/__init__.py similarity index 100% rename from src/fides/api/ops/service/email/__init__.py rename to src/fides/api/ops/service/messaging/__init__.py diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py new file mode 100644 index 00000000000..0e87d0c539e --- /dev/null +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -0,0 +1,321 @@ +from __future__ import annotations + +import logging +from typing import Any, Callable, Dict, List, Optional, Union + +import requests +from sqlalchemy.orm import Session +from twilio.base.exceptions import TwilioRestException +from twilio.rest import Client + +from fides.api.ops.common_exceptions import MessageDispatchException +from fides.api.ops.email_templates import get_email_template +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.models.privacy_request import CheckpointActionRequired +from fides.api.ops.schemas.messaging.messaging import ( + AccessRequestCompleteBodyParams, + EmailForActionType, + FidesopsMessage, + MessagingActionType, + MessagingMethod, + MessagingServiceDetails, + MessagingServiceSecrets, + MessagingServiceType, + RequestReceiptBodyParams, + RequestReviewDenyBodyParams, + SubjectIdentityVerificationBodyParams, +) +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.tasks import DatabaseTask, celery_app +from fides.api.ops.util.logger import Pii +from fides.ctl.core.config import get_config + +CONFIG = get_config() + +logger = logging.getLogger(__name__) + + +@celery_app.task(base=DatabaseTask, bind=True) +def dispatch_message_task( + self: DatabaseTask, + message_meta: Dict[str, Any], + messaging_method: Optional[MessagingMethod], + to_identity: Identity, +) -> None: + """ + A wrapper function to dispatch a message task into the Celery queues + """ + schema = FidesopsMessage.parse_obj(message_meta) + with self.session as db: + dispatch_message( + db, + schema.action_type, + to_identity, + messaging_method, + schema.body_params, + ) + + +def dispatch_message( + db: Session, + action_type: MessagingActionType, + to_identity: Optional[Identity], + messaging_method: Optional[MessagingMethod], + message_body_params: Optional[ + Union[ + AccessRequestCompleteBodyParams, + SubjectIdentityVerificationBodyParams, + RequestReceiptBodyParams, + RequestReviewDenyBodyParams, + List[CheckpointActionRequired], + ] + ] = None, +) -> None: + """ + Sends a message to `to_identity` with content supplied in `message_body_params` + """ + if not to_identity: + logger.error("Message failed to send. No identity supplied.") + raise MessageDispatchException("No identity supplied.") + + logger.info("Retrieving message config") + messaging_config: MessagingConfig = MessagingConfig.get_configuration(db=db) + logger.info( + "Building appropriate message template for action type: %s", action_type + ) + message: Optional[Union[EmailForActionType, str]] = None + if messaging_method == MessagingMethod.EMAIL: + message = _build_email( + action_type=action_type, + body_params=message_body_params, + ) + elif messaging_method == MessagingMethod.SMS: + message = _build_sms( + action_type=action_type, + body_params=message_body_params, + ) + else: + logger.error( + "Notification service type is not valid: %s", + CONFIG.notifications.notification_service_type, + ) + raise MessageDispatchException( + f"Notification service type is not valid: {CONFIG.notifications.notification_service_type}" + ) + messaging_service: MessagingServiceType = messaging_config.service_type # type: ignore + logger.info( + "Retrieving appropriate dispatcher for email service: %s", messaging_service + ) + dispatcher: Optional[ + Callable[[MessagingConfig, Any, Optional[str]], None] + ] = _get_dispatcher_from_config_type(message_service_type=messaging_service) + if not dispatcher: + logger.error( + "Dispatcher has not been implemented for message service type: %s", + messaging_service, + ) + raise MessageDispatchException( + f"Dispatcher has not been implemented for message service type: {messaging_service}" + ) + logger.info( + "Starting message dispatch for messaging service with action type: %s", + action_type, + ) + dispatcher( + messaging_config, + message, + to_identity.email + if messaging_method == MessagingMethod.EMAIL + else to_identity.phone_number, + ) + + +def _build_sms( + action_type: MessagingActionType, + body_params: Any, +) -> str: + if action_type == MessagingActionType.CONSENT_REQUEST: + return "Hello, this message was sent from Fides!" + logger.error("Message action type %s is not implemented", action_type) + raise MessageDispatchException( + f"Message action type {action_type} is not implemented" + ) + + +def _build_email( # pylint: disable=too-many-return-statements + action_type: MessagingActionType, + body_params: Any, +) -> EmailForActionType: + if action_type == MessagingActionType.CONSENT_REQUEST: + template = get_email_template(action_type) + return EmailForActionType( + subject="Your one-time code", + body=template.render( + { + "code": body_params.verification_code, + "minutes": body_params.get_verification_code_ttl_minutes(), + } + ), + ) + if action_type == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION: + template = get_email_template(action_type) + return EmailForActionType( + subject="Your one-time code", + body=template.render( + { + "code": body_params.verification_code, + "minutes": body_params.get_verification_code_ttl_minutes(), + } + ), + ) + if action_type == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Data erasure request", + body=base_template.render( + {"dataset_collection_action_required": body_params} + ), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_RECEIPT: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your request has been received", + body=base_template.render({"request_types": body_params.request_types}), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your data is ready to be downloaded", + body=base_template.render( + { + "download_links": body_params.download_links, + } + ), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your data has been deleted", + body=base_template.render(), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your request has been approved", + body=base_template.render(), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY: + base_template = get_email_template(action_type) + return EmailForActionType( + subject="Your request has been denied", + body=base_template.render( + {"rejection_reason": body_params.rejection_reason} + ), + ) + logger.error("Message action type %s is not implemented", action_type) + raise MessageDispatchException( + f"Message action type {action_type} is not implemented" + ) + + +def _get_dispatcher_from_config_type( + message_service_type: MessagingServiceType, +) -> Optional[Callable[[MessagingConfig, Any, Optional[str]], None]]: + """Determines which dispatcher to use based on message service type""" + if message_service_type == MessagingServiceType.MAILGUN: + return _mailgun_dispatcher + if message_service_type == MessagingServiceType.TWILIO_TEXT: + return _twilio_sms_dispatcher + return None + + +def _mailgun_dispatcher( + messaging_config: MessagingConfig, + message: EmailForActionType, + to: Optional[str], +) -> None: + """Dispatches email using mailgun""" + if not to: + logger.error("Message failed to send. No email identity supplied.") + raise MessageDispatchException("No email identity supplied.") + if not messaging_config.details or not messaging_config.secrets: + logger.error( + "Message failed to send. No mailgun config details or secrets supplied." + ) + raise MessageDispatchException("No mailgun config details or secrets supplied.") + base_url = ( + "https://api.mailgun.net" + if messaging_config.details[MessagingServiceDetails.IS_EU_DOMAIN.value] is False + else "https://api.eu.mailgun.net" + ) + domain = messaging_config.details[MessagingServiceDetails.DOMAIN.value] + data = { + "from": f"", + "to": [to.strip()], + "subject": message.subject, + "html": message.body, + } + try: + response: requests.Response = requests.post( + f"{base_url}/{messaging_config.details[MessagingServiceDetails.API_VERSION.value]}/{domain}/messages", + auth=( + "api", + messaging_config.secrets[MessagingServiceSecrets.MAILGUN_API_KEY.value], + ), + data=data, + ) + if not response.ok: + logger.error( + "Email failed to send with status code: %s", response.status_code + ) + raise MessageDispatchException( + f"Email failed to send with status code {response.status_code}" + ) + except Exception as e: + logger.error("Email failed to send: %s", Pii(str(e))) + raise MessageDispatchException(f"Email failed to send due to: {Pii(e)}") + + +def _twilio_sms_dispatcher( + messaging_config: MessagingConfig, + message: str, + to: Optional[str], +) -> None: + """Dispatches SMS using Twilio""" + if not to: + logger.error("Message failed to send. No phone identity supplied.") + raise MessageDispatchException("No phone identity supplied.") + if messaging_config.secrets is None: + logger.error("Message failed to send. No config secrets supplied.") + raise MessageDispatchException("No config secrets supplied.") + account_sid = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value + ] + auth_token = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value + ] + messaging_service_id = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] + sender_phone_number = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value + ] + + client = Client(account_sid, auth_token) + try: + if messaging_service_id: + client.messages.create( + to=to, messaging_service_sid=messaging_service_id, body=message + ) + elif sender_phone_number: + client.messages.create(to=to, from_=sender_phone_number, body=message) + else: + logger.error( + "Message failed to send. Either sender phone number or messaging service sid must be provided." + ) + raise MessageDispatchException( + "Message failed to send. Either sender phone number or messaging service sid must be provided." + ) + except TwilioRestException as e: + logger.error("Twilio SMS failed to send: %s", Pii(str(e))) + raise MessageDispatchException(f"Twilio SMS failed to send due to: {Pii(e)}") diff --git a/src/fides/api/ops/service/messaging/messaging_crud_service.py b/src/fides/api/ops/service/messaging/messaging_crud_service.py new file mode 100644 index 00000000000..2a8f204a415 --- /dev/null +++ b/src/fides/api/ops/service/messaging/messaging_crud_service.py @@ -0,0 +1,75 @@ +import logging + +from fideslang.validation import FidesKey +from sqlalchemy.orm import Session + +from fides.api.ops.common_exceptions import MessagingConfigNotFoundException +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.schemas.messaging.messaging import ( + MessagingConfigRequest, + MessagingConfigResponse, +) + +logger = logging.getLogger(__name__) + + +def update_messaging_config( + db: Session, key: FidesKey, config: MessagingConfigRequest +) -> MessagingConfigResponse: + existing_config_with_key: MessagingConfig = MessagingConfig.get_by( + db=db, field="key", value=key + ) + if not existing_config_with_key: + raise MessagingConfigNotFoundException( + f"No messaging config found with key {key}" + ) + return create_or_update_messaging_config(db=db, config=config) + + +def create_or_update_messaging_config( + db: Session, config: MessagingConfigRequest +) -> MessagingConfigResponse: + data = { + "key": config.key, + "name": config.name, + "service_type": config.service_type, + } + if config.details: + data["details"] = config.details.__dict__ # type: ignore + messaging_config: MessagingConfig = MessagingConfig.create_or_update( + db=db, + data=data, + ) + return MessagingConfigResponse( + name=messaging_config.name, + key=messaging_config.key, + service_type=messaging_config.service_type, + details=messaging_config.details, + ) + + +def delete_messaging_config(db: Session, key: FidesKey) -> None: + logger.info("Finding messaging config with key '%s'", key) + messaging_config: MessagingConfig = MessagingConfig.get_by( + db, field="key", value=key + ) + if not messaging_config: + raise MessagingConfigNotFoundException( + f"No messaging config found with key {key}" + ) + logger.info("Deleting messaging config with key '%s'", key) + messaging_config.delete(db) + + +def get_messaging_config_by_key(db: Session, key: FidesKey) -> MessagingConfigResponse: + config = MessagingConfig.get_by(db=db, field="key", value=key) + if not config: + raise MessagingConfigNotFoundException( + f"No messaging config found with key {key}" + ) + return MessagingConfigResponse( + name=config.name, + key=config.key, + service_type=config.service_type, + details=config.details, + ) diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py index b84efa65f3d..69c5460487b 100644 --- a/src/fides/api/ops/service/privacy_request/request_runner_service.py +++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py @@ -14,9 +14,9 @@ from fides.api.ops import common_exceptions from fides.api.ops.common_exceptions import ( ClientUnsuccessfulException, - EmailDispatchException, IdentityNotFoundException, ManualWebhookFieldsUnset, + MessageDispatchException, NoCachedManualWebhookEntry, PrivacyRequestPaused, ) @@ -28,6 +28,7 @@ from fides.api.ops.models.connectionconfig import ConnectionConfig from fides.api.ops.models.datasetconfig import DatasetConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook +from fides.api.ops.models.messaging import get_messaging_method from fides.api.ops.models.policy import ( ActionType, CurrentStep, @@ -42,14 +43,15 @@ ProvidedIdentityType, can_run_checkpoint, ) -from fides.api.ops.schemas.email.email import ( +from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, - EmailActionType, + MessagingActionType, ) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( email_connector_erasure_send, ) -from fides.api.ops.service.email.email_dispatch_service import dispatch_email +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message from fides.api.ops.service.storage.storage_uploader_service import upload from fides.api.ops.task.filter_results import filter_data_categories from fides.api.ops.task.graph_task import ( @@ -380,7 +382,7 @@ async def run_privacy_request( email_connector_erasure_send( db=session, privacy_request=privacy_request ) - except EmailDispatchException as exc: + except MessageDispatchException as exc: privacy_request.cache_failed_checkpoint_details( step=CurrentStep.erasure_email_post_send, collection=None ) @@ -405,7 +407,7 @@ async def run_privacy_request( initiate_privacy_request_completion_email( session, policy, access_result_urls, identity_data ) - except (IdentityNotFoundException, EmailDispatchException) as e: + except (IdentityNotFoundException, MessageDispatchException) as e: privacy_request.error_processing(db=session) # If dev mode, log traceback await fideslog_graph_failure( @@ -444,22 +446,32 @@ def initiate_privacy_request_completion_email( raise IdentityNotFoundException( "Identity email was not found, so request completion email could not be sent." ) + to_identity: Identity = Identity( + email=identity_data.get(ProvidedIdentityType.email.value), + phone_number=identity_data.get(ProvidedIdentityType.phone_number.value), + ) if policy.get_rules_for_action(action_type=ActionType.access): # synchronous for now since failure to send complete emails is fatal to request - dispatch_email( + dispatch_message( db=session, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, - to_email=identity_data.get(ProvidedIdentityType.email.value), - email_body_params=AccessRequestCompleteBodyParams( + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, + to_identity=to_identity, + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), + message_body_params=AccessRequestCompleteBodyParams( download_links=access_result_urls ), ) if policy.get_rules_for_action(action_type=ActionType.erasure): - dispatch_email( + dispatch_message( db=session, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION, - to_email=identity_data.get(ProvidedIdentityType.email.value), - email_body_params=None, + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, + to_identity=to_identity, + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), + message_body_params=None, ) diff --git a/src/fides/api/ops/tasks/__init__.py b/src/fides/api/ops/tasks/__init__.py index d583814fa2d..653aee0b446 100644 --- a/src/fides/api/ops/tasks/__init__.py +++ b/src/fides/api/ops/tasks/__init__.py @@ -11,7 +11,7 @@ logger = get_task_logger(__name__) CONFIG = get_config() -EMAIL_QUEUE_NAME = "fidesops.email" +MESSAGING_QUEUE_NAME = "fidesops.messaging" class DatabaseTask(Task): # pylint: disable=W0223 @@ -87,7 +87,7 @@ def start_worker() -> None: "worker", "--loglevel=info", "--concurrency=2", - f"--queues={default_queue_name},{EMAIL_QUEUE_NAME}", + f"--queues={default_queue_name},{MESSAGING_QUEUE_NAME}", ] ) diff --git a/src/fides/ctl/core/config/__init__.py b/src/fides/ctl/core/config/__init__.py index 5c1afc2e954..0afed3547cb 100644 --- a/src/fides/ctl/core/config/__init__.py +++ b/src/fides/ctl/core/config/__init__.py @@ -65,6 +65,7 @@ def log_all_config_values(self) -> None: self.user, self.logging, self.database, + self.notifications, self.redis, self.security, self.execution, @@ -128,6 +129,12 @@ def handle_deprecated_env_variables(settings: MutableMapping) -> MutableMapping: "db", "test_db", ], + "notifications": [ + "send_request_completion_notification", + "send_request_receipt_notification", + "send_request_review_notification", + "notification_service_type", + ], "redis": [ "host", "port", diff --git a/src/fides/ctl/core/config/notification_settings.py b/src/fides/ctl/core/config/notification_settings.py index 4396ab0a135..efd0e71390c 100644 --- a/src/fides/ctl/core/config/notification_settings.py +++ b/src/fides/ctl/core/config/notification_settings.py @@ -1,4 +1,7 @@ import logging +from typing import Optional + +from pydantic import validator from .fides_settings import FidesSettings @@ -13,6 +16,22 @@ class NotificationSettings(FidesSettings): send_request_completion_notification: bool = False send_request_receipt_notification: bool = False send_request_review_notification: bool = False + notification_service_type: Optional[str] = None + + @validator("notification_service_type", pre=True) + @classmethod + def validate_notification_service_type(cls, value: Optional[str]) -> Optional[str]: + """Ensure the provided type is a valid value.""" + if value: + valid_values = ["MAILGUN", "TWILIO_TEXT", "TWILIO_EMAIL"] + value = value.upper() # force uppercase for safety + + if value not in valid_values: + raise ValueError( + f"Invalid NOTIFICATION_SERVICE_TYPE provided '{value}', must be one of: {', '.join([level for level in valid_values])}" + ) + + return value class Config: env_prefix = ENV_PREFIX diff --git a/src/fides/ctl/core/deploy.py b/src/fides/ctl/core/deploy.py index 24e9c7ccb65..b81d06d2cd3 100644 --- a/src/fides/ctl/core/deploy.py +++ b/src/fides/ctl/core/deploy.py @@ -96,7 +96,8 @@ def check_docker_version() -> bool: def seed_example_data() -> None: run_shell( - DOCKER_COMPOSE_COMMAND + "run --no-deps --rm fides fides push src/fides/data/sample_project/sample_resources/" + DOCKER_COMPOSE_COMMAND + + "run --no-deps --rm fides fides push src/fides/data/sample_project/sample_resources/" ) run_shell( DOCKER_COMPOSE_COMMAND diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 422a763c3f9..28fba585a64 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -27,8 +27,9 @@ ManualAction, PrivacyRequestStatus, ) -from fides.api.ops.schemas.email.email import EmailActionType -from fides.api.ops.tasks import EMAIL_QUEUE_NAME +from fides.api.ops.schemas.messaging.messaging import MessagingActionType +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.tasks import MESSAGING_QUEUE_NAME page_size = Params().size @@ -1421,10 +1422,10 @@ def test_put_saas_example_connection_config_secrets_missing_saas_config( == f"A SaaS config to validate the secrets is unavailable for this connection config, please add one via {SAAS_CONFIG}" ) - @mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_email") + @mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_message") def test_put_email_connection_config_secrets( self, - mock_dispatch_email, + mock_dispatch_message, api_client: TestClient, db: Session, generate_auth_header, @@ -1461,13 +1462,14 @@ def test_put_email_connection_config_secrets( assert email_connection_config.last_test_timestamp is not None assert email_connection_config.last_test_succeeded is not None - assert mock_dispatch_email.called - kwargs = mock_dispatch_email.call_args.kwargs + assert mock_dispatch_message.called + kwargs = mock_dispatch_message.call_args.kwargs assert ( - kwargs["action_type"] == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT + kwargs["action_type"] + == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT ) - assert kwargs["to_email"] == "test@example.com" - assert kwargs["email_body_params"] == [ + assert kwargs["to_identity"] == Identity(email="test@example.com") + assert kwargs["message_body_params"] == [ CheckpointActionRequired( step=CurrentStep.erasure, collection=CollectionAddress("test_dataset", "test_collection"), diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index d8798e724f9..01e72a69837 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -56,28 +56,28 @@ def url(self) -> str: return f"{V1_URL_PREFIX}{CONSENT_REQUEST}" @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", ) - @patch("fides.api.ops.service._verification.dispatch_email") - def test_consent_request(self, mock_dispatch_email, api_client, url): + @patch("fides.api.ops.service._verification.dispatch_message") + def test_consent_request(self, mock_dispatch_message, api_client, url): data = {"email": "test@example.com"} response = api_client.post(url, json=data) assert response.status_code == 200 - assert mock_dispatch_email.called + assert mock_dispatch_message.called @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", ) - @patch("fides.api.ops.service._verification.dispatch_email") + @patch("fides.api.ops.service._verification.dispatch_message") def test_consent_request_identity_present( self, - mock_dispatch_email, + mock_dispatch_message, provided_identity_and_consent_request, api_client, url, @@ -86,10 +86,10 @@ def test_consent_request_identity_present( data = {"email": provided_identity.encrypted_value["value"]} response = api_client.post(url, json=data) assert response.status_code == 200 - assert mock_dispatch_email.called + assert mock_dispatch_message.called @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", @@ -102,21 +102,21 @@ def test_consent_request_redis_disabled(self, api_client, url): assert "redis cache required" in response.json()["message"] @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", ) - @patch("fides.api.ops.service._verification.dispatch_email") + @patch("fides.api.ops.service._verification.dispatch_message") def test_consent_request_subject_verification_disabled_no_email( - self, mock_dispatch_email, api_client, url + self, mock_dispatch_message, api_client, url ): data = {"email": "test@example.com"} response = api_client.post(url, json=data) assert response.status_code == 200 - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", diff --git a/tests/ops/api/v1/endpoints/test_email_endpoints.py b/tests/ops/api/v1/endpoints/test_email_endpoints.py deleted file mode 100644 index 7aa893c6815..00000000000 --- a/tests/ops/api/v1/endpoints/test_email_endpoints.py +++ /dev/null @@ -1,498 +0,0 @@ -import json - -import pytest -from fastapi_pagination import Params -from sqlalchemy.orm import Session -from starlette.testclient import TestClient - -from fides.api.ops.api.v1.scope_registry import ( - EMAIL_CREATE_OR_UPDATE, - EMAIL_DELETE, - EMAIL_READ, -) -from fides.api.ops.api.v1.urn_registry import ( - EMAIL_BY_KEY, - EMAIL_CONFIG, - EMAIL_SECRETS, - V1_URL_PREFIX, -) -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.schemas.email.email import ( - EmailServiceDetails, - EmailServiceSecrets, - EmailServiceType, -) - -PAGE_SIZE = Params().size - - -class TestPostEmailConfig: - @pytest.fixture(scope="function") - def url(self) -> str: - return V1_URL_PREFIX + EMAIL_CONFIG - - @pytest.fixture(scope="function") - def payload(self): - return { - "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, - } - - def test_post_email_config_not_authenticated( - self, api_client: TestClient, payload, url - ): - response = api_client.post(url, headers={}, json=payload) - assert 401 == response.status_code - - def test_post_email_config_incorrect_scope( - self, - api_client: TestClient, - payload, - url, - generate_auth_header, - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.post(url, headers=auth_header, json=payload) - assert 403 == response.status_code - - def test_post_email_config_with_invalid_mailgun_details( - self, - db: Session, - api_client: TestClient, - url, - payload, - generate_auth_header, - ): - payload["details"] = {"invalid": "invalid"} - - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.post(url, headers=auth_header, json=payload) - assert 422 == response.status_code - assert json.loads(response.text)["detail"][0]["msg"] == "field required" - assert ( - json.loads(response.text)["detail"][1]["msg"] - == "extra fields not permitted" - ) - - def test_post_email_config_with_not_supported_service_type( - self, - db: Session, - api_client: TestClient, - url, - payload, - generate_auth_header, - ): - payload["service_type"] = "twilio" - - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.post(url, headers=auth_header, json=payload) - assert 422 == response.status_code - assert ( - json.loads(response.text)["detail"][0]["msg"] - == "value is not a valid enumeration member; permitted: 'mailgun'" - ) - - def test_post_email_config_with_no_key( - self, - db: Session, - api_client: TestClient, - payload, - url, - generate_auth_header, - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.post(url, headers=auth_header, json=payload) - - assert 200 == response.status_code - response_body = json.loads(response.text) - - assert response_body["key"] == "mailgun" - email_config = db.query(EmailConfig).filter_by(key="mailgun")[0] - email_config.delete(db) - - def test_post_email_config_with_invalid_key( - self, - db: Session, - api_client: TestClient, - payload, - url, - generate_auth_header, - ): - payload["key"] = "*invalid-key" - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.post(url, headers=auth_header, json=payload) - assert 422 == response.status_code - assert ( - json.loads(response.text)["detail"][0]["msg"] - == "FidesKey must only contain alphanumeric characters, '.', '_' or '-'." - ) - - def test_post_email_config_with_key( - self, - db: Session, - api_client: TestClient, - payload, - url, - generate_auth_header, - ): - payload["key"] = "my_email_config" - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - - response = api_client.post(url, headers=auth_header, json=payload) - assert 200 == response.status_code - - response_body = json.loads(response.text) - email_config = db.query(EmailConfig).filter_by(key="my_email_config")[0] - - expected_response = { - "key": "my_email_config", - "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, - "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "my.mailgun.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, - }, - } - assert expected_response == response_body - email_config.delete(db) - - def test_post_email_config_missing_detail( - self, - api_client: TestClient, - url, - generate_auth_header, - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.post( - url, - headers=auth_header, - json={ - "key": "my_email_config", - "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, - "details": { - # "domain": "my.mailgun.domain" - }, - }, - ) - assert response.status_code == 422 - errors = response.json()["detail"] - assert "details" in errors[0]["loc"] - assert errors[0]["msg"] == "field required" - - def test_post_email_config_already_exists( - self, - api_client: TestClient, - url, - email_config, - generate_auth_header, - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.post( - url, - headers=auth_header, - json={ - "key": "my_email_config", - "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, - }, - ) - assert response.status_code == 400 - assert response.json() == { - "detail": "Only one email config is supported at a time. Config with key my_email_config is already configured." - } - - -class TestPatchEmailConfig: - @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) - - @pytest.fixture(scope="function") - def payload(self): - return { - "key": "my_email_config", - "name": "mailgun new name", - "service_type": EmailServiceType.MAILGUN.value, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, - } - - def test_patch_email_config_not_authenticated( - self, api_client: TestClient, payload, url - ): - response = api_client.patch(url, headers={}, json=payload) - assert 401 == response.status_code - - def test_patch_email_config_incorrect_scope( - self, - api_client: TestClient, - payload, - url, - generate_auth_header, - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.patch(url, headers=auth_header, json=payload) - assert 403 == response.status_code - - def test_patch_email_config_with_key_not_found( - self, - db: Session, - api_client: TestClient, - payload, - email_config, - generate_auth_header, - ): - url = (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key="nonexistent_key") - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - - response = api_client.patch(url, headers=auth_header, json=payload) - assert response.status_code == 404 - assert response.json() == { - "detail": "No email config found with key nonexistent_key" - } - - def test_patch_email_config_with_key( - self, - db: Session, - api_client: TestClient, - payload, - url, - generate_auth_header, - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - - response = api_client.patch(url, headers=auth_header, json=payload) - assert 200 == response.status_code - - response_body = json.loads(response.text) - email_config = db.query(EmailConfig).filter_by(key="my_email_config")[0] - - expected_response = { - "key": "my_email_config", - "name": "mailgun new name", - "service_type": EmailServiceType.MAILGUN.value, - "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "my.mailgun.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, - }, - } - assert expected_response == response_body - email_config.delete(db) - - -class TestPutEmailConfigSecretsMailgun: - @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_SECRETS).format(config_key=email_config.key) - - @pytest.fixture(scope="function") - def payload(self): - return { - EmailServiceSecrets.MAILGUN_API_KEY.value: "1345234524", - } - - def test_put_config_secrets_unauthenticated( - self, api_client: TestClient, payload, url - ): - response = api_client.put(url, headers={}, json=payload) - assert 401 == response.status_code - - def test_put_config_secrets_wrong_scope( - self, api_client: TestClient, payload, url, generate_auth_header - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.put(url, headers=auth_header, json=payload) - assert 403 == response.status_code - - def test_put_config_secret_invalid_config( - self, api_client: TestClient, payload, generate_auth_header - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - url = (V1_URL_PREFIX + EMAIL_SECRETS).format(config_key="invalid_key") - response = api_client.put(url, headers=auth_header, json=payload) - assert 404 == response.status_code - - def test_update_with_invalid_secrets_key( - self, api_client: TestClient, generate_auth_header, url - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.put(url, headers=auth_header, json={"bad_key": "12345"}) - - assert response.status_code == 400 - assert response.json() == { - "detail": [ - "field required ('mailgun_api_key',)", - "extra fields not permitted ('bad_key',)", - ] - } - - def test_put_config_secrets( - self, - db: Session, - api_client: TestClient, - payload, - url, - generate_auth_header, - email_config, - ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - response = api_client.put(url, headers=auth_header, json=payload) - assert 200 == response.status_code - - db.refresh(email_config) - - assert json.loads(response.text) == { - "msg": "Secrets updated for EmailConfig with key: my_email_config.", - "test_status": None, - "failure_reason": None, - } - assert ( - email_config.secrets[EmailServiceSecrets.MAILGUN_API_KEY.value] - == "1345234524" - ) - - -class TestGetEmailConfigs: - @pytest.fixture(scope="function") - def url(self) -> str: - return V1_URL_PREFIX + EMAIL_CONFIG - - def test_get_configs_not_authenticated(self, api_client: TestClient, url) -> None: - response = api_client.get(url) - assert 401 == response.status_code - - def test_get_configs_wrong_scope( - self, api_client: TestClient, url, generate_auth_header - ) -> None: - auth_header = generate_auth_header([EMAIL_DELETE]) - response = api_client.get(url, headers=auth_header) - assert 403 == response.status_code - - def test_get_configs( - self, db, api_client: TestClient, url, generate_auth_header, email_config - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.get(url, headers=auth_header) - assert 200 == response.status_code - - expected_response = { - "items": [ - { - "key": "my_email_config", - "name": email_config.name, - "service_type": EmailServiceType.MAILGUN.value, - "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "some.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, - }, - } - ], - "page": 1, - "size": PAGE_SIZE, - "total": 1, - } - response_body = json.loads(response.text) - assert expected_response == response_body - - -class TestGetEmailConfig: - @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) - - def test_get_config_not_authenticated(self, url, api_client: TestClient): - response = api_client.get(url) - assert 401 == response.status_code - - def test_get_config_wrong_scope( - self, url, api_client: TestClient, generate_auth_header - ): - auth_header = generate_auth_header([EMAIL_DELETE]) - response = api_client.get(url, headers=auth_header) - assert 403 == response.status_code - - def test_get_config_invalid( - self, api_client: TestClient, generate_auth_header, email_config - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.get( - (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key="invalid"), - headers=auth_header, - ) - assert 404 == response.status_code - - def test_get_config( - self, url, api_client: TestClient, generate_auth_header, email_config - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.get(url, headers=auth_header) - assert response.status_code == 200 - - response_body = json.loads(response.text) - - assert response_body == { - "key": "my_email_config", - "name": email_config.name, - "service_type": EmailServiceType.MAILGUN.value, - "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "some.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, - }, - } - - -class TestDeleteConfig: - @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) - - def test_delete_config_not_authenticated(self, url, api_client: TestClient): - response = api_client.delete(url) - assert 401 == response.status_code - - def test_delete_config_wrong_scope( - self, url, api_client: TestClient, generate_auth_header - ): - auth_header = generate_auth_header([EMAIL_READ]) - response = api_client.delete(url, headers=auth_header) - assert 403 == response.status_code - - def test_delete_config_invalid(self, api_client: TestClient, generate_auth_header): - auth_header = generate_auth_header([EMAIL_DELETE]) - response = api_client.delete( - (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key="invalid"), - headers=auth_header, - ) - assert 404 == response.status_code - - def test_delete_config( - self, - db: Session, - url, - api_client: TestClient, - generate_auth_header, - ): - # Creating new config, so we don't run into issues trying to clean up a deleted fixture - email_config = EmailConfig.create( - db=db, - data={ - "key": "my_different_email_config", - "name": "mailgun", - "service_type": EmailServiceType.MAILGUN, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, - }, - ) - url = (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) - auth_header = generate_auth_header([EMAIL_DELETE]) - response = api_client.delete(url, headers=auth_header) - assert response.status_code == 204 - - db.expunge_all() - config = db.query(EmailConfig).filter_by(key=email_config.key).first() - assert config is None diff --git a/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py b/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py index dc72bec474b..1a57913df06 100644 --- a/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py @@ -37,7 +37,7 @@ def test_get_config_with_verification_required_with_email_config( url, db, api_client: TestClient, - email_config, + messaging_config, subject_identity_verification_required, ): resp = api_client.get(url) @@ -51,7 +51,7 @@ def test_get_config_with_verification_not_required_with_email_config( url, db, api_client: TestClient, - email_config, + messaging_config, ): resp = api_client.get(url) assert resp.status_code == 200 diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py new file mode 100644 index 00000000000..793c6dd4fc0 --- /dev/null +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -0,0 +1,754 @@ +import json + +import pytest +from fastapi_pagination import Params +from sqlalchemy.orm import Session +from starlette.testclient import TestClient + +from fides.api.ops.api.v1.scope_registry import ( + MESSAGING_CREATE_OR_UPDATE, + MESSAGING_DELETE, + MESSAGING_READ, +) +from fides.api.ops.api.v1.urn_registry import ( + MESSAGING_BY_KEY, + MESSAGING_CONFIG, + MESSAGING_SECRETS, + V1_URL_PREFIX, +) +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceDetails, + MessagingServiceSecrets, + MessagingServiceType, +) + +PAGE_SIZE = Params().size + + +class TestPostMessagingConfig: + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + MESSAGING_CONFIG + + @pytest.fixture(scope="function") + def payload(self): + return { + "name": "mailgun", + "service_type": MessagingServiceType.MAILGUN.value, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + } + + @pytest.fixture(scope="function") + def payload_twilio_email(self): + return { + "name": "twilio_email", + "service_type": MessagingServiceType.TWILIO_EMAIL.value, + } + + @pytest.fixture(scope="function") + def payload_twilio_sms(self): + return { + "name": "twilio_sms", + "service_type": MessagingServiceType.TWILIO_TEXT.value, + } + + def test_post_email_config_not_authenticated( + self, api_client: TestClient, payload, url + ): + response = api_client.post(url, headers={}, json=payload) + assert 401 == response.status_code + + def test_post_email_config_incorrect_scope( + self, + api_client: TestClient, + payload, + url, + generate_auth_header, + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.post(url, headers=auth_header, json=payload) + assert 403 == response.status_code + + def test_post_email_config_with_invalid_mailgun_details( + self, + db: Session, + api_client: TestClient, + url, + payload, + generate_auth_header, + ): + payload["details"] = {"invalid": "invalid"} + + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.post(url, headers=auth_header, json=payload) + assert 422 == response.status_code + assert response.json()["detail"][0]["msg"] == "field required" + assert response.json()["detail"][1]["msg"] == "extra fields not permitted" + + def test_post_email_config_with_not_supported_service_type( + self, + db: Session, + api_client: TestClient, + url, + payload, + generate_auth_header, + ): + payload["service_type"] = "twilio" + + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.post(url, headers=auth_header, json=payload) + assert 422 == response.status_code + assert ( + json.loads(response.text)["detail"][0]["msg"] + == "value is not a valid enumeration member; permitted: 'mailgun', 'twilio_text', 'twilio_email'" + ) + + def test_post_email_config_with_no_key( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.post(url, headers=auth_header, json=payload) + + assert 200 == response.status_code + response_body = json.loads(response.text) + + assert response_body["key"] == "mailgun" + email_config = db.query(MessagingConfig).filter_by(key="mailgun")[0] + email_config.delete(db) + + def test_post_email_config_with_invalid_key( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + ): + payload["key"] = "*invalid-key" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.post(url, headers=auth_header, json=payload) + assert 422 == response.status_code + assert ( + json.loads(response.text)["detail"][0]["msg"] + == "FidesKey must only contain alphanumeric characters, '.', '_' or '-'." + ) + + def test_post_email_config_with_key( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + ): + payload["key"] = "my_mailgun_messaging_config" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.post(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by( + key="my_mailgun_messaging_config" + )[0] + + expected_response = { + "key": "my_mailgun_messaging_config", + "name": "mailgun", + "service_type": MessagingServiceType.MAILGUN.value, + "details": { + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, + }, + } + assert expected_response == response_body + email_config.delete(db) + + def test_post_email_config_missing_detail( + self, + api_client: TestClient, + url, + generate_auth_header, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.post( + url, + headers=auth_header, + json={ + "key": "my_mailgun_messaging_config", + "name": "mailgun", + "service_type": MessagingServiceType.MAILGUN.value, + }, + ) + assert response.status_code == 422 + errors = response.json()["detail"] + assert errors[0]["msg"] == "Mailgun messaging config must include details" + + def test_post_email_config_service_already_exists( + self, + api_client: TestClient, + url, + messaging_config, + generate_auth_header, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.post( + url, + headers=auth_header, + json={ + "key": "my_new_mailgun_messaging_config", + "name": "mailgun", + "service_type": MessagingServiceType.MAILGUN.value, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + }, + ) + assert response.status_code == 500 + assert ( + f"Key (service_type)=(MAILGUN) already exists" in response.json()["detail"] + ) + + def test_post_twilio_email_config( + self, + db: Session, + api_client: TestClient, + payload_twilio_email, + url, + generate_auth_header, + ): + payload_twilio_email["key"] = "my_twilio_email_config" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.post(url, headers=auth_header, json=payload_twilio_email) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by( + key="my_twilio_email_config" + )[0] + + expected_response = { + "key": "my_twilio_email_config", + "name": "twilio_email", + "service_type": MessagingServiceType.TWILIO_EMAIL.value, + "details": None, + } + assert expected_response == response_body + email_config.delete(db) + + def test_post_twilio_sms_config( + self, + db: Session, + api_client: TestClient, + payload_twilio_sms, + url, + generate_auth_header, + ): + payload_twilio_sms["key"] = "my_twilio_sms_config" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.post(url, headers=auth_header, json=payload_twilio_sms) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by(key="my_twilio_sms_config")[ + 0 + ] + + expected_response = { + "key": "my_twilio_sms_config", + "name": "twilio_sms", + "service_type": MessagingServiceType.TWILIO_TEXT.value, + "details": None, + } + assert expected_response == response_body + email_config.delete(db) + + +class TestPatchMessagingConfig: + @pytest.fixture(scope="function") + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=messaging_config.key + ) + + @pytest.fixture(scope="function") + def payload(self): + return { + "key": "my_mailgun_messaging_config", + "name": "mailgun new name", + "service_type": MessagingServiceType.MAILGUN.value, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + } + + def test_patch_email_config_not_authenticated( + self, api_client: TestClient, payload, url + ): + response = api_client.patch(url, headers={}, json=payload) + assert 401 == response.status_code + + def test_patch_email_config_incorrect_scope( + self, + api_client: TestClient, + payload, + url, + generate_auth_header, + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.patch(url, headers=auth_header, json=payload) + assert 403 == response.status_code + + def test_patch_email_config_with_key_not_found( + self, + db: Session, + api_client: TestClient, + payload, + messaging_config, + generate_auth_header, + ): + url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key="nonexistent_key") + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.patch(url, headers=auth_header, json=payload) + assert response.status_code == 404 + assert response.json() == { + "detail": "No messaging config found with key nonexistent_key" + } + + def test_patch_email_config_with_key( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.patch(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by( + key="my_mailgun_messaging_config" + )[0] + + expected_response = { + "key": "my_mailgun_messaging_config", + "name": "mailgun new name", + "service_type": MessagingServiceType.MAILGUN.value, + "details": { + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, + }, + } + assert expected_response == response_body + email_config.delete(db) + + +class TestPutMessagingConfigSecretsMailgun: + @pytest.fixture(scope="function") + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config.key + ) + + @pytest.fixture(scope="function") + def payload(self): + return { + MessagingServiceSecrets.MAILGUN_API_KEY.value: "1345234524", + } + + def test_put_config_secrets_unauthenticated( + self, api_client: TestClient, payload, url + ): + response = api_client.put(url, headers={}, json=payload) + assert 401 == response.status_code + + def test_put_config_secrets_wrong_scope( + self, api_client: TestClient, payload, url, generate_auth_header + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 403 == response.status_code + + def test_put_config_secret_invalid_config( + self, api_client: TestClient, payload, generate_auth_header + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + url = (V1_URL_PREFIX + MESSAGING_SECRETS).format(config_key="invalid_key") + response = api_client.put(url, headers=auth_header, json=payload) + assert 404 == response.status_code + + def test_update_with_invalid_secrets_key( + self, api_client: TestClient, generate_auth_header, url + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json={"bad_key": "12345"}) + + assert response.status_code == 400 + assert response.json() == { + "detail": [ + "field required ('mailgun_api_key',)", + "extra fields not permitted ('bad_key',)", + ] + } + + def test_put_config_secrets( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + messaging_config, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_mailgun_messaging_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config.secrets[MessagingServiceSecrets.MAILGUN_API_KEY.value] + == "1345234524" + ) + + +class TestPutMessagingConfigSecretTwilioEmail: + @pytest.fixture(scope="function") + def url(self, messaging_config_twilio_email) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config_twilio_email.key + ) + + @pytest.fixture(scope="function") + def payload(self): + return {MessagingServiceSecrets.TWILIO_API_KEY.value: "23p48btcpy14b"} + + def test_put_config_secrets( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + messaging_config_twilio_email, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config_twilio_email) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_twilio_email_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_twilio_email.secrets[ + MessagingServiceSecrets.TWILIO_API_KEY.value + ] + == "23p48btcpy14b" + ) + + +class TestPutMessagingConfigSecretTwilioSms: + @pytest.fixture(scope="function") + def url(self, messaging_config_twilio_sms) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config_twilio_sms.key + ) + + def test_put_config_secrets_with_messaging_service_sid( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "234ct324", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "3rcuinhewrf", + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value: "asdfasdf", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config_twilio_sms) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_twilio_sms_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value + ] + == "234ct324" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value + ] + == "3rcuinhewrf" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] + == "asdfasdf" + ) + + def test_put_config_secrets_with_sender_phone( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23tc3", + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value: "+12345436543", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config_twilio_sms) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_twilio_sms_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value + ] + == "2asdf35tv5wsdf" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value + ] + == "23tc3" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value + ] + == "+12345436543" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] + is None + ) + + def test_put_config_secrets_with_sender_phone_incorrect_format( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23tc3", + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value: "12345436543", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert response.status_code == 400 + assert ( + f"Sender phone number must include country code, formatted like +15558675309 ('__root__',)" + in response.json()["detail"] + ) + + def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23tc3", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert response.status_code == 400 + assert ( + f"Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied. ('__root__',)" + in response.json()["detail"] + ) + + +class TestGetMessagingConfigs: + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + MESSAGING_CONFIG + + def test_get_configs_not_authenticated(self, api_client: TestClient, url) -> None: + response = api_client.get(url) + assert 401 == response.status_code + + def test_get_configs_wrong_scope( + self, api_client: TestClient, url, generate_auth_header + ) -> None: + auth_header = generate_auth_header([MESSAGING_DELETE]) + response = api_client.get(url, headers=auth_header) + assert 403 == response.status_code + + def test_get_configs( + self, db, api_client: TestClient, url, generate_auth_header, messaging_config + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.get(url, headers=auth_header) + assert 200 == response.status_code + + expected_response = { + "items": [ + { + "key": "my_mailgun_messaging_config", + "name": messaging_config.name, + "service_type": MessagingServiceType.MAILGUN.value, + "details": { + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, + }, + } + ], + "page": 1, + "size": PAGE_SIZE, + "total": 1, + } + response_body = json.loads(response.text) + assert expected_response == response_body + + +class TestGetMessagingConfig: + @pytest.fixture(scope="function") + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=messaging_config.key + ) + + def test_get_config_not_authenticated(self, url, api_client: TestClient): + response = api_client.get(url) + assert 401 == response.status_code + + def test_get_config_wrong_scope( + self, url, api_client: TestClient, generate_auth_header + ): + auth_header = generate_auth_header([MESSAGING_DELETE]) + response = api_client.get(url, headers=auth_header) + assert 403 == response.status_code + + def test_get_config_invalid( + self, api_client: TestClient, generate_auth_header, messaging_config + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.get( + (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key="invalid"), + headers=auth_header, + ) + assert 404 == response.status_code + + def test_get_config( + self, url, api_client: TestClient, generate_auth_header, messaging_config + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.get(url, headers=auth_header) + assert response.status_code == 200 + + response_body = json.loads(response.text) + + assert response_body == { + "key": "my_mailgun_messaging_config", + "name": messaging_config.name, + "service_type": MessagingServiceType.MAILGUN.value, + "details": { + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, + }, + } + + +class TestDeleteConfig: + @pytest.fixture(scope="function") + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=messaging_config.key + ) + + def test_delete_config_not_authenticated(self, url, api_client: TestClient): + response = api_client.delete(url) + assert 401 == response.status_code + + def test_delete_config_wrong_scope( + self, url, api_client: TestClient, generate_auth_header + ): + auth_header = generate_auth_header([MESSAGING_READ]) + response = api_client.delete(url, headers=auth_header) + assert 403 == response.status_code + + def test_delete_config_invalid(self, api_client: TestClient, generate_auth_header): + auth_header = generate_auth_header([MESSAGING_DELETE]) + response = api_client.delete( + (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key="invalid"), + headers=auth_header, + ) + assert 404 == response.status_code + + def test_delete_config( + self, + db: Session, + url, + api_client: TestClient, + generate_auth_header, + ): + # Creating new config, so we don't run into issues trying to clean up a deleted fixture + twilio_sms_config = MessagingConfig.create( + db=db, + data={ + "key": "my_twilio_sms_config", + "name": "twilio_sms", + "service_type": MessagingServiceType.TWILIO_TEXT, + }, + ) + url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=twilio_sms_config.key + ) + auth_header = generate_auth_header([MESSAGING_DELETE]) + response = api_client.delete(url, headers=auth_header) + assert response.status_code == 204 + + db.expunge_all() + config = db.query(MessagingConfig).filter_by(key=twilio_sms_config.key).first() + assert config is None diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index e7d31e07669..15ade528c3e 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -62,16 +62,17 @@ PrivacyRequestStatus, ) from fides.api.ops.schemas.dataset import DryRunDatasetResponse -from fides.api.ops.schemas.email.email import ( - EmailActionType, +from fides.api.ops.schemas.masking.masking_secrets import SecretType +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, + MessagingMethod, RequestReceiptBodyParams, RequestReviewDenyBodyParams, SubjectIdentityVerificationBodyParams, ) -from fides.api.ops.schemas.masking.masking_secrets import SecretType from fides.api.ops.schemas.policy import PolicyResponse from fides.api.ops.schemas.redis_cache import Identity -from fides.api.ops.tasks import EMAIL_QUEUE_NAME +from fides.api.ops.tasks import MESSAGING_QUEUE_NAME from fides.api.ops.util.cache import ( get_encryption_cache_key, get_identity_cache_key, @@ -96,11 +97,11 @@ def url(self, oauth_client: ClientDetail, policy) -> str: "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_create_privacy_request( self, - mock_dispatch_email, + mock_dispatch_message, run_access_request_mock, url, db, @@ -121,7 +122,7 @@ def test_create_privacy_request( pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) pr.delete(db=db) assert run_access_request_mock.called - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" @@ -1693,8 +1694,8 @@ def url(self, db, privacy_request): return V1_URL_PREFIX + PRIVACY_REQUEST_APPROVE @pytest.fixture(scope="function") - def privacy_request_review_email_notification_enabled(self): - """Enable request review email""" + def privacy_request_review_notification_enabled(self): + """Enable request review notification""" original_value = CONFIG.notifications.send_request_review_notification CONFIG.notifications.send_request_review_notification = True yield @@ -1801,11 +1802,11 @@ def test_approve_privacy_request_no_user_on_client( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_approve_privacy_request( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -1841,7 +1842,7 @@ def test_approve_privacy_request( assert response_body["succeeded"][0]["reviewed_by"] == user.id assert submit_mock.called - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called privacy_request.delete(db) @@ -1849,11 +1850,11 @@ def test_approve_privacy_request( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_approve_privacy_request_creates_audit_log_and_sends_email( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -1861,7 +1862,7 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( generate_auth_header, user, privacy_request_status_pending, - privacy_request_review_email_notification_enabled, + privacy_request_review_notification_enabled, ): payload = { JWE_PAYLOAD_SCOPES: user.client.scopes, @@ -1889,17 +1890,19 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( approval_audit_log.delete(db) - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] + message_meta = task_kwargs["message_meta"] assert ( - email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE + message_meta["action_type"] + == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE ) - assert email_meta["body_params"] is None + assert message_meta["body_params"] is None queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME class TestDenyPrivacyRequest: @@ -1908,8 +1911,8 @@ def url(self, db, privacy_request): return V1_URL_PREFIX + PRIVACY_REQUEST_DENY @pytest.fixture(autouse=True, scope="function") - def privacy_request_review_email_notification_enabled(self): - """Enable request review email""" + def privacy_request_review_notification_enabled(self): + """Enable request review notification""" original_value = CONFIG.notifications.send_request_review_notification CONFIG.notifications.send_request_review_notification = True yield @@ -1972,11 +1975,11 @@ def test_deny_completed_privacy_request( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_deny_privacy_request_without_denial_reason( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -2017,17 +2020,21 @@ def test_deny_privacy_request_without_denial_reason( ), ).first() - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY - assert email_meta["body_params"] == RequestReviewDenyBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] + == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY + ) + assert message_meta["body_params"] == RequestReviewDenyBodyParams( rejection_reason=None ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME assert denial_audit_log.message is None @@ -2039,11 +2046,11 @@ def test_deny_privacy_request_without_denial_reason( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_deny_privacy_request_with_denial_reason( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -2084,17 +2091,21 @@ def test_deny_privacy_request_with_denial_reason( ), ).first() - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY - assert email_meta["body_params"] == RequestReviewDenyBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] + == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY + ) + assert message_meta["body_params"] == RequestReviewDenyBodyParams( rejection_reason=denial_reason ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME assert denial_audit_log.message == denial_reason @@ -2849,8 +2860,8 @@ def url(self, db, privacy_request): ) @pytest.fixture(scope="function") - def privacy_request_receipt_email_notification_enabled(self): - """Enable request receipt email""" + def privacy_request_receipt_notification_enabled(self): + """Enable request receipt""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = True yield @@ -2866,16 +2877,16 @@ def test_incorrect_privacy_request_status(self, api_client, url, privacy_request ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verification_code_expired( self, - mock_dispatch_email, + mock_dispatch_message, db, api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -2887,19 +2898,19 @@ def test_verification_code_expired( resp.json()["detail"] == f"Identification code expired for {privacy_request.id}." ) - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_invalid_code( self, - mock_dispatch_email, + mock_dispatch_message, db, api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -2912,23 +2923,23 @@ def test_invalid_code( resp.json()["detail"] == f"Incorrect identification code for '{privacy_request.id}'" ) - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verify_identity_no_admin_approval_needed( self, - mock_dispatch_email, + mock_dispatch_message, mock_run_privacy_request, db, api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -2958,29 +2969,34 @@ def test_verify_identity_no_admin_approval_needed( assert mock_run_privacy_request.called - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity( + phone_number="+1 234 567 8910", email="test@example.com" + ) + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verify_identity_no_admin_approval_needed_email_disabled( self, - mock_dispatch_email, + mock_dispatch_message, mock_run_privacy_request, db, api_client, @@ -3015,24 +3031,24 @@ def test_verify_identity_no_admin_approval_needed_email_disabled( assert mock_run_privacy_request.called - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verify_identity_admin_approval_needed( self, - mock_dispatch_email, + mock_dispatch_message, mock_run_privacy_request, require_manual_request_approval, db, api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -3061,19 +3077,24 @@ def test_verify_identity_admin_approval_needed( assert approved_audit_log is None assert not mock_run_privacy_request.called - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity( + phone_number="+1 234 567 8910", email="test@example.com" + ) + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME class TestCreatePrivacyRequestEmailVerificationRequired: @@ -3121,16 +3142,16 @@ def test_create_privacy_request_no_email_config( @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) - @mock.patch("fides.api.ops.service._verification.dispatch_email") + @mock.patch("fides.api.ops.service._verification.dispatch_message") def test_create_privacy_request_with_email_config( self, - mock_dispatch_email, + mock_dispatch_message, mock_execute_request, url, db, api_client: TestClient, policy, - email_config, + messaging_config, subject_identity_verification_required, ): data = [ @@ -3157,11 +3178,14 @@ def test_create_privacy_request_with_email_config( assert response_data[0]["status"] == PrivacyRequestStatus.identity_unverified - assert mock_dispatch_email.called - kwargs = mock_dispatch_email.call_args.kwargs - assert kwargs["action_type"] == EmailActionType.SUBJECT_IDENTITY_VERIFICATION - assert kwargs["to_email"] == "test@example.com" - assert kwargs["email_body_params"] == SubjectIdentityVerificationBodyParams( + assert mock_dispatch_message.called + kwargs = mock_dispatch_message.call_args.kwargs + assert ( + kwargs["action_type"] == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION + ) + assert kwargs["to_identity"] == Identity(email="test@example.com") + assert kwargs["messaging_method"] == MessagingMethod.EMAIL + assert kwargs["message_body_params"] == SubjectIdentityVerificationBodyParams( verification_code=pr.get_cached_verification_code(), verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, ) @@ -3633,8 +3657,8 @@ def url(self, oauth_client: ClientDetail, policy) -> str: return V1_URL_PREFIX + PRIVACY_REQUESTS @pytest.fixture(scope="function") - def privacy_request_receipt_email_notification_enabled(self): - """Enable request receipt email""" + def privacy_request_receipt_notification_enabled(self): + """Enable request receipt notification""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = True yield @@ -3644,17 +3668,17 @@ def privacy_request_receipt_email_notification_enabled(self): "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_create_privacy_request_no_email_config( self, - mock_dispatch_email, + mock_dispatch_message, mock_execute_request, url, db, api_client: TestClient, policy, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): data = [ { @@ -3672,19 +3696,22 @@ def test_create_privacy_request_no_email_config( assert mock_execute_request.called assert response_data[0]["status"] == PrivacyRequestStatus.pending - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME pr.delete(db=db) @@ -3692,18 +3719,18 @@ def test_create_privacy_request_no_email_config( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_create_privacy_request_with_email_config( self, - mock_dispatch_email, + mock_dispatch_message, mock_execute_request, url, db, api_client: TestClient, policy, - email_config, - privacy_request_receipt_email_notification_enabled, + messaging_config, + privacy_request_receipt_notification_enabled, ): data = [ { @@ -3720,18 +3747,21 @@ def test_create_privacy_request_with_email_config( assert mock_execute_request.called assert response_data[0]["status"] == PrivacyRequestStatus.pending - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME pr.delete(db=db) diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 3fa163b9202..3e815684ce0 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -283,8 +283,8 @@ def privacy_request_complete_email_notification_disabled(): @pytest.fixture(autouse=True, scope="function") -def privacy_request_receipt_email_notification_disabled(): - """Disable request receipt email for most tests unless overridden""" +def privacy_request_receipt_notification_disabled(): + """Disable request receipt notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = False yield @@ -292,9 +292,18 @@ def privacy_request_receipt_email_notification_disabled(): @pytest.fixture(autouse=True, scope="function") -def privacy_request_review_email_notification_disabled(): - """Disable request review email for most tests unless overridden""" +def privacy_request_review_notification_disabled(): + """Disable request review notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_review_notification CONFIG.notifications.send_request_review_notification = False yield CONFIG.notifications.send_request_review_notification = original_value + + +@pytest.fixture(scope="function", autouse=True) +def set_notification_service_type_mailgun(): + """Set default notification service type""" + original_value = CONFIG.notifications.notification_service_type + CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value + yield + CONFIG.notifications.notification_service_type = original_value diff --git a/tests/ops/email_templates/test_get_email_template.py b/tests/ops/email_templates/test_get_email_template.py index 318d7a28aca..d9ea4c112fa 100644 --- a/tests/ops/email_templates/test_get_email_template.py +++ b/tests/ops/email_templates/test_get_email_template.py @@ -3,11 +3,11 @@ from fides.api.ops.common_exceptions import EmailTemplateUnhandledActionType from fides.api.ops.email_templates import get_email_template -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import MessagingActionType def test_get_email_template_returns_template(): - result = get_email_template(EmailActionType.SUBJECT_IDENTITY_VERIFICATION) + result = get_email_template(MessagingActionType.SUBJECT_IDENTITY_VERIFICATION) assert type(result) == Template diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 9ca5fc9f414..b50d04a4ea5 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -23,7 +23,7 @@ ConnectionType, ) from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import ( ActionType, Policy, @@ -35,10 +35,10 @@ from fides.api.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus from fides.api.ops.models.registration import UserRegistration from fides.api.ops.models.storage import ResponseFormat, StorageConfig -from fides.api.ops.schemas.email.email import ( - EmailServiceDetails, - EmailServiceSecrets, - EmailServiceType, +from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceDetails, + MessagingServiceSecrets, + MessagingServiceType, ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.schemas.storage.storage import ( @@ -153,26 +153,73 @@ def storage_config(db: Session) -> Generator: @pytest.fixture(scope="function") -def email_config(db: Session) -> Generator: +def messaging_config(db: Session) -> Generator: name = str(uuid4()) - email_config = EmailConfig.create( + messaging_config = MessagingConfig.create( db=db, data={ "name": name, - "key": "my_email_config", - "service_type": EmailServiceType.MAILGUN, + "key": "my_mailgun_messaging_config", + "service_type": MessagingServiceType.MAILGUN, "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "some.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, }, }, ) - email_config.set_secrets( - db=db, email_secrets={EmailServiceSecrets.MAILGUN_API_KEY.value: "12984r70298r"} + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.MAILGUN_API_KEY.value: "12984r70298r" + }, + ) + yield messaging_config + messaging_config.delete(db) + + +@pytest.fixture(scope="function") +def messaging_config_twilio_email(db: Session) -> Generator: + name = str(uuid4()) + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": name, + "key": "my_twilio_email_config", + "service_type": MessagingServiceType.TWILIO_EMAIL, + }, + ) + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.TWILIO_API_KEY.value: "123489ctynpiqurwfh" + }, + ) + yield messaging_config + messaging_config.delete(db) + + +@pytest.fixture(scope="function") +def messaging_config_twilio_sms(db: Session) -> Generator: + name = str(uuid4()) + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": name, + "key": "my_twilio_sms_config", + "service_type": MessagingServiceType.TWILIO_TEXT, + }, + ) + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "23rwrfwxwef", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23984y29384y598432", + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value: "2ieurnoqw", + }, ) - yield email_config - email_config.delete(db) + yield messaging_config + messaging_config.delete(db) @pytest.fixture(scope="function") diff --git a/tests/ops/integration_tests/test_integration_email.py b/tests/ops/integration_tests/test_integration_email.py index 70ebd7bd72e..6cb91b93d11 100644 --- a/tests/ops/integration_tests/test_integration_email.py +++ b/tests/ops/integration_tests/test_integration_email.py @@ -14,7 +14,11 @@ ManualAction, ) from fides.api.ops.schemas.dataset import FidesopsDataset -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, + MessagingMethod, +) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( email_connector_erasure_send, ) @@ -23,7 +27,7 @@ @pytest.mark.integration_postgres @pytest.mark.integration -@mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_email") +@mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_message") @pytest.mark.asyncio async def test_email_connector_cache_and_delayed_send( mock_email_dispatch, @@ -34,7 +38,7 @@ async def test_email_connector_cache_and_delayed_send( privacy_request, example_datasets, email_dataset_config, - email_config, + messaging_config, ) -> None: """Run an erasure privacy request with a postgres dataset and an email dataset. The email dataset has three separate collections. @@ -178,9 +182,13 @@ async def test_email_connector_cache_and_delayed_send( email_connector_erasure_send(db, privacy_request) assert mock_email_dispatch.called call_args = mock_email_dispatch.call_args[1] - assert call_args["action_type"] == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT - assert call_args["to_email"] == "test@example.com" - assert call_args["email_body_params"] == raw_email_template_values + assert ( + call_args["action_type"] + == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT + ) + assert call_args["to_identity"] == Identity(email="test@example.com") + assert call_args["messaging_method"] == MessagingMethod.EMAIL + assert call_args["message_body_params"] == raw_email_template_values created_email_audit_log = ( db.query(AuditLog) diff --git a/tests/ops/schemas/email/email_test.py b/tests/ops/schemas/messaging/messaging_test.py similarity index 77% rename from tests/ops/schemas/email/email_test.py rename to tests/ops/schemas/messaging/messaging_test.py index e3caea5f3bc..65c31b2ea2f 100644 --- a/tests/ops/schemas/email/email_test.py +++ b/tests/ops/schemas/messaging/messaging_test.py @@ -1,6 +1,8 @@ import pytest -from fides.api.ops.schemas.email.email import SubjectIdentityVerificationBodyParams +from fides.api.ops.schemas.messaging.messaging import ( + SubjectIdentityVerificationBodyParams, +) @pytest.mark.parametrize("ttl, expected", [(600, 10), (155, 2), (33, 0)]) diff --git a/tests/ops/service/email/email_dispatch_service_test.py b/tests/ops/service/email/email_dispatch_service_test.py deleted file mode 100644 index 0b0c86c8dc7..00000000000 --- a/tests/ops/service/email/email_dispatch_service_test.py +++ /dev/null @@ -1,159 +0,0 @@ -from unittest import mock -from unittest.mock import Mock - -import pytest -import requests.exceptions -import requests_mock -from sqlalchemy.orm import Session - -from fides.api.ops.common_exceptions import EmailDispatchException -from fides.api.ops.graph.config import CollectionAddress -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.models.policy import CurrentStep -from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction -from fides.api.ops.schemas.email.email import ( - EmailActionType, - EmailForActionType, - EmailServiceDetails, - EmailServiceType, - FidesopsEmail, - SubjectIdentityVerificationBodyParams, -) -from fides.api.ops.service.email.email_dispatch_service import dispatch_email - - -@mock.patch("fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher") -def test_email_dispatch_mailgun_success( - mock_mailgun_dispatcher: Mock, db: Session, email_config -) -> None: - - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - body = '\n\n\n \n ID Code\n\n\n
\n

\n Your privacy request verification code is 2348.\n Please return to the Privacy Center and enter the code to\n continue. This code will expire in 10 minutes\n

\n
\n\n' - mock_mailgun_dispatcher.assert_called_with( - email_config=email_config, - email=EmailForActionType( - subject="Your one-time code", - body=body, - ), - to_email="test@email.com", - ) - - -@mock.patch("fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher") -def test_email_dispatch_mailgun_config_not_found( - mock_mailgun_dispatcher: Mock, db: Session -) -> None: - - with pytest.raises(EmailDispatchException) as exc: - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - assert exc.value.args[0] == "No email config found." - - mock_mailgun_dispatcher.assert_not_called() - - -@mock.patch("fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher") -def test_email_dispatch_mailgun_config_no_secrets( - mock_mailgun_dispatcher: Mock, db: Session -) -> None: - - email_config = EmailConfig.create( - db=db, - data={ - "name": "mailgun config", - "key": "my_email_config", - "service_type": EmailServiceType.MAILGUN, - "details": { - EmailServiceDetails.DOMAIN.value: "some.domain", - }, - }, - ) - - with pytest.raises(EmailDispatchException) as exc: - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - assert ( - exc.value.args[0] - == "Email secrets not found for config with key: my_email_config" - ) - - mock_mailgun_dispatcher.assert_not_called() - - email_config.delete(db) - - -def test_email_dispatch_mailgun_failed_email(db: Session, email_config) -> None: - with requests_mock.Mocker() as mock_response: - mock_response.post( - f"https://api.mailgun.net/{email_config.details[EmailServiceDetails.API_VERSION.value]}/{email_config.details[EmailServiceDetails.DOMAIN.value]}/messages", - json={ - "message": "Rejected: IP can’t be used to send the message", - "id": "<20111114174239.25659.5817@samples.mailgun.org>", - }, - status_code=403, - ) - with pytest.raises(EmailDispatchException) as exc: - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - assert ( - exc.value.args[0] - == "Email failed to send due to: Email failed to send with status code 403" - ) - - -def test_fidesops_email_parse_object(): - body = [ - CheckpointActionRequired( - step=CurrentStep.erasure, - collection=CollectionAddress("email_dataset", "test_collection"), - action_needed=[ - ManualAction( - locators={"email": "test@example.com"}, - get=None, - update={"phone": "null_rewrite"}, - ) - ], - ) - ] - - FidesopsEmail.parse_obj( - { - "action_type": EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT, - "body_params": [action.dict() for action in body], - } - ) - - FidesopsEmail.parse_obj( - { - "action_type": EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - "body_params": { - "verification_code": "123456", - "verification_code_ttl_seconds": 1000, - }, - } - ) diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py new file mode 100644 index 00000000000..f3b9e85d30d --- /dev/null +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -0,0 +1,174 @@ +from unittest import mock +from unittest.mock import Mock + +import pytest +import requests_mock +from sqlalchemy.orm import Session + +from fides.api.ops.common_exceptions import MessageDispatchException +from fides.api.ops.graph.config import CollectionAddress +from fides.api.ops.models.messaging import MessagingConfig, get_messaging_method +from fides.api.ops.models.policy import CurrentStep +from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction +from fides.api.ops.schemas.messaging.messaging import ( + EmailForActionType, + FidesopsMessage, + MessagingActionType, + MessagingServiceDetails, + MessagingServiceType, + SubjectIdentityVerificationBodyParams, +) +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message +from fides.ctl.core.config import get_config + +CONFIG = get_config() + + +@mock.patch( + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" +) +def test_email_dispatch_mailgun_success( + mock_mailgun_dispatcher: Mock, db: Session, messaging_config +) -> None: + + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=get_messaging_method(MessagingServiceType.MAILGUN.value), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + body = '\n\n\n \n ID Code\n\n\n
\n

\n Your privacy request verification code is 2348.\n Please return to the Privacy Center and enter the code to\n continue. This code will expire in 10 minutes\n

\n
\n\n' + mock_mailgun_dispatcher.assert_called_with( + messaging_config, + EmailForActionType( + subject="Your one-time code", + body=body, + ), + "test@email.com", + ) + + +@mock.patch( + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" +) +def test_email_dispatch_mailgun_config_not_found( + mock_mailgun_dispatcher: Mock, db: Session +) -> None: + + with pytest.raises(MessageDispatchException) as exc: + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=get_messaging_method(MessagingServiceType.MAILGUN.value), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + assert exc.value.args[0] == "No messaging config found." + + mock_mailgun_dispatcher.assert_not_called() + + +@mock.patch( + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" +) +def test_email_dispatch_mailgun_config_no_secrets( + mock_mailgun_dispatcher: Mock, db: Session +) -> None: + + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": "mailgun config", + "key": "my_mailgun_messaging_config", + "service_type": MessagingServiceType.MAILGUN, + "details": { + MessagingServiceDetails.DOMAIN.value: "some.domain", + }, + }, + ) + + with pytest.raises(MessageDispatchException) as exc: + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=get_messaging_method(MessagingServiceType.MAILGUN.value), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + assert ( + exc.value.args[0] + == "Messaging secrets not found for config with key: my_mailgun_messaging_config" + ) + + mock_mailgun_dispatcher.assert_not_called() + + messaging_config.delete(db) + + +def test_email_dispatch_mailgun_failed_email(db: Session, messaging_config) -> None: + with requests_mock.Mocker() as mock_response: + mock_response.post( + f"https://api.mailgun.net/{messaging_config.details[MessagingServiceDetails.API_VERSION.value]}/{messaging_config.details[MessagingServiceDetails.DOMAIN.value]}/messages", + json={ + "message": "Rejected: IP can’t be used to send the message", + "id": "<20111114174239.25659.5817@samples.mailgun.org>", + }, + status_code=403, + ) + with pytest.raises(MessageDispatchException) as exc: + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=get_messaging_method( + MessagingServiceType.MAILGUN.value + ), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + assert ( + exc.value.args[0] + == "Email failed to send due to: Email failed to send with status code 403" + ) + + +def test_fidesops_email_parse_object(): + body = [ + CheckpointActionRequired( + step=CurrentStep.erasure, + collection=CollectionAddress("email_dataset", "test_collection"), + action_needed=[ + ManualAction( + locators={"email": "test@example.com"}, + get=None, + update={"phone": "null_rewrite"}, + ) + ], + ) + ] + + FidesopsMessage.parse_obj( + { + "action_type": MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, + "body_params": [action.dict() for action in body], + } + ) + + FidesopsMessage.parse_obj( + { + "action_type": MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + "body_params": { + "verification_code": "123456", + "verification_code_ttl_seconds": 1000, + }, + } + ) diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 236ea6ff13f..1c06c19be21 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -16,7 +16,7 @@ PrivacyRequestPaused, ) from fides.api.ops.models.connectionconfig import AccessLevel -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import CurrentStep, PolicyPostWebhook from fides.api.ops.models.privacy_request import ( ActionType, @@ -26,18 +26,20 @@ PrivacyRequest, PrivacyRequestStatus, ) -from fides.api.ops.schemas.email.email import ( - AccessRequestCompleteBodyParams, - EmailActionType, - EmailForActionType, -) from fides.api.ops.schemas.external_https import SecondPartyResponseFormat from fides.api.ops.schemas.masking.masking_configuration import ( HmacMaskingConfiguration, MaskingConfiguration, ) from fides.api.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fides.api.ops.schemas.messaging.messaging import ( + AccessRequestCompleteBodyParams, + EmailForActionType, + MessagingActionType, + MessagingMethod, +) from fides.api.ops.schemas.policy import Rule +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.schemas.saas.saas_config import SaaSRequest from fides.api.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fides.api.ops.service.connectors.saas_connector import SaaSConnector @@ -71,10 +73,10 @@ def privacy_request_complete_email_notification_enabled(): @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") -def test_policy_upload_dispatch_email_called( +def test_policy_upload_dispatch_message_called( upload_mock: Mock, mock_email_dispatch: Mock, privacy_request_status_pending: PrivacyRequest, @@ -90,7 +92,7 @@ def test_policy_upload_dispatch_email_called( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_start_processing_sets_started_processing_at( @@ -116,7 +118,7 @@ def test_start_processing_sets_started_processing_at( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_start_processing_doesnt_overwrite_started_processing_at( @@ -167,7 +169,7 @@ def test_halts_proceeding_if_cancelled( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.upload_access_results" @@ -217,7 +219,7 @@ def test_from_graph_resume_does_not_run_pre_webhooks( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", @@ -1637,7 +1639,7 @@ def test_privacy_request_log_failure( class TestPrivacyRequestsEmailConnector: @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_create_and_process_erasure_request_email_connector( @@ -1649,7 +1651,7 @@ def test_create_and_process_erasure_request_email_connector( run_privacy_request_task, email_dataset_config, postgres_example_test_dataset_config_read_access, - email_config, + messaging_config, db, ): """ @@ -1675,12 +1677,12 @@ def test_create_and_process_erasure_request_email_connector( ) pr.delete(db=db) assert mailgun_send.called - kwargs = mailgun_send.call_args.kwargs - assert type(kwargs["email_config"]) == EmailConfig - assert type(kwargs["email"]) == EmailForActionType + args = mailgun_send.call_args.args + assert type(args[0]) == MessagingConfig + assert type(args[1]) == EmailForActionType @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_create_and_process_erasure_request_email_connector_email_send_error( @@ -1695,7 +1697,7 @@ def test_create_and_process_erasure_request_email_connector_email_send_error( db, ): """ - Force error by having no email config setup + Force error by having no messaging config setup """ rule = erasure_policy.rules[0] target = rule.targets[0] @@ -1734,7 +1736,7 @@ def test_create_and_process_erasure_request_email_connector_email_send_error( assert mailgun_send.called is False @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_email_connector_read_only_permissions( @@ -1745,12 +1747,12 @@ def test_email_connector_read_only_permissions( integration_postgres_config, run_privacy_request_task, email_dataset_config, - email_config, + messaging_config, postgres_example_test_dataset_config_read_access, db, ): """ - Set email config to read only - don't send email in this case. + Set messaging config to read only - don't send message in this case. """ rule = erasure_policy.rules[0] target = rule.targets[0] @@ -1788,7 +1790,7 @@ def test_email_connector_read_only_permissions( ), "Email not sent because the connection was read only" @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_email_connector_no_updates_needed( @@ -1799,7 +1801,7 @@ def test_email_connector_no_updates_needed( integration_postgres_config, run_privacy_request_task, email_dataset_config, - email_config, + messaging_config, postgres_example_test_dataset_config_read_access, db, ): @@ -1855,7 +1857,7 @@ def privacy_request_complete_email_notification_enabled(self): @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) def test_email_complete_send_erasure( self, @@ -1867,7 +1869,7 @@ def test_email_complete_send_erasure( generate_auth_header, erasure_policy, read_connection_config, - email_config, + messaging_config, privacy_request_complete_email_notification_enabled, run_privacy_request_task, ): @@ -1891,7 +1893,7 @@ def test_email_complete_send_erasure( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_email_complete_send_access( @@ -1905,7 +1907,7 @@ def test_email_complete_send_access( generate_auth_header, policy, read_connection_config, - email_config, + messaging_config, privacy_request_complete_email_notification_enabled, run_privacy_request_task, ): @@ -1930,7 +1932,7 @@ def test_email_complete_send_access( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_email_complete_send_access_and_erasure( @@ -1944,7 +1946,7 @@ def test_email_complete_send_access_and_erasure( generate_auth_header, access_and_erasure_policy, read_connection_config, - email_config, + messaging_config, privacy_request_complete_email_notification_enabled, run_privacy_request_task, ): @@ -1963,22 +1965,25 @@ def test_email_complete_send_access_and_erasure( data, ) pr.delete(db=db) + identity = Identity(email=customer_email) mailgun_send.assert_has_calls( [ call( db=ANY, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, - to_email=customer_email, - email_body_params=AccessRequestCompleteBodyParams( + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, + to_identity=identity, + messaging_method=MessagingMethod.EMAIL, + message_body_params=AccessRequestCompleteBodyParams( download_links=[upload_mock.return_value] ), ), call( db=ANY, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION, - to_email=customer_email, - email_body_params=None, + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, + to_identity=identity, + messaging_method=MessagingMethod.EMAIL, + message_body_params=None, ), ], any_order=True, @@ -1987,10 +1992,10 @@ def test_email_complete_send_access_and_erasure( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") - def test_email_complete_send_access_no_email_config( + def test_email_complete_send_access_no_messaging_config( self, upload_mock, mailgun_send, @@ -2027,7 +2032,7 @@ def test_email_complete_send_access_no_email_config( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_email_complete_send_access_no_email_identity(