From f5bd39edf4fb3d7295edb6d465284d2abf0d4f70 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Fri, 21 Oct 2022 14:45:27 -0500 Subject: [PATCH 01/28] Adds config for notification service type, starts work to genericize email dispatcher --- .../postman/Fides.postman_collection.json | 61 +++-- .../fides/docs/guides/email_communications.md | 14 +- docs/fides/docs/guides/messaging.md | 59 ++++ scripts/setup/email.py | 18 +- .../2d5ff3096959_update_table_for_twilio.py | 64 +++++ src/fides/api/ops/api/v1/api.py | 2 +- .../v1/endpoints/consent_request_endpoints.py | 10 +- .../ops/api/v1/endpoints/email_endpoints.py | 230 ---------------- .../identity_verification_endpoints.py | 6 +- .../api/v1/endpoints/messaging_endpoints.py | 226 +++++++++++++++ .../v1/endpoints/privacy_request_endpoints.py | 91 +++--- src/fides/api/ops/api/v1/scope_registry.py | 12 +- src/fides/api/ops/api/v1/urn_registry.py | 6 +- src/fides/api/ops/common_exceptions.py | 12 +- src/fides/api/ops/db/base.py | 2 +- .../ops/email_templates/get_email_template.py | 20 +- .../api/ops/email_templates/template_names.py | 2 +- ...=> message_request_email_fulfillment.html} | 0 .../api/ops/models/{email.py => messaging.py} | 49 ++-- .../schemas/email/email_secrets_docs_only.py | 11 - .../schemas/{email => messaging}/__init__.py | 0 .../email.py => messaging/messaging.py} | 117 +++++--- .../messaging/messaging_secrets_docs_only.py | 20 ++ src/fides/api/ops/service/_verification.py | 32 ++- .../ops/service/connectors/email_connector.py | 30 +- .../ops/service/email/email_crud_service.py | 74 ----- .../service/email/email_dispatch_service.py | 208 -------------- .../service/{email => messaging}/__init__.py | 0 .../messaging/message_dispatch_service.py | 259 ++++++++++++++++++ .../messaging/messaging_crud_service.py | 73 +++++ .../privacy_request/request_runner_service.py | 35 ++- src/fides/api/ops/tasks/__init__.py | 4 +- .../ctl/core/config/notification_settings.py | 16 ++ .../test_connection_config_endpoints.py | 9 +- .../test_identity_verification_endpoints.py | 4 +- ...dpoints.py => test_messaging_endpoints.py} | 212 +++++++------- .../test_privacy_request_endpoints.py | 113 ++++---- .../test_get_email_template.py | 4 +- tests/ops/fixtures/application_fixtures.py | 35 +-- .../test_integration_email.py | 11 +- .../messaging_test.py} | 4 +- .../email/email_dispatch_service_test.py | 159 ----------- .../message_dispatch_service_test.py | 173 ++++++++++++ .../request_runner_service_test.py | 54 ++-- 44 files changed, 1414 insertions(+), 1127 deletions(-) create mode 100644 docs/fides/docs/guides/messaging.md create mode 100644 src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py delete mode 100644 src/fides/api/ops/api/v1/endpoints/email_endpoints.py create mode 100644 src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py rename src/fides/api/ops/email_templates/templates/{erasure_request_email_fulfillment.html => message_request_email_fulfillment.html} (100%) rename src/fides/api/ops/models/{email.py => messaging.py} (68%) delete mode 100644 src/fides/api/ops/schemas/email/email_secrets_docs_only.py rename src/fides/api/ops/schemas/{email => messaging}/__init__.py (100%) rename src/fides/api/ops/schemas/{email/email.py => messaging/messaging.py} (55%) create mode 100644 src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py delete mode 100644 src/fides/api/ops/service/email/email_crud_service.py delete mode 100644 src/fides/api/ops/service/email/email_dispatch_service.py rename src/fides/api/ops/service/{email => messaging}/__init__.py (100%) create mode 100644 src/fides/api/ops/service/messaging/message_dispatch_service.py create mode 100644 src/fides/api/ops/service/messaging/messaging_crud_service.py rename tests/ops/api/v1/endpoints/{test_email_endpoints.py => test_messaging_endpoints.py} (63%) rename tests/ops/schemas/{email/email_test.py => messaging/messaging_test.py} (77%) delete mode 100644 tests/ops/service/email/email_dispatch_service_test.py create mode 100644 tests/ops/service/messaging/message_dispatch_service_test.py diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 148e16553bd..371bd788da3 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -1,8 +1,9 @@ { "info": { - "_postman_id": "002813f4-12b7-4467-9377-a57706b6dbc8", + "_postman_id": "652de86a-b40e-4711-b488-4b26252683ab", "name": "Fidesops", - "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json" + "schema": "https://schema.getpostman.com/json/collection/v2.1.0/collection.json", + "_exporter_id": "13396647" }, "item": [ { @@ -4104,10 +4105,10 @@ ] }, { - "name": "Primary Email Config", + "name": "Primary Messaging Config", "item": [ { - "name": "Post Email Config", + "name": "Post Messaging Config", "request": { "auth": { "type": "bearer", @@ -4123,7 +4124,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"key\": \"{{email_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", + "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", "options": { "raw": { "language": "json" @@ -4131,12 +4132,12 @@ } }, "url": { - "raw": "{{host}}/email/config/", + "raw": "{{host}}/messaging/config/", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", "" ] @@ -4145,7 +4146,7 @@ "response": [] }, { - "name": "Patch Email Config By Key", + "name": "Patch Messaging Config By Key", "request": { "auth": { "type": "bearer", @@ -4161,7 +4162,7 @@ "header": [], "body": { "mode": "raw", - "raw": "{\n \"key\": \"{{email_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", + "raw": "{\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n}\n\n", "options": { "raw": { "language": "json" @@ -4169,21 +4170,21 @@ } }, "url": { - "raw": "{{host}}/email/config/{{email_config_key}}", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}" + "{{mailgun_config_key}}" ] } }, "response": [] }, { - "name": "Get Email Configs", + "name": "Get Messaging Configs", "request": { "auth": { "type": "bearer", @@ -4198,12 +4199,12 @@ "method": "GET", "header": [], "url": { - "raw": "{{host}}/email/config/", + "raw": "{{host}}/messaging/config/", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", "" ] @@ -4212,7 +4213,7 @@ "response": [] }, { - "name": "Get Email Config By Key", + "name": "Get Messaging Config By Key", "request": { "auth": { "type": "bearer", @@ -4227,21 +4228,21 @@ "method": "GET", "header": [], "url": { - "raw": "{{host}}/email/config/{{email_config_key}}", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}" + "{{mailgun_config_key}}" ] } }, "response": [] }, { - "name": "Delete Email Config By Key", + "name": "Delete Messaging Config By Key", "request": { "auth": { "type": "bearer", @@ -4257,7 +4258,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n {\n \"key\": \"{{email_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n }\n]\n", + "raw": "[\n {\n \"key\": \"{{mailgun_config_key}}\",\n \"name\": \"mailgun\",\n \"service_type\": \"mailgun\",\n \"details\": {\n \"domain\": \"{{mailgun_domain}}\"\n }\n }\n]\n", "options": { "raw": { "language": "json" @@ -4265,14 +4266,14 @@ } }, "url": { - "raw": "{{host}}/email/config/{{email_config_key}}", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}" + "{{mailgun_config_key}}" ] } }, @@ -4303,14 +4304,14 @@ } }, "url": { - "raw": "{{host}}/email/config/{{email_config_key}}/secret", + "raw": "{{host}}/messaging/config/{{mailgun_config_key}}/secret", "host": [ "{{host}}" ], "path": [ - "email", + "messaging", "config", - "{{email_config_key}}", + "{{mailgun_config_key}}", "secret" ] } @@ -4732,8 +4733,8 @@ "value": "manual_key" }, { - "key": "email_config_key", - "value": "my_email_config", + "key": "mailgun_config_key", + "value": "my_mailgun_config", "type": "string" }, { @@ -4787,4 +4788,4 @@ "type": "string" } ] -} +} \ No newline at end of file diff --git a/docs/fides/docs/guides/email_communications.md b/docs/fides/docs/guides/email_communications.md index 54350f11695..97fda6d052e 100644 --- a/docs/fides/docs/guides/email_communications.md +++ b/docs/fides/docs/guides/email_communications.md @@ -21,11 +21,11 @@ Fides currently supports Mailgun for email integrations. Ensure you register or ## Configuration -### Create the email configuration +### Create the messaging configuration -```json title="POST api/v1/email/config" +```json title="POST api/v1/messaging/config" { - "key": "{{email_config_key}}", + "key": "{{messaging_config_key}}", "name": "mailgun", "service_type": "mailgun", "details": { @@ -36,8 +36,8 @@ Fides currently supports Mailgun for email integrations. Ensure you register or | Field | Description | |----|----| -| `key` | *Optional.* A unique key used to manage your email config. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`. | -| `name` | A unique user-friendly name for your email config. | +| `key` | *Optional.* A unique key used to manage your messaging config. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`. | +| `name` | A unique user-friendly name for your messaging config. | | `service_type` | The email service to configure. Currently, Fides supports `mailgun`. | | `details` | A dict of key/val config vars specific to Mailgun. | | `domain` | Your unique Mailgun domain. | @@ -45,9 +45,9 @@ Fides currently supports Mailgun for email integrations. Ensure you register or | `api_version` | *Optional.* A string that denotes the API version. Defaults to `v3`. | -### Add the email configuration secrets +### Add the messaging configuration secrets -```json title="POST api/v1/email/config/{{email_config_key}}/secret" +```json title="POST api/v1/messaging/config/{{messaging_config_key}}/secret" { "mailgun_api_key": "nc123849ycnpq98fnu" } diff --git a/docs/fides/docs/guides/messaging.md b/docs/fides/docs/guides/messaging.md new file mode 100644 index 00000000000..21471cb6fb4 --- /dev/null +++ b/docs/fides/docs/guides/messaging.md @@ -0,0 +1,59 @@ +# Configure Email/SMS Messaging +## What is email/SMS used for? + +Fides supports email and SMS server configurations for sending processing notices to privacy request subjects. Future updates will support outbound email communications with data processors. + +Supported modes of use: + +- Subject Identity Verification - for more information on identity verification in subject requests, see the [Privacy Requests](../getting-started/privacy_requests.md#subject-identity-verification) guide. + + +## Prerequisites + +Fides currently supports Mailgun for email integrations. Ensure you register or use an existing Mailgun account in order to get up and running with email communications. + +1. Generate a Mailgun Domain Sending Key + + Follow the [Mailgun documentation](https://documentation.mailgun.com/en/latest/api-intro.html#authentication-1) to create a new Domain Sending Key for Fides. + + !!! Note + Mailgun automatically generates a **primary account API key** when you sign up for an account. This key allows you to perform all CRUD operations via Mailgun's API endpoints, and for any of your sending domains. For security purposes, using a new **domain sending key** is recommended over your primary API key. + +## Configuration + +### Create the messaging configuration + +```json title="POST api/v1/messaging/config" +{ + "key": "{{messaging_config_key}}", + "name": "mailgun", + "service_type": "mailgun", + "details": { + "domain": "your.mailgun.domain" + } +} +``` + +| Field | Description | +|----|----| +| `key` | *Optional.* A unique key used to manage your messaging config. This is auto-generated from `name` if left blank. Accepted values are alphanumeric, `_`, and `.`. | +| `name` | A unique user-friendly name for your messaging config. | +| `service_type` | The email service to configure. Currently, Fides supports `mailgun`. | +| `details` | A dict of key/val config vars specific to Mailgun. | +| `domain` | Your unique Mailgun domain. | +| `is_eu_domain` | *Optional.* A boolean that denotes whether your Mailgun domain was created in the EU region. Defaults to `False`. | +| `api_version` | *Optional.* A string that denotes the API version. Defaults to `v3`. | + + +### Add the messaging configuration secrets + +```json title="POST api/v1/messaging/config/{{messaging_config_key}}/secret" +{ + "mailgun_api_key": "nc123849ycnpq98fnu" +} + +``` + +| Field | Description | +|---|----| +| `mailgun_api_key` | Your Mailgun Domain Sending Key. | diff --git a/scripts/setup/email.py b/scripts/setup/email.py index 99e3df9b938..f15f538db68 100644 --- a/scripts/setup/email.py +++ b/scripts/setup/email.py @@ -17,7 +17,7 @@ def create_email_integration( key: str = "fides_email", ): response = requests.post( - f"{constants.BASE_URL}{urls.EMAIL_CONFIG}", + f"{constants.BASE_URL}{urls.MESSAGING_CONFIG}", headers=auth_header, json={ "name": "fides Emails", @@ -32,20 +32,12 @@ def create_email_integration( ) if not response.ok: - if ( - response.json()["detail"] - != f"Only one email config is supported at a time. Config with key {key} is already configured." - ): - raise RuntimeError( - f"fides email config creation failed! response.status_code={response.status_code}, response.json()={response.json()}" - ) - logger.info( - f"fides email config is already created. Using the existing config." + raise RuntimeError( + f"fides messaging config creation failed! response.status_code={response.status_code}, response.json()={response.json()}" ) - return # Now add secrets - email_secrets_path = urls.EMAIL_SECRETS.format(config_key=key) + email_secrets_path = urls.MESSAGING_SECRETS.format(config_key=key) response = requests.put( f"{constants.BASE_URL}{email_secrets_path}", headers=auth_header, @@ -56,7 +48,7 @@ def create_email_integration( if not response.ok: raise RuntimeError( - f"fides email config secrets update failed! response.status_code={response.status_code}, response.json()={response.json()}" + f"fides messaging config secrets update failed! response.status_code={response.status_code}, response.json()={response.json()}" ) logger.info(response.json()["msg"]) diff --git a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py new file mode 100644 index 00000000000..cf43bffd70d --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py @@ -0,0 +1,64 @@ +"""Update table for twilio +Revision ID: 2d5ff3096959 +Revises: fb6b0150d6e4 +Create Date: 2022-10-21 22:10:48.899562 +""" +from alembic import op +import sqlalchemy as sa +from sqlalchemy.dialects import postgresql + +# revision identifiers, used by Alembic. +revision = '2d5ff3096959' +down_revision = 'fb6b0150d6e4' +branch_labels = None +depends_on = None + + +def upgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('messagingconfig', + sa.Column('id', sa.String(length=255), nullable=False), + sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), + sa.Column('key', sa.String(), nullable=False), + sa.Column('name', sa.String(), nullable=True), + sa.Column('service_type', sa.Enum('MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL', name='messagingservicetype'), nullable=False), + sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('secrets', sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), nullable=True), + sa.PrimaryKeyConstraint('id') + ) + op.create_index(op.f('ix_messagingconfig_id'), 'messagingconfig', ['id'], unique=False) + op.create_index(op.f('ix_messagingconfig_key'), 'messagingconfig', ['key'], unique=True) + op.create_index(op.f('ix_messagingconfig_name'), 'messagingconfig', ['name'], unique=True) + op.create_index(op.f('ix_messagingconfig_service_type'), 'messagingconfig', ['service_type'], unique=True) + op.drop_index('ix_emailconfig_id', table_name='emailconfig') + op.drop_index('ix_emailconfig_key', table_name='emailconfig') + op.drop_index('ix_emailconfig_name', table_name='emailconfig') + op.drop_index('ix_emailconfig_service_type', table_name='emailconfig') + op.drop_table('emailconfig') + # ### end Alembic commands ### + + +def downgrade(): + # ### commands auto generated by Alembic - please adjust! ### + op.create_table('emailconfig', + sa.Column('id', sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=True), + sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=True), + sa.Column('key', sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column('name', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column('service_type', postgresql.ENUM('MAILGUN', name='emailservicetype'), autoincrement=False, nullable=False), + sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False), + sa.Column('secrets', sa.VARCHAR(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint('id', name='emailconfig_pkey') + ) + op.create_index('ix_emailconfig_service_type', 'emailconfig', ['service_type'], unique=False) + op.create_index('ix_emailconfig_name', 'emailconfig', ['name'], unique=False) + op.create_index('ix_emailconfig_key', 'emailconfig', ['key'], unique=False) + op.create_index('ix_emailconfig_id', 'emailconfig', ['id'], unique=False) + op.drop_index(op.f('ix_messagingconfig_service_type'), table_name='messagingconfig') + op.drop_index(op.f('ix_messagingconfig_name'), table_name='messagingconfig') + op.drop_index(op.f('ix_messagingconfig_key'), table_name='messagingconfig') + op.drop_index(op.f('ix_messagingconfig_id'), table_name='messagingconfig') + op.drop_table('messagingconfig') + # ### end Alembic commands ### \ No newline at end of file diff --git a/src/fides/api/ops/api/v1/api.py b/src/fides/api/ops/api/v1/api.py index 7a3a83dcdac..99c76ff90be 100644 --- a/src/fides/api/ops/api/v1/api.py +++ b/src/fides/api/ops/api/v1/api.py @@ -5,7 +5,7 @@ consent_request_endpoints, dataset_endpoints, drp_endpoints, - email_endpoints, + messaging_endpoints, encryption_endpoints, identity_verification_endpoints, manual_webhook_endpoints, diff --git a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py index 1946eabba9e..ccf5a2a8a7a 100644 --- a/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/consent_request_endpoints.py @@ -24,9 +24,9 @@ V1_URL_PREFIX, ) from fides.api.ops.common_exceptions import ( - EmailDispatchException, FunctionalityNotConfigured, IdentityVerificationException, + MessageDispatchException, ) from fides.api.ops.models.privacy_request import ( Consent, @@ -98,12 +98,12 @@ def create_consent_request( if CONFIG.execution.subject_identity_verification_required: try: - send_verification_code_to_user(db, consent_request, data.email) - except EmailDispatchException as exc: - logger.error("Error sending the verification code email: %s", str(exc)) + send_verification_code_to_user(db, consent_request, data) + except MessageDispatchException as exc: + logger.error("Error sending the verification code message: %s", str(exc)) raise HTTPException( status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Error sending the verification code email: {str(exc)}", + detail=f"Error sending the verification code message: {str(exc)}", ) return ConsentRequestResponse( identity=data, diff --git a/src/fides/api/ops/api/v1/endpoints/email_endpoints.py b/src/fides/api/ops/api/v1/endpoints/email_endpoints.py deleted file mode 100644 index e375c6ad294..00000000000 --- a/src/fides/api/ops/api/v1/endpoints/email_endpoints.py +++ /dev/null @@ -1,230 +0,0 @@ -import logging -from typing import Optional - -from fastapi import Depends, Security -from fastapi_pagination import Page, Params -from fastapi_pagination.bases import AbstractPage -from fastapi_pagination.ext.sqlalchemy import paginate -from sqlalchemy.orm import Session -from starlette.exceptions import HTTPException -from starlette.status import ( - HTTP_200_OK, - HTTP_204_NO_CONTENT, - HTTP_400_BAD_REQUEST, - HTTP_404_NOT_FOUND, - HTTP_422_UNPROCESSABLE_ENTITY, - HTTP_500_INTERNAL_SERVER_ERROR, -) - -from fides.api.ops.api import deps -from fides.api.ops.api.v1.scope_registry import ( - EMAIL_CREATE_OR_UPDATE, - EMAIL_DELETE, - EMAIL_READ, -) -from fides.api.ops.api.v1.urn_registry import ( - EMAIL_BY_KEY, - EMAIL_CONFIG, - EMAIL_SECRETS, - V1_URL_PREFIX, -) -from fides.api.ops.common_exceptions import ( - EmailConfigAlreadyExistsException, - EmailConfigNotFoundException, -) -from fides.api.ops.models.email import EmailConfig, get_schema_for_secrets -from fides.api.ops.schemas.email.email import ( - EmailConfigRequest, - EmailConfigResponse, - TestEmailStatusMessage, -) -from fides.api.ops.schemas.email.email_secrets_docs_only import possible_email_secrets -from fides.api.ops.schemas.shared_schemas import FidesOpsKey -from fides.api.ops.service.email.email_crud_service import ( - create_email_config, - delete_email_config, - get_email_config_by_key, - update_email_config, -) -from fides.api.ops.util.api_router import APIRouter -from fides.api.ops.util.logger import Pii -from fides.api.ops.util.oauth_util import verify_oauth_client - -router = APIRouter(tags=["email"], prefix=V1_URL_PREFIX) -logger = logging.getLogger(__name__) - - -@router.post( - EMAIL_CONFIG, - status_code=HTTP_200_OK, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_CREATE_OR_UPDATE])], - response_model=EmailConfigResponse, -) -def post_config( - *, - db: Session = Depends(deps.get_db), - email_config: EmailConfigRequest, -) -> EmailConfigResponse: - """ - Given an email config, create corresponding EmailConfig object, provided no config already exists - """ - - try: - return create_email_config(db=db, config=email_config) - except EmailConfigAlreadyExistsException as e: - logger.warning(e.message) - raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=e.message) - - except Exception as exc: - logger.warning( - "Create failed for email config %s: %s", email_config.key, Pii(str(exc)) - ) - raise HTTPException( - status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Config with key {email_config.key} failed to be added", - ) - - -@router.patch( - EMAIL_BY_KEY, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_CREATE_OR_UPDATE])], - response_model=EmailConfigResponse, -) -def patch_config_by_key( - config_key: FidesOpsKey, - *, - db: Session = Depends(deps.get_db), - email_config: EmailConfigRequest, -) -> Optional[EmailConfigResponse]: - """ - Updates config for email by key, provided config with key can be found. - """ - try: - return update_email_config(db=db, key=config_key, config=email_config) - except EmailConfigNotFoundException: - logger.warning("No email config found with key %s", config_key) - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No email config found with key {config_key}", - ) - - except Exception as exc: - logger.warning( - "Patch failed for email config %s: %s", email_config.key, Pii(str(exc)) - ) - raise HTTPException( - status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Config with key {email_config.key} failed to be added", - ) - - -@router.put( - EMAIL_SECRETS, - status_code=HTTP_200_OK, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_CREATE_OR_UPDATE])], - response_model=TestEmailStatusMessage, -) -def put_config_secrets( - config_key: FidesOpsKey, - *, - db: Session = Depends(deps.get_db), - unvalidated_email_secrets: possible_email_secrets, -) -> TestEmailStatusMessage: - """ - Add or update secrets for email config. - """ - logger.info("Finding email config with key '%s'", config_key) - email_config = EmailConfig.get_by(db=db, field="key", value=config_key) - if not email_config: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=f"No email configuration with key {config_key}.", - ) - - try: - secrets_schema = get_schema_for_secrets( - service_type=email_config.service_type, - secrets=unvalidated_email_secrets, - ) - except KeyError as exc: - raise HTTPException( - status_code=HTTP_422_UNPROCESSABLE_ENTITY, - detail=exc.args[0], - ) - except ValueError as exc: - raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, - detail=exc.args[0], - ) - - logger.info("Updating email config secrets for config with key '%s'", config_key) - try: - email_config.set_secrets(db=db, email_secrets=secrets_schema.dict()) - except ValueError as exc: - raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, - detail=exc.args[0], - ) - - msg = f"Secrets updated for EmailConfig with key: {config_key}." - # todo- implement test status for email service - return TestEmailStatusMessage(msg=msg, test_status=None) - - -@router.get( - EMAIL_CONFIG, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_READ])], - response_model=Page[EmailConfigResponse], -) -def get_configs( - *, db: Session = Depends(deps.get_db), params: Params = Depends() -) -> AbstractPage[EmailConfig]: - """ - Retrieves configs for email. - """ - logger.info("Finding all email configurations with pagination params %s", params) - return paginate( - EmailConfig.query(db=db).order_by(EmailConfig.created_at.desc()), params=params - ) - - -@router.get( - EMAIL_BY_KEY, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_READ])], - response_model=EmailConfigResponse, -) -def get_config_by_key( - config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) -) -> EmailConfigResponse: - """ - Retrieves configs for email by key. - """ - logger.info("Finding email config with key '%s'", config_key) - - try: - return get_email_config_by_key(db=db, key=config_key) - except EmailConfigNotFoundException as e: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=e.message, - ) - - -@router.delete( - EMAIL_BY_KEY, - status_code=HTTP_204_NO_CONTENT, - dependencies=[Security(verify_oauth_client, scopes=[EMAIL_DELETE])], -) -def delete_config_by_key( - config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) -) -> None: - """ - Deletes email configs by key. - """ - try: - delete_email_config(db=db, key=config_key) - except EmailConfigNotFoundException as e: - raise HTTPException( - status_code=HTTP_404_NOT_FOUND, - detail=e.message, - ) diff --git a/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py b/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py index 56fa8a67412..be6aa16d5fb 100644 --- a/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/identity_verification_endpoints.py @@ -6,7 +6,7 @@ from fides.api.ops.api import deps from fides.api.ops.api.v1 import urn_registry as urls -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.schemas.identity_verification import ( IdentityVerificationConfigResponse, ) @@ -27,8 +27,8 @@ def get_id_verification_config( ) -> IdentityVerificationConfigResponse: """Returns id verification config.""" config = get_config() - email_config: Optional[EmailConfig] = db.query(EmailConfig).first() + messaging_config: Optional[MessagingConfig] = db.query(MessagingConfig).first() return IdentityVerificationConfigResponse( identity_verification_required=config.execution.subject_identity_verification_required, - valid_email_config_exists=bool(email_config and email_config.secrets), + valid_email_config_exists=bool(messaging_config and messaging_config.secrets), ) diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py new file mode 100644 index 00000000000..799db5ff1b0 --- /dev/null +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -0,0 +1,226 @@ +import logging +from typing import Optional + +from fastapi import Depends, Security +from fastapi_pagination import Page, Params +from fastapi_pagination.bases import AbstractPage +from fastapi_pagination.ext.sqlalchemy import paginate +from sqlalchemy.orm import Session +from starlette.exceptions import HTTPException +from starlette.status import ( + HTTP_200_OK, + HTTP_204_NO_CONTENT, + HTTP_400_BAD_REQUEST, + HTTP_404_NOT_FOUND, + HTTP_422_UNPROCESSABLE_ENTITY, + HTTP_500_INTERNAL_SERVER_ERROR, +) + +from fides.api.ops.api import deps +from fides.api.ops.api.v1.scope_registry import ( + MESSAGING_CREATE_OR_UPDATE, + MESSAGING_DELETE, + MESSAGING_READ, +) +from fides.api.ops.api.v1.urn_registry import ( + MESSAGING_BY_KEY, + MESSAGING_CONFIG, + MESSAGING_SECRETS, + V1_URL_PREFIX, +) +from fides.api.ops.common_exceptions import MessagingConfigNotFoundException +from fides.api.ops.models.messaging import MessagingConfig, get_schema_for_secrets +from fides.api.ops.schemas.messaging.messaging import ( + MessagingConfigRequest, + MessagingConfigResponse, + TestMessagingStatusMessage, +) +from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import ( + possible_messaging_secrets, +) +from fides.api.ops.schemas.shared_schemas import FidesOpsKey +from fides.api.ops.service.messaging.messaging_crud_service import ( + create_or_update_messaging_config, + delete_messaging_config, + get_messaging_config_by_key, + update_messaging_config, +) +from fides.api.ops.util.api_router import APIRouter +from fides.api.ops.util.logger import Pii +from fides.api.ops.util.oauth_util import verify_oauth_client + +router = APIRouter(tags=["email"], prefix=V1_URL_PREFIX) +logger = logging.getLogger(__name__) + + +@router.post( + MESSAGING_CONFIG, + status_code=HTTP_200_OK, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])], + response_model=MessagingConfigResponse, +) +def post_config( + *, + db: Session = Depends(deps.get_db), + messaging_config: MessagingConfigRequest, +) -> MessagingConfigResponse: + """ + Given a messaging config, create corresponding EmailConfig object, provided no config already exists + """ + + try: + return create_or_update_messaging_config(db=db, config=messaging_config) + except Exception as exc: + logger.warning( + "Create failed for messaging config %s: %s", messaging_config.key, Pii(str(exc)) + ) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Config with key {messaging_config.key} failed to be added", + ) + + +@router.patch( + MESSAGING_BY_KEY, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])], + response_model=MessagingConfigResponse, +) +def patch_config_by_key( + config_key: FidesOpsKey, + *, + db: Session = Depends(deps.get_db), + messaging_config: MessagingConfigRequest, +) -> Optional[MessagingConfigResponse]: + """ + Updates config for messaging by key, provided config with key can be found. + """ + try: + return update_messaging_config(db=db, key=config_key, config=messaging_config) + except MessagingConfigNotFoundException: + logger.warning("No messaging config found with key %s", config_key) + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No messaging config found with key {config_key}", + ) + + except Exception as exc: + logger.warning( + "Patch failed for messaging config %s: %s", messaging_config.key, Pii(str(exc)) + ) + raise HTTPException( + status_code=HTTP_500_INTERNAL_SERVER_ERROR, + detail=f"Config with key {messaging_config.key} failed to be added", + ) + + +@router.put( + MESSAGING_SECRETS, + status_code=HTTP_200_OK, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_CREATE_OR_UPDATE])], + response_model=TestMessagingStatusMessage, +) +def put_config_secrets( + config_key: FidesOpsKey, + *, + db: Session = Depends(deps.get_db), + unvalidated_messaging_secrets: possible_messaging_secrets, +) -> TestMessagingStatusMessage: + """ + Add or update secrets for messaging config. + """ + logger.info("Finding messaging config with key '%s'", config_key) + messaging_config = MessagingConfig.get_by(db=db, field="key", value=config_key) + if not messaging_config: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=f"No messaging configuration with key {config_key}.", + ) + + try: + secrets_schema = get_schema_for_secrets( + service_type=messaging_config.service_type, + secrets=unvalidated_messaging_secrets, + ) + except KeyError as exc: + raise HTTPException( + status_code=HTTP_422_UNPROCESSABLE_ENTITY, + detail=exc.args[0], + ) + except ValueError as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=exc.args[0], + ) + + logger.info("Updating messaging config secrets for config with key '%s'", config_key) + try: + messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.dict()) + except ValueError as exc: + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=exc.args[0], + ) + + msg = f"Secrets updated for MessagingConfig with key: {config_key}." + # todo- implement test status for messaging service + return TestMessagingStatusMessage(msg=msg, test_status=None) + + +@router.get( + MESSAGING_CONFIG, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_READ])], + response_model=Page[MessagingConfigResponse], +) +def get_configs( + *, db: Session = Depends(deps.get_db), params: Params = Depends() +) -> AbstractPage[MessagingConfig]: + """ + Retrieves configs for messaging. + """ + logger.info("Finding all messaging configurations with pagination params %s", params) + return paginate( + MessagingConfig.query(db=db).order_by(MessagingConfig.created_at.desc()), + params=params, + ) + + +@router.get( + MESSAGING_BY_KEY, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_READ])], + response_model=MessagingConfigResponse, +) +def get_config_by_key( + config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) +) -> MessagingConfigResponse: + """ + Retrieves configs for messaging service by key. + """ + logger.info("Finding messaging config with key '%s'", config_key) + + try: + return get_messaging_config_by_key(db=db, key=config_key) + except MessagingConfigNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=e.message, + ) + + +@router.delete( + MESSAGING_BY_KEY, + status_code=HTTP_204_NO_CONTENT, + dependencies=[Security(verify_oauth_client, scopes=[MESSAGING_DELETE])], +) +def delete_config_by_key( + config_key: FidesOpsKey, *, db: Session = Depends(deps.get_db) +) -> None: + """ + Deletes messaging configs by key. + """ + try: + delete_messaging_config(db=db, key=config_key) + except MessagingConfigNotFoundException as e: + raise HTTPException( + status_code=HTTP_404_NOT_FOUND, + detail=e.message, + ) diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index b178a3229bb..8d32b4f3288 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -59,11 +59,11 @@ REQUEST_PREVIEW, ) from fides.api.ops.common_exceptions import ( - EmailDispatchException, FunctionalityNotConfigured, IdentityNotFoundException, IdentityVerificationException, ManualWebhookFieldsUnset, + MessageDispatchException, NoCachedManualWebhookEntry, PolicyNotFoundException, TraversalError, @@ -93,13 +93,13 @@ CollectionAddressResponse, DryRunDatasetResponse, ) -from fides.api.ops.schemas.email.email import ( - EmailActionType, - FidesopsEmail, +from fides.api.ops.schemas.external_https import PrivacyRequestResumeFormat +from fides.api.ops.schemas.messaging.messaging import ( + FidesopsMessage, + MessagingActionType, RequestReceiptBodyParams, RequestReviewDenyBodyParams, ) -from fides.api.ops.schemas.external_https import PrivacyRequestResumeFormat from fides.api.ops.schemas.privacy_request import ( BulkPostPrivacyRequests, BulkReviewResponse, @@ -113,8 +113,11 @@ RowCountRequest, VerificationCode, ) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service._verification import send_verification_code_to_user -from fides.api.ops.service.email.email_dispatch_service import dispatch_email_task +from fides.api.ops.service.messaging.message_dispatch_service import ( + dispatch_message_task, +) from fides.api.ops.service.privacy_request.request_runner_service import ( queue_privacy_request, ) @@ -124,7 +127,7 @@ ) from fides.api.ops.task.graph_task import EMPTY_REQUEST, collect_queries from fides.api.ops.task.task_resources import TaskResources -from fides.api.ops.tasks import EMAIL_QUEUE_NAME +from fides.api.ops.tasks import MESSAGING_QUEUE_NAME from fides.api.ops.util.api_router import APIRouter from fides.api.ops.util.cache import FidesopsRedis from fides.api.ops.util.collection_util import Row @@ -240,13 +243,13 @@ async def create_privacy_request( if CONFIG.execution.subject_identity_verification_required: send_verification_code_to_user( - db, privacy_request, privacy_request_data.identity.email + db, privacy_request, privacy_request_data.identity ) created.append(privacy_request) continue # Skip further processing for this privacy request if CONFIG.notifications.send_request_receipt_notification: - _send_privacy_request_receipt_email_to_user( - policy, privacy_request_data.identity.email + _send_privacy_request_receipt_message_to_user( + policy, privacy_request_data.identity ) if not CONFIG.execution.require_manual_request_approval: AuditLog.create( @@ -259,7 +262,7 @@ async def create_privacy_request( }, ) queue_privacy_request(privacy_request.id) - except EmailDispatchException as exc: + except MessageDispatchException as exc: kwargs["privacy_request_id"] = privacy_request.id logger.error("EmailDispatchException: %s", exc) failure = { @@ -290,14 +293,14 @@ async def create_privacy_request( ) -def _send_privacy_request_receipt_email_to_user( - policy: Optional[Policy], email: Optional[str] +def _send_privacy_request_receipt_message_to_user( + policy: Optional[Policy], to_identity: Optional[Identity] ) -> None: """Helper function to send request receipt email to the user""" - if not email: + if not to_identity: logger.error( IdentityNotFoundException( - "Identity email was not found, so request receipt email could not be sent." + "Identity was not found, so request receipt email could not be sent." ) ) return @@ -312,14 +315,15 @@ def _send_privacy_request_receipt_email_to_user( for action_type in ActionType: if policy.get_rules_for_action(action_type=ActionType(action_type)): request_types.add(action_type) - dispatch_email_task.apply_async( - queue=EMAIL_QUEUE_NAME, + dispatch_message_task.apply_async( + queue=MESSAGING_QUEUE_NAME, kwargs={ - "email_meta": FidesopsEmail( - action_type=EmailActionType.PRIVACY_REQUEST_RECEIPT, + "message_meta": FidesopsMessage( + action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT, body_params=RequestReceiptBodyParams(request_types=request_types), ).dict(), - "to_email": email, + "messaging_method": CONFIG.notifications.get_messaging_method(), + "to_identity": to_identity, }, ) @@ -1143,30 +1147,35 @@ def review_privacy_request( ) -def _send_privacy_request_review_email_to_user( - action_type: EmailActionType, - email: Optional[str], +def _send_privacy_request_review_message_to_user( + action_type: MessagingActionType, + identity_data: Dict[str, Any], rejection_reason: Optional[str], ) -> None: - """Helper method to send review notification email to user, shared between approve and deny""" - if not email: + """Helper method to send review notification message to user, shared between approve and deny""" + if not identity_data: logger.error( IdentityNotFoundException( - "Identity email was not found, so request review email could not be sent." + "Identity was not found, so request review email could not be sent." ) ) - dispatch_email_task.apply_async( - queue=EMAIL_QUEUE_NAME, + to_identity: Identity = Identity( + email=identity_data.get(ProvidedIdentityType.email.value), + phone_number=identity_data.get(ProvidedIdentityType.phone_number.value), + ) + dispatch_message_task.apply_async( + queue=MESSAGING_QUEUE_NAME, kwargs={ - "email_meta": FidesopsEmail( + "message_meta": FidesopsMessage( action_type=action_type, body_params=RequestReviewDenyBodyParams( rejection_reason=rejection_reason ) - if action_type is EmailActionType.PRIVACY_REQUEST_REVIEW_DENY + if action_type is MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY else None, ).dict(), - "to_email": email, + "messaging_method": CONFIG.notifications.get_messaging_method(), + "to_identity": to_identity, }, ) @@ -1197,8 +1206,8 @@ async def verify_identification_code( db=db, object_id=privacy_request.policy_id ) if CONFIG.notifications.send_request_receipt_notification: - _send_privacy_request_receipt_email_to_user( - policy, privacy_request.get_persisted_identity().email + _send_privacy_request_receipt_message_to_user( + policy, privacy_request.get_persisted_identity() ) except IdentityVerificationException as exc: raise HTTPException(status_code=HTTP_400_BAD_REQUEST, detail=exc.message) @@ -1256,11 +1265,9 @@ def _approve_request(privacy_request: PrivacyRequest) -> None: }, ) if CONFIG.notifications.send_request_review_notification: - _send_privacy_request_review_email_to_user( - action_type=EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE, - email=privacy_request.get_cached_identity_data().get( - ProvidedIdentityType.email.value - ), + _send_privacy_request_review_message_to_user( + action_type=MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE, + identity_data=privacy_request.get_cached_identity_data(), rejection_reason=None, ) @@ -1308,11 +1315,9 @@ def _deny_request( }, ) if CONFIG.notifications.send_request_review_notification: - _send_privacy_request_review_email_to_user( - action_type=EmailActionType.PRIVACY_REQUEST_REVIEW_DENY, - email=privacy_request.get_cached_identity_data().get( - ProvidedIdentityType.email.value - ), + _send_privacy_request_review_message_to_user( + action_type=MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY, + identity_data=privacy_request.get_cached_identity_data(), rejection_reason=privacy_requests.reason, ) diff --git a/src/fides/api/ops/api/v1/scope_registry.py b/src/fides/api/ops/api/v1/scope_registry.py index 9b9e8d24e32..c13199c27a4 100644 --- a/src/fides/api/ops/api/v1/scope_registry.py +++ b/src/fides/api/ops/api/v1/scope_registry.py @@ -40,9 +40,9 @@ STORAGE_READ = "storage:read" STORAGE_DELETE = "storage:delete" -EMAIL_CREATE_OR_UPDATE = "email:create_or_update" -EMAIL_READ = "email:read" -EMAIL_DELETE = "email:delete" +MESSAGING_CREATE_OR_UPDATE = "messaging:create_or_update" +MESSAGING_READ = "messaging:read" +MESSAGING_DELETE = "messaging:delete" SCOPE_READ = "scope:read" @@ -99,9 +99,9 @@ STORAGE_CREATE_OR_UPDATE, STORAGE_DELETE, STORAGE_READ, - EMAIL_CREATE_OR_UPDATE, - EMAIL_DELETE, - EMAIL_READ, + MESSAGING_CREATE_OR_UPDATE, + MESSAGING_DELETE, + MESSAGING_READ, WEBHOOK_CREATE_OR_UPDATE, WEBHOOK_READ, WEBHOOK_DELETE, diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py index bff93e7d6b0..f3624da52a9 100644 --- a/src/fides/api/ops/api/v1/urn_registry.py +++ b/src/fides/api/ops/api/v1/urn_registry.py @@ -38,9 +38,9 @@ STORAGE_UPLOAD = "/storage/{request_id}" # Email URLs -EMAIL_CONFIG = "/email/config" -EMAIL_SECRETS = "/email/config/{config_key}/secret" -EMAIL_BY_KEY = "/email/config/{config_key}" +MESSAGING_CONFIG = "/messaging/config" +MESSAGING_SECRETS = "/messaging/config/{config_key}/secret" +MESSAGING_BY_KEY = "/messaging/config/{config_key}" # Policy URLs POLICY_LIST = "/dsr/policy" diff --git a/src/fides/api/ops/common_exceptions.py b/src/fides/api/ops/common_exceptions.py index 749bc596b14..587e3d05541 100644 --- a/src/fides/api/ops/common_exceptions.py +++ b/src/fides/api/ops/common_exceptions.py @@ -121,16 +121,12 @@ class SaaSConfigNotFoundException(FidesopsException): """Custom Exception - SaaS Config Not Found""" -class EmailConfigAlreadyExistsException(FidesopsException): - """Custom Exception - Email Config already exists""" +class MessagingConfigNotFoundException(FidesopsException): + """Custom Exception - Messaging Config Not Found""" -class EmailConfigNotFoundException(FidesopsException): - """Custom Exception - Email Config Not Found""" - - -class EmailDispatchException(FidesopsException): - """Custom Exception - Email Dispatch Error""" +class MessageDispatchException(FidesopsException): + """Custom Exception - Message Dispatch Error""" class EmailTemplateUnhandledActionType(FidesopsException): diff --git a/src/fides/api/ops/db/base.py b/src/fides/api/ops/db/base.py index 668e84d436a..b8aee77edc0 100644 --- a/src/fides/api/ops/db/base.py +++ b/src/fides/api/ops/db/base.py @@ -10,8 +10,8 @@ from fides.api.ops.models.authentication_request import AuthenticationRequest from fides.api.ops.models.connectionconfig import ConnectionConfig from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.models.email import EmailConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import Policy, Rule, RuleTarget from fides.api.ops.models.privacy_request import PrivacyRequest from fides.api.ops.models.registration import UserRegistration diff --git a/src/fides/api/ops/email_templates/get_email_template.py b/src/fides/api/ops/email_templates/get_email_template.py index 5ed51f506ea..a128fa878eb 100644 --- a/src/fides/api/ops/email_templates/get_email_template.py +++ b/src/fides/api/ops/email_templates/get_email_template.py @@ -14,7 +14,7 @@ PRIVACY_REQUEST_REVIEW_DENY_TEMPLATE, SUBJECT_IDENTITY_VERIFICATION_TEMPLATE, ) -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import MessagingActionType pathlib.Path(__file__).parent.resolve() logger = logging.getLogger(__name__) @@ -27,23 +27,23 @@ def get_email_template( # pylint: disable=too-many-return-statements - action_type: EmailActionType, + action_type: MessagingActionType, ) -> Template: - if action_type == EmailActionType.CONSENT_REQUEST: + if action_type == MessagingActionType.CONSENT_REQUEST: return template_env.get_template(CONSENT_REQUEST_VERIFICATION_TEMPLATE) - if action_type == EmailActionType.SUBJECT_IDENTITY_VERIFICATION: + if action_type == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION: return template_env.get_template(SUBJECT_IDENTITY_VERIFICATION_TEMPLATE) - if action_type == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT: + if action_type == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT: return template_env.get_template(EMAIL_ERASURE_REQUEST_FULFILLMENT) - if action_type == EmailActionType.PRIVACY_REQUEST_RECEIPT: + if action_type == MessagingActionType.PRIVACY_REQUEST_RECEIPT: return template_env.get_template(PRIVACY_REQUEST_RECEIPT_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: return template_env.get_template(PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION: + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION: return template_env.get_template(PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY: + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY: return template_env.get_template(PRIVACY_REQUEST_REVIEW_DENY_TEMPLATE) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE: + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE: return template_env.get_template(PRIVACY_REQUEST_REVIEW_APPROVE_TEMPLATE) logger.error("No corresponding template linked to the %s", action_type) diff --git a/src/fides/api/ops/email_templates/template_names.py b/src/fides/api/ops/email_templates/template_names.py index 8ceb5cff023..857053cb716 100644 --- a/src/fides/api/ops/email_templates/template_names.py +++ b/src/fides/api/ops/email_templates/template_names.py @@ -1,6 +1,6 @@ CONSENT_REQUEST_VERIFICATION_TEMPLATE = "consent_request_verification.html" SUBJECT_IDENTITY_VERIFICATION_TEMPLATE = "subject_identity_verification.html" -EMAIL_ERASURE_REQUEST_FULFILLMENT = "erasure_request_email_fulfillment.html" +EMAIL_ERASURE_REQUEST_FULFILLMENT = "message_request_email_fulfillment.html" PRIVACY_REQUEST_RECEIPT_TEMPLATE = "privacy_request_receipt.html" PRIVACY_REQUEST_COMPLETE_DELETION_TEMPLATE = "privacy_request_complete_deletion.html" PRIVACY_REQUEST_COMPLETE_ACCESS_TEMPLATE = "privacy_request_complete_access.html" diff --git a/src/fides/api/ops/email_templates/templates/erasure_request_email_fulfillment.html b/src/fides/api/ops/email_templates/templates/message_request_email_fulfillment.html similarity index 100% rename from src/fides/api/ops/email_templates/templates/erasure_request_email_fulfillment.html rename to src/fides/api/ops/email_templates/templates/message_request_email_fulfillment.html diff --git a/src/fides/api/ops/models/email.py b/src/fides/api/ops/models/messaging.py similarity index 68% rename from src/fides/api/ops/models/email.py rename to src/fides/api/ops/models/messaging.py index 82dc66d2c7e..8f27d95667a 100644 --- a/src/fides/api/ops/models/email.py +++ b/src/fides/api/ops/models/messaging.py @@ -12,14 +12,16 @@ StringEncryptedType, ) -from fides.api.ops.common_exceptions import EmailDispatchException +from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.db.base_class import JSONTypeOverride -from fides.api.ops.schemas.email.email import ( - SUPPORTED_EMAIL_SERVICE_SECRETS, - EmailServiceSecretsMailgun, - EmailServiceType, +from fides.api.ops.schemas.messaging.messaging import ( + SUPPORTED_MESSAGING_SERVICE_SECRETS, + MessagingServiceSecretsMailgun, + MessagingServiceType, +) +from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import ( + possible_messaging_secrets, ) -from fides.api.ops.schemas.email.email_secrets_docs_only import possible_email_secrets from fides.api.ops.util.logger import Pii from fides.ctl.core.config import get_config @@ -28,16 +30,16 @@ def get_schema_for_secrets( - service_type: EmailServiceType, - secrets: possible_email_secrets, -) -> SUPPORTED_EMAIL_SERVICE_SECRETS: + service_type: MessagingServiceType, + secrets: possible_messaging_secrets, +) -> SUPPORTED_MESSAGING_SERVICE_SECRETS: """ Returns the secrets that pertain to `service_type` represented as a Pydantic schema for validation purposes. """ try: schema = { - EmailServiceType.MAILGUN: EmailServiceSecretsMailgun, + MessagingServiceType.MAILGUN: MessagingServiceSecretsMailgun, }[service_type] except KeyError: raise ValueError( @@ -53,12 +55,14 @@ def get_schema_for_secrets( raise ValueError(errors) -class EmailConfig(Base): - """The DB ORM model for EmailConfig""" +class MessagingConfig(Base): + """The DB ORM model for MessagingConfig""" key = Column(String, index=True, unique=True, nullable=False) name = Column(String, unique=True, index=True) - service_type = Column(Enum(EmailServiceType), index=True, nullable=False) + service_type = Column( + Enum(MessagingServiceType), index=True, unique=True, nullable=False + ) details = Column(MutableDict.as_mutable(JSONB), nullable=False) secrets = Column( MutableDict.as_mutable( @@ -75,19 +79,18 @@ class EmailConfig(Base): @classmethod def get_configuration(cls, db: Session) -> Base: """ - Fetches the first configured EmailConfig record. As of v1.7.3 Fidesops does not support - multiple configured email connectors. Once fetched this function validates that - the EmailConfig is configured with secrets. + Fetches the first configured MessagingConfig record. Once fetched this function validates that + the MessagingConfig is configured with secrets. """ instance: Optional[Base] = cls.query(db=db).first() if not instance: - raise EmailDispatchException("No email config found.") + raise MessageDispatchException("No messaging config found.") if not instance.secrets: logger.warning( - "Email secrets not found for config with key: %s", instance.key + "Messaging secrets not found for config with key: %s", instance.key ) - raise EmailDispatchException( - f"Email secrets not found for config with key: {instance.key}" + raise MessageDispatchException( + f"Messaging secrets not found for config with key: {instance.key}" ) return instance @@ -95,7 +98,7 @@ def set_secrets( self, *, db: Session, - email_secrets: possible_email_secrets, + messaging_secrets: possible_messaging_secrets, ) -> None: """Creates or updates secrets associated with a config id""" @@ -108,7 +111,7 @@ def set_secrets( try: get_schema_for_secrets( service_type=service_type, # type: ignore - secrets=email_secrets, + secrets=messaging_secrets, ) except ( KeyError, @@ -118,5 +121,5 @@ def set_secrets( # We don't want to handle these explicitly here, only in the API view raise - self.secrets = email_secrets + self.secrets = messaging_secrets self.save(db=db) diff --git a/src/fides/api/ops/schemas/email/email_secrets_docs_only.py b/src/fides/api/ops/schemas/email/email_secrets_docs_only.py deleted file mode 100644 index 6620939b01f..00000000000 --- a/src/fides/api/ops/schemas/email/email_secrets_docs_only.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Union - -from fides.api.ops.schemas.base_class import NoValidationSchema -from fides.api.ops.schemas.email.email import EmailServiceSecretsMailgun - - -class EmailSecretsMailgunDocs(EmailServiceSecretsMailgun, NoValidationSchema): - """The secrets required to connect to Mailgun, for documentation""" - - -possible_email_secrets = Union[EmailSecretsMailgunDocs] diff --git a/src/fides/api/ops/schemas/email/__init__.py b/src/fides/api/ops/schemas/messaging/__init__.py similarity index 100% rename from src/fides/api/ops/schemas/email/__init__.py rename to src/fides/api/ops/schemas/messaging/__init__.py diff --git a/src/fides/api/ops/schemas/email/email.py b/src/fides/api/ops/schemas/messaging/messaging.py similarity index 55% rename from src/fides/api/ops/schemas/email/email.py rename to src/fides/api/ops/schemas/messaging/messaging.py index c90d61acae0..388437c8e3d 100644 --- a/src/fides/api/ops/schemas/email/email.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -8,20 +8,36 @@ from fides.api.ops.schemas.shared_schemas import FidesOpsKey -class EmailServiceType(Enum): - """Enum for email service type""" +class MessagingMethod(Enum): + """Enum for messaging method""" + + EMAIL = "email" + SMS = "sms" + + +class MessagingServiceType(Enum): + """Enum for messaging service type""" - # may support twilio or google in the future MAILGUN = "mailgun" + TWILIO_TEXT = "twilio_text" + TWILIO_EMAIL = "twilio_email" + + +EMAIL_MESSAGING_SERVICES = [ + MessagingServiceType.MAILGUN, + MessagingServiceType.TWILIO_EMAIL, +] +SMS_MESSAGING_SERVICES = [MessagingServiceType.TWILIO_TEXT] -class EmailActionType(str, Enum): - """Enum for email action type""" + +class MessagingActionType(str, Enum): + """Enum for messaging action type""" # verify email upon acct creation CONSENT_REQUEST = "consent_request" SUBJECT_IDENTITY_VERIFICATION = "subject_identity_verification" - EMAIL_ERASURE_REQUEST_FULFILLMENT = "email_erasure_fulfillment" + MESSAGE_ERASURE_REQUEST_FULFILLMENT = "message_erasure_fulfillment" PRIVACY_REQUEST_RECEIPT = "privacy_request_receipt" PRIVACY_REQUEST_COMPLETE_ACCESS = "privacy_request_complete_access" PRIVACY_REQUEST_COMPLETE_DELETION = "privacy_request_complete_deletion" @@ -29,14 +45,8 @@ class EmailActionType(str, Enum): PRIVACY_REQUEST_REVIEW_APPROVE = "privacy_request_review_approve" -class EmailTemplateBodyParams(Enum): - """Enum for all possible email template body params""" - - VERIFICATION_CODE = "verification_code" - - class SubjectIdentityVerificationBodyParams(BaseModel): - """Body params required for subject identity verification email template""" + """Body params required for subject identity verification email/sms template""" verification_code: str verification_code_ttl_seconds: int @@ -49,31 +59,31 @@ def get_verification_code_ttl_minutes(self) -> int: class RequestReceiptBodyParams(BaseModel): - """Body params required for privacy request receipt email template""" + """Body params required for privacy request receipt template""" request_types: List[str] class AccessRequestCompleteBodyParams(BaseModel): - """Body params required for privacy request completion access email template""" + """Body params required for privacy request completion access template""" download_links: List[str] class RequestReviewDenyBodyParams(BaseModel): - """Body params required for privacy request review deny email template""" + """Body params required for privacy request review deny template""" rejection_reason: Optional[str] -class FidesopsEmail( +class FidesopsMessage( BaseModel, smart_union=True, arbitrary_types_allowed=True, ): """A mapping of action_type to body_params""" - action_type: EmailActionType + action_type: MessagingActionType body_params: Optional[ Union[ SubjectIdentityVerificationBodyParams, @@ -85,23 +95,28 @@ class FidesopsEmail( ] -class EmailForActionType(BaseModel): - """Email details that depend on action type""" +class MessageForActionType(BaseModel): + """Message details that depend on action type""" subject: str body: str -class EmailServiceDetails(Enum): - """Enum for email service details""" +class MessagingServiceDetails(Enum): + """Enum for messaging service details""" # mailgun-specific IS_EU_DOMAIN = "is_eu_domain" API_VERSION = "api_version" DOMAIN = "domain" + # twilio + TWILIO_ACCOUNT_SID = "twilio_account_sid" + TWILIO_AUTH_TOKEN = "twilio_auth_token" + TWILIO_MESSAGING_SERVICE_ID = "twilio_messaging_service_id" + -class EmailServiceDetailsMailgun(BaseModel): +class MessagingServiceDetailsMailgun(BaseModel): """The details required to represent a Mailgun email configuration.""" is_eu_domain: Optional[bool] = False @@ -114,14 +129,19 @@ class Config: extra = Extra.forbid -class EmailServiceSecrets(Enum): - """Enum for email service secrets""" +class MessagingServiceSecrets(Enum): + """Enum for message service secrets""" # mailgun-specific MAILGUN_API_KEY = "mailgun_api_key" + # twilio + TWILIO_ACCOUNT_SID = "twilio_account_sid" + TWILIO_AUTH_TOKEN = "twilio_auth_token" + TWILIO_MESSAGING_SERVICE_ID = "twilio_messaging_service_id" -class EmailServiceSecretsMailgun(BaseModel): + +class MessagingServiceSecretsMailgun(BaseModel): """The secrets required to connect to mailgun.""" mailgun_api_key: str @@ -132,47 +152,60 @@ class Config: extra = Extra.forbid -class EmailConfigRequest(BaseModel): - """Email Config Request Schema""" +class MessagingServiceSecretsTwilio(BaseModel): + """The secrets required to connect to twilio email.""" + + account_sid: str + auth_token: str + messaging_service_id: str + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + +class MessagingConfigRequest(BaseModel): + """Messaging Config Request Schema""" name: str key: Optional[FidesOpsKey] - service_type: EmailServiceType - details: Union[ - EmailServiceDetailsMailgun, - ] + service_type: MessagingServiceType + details: Union[MessagingServiceDetailsMailgun] class Config: use_enum_values = False orm_mode = True -class EmailConfigResponse(BaseModel): - """Email Config Response Schema""" +class MessagingConfigResponse(BaseModel): + """Messaging Config Response Schema""" name: str key: FidesOpsKey - service_type: EmailServiceType - details: Dict[EmailServiceDetails, Any] + service_type: MessagingServiceType + details: Dict[MessagingServiceDetails, Any] class Config: orm_mode = True use_enum_values = True -SUPPORTED_EMAIL_SERVICE_SECRETS = Union[EmailServiceSecretsMailgun] +SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[ + MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilio +] -class EmailConnectionTestStatus(Enum): - """Enum for supplying statuses of validating credentials for an Email Config""" +class MessagingConnectionTestStatus(Enum): + """Enum for supplying statuses of validating credentials for an Messaging Config""" succeeded = "succeeded" failed = "failed" skipped = "skipped" -class TestEmailStatusMessage(Msg): - """A schema for checking status of email config.""" +class TestMessagingStatusMessage(Msg): + """A schema for checking status of message config.""" - test_status: Optional[EmailConnectionTestStatus] = None + test_status: Optional[MessagingConnectionTestStatus] = None failure_reason: Optional[str] = None diff --git a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py new file mode 100644 index 00000000000..3a9f60a4c69 --- /dev/null +++ b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py @@ -0,0 +1,20 @@ +from typing import Union + +from fides.api.ops.schemas.base_class import NoValidationSchema +from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceSecretsMailgun, + MessagingServiceSecretsTwilio, +) + + +class MessagingSecretsMailgunDocs(MessagingServiceSecretsMailgun, NoValidationSchema): + """The secrets required to connect to Mailgun, for documentation""" + + +class MessagingSecretsTwilioDocs(MessagingServiceSecretsTwilio, NoValidationSchema): + """The secrets required to connect to Twilio, for documentation""" + + +possible_messaging_secrets = Union[ + MessagingSecretsMailgunDocs, MessagingSecretsTwilioDocs +] diff --git a/src/fides/api/ops/service/_verification.py b/src/fides/api/ops/service/_verification.py index 84d0cb873ae..7193b21f1c4 100644 --- a/src/fides/api/ops/service/_verification.py +++ b/src/fides/api/ops/service/_verification.py @@ -2,13 +2,14 @@ from sqlalchemy.orm import Session -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.privacy_request import ConsentRequest, PrivacyRequest -from fides.api.ops.schemas.email.email import ( - EmailActionType, +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, SubjectIdentityVerificationBodyParams, ) -from fides.api.ops.service.email.email_dispatch_service import dispatch_email +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message from fides.api.ops.service.privacy_request.request_runner_service import ( generate_id_verification_code, ) @@ -18,24 +19,25 @@ def send_verification_code_to_user( - db: Session, request: ConsentRequest | PrivacyRequest, email: str | None + db: Session, request: ConsentRequest | PrivacyRequest, to_identity: Identity | None ) -> str: - """Generate and cache a verification code, and then email to the user""" - EmailConfig.get_configuration( + """Generate and cache a verification code, and then message the user""" + MessagingConfig.get_configuration( db=db - ) # Validates Fidesops is currently configured to send emails + ) # Validates Fidesops is currently configured to send messages verification_code = generate_id_verification_code() request.cache_identity_verification_code(verification_code) - email_action_type = ( - EmailActionType.CONSENT_REQUEST + messaging_action_type = ( + MessagingActionType.CONSENT_REQUEST if isinstance(request, ConsentRequest) - else EmailActionType.SUBJECT_IDENTITY_VERIFICATION + else MessagingActionType.SUBJECT_IDENTITY_VERIFICATION ) - dispatch_email( + dispatch_message( db, - action_type=email_action_type, - to_email=email, - email_body_params=SubjectIdentityVerificationBodyParams( + action_type=messaging_action_type, + to_identity=to_identity, + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=SubjectIdentityVerificationBodyParams( verification_code=verification_code, verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, ), diff --git a/src/fides/api/ops/service/connectors/email_connector.py b/src/fides/api/ops/service/connectors/email_connector.py index 57eef852648..7f8fa717498 100644 --- a/src/fides/api/ops/service/connectors/email_connector.py +++ b/src/fides/api/ops/service/connectors/email_connector.py @@ -5,7 +5,7 @@ from sqlalchemy.orm import Session from fides.api.ops.common_exceptions import ( - EmailDispatchException, + MessageDispatchException, PrivacyRequestErasureEmailSendRequired, ) from fides.api.ops.graph.config import CollectionAddress, FieldPath @@ -23,10 +23,14 @@ PrivacyRequest, ) from fides.api.ops.schemas.connection_configuration import EmailSchema -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, + MessagingMethod, +) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.base_connector import BaseConnector from fides.api.ops.service.connectors.query_config import ManualQueryConfig -from fides.api.ops.service.email.email_dispatch_service import dispatch_email +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message from fides.api.ops.util.collection_util import Row, append logger = logging.getLogger(__name__) @@ -56,11 +60,12 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: try: # synchronous for now since failure to send is considered a connection test failure - dispatch_email( + dispatch_message( db=db, - action_type=EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT, - to_email=config.test_email, - email_body_params=[ + action_type=MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, + to_identity=Identity(**{"email": config.test_email}), + messaging_method=MessagingMethod.EMAIL, + message_body_params=[ CheckpointActionRequired( step=CurrentStep.erasure, collection=CollectionAddress("test_dataset", "test_collection"), @@ -76,7 +81,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: ) ], ) - except EmailDispatchException as exc: + except MessageDispatchException as exc: logger.info("Email connector test failed with exception %s", exc) return ConnectionTestStatus.failed return ConnectionTestStatus.succeeded @@ -194,11 +199,12 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - ) return - dispatch_email( + dispatch_message( db, - action_type=EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT, - to_email=cc.secrets.get("to_email"), - email_body_params=template_values, + action_type=MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, + to_identity=Identity(**{"email": cc.secrets.get("to_email")}), + messaging_method=MessagingMethod.EMAIL, + message_body_params=template_values, ) logger.info( diff --git a/src/fides/api/ops/service/email/email_crud_service.py b/src/fides/api/ops/service/email/email_crud_service.py deleted file mode 100644 index 5497682fd72..00000000000 --- a/src/fides/api/ops/service/email/email_crud_service.py +++ /dev/null @@ -1,74 +0,0 @@ -import logging - -from fideslang.validation import FidesKey -from sqlalchemy.orm import Session - -from fides.api.ops.common_exceptions import ( - EmailConfigAlreadyExistsException, - EmailConfigNotFoundException, -) -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.schemas.email.email import EmailConfigRequest, EmailConfigResponse - -logger = logging.getLogger(__name__) - - -def create_email_config(db: Session, config: EmailConfigRequest) -> EmailConfigResponse: - existing_config = db.query(EmailConfig).first() - if existing_config: - raise EmailConfigAlreadyExistsException( - f"Only one email config is supported at a time. Config with key {config.key} is already configured." - ) - return _create_or_update_email_config(db=db, config=config) - - -def update_email_config( - db: Session, key: FidesKey, config: EmailConfigRequest -) -> EmailConfigResponse: - existing_config_with_key: EmailConfig = EmailConfig.get_by( - db=db, field="key", value=key - ) - if not existing_config_with_key: - raise EmailConfigNotFoundException(f"No email config found with key {key}") - return _create_or_update_email_config(db=db, config=config) - - -def _create_or_update_email_config( - db: Session, config: EmailConfigRequest -) -> EmailConfigResponse: - email_config: EmailConfig = EmailConfig.create_or_update( - db=db, - data={ - "key": config.key, - "name": config.name, - "service_type": config.service_type, - "details": config.details.__dict__, - }, - ) - return EmailConfigResponse( - name=email_config.name, - key=email_config.key, - service_type=email_config.service_type, - details=email_config.details, - ) - - -def delete_email_config(db: Session, key: FidesKey) -> None: - logger.info("Finding email config with key '%s'", key) - email_config: EmailConfig = EmailConfig.get_by(db, field="key", value=key) - if not email_config: - raise EmailConfigNotFoundException(f"No email config found with key {key}") - logger.info("Deleting email config with key '%s'", key) - email_config.delete(db) - - -def get_email_config_by_key(db: Session, key: FidesKey) -> EmailConfigResponse: - config = EmailConfig.get_by(db=db, field="key", value=key) - if not config: - raise EmailConfigNotFoundException(f"No email config found with key {key}") - return EmailConfigResponse( - name=config.name, - key=config.key, - service_type=config.service_type, - details=config.details, - ) diff --git a/src/fides/api/ops/service/email/email_dispatch_service.py b/src/fides/api/ops/service/email/email_dispatch_service.py deleted file mode 100644 index ebe6e885a20..00000000000 --- a/src/fides/api/ops/service/email/email_dispatch_service.py +++ /dev/null @@ -1,208 +0,0 @@ -from __future__ import annotations - -import logging -from typing import Any, Dict, List, Optional, Union - -import requests -from sqlalchemy.orm import Session - -from fides.api.ops.common_exceptions import EmailDispatchException -from fides.api.ops.email_templates import get_email_template -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.models.privacy_request import CheckpointActionRequired -from fides.api.ops.schemas.email.email import ( - AccessRequestCompleteBodyParams, - EmailActionType, - EmailForActionType, - EmailServiceDetails, - EmailServiceSecrets, - EmailServiceType, - FidesopsEmail, - RequestReceiptBodyParams, - RequestReviewDenyBodyParams, - SubjectIdentityVerificationBodyParams, -) -from fides.api.ops.tasks import DatabaseTask, celery_app -from fides.api.ops.util.logger import Pii - -logger = logging.getLogger(__name__) - - -@celery_app.task(base=DatabaseTask, bind=True) -def dispatch_email_task( - self: DatabaseTask, - email_meta: Dict[str, Any], - to_email: str, -) -> None: - """ - A wrapper function to dispatch an email task into the Celery queues - """ - schema = FidesopsEmail.parse_obj(email_meta) - with self.session as db: - dispatch_email( - db, - schema.action_type, - to_email, - schema.body_params, - ) - - -def dispatch_email( - db: Session, - action_type: EmailActionType, - to_email: Optional[str], - email_body_params: Optional[ - Union[ - AccessRequestCompleteBodyParams, - SubjectIdentityVerificationBodyParams, - RequestReceiptBodyParams, - RequestReviewDenyBodyParams, - List[CheckpointActionRequired], - ] - ] = None, -) -> None: - """ - Sends an email to `to_email` with content supplied in `email_body_params` - """ - if not to_email: - logger.error("Email failed to send. No email supplied.") - raise EmailDispatchException("No email supplied.") - - logger.info("Retrieving email config") - email_config: EmailConfig = EmailConfig.get_configuration(db=db) - logger.info("Building appropriate email template for action type: %s", action_type) - email: EmailForActionType = _build_email( - action_type=action_type, - body_params=email_body_params, - ) - email_service: EmailServiceType = email_config.service_type # type: ignore - logger.info( - "Retrieving appropriate dispatcher for email service: %s", email_service - ) - dispatcher: Any = _get_dispatcher_from_config_type(email_service_type=email_service) - logger.info( - "Starting email dispatch for email service with action type: %s", action_type - ) - dispatcher( - email_config=email_config, - email=email, - to_email=to_email, - ) - - -def _build_email( # pylint: disable=too-many-return-statements - action_type: EmailActionType, - body_params: Any, -) -> EmailForActionType: - if action_type == EmailActionType.CONSENT_REQUEST: - template = get_email_template(action_type) - return EmailForActionType( - subject="Your one-time code", - body=template.render( - { - "code": body_params.verification_code, - "minutes": body_params.get_verification_code_ttl_minutes(), - } - ), - ) - if action_type == EmailActionType.SUBJECT_IDENTITY_VERIFICATION: - template = get_email_template(action_type) - return EmailForActionType( - subject="Your one-time code", - body=template.render( - { - "code": body_params.verification_code, - "minutes": body_params.get_verification_code_ttl_minutes(), - } - ), - ) - if action_type == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Data erasure request", - body=base_template.render( - {"dataset_collection_action_required": body_params} - ), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_RECEIPT: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your request has been received", - body=base_template.render({"request_types": body_params.request_types}), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your data is ready to be downloaded", - body=base_template.render( - { - "download_links": body_params.download_links, - } - ), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your data has been deleted", - body=base_template.render(), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your request has been approved", - body=base_template.render(), - ) - if action_type == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY: - base_template = get_email_template(action_type) - return EmailForActionType( - subject="Your request has been denied", - body=base_template.render( - {"rejection_reason": body_params.rejection_reason} - ), - ) - logger.error("Email action type %s is not implemented", action_type) - raise EmailDispatchException(f"Email action type {action_type} is not implemented") - - -def _get_dispatcher_from_config_type(email_service_type: EmailServiceType) -> Any: - """Determines which dispatcher to use based on email service type""" - return { - EmailServiceType.MAILGUN.value: _mailgun_dispatcher, - }[email_service_type.value] - - -def _mailgun_dispatcher( - email_config: EmailConfig, email: EmailForActionType, to_email: str -) -> None: - """Dispatches email using mailgun""" - base_url = ( - "https://api.mailgun.net" - if email_config.details[EmailServiceDetails.IS_EU_DOMAIN.value] is False - else "https://api.eu.mailgun.net" - ) - domain = email_config.details[EmailServiceDetails.DOMAIN.value] - data = { - "from": f"", - "to": [to_email], - "subject": email.subject, - "html": email.body, - } - try: - response: requests.Response = requests.post( - f"{base_url}/{email_config.details[EmailServiceDetails.API_VERSION.value]}/{domain}/messages", - auth=( - "api", - email_config.secrets[EmailServiceSecrets.MAILGUN_API_KEY.value], # type: ignore - ), - data=data, - ) - if not response.ok: - logger.error( - "Email failed to send with status code: %s", response.status_code - ) - raise EmailDispatchException( - f"Email failed to send with status code {response.status_code}" - ) - except Exception as e: - logger.error("Email failed to send: %s", Pii(str(e))) - raise EmailDispatchException(f"Email failed to send due to: {Pii(e)}") diff --git a/src/fides/api/ops/service/email/__init__.py b/src/fides/api/ops/service/messaging/__init__.py similarity index 100% rename from src/fides/api/ops/service/email/__init__.py rename to src/fides/api/ops/service/messaging/__init__.py diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py new file mode 100644 index 00000000000..98d54f31877 --- /dev/null +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -0,0 +1,259 @@ +from __future__ import annotations + +import logging +from typing import Any, Dict, List, Optional, Union + +import requests +from sqlalchemy.orm import Session + +from fides.api.ops.common_exceptions import MessageDispatchException +from fides.api.ops.email_templates import get_email_template +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.models.privacy_request import CheckpointActionRequired +from fides.api.ops.schemas.messaging.messaging import ( + AccessRequestCompleteBodyParams, + FidesopsMessage, + MessageForActionType, + MessagingActionType, + MessagingMethod, + MessagingServiceDetails, + MessagingServiceSecrets, + MessagingServiceType, + RequestReceiptBodyParams, + RequestReviewDenyBodyParams, + SubjectIdentityVerificationBodyParams, +) +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.tasks import DatabaseTask, celery_app +from fides.api.ops.util.logger import Pii +from fides.ctl.core.config import get_config + +CONFIG = get_config() + +logger = logging.getLogger(__name__) + + +@celery_app.task(base=DatabaseTask, bind=True) +def dispatch_message_task( + self: DatabaseTask, + message_meta: Dict[str, Any], + messaging_method: Optional[MessagingMethod], + to_identity: Identity, +) -> None: + """ + A wrapper function to dispatch a message task into the Celery queues + """ + schema = FidesopsMessage.parse_obj(message_meta) + with self.session as db: + dispatch_message( + db, + schema.action_type, + to_identity, + messaging_method, + schema.body_params, + ) + + +def dispatch_message( + db: Session, + action_type: MessagingActionType, + to_identity: Optional[Identity], + messaging_method: Optional[MessagingMethod], + message_body_params: Optional[ + Union[ + AccessRequestCompleteBodyParams, + SubjectIdentityVerificationBodyParams, + RequestReceiptBodyParams, + RequestReviewDenyBodyParams, + List[CheckpointActionRequired], + ] + ] = None, +) -> None: + """ + Sends a message to `to_identity` with content supplied in `message_body_params` + """ + if not to_identity: + logger.error("Message failed to send. No identity supplied.") + raise MessageDispatchException("No identity supplied.") + + logger.info("Retrieving message config") + messaging_config: MessagingConfig = MessagingConfig.get_configuration(db=db) + logger.info( + "Building appropriate message template for action type: %s", action_type + ) + message: Optional[MessageForActionType] = None # fixme- huh?? + if messaging_method == MessagingMethod.EMAIL: + message = _build_email( + action_type=action_type, + body_params=message_body_params, + ) + elif messaging_method == MessagingMethod.SMS: + message = _build_sms( + action_type=action_type, + body_params=message_body_params, + ) + else: + logger.error( + "Notification service type is not valid: %s", CONFIG.notifications.notification_service_type + ) + raise MessageDispatchException( + f"Notification service type is not valid: {CONFIG.notifications.notification_service_type}" + ) + messaging_service: MessagingServiceType = messaging_config.service_type # type: ignore + logger.info( + "Retrieving appropriate dispatcher for email service: %s", messaging_service + ) + dispatcher: Any = _get_dispatcher_from_config_type( + message_service_type=messaging_service + ) + logger.info( + "Starting email dispatch for messaging service with action type: %s", + action_type, + ) + dispatcher( + messaging_config=messaging_config, + message=message, + to=to_identity.email + if messaging_method == MessagingMethod.EMAIL + else to_identity.phone_number, + ) + + +def _build_sms( + action_type: MessagingActionType, + body_params: Any, # fixme- create message body based on params +) -> MessageForActionType: + if action_type == MessagingActionType.CONSENT_REQUEST: + return MessageForActionType( + subject="Your one-time code", + body="body", + ) + logger.error("Message action type %s is not implemented", action_type) + raise MessageDispatchException( + f"Message action type {action_type} is not implemented" + ) + + +def _build_email( # pylint: disable=too-many-return-statements + action_type: MessagingActionType, + body_params: Any, +) -> MessageForActionType: + if action_type == MessagingActionType.CONSENT_REQUEST: + template = get_email_template(action_type) + return MessageForActionType( + subject="Your one-time code", + body=template.render( + { + "code": body_params.verification_code, + "minutes": body_params.get_verification_code_ttl_minutes(), + } + ), + ) + if action_type == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION: + template = get_email_template(action_type) + return MessageForActionType( + subject="Your one-time code", + body=template.render( + { + "code": body_params.verification_code, + "minutes": body_params.get_verification_code_ttl_minutes(), + } + ), + ) + if action_type == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT: + base_template = get_email_template(action_type) + return MessageForActionType( + subject="Data erasure request", + body=base_template.render( + {"dataset_collection_action_required": body_params} + ), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_RECEIPT: + base_template = get_email_template(action_type) + return MessageForActionType( + subject="Your request has been received", + body=base_template.render({"request_types": body_params.request_types}), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: + base_template = get_email_template(action_type) + return MessageForActionType( + subject="Your data is ready to be downloaded", + body=base_template.render( + { + "download_links": body_params.download_links, + } + ), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION: + base_template = get_email_template(action_type) + return MessageForActionType( + subject="Your data has been deleted", + body=base_template.render(), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE: + base_template = get_email_template(action_type) + return MessageForActionType( + subject="Your request has been approved", + body=base_template.render(), + ) + if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY: + base_template = get_email_template(action_type) + return MessageForActionType( + subject="Your request has been denied", + body=base_template.render( + {"rejection_reason": body_params.rejection_reason} + ), + ) + logger.error("Message action type %s is not implemented", action_type) + raise MessageDispatchException( + f"Message action type {action_type} is not implemented" + ) + + +def _get_dispatcher_from_config_type(message_service_type: MessagingServiceType) -> Any: + """Determines which dispatcher to use based on message service type""" + return { + MessagingServiceType.MAILGUN.value: _mailgun_dispatcher, + }[message_service_type.value] + + +def _mailgun_dispatcher( + messaging_config: MessagingConfig, + message: MessageForActionType, + to_email: Optional[str], +) -> None: + """Dispatches email using mailgun""" + if not to_email: + logger.error("Message failed to send. No email identity supplied.") + raise MessageDispatchException("No email identity supplied.") + base_url = ( + "https://api.mailgun.net" + if messaging_config.details[MessagingServiceDetails.IS_EU_DOMAIN.value] is False + else "https://api.eu.mailgun.net" + ) + domain = messaging_config.details[MessagingServiceDetails.DOMAIN.value] + data = { + "from": f"", + "to": [to_email], + "subject": message.subject, + "html": message.body, + } + try: + response: requests.Response = requests.post( + f"{base_url}/{messaging_config.details[MessagingServiceDetails.API_VERSION.value]}/{domain}/messages", + auth=( + "api", + messaging_config.secrets[MessagingServiceSecrets.MAILGUN_API_KEY.value], # type: ignore + ), + data=data, + ) + if not response.ok: + logger.error( + "Email failed to send with status code: %s", response.status_code + ) + raise MessageDispatchException( + f"Email failed to send with status code {response.status_code}" + ) + except Exception as e: + logger.error("Email failed to send: %s", Pii(str(e))) + raise MessageDispatchException(f"Email failed to send due to: {Pii(e)}") diff --git a/src/fides/api/ops/service/messaging/messaging_crud_service.py b/src/fides/api/ops/service/messaging/messaging_crud_service.py new file mode 100644 index 00000000000..8509dccccc9 --- /dev/null +++ b/src/fides/api/ops/service/messaging/messaging_crud_service.py @@ -0,0 +1,73 @@ +import logging + +from fideslang.validation import FidesKey +from sqlalchemy.orm import Session + +from fides.api.ops.common_exceptions import MessagingConfigNotFoundException +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.schemas.messaging.messaging import ( + MessagingConfigRequest, + MessagingConfigResponse, +) + +logger = logging.getLogger(__name__) + + +def update_messaging_config( + db: Session, key: FidesKey, config: MessagingConfigRequest +) -> MessagingConfigResponse: + existing_config_with_key: MessagingConfig = MessagingConfig.get_by( + db=db, field="key", value=key + ) + if not existing_config_with_key: + raise MessagingConfigNotFoundException( + f"No messaging config found with key {key}" + ) + return create_or_update_messaging_config(db=db, config=config) + + +def create_or_update_messaging_config( + db: Session, config: MessagingConfigRequest +) -> MessagingConfigResponse: + messaging_config: MessagingConfig = MessagingConfig.create_or_update( + db=db, + data={ + "key": config.key, + "name": config.name, + "service_type": config.service_type, + "details": config.details.__dict__, + }, + ) + return MessagingConfigResponse( + name=messaging_config.name, + key=messaging_config.key, + service_type=messaging_config.service_type, + details=messaging_config.details, + ) + + +def delete_messaging_config(db: Session, key: FidesKey) -> None: + logger.info("Finding messaging config with key '%s'", key) + messaging_config: MessagingConfig = MessagingConfig.get_by( + db, field="key", value=key + ) + if not messaging_config: + raise MessagingConfigNotFoundException( + f"No messaging config found with key {key}" + ) + logger.info("Deleting messaging config with key '%s'", key) + messaging_config.delete(db) + + +def get_messaging_config_by_key(db: Session, key: FidesKey) -> MessagingConfigResponse: + config = MessagingConfig.get_by(db=db, field="key", value=key) + if not config: + raise MessagingConfigNotFoundException( + f"No messaging config found with key {key}" + ) + return MessagingConfigResponse( + name=config.name, + key=config.key, + service_type=config.service_type, + details=config.details, + ) diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py index b84efa65f3d..13c4df28b19 100644 --- a/src/fides/api/ops/service/privacy_request/request_runner_service.py +++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py @@ -14,9 +14,9 @@ from fides.api.ops import common_exceptions from fides.api.ops.common_exceptions import ( ClientUnsuccessfulException, - EmailDispatchException, IdentityNotFoundException, ManualWebhookFieldsUnset, + MessageDispatchException, NoCachedManualWebhookEntry, PrivacyRequestPaused, ) @@ -42,14 +42,15 @@ ProvidedIdentityType, can_run_checkpoint, ) -from fides.api.ops.schemas.email.email import ( +from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, - EmailActionType, + MessagingActionType, ) +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( email_connector_erasure_send, ) -from fides.api.ops.service.email.email_dispatch_service import dispatch_email +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message from fides.api.ops.service.storage.storage_uploader_service import upload from fides.api.ops.task.filter_results import filter_data_categories from fides.api.ops.task.graph_task import ( @@ -380,7 +381,7 @@ async def run_privacy_request( email_connector_erasure_send( db=session, privacy_request=privacy_request ) - except EmailDispatchException as exc: + except MessageDispatchException as exc: privacy_request.cache_failed_checkpoint_details( step=CurrentStep.erasure_email_post_send, collection=None ) @@ -405,7 +406,7 @@ async def run_privacy_request( initiate_privacy_request_completion_email( session, policy, access_result_urls, identity_data ) - except (IdentityNotFoundException, EmailDispatchException) as e: + except (IdentityNotFoundException, MessageDispatchException) as e: privacy_request.error_processing(db=session) # If dev mode, log traceback await fideslog_graph_failure( @@ -444,22 +445,28 @@ def initiate_privacy_request_completion_email( raise IdentityNotFoundException( "Identity email was not found, so request completion email could not be sent." ) + to_identity: Identity = Identity( + email=identity_data.get(ProvidedIdentityType.email.value), + phone_number=identity_data.get(ProvidedIdentityType.phone_number.value), + ) if policy.get_rules_for_action(action_type=ActionType.access): # synchronous for now since failure to send complete emails is fatal to request - dispatch_email( + dispatch_message( db=session, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, - to_email=identity_data.get(ProvidedIdentityType.email.value), - email_body_params=AccessRequestCompleteBodyParams( + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, + to_identity=to_identity, + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=AccessRequestCompleteBodyParams( download_links=access_result_urls ), ) if policy.get_rules_for_action(action_type=ActionType.erasure): - dispatch_email( + dispatch_message( db=session, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION, - to_email=identity_data.get(ProvidedIdentityType.email.value), - email_body_params=None, + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, + to_identity=to_identity, + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=None, ) diff --git a/src/fides/api/ops/tasks/__init__.py b/src/fides/api/ops/tasks/__init__.py index d583814fa2d..653aee0b446 100644 --- a/src/fides/api/ops/tasks/__init__.py +++ b/src/fides/api/ops/tasks/__init__.py @@ -11,7 +11,7 @@ logger = get_task_logger(__name__) CONFIG = get_config() -EMAIL_QUEUE_NAME = "fidesops.email" +MESSAGING_QUEUE_NAME = "fidesops.messaging" class DatabaseTask(Task): # pylint: disable=W0223 @@ -87,7 +87,7 @@ def start_worker() -> None: "worker", "--loglevel=info", "--concurrency=2", - f"--queues={default_queue_name},{EMAIL_QUEUE_NAME}", + f"--queues={default_queue_name},{MESSAGING_QUEUE_NAME}", ] ) diff --git a/src/fides/ctl/core/config/notification_settings.py b/src/fides/ctl/core/config/notification_settings.py index 4396ab0a135..4afa33550fd 100644 --- a/src/fides/ctl/core/config/notification_settings.py +++ b/src/fides/ctl/core/config/notification_settings.py @@ -1,4 +1,11 @@ import logging +from typing import Optional + +from fides.api.ops.schemas.messaging.messaging import ( + EMAIL_MESSAGING_SERVICES, + SMS_MESSAGING_SERVICES, + MessagingMethod, +) from .fides_settings import FidesSettings @@ -13,6 +20,15 @@ class NotificationSettings(FidesSettings): send_request_completion_notification: bool = False send_request_receipt_notification: bool = False send_request_review_notification: bool = False + notification_service_type: Optional[str] = None + + def get_messaging_method(self) -> Optional[MessagingMethod]: + """returns messaging method based on configured notification service type""" + if self in EMAIL_MESSAGING_SERVICES: + return MessagingMethod.EMAIL + if self in SMS_MESSAGING_SERVICES: + return MessagingMethod.SMS + return None class Config: env_prefix = ENV_PREFIX diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index 422a763c3f9..df9134aa882 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -27,8 +27,8 @@ ManualAction, PrivacyRequestStatus, ) -from fides.api.ops.schemas.email.email import EmailActionType -from fides.api.ops.tasks import EMAIL_QUEUE_NAME +from fides.api.ops.schemas.messaging.messaging import MessagingActionType +from fides.api.ops.tasks import MESSAGING_QUEUE_NAME page_size = Params().size @@ -1464,10 +1464,11 @@ def test_put_email_connection_config_secrets( assert mock_dispatch_email.called kwargs = mock_dispatch_email.call_args.kwargs assert ( - kwargs["action_type"] == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT + kwargs["action_type"] + == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT ) assert kwargs["to_email"] == "test@example.com" - assert kwargs["email_body_params"] == [ + assert kwargs["message_body_params"] == [ CheckpointActionRequired( step=CurrentStep.erasure, collection=CollectionAddress("test_dataset", "test_collection"), diff --git a/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py b/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py index dc72bec474b..1a57913df06 100644 --- a/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_identity_verification_endpoints.py @@ -37,7 +37,7 @@ def test_get_config_with_verification_required_with_email_config( url, db, api_client: TestClient, - email_config, + messaging_config, subject_identity_verification_required, ): resp = api_client.get(url) @@ -51,7 +51,7 @@ def test_get_config_with_verification_not_required_with_email_config( url, db, api_client: TestClient, - email_config, + messaging_config, ): resp = api_client.get(url) assert resp.status_code == 200 diff --git a/tests/ops/api/v1/endpoints/test_email_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py similarity index 63% rename from tests/ops/api/v1/endpoints/test_email_endpoints.py rename to tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 7aa893c6815..30792e3476c 100644 --- a/tests/ops/api/v1/endpoints/test_email_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -6,21 +6,21 @@ from starlette.testclient import TestClient from fides.api.ops.api.v1.scope_registry import ( - EMAIL_CREATE_OR_UPDATE, - EMAIL_DELETE, - EMAIL_READ, + MESSAGING_CREATE_OR_UPDATE, + MESSAGING_DELETE, + MESSAGING_READ, ) from fides.api.ops.api.v1.urn_registry import ( - EMAIL_BY_KEY, - EMAIL_CONFIG, - EMAIL_SECRETS, + MESSAGING_BY_KEY, + MESSAGING_CONFIG, + MESSAGING_SECRETS, V1_URL_PREFIX, ) -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.schemas.email.email import ( - EmailServiceDetails, - EmailServiceSecrets, - EmailServiceType, +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceDetails, + MessagingServiceSecrets, + MessagingServiceType, ) PAGE_SIZE = Params().size @@ -29,14 +29,14 @@ class TestPostEmailConfig: @pytest.fixture(scope="function") def url(self) -> str: - return V1_URL_PREFIX + EMAIL_CONFIG + return V1_URL_PREFIX + MESSAGING_CONFIG @pytest.fixture(scope="function") def payload(self): return { "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + "service_type": MessagingServiceType.MAILGUN.value, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, } def test_post_email_config_not_authenticated( @@ -52,7 +52,7 @@ def test_post_email_config_incorrect_scope( url, generate_auth_header, ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.post(url, headers=auth_header, json=payload) assert 403 == response.status_code @@ -66,7 +66,7 @@ def test_post_email_config_with_invalid_mailgun_details( ): payload["details"] = {"invalid": "invalid"} - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code assert json.loads(response.text)["detail"][0]["msg"] == "field required" @@ -85,7 +85,7 @@ def test_post_email_config_with_not_supported_service_type( ): payload["service_type"] = "twilio" - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code assert ( @@ -101,14 +101,14 @@ def test_post_email_config_with_no_key( url, generate_auth_header, ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 200 == response.status_code response_body = json.loads(response.text) assert response_body["key"] == "mailgun" - email_config = db.query(EmailConfig).filter_by(key="mailgun")[0] + email_config = db.query(MessagingConfig).filter_by(key="mailgun")[0] email_config.delete(db) def test_post_email_config_with_invalid_key( @@ -120,7 +120,7 @@ def test_post_email_config_with_invalid_key( generate_auth_header, ): payload["key"] = "*invalid-key" - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code assert ( @@ -136,23 +136,23 @@ def test_post_email_config_with_key( url, generate_auth_header, ): - payload["key"] = "my_email_config" - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + payload["key"] = "my_mailgun_messaging_config" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 200 == response.status_code response_body = json.loads(response.text) - email_config = db.query(EmailConfig).filter_by(key="my_email_config")[0] + email_config = db.query(MessagingConfig).filter_by(key="my_mailgun_messaging_config")[0] expected_response = { - "key": "my_email_config", + "key": "my_mailgun_messaging_config", "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, + "service_type": MessagingServiceType.MAILGUN.value, "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "my.mailgun.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, }, } assert expected_response == response_body @@ -164,14 +164,14 @@ def test_post_email_config_missing_detail( url, generate_auth_header, ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post( url, headers=auth_header, json={ - "key": "my_email_config", + "key": "my_mailgun_messaging_config", "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, + "service_type": MessagingServiceType.MAILGUN.value, "details": { # "domain": "my.mailgun.domain" }, @@ -186,38 +186,40 @@ def test_post_email_config_already_exists( self, api_client: TestClient, url, - email_config, + messaging_config, generate_auth_header, ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post( url, headers=auth_header, json={ - "key": "my_email_config", + "key": "my_mailgun_messaging_config", "name": "mailgun", - "service_type": EmailServiceType.MAILGUN.value, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + "service_type": MessagingServiceType.MAILGUN.value, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, }, ) assert response.status_code == 400 assert response.json() == { - "detail": "Only one email config is supported at a time. Config with key my_email_config is already configured." - } + "detail": "Only one messaging config is supported at a time. Config with key my_mailgun_messaging_config is already configured." + } # fixme- what's the error here? class TestPatchEmailConfig: @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=messaging_config.key + ) @pytest.fixture(scope="function") def payload(self): return { - "key": "my_email_config", + "key": "my_mailgun_messaging_config", "name": "mailgun new name", - "service_type": EmailServiceType.MAILGUN.value, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + "service_type": MessagingServiceType.MAILGUN.value, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, } def test_patch_email_config_not_authenticated( @@ -233,7 +235,7 @@ def test_patch_email_config_incorrect_scope( url, generate_auth_header, ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.patch(url, headers=auth_header, json=payload) assert 403 == response.status_code @@ -242,16 +244,16 @@ def test_patch_email_config_with_key_not_found( db: Session, api_client: TestClient, payload, - email_config, + messaging_config, generate_auth_header, ): - url = (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key="nonexistent_key") - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key="nonexistent_key") + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.patch(url, headers=auth_header, json=payload) assert response.status_code == 404 assert response.json() == { - "detail": "No email config found with key nonexistent_key" + "detail": "No messaging config found with key nonexistent_key" } def test_patch_email_config_with_key( @@ -262,22 +264,22 @@ def test_patch_email_config_with_key( url, generate_auth_header, ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.patch(url, headers=auth_header, json=payload) assert 200 == response.status_code response_body = json.loads(response.text) - email_config = db.query(EmailConfig).filter_by(key="my_email_config")[0] + email_config = db.query(MessagingConfig).filter_by(key="my_mailgun_messaging_config")[0] expected_response = { - "key": "my_email_config", + "key": "my_mailgun_messaging_config", "name": "mailgun new name", - "service_type": EmailServiceType.MAILGUN.value, + "service_type": MessagingServiceType.MAILGUN.value, "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "my.mailgun.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, }, } assert expected_response == response_body @@ -286,13 +288,15 @@ def test_patch_email_config_with_key( class TestPutEmailConfigSecretsMailgun: @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_SECRETS).format(config_key=email_config.key) + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config.key + ) @pytest.fixture(scope="function") def payload(self): return { - EmailServiceSecrets.MAILGUN_API_KEY.value: "1345234524", + MessagingServiceSecrets.MAILGUN_API_KEY.value: "1345234524", } def test_put_config_secrets_unauthenticated( @@ -304,22 +308,22 @@ def test_put_config_secrets_unauthenticated( def test_put_config_secrets_wrong_scope( self, api_client: TestClient, payload, url, generate_auth_header ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.put(url, headers=auth_header, json=payload) assert 403 == response.status_code def test_put_config_secret_invalid_config( self, api_client: TestClient, payload, generate_auth_header ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) - url = (V1_URL_PREFIX + EMAIL_SECRETS).format(config_key="invalid_key") + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + url = (V1_URL_PREFIX + MESSAGING_SECRETS).format(config_key="invalid_key") response = api_client.put(url, headers=auth_header, json=payload) assert 404 == response.status_code def test_update_with_invalid_secrets_key( self, api_client: TestClient, generate_auth_header, url ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json={"bad_key": "12345"}) assert response.status_code == 400 @@ -337,21 +341,21 @@ def test_put_config_secrets( payload, url, generate_auth_header, - email_config, + messaging_config, ): - auth_header = generate_auth_header([EMAIL_CREATE_OR_UPDATE]) + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) assert 200 == response.status_code - db.refresh(email_config) + db.refresh(messaging_config) assert json.loads(response.text) == { - "msg": "Secrets updated for EmailConfig with key: my_email_config.", + "msg": "Secrets updated for MessagingConfig with key: my_mailgun_messaging_config.", "test_status": None, "failure_reason": None, } assert ( - email_config.secrets[EmailServiceSecrets.MAILGUN_API_KEY.value] + messaging_config.secrets[MessagingServiceSecrets.MAILGUN_API_KEY.value] == "1345234524" ) @@ -359,7 +363,7 @@ def test_put_config_secrets( class TestGetEmailConfigs: @pytest.fixture(scope="function") def url(self) -> str: - return V1_URL_PREFIX + EMAIL_CONFIG + return V1_URL_PREFIX + MESSAGING_CONFIG def test_get_configs_not_authenticated(self, api_client: TestClient, url) -> None: response = api_client.get(url) @@ -368,27 +372,27 @@ def test_get_configs_not_authenticated(self, api_client: TestClient, url) -> Non def test_get_configs_wrong_scope( self, api_client: TestClient, url, generate_auth_header ) -> None: - auth_header = generate_auth_header([EMAIL_DELETE]) + auth_header = generate_auth_header([MESSAGING_DELETE]) response = api_client.get(url, headers=auth_header) assert 403 == response.status_code def test_get_configs( - self, db, api_client: TestClient, url, generate_auth_header, email_config + self, db, api_client: TestClient, url, generate_auth_header, messaging_config ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.get(url, headers=auth_header) assert 200 == response.status_code expected_response = { "items": [ { - "key": "my_email_config", - "name": email_config.name, - "service_type": EmailServiceType.MAILGUN.value, + "key": "my_mailgun_messaging_config", + "name": messaging_config.name, + "service_type": MessagingServiceType.MAILGUN.value, "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "some.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, }, } ], @@ -402,8 +406,10 @@ def test_get_configs( class TestGetEmailConfig: @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=messaging_config.key + ) def test_get_config_not_authenticated(self, url, api_client: TestClient): response = api_client.get(url) @@ -412,45 +418,47 @@ def test_get_config_not_authenticated(self, url, api_client: TestClient): def test_get_config_wrong_scope( self, url, api_client: TestClient, generate_auth_header ): - auth_header = generate_auth_header([EMAIL_DELETE]) + auth_header = generate_auth_header([MESSAGING_DELETE]) response = api_client.get(url, headers=auth_header) assert 403 == response.status_code def test_get_config_invalid( - self, api_client: TestClient, generate_auth_header, email_config + self, api_client: TestClient, generate_auth_header, messaging_config ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.get( - (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key="invalid"), + (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key="invalid"), headers=auth_header, ) assert 404 == response.status_code def test_get_config( - self, url, api_client: TestClient, generate_auth_header, email_config + self, url, api_client: TestClient, generate_auth_header, messaging_config ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.get(url, headers=auth_header) assert response.status_code == 200 response_body = json.loads(response.text) assert response_body == { - "key": "my_email_config", - "name": email_config.name, - "service_type": EmailServiceType.MAILGUN.value, + "key": "my_mailgun_messaging_config", + "name": messaging_config.name, + "service_type": MessagingServiceType.MAILGUN.value, "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "some.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, }, } class TestDeleteConfig: @pytest.fixture(scope="function") - def url(self, email_config) -> str: - return (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=messaging_config.key + ) def test_delete_config_not_authenticated(self, url, api_client: TestClient): response = api_client.delete(url) @@ -459,14 +467,14 @@ def test_delete_config_not_authenticated(self, url, api_client: TestClient): def test_delete_config_wrong_scope( self, url, api_client: TestClient, generate_auth_header ): - auth_header = generate_auth_header([EMAIL_READ]) + auth_header = generate_auth_header([MESSAGING_READ]) response = api_client.delete(url, headers=auth_header) assert 403 == response.status_code def test_delete_config_invalid(self, api_client: TestClient, generate_auth_header): - auth_header = generate_auth_header([EMAIL_DELETE]) + auth_header = generate_auth_header([MESSAGING_DELETE]) response = api_client.delete( - (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key="invalid"), + (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key="invalid"), headers=auth_header, ) assert 404 == response.status_code @@ -479,20 +487,20 @@ def test_delete_config( generate_auth_header, ): # Creating new config, so we don't run into issues trying to clean up a deleted fixture - email_config = EmailConfig.create( + email_config = MessagingConfig.create( db=db, data={ "key": "my_different_email_config", "name": "mailgun", - "service_type": EmailServiceType.MAILGUN, - "details": {EmailServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + "service_type": MessagingServiceType.MAILGUN, + "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, }, ) - url = (V1_URL_PREFIX + EMAIL_BY_KEY).format(config_key=email_config.key) - auth_header = generate_auth_header([EMAIL_DELETE]) + url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key=email_config.key) + auth_header = generate_auth_header([MESSAGING_DELETE]) response = api_client.delete(url, headers=auth_header) assert response.status_code == 204 db.expunge_all() - config = db.query(EmailConfig).filter_by(key=email_config.key).first() + config = db.query(MessagingConfig).filter_by(key=email_config.key).first() assert config is None diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index e7d31e07669..56bb986ba2f 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -62,16 +62,16 @@ PrivacyRequestStatus, ) from fides.api.ops.schemas.dataset import DryRunDatasetResponse -from fides.api.ops.schemas.email.email import ( - EmailActionType, +from fides.api.ops.schemas.masking.masking_secrets import SecretType +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, RequestReceiptBodyParams, RequestReviewDenyBodyParams, SubjectIdentityVerificationBodyParams, ) -from fides.api.ops.schemas.masking.masking_secrets import SecretType from fides.api.ops.schemas.policy import PolicyResponse from fides.api.ops.schemas.redis_cache import Identity -from fides.api.ops.tasks import EMAIL_QUEUE_NAME +from fides.api.ops.tasks import MESSAGING_QUEUE_NAME from fides.api.ops.util.cache import ( get_encryption_cache_key, get_identity_cache_key, @@ -96,7 +96,7 @@ def url(self, oauth_client: ClientDetail, policy) -> str: "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_create_privacy_request( self, @@ -1801,7 +1801,7 @@ def test_approve_privacy_request_no_user_on_client( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_approve_privacy_request( self, @@ -1849,7 +1849,7 @@ def test_approve_privacy_request( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_approve_privacy_request_creates_audit_log_and_sends_email( self, @@ -1893,13 +1893,14 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] + message_meta = task_kwargs["message_meta"] assert ( - email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_REVIEW_APPROVE + message_meta["action_type"] + == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE ) - assert email_meta["body_params"] is None + assert message_meta["body_params"] is None queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME class TestDenyPrivacyRequest: @@ -1972,7 +1973,7 @@ def test_deny_completed_privacy_request( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_deny_privacy_request_without_denial_reason( self, @@ -2021,13 +2022,16 @@ def test_deny_privacy_request_without_denial_reason( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY - assert email_meta["body_params"] == RequestReviewDenyBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] + == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY + ) + assert message_meta["body_params"] == RequestReviewDenyBodyParams( rejection_reason=None ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME assert denial_audit_log.message is None @@ -2039,7 +2043,7 @@ def test_deny_privacy_request_without_denial_reason( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_deny_privacy_request_with_denial_reason( self, @@ -2088,13 +2092,16 @@ def test_deny_privacy_request_with_denial_reason( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_REVIEW_DENY - assert email_meta["body_params"] == RequestReviewDenyBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] + == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY + ) + assert message_meta["body_params"] == RequestReviewDenyBodyParams( rejection_reason=denial_reason ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME assert denial_audit_log.message == denial_reason @@ -2866,7 +2873,7 @@ def test_incorrect_privacy_request_status(self, api_client, url, privacy_request ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verification_code_expired( self, @@ -2890,7 +2897,7 @@ def test_verification_code_expired( assert not mock_dispatch_email.called @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_invalid_code( self, @@ -2918,7 +2925,7 @@ def test_invalid_code( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verify_identity_no_admin_approval_needed( self, @@ -2964,19 +2971,21 @@ def test_verify_identity_no_admin_approval_needed( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verify_identity_no_admin_approval_needed_email_disabled( self, @@ -3021,7 +3030,7 @@ def test_verify_identity_no_admin_approval_needed_email_disabled( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_verify_identity_admin_approval_needed( self, @@ -3067,13 +3076,15 @@ def test_verify_identity_admin_approval_needed( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME class TestCreatePrivacyRequestEmailVerificationRequired: @@ -3130,7 +3141,7 @@ def test_create_privacy_request_with_email_config( db, api_client: TestClient, policy, - email_config, + messaging_config, subject_identity_verification_required, ): data = [ @@ -3159,9 +3170,11 @@ def test_create_privacy_request_with_email_config( assert mock_dispatch_email.called kwargs = mock_dispatch_email.call_args.kwargs - assert kwargs["action_type"] == EmailActionType.SUBJECT_IDENTITY_VERIFICATION + assert ( + kwargs["action_type"] == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION + ) assert kwargs["to_email"] == "test@example.com" - assert kwargs["email_body_params"] == SubjectIdentityVerificationBodyParams( + assert kwargs["message_body_params"] == SubjectIdentityVerificationBodyParams( verification_code=pr.get_cached_verification_code(), verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, ) @@ -3644,7 +3657,7 @@ def privacy_request_receipt_email_notification_enabled(self): "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_create_privacy_request_no_email_config( self, @@ -3678,13 +3691,15 @@ def test_create_privacy_request_no_email_config( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME pr.delete(db=db) @@ -3692,7 +3707,7 @@ def test_create_privacy_request_no_email_config( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @mock.patch( - "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_email_task.apply_async" + "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_create_privacy_request_with_email_config( self, @@ -3702,7 +3717,7 @@ def test_create_privacy_request_with_email_config( db, api_client: TestClient, policy, - email_config, + messaging_config, privacy_request_receipt_email_notification_enabled, ): data = [ @@ -3726,12 +3741,14 @@ def test_create_privacy_request_with_email_config( task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" - email_meta = task_kwargs["email_meta"] - assert email_meta["action_type"] == EmailActionType.PRIVACY_REQUEST_RECEIPT - assert email_meta["body_params"] == RequestReceiptBodyParams( + message_meta = task_kwargs["message_meta"] + assert ( + message_meta["action_type"] == MessagingActionType.PRIVACY_REQUEST_RECEIPT + ) + assert message_meta["body_params"] == RequestReceiptBodyParams( request_types={ActionType.access.value} ) queue = call_args["queue"] - assert queue == EMAIL_QUEUE_NAME + assert queue == MESSAGING_QUEUE_NAME pr.delete(db=db) diff --git a/tests/ops/email_templates/test_get_email_template.py b/tests/ops/email_templates/test_get_email_template.py index 318d7a28aca..d9ea4c112fa 100644 --- a/tests/ops/email_templates/test_get_email_template.py +++ b/tests/ops/email_templates/test_get_email_template.py @@ -3,11 +3,11 @@ from fides.api.ops.common_exceptions import EmailTemplateUnhandledActionType from fides.api.ops.email_templates import get_email_template -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import MessagingActionType def test_get_email_template_returns_template(): - result = get_email_template(EmailActionType.SUBJECT_IDENTITY_VERIFICATION) + result = get_email_template(MessagingActionType.SUBJECT_IDENTITY_VERIFICATION) assert type(result) == Template diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 9ca5fc9f414..7f821792356 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -23,7 +23,7 @@ ConnectionType, ) from fides.api.ops.models.datasetconfig import DatasetConfig -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import ( ActionType, Policy, @@ -35,10 +35,10 @@ from fides.api.ops.models.privacy_request import PrivacyRequest, PrivacyRequestStatus from fides.api.ops.models.registration import UserRegistration from fides.api.ops.models.storage import ResponseFormat, StorageConfig -from fides.api.ops.schemas.email.email import ( - EmailServiceDetails, - EmailServiceSecrets, - EmailServiceType, +from fides.api.ops.schemas.messaging.messaging import ( + MessagingServiceDetails, + MessagingServiceSecrets, + MessagingServiceType, ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.schemas.storage.storage import ( @@ -153,26 +153,29 @@ def storage_config(db: Session) -> Generator: @pytest.fixture(scope="function") -def email_config(db: Session) -> Generator: +def messaging_config(db: Session) -> Generator: name = str(uuid4()) - email_config = EmailConfig.create( + messaging_config = MessagingConfig.create( db=db, data={ "name": name, - "key": "my_email_config", - "service_type": EmailServiceType.MAILGUN, + "key": "my_mailgun_messaging_config", + "service_type": MessagingServiceType.MAILGUN, "details": { - EmailServiceDetails.API_VERSION.value: "v3", - EmailServiceDetails.DOMAIN.value: "some.domain", - EmailServiceDetails.IS_EU_DOMAIN.value: False, + MessagingServiceDetails.API_VERSION.value: "v3", + MessagingServiceDetails.DOMAIN.value: "some.domain", + MessagingServiceDetails.IS_EU_DOMAIN.value: False, }, }, ) - email_config.set_secrets( - db=db, email_secrets={EmailServiceSecrets.MAILGUN_API_KEY.value: "12984r70298r"} + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.MAILGUN_API_KEY.value: "12984r70298r" + }, ) - yield email_config - email_config.delete(db) + yield messaging_config + messaging_config.delete(db) @pytest.fixture(scope="function") diff --git a/tests/ops/integration_tests/test_integration_email.py b/tests/ops/integration_tests/test_integration_email.py index 70ebd7bd72e..ad970dbc83b 100644 --- a/tests/ops/integration_tests/test_integration_email.py +++ b/tests/ops/integration_tests/test_integration_email.py @@ -14,7 +14,7 @@ ManualAction, ) from fides.api.ops.schemas.dataset import FidesopsDataset -from fides.api.ops.schemas.email.email import EmailActionType +from fides.api.ops.schemas.messaging.messaging import MessagingActionType from fides.api.ops.service.connectors.email_connector import ( email_connector_erasure_send, ) @@ -34,7 +34,7 @@ async def test_email_connector_cache_and_delayed_send( privacy_request, example_datasets, email_dataset_config, - email_config, + messaging_config, ) -> None: """Run an erasure privacy request with a postgres dataset and an email dataset. The email dataset has three separate collections. @@ -178,9 +178,12 @@ async def test_email_connector_cache_and_delayed_send( email_connector_erasure_send(db, privacy_request) assert mock_email_dispatch.called call_args = mock_email_dispatch.call_args[1] - assert call_args["action_type"] == EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT + assert ( + call_args["action_type"] + == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT + ) assert call_args["to_email"] == "test@example.com" - assert call_args["email_body_params"] == raw_email_template_values + assert call_args["message_body_params"] == raw_email_template_values created_email_audit_log = ( db.query(AuditLog) diff --git a/tests/ops/schemas/email/email_test.py b/tests/ops/schemas/messaging/messaging_test.py similarity index 77% rename from tests/ops/schemas/email/email_test.py rename to tests/ops/schemas/messaging/messaging_test.py index e3caea5f3bc..65c31b2ea2f 100644 --- a/tests/ops/schemas/email/email_test.py +++ b/tests/ops/schemas/messaging/messaging_test.py @@ -1,6 +1,8 @@ import pytest -from fides.api.ops.schemas.email.email import SubjectIdentityVerificationBodyParams +from fides.api.ops.schemas.messaging.messaging import ( + SubjectIdentityVerificationBodyParams, +) @pytest.mark.parametrize("ttl, expected", [(600, 10), (155, 2), (33, 0)]) diff --git a/tests/ops/service/email/email_dispatch_service_test.py b/tests/ops/service/email/email_dispatch_service_test.py deleted file mode 100644 index 0b0c86c8dc7..00000000000 --- a/tests/ops/service/email/email_dispatch_service_test.py +++ /dev/null @@ -1,159 +0,0 @@ -from unittest import mock -from unittest.mock import Mock - -import pytest -import requests.exceptions -import requests_mock -from sqlalchemy.orm import Session - -from fides.api.ops.common_exceptions import EmailDispatchException -from fides.api.ops.graph.config import CollectionAddress -from fides.api.ops.models.email import EmailConfig -from fides.api.ops.models.policy import CurrentStep -from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction -from fides.api.ops.schemas.email.email import ( - EmailActionType, - EmailForActionType, - EmailServiceDetails, - EmailServiceType, - FidesopsEmail, - SubjectIdentityVerificationBodyParams, -) -from fides.api.ops.service.email.email_dispatch_service import dispatch_email - - -@mock.patch("fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher") -def test_email_dispatch_mailgun_success( - mock_mailgun_dispatcher: Mock, db: Session, email_config -) -> None: - - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - body = '\n\n\n \n ID Code\n\n\n
\n

\n Your privacy request verification code is 2348.\n Please return to the Privacy Center and enter the code to\n continue. This code will expire in 10 minutes\n

\n
\n\n' - mock_mailgun_dispatcher.assert_called_with( - email_config=email_config, - email=EmailForActionType( - subject="Your one-time code", - body=body, - ), - to_email="test@email.com", - ) - - -@mock.patch("fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher") -def test_email_dispatch_mailgun_config_not_found( - mock_mailgun_dispatcher: Mock, db: Session -) -> None: - - with pytest.raises(EmailDispatchException) as exc: - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - assert exc.value.args[0] == "No email config found." - - mock_mailgun_dispatcher.assert_not_called() - - -@mock.patch("fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher") -def test_email_dispatch_mailgun_config_no_secrets( - mock_mailgun_dispatcher: Mock, db: Session -) -> None: - - email_config = EmailConfig.create( - db=db, - data={ - "name": "mailgun config", - "key": "my_email_config", - "service_type": EmailServiceType.MAILGUN, - "details": { - EmailServiceDetails.DOMAIN.value: "some.domain", - }, - }, - ) - - with pytest.raises(EmailDispatchException) as exc: - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - assert ( - exc.value.args[0] - == "Email secrets not found for config with key: my_email_config" - ) - - mock_mailgun_dispatcher.assert_not_called() - - email_config.delete(db) - - -def test_email_dispatch_mailgun_failed_email(db: Session, email_config) -> None: - with requests_mock.Mocker() as mock_response: - mock_response.post( - f"https://api.mailgun.net/{email_config.details[EmailServiceDetails.API_VERSION.value]}/{email_config.details[EmailServiceDetails.DOMAIN.value]}/messages", - json={ - "message": "Rejected: IP can’t be used to send the message", - "id": "<20111114174239.25659.5817@samples.mailgun.org>", - }, - status_code=403, - ) - with pytest.raises(EmailDispatchException) as exc: - dispatch_email( - db=db, - action_type=EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - to_email="test@email.com", - email_body_params=SubjectIdentityVerificationBodyParams( - verification_code="2348", verification_code_ttl_seconds=600 - ), - ) - assert ( - exc.value.args[0] - == "Email failed to send due to: Email failed to send with status code 403" - ) - - -def test_fidesops_email_parse_object(): - body = [ - CheckpointActionRequired( - step=CurrentStep.erasure, - collection=CollectionAddress("email_dataset", "test_collection"), - action_needed=[ - ManualAction( - locators={"email": "test@example.com"}, - get=None, - update={"phone": "null_rewrite"}, - ) - ], - ) - ] - - FidesopsEmail.parse_obj( - { - "action_type": EmailActionType.EMAIL_ERASURE_REQUEST_FULFILLMENT, - "body_params": [action.dict() for action in body], - } - ) - - FidesopsEmail.parse_obj( - { - "action_type": EmailActionType.SUBJECT_IDENTITY_VERIFICATION, - "body_params": { - "verification_code": "123456", - "verification_code_ttl_seconds": 1000, - }, - } - ) diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py new file mode 100644 index 00000000000..7fc0ad60b37 --- /dev/null +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -0,0 +1,173 @@ +from unittest import mock +from unittest.mock import Mock + +import pytest +import requests.exceptions +import requests_mock +from sqlalchemy.orm import Session + +from fides.api.ops.common_exceptions import MessageDispatchException +from fides.api.ops.graph.config import CollectionAddress +from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.models.policy import CurrentStep +from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction +from fides.api.ops.schemas.messaging.messaging import ( + FidesopsMessage, + MessageForActionType, + MessagingActionType, + MessagingServiceDetails, + MessagingServiceType, + SubjectIdentityVerificationBodyParams, +) +from fides.api.ops.schemas.redis_cache import Identity +from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message +from fides.ctl.core.config import get_config + +CONFIG = get_config() + + +@mock.patch( + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" +) +def test_email_dispatch_mailgun_success( + mock_mailgun_dispatcher: Mock, db: Session, messaging_config +) -> None: + + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + body = '\n\n\n \n ID Code\n\n\n
\n

\n Your privacy request verification code is 2348.\n Please return to the Privacy Center and enter the code to\n continue. This code will expire in 10 minutes\n

\n
\n\n' + mock_mailgun_dispatcher.assert_called_with( + messaging_config=messaging_config, + message=MessageForActionType( + subject="Your one-time code", + body=body, + ), + to_email="test@email.com", + ) + + +@mock.patch( + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" +) +def test_email_dispatch_mailgun_config_not_found( + mock_mailgun_dispatcher: Mock, db: Session +) -> None: + + with pytest.raises(MessageDispatchException) as exc: + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + assert exc.value.args[0] == "No messaging config found." + + mock_mailgun_dispatcher.assert_not_called() + + +@mock.patch( + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" +) +def test_email_dispatch_mailgun_config_no_secrets( + mock_mailgun_dispatcher: Mock, db: Session +) -> None: + + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": "mailgun config", + "key": "my_mailgun_messaging_config", + "service_type": MessagingServiceType.MAILGUN, + "details": { + MessagingServiceDetails.DOMAIN.value: "some.domain", + }, + }, + ) + + with pytest.raises(MessageDispatchException) as exc: + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + assert ( + exc.value.args[0] + == "Messaging secrets not found for config with key: my_mailgun_messaging_config" + ) + + mock_mailgun_dispatcher.assert_not_called() + + messaging_config.delete(db) + + +def test_email_dispatch_mailgun_failed_email(db: Session, messaging_config) -> None: + with requests_mock.Mocker() as mock_response: + mock_response.post( + f"https://api.mailgun.net/{messaging_config.details[MessagingServiceDetails.API_VERSION.value]}/{messaging_config.details[MessagingServiceDetails.DOMAIN.value]}/messages", + json={ + "message": "Rejected: IP can’t be used to send the message", + "id": "<20111114174239.25659.5817@samples.mailgun.org>", + }, + status_code=403, + ) + with pytest.raises(MessageDispatchException) as exc: + dispatch_message( + db=db, + action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + to_identity=Identity(**{"email": "test@email.com"}), + messaging_method=CONFIG.notifications.get_messaging_method(), + message_body_params=SubjectIdentityVerificationBodyParams( + verification_code="2348", verification_code_ttl_seconds=600 + ), + ) + assert ( + exc.value.args[0] + == "Email failed to send due to: Email failed to send with status code 403" + ) + + +def test_fidesops_email_parse_object(): + body = [ + CheckpointActionRequired( + step=CurrentStep.erasure, + collection=CollectionAddress("email_dataset", "test_collection"), + action_needed=[ + ManualAction( + locators={"email": "test@example.com"}, + get=None, + update={"phone": "null_rewrite"}, + ) + ], + ) + ] + + FidesopsMessage.parse_obj( + { + "action_type": MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, + "body_params": [action.dict() for action in body], + } + ) + + FidesopsMessage.parse_obj( + { + "action_type": MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, + "body_params": { + "verification_code": "123456", + "verification_code_ttl_seconds": 1000, + }, + } + ) diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 236ea6ff13f..b6ea6ddd6dd 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -16,7 +16,7 @@ PrivacyRequestPaused, ) from fides.api.ops.models.connectionconfig import AccessLevel -from fides.api.ops.models.email import EmailConfig +from fides.api.ops.models.messaging import MessagingConfig from fides.api.ops.models.policy import CurrentStep, PolicyPostWebhook from fides.api.ops.models.privacy_request import ( ActionType, @@ -26,17 +26,17 @@ PrivacyRequest, PrivacyRequestStatus, ) -from fides.api.ops.schemas.email.email import ( - AccessRequestCompleteBodyParams, - EmailActionType, - EmailForActionType, -) from fides.api.ops.schemas.external_https import SecondPartyResponseFormat from fides.api.ops.schemas.masking.masking_configuration import ( HmacMaskingConfiguration, MaskingConfiguration, ) from fides.api.ops.schemas.masking.masking_secrets import MaskingSecretCache +from fides.api.ops.schemas.messaging.messaging import ( + AccessRequestCompleteBodyParams, + MessageForActionType, + MessagingActionType, +) from fides.api.ops.schemas.policy import Rule from fides.api.ops.schemas.saas.saas_config import SaaSRequest from fides.api.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams @@ -1637,7 +1637,7 @@ def test_privacy_request_log_failure( class TestPrivacyRequestsEmailConnector: @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_create_and_process_erasure_request_email_connector( @@ -1649,7 +1649,7 @@ def test_create_and_process_erasure_request_email_connector( run_privacy_request_task, email_dataset_config, postgres_example_test_dataset_config_read_access, - email_config, + messaging_config, db, ): """ @@ -1676,11 +1676,11 @@ def test_create_and_process_erasure_request_email_connector( pr.delete(db=db) assert mailgun_send.called kwargs = mailgun_send.call_args.kwargs - assert type(kwargs["email_config"]) == EmailConfig - assert type(kwargs["email"]) == EmailForActionType + assert type(kwargs["messaging_config"]) == MessagingConfig + assert type(kwargs["email"]) == MessageForActionType @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_create_and_process_erasure_request_email_connector_email_send_error( @@ -1695,7 +1695,7 @@ def test_create_and_process_erasure_request_email_connector_email_send_error( db, ): """ - Force error by having no email config setup + Force error by having no messaging config setup """ rule = erasure_policy.rules[0] target = rule.targets[0] @@ -1734,7 +1734,7 @@ def test_create_and_process_erasure_request_email_connector_email_send_error( assert mailgun_send.called is False @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_email_connector_read_only_permissions( @@ -1745,12 +1745,12 @@ def test_email_connector_read_only_permissions( integration_postgres_config, run_privacy_request_task, email_dataset_config, - email_config, + messaging_config, postgres_example_test_dataset_config_read_access, db, ): """ - Set email config to read only - don't send email in this case. + Set messaging config to read only - don't send message in this case. """ rule = erasure_policy.rules[0] target = rule.targets[0] @@ -1788,7 +1788,7 @@ def test_email_connector_read_only_permissions( ), "Email not sent because the connection was read only" @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @pytest.mark.integration def test_email_connector_no_updates_needed( @@ -1799,7 +1799,7 @@ def test_email_connector_no_updates_needed( integration_postgres_config, run_privacy_request_task, email_dataset_config, - email_config, + messaging_config, postgres_example_test_dataset_config_read_access, db, ): @@ -1867,7 +1867,7 @@ def test_email_complete_send_erasure( generate_auth_header, erasure_policy, read_connection_config, - email_config, + messaging_config, privacy_request_complete_email_notification_enabled, run_privacy_request_task, ): @@ -1905,7 +1905,7 @@ def test_email_complete_send_access( generate_auth_header, policy, read_connection_config, - email_config, + messaging_config, privacy_request_complete_email_notification_enabled, run_privacy_request_task, ): @@ -1944,7 +1944,7 @@ def test_email_complete_send_access_and_erasure( generate_auth_header, access_and_erasure_policy, read_connection_config, - email_config, + messaging_config, privacy_request_complete_email_notification_enabled, run_privacy_request_task, ): @@ -1968,17 +1968,17 @@ def test_email_complete_send_access_and_erasure( [ call( db=ANY, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, to_email=customer_email, - email_body_params=AccessRequestCompleteBodyParams( + message_body_params=AccessRequestCompleteBodyParams( download_links=[upload_mock.return_value] ), ), call( db=ANY, - action_type=EmailActionType.PRIVACY_REQUEST_COMPLETE_DELETION, + action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, to_email=customer_email, - email_body_params=None, + message_body_params=None, ), ], any_order=True, @@ -1987,10 +1987,10 @@ def test_email_complete_send_access_and_erasure( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") - def test_email_complete_send_access_no_email_config( + def test_email_complete_send_access_no_messaging_config( self, upload_mock, mailgun_send, @@ -2027,7 +2027,7 @@ def test_email_complete_send_access_no_email_config( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.email.email_dispatch_service._mailgun_dispatcher" + "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_email_complete_send_access_no_email_identity( From b9dd6d9a15b3848792de27d1ddd3fc8654e58d28 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Thu, 27 Oct 2022 18:04:29 -0500 Subject: [PATCH 02/28] make messaging details nullable --- .../2d5ff3096959_update_table_for_twilio.py | 2 +- src/fides/api/ops/api/v1/api.py | 2 +- src/fides/api/ops/models/messaging.py | 2 +- .../api/ops/schemas/messaging/messaging.py | 13 +++++--- src/fides/api/ops/service/_verification.py | 4 +-- .../messaging/message_dispatch_service.py | 7 ++++ .../privacy_request/request_runner_service.py | 6 ++-- .../ctl/core/config/notification_settings.py | 32 ++++++++++++------- .../message_dispatch_service_test.py | 8 ++--- 9 files changed, 47 insertions(+), 29 deletions(-) diff --git a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py index cf43bffd70d..962efbd0959 100644 --- a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py @@ -23,7 +23,7 @@ def upgrade(): sa.Column('key', sa.String(), nullable=False), sa.Column('name', sa.String(), nullable=True), sa.Column('service_type', sa.Enum('MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL', name='messagingservicetype'), nullable=False), - sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), nullable=False), + sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), nullable=True), sa.Column('secrets', sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), nullable=True), sa.PrimaryKeyConstraint('id') ) diff --git a/src/fides/api/ops/api/v1/api.py b/src/fides/api/ops/api/v1/api.py index 99c76ff90be..f87d8abf337 100644 --- a/src/fides/api/ops/api/v1/api.py +++ b/src/fides/api/ops/api/v1/api.py @@ -37,7 +37,7 @@ api_router.include_router(privacy_request_endpoints.router) api_router.include_router(identity_verification_endpoints.router) api_router.include_router(storage_endpoints.router) -api_router.include_router(email_endpoints.router) +api_router.include_router(messaging_endpoints.router) api_router.include_router(saas_config_endpoints.router) api_router.include_router(user_endpoints.router) api_router.include_router(user_permission_endpoints.router) diff --git a/src/fides/api/ops/models/messaging.py b/src/fides/api/ops/models/messaging.py index 8f27d95667a..cb0b738a578 100644 --- a/src/fides/api/ops/models/messaging.py +++ b/src/fides/api/ops/models/messaging.py @@ -63,7 +63,7 @@ class MessagingConfig(Base): service_type = Column( Enum(MessagingServiceType), index=True, unique=True, nullable=False ) - details = Column(MutableDict.as_mutable(JSONB), nullable=False) + details = Column(MutableDict.as_mutable(JSONB), nullable=True) secrets = Column( MutableDict.as_mutable( StringEncryptedType( diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index 388437c8e3d..fc7c671febd 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -23,6 +23,14 @@ class MessagingServiceType(Enum): TWILIO_TEXT = "twilio_text" TWILIO_EMAIL = "twilio_email" + def get_messaging_method(self) -> Optional[MessagingMethod]: + """returns messaging method based on configured service type""" + if self in EMAIL_MESSAGING_SERVICES: + return MessagingMethod.EMAIL + if self in SMS_MESSAGING_SERVICES: + return MessagingMethod.SMS + return None + EMAIL_MESSAGING_SERVICES = [ MessagingServiceType.MAILGUN, @@ -110,11 +118,6 @@ class MessagingServiceDetails(Enum): API_VERSION = "api_version" DOMAIN = "domain" - # twilio - TWILIO_ACCOUNT_SID = "twilio_account_sid" - TWILIO_AUTH_TOKEN = "twilio_auth_token" - TWILIO_MESSAGING_SERVICE_ID = "twilio_messaging_service_id" - class MessagingServiceDetailsMailgun(BaseModel): """The details required to represent a Mailgun email configuration.""" diff --git a/src/fides/api/ops/service/_verification.py b/src/fides/api/ops/service/_verification.py index 7193b21f1c4..4a8bddb13e6 100644 --- a/src/fides/api/ops/service/_verification.py +++ b/src/fides/api/ops/service/_verification.py @@ -6,7 +6,7 @@ from fides.api.ops.models.privacy_request import ConsentRequest, PrivacyRequest from fides.api.ops.schemas.messaging.messaging import ( MessagingActionType, - SubjectIdentityVerificationBodyParams, + SubjectIdentityVerificationBodyParams, MessagingServiceType, ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message @@ -36,7 +36,7 @@ def send_verification_code_to_user( db, action_type=messaging_action_type, to_identity=to_identity, - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=SubjectIdentityVerificationBodyParams( verification_code=verification_code, verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index 98d54f31877..b69947dad4b 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -106,6 +106,13 @@ def dispatch_message( dispatcher: Any = _get_dispatcher_from_config_type( message_service_type=messaging_service ) + if not dispatcher: + logger.error( + "Dispatcher has not been implemented for message service type: %s", messaging_service + ) + raise MessageDispatchException( + f"Dispatcher has not been implemented for message service type: {messaging_service}" + ) logger.info( "Starting email dispatch for messaging service with action type: %s", action_type, diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py index 13c4df28b19..348f2b9dbe9 100644 --- a/src/fides/api/ops/service/privacy_request/request_runner_service.py +++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py @@ -44,7 +44,7 @@ ) from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, - MessagingActionType, + MessagingActionType, MessagingServiceType, ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( @@ -455,7 +455,7 @@ def initiate_privacy_request_completion_email( db=session, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, to_identity=to_identity, - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=AccessRequestCompleteBodyParams( download_links=access_result_urls ), @@ -465,7 +465,7 @@ def initiate_privacy_request_completion_email( db=session, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, to_identity=to_identity, - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=None, ) diff --git a/src/fides/ctl/core/config/notification_settings.py b/src/fides/ctl/core/config/notification_settings.py index 4afa33550fd..8aac0776d44 100644 --- a/src/fides/ctl/core/config/notification_settings.py +++ b/src/fides/ctl/core/config/notification_settings.py @@ -1,11 +1,7 @@ import logging from typing import Optional -from fides.api.ops.schemas.messaging.messaging import ( - EMAIL_MESSAGING_SERVICES, - SMS_MESSAGING_SERVICES, - MessagingMethod, -) +from pydantic import validator from .fides_settings import FidesSettings @@ -22,13 +18,25 @@ class NotificationSettings(FidesSettings): send_request_review_notification: bool = False notification_service_type: Optional[str] = None - def get_messaging_method(self) -> Optional[MessagingMethod]: - """returns messaging method based on configured notification service type""" - if self in EMAIL_MESSAGING_SERVICES: - return MessagingMethod.EMAIL - if self in SMS_MESSAGING_SERVICES: - return MessagingMethod.SMS - return None + @validator("notification_service_type", pre=True) + @classmethod + def validate_notification_service_type(cls, value: Optional[str]) -> str: + """Ensure the provided type is a valid value.""" + if value: + + valid_values = [ + "mailgun", + "twilio_text", + "twilio_email" + ] + value = value.lower() # force lowercase for safety + + if value not in valid_values: + raise ValueError( + f"Invalid NOTIFICATION_SERVICE_TYPE provided '{value}', must be one of: {', '.join([level for level in valid_values])}" + ) + + return value class Config: env_prefix = ENV_PREFIX diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index 7fc0ad60b37..e4d911f7aff 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -37,7 +37,7 @@ def test_email_dispatch_mailgun_success( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -65,7 +65,7 @@ def test_email_dispatch_mailgun_config_not_found( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -99,7 +99,7 @@ def test_email_dispatch_mailgun_config_no_secrets( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -129,7 +129,7 @@ def test_email_dispatch_mailgun_failed_email(db: Session, messaging_config) -> N db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=CONFIG.notifications.get_messaging_method(), + messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), From c2367f679034ae80b2816c65dd34ee36a5daf98e Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Fri, 28 Oct 2022 14:04:46 -0500 Subject: [PATCH 03/28] Adds twilio client, config, and refactor get_messaging_method --- requirements.txt | 1 + .../2d5ff3096959_update_table_for_twilio.py | 1 + .../api/v1/endpoints/messaging_endpoints.py | 3 +- .../v1/endpoints/privacy_request_endpoints.py | 7 +- src/fides/api/ops/models/messaging.py | 14 +++- .../api/ops/schemas/messaging/messaging.py | 52 +++++++----- .../messaging/messaging_secrets_docs_only.py | 4 +- src/fides/api/ops/service/_verification.py | 4 +- .../messaging/message_dispatch_service.py | 81 ++++++++++++++----- .../messaging/messaging_crud_service.py | 14 ++-- .../privacy_request/request_runner_service.py | 5 +- src/fides/ctl/core/config/__init__.py | 7 ++ .../ctl/core/config/notification_settings.py | 9 +-- .../message_dispatch_service_test.py | 17 ++-- .../request_runner_service_test.py | 4 +- 15 files changed, 146 insertions(+), 77 deletions(-) diff --git a/requirements.txt b/requirements.txt index cc72a5342b1..3caa6ca150b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -33,5 +33,6 @@ sqlalchemy-stubs==0.4 SQLAlchemy-Utils==0.38.3 sqlalchemy[asyncio]==1.4.42 toml>=0.10.1 +twilio==7.15.0 Unidecode==1.3.4 versioneer==0.19 diff --git a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py index 962efbd0959..a54e2a228db 100644 --- a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py @@ -3,6 +3,7 @@ Revises: fb6b0150d6e4 Create Date: 2022-10-21 22:10:48.899562 """ +import sqlalchemy_utils from alembic import op import sqlalchemy as sa from sqlalchemy.dialects import postgresql diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py index 799db5ff1b0..8484c3c34a7 100644 --- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -76,7 +76,7 @@ def post_config( ) raise HTTPException( status_code=HTTP_500_INTERNAL_SERVER_ERROR, - detail=f"Config with key {messaging_config.key} failed to be added", + detail=f"Config with key {messaging_config.key} failed to be added: {exc}", ) @@ -160,7 +160,6 @@ def put_config_secrets( status_code=HTTP_400_BAD_REQUEST, detail=exc.args[0], ) - msg = f"Secrets updated for MessagingConfig with key: {config_key}." # todo- implement test status for messaging service return TestMessagingStatusMessage(msg=msg, test_status=None) diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index 8d32b4f3288..97a70c9022c 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -75,6 +75,7 @@ from fides.api.ops.models.connectionconfig import ConnectionConfig from fides.api.ops.models.datasetconfig import DatasetConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook +from fides.api.ops.models.messaging import get_messaging_method from fides.api.ops.models.policy import ( ActionType, CurrentStep, @@ -98,7 +99,7 @@ FidesopsMessage, MessagingActionType, RequestReceiptBodyParams, - RequestReviewDenyBodyParams, + RequestReviewDenyBodyParams, MessagingServiceType, ) from fides.api.ops.schemas.privacy_request import ( BulkPostPrivacyRequests, @@ -322,7 +323,7 @@ def _send_privacy_request_receipt_message_to_user( action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT, body_params=RequestReceiptBodyParams(request_types=request_types), ).dict(), - "messaging_method": CONFIG.notifications.get_messaging_method(), + "messaging_method": get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), "to_identity": to_identity, }, ) @@ -1174,7 +1175,7 @@ def _send_privacy_request_review_message_to_user( if action_type is MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY else None, ).dict(), - "messaging_method": CONFIG.notifications.get_messaging_method(), + "messaging_method": get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), "to_identity": to_identity, }, ) diff --git a/src/fides/api/ops/models/messaging.py b/src/fides/api/ops/models/messaging.py index cb0b738a578..5ab890b08a4 100644 --- a/src/fides/api/ops/models/messaging.py +++ b/src/fides/api/ops/models/messaging.py @@ -17,7 +17,8 @@ from fides.api.ops.schemas.messaging.messaging import ( SUPPORTED_MESSAGING_SERVICE_SECRETS, MessagingServiceSecretsMailgun, - MessagingServiceType, + MessagingServiceType, MessagingServiceSecretsTwilioSMS, MessagingServiceSecretsTwilioEmail, MessagingMethod, + EMAIL_MESSAGING_SERVICES, SMS_MESSAGING_SERVICES, ) from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import ( possible_messaging_secrets, @@ -29,6 +30,15 @@ logger = logging.getLogger(__name__) +def get_messaging_method(service_type: Optional[MessagingServiceType]) -> Optional[MessagingMethod]: + """returns messaging method based on configured service type""" + if service_type in EMAIL_MESSAGING_SERVICES: + return MessagingMethod.EMAIL + if service_type in SMS_MESSAGING_SERVICES: + return MessagingMethod.SMS + return None + + def get_schema_for_secrets( service_type: MessagingServiceType, secrets: possible_messaging_secrets, @@ -40,6 +50,8 @@ def get_schema_for_secrets( try: schema = { MessagingServiceType.MAILGUN: MessagingServiceSecretsMailgun, + MessagingServiceType.TWILIO_TEXT: MessagingServiceSecretsTwilioSMS, + MessagingServiceType.TWILIO_EMAIL: MessagingServiceSecretsTwilioEmail, }[service_type] except KeyError: raise ValueError( diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index fc7c671febd..620c28e217e 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -23,14 +23,6 @@ class MessagingServiceType(Enum): TWILIO_TEXT = "twilio_text" TWILIO_EMAIL = "twilio_email" - def get_messaging_method(self) -> Optional[MessagingMethod]: - """returns messaging method based on configured service type""" - if self in EMAIL_MESSAGING_SERVICES: - return MessagingMethod.EMAIL - if self in SMS_MESSAGING_SERVICES: - return MessagingMethod.SMS - return None - EMAIL_MESSAGING_SERVICES = [ MessagingServiceType.MAILGUN, @@ -103,8 +95,8 @@ class FidesopsMessage( ] -class MessageForActionType(BaseModel): - """Message details that depend on action type""" +class EmailForActionType(BaseModel): + """Email details that depend on action type""" subject: str body: str @@ -113,7 +105,7 @@ class MessageForActionType(BaseModel): class MessagingServiceDetails(Enum): """Enum for messaging service details""" - # mailgun-specific + # Mailgun IS_EU_DOMAIN = "is_eu_domain" API_VERSION = "api_version" DOMAIN = "domain" @@ -135,13 +127,17 @@ class Config: class MessagingServiceSecrets(Enum): """Enum for message service secrets""" - # mailgun-specific + # Mailgun MAILGUN_API_KEY = "mailgun_api_key" - # twilio + # Twilio SMS TWILIO_ACCOUNT_SID = "twilio_account_sid" TWILIO_AUTH_TOKEN = "twilio_auth_token" - TWILIO_MESSAGING_SERVICE_ID = "twilio_messaging_service_id" + TWILIO_MESSAGING_SERVICE_SID = "twilio_messaging_service_sid" + TWILIO_SENDER_PHONE_NUMBER = "twilio_sender_phone_number" # formatted like +15558675309 + + # Twilio Sendgrid/Email + TWILIO_API_KEY = "twilio_api_key" class MessagingServiceSecretsMailgun(BaseModel): @@ -155,12 +151,24 @@ class Config: extra = Extra.forbid -class MessagingServiceSecretsTwilio(BaseModel): +class MessagingServiceSecretsTwilioSMS(BaseModel): + """The secrets required to connect to twilio SMS.""" + + twilio_account_sid: str + twilio_auth_token: str + twilio_messaging_service_sid: Optional[str] + twilio_sender_phone_number: Optional[str] # Either the twilio_messaging_service_id *OR* the twilio_sender_phone_number should be supplied. + + class Config: + """Restrict adding other fields through this schema.""" + + extra = Extra.forbid + + +class MessagingServiceSecretsTwilioEmail(BaseModel): """The secrets required to connect to twilio email.""" - account_sid: str - auth_token: str - messaging_service_id: str + twilio_api_key: str class Config: """Restrict adding other fields through this schema.""" @@ -174,7 +182,7 @@ class MessagingConfigRequest(BaseModel): name: str key: Optional[FidesOpsKey] service_type: MessagingServiceType - details: Union[MessagingServiceDetailsMailgun] + details: Optional[Union[MessagingServiceDetailsMailgun]] class Config: use_enum_values = False @@ -187,7 +195,7 @@ class MessagingConfigResponse(BaseModel): name: str key: FidesOpsKey service_type: MessagingServiceType - details: Dict[MessagingServiceDetails, Any] + details: Optional[Dict[MessagingServiceDetails, Any]] class Config: orm_mode = True @@ -195,12 +203,12 @@ class Config: SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[ - MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilio + MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilioSMS, MessagingServiceSecretsTwilioEmail ] class MessagingConnectionTestStatus(Enum): - """Enum for supplying statuses of validating credentials for an Messaging Config""" + """Enum for supplying statuses of validating credentials for a messaging Config""" succeeded = "succeeded" failed = "failed" diff --git a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py index 3a9f60a4c69..ab3860a1dec 100644 --- a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py +++ b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py @@ -3,7 +3,7 @@ from fides.api.ops.schemas.base_class import NoValidationSchema from fides.api.ops.schemas.messaging.messaging import ( MessagingServiceSecretsMailgun, - MessagingServiceSecretsTwilio, + MessagingServiceSecretsTwilioSMS, ) @@ -11,7 +11,7 @@ class MessagingSecretsMailgunDocs(MessagingServiceSecretsMailgun, NoValidationSc """The secrets required to connect to Mailgun, for documentation""" -class MessagingSecretsTwilioDocs(MessagingServiceSecretsTwilio, NoValidationSchema): +class MessagingSecretsTwilioDocs(MessagingServiceSecretsTwilioSMS, NoValidationSchema): """The secrets required to connect to Twilio, for documentation""" diff --git a/src/fides/api/ops/service/_verification.py b/src/fides/api/ops/service/_verification.py index 4a8bddb13e6..3d37f55d41b 100644 --- a/src/fides/api/ops/service/_verification.py +++ b/src/fides/api/ops/service/_verification.py @@ -2,7 +2,7 @@ from sqlalchemy.orm import Session -from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.models.messaging import MessagingConfig, get_messaging_method from fides.api.ops.models.privacy_request import ConsentRequest, PrivacyRequest from fides.api.ops.schemas.messaging.messaging import ( MessagingActionType, @@ -36,7 +36,7 @@ def send_verification_code_to_user( db, action_type=messaging_action_type, to_identity=to_identity, - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=SubjectIdentityVerificationBodyParams( verification_code=verification_code, verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index b69947dad4b..fbd41271695 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -5,6 +5,9 @@ import requests from sqlalchemy.orm import Session +from twilio.base.exceptions import TwilioRestException + +from twilio.rest import Client from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.email_templates import get_email_template @@ -13,7 +16,7 @@ from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, FidesopsMessage, - MessageForActionType, + EmailForActionType, MessagingActionType, MessagingMethod, MessagingServiceDetails, @@ -27,6 +30,7 @@ from fides.api.ops.tasks import DatabaseTask, celery_app from fides.api.ops.util.logger import Pii from fides.ctl.core.config import get_config +from loguru import logger as log CONFIG = get_config() @@ -81,7 +85,7 @@ def dispatch_message( logger.info( "Building appropriate message template for action type: %s", action_type ) - message: Optional[MessageForActionType] = None # fixme- huh?? + message: Optional[Union[EmailForActionType, str]] = None # fixme- huh?? if messaging_method == MessagingMethod.EMAIL: message = _build_email( action_type=action_type, @@ -128,13 +132,10 @@ def dispatch_message( def _build_sms( action_type: MessagingActionType, - body_params: Any, # fixme- create message body based on params -) -> MessageForActionType: + body_params: Any, +) -> str: if action_type == MessagingActionType.CONSENT_REQUEST: - return MessageForActionType( - subject="Your one-time code", - body="body", - ) + return "Hello, this message was sent from Fides!" logger.error("Message action type %s is not implemented", action_type) raise MessageDispatchException( f"Message action type {action_type} is not implemented" @@ -144,10 +145,10 @@ def _build_sms( def _build_email( # pylint: disable=too-many-return-statements action_type: MessagingActionType, body_params: Any, -) -> MessageForActionType: +) -> EmailForActionType: if action_type == MessagingActionType.CONSENT_REQUEST: template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your one-time code", body=template.render( { @@ -158,7 +159,7 @@ def _build_email( # pylint: disable=too-many-return-statements ) if action_type == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION: template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your one-time code", body=template.render( { @@ -169,7 +170,7 @@ def _build_email( # pylint: disable=too-many-return-statements ) if action_type == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT: base_template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Data erasure request", body=base_template.render( {"dataset_collection_action_required": body_params} @@ -177,13 +178,13 @@ def _build_email( # pylint: disable=too-many-return-statements ) if action_type == MessagingActionType.PRIVACY_REQUEST_RECEIPT: base_template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your request has been received", body=base_template.render({"request_types": body_params.request_types}), ) if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS: base_template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your data is ready to be downloaded", body=base_template.render( { @@ -193,19 +194,19 @@ def _build_email( # pylint: disable=too-many-return-statements ) if action_type == MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION: base_template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your data has been deleted", body=base_template.render(), ) if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_APPROVE: base_template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your request has been approved", body=base_template.render(), ) if action_type == MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY: base_template = get_email_template(action_type) - return MessageForActionType( + return EmailForActionType( subject="Your request has been denied", body=base_template.render( {"rejection_reason": body_params.rejection_reason} @@ -221,16 +222,17 @@ def _get_dispatcher_from_config_type(message_service_type: MessagingServiceType) """Determines which dispatcher to use based on message service type""" return { MessagingServiceType.MAILGUN.value: _mailgun_dispatcher, + MessagingServiceType.TWILIO_TEXT.value: _twilio_sms_dispatcher, }[message_service_type.value] def _mailgun_dispatcher( messaging_config: MessagingConfig, - message: MessageForActionType, - to_email: Optional[str], + message: EmailForActionType, + to: Optional[str], ) -> None: """Dispatches email using mailgun""" - if not to_email: + if not to: logger.error("Message failed to send. No email identity supplied.") raise MessageDispatchException("No email identity supplied.") base_url = ( @@ -241,7 +243,7 @@ def _mailgun_dispatcher( domain = messaging_config.details[MessagingServiceDetails.DOMAIN.value] data = { "from": f"", - "to": [to_email], + "to": [to], "subject": message.subject, "html": message.body, } @@ -264,3 +266,40 @@ def _mailgun_dispatcher( except Exception as e: logger.error("Email failed to send: %s", Pii(str(e))) raise MessageDispatchException(f"Email failed to send due to: {Pii(e)}") + + +def _twilio_sms_dispatcher( + messaging_config: MessagingConfig, + message: str, + to: Optional[str], +) -> None: + """Dispatches SMS using Twilio""" + if not to: + logger.error("Message failed to send. No phone identity supplied.") + raise MessageDispatchException("No phone identity supplied.") + + account_sid = messaging_config.secrets[MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value] # type: ignore + auth_token = messaging_config.secrets[MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value] # type: ignore + messaging_service_id = messaging_config.secrets[MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value] # type:ignore + sender_phone_number = messaging_config.secrets[MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value] # type:ignore + + client = Client(account_sid, auth_token) + try: + if messaging_service_id: + client.messages.create( + to=to, + messaging_service_sid=messaging_service_id, + body=message + ) + elif sender_phone_number: + client.messages.create( + to=to, + from_=sender_phone_number, + body=message + ) + else: + logger.error("Message failed to send. Either sender phone number or messaging service sid must be provided.") + raise MessageDispatchException("Message failed to send. Either sender phone number or messaging service sid must be provided.") + except TwilioRestException as e: + logger.error("Twilio SMS failed to send: %s", Pii(str(e))) + raise MessageDispatchException(f"Twilio SMS failed to send due to: {Pii(e)}") diff --git a/src/fides/api/ops/service/messaging/messaging_crud_service.py b/src/fides/api/ops/service/messaging/messaging_crud_service.py index 8509dccccc9..3b6d8c69c8c 100644 --- a/src/fides/api/ops/service/messaging/messaging_crud_service.py +++ b/src/fides/api/ops/service/messaging/messaging_crud_service.py @@ -29,14 +29,16 @@ def update_messaging_config( def create_or_update_messaging_config( db: Session, config: MessagingConfigRequest ) -> MessagingConfigResponse: + data = { + "key": config.key, + "name": config.name, + "service_type": config.service_type, + } + if config.details: + data["details"] = config.details.__dict__ messaging_config: MessagingConfig = MessagingConfig.create_or_update( db=db, - data={ - "key": config.key, - "name": config.name, - "service_type": config.service_type, - "details": config.details.__dict__, - }, + data=data, ) return MessagingConfigResponse( name=messaging_config.name, diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py index 348f2b9dbe9..f8d35d1b698 100644 --- a/src/fides/api/ops/service/privacy_request/request_runner_service.py +++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py @@ -28,6 +28,7 @@ from fides.api.ops.models.connectionconfig import ConnectionConfig from fides.api.ops.models.datasetconfig import DatasetConfig from fides.api.ops.models.manual_webhook import AccessManualWebhook +from fides.api.ops.models.messaging import get_messaging_method from fides.api.ops.models.policy import ( ActionType, CurrentStep, @@ -455,7 +456,7 @@ def initiate_privacy_request_completion_email( db=session, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, to_identity=to_identity, - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=AccessRequestCompleteBodyParams( download_links=access_result_urls ), @@ -465,7 +466,7 @@ def initiate_privacy_request_completion_email( db=session, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, to_identity=to_identity, - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=None, ) diff --git a/src/fides/ctl/core/config/__init__.py b/src/fides/ctl/core/config/__init__.py index 5c1afc2e954..0afed3547cb 100644 --- a/src/fides/ctl/core/config/__init__.py +++ b/src/fides/ctl/core/config/__init__.py @@ -65,6 +65,7 @@ def log_all_config_values(self) -> None: self.user, self.logging, self.database, + self.notifications, self.redis, self.security, self.execution, @@ -128,6 +129,12 @@ def handle_deprecated_env_variables(settings: MutableMapping) -> MutableMapping: "db", "test_db", ], + "notifications": [ + "send_request_completion_notification", + "send_request_receipt_notification", + "send_request_review_notification", + "notification_service_type", + ], "redis": [ "host", "port", diff --git a/src/fides/ctl/core/config/notification_settings.py b/src/fides/ctl/core/config/notification_settings.py index 8aac0776d44..1e0bd15ba84 100644 --- a/src/fides/ctl/core/config/notification_settings.py +++ b/src/fides/ctl/core/config/notification_settings.py @@ -23,13 +23,12 @@ class NotificationSettings(FidesSettings): def validate_notification_service_type(cls, value: Optional[str]) -> str: """Ensure the provided type is a valid value.""" if value: - valid_values = [ - "mailgun", - "twilio_text", - "twilio_email" + "MAILGUN", + "TWILIO_TEXT", + "TWILIO_EMAIL" ] - value = value.lower() # force lowercase for safety + value = value.upper() # force lowercase for safety if value not in valid_values: raise ValueError( diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index e4d911f7aff..4d32ec54d26 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -2,18 +2,17 @@ from unittest.mock import Mock import pytest -import requests.exceptions import requests_mock from sqlalchemy.orm import Session from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.graph.config import CollectionAddress -from fides.api.ops.models.messaging import MessagingConfig +from fides.api.ops.models.messaging import MessagingConfig, get_messaging_method from fides.api.ops.models.policy import CurrentStep from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction from fides.api.ops.schemas.messaging.messaging import ( FidesopsMessage, - MessageForActionType, + EmailForActionType, MessagingActionType, MessagingServiceDetails, MessagingServiceType, @@ -37,7 +36,7 @@ def test_email_dispatch_mailgun_success( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -45,11 +44,11 @@ def test_email_dispatch_mailgun_success( body = '\n\n\n \n ID Code\n\n\n
\n

\n Your privacy request verification code is 2348.\n Please return to the Privacy Center and enter the code to\n continue. This code will expire in 10 minutes\n

\n
\n\n' mock_mailgun_dispatcher.assert_called_with( messaging_config=messaging_config, - message=MessageForActionType( + message=EmailForActionType( subject="Your one-time code", body=body, ), - to_email="test@email.com", + to="test@email.com", ) @@ -65,7 +64,7 @@ def test_email_dispatch_mailgun_config_not_found( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -99,7 +98,7 @@ def test_email_dispatch_mailgun_config_no_secrets( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -129,7 +128,7 @@ def test_email_dispatch_mailgun_failed_email(db: Session, messaging_config) -> N db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=MessagingServiceType[CONFIG.notifications.notification_service_type].get_messaging_method(), + messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index b6ea6ddd6dd..bf119ea6798 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -34,7 +34,7 @@ from fides.api.ops.schemas.masking.masking_secrets import MaskingSecretCache from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, - MessageForActionType, + EmailForActionType, MessagingActionType, ) from fides.api.ops.schemas.policy import Rule @@ -1677,7 +1677,7 @@ def test_create_and_process_erasure_request_email_connector( assert mailgun_send.called kwargs = mailgun_send.call_args.kwargs assert type(kwargs["messaging_config"]) == MessagingConfig - assert type(kwargs["email"]) == MessageForActionType + assert type(kwargs["email"]) == EmailForActionType @mock.patch( "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" From 3e3a9d54b617a0d12bb10c7fd24a40ac83338223 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 31 Oct 2022 07:34:56 -0600 Subject: [PATCH 04/28] linter fixes --- .../2d5ff3096959_update_table_for_twilio.py | 145 ++++++++++++------ src/fides/api/ops/api/v1/api.py | 2 +- .../api/v1/endpoints/messaging_endpoints.py | 16 +- .../v1/endpoints/privacy_request_endpoints.py | 10 +- src/fides/api/ops/models/messaging.py | 12 +- .../api/ops/schemas/messaging/messaging.py | 12 +- src/fides/api/ops/service/_verification.py | 6 +- .../messaging/message_dispatch_service.py | 49 +++--- .../messaging/messaging_crud_service.py | 2 +- .../privacy_request/request_runner_service.py | 10 +- .../ctl/core/config/notification_settings.py | 8 +- .../v1/endpoints/test_messaging_endpoints.py | 8 +- .../message_dispatch_service_test.py | 18 ++- 13 files changed, 199 insertions(+), 99 deletions(-) diff --git a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py index a54e2a228db..2f4e18da9f2 100644 --- a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py @@ -3,63 +3,120 @@ Revises: fb6b0150d6e4 Create Date: 2022-10-21 22:10:48.899562 """ +import sqlalchemy as sa import sqlalchemy_utils from alembic import op -import sqlalchemy as sa from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '2d5ff3096959' -down_revision = 'fb6b0150d6e4' +revision = "2d5ff3096959" +down_revision = "fb6b0150d6e4" branch_labels = None depends_on = None def upgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('messagingconfig', - sa.Column('id', sa.String(length=255), nullable=False), - sa.Column('created_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('updated_at', sa.DateTime(timezone=True), server_default=sa.text('now()'), nullable=True), - sa.Column('key', sa.String(), nullable=False), - sa.Column('name', sa.String(), nullable=True), - sa.Column('service_type', sa.Enum('MAILGUN', 'TWILIO_TEXT', 'TWILIO_EMAIL', name='messagingservicetype'), nullable=False), - sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), nullable=True), - sa.Column('secrets', sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), nullable=True), - sa.PrimaryKeyConstraint('id') - ) - op.create_index(op.f('ix_messagingconfig_id'), 'messagingconfig', ['id'], unique=False) - op.create_index(op.f('ix_messagingconfig_key'), 'messagingconfig', ['key'], unique=True) - op.create_index(op.f('ix_messagingconfig_name'), 'messagingconfig', ['name'], unique=True) - op.create_index(op.f('ix_messagingconfig_service_type'), 'messagingconfig', ['service_type'], unique=True) - op.drop_index('ix_emailconfig_id', table_name='emailconfig') - op.drop_index('ix_emailconfig_key', table_name='emailconfig') - op.drop_index('ix_emailconfig_name', table_name='emailconfig') - op.drop_index('ix_emailconfig_service_type', table_name='emailconfig') - op.drop_table('emailconfig') + op.create_table( + "messagingconfig", + sa.Column("id", sa.String(length=255), nullable=False), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.text("now()"), + nullable=True, + ), + sa.Column("key", sa.String(), nullable=False), + sa.Column("name", sa.String(), nullable=True), + sa.Column( + "service_type", + sa.Enum( + "MAILGUN", "TWILIO_TEXT", "TWILIO_EMAIL", name="messagingservicetype" + ), + nullable=False, + ), + sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=True), + sa.Column( + "secrets", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + sa.PrimaryKeyConstraint("id"), + ) + op.create_index( + op.f("ix_messagingconfig_id"), "messagingconfig", ["id"], unique=False + ) + op.create_index( + op.f("ix_messagingconfig_key"), "messagingconfig", ["key"], unique=True + ) + op.create_index( + op.f("ix_messagingconfig_name"), "messagingconfig", ["name"], unique=True + ) + op.create_index( + op.f("ix_messagingconfig_service_type"), + "messagingconfig", + ["service_type"], + unique=True, + ) + op.drop_index("ix_emailconfig_id", table_name="emailconfig") + op.drop_index("ix_emailconfig_key", table_name="emailconfig") + op.drop_index("ix_emailconfig_name", table_name="emailconfig") + op.drop_index("ix_emailconfig_service_type", table_name="emailconfig") + op.drop_table("emailconfig") # ### end Alembic commands ### def downgrade(): # ### commands auto generated by Alembic - please adjust! ### - op.create_table('emailconfig', - sa.Column('id', sa.VARCHAR(length=255), autoincrement=False, nullable=False), - sa.Column('created_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=True), - sa.Column('updated_at', postgresql.TIMESTAMP(timezone=True), server_default=sa.text('now()'), autoincrement=False, nullable=True), - sa.Column('key', sa.VARCHAR(), autoincrement=False, nullable=False), - sa.Column('name', sa.VARCHAR(), autoincrement=False, nullable=True), - sa.Column('service_type', postgresql.ENUM('MAILGUN', name='emailservicetype'), autoincrement=False, nullable=False), - sa.Column('details', postgresql.JSONB(astext_type=sa.Text()), autoincrement=False, nullable=False), - sa.Column('secrets', sa.VARCHAR(), autoincrement=False, nullable=True), - sa.PrimaryKeyConstraint('id', name='emailconfig_pkey') - ) - op.create_index('ix_emailconfig_service_type', 'emailconfig', ['service_type'], unique=False) - op.create_index('ix_emailconfig_name', 'emailconfig', ['name'], unique=False) - op.create_index('ix_emailconfig_key', 'emailconfig', ['key'], unique=False) - op.create_index('ix_emailconfig_id', 'emailconfig', ['id'], unique=False) - op.drop_index(op.f('ix_messagingconfig_service_type'), table_name='messagingconfig') - op.drop_index(op.f('ix_messagingconfig_name'), table_name='messagingconfig') - op.drop_index(op.f('ix_messagingconfig_key'), table_name='messagingconfig') - op.drop_index(op.f('ix_messagingconfig_id'), table_name='messagingconfig') - op.drop_table('messagingconfig') - # ### end Alembic commands ### \ No newline at end of file + op.create_table( + "emailconfig", + sa.Column("id", sa.VARCHAR(length=255), autoincrement=False, nullable=False), + sa.Column( + "created_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=True, + ), + sa.Column( + "updated_at", + postgresql.TIMESTAMP(timezone=True), + server_default=sa.text("now()"), + autoincrement=False, + nullable=True, + ), + sa.Column("key", sa.VARCHAR(), autoincrement=False, nullable=False), + sa.Column("name", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.Column( + "service_type", + postgresql.ENUM("MAILGUN", name="emailservicetype"), + autoincrement=False, + nullable=False, + ), + sa.Column( + "details", + postgresql.JSONB(astext_type=sa.Text()), + autoincrement=False, + nullable=False, + ), + sa.Column("secrets", sa.VARCHAR(), autoincrement=False, nullable=True), + sa.PrimaryKeyConstraint("id", name="emailconfig_pkey"), + ) + op.create_index( + "ix_emailconfig_service_type", "emailconfig", ["service_type"], unique=False + ) + op.create_index("ix_emailconfig_name", "emailconfig", ["name"], unique=False) + op.create_index("ix_emailconfig_key", "emailconfig", ["key"], unique=False) + op.create_index("ix_emailconfig_id", "emailconfig", ["id"], unique=False) + op.drop_index(op.f("ix_messagingconfig_service_type"), table_name="messagingconfig") + op.drop_index(op.f("ix_messagingconfig_name"), table_name="messagingconfig") + op.drop_index(op.f("ix_messagingconfig_key"), table_name="messagingconfig") + op.drop_index(op.f("ix_messagingconfig_id"), table_name="messagingconfig") + op.drop_table("messagingconfig") + # ### end Alembic commands ### diff --git a/src/fides/api/ops/api/v1/api.py b/src/fides/api/ops/api/v1/api.py index f87d8abf337..ccdf32fd6b2 100644 --- a/src/fides/api/ops/api/v1/api.py +++ b/src/fides/api/ops/api/v1/api.py @@ -5,11 +5,11 @@ consent_request_endpoints, dataset_endpoints, drp_endpoints, - messaging_endpoints, encryption_endpoints, identity_verification_endpoints, manual_webhook_endpoints, masking_endpoints, + messaging_endpoints, oauth_endpoints, policy_endpoints, policy_webhook_endpoints, diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py index 8484c3c34a7..e1e2dd6a66d 100644 --- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -72,7 +72,9 @@ def post_config( return create_or_update_messaging_config(db=db, config=messaging_config) except Exception as exc: logger.warning( - "Create failed for messaging config %s: %s", messaging_config.key, Pii(str(exc)) + "Create failed for messaging config %s: %s", + messaging_config.key, + Pii(str(exc)), ) raise HTTPException( status_code=HTTP_500_INTERNAL_SERVER_ERROR, @@ -105,7 +107,9 @@ def patch_config_by_key( except Exception as exc: logger.warning( - "Patch failed for messaging config %s: %s", messaging_config.key, Pii(str(exc)) + "Patch failed for messaging config %s: %s", + messaging_config.key, + Pii(str(exc)), ) raise HTTPException( status_code=HTTP_500_INTERNAL_SERVER_ERROR, @@ -152,7 +156,9 @@ def put_config_secrets( detail=exc.args[0], ) - logger.info("Updating messaging config secrets for config with key '%s'", config_key) + logger.info( + "Updating messaging config secrets for config with key '%s'", config_key + ) try: messaging_config.set_secrets(db=db, messaging_secrets=secrets_schema.dict()) except ValueError as exc: @@ -176,7 +182,9 @@ def get_configs( """ Retrieves configs for messaging. """ - logger.info("Finding all messaging configurations with pagination params %s", params) + logger.info( + "Finding all messaging configurations with pagination params %s", params + ) return paginate( MessagingConfig.query(db=db).order_by(MessagingConfig.created_at.desc()), params=params, diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index 97a70c9022c..56ebe3e2689 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -99,7 +99,7 @@ FidesopsMessage, MessagingActionType, RequestReceiptBodyParams, - RequestReviewDenyBodyParams, MessagingServiceType, + RequestReviewDenyBodyParams, ) from fides.api.ops.schemas.privacy_request import ( BulkPostPrivacyRequests, @@ -323,7 +323,9 @@ def _send_privacy_request_receipt_message_to_user( action_type=MessagingActionType.PRIVACY_REQUEST_RECEIPT, body_params=RequestReceiptBodyParams(request_types=request_types), ).dict(), - "messaging_method": get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + "messaging_method": get_messaging_method( + CONFIG.notifications.notification_service_type + ), "to_identity": to_identity, }, ) @@ -1175,7 +1177,9 @@ def _send_privacy_request_review_message_to_user( if action_type is MessagingActionType.PRIVACY_REQUEST_REVIEW_DENY else None, ).dict(), - "messaging_method": get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + "messaging_method": get_messaging_method( + CONFIG.notifications.notification_service_type + ), "to_identity": to_identity, }, ) diff --git a/src/fides/api/ops/models/messaging.py b/src/fides/api/ops/models/messaging.py index 5ab890b08a4..985a62615f1 100644 --- a/src/fides/api/ops/models/messaging.py +++ b/src/fides/api/ops/models/messaging.py @@ -15,10 +15,14 @@ from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.db.base_class import JSONTypeOverride from fides.api.ops.schemas.messaging.messaging import ( + EMAIL_MESSAGING_SERVICES, + SMS_MESSAGING_SERVICES, SUPPORTED_MESSAGING_SERVICE_SECRETS, + MessagingMethod, MessagingServiceSecretsMailgun, - MessagingServiceType, MessagingServiceSecretsTwilioSMS, MessagingServiceSecretsTwilioEmail, MessagingMethod, - EMAIL_MESSAGING_SERVICES, SMS_MESSAGING_SERVICES, + MessagingServiceSecretsTwilioEmail, + MessagingServiceSecretsTwilioSMS, + MessagingServiceType, ) from fides.api.ops.schemas.messaging.messaging_secrets_docs_only import ( possible_messaging_secrets, @@ -30,7 +34,9 @@ logger = logging.getLogger(__name__) -def get_messaging_method(service_type: Optional[MessagingServiceType]) -> Optional[MessagingMethod]: +def get_messaging_method( + service_type: Optional[str], +) -> Optional[MessagingMethod]: """returns messaging method based on configured service type""" if service_type in EMAIL_MESSAGING_SERVICES: return MessagingMethod.EMAIL diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index 620c28e217e..c3477f9551d 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -134,7 +134,9 @@ class MessagingServiceSecrets(Enum): TWILIO_ACCOUNT_SID = "twilio_account_sid" TWILIO_AUTH_TOKEN = "twilio_auth_token" TWILIO_MESSAGING_SERVICE_SID = "twilio_messaging_service_sid" - TWILIO_SENDER_PHONE_NUMBER = "twilio_sender_phone_number" # formatted like +15558675309 + TWILIO_SENDER_PHONE_NUMBER = ( + "twilio_sender_phone_number" # formatted like +15558675309 + ) # Twilio Sendgrid/Email TWILIO_API_KEY = "twilio_api_key" @@ -157,7 +159,9 @@ class MessagingServiceSecretsTwilioSMS(BaseModel): twilio_account_sid: str twilio_auth_token: str twilio_messaging_service_sid: Optional[str] - twilio_sender_phone_number: Optional[str] # Either the twilio_messaging_service_id *OR* the twilio_sender_phone_number should be supplied. + twilio_sender_phone_number: Optional[ + str + ] # Either the twilio_messaging_service_id *OR* the twilio_sender_phone_number should be supplied. class Config: """Restrict adding other fields through this schema.""" @@ -203,7 +207,9 @@ class Config: SUPPORTED_MESSAGING_SERVICE_SECRETS = Union[ - MessagingServiceSecretsMailgun, MessagingServiceSecretsTwilioSMS, MessagingServiceSecretsTwilioEmail + MessagingServiceSecretsMailgun, + MessagingServiceSecretsTwilioSMS, + MessagingServiceSecretsTwilioEmail, ] diff --git a/src/fides/api/ops/service/_verification.py b/src/fides/api/ops/service/_verification.py index 3d37f55d41b..36979da6562 100644 --- a/src/fides/api/ops/service/_verification.py +++ b/src/fides/api/ops/service/_verification.py @@ -6,7 +6,7 @@ from fides.api.ops.models.privacy_request import ConsentRequest, PrivacyRequest from fides.api.ops.schemas.messaging.messaging import ( MessagingActionType, - SubjectIdentityVerificationBodyParams, MessagingServiceType, + SubjectIdentityVerificationBodyParams, ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.messaging.message_dispatch_service import dispatch_message @@ -36,7 +36,9 @@ def send_verification_code_to_user( db, action_type=messaging_action_type, to_identity=to_identity, - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code=verification_code, verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index fbd41271695..1aaf2ebf5b0 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -5,9 +5,8 @@ import requests from sqlalchemy.orm import Session -from twilio.base.exceptions import TwilioRestException - -from twilio.rest import Client +from twilio.base.exceptions import TwilioRestException # type: ignore +from twilio.rest import Client # type: ignore from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.email_templates import get_email_template @@ -15,8 +14,8 @@ from fides.api.ops.models.privacy_request import CheckpointActionRequired from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, - FidesopsMessage, EmailForActionType, + FidesopsMessage, MessagingActionType, MessagingMethod, MessagingServiceDetails, @@ -30,7 +29,6 @@ from fides.api.ops.tasks import DatabaseTask, celery_app from fides.api.ops.util.logger import Pii from fides.ctl.core.config import get_config -from loguru import logger as log CONFIG = get_config() @@ -98,7 +96,8 @@ def dispatch_message( ) else: logger.error( - "Notification service type is not valid: %s", CONFIG.notifications.notification_service_type + "Notification service type is not valid: %s", + CONFIG.notifications.notification_service_type, ) raise MessageDispatchException( f"Notification service type is not valid: {CONFIG.notifications.notification_service_type}" @@ -112,7 +111,8 @@ def dispatch_message( ) if not dispatcher: logger.error( - "Dispatcher has not been implemented for message service type: %s", messaging_service + "Dispatcher has not been implemented for message service type: %s", + messaging_service, ) raise MessageDispatchException( f"Dispatcher has not been implemented for message service type: {messaging_service}" @@ -235,6 +235,9 @@ def _mailgun_dispatcher( if not to: logger.error("Message failed to send. No email identity supplied.") raise MessageDispatchException("No email identity supplied.") + if not messaging_config.details: + logger.error("Message failed to send. No mailgun config details supplied.") + raise MessageDispatchException("No mailgun config details supplied.") base_url = ( "https://api.mailgun.net" if messaging_config.details[MessagingServiceDetails.IS_EU_DOMAIN.value] is False @@ -269,9 +272,9 @@ def _mailgun_dispatcher( def _twilio_sms_dispatcher( - messaging_config: MessagingConfig, - message: str, - to: Optional[str], + messaging_config: MessagingConfig, + message: str, + to: Optional[str], ) -> None: """Dispatches SMS using Twilio""" if not to: @@ -280,26 +283,28 @@ def _twilio_sms_dispatcher( account_sid = messaging_config.secrets[MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value] # type: ignore auth_token = messaging_config.secrets[MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value] # type: ignore - messaging_service_id = messaging_config.secrets[MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value] # type:ignore - sender_phone_number = messaging_config.secrets[MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value] # type:ignore + messaging_service_id = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] # type:ignore + sender_phone_number = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value + ] # type:ignore client = Client(account_sid, auth_token) try: if messaging_service_id: client.messages.create( - to=to, - messaging_service_sid=messaging_service_id, - body=message + to=to, messaging_service_sid=messaging_service_id, body=message ) elif sender_phone_number: - client.messages.create( - to=to, - from_=sender_phone_number, - body=message - ) + client.messages.create(to=to, from_=sender_phone_number, body=message) else: - logger.error("Message failed to send. Either sender phone number or messaging service sid must be provided.") - raise MessageDispatchException("Message failed to send. Either sender phone number or messaging service sid must be provided.") + logger.error( + "Message failed to send. Either sender phone number or messaging service sid must be provided." + ) + raise MessageDispatchException( + "Message failed to send. Either sender phone number or messaging service sid must be provided." + ) except TwilioRestException as e: logger.error("Twilio SMS failed to send: %s", Pii(str(e))) raise MessageDispatchException(f"Twilio SMS failed to send due to: {Pii(e)}") diff --git a/src/fides/api/ops/service/messaging/messaging_crud_service.py b/src/fides/api/ops/service/messaging/messaging_crud_service.py index 3b6d8c69c8c..2a8f204a415 100644 --- a/src/fides/api/ops/service/messaging/messaging_crud_service.py +++ b/src/fides/api/ops/service/messaging/messaging_crud_service.py @@ -35,7 +35,7 @@ def create_or_update_messaging_config( "service_type": config.service_type, } if config.details: - data["details"] = config.details.__dict__ + data["details"] = config.details.__dict__ # type: ignore messaging_config: MessagingConfig = MessagingConfig.create_or_update( db=db, data=data, diff --git a/src/fides/api/ops/service/privacy_request/request_runner_service.py b/src/fides/api/ops/service/privacy_request/request_runner_service.py index f8d35d1b698..69c5460487b 100644 --- a/src/fides/api/ops/service/privacy_request/request_runner_service.py +++ b/src/fides/api/ops/service/privacy_request/request_runner_service.py @@ -45,7 +45,7 @@ ) from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, - MessagingActionType, MessagingServiceType, + MessagingActionType, ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( @@ -456,7 +456,9 @@ def initiate_privacy_request_completion_email( db=session, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, to_identity=to_identity, - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=AccessRequestCompleteBodyParams( download_links=access_result_urls ), @@ -466,7 +468,9 @@ def initiate_privacy_request_completion_email( db=session, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, to_identity=to_identity, - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=None, ) diff --git a/src/fides/ctl/core/config/notification_settings.py b/src/fides/ctl/core/config/notification_settings.py index 1e0bd15ba84..8da188df1ab 100644 --- a/src/fides/ctl/core/config/notification_settings.py +++ b/src/fides/ctl/core/config/notification_settings.py @@ -20,14 +20,10 @@ class NotificationSettings(FidesSettings): @validator("notification_service_type", pre=True) @classmethod - def validate_notification_service_type(cls, value: Optional[str]) -> str: + def validate_notification_service_type(cls, value: Optional[str]) -> Optional[str]: """Ensure the provided type is a valid value.""" if value: - valid_values = [ - "MAILGUN", - "TWILIO_TEXT", - "TWILIO_EMAIL" - ] + valid_values = ["MAILGUN", "TWILIO_TEXT", "TWILIO_EMAIL"] value = value.upper() # force lowercase for safety if value not in valid_values: diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 30792e3476c..90946108613 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -143,7 +143,9 @@ def test_post_email_config_with_key( assert 200 == response.status_code response_body = json.loads(response.text) - email_config = db.query(MessagingConfig).filter_by(key="my_mailgun_messaging_config")[0] + email_config = db.query(MessagingConfig).filter_by( + key="my_mailgun_messaging_config" + )[0] expected_response = { "key": "my_mailgun_messaging_config", @@ -270,7 +272,9 @@ def test_patch_email_config_with_key( assert 200 == response.status_code response_body = json.loads(response.text) - email_config = db.query(MessagingConfig).filter_by(key="my_mailgun_messaging_config")[0] + email_config = db.query(MessagingConfig).filter_by( + key="my_mailgun_messaging_config" + )[0] expected_response = { "key": "my_mailgun_messaging_config", diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index 4d32ec54d26..823b4071cf8 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -11,8 +11,8 @@ from fides.api.ops.models.policy import CurrentStep from fides.api.ops.models.privacy_request import CheckpointActionRequired, ManualAction from fides.api.ops.schemas.messaging.messaging import ( - FidesopsMessage, EmailForActionType, + FidesopsMessage, MessagingActionType, MessagingServiceDetails, MessagingServiceType, @@ -36,7 +36,9 @@ def test_email_dispatch_mailgun_success( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -64,7 +66,9 @@ def test_email_dispatch_mailgun_config_not_found( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -98,7 +102,9 @@ def test_email_dispatch_mailgun_config_no_secrets( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -128,7 +134,9 @@ def test_email_dispatch_mailgun_failed_email(db: Session, messaging_config) -> N db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method(MessagingServiceType[CONFIG.notifications.notification_service_type]), + messaging_method=get_messaging_method( + CONFIG.notifications.notification_service_type + ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), From 2b11a81d294ca0e0529e2b4313cc7fc7a4eb6fc3 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 31 Oct 2022 07:37:06 -0600 Subject: [PATCH 05/28] update twilio postman --- .../postman/Fides.postman_collection.json | 146 +++++++++++++++++- 1 file changed, 143 insertions(+), 3 deletions(-) diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index 371bd788da3..6e1ea6d23e3 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -73,7 +73,7 @@ "header": [], "body": { "mode": "raw", - "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"consent:read\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"email:create_or_update\",\n \"email:read\",\n \"email:delete\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", + "raw": "[\n \"client:create\",\n \"client:update\",\n \"client:read\",\n \"client:delete\",\n \"config:read\",\n \"connection_type:read\",\n \"connection:read\",\n \"connection:create_or_update\",\n \"connection:delete\",\n \"connection:instantiate\",\n \"consent:read\",\n \"dataset:create_or_update\",\n \"dataset:delete\",\n \"dataset:read\",\n \"encryption:exec\",\n \"messaging:create_or_update\",\n \"messaging:read\",\n \"messaging:delete\",\n \"policy:create_or_update\",\n \"policy:read\",\n \"policy:delete\",\n \"privacy-request:read\",\n \"privacy-request:delete\",\n \"rule:create_or_update\",\n \"rule:read\",\n \"rule:delete\",\n \"scope:read\",\n \"storage:create_or_update\",\n \"storage:delete\",\n \"storage:read\",\n \"privacy-request:resume\",\n \"webhook:create_or_update\",\n \"webhook:read\",\n \"webhook:delete\",\n \"saas_config:create_or_update\",\n \"saas_config:read\",\n \"saas_config:delete\",\n \"privacy-request:review\",\n \"user:create\",\n \"user:delete\"\n]", "options": { "raw": { "language": "json" @@ -4105,7 +4105,7 @@ ] }, { - "name": "Primary Messaging Config", + "name": "Messaging Config - Email", "item": [ { "name": "Post Messaging Config", @@ -4280,7 +4280,7 @@ "response": [] }, { - "name": "Email Config Secrets", + "name": "Messaging Config Secrets", "request": { "auth": { "type": "bearer", @@ -4320,6 +4320,126 @@ } ] }, + { + "name": "Messaging Config - SMS", + "item": [ + { + "name": "Post Messaging Config", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "POST", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/", + "host": [ + "{{host}}" + ], + "path": [ + "messaging", + "config", + "" + ] + } + }, + "response": [] + }, + { + "name": "Patch Messaging Config By Key", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"key\": \"{{twilio_config_key}}\",\n \"name\": \"twilio\",\n \"service_type\": \"twilio_text\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{twilio_config_key}}", + "host": [ + "{{host}}" + ], + "path": [ + "messaging", + "config", + "{{twilio_config_key}}" + ] + } + }, + "response": [] + }, + { + "name": "Messaging Config Secrets", + "request": { + "auth": { + "type": "bearer", + "bearer": [ + { + "key": "token", + "value": "{{client_token}}", + "type": "string" + } + ] + }, + "method": "PUT", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"twilio_account_sid\": \"{{twilio_account_sid}}\",\n \"twilio_auth_token\": \"{{twilio_auth_token}}\",\n \"twilio_messaging_service_sid\": \"{{twilio_messaging_service_id}}\"\n}\n\n", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/messaging/config/{{twilio_config_key}}/secret", + "host": [ + "{{host}}" + ], + "path": [ + "messaging", + "config", + "{{twilio_config_key}}", + "secret" + ] + } + }, + "response": [] + } + ] + }, { "name": "Identity Verification", "item": [ @@ -4786,6 +4906,26 @@ "key": "verification_code", "value": "", "type": "string" + }, + { + "key": "twilio_config_key", + "value": "my_twilio_config", + "type": "string" + }, + { + "key": "twilio_account_sid", + "value": "", + "type": "string" + }, + { + "key": "twilio_auth_token", + "value": "", + "type": "string" + }, + { + "key": "twilio_messaging_service_id", + "value": "", + "type": "string" } ] } \ No newline at end of file From bb74f61dd519669c734fdfde25961d0b0363e6ce Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 31 Oct 2022 12:31:11 -0600 Subject: [PATCH 06/28] regenerate migration --- ...py => 179f2bb623ae_update_table_for_twilio.py} | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) rename src/fides/api/ctl/migrations/versions/{2d5ff3096959_update_table_for_twilio.py => 179f2bb623ae_update_table_for_twilio.py} (96%) diff --git a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py similarity index 96% rename from src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py rename to src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py index 2f4e18da9f2..a69024fa397 100644 --- a/src/fides/api/ctl/migrations/versions/2d5ff3096959_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py @@ -1,16 +1,17 @@ """Update table for twilio -Revision ID: 2d5ff3096959 -Revises: fb6b0150d6e4 -Create Date: 2022-10-21 22:10:48.899562 + +Revision ID: 179f2bb623ae +Revises: 8f1a19465239 +Create Date: 2022-10-31 18:19:26.845723 + """ -import sqlalchemy as sa -import sqlalchemy_utils from alembic import op +import sqlalchemy as sa from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = "2d5ff3096959" -down_revision = "fb6b0150d6e4" +revision = '179f2bb623ae' +down_revision = '8f1a19465239' branch_labels = None depends_on = None From 6adcf54e92d6ea99b6e22522006249ad8419bb42 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 31 Oct 2022 15:25:22 -0600 Subject: [PATCH 07/28] adds sqlalchemy import --- .../migrations/versions/179f2bb623ae_update_table_for_twilio.py | 1 + 1 file changed, 1 insertion(+) diff --git a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py index a69024fa397..2e553b06a0e 100644 --- a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py @@ -6,6 +6,7 @@ """ from alembic import op +import sqlalchemy_utils import sqlalchemy as sa from sqlalchemy.dialects import postgresql From c4dfd2c568ed9e0dea15898fa302c5f669323db3 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 31 Oct 2022 16:22:24 -0600 Subject: [PATCH 08/28] rename emailconfig to messagingconfig in db_dataset.yml --- .fides/db_dataset.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 0b5678ef35e..4d6dd204fd4 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -871,8 +871,8 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - - name: emailconfig - description: 'Fides Generated Description for Table: emailconfig' + - name: messagingconfig + description: 'Fides Generated Description for Table: messagingconfig' data_categories: [] data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified fields: From 7ebbbbd984389cf636f89a6d4c839aa549c5651f Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Tue, 1 Nov 2022 09:44:37 -0600 Subject: [PATCH 09/28] more renaming, adding validator for twilio sms secrets, add mypy ignore for twilio module --- pyproject.toml | 1 + .../179f2bb623ae_update_table_for_twilio.py | 8 ++--- .../api/v1/endpoints/messaging_endpoints.py | 2 +- .../v1/endpoints/privacy_request_endpoints.py | 8 ++--- .../api/ops/schemas/messaging/messaging.py | 28 ++++++++++++----- .../messaging/messaging_secrets_docs_only.py | 17 +++++++++-- .../ops/service/connectors/email_connector.py | 4 +-- .../messaging/message_dispatch_service.py | 6 ++-- .../ctl/core/config/notification_settings.py | 2 +- .../v1/endpoints/test_messaging_endpoints.py | 10 +++---- .../test_privacy_request_endpoints.py | 30 +++++++++---------- tests/ops/conftest.py | 8 ++--- 12 files changed, 74 insertions(+), 50 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 1c9a6300520..2d382c793e9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,6 +57,7 @@ module = [ "sqlalchemy.ext.*", "sqlalchemy.future.*", "sqlalchemy_utils.*", + "twilio.*", "uvicorn.*" ] ignore_missing_imports = true diff --git a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py index 2e553b06a0e..68dc4125083 100644 --- a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py @@ -5,14 +5,14 @@ Create Date: 2022-10-31 18:19:26.845723 """ -from alembic import op -import sqlalchemy_utils import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. -revision = '179f2bb623ae' -down_revision = '8f1a19465239' +revision = "179f2bb623ae" +down_revision = "8f1a19465239" branch_labels = None depends_on = None diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py index e1e2dd6a66d..5cf7c365f1a 100644 --- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -65,7 +65,7 @@ def post_config( messaging_config: MessagingConfigRequest, ) -> MessagingConfigResponse: """ - Given a messaging config, create corresponding EmailConfig object, provided no config already exists + Given a messaging config, create corresponding MessagingConfig object, provided no config already exists """ try: diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py index 56ebe3e2689..70a966fa7ff 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_request_endpoints.py @@ -297,18 +297,18 @@ async def create_privacy_request( def _send_privacy_request_receipt_message_to_user( policy: Optional[Policy], to_identity: Optional[Identity] ) -> None: - """Helper function to send request receipt email to the user""" + """Helper function to send request receipt message to the user""" if not to_identity: logger.error( IdentityNotFoundException( - "Identity was not found, so request receipt email could not be sent." + "Identity was not found, so request receipt message could not be sent." ) ) return if not policy: logger.error( PolicyNotFoundException( - "Policy was not found, so request receipt email could not be sent." + "Policy was not found, so request receipt message could not be sent." ) ) return @@ -1159,7 +1159,7 @@ def _send_privacy_request_review_message_to_user( if not identity_data: logger.error( IdentityNotFoundException( - "Identity was not found, so request review email could not be sent." + "Identity was not found, so request review message could not be sent." ) ) to_identity: Identity = Identity( diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index c3477f9551d..5c4bf5654b4 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -1,7 +1,8 @@ from enum import Enum +from re import compile as regex from typing import Any, Dict, List, Optional, Union -from pydantic import BaseModel, Extra +from pydantic import BaseModel, Extra, root_validator from fides.api.ops.models.privacy_request import CheckpointActionRequired from fides.api.ops.schemas import Msg @@ -134,9 +135,7 @@ class MessagingServiceSecrets(Enum): TWILIO_ACCOUNT_SID = "twilio_account_sid" TWILIO_AUTH_TOKEN = "twilio_auth_token" TWILIO_MESSAGING_SERVICE_SID = "twilio_messaging_service_sid" - TWILIO_SENDER_PHONE_NUMBER = ( - "twilio_sender_phone_number" # formatted like +15558675309 - ) + TWILIO_SENDER_PHONE_NUMBER = "twilio_sender_phone_number" # Twilio Sendgrid/Email TWILIO_API_KEY = "twilio_api_key" @@ -159,15 +158,28 @@ class MessagingServiceSecretsTwilioSMS(BaseModel): twilio_account_sid: str twilio_auth_token: str twilio_messaging_service_sid: Optional[str] - twilio_sender_phone_number: Optional[ - str - ] # Either the twilio_messaging_service_id *OR* the twilio_sender_phone_number should be supplied. + twilio_sender_phone_number: Optional[str] class Config: """Restrict adding other fields through this schema.""" extra = Extra.forbid + @root_validator + def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + sender_phone = values.get("twilio_sender_phone_number") + if not values.get("twilio_messaging_service_sid") or sender_phone: + raise ValueError( + "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." + ) + if sender_phone: + pattern = regex(r"^\+\d+$") + if not pattern.search(sender_phone): + raise ValueError( + "Sender phone number must include country code, formatted like +15558675309" + ) + return values + class MessagingServiceSecretsTwilioEmail(BaseModel): """The secrets required to connect to twilio email.""" @@ -186,7 +198,7 @@ class MessagingConfigRequest(BaseModel): name: str key: Optional[FidesOpsKey] service_type: MessagingServiceType - details: Optional[Union[MessagingServiceDetailsMailgun]] + details: Optional[MessagingServiceDetailsMailgun] class Config: use_enum_values = False diff --git a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py index ab3860a1dec..18e24a52f46 100644 --- a/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py +++ b/src/fides/api/ops/schemas/messaging/messaging_secrets_docs_only.py @@ -3,6 +3,7 @@ from fides.api.ops.schemas.base_class import NoValidationSchema from fides.api.ops.schemas.messaging.messaging import ( MessagingServiceSecretsMailgun, + MessagingServiceSecretsTwilioEmail, MessagingServiceSecretsTwilioSMS, ) @@ -11,10 +12,20 @@ class MessagingSecretsMailgunDocs(MessagingServiceSecretsMailgun, NoValidationSc """The secrets required to connect to Mailgun, for documentation""" -class MessagingSecretsTwilioDocs(MessagingServiceSecretsTwilioSMS, NoValidationSchema): - """The secrets required to connect to Twilio, for documentation""" +class MessagingSecretsTwilioSMSDocs( + MessagingServiceSecretsTwilioSMS, NoValidationSchema +): + """The secrets required to connect to Twilio sms, for documentation""" + + +class MessagingSecretsTwilioEmailDocs( + MessagingServiceSecretsTwilioEmail, NoValidationSchema +): + """The secrets required to connect to Twilio email, for documentation""" possible_messaging_secrets = Union[ - MessagingSecretsMailgunDocs, MessagingSecretsTwilioDocs + MessagingSecretsMailgunDocs, + MessagingSecretsTwilioSMSDocs, + MessagingSecretsTwilioEmailDocs, ] diff --git a/src/fides/api/ops/service/connectors/email_connector.py b/src/fides/api/ops/service/connectors/email_connector.py index 7f8fa717498..c85447ea152 100644 --- a/src/fides/api/ops/service/connectors/email_connector.py +++ b/src/fides/api/ops/service/connectors/email_connector.py @@ -63,7 +63,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]: dispatch_message( db=db, action_type=MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, - to_identity=Identity(**{"email": config.test_email}), + to_identity=Identity(email=config.test_email), messaging_method=MessagingMethod.EMAIL, message_body_params=[ CheckpointActionRequired( @@ -202,7 +202,7 @@ def email_connector_erasure_send(db: Session, privacy_request: PrivacyRequest) - dispatch_message( db, action_type=MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT, - to_identity=Identity(**{"email": cc.secrets.get("to_email")}), + to_identity=Identity(email=cc.secrets.get("to_email")), messaging_method=MessagingMethod.EMAIL, message_body_params=template_values, ) diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index 1aaf2ebf5b0..e63d348ffb1 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -5,8 +5,8 @@ import requests from sqlalchemy.orm import Session -from twilio.base.exceptions import TwilioRestException # type: ignore -from twilio.rest import Client # type: ignore +from twilio.base.exceptions import TwilioRestException +from twilio.rest import Client from fides.api.ops.common_exceptions import MessageDispatchException from fides.api.ops.email_templates import get_email_template @@ -118,7 +118,7 @@ def dispatch_message( f"Dispatcher has not been implemented for message service type: {messaging_service}" ) logger.info( - "Starting email dispatch for messaging service with action type: %s", + "Starting message dispatch for messaging service with action type: %s", action_type, ) dispatcher( diff --git a/src/fides/ctl/core/config/notification_settings.py b/src/fides/ctl/core/config/notification_settings.py index 8da188df1ab..efd0e71390c 100644 --- a/src/fides/ctl/core/config/notification_settings.py +++ b/src/fides/ctl/core/config/notification_settings.py @@ -24,7 +24,7 @@ def validate_notification_service_type(cls, value: Optional[str]) -> Optional[st """Ensure the provided type is a valid value.""" if value: valid_values = ["MAILGUN", "TWILIO_TEXT", "TWILIO_EMAIL"] - value = value.upper() # force lowercase for safety + value = value.upper() # force uppercase for safety if value not in valid_values: raise ValueError( diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 90946108613..091ce9e6737 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -26,7 +26,7 @@ PAGE_SIZE = Params().size -class TestPostEmailConfig: +class TestPostMessagingConfig: @pytest.fixture(scope="function") def url(self) -> str: return V1_URL_PREFIX + MESSAGING_CONFIG @@ -208,7 +208,7 @@ def test_post_email_config_already_exists( } # fixme- what's the error here? -class TestPatchEmailConfig: +class TestPatchMessagingConfig: @pytest.fixture(scope="function") def url(self, messaging_config) -> str: return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( @@ -290,7 +290,7 @@ def test_patch_email_config_with_key( email_config.delete(db) -class TestPutEmailConfigSecretsMailgun: +class TestPutMessagingConfigSecretsMailgun: @pytest.fixture(scope="function") def url(self, messaging_config) -> str: return (V1_URL_PREFIX + MESSAGING_SECRETS).format( @@ -364,7 +364,7 @@ def test_put_config_secrets( ) -class TestGetEmailConfigs: +class TestGetMessagingConfigs: @pytest.fixture(scope="function") def url(self) -> str: return V1_URL_PREFIX + MESSAGING_CONFIG @@ -408,7 +408,7 @@ def test_get_configs( assert expected_response == response_body -class TestGetEmailConfig: +class TestGetMessagingConfig: @pytest.fixture(scope="function") def url(self, messaging_config) -> str: return (V1_URL_PREFIX + MESSAGING_BY_KEY).format( diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 56bb986ba2f..dc0a6ca262d 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -1693,8 +1693,8 @@ def url(self, db, privacy_request): return V1_URL_PREFIX + PRIVACY_REQUEST_APPROVE @pytest.fixture(scope="function") - def privacy_request_review_email_notification_enabled(self): - """Enable request review email""" + def privacy_request_review_notification_enabled(self): + """Enable request review notification""" original_value = CONFIG.notifications.send_request_review_notification CONFIG.notifications.send_request_review_notification = True yield @@ -1861,7 +1861,7 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( generate_auth_header, user, privacy_request_status_pending, - privacy_request_review_email_notification_enabled, + privacy_request_review_notification_enabled, ): payload = { JWE_PAYLOAD_SCOPES: user.client.scopes, @@ -1909,8 +1909,8 @@ def url(self, db, privacy_request): return V1_URL_PREFIX + PRIVACY_REQUEST_DENY @pytest.fixture(autouse=True, scope="function") - def privacy_request_review_email_notification_enabled(self): - """Enable request review email""" + def privacy_request_review_notification_enabled(self): + """Enable request review notification""" original_value = CONFIG.notifications.send_request_review_notification CONFIG.notifications.send_request_review_notification = True yield @@ -2856,8 +2856,8 @@ def url(self, db, privacy_request): ) @pytest.fixture(scope="function") - def privacy_request_receipt_email_notification_enabled(self): - """Enable request receipt email""" + def privacy_request_receipt_notification_enabled(self): + """Enable request receipt""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = True yield @@ -2882,7 +2882,7 @@ def test_verification_code_expired( api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -2906,7 +2906,7 @@ def test_invalid_code( api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -2935,7 +2935,7 @@ def test_verify_identity_no_admin_approval_needed( api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -3041,7 +3041,7 @@ def test_verify_identity_admin_approval_needed( api_client, url, privacy_request, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_notification_enabled, ): privacy_request.status = PrivacyRequestStatus.identity_unverified privacy_request.save(db) @@ -3646,8 +3646,8 @@ def url(self, oauth_client: ClientDetail, policy) -> str: return V1_URL_PREFIX + PRIVACY_REQUESTS @pytest.fixture(scope="function") - def privacy_request_receipt_email_notification_enabled(self): - """Enable request receipt email""" + def privacy_request_receipt_notification_enabled(self): + """Enable request receipt notification""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = True yield @@ -3667,7 +3667,7 @@ def test_create_privacy_request_no_email_config( db, api_client: TestClient, policy, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_email_enabled, ): data = [ { @@ -3718,7 +3718,7 @@ def test_create_privacy_request_with_email_config( api_client: TestClient, policy, messaging_config, - privacy_request_receipt_email_notification_enabled, + privacy_request_receipt_email_enabled, ): data = [ { diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 3fa163b9202..e1217ad1291 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -283,8 +283,8 @@ def privacy_request_complete_email_notification_disabled(): @pytest.fixture(autouse=True, scope="function") -def privacy_request_receipt_email_notification_disabled(): - """Disable request receipt email for most tests unless overridden""" +def privacy_request_receipt_notification_disabled(): + """Disable request receipt notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_receipt_notification CONFIG.notifications.send_request_receipt_notification = False yield @@ -292,8 +292,8 @@ def privacy_request_receipt_email_notification_disabled(): @pytest.fixture(autouse=True, scope="function") -def privacy_request_review_email_notification_disabled(): - """Disable request review email for most tests unless overridden""" +def privacy_request_review_notification_disabled(): + """Disable request review notification for most tests unless overridden""" original_value = CONFIG.notifications.send_request_review_notification CONFIG.notifications.send_request_review_notification = False yield From a510b905fbd904bc2a6c67f89dba8d7914417fb7 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Tue, 1 Nov 2022 14:26:23 -0600 Subject: [PATCH 10/28] removing original email config table migration --- .../179f2bb623ae_update_table_for_twilio.py | 5 --- .../versions/c61fd9d4f73e_emailconfig.py | 45 ++----------------- 2 files changed, 3 insertions(+), 47 deletions(-) diff --git a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py index 68dc4125083..30911f27480 100644 --- a/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py +++ b/src/fides/api/ctl/migrations/versions/179f2bb623ae_update_table_for_twilio.py @@ -66,11 +66,6 @@ def upgrade(): ["service_type"], unique=True, ) - op.drop_index("ix_emailconfig_id", table_name="emailconfig") - op.drop_index("ix_emailconfig_key", table_name="emailconfig") - op.drop_index("ix_emailconfig_name", table_name="emailconfig") - op.drop_index("ix_emailconfig_service_type", table_name="emailconfig") - op.drop_table("emailconfig") # ### end Alembic commands ### diff --git a/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py b/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py index 3aa4320c3b0..bbddb080fcf 100644 --- a/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py +++ b/src/fides/api/ctl/migrations/versions/c61fd9d4f73e_emailconfig.py @@ -6,12 +6,10 @@ Create Date: 2022-08-10 19:29:12.119401 """ -import sqlalchemy as sa -import sqlalchemy_utils from alembic import op -from sqlalchemy.dialects import postgresql # revision identifiers, used by Alembic. + revision = "c61fd9d4f73e" down_revision = "7abe778b7082" branch_labels = None @@ -19,45 +17,8 @@ def upgrade(): - # ### commands auto generated by Alembic - please adjust! ### - op.create_table( - "emailconfig", - sa.Column("id", sa.String(length=255), nullable=False), - sa.Column( - "created_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - ), - sa.Column( - "updated_at", - sa.DateTime(timezone=True), - server_default=sa.text("now()"), - nullable=True, - ), - sa.Column("key", sa.String(), nullable=False), - sa.Column("name", sa.String(), nullable=True), - sa.Column( - "service_type", sa.Enum("MAILGUN", name="emailservicetype"), nullable=False - ), - sa.Column("details", postgresql.JSONB(astext_type=sa.Text()), nullable=False), - sa.Column( - "secrets", - sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), - nullable=True, - ), - sa.PrimaryKeyConstraint("id"), - ) - op.create_index(op.f("ix_emailconfig_id"), "emailconfig", ["id"], unique=False) - op.create_index(op.f("ix_emailconfig_key"), "emailconfig", ["key"], unique=True) - op.create_index(op.f("ix_emailconfig_name"), "emailconfig", ["name"], unique=True) - op.create_index( - op.f("ix_emailconfig_service_type"), - "emailconfig", - ["service_type"], - unique=False, - ) - # ### end Alembic commands ### + # Removing due to the following error: type "emailservicetype" already exists + pass def downgrade(): From 201e2c6ace3c5430cfe898e16747ca1bd3858a06 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Wed, 2 Nov 2022 11:55:54 -0600 Subject: [PATCH 11/28] rename dispatch_email to dispatch_message --- .../test_connection_config_endpoints.py | 8 +-- .../test_consent_request_endpoints.py | 18 +++--- .../test_privacy_request_endpoints.py | 64 +++++++++---------- .../test_integration_email.py | 2 +- .../request_runner_service_test.py | 18 +++--- 5 files changed, 55 insertions(+), 55 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index df9134aa882..ba3f30d6953 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -1421,10 +1421,10 @@ def test_put_saas_example_connection_config_secrets_missing_saas_config( == f"A SaaS config to validate the secrets is unavailable for this connection config, please add one via {SAAS_CONFIG}" ) - @mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_email") + @mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_message") def test_put_email_connection_config_secrets( self, - mock_dispatch_email, + mock_dispatch_message, api_client: TestClient, db: Session, generate_auth_header, @@ -1461,8 +1461,8 @@ def test_put_email_connection_config_secrets( assert email_connection_config.last_test_timestamp is not None assert email_connection_config.last_test_succeeded is not None - assert mock_dispatch_email.called - kwargs = mock_dispatch_email.call_args.kwargs + assert mock_dispatch_message.called + kwargs = mock_dispatch_message.call_args.kwargs assert ( kwargs["action_type"] == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index d8798e724f9..1178252ada9 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -61,12 +61,12 @@ def url(self) -> str: "email_dataset_config", "subject_identity_verification_required", ) - @patch("fides.api.ops.service._verification.dispatch_email") - def test_consent_request(self, mock_dispatch_email, api_client, url): + @patch("fides.api.ops.service._verification.dispatch_message") + def test_consent_request(self, mock_dispatch_message, api_client, url): data = {"email": "test@example.com"} response = api_client.post(url, json=data) assert response.status_code == 200 - assert mock_dispatch_email.called + assert mock_dispatch_message.called @pytest.mark.usefixtures( "email_config", @@ -74,10 +74,10 @@ def test_consent_request(self, mock_dispatch_email, api_client, url): "email_dataset_config", "subject_identity_verification_required", ) - @patch("fides.api.ops.service._verification.dispatch_email") + @patch("fides.api.ops.service._verification.dispatch_message") def test_consent_request_identity_present( self, - mock_dispatch_email, + mock_dispatch_message, provided_identity_and_consent_request, api_client, url, @@ -86,7 +86,7 @@ def test_consent_request_identity_present( data = {"email": provided_identity.encrypted_value["value"]} response = api_client.post(url, json=data) assert response.status_code == 200 - assert mock_dispatch_email.called + assert mock_dispatch_message.called @pytest.mark.usefixtures( "email_config", @@ -106,14 +106,14 @@ def test_consent_request_redis_disabled(self, api_client, url): "email_connection_config", "email_dataset_config", ) - @patch("fides.api.ops.service._verification.dispatch_email") + @patch("fides.api.ops.service._verification.dispatch_message") def test_consent_request_subject_verification_disabled_no_email( - self, mock_dispatch_email, api_client, url + self, mock_dispatch_message, api_client, url ): data = {"email": "test@example.com"} response = api_client.post(url, json=data) assert response.status_code == 200 - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @pytest.mark.usefixtures( "email_config", diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index dc0a6ca262d..9b7517a463b 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -100,7 +100,7 @@ def url(self, oauth_client: ClientDetail, policy) -> str: ) def test_create_privacy_request( self, - mock_dispatch_email, + mock_dispatch_message, run_access_request_mock, url, db, @@ -121,7 +121,7 @@ def test_create_privacy_request( pr = PrivacyRequest.get(db=db, object_id=response_data[0]["id"]) pr.delete(db=db) assert run_access_request_mock.called - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" @@ -1805,7 +1805,7 @@ def test_approve_privacy_request_no_user_on_client( ) def test_approve_privacy_request( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -1841,7 +1841,7 @@ def test_approve_privacy_request( assert response_body["succeeded"][0]["reviewed_by"] == user.id assert submit_mock.called - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called privacy_request.delete(db) @@ -1853,7 +1853,7 @@ def test_approve_privacy_request( ) def test_approve_privacy_request_creates_audit_log_and_sends_email( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -1889,7 +1889,7 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( approval_audit_log.delete(db) - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" @@ -1977,7 +1977,7 @@ def test_deny_completed_privacy_request( ) def test_deny_privacy_request_without_denial_reason( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -2018,7 +2018,7 @@ def test_deny_privacy_request_without_denial_reason( ), ).first() - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" @@ -2047,7 +2047,7 @@ def test_deny_privacy_request_without_denial_reason( ) def test_deny_privacy_request_with_denial_reason( self, - mock_dispatch_email, + mock_dispatch_message, submit_mock, db, url, @@ -2088,7 +2088,7 @@ def test_deny_privacy_request_with_denial_reason( ), ).first() - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" @@ -2877,7 +2877,7 @@ def test_incorrect_privacy_request_status(self, api_client, url, privacy_request ) def test_verification_code_expired( self, - mock_dispatch_email, + mock_dispatch_message, db, api_client, url, @@ -2894,14 +2894,14 @@ def test_verification_code_expired( resp.json()["detail"] == f"Identification code expired for {privacy_request.id}." ) - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.api.v1.endpoints.privacy_request_endpoints.dispatch_message_task.apply_async" ) def test_invalid_code( self, - mock_dispatch_email, + mock_dispatch_message, db, api_client, url, @@ -2919,7 +2919,7 @@ def test_invalid_code( resp.json()["detail"] == f"Incorrect identification code for '{privacy_request.id}'" ) - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" @@ -2929,7 +2929,7 @@ def test_invalid_code( ) def test_verify_identity_no_admin_approval_needed( self, - mock_dispatch_email, + mock_dispatch_message, mock_run_privacy_request, db, api_client, @@ -2965,9 +2965,9 @@ def test_verify_identity_no_admin_approval_needed( assert mock_run_privacy_request.called - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" @@ -2989,7 +2989,7 @@ def test_verify_identity_no_admin_approval_needed( ) def test_verify_identity_no_admin_approval_needed_email_disabled( self, - mock_dispatch_email, + mock_dispatch_message, mock_run_privacy_request, db, api_client, @@ -3024,7 +3024,7 @@ def test_verify_identity_no_admin_approval_needed_email_disabled( assert mock_run_privacy_request.called - assert not mock_dispatch_email.called + assert not mock_dispatch_message.called @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" @@ -3034,7 +3034,7 @@ def test_verify_identity_no_admin_approval_needed_email_disabled( ) def test_verify_identity_admin_approval_needed( self, - mock_dispatch_email, + mock_dispatch_message, mock_run_privacy_request, require_manual_request_approval, db, @@ -3070,9 +3070,9 @@ def test_verify_identity_admin_approval_needed( assert approved_audit_log is None assert not mock_run_privacy_request.called - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" @@ -3132,10 +3132,10 @@ def test_create_privacy_request_no_email_config( @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) - @mock.patch("fides.api.ops.service._verification.dispatch_email") + @mock.patch("fides.api.ops.service._verification.dispatch_message") def test_create_privacy_request_with_email_config( self, - mock_dispatch_email, + mock_dispatch_message, mock_execute_request, url, db, @@ -3168,8 +3168,8 @@ def test_create_privacy_request_with_email_config( assert response_data[0]["status"] == PrivacyRequestStatus.identity_unverified - assert mock_dispatch_email.called - kwargs = mock_dispatch_email.call_args.kwargs + assert mock_dispatch_message.called + kwargs = mock_dispatch_message.call_args.kwargs assert ( kwargs["action_type"] == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION ) @@ -3661,7 +3661,7 @@ def privacy_request_receipt_notification_enabled(self): ) def test_create_privacy_request_no_email_config( self, - mock_dispatch_email, + mock_dispatch_message, mock_execute_request, url, db, @@ -3685,9 +3685,9 @@ def test_create_privacy_request_no_email_config( assert mock_execute_request.called assert response_data[0]["status"] == PrivacyRequestStatus.pending - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" @@ -3711,7 +3711,7 @@ def test_create_privacy_request_no_email_config( ) def test_create_privacy_request_with_email_config( self, - mock_dispatch_email, + mock_dispatch_message, mock_execute_request, url, db, @@ -3735,9 +3735,9 @@ def test_create_privacy_request_with_email_config( assert mock_execute_request.called assert response_data[0]["status"] == PrivacyRequestStatus.pending - assert mock_dispatch_email.called + assert mock_dispatch_message.called - call_args = mock_dispatch_email.call_args[1] + call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] assert task_kwargs["to_email"] == "test@example.com" diff --git a/tests/ops/integration_tests/test_integration_email.py b/tests/ops/integration_tests/test_integration_email.py index ad970dbc83b..6c38cf0ed26 100644 --- a/tests/ops/integration_tests/test_integration_email.py +++ b/tests/ops/integration_tests/test_integration_email.py @@ -23,7 +23,7 @@ @pytest.mark.integration_postgres @pytest.mark.integration -@mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_email") +@mock.patch("fides.api.ops.service.connectors.email_connector.dispatch_message") @pytest.mark.asyncio async def test_email_connector_cache_and_delayed_send( mock_email_dispatch, diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index bf119ea6798..ddb5053a8dc 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -71,10 +71,10 @@ def privacy_request_complete_email_notification_enabled(): @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") -def test_policy_upload_dispatch_email_called( +def test_policy_upload_dispatch_message_called( upload_mock: Mock, mock_email_dispatch: Mock, privacy_request_status_pending: PrivacyRequest, @@ -90,7 +90,7 @@ def test_policy_upload_dispatch_email_called( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_start_processing_sets_started_processing_at( @@ -116,7 +116,7 @@ def test_start_processing_sets_started_processing_at( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_start_processing_doesnt_overwrite_started_processing_at( @@ -167,7 +167,7 @@ def test_halts_proceeding_if_cancelled( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.upload_access_results" @@ -217,7 +217,7 @@ def test_from_graph_resume_does_not_run_pre_webhooks( @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_webhooks_and_report_status", @@ -1855,7 +1855,7 @@ def privacy_request_complete_email_notification_enabled(self): @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) def test_email_complete_send_erasure( self, @@ -1891,7 +1891,7 @@ def test_email_complete_send_erasure( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_email_complete_send_access( @@ -1930,7 +1930,7 @@ def test_email_complete_send_access( @pytest.mark.integration_postgres @pytest.mark.integration @mock.patch( - "fides.api.ops.service.privacy_request.request_runner_service.dispatch_email" + "fides.api.ops.service.privacy_request.request_runner_service.dispatch_message" ) @mock.patch("fides.api.ops.service.privacy_request.request_runner_service.upload") def test_email_complete_send_access_and_erasure( From aebb9010f0124c43327b5c03f6be23d56ba60d4b Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Wed, 2 Nov 2022 14:19:00 -0600 Subject: [PATCH 12/28] fixes some test data --- .../test_privacy_request_endpoints.py | 26 ++++++++++++------- .../test_integration_email.py | 6 +++-- .../message_dispatch_service_test.py | 8 +++--- .../request_runner_service_test.py | 12 ++++++--- 4 files changed, 33 insertions(+), 19 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 9b7517a463b..9d09bb92459 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -67,7 +67,7 @@ MessagingActionType, RequestReceiptBodyParams, RequestReviewDenyBodyParams, - SubjectIdentityVerificationBodyParams, + SubjectIdentityVerificationBodyParams, MessagingMethod, ) from fides.api.ops.schemas.policy import PolicyResponse from fides.api.ops.schemas.redis_cache import Identity @@ -1891,7 +1891,8 @@ def test_approve_privacy_request_creates_audit_log_and_sends_email( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( @@ -2020,7 +2021,8 @@ def test_deny_privacy_request_without_denial_reason( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( @@ -2090,7 +2092,8 @@ def test_deny_privacy_request_with_denial_reason( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( @@ -2969,7 +2972,8 @@ def test_verify_identity_no_admin_approval_needed( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( @@ -3074,7 +3078,8 @@ def test_verify_identity_admin_approval_needed( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( @@ -3173,7 +3178,8 @@ def test_create_privacy_request_with_email_config( assert ( kwargs["action_type"] == MessagingActionType.SUBJECT_IDENTITY_VERIFICATION ) - assert kwargs["to_email"] == "test@example.com" + assert kwargs["to_identity"] == Identity(email="test@example.com") + assert kwargs["messaging_method"] == MessagingMethod.EMAIL assert kwargs["message_body_params"] == SubjectIdentityVerificationBodyParams( verification_code=pr.get_cached_verification_code(), verification_code_ttl_seconds=CONFIG.redis.identity_verification_code_ttl_seconds, @@ -3689,7 +3695,8 @@ def test_create_privacy_request_no_email_config( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( @@ -3739,7 +3746,8 @@ def test_create_privacy_request_with_email_config( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_email"] == "test@example.com" + assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] assert ( diff --git a/tests/ops/integration_tests/test_integration_email.py b/tests/ops/integration_tests/test_integration_email.py index 6c38cf0ed26..286bf86e4bc 100644 --- a/tests/ops/integration_tests/test_integration_email.py +++ b/tests/ops/integration_tests/test_integration_email.py @@ -14,7 +14,8 @@ ManualAction, ) from fides.api.ops.schemas.dataset import FidesopsDataset -from fides.api.ops.schemas.messaging.messaging import MessagingActionType +from fides.api.ops.schemas.messaging.messaging import MessagingActionType, MessagingMethod +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( email_connector_erasure_send, ) @@ -182,7 +183,8 @@ async def test_email_connector_cache_and_delayed_send( call_args["action_type"] == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT ) - assert call_args["to_email"] == "test@example.com" + assert call_args["to_identity"] == Identity(email="test@example.com") + assert call_args["messaging_method"] == MessagingMethod.EMAIL assert call_args["message_body_params"] == raw_email_template_values created_email_audit_log = ( diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index 823b4071cf8..a438efa54ee 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -37,7 +37,7 @@ def test_email_dispatch_mailgun_success( action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), messaging_method=get_messaging_method( - CONFIG.notifications.notification_service_type + MessagingServiceType.MAILGUN.value ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 @@ -67,7 +67,7 @@ def test_email_dispatch_mailgun_config_not_found( action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), messaging_method=get_messaging_method( - CONFIG.notifications.notification_service_type + MessagingServiceType.MAILGUN.value ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 @@ -103,7 +103,7 @@ def test_email_dispatch_mailgun_config_no_secrets( action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), messaging_method=get_messaging_method( - CONFIG.notifications.notification_service_type + MessagingServiceType.MAILGUN.value ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 @@ -135,7 +135,7 @@ def test_email_dispatch_mailgun_failed_email(db: Session, messaging_config) -> N action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), messaging_method=get_messaging_method( - CONFIG.notifications.notification_service_type + MessagingServiceType.MAILGUN.value ), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index ddb5053a8dc..78c93da3a51 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -35,9 +35,10 @@ from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, EmailForActionType, - MessagingActionType, + MessagingActionType, MessagingMethod, ) from fides.api.ops.schemas.policy import Rule +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.schemas.saas.saas_config import SaaSRequest from fides.api.ops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fides.api.ops.service.connectors.saas_connector import SaaSConnector @@ -1677,7 +1678,7 @@ def test_create_and_process_erasure_request_email_connector( assert mailgun_send.called kwargs = mailgun_send.call_args.kwargs assert type(kwargs["messaging_config"]) == MessagingConfig - assert type(kwargs["email"]) == EmailForActionType + assert type(kwargs["message"]) == EmailForActionType @mock.patch( "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" @@ -1963,13 +1964,15 @@ def test_email_complete_send_access_and_erasure( data, ) pr.delete(db=db) + identity = Identity(email=customer_email) mailgun_send.assert_has_calls( [ call( db=ANY, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_ACCESS, - to_email=customer_email, + to_identity=identity, + messaging_method=MessagingMethod.EMAIL, message_body_params=AccessRequestCompleteBodyParams( download_links=[upload_mock.return_value] ), @@ -1977,7 +1980,8 @@ def test_email_complete_send_access_and_erasure( call( db=ANY, action_type=MessagingActionType.PRIVACY_REQUEST_COMPLETE_DELETION, - to_email=customer_email, + to_identity=identity, + messaging_method=MessagingMethod.EMAIL, message_body_params=None, ), ], From 96cf23ade50f68cd3e8438b7db51ee8755184c4c Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Wed, 2 Nov 2022 14:19:33 -0600 Subject: [PATCH 13/28] format --- .../v1/endpoints/test_privacy_request_endpoints.py | 3 ++- .../ops/integration_tests/test_integration_email.py | 5 ++++- .../messaging/message_dispatch_service_test.py | 12 +++--------- .../privacy_request/request_runner_service_test.py | 3 ++- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 9d09bb92459..4b16b9f703d 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -65,9 +65,10 @@ from fides.api.ops.schemas.masking.masking_secrets import SecretType from fides.api.ops.schemas.messaging.messaging import ( MessagingActionType, + MessagingMethod, RequestReceiptBodyParams, RequestReviewDenyBodyParams, - SubjectIdentityVerificationBodyParams, MessagingMethod, + SubjectIdentityVerificationBodyParams, ) from fides.api.ops.schemas.policy import PolicyResponse from fides.api.ops.schemas.redis_cache import Identity diff --git a/tests/ops/integration_tests/test_integration_email.py b/tests/ops/integration_tests/test_integration_email.py index 286bf86e4bc..6cb91b93d11 100644 --- a/tests/ops/integration_tests/test_integration_email.py +++ b/tests/ops/integration_tests/test_integration_email.py @@ -14,7 +14,10 @@ ManualAction, ) from fides.api.ops.schemas.dataset import FidesopsDataset -from fides.api.ops.schemas.messaging.messaging import MessagingActionType, MessagingMethod +from fides.api.ops.schemas.messaging.messaging import ( + MessagingActionType, + MessagingMethod, +) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.service.connectors.email_connector import ( email_connector_erasure_send, diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index a438efa54ee..7a2f9da39bf 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -36,9 +36,7 @@ def test_email_dispatch_mailgun_success( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method( - MessagingServiceType.MAILGUN.value - ), + messaging_method=get_messaging_method(MessagingServiceType.MAILGUN.value), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -66,9 +64,7 @@ def test_email_dispatch_mailgun_config_not_found( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method( - MessagingServiceType.MAILGUN.value - ), + messaging_method=get_messaging_method(MessagingServiceType.MAILGUN.value), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), @@ -102,9 +98,7 @@ def test_email_dispatch_mailgun_config_no_secrets( db=db, action_type=MessagingActionType.SUBJECT_IDENTITY_VERIFICATION, to_identity=Identity(**{"email": "test@email.com"}), - messaging_method=get_messaging_method( - MessagingServiceType.MAILGUN.value - ), + messaging_method=get_messaging_method(MessagingServiceType.MAILGUN.value), message_body_params=SubjectIdentityVerificationBodyParams( verification_code="2348", verification_code_ttl_seconds=600 ), diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index 78c93da3a51..a122057a3fa 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -35,7 +35,8 @@ from fides.api.ops.schemas.messaging.messaging import ( AccessRequestCompleteBodyParams, EmailForActionType, - MessagingActionType, MessagingMethod, + MessagingActionType, + MessagingMethod, ) from fides.api.ops.schemas.policy import Rule from fides.api.ops.schemas.redis_cache import Identity From 441166e13bb505e58f4a0130a09cff1584f67839 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Wed, 2 Nov 2022 15:12:51 -0600 Subject: [PATCH 14/28] fixes type error, adds tests for messaging config endpoints --- .../api/ops/schemas/messaging/messaging.py | 6 +- .../v1/endpoints/test_messaging_endpoints.py | 242 ++++++++++++++++++ tests/ops/fixtures/application_fixtures.py | 44 ++++ 3 files changed, 289 insertions(+), 3 deletions(-) diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index 5c4bf5654b4..acf60321978 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -26,10 +26,10 @@ class MessagingServiceType(Enum): EMAIL_MESSAGING_SERVICES = [ - MessagingServiceType.MAILGUN, - MessagingServiceType.TWILIO_EMAIL, + MessagingServiceType.MAILGUN.value, + MessagingServiceType.TWILIO_EMAIL.value, ] -SMS_MESSAGING_SERVICES = [MessagingServiceType.TWILIO_TEXT] +SMS_MESSAGING_SERVICES = [MessagingServiceType.TWILIO_TEXT.value] class MessagingActionType(str, Enum): diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 091ce9e6737..b778b74c50d 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -39,6 +39,20 @@ def payload(self): "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, } + @pytest.fixture(scope="function") + def payload_twilio_email(self): + return { + "name": "twilio email", + "service_type": MessagingServiceType.TWILIO_EMAIL, + } + + @pytest.fixture(scope="function") + def payload_twilio_sms(self): + return { + "name": "twilio sms", + "service_type": MessagingServiceType.TWILIO_TEXT, + } + def test_post_email_config_not_authenticated( self, api_client: TestClient, payload, url ): @@ -207,6 +221,62 @@ def test_post_email_config_already_exists( "detail": "Only one messaging config is supported at a time. Config with key my_mailgun_messaging_config is already configured." } # fixme- what's the error here? + def test_post_twilio_email_config( + self, + db: Session, + api_client: TestClient, + payload_twilio_email, + url, + generate_auth_header, + ): + payload_twilio_email["key"] = "my_twilio_email_config" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.post(url, headers=auth_header, json=payload_twilio_email) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by( + key="my_twilio_email_config" + )[0] + + expected_response = { + "key": "my_twilio_email_config", + "name": "twilio_email", + "service_type": MessagingServiceType.TWILIO_EMAIL.value, + "details": None, + } + assert expected_response == response_body + email_config.delete(db) + + def test_post_twilio_sms_config( + self, + db: Session, + api_client: TestClient, + payload_twilio_sms, + url, + generate_auth_header, + ): + payload_twilio_sms["key"] = "my_twilio_sms_config" + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + + response = api_client.post(url, headers=auth_header, json=payload_twilio_sms) + assert 200 == response.status_code + + response_body = json.loads(response.text) + email_config = db.query(MessagingConfig).filter_by(key="my_twilio_sms_config")[ + 0 + ] + + expected_response = { + "key": "my_twilio_sms_config", + "name": "twilio_sms", + "service_type": MessagingServiceType.TWILIO_TEXT.value, + "details": None, + } + assert expected_response == response_body + email_config.delete(db) + class TestPatchMessagingConfig: @pytest.fixture(scope="function") @@ -364,6 +434,178 @@ def test_put_config_secrets( ) +class TestPutMessagingConfigSecretTwilioEmail: + @pytest.fixture(scope="function") + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config.key + ) + + @pytest.fixture(scope="function") + def payload(self): + return {MessagingServiceSecrets.TWILIO_API_KEY.value: "23p48btcpy14b"} + + def test_put_config_secrets( + self, + db: Session, + api_client: TestClient, + payload, + url, + generate_auth_header, + messaging_config_twilio_email, + ): + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config_twilio_email) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_twilio_email_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_twilio_email.secrets[ + MessagingServiceSecrets.TWILIO_API_KEY.value + ] + == "23p48btcpy14b" + ) + + +class TestPutMessagingConfigSecretTwilioSms: + @pytest.fixture(scope="function") + def url(self, messaging_config) -> str: + return (V1_URL_PREFIX + MESSAGING_SECRETS).format( + config_key=messaging_config.key + ) + + def test_put_config_secrets_with_messaging_service_sid( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "234ct324", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "3rcuinhewrf", + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID: "asdfasdf", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config_twilio_sms) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_twilio_email_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value + ] + == "234ct324" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value + ] + == "3rcuinhewrf" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] + == "asdfasdf" + ) + + def test_put_config_secrets_with_sender_phone( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23tc3", + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER: "+12345436543", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert 200 == response.status_code + + db.refresh(messaging_config_twilio_sms) + + assert json.loads(response.text) == { + "msg": "Secrets updated for MessagingConfig with key: my_twilio_sms_config.", + "test_status": None, + "failure_reason": None, + } + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value + ] + == "2asdf35tv5wsdf" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value + ] + == "23tc3" + ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] + == "+12345436543" + ) + + def test_put_config_secrets_with_sender_phone_incorrect_format( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23tc3", + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER: "12345436543", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert response.status_code == 400 + assert response.json() == { + "detail": "Sender phone number must include country code, formatted like +15558675309" + } + + def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( + self, + db: Session, + api_client: TestClient, + url, + generate_auth_header, + messaging_config_twilio_sms, + ): + payload = { + MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23tc3", + } + auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) + response = api_client.put(url, headers=auth_header, json=payload) + assert response.status_code == 400 + assert response.json() == { + "detail": "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." + } + + class TestGetMessagingConfigs: @pytest.fixture(scope="function") def url(self) -> str: diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 7f821792356..463278ee208 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -178,6 +178,50 @@ def messaging_config(db: Session) -> Generator: messaging_config.delete(db) +@pytest.fixture(scope="function") +def messaging_config_twilio_email(db: Session) -> Generator: + name = str(uuid4()) + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": name, + "key": "my_twilio_emailconfig", + "service_type": MessagingServiceType.TWILIO_EMAIL, + }, + ) + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.TWILIO_API_KEY.value: "123489ctynpiqurwfh" + }, + ) + yield messaging_config + messaging_config.delete(db) + + +@pytest.fixture(scope="function") +def messaging_config_twilio_sms(db: Session) -> Generator: + name = str(uuid4()) + messaging_config = MessagingConfig.create( + db=db, + data={ + "name": name, + "key": "my_twilio_sms_config", + "service_type": MessagingServiceType.TWILIO_TEXT, + }, + ) + messaging_config.set_secrets( + db=db, + messaging_secrets={ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "23rwrfwxwef", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23984y29384y598432", + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID: "2ieurnoqw", + }, + ) + yield messaging_config + messaging_config.delete(db) + + @pytest.fixture(scope="function") def https_connection_config(db: Session) -> Generator: name = str(uuid4()) From 3c18ca8ec857a7fdadfed55b0747a69a7a2ac68d Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Wed, 2 Nov 2022 17:59:38 -0600 Subject: [PATCH 15/28] fix unit/integration tests --- .../messaging/message_dispatch_service.py | 1 + .../test_connection_config_endpoints.py | 3 ++- .../test_consent_request_endpoints.py | 10 ++++---- .../v1/endpoints/test_messaging_endpoints.py | 25 +++++++++---------- .../test_privacy_request_endpoints.py | 9 ++++--- tests/ops/conftest.py | 9 +++++++ tests/ops/fixtures/application_fixtures.py | 6 ++--- 7 files changed, 37 insertions(+), 26 deletions(-) diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index e63d348ffb1..c29103d9e8c 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -244,6 +244,7 @@ def _mailgun_dispatcher( else "https://api.eu.mailgun.net" ) domain = messaging_config.details[MessagingServiceDetails.DOMAIN.value] + # todo: ensure formatting of to phone number is correct data = { "from": f"", "to": [to], diff --git a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py index ba3f30d6953..28fba585a64 100644 --- a/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_connection_config_endpoints.py @@ -28,6 +28,7 @@ PrivacyRequestStatus, ) from fides.api.ops.schemas.messaging.messaging import MessagingActionType +from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.tasks import MESSAGING_QUEUE_NAME page_size = Params().size @@ -1467,7 +1468,7 @@ def test_put_email_connection_config_secrets( kwargs["action_type"] == MessagingActionType.MESSAGE_ERASURE_REQUEST_FULFILLMENT ) - assert kwargs["to_email"] == "test@example.com" + assert kwargs["to_identity"] == Identity(email="test@example.com") assert kwargs["message_body_params"] == [ CheckpointActionRequired( step=CurrentStep.erasure, diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index 1178252ada9..01e72a69837 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -56,7 +56,7 @@ def url(self) -> str: return f"{V1_URL_PREFIX}{CONSENT_REQUEST}" @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", @@ -69,7 +69,7 @@ def test_consent_request(self, mock_dispatch_message, api_client, url): assert mock_dispatch_message.called @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", @@ -89,7 +89,7 @@ def test_consent_request_identity_present( assert mock_dispatch_message.called @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", @@ -102,7 +102,7 @@ def test_consent_request_redis_disabled(self, api_client, url): assert "redis cache required" in response.json()["message"] @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", ) @@ -116,7 +116,7 @@ def test_consent_request_subject_verification_disabled_no_email( assert not mock_dispatch_message.called @pytest.mark.usefixtures( - "email_config", + "messaging_config", "email_connection_config", "email_dataset_config", "subject_identity_verification_required", diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index b778b74c50d..9617401ee4e 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -43,14 +43,14 @@ def payload(self): def payload_twilio_email(self): return { "name": "twilio email", - "service_type": MessagingServiceType.TWILIO_EMAIL, + "service_type": MessagingServiceType.TWILIO_EMAIL.value, } @pytest.fixture(scope="function") def payload_twilio_sms(self): return { "name": "twilio sms", - "service_type": MessagingServiceType.TWILIO_TEXT, + "service_type": MessagingServiceType.TWILIO_TEXT.value, } def test_post_email_config_not_authenticated( @@ -104,7 +104,7 @@ def test_post_email_config_with_not_supported_service_type( assert 422 == response.status_code assert ( json.loads(response.text)["detail"][0]["msg"] - == "value is not a valid enumeration member; permitted: 'mailgun'" + == "value is not a valid enumeration member; permitted: 'mailgun', 'twilio_text', 'twilio_email'" ) def test_post_email_config_with_no_key( @@ -198,7 +198,7 @@ def test_post_email_config_missing_detail( assert "details" in errors[0]["loc"] assert errors[0]["msg"] == "field required" - def test_post_email_config_already_exists( + def test_post_email_config_service_already_exists( self, api_client: TestClient, url, @@ -210,7 +210,7 @@ def test_post_email_config_already_exists( url, headers=auth_header, json={ - "key": "my_mailgun_messaging_config", + "key": "my_new_mailgun_messaging_config", "name": "mailgun", "service_type": MessagingServiceType.MAILGUN.value, "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, @@ -218,7 +218,7 @@ def test_post_email_config_already_exists( ) assert response.status_code == 400 assert response.json() == { - "detail": "Only one messaging config is supported at a time. Config with key my_mailgun_messaging_config is already configured." + "detail": "" } # fixme- what's the error here? def test_post_twilio_email_config( @@ -733,20 +733,19 @@ def test_delete_config( generate_auth_header, ): # Creating new config, so we don't run into issues trying to clean up a deleted fixture - email_config = MessagingConfig.create( + twilio_sms_config = MessagingConfig.create( db=db, data={ - "key": "my_different_email_config", - "name": "mailgun", - "service_type": MessagingServiceType.MAILGUN, - "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, + "key": "my_twilio_sms_config", + "name": "twilio sms", + "service_type": MessagingServiceType.TWILIO_TEXT, }, ) - url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key=email_config.key) + url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key=twilio_sms_config.key) auth_header = generate_auth_header([MESSAGING_DELETE]) response = api_client.delete(url, headers=auth_header) assert response.status_code == 204 db.expunge_all() - config = db.query(MessagingConfig).filter_by(key=email_config.key).first() + config = db.query(MessagingConfig).filter_by(key=twilio_sms_config.key).first() assert config is None diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 4b16b9f703d..0825fe4445a 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -2973,7 +2973,7 @@ def test_verify_identity_no_admin_approval_needed( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity(phone_number='+1 234 567 8910', email='test@example.com') assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] @@ -3079,7 +3079,7 @@ def test_verify_identity_admin_approval_needed( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(email="test@example.com") + assert task_kwargs["to_identity"] == Identity(phone_number='+1 234 567 8910', email='test@example.com') assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] @@ -3660,6 +3660,7 @@ def privacy_request_receipt_notification_enabled(self): yield CONFIG.notifications.send_request_receipt_notification = original_value + @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) @@ -3674,7 +3675,7 @@ def test_create_privacy_request_no_email_config( db, api_client: TestClient, policy, - privacy_request_receipt_email_enabled, + privacy_request_receipt_notification_enabled, ): data = [ { @@ -3726,7 +3727,7 @@ def test_create_privacy_request_with_email_config( api_client: TestClient, policy, messaging_config, - privacy_request_receipt_email_enabled, + privacy_request_receipt_notification_enabled, ): data = [ { diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index e1217ad1291..1a6d3016dd0 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -298,3 +298,12 @@ def privacy_request_review_notification_disabled(): CONFIG.notifications.send_request_review_notification = False yield CONFIG.notifications.send_request_review_notification = original_value + + +@pytest.fixture(scope="function", autouse=True) +def set_notification_service_type_mailgun(): + """Set default notification service type""" + original_value = CONFIG.notifications.notification_service_type + CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value + yield + CONFIG.notifications.notification_service_type = original_value \ No newline at end of file diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 463278ee208..1c1c81755b6 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -213,9 +213,9 @@ def messaging_config_twilio_sms(db: Session) -> Generator: messaging_config.set_secrets( db=db, messaging_secrets={ - MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "23rwrfwxwef", - MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23984y29384y598432", - MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID: "2ieurnoqw", + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "23rwrfwxwef", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23984y29384y598432", + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value: "2ieurnoqw", }, ) yield messaging_config From 5fa23bd288805ff4d4944bd876bc6dc5ac0444f7 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Wed, 2 Nov 2022 18:00:01 -0600 Subject: [PATCH 16/28] format --- tests/ops/api/v1/endpoints/test_messaging_endpoints.py | 8 ++++---- .../api/v1/endpoints/test_privacy_request_endpoints.py | 9 ++++++--- tests/ops/conftest.py | 2 +- 3 files changed, 11 insertions(+), 8 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 9617401ee4e..6b601884e9a 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -217,9 +217,7 @@ def test_post_email_config_service_already_exists( }, ) assert response.status_code == 400 - assert response.json() == { - "detail": "" - } # fixme- what's the error here? + assert response.json() == {"detail": ""} # fixme- what's the error here? def test_post_twilio_email_config( self, @@ -741,7 +739,9 @@ def test_delete_config( "service_type": MessagingServiceType.TWILIO_TEXT, }, ) - url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format(config_key=twilio_sms_config.key) + url = (V1_URL_PREFIX + MESSAGING_BY_KEY).format( + config_key=twilio_sms_config.key + ) auth_header = generate_auth_header([MESSAGING_DELETE]) response = api_client.delete(url, headers=auth_header) assert response.status_code == 204 diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index 0825fe4445a..15ade528c3e 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -2973,7 +2973,9 @@ def test_verify_identity_no_admin_approval_needed( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(phone_number='+1 234 567 8910', email='test@example.com') + assert task_kwargs["to_identity"] == Identity( + phone_number="+1 234 567 8910", email="test@example.com" + ) assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] @@ -3079,7 +3081,9 @@ def test_verify_identity_admin_approval_needed( call_args = mock_dispatch_message.call_args[1] task_kwargs = call_args["kwargs"] - assert task_kwargs["to_identity"] == Identity(phone_number='+1 234 567 8910', email='test@example.com') + assert task_kwargs["to_identity"] == Identity( + phone_number="+1 234 567 8910", email="test@example.com" + ) assert task_kwargs["messaging_method"] == MessagingMethod.EMAIL message_meta = task_kwargs["message_meta"] @@ -3660,7 +3664,6 @@ def privacy_request_receipt_notification_enabled(self): yield CONFIG.notifications.send_request_receipt_notification = original_value - @mock.patch( "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" ) diff --git a/tests/ops/conftest.py b/tests/ops/conftest.py index 1a6d3016dd0..3e815684ce0 100644 --- a/tests/ops/conftest.py +++ b/tests/ops/conftest.py @@ -306,4 +306,4 @@ def set_notification_service_type_mailgun(): original_value = CONFIG.notifications.notification_service_type CONFIG.notifications.notification_service_type = MessagingServiceType.MAILGUN.value yield - CONFIG.notifications.notification_service_type = original_value \ No newline at end of file + CONFIG.notifications.notification_service_type = original_value From 51d57e28f8d9cfa7d3b03a4e6cd97a7c9ef2f0b0 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Thu, 3 Nov 2022 08:43:58 -0600 Subject: [PATCH 17/28] formatting to phone number --- .../api/ops/service/messaging/message_dispatch_service.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index c29103d9e8c..0de1d85310f 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -244,10 +244,9 @@ def _mailgun_dispatcher( else "https://api.eu.mailgun.net" ) domain = messaging_config.details[MessagingServiceDetails.DOMAIN.value] - # todo: ensure formatting of to phone number is correct data = { "from": f"", - "to": [to], + "to": [to.strip()], "subject": message.subject, "html": message.body, } From e2ae08aa376700cc94b538d5e170e8f902cf5508 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Thu, 3 Nov 2022 09:18:17 -0600 Subject: [PATCH 18/28] fixing tests --- .../v1/endpoints/test_messaging_endpoints.py | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 6b601884e9a..0601bc63475 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -42,14 +42,14 @@ def payload(self): @pytest.fixture(scope="function") def payload_twilio_email(self): return { - "name": "twilio email", + "name": "twilio_email", "service_type": MessagingServiceType.TWILIO_EMAIL.value, } @pytest.fixture(scope="function") def payload_twilio_sms(self): return { - "name": "twilio sms", + "name": "twilio_sms", "service_type": MessagingServiceType.TWILIO_TEXT.value, } @@ -216,7 +216,7 @@ def test_post_email_config_service_already_exists( "details": {MessagingServiceDetails.DOMAIN.value: "my.mailgun.domain"}, }, ) - assert response.status_code == 400 + assert response.status_code == 500 assert response.json() == {"detail": ""} # fixme- what's the error here? def test_post_twilio_email_config( @@ -434,9 +434,9 @@ def test_put_config_secrets( class TestPutMessagingConfigSecretTwilioEmail: @pytest.fixture(scope="function") - def url(self, messaging_config) -> str: + def url(self, messaging_config_twilio_email) -> str: return (V1_URL_PREFIX + MESSAGING_SECRETS).format( - config_key=messaging_config.key + config_key=messaging_config_twilio_email.key ) @pytest.fixture(scope="function") @@ -473,9 +473,9 @@ def test_put_config_secrets( class TestPutMessagingConfigSecretTwilioSms: @pytest.fixture(scope="function") - def url(self, messaging_config) -> str: + def url(self, messaging_config_twilio_sms) -> str: return (V1_URL_PREFIX + MESSAGING_SECRETS).format( - config_key=messaging_config.key + config_key=messaging_config_twilio_sms.key ) def test_put_config_secrets_with_messaging_service_sid( @@ -487,9 +487,9 @@ def test_put_config_secrets_with_messaging_service_sid( messaging_config_twilio_sms, ): payload = { - MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "234ct324", - MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "3rcuinhewrf", - MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID: "asdfasdf", + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "234ct324", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "3rcuinhewrf", + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value: "asdfasdf", } auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) @@ -530,9 +530,9 @@ def test_put_config_secrets_with_sender_phone( messaging_config_twilio_sms, ): payload = { - MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "2asdf35tv5wsdf", - MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23tc3", - MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER: "+12345436543", + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23tc3", + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value: "+12345436543", } auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) @@ -573,9 +573,9 @@ def test_put_config_secrets_with_sender_phone_incorrect_format( messaging_config_twilio_sms, ): payload = { - MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "2asdf35tv5wsdf", - MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23tc3", - MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER: "12345436543", + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23tc3", + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value: "12345436543", } auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) @@ -593,8 +593,8 @@ def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( messaging_config_twilio_sms, ): payload = { - MessagingServiceSecrets.TWILIO_ACCOUNT_SID: "2asdf35tv5wsdf", - MessagingServiceSecrets.TWILIO_AUTH_TOKEN: "23tc3", + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value: "2asdf35tv5wsdf", + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value: "23tc3", } auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) @@ -735,7 +735,7 @@ def test_delete_config( db=db, data={ "key": "my_twilio_sms_config", - "name": "twilio sms", + "name": "twilio_sms", "service_type": MessagingServiceType.TWILIO_TEXT, }, ) From 9f869145866950da95084ec90beb90f795d6356d Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Thu, 3 Nov 2022 10:52:19 -0600 Subject: [PATCH 19/28] more unit test fixes, fix bug in twilio sms validation --- src/fides/api/ops/schemas/messaging/messaging.py | 2 +- src/fides/ctl/core/deploy.py | 3 ++- tests/ops/api/v1/endpoints/test_messaging_endpoints.py | 10 +++++++--- tests/ops/fixtures/application_fixtures.py | 2 +- 4 files changed, 11 insertions(+), 6 deletions(-) diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index acf60321978..a88d1885e14 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -168,7 +168,7 @@ class Config: @root_validator def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: sender_phone = values.get("twilio_sender_phone_number") - if not values.get("twilio_messaging_service_sid") or sender_phone: + if not values.get("twilio_messaging_service_sid") and not sender_phone: raise ValueError( "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." ) diff --git a/src/fides/ctl/core/deploy.py b/src/fides/ctl/core/deploy.py index 24e9c7ccb65..b81d06d2cd3 100644 --- a/src/fides/ctl/core/deploy.py +++ b/src/fides/ctl/core/deploy.py @@ -96,7 +96,8 @@ def check_docker_version() -> bool: def seed_example_data() -> None: run_shell( - DOCKER_COMPOSE_COMMAND + "run --no-deps --rm fides fides push src/fides/data/sample_project/sample_resources/" + DOCKER_COMPOSE_COMMAND + + "run --no-deps --rm fides fides push src/fides/data/sample_project/sample_resources/" ) run_shell( DOCKER_COMPOSE_COMMAND diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 0601bc63475..b05758c1da0 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -217,7 +217,9 @@ def test_post_email_config_service_already_exists( }, ) assert response.status_code == 500 - assert response.json() == {"detail": ""} # fixme- what's the error here? + assert ( + f"Key (service_type)=(MAILGUN) already exists" in response.json()["detail"] + ) def test_post_twilio_email_config( self, @@ -498,7 +500,7 @@ def test_put_config_secrets_with_messaging_service_sid( db.refresh(messaging_config_twilio_sms) assert json.loads(response.text) == { - "msg": "Secrets updated for MessagingConfig with key: my_twilio_email_config.", + "msg": "Secrets updated for MessagingConfig with key: my_twilio_sms_config.", "test_status": None, "failure_reason": None, } @@ -600,7 +602,9 @@ def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( response = api_client.put(url, headers=auth_header, json=payload) assert response.status_code == 400 assert response.json() == { - "detail": "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." + "detail": [ + "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied. ('__root__',)" + ] } diff --git a/tests/ops/fixtures/application_fixtures.py b/tests/ops/fixtures/application_fixtures.py index 1c1c81755b6..b50d04a4ea5 100644 --- a/tests/ops/fixtures/application_fixtures.py +++ b/tests/ops/fixtures/application_fixtures.py @@ -185,7 +185,7 @@ def messaging_config_twilio_email(db: Session) -> Generator: db=db, data={ "name": name, - "key": "my_twilio_emailconfig", + "key": "my_twilio_email_config", "service_type": MessagingServiceType.TWILIO_EMAIL, }, ) From 091cf3ff44401afb11179503c673b39e9212d22a Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Thu, 3 Nov 2022 11:19:29 -0600 Subject: [PATCH 20/28] more tests fixes --- .../v1/endpoints/test_messaging_endpoints.py | 24 ++++++++++++------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index b05758c1da0..8278a45d8cb 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -561,10 +561,16 @@ def test_put_config_secrets_with_sender_phone( ) assert ( messaging_config_twilio_sms.secrets[ - MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value ] == "+12345436543" ) + assert ( + messaging_config_twilio_sms.secrets[ + MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value + ] + is None + ) def test_put_config_secrets_with_sender_phone_incorrect_format( self, @@ -582,9 +588,10 @@ def test_put_config_secrets_with_sender_phone_incorrect_format( auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) assert response.status_code == 400 - assert response.json() == { - "detail": "Sender phone number must include country code, formatted like +15558675309" - } + assert ( + f"Sender phone number must include country code, formatted like +15558675309" + in response.json()["detail"] + ) def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( self, @@ -601,11 +608,10 @@ def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.put(url, headers=auth_header, json=payload) assert response.status_code == 400 - assert response.json() == { - "detail": [ - "Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied. ('__root__',)" - ] - } + assert ( + f"Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." + in response.json()["detail"] + ) class TestGetMessagingConfigs: From 9cba9f195ff5c8785afb8a7439e9133acbe4754a Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Thu, 3 Nov 2022 12:00:05 -0600 Subject: [PATCH 21/28] fix err message --- tests/ops/api/v1/endpoints/test_messaging_endpoints.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 8278a45d8cb..7045f49a2cd 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -589,7 +589,7 @@ def test_put_config_secrets_with_sender_phone_incorrect_format( response = api_client.put(url, headers=auth_header, json=payload) assert response.status_code == 400 assert ( - f"Sender phone number must include country code, formatted like +15558675309" + f"Sender phone number must include country code, formatted like +15558675309 ('__root__',)" in response.json()["detail"] ) @@ -609,7 +609,7 @@ def test_put_config_secrets_with_no_sender_phone_nor_messaging_service_id( response = api_client.put(url, headers=auth_header, json=payload) assert response.status_code == 400 assert ( - f"Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied." + f"Either the twilio_messaging_service_id or the twilio_sender_phone_number should be supplied. ('__root__',)" in response.json()["detail"] ) From a61c0558958a5d916f4c84f10f747e1aeb4427b7 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Fri, 4 Nov 2022 10:59:53 -0600 Subject: [PATCH 22/28] changes from CR --- .../api/v1/endpoints/messaging_endpoints.py | 2 +- .../api/ops/schemas/messaging/messaging.py | 8 ++-- .../messaging/message_dispatch_service.py | 45 +++++++++++-------- .../v1/endpoints/test_messaging_endpoints.py | 10 +---- 4 files changed, 33 insertions(+), 32 deletions(-) diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py index 5cf7c365f1a..be2f5737760 100644 --- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -49,7 +49,7 @@ from fides.api.ops.util.logger import Pii from fides.api.ops.util.oauth_util import verify_oauth_client -router = APIRouter(tags=["email"], prefix=V1_URL_PREFIX) +router = APIRouter(tags=["messaging"], prefix=V1_URL_PREFIX) logger = logging.getLogger(__name__) diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index a88d1885e14..ebd9dd6728c 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -1,6 +1,6 @@ from enum import Enum from re import compile as regex -from typing import Any, Dict, List, Optional, Union +from typing import Any, Dict, List, Optional, Tuple, Union from pydantic import BaseModel, Extra, root_validator @@ -25,11 +25,11 @@ class MessagingServiceType(Enum): TWILIO_EMAIL = "twilio_email" -EMAIL_MESSAGING_SERVICES = [ +EMAIL_MESSAGING_SERVICES: Tuple[str, ...] = ( MessagingServiceType.MAILGUN.value, MessagingServiceType.TWILIO_EMAIL.value, -] -SMS_MESSAGING_SERVICES = [MessagingServiceType.TWILIO_TEXT.value] +) +SMS_MESSAGING_SERVICES: Tuple[str, ...] = tuple(MessagingServiceType.TWILIO_TEXT.value) class MessagingActionType(str, Enum): diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index 0de1d85310f..ffcd0a3baff 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -1,7 +1,7 @@ from __future__ import annotations import logging -from typing import Any, Dict, List, Optional, Union +from typing import Any, Callable, Dict, List, Optional, Union import requests from sqlalchemy.orm import Session @@ -83,7 +83,7 @@ def dispatch_message( logger.info( "Building appropriate message template for action type: %s", action_type ) - message: Optional[Union[EmailForActionType, str]] = None # fixme- huh?? + message: Optional[Union[EmailForActionType, str]] = None if messaging_method == MessagingMethod.EMAIL: message = _build_email( action_type=action_type, @@ -106,9 +106,9 @@ def dispatch_message( logger.info( "Retrieving appropriate dispatcher for email service: %s", messaging_service ) - dispatcher: Any = _get_dispatcher_from_config_type( - message_service_type=messaging_service - ) + dispatcher: Optional[Callable[ + [MessagingConfig, Any, Optional[str]], None + ]] = _get_dispatcher_from_config_type(message_service_type=messaging_service) if not dispatcher: logger.error( "Dispatcher has not been implemented for message service type: %s", @@ -122,9 +122,9 @@ def dispatch_message( action_type, ) dispatcher( - messaging_config=messaging_config, - message=message, - to=to_identity.email + messaging_config, + message, + to_identity.email if messaging_method == MessagingMethod.EMAIL else to_identity.phone_number, ) @@ -218,12 +218,15 @@ def _build_email( # pylint: disable=too-many-return-statements ) -def _get_dispatcher_from_config_type(message_service_type: MessagingServiceType) -> Any: +def _get_dispatcher_from_config_type( + message_service_type: MessagingServiceType, +) -> Optional[Callable[[MessagingConfig, Any, Optional[str]], None]]: """Determines which dispatcher to use based on message service type""" - return { - MessagingServiceType.MAILGUN.value: _mailgun_dispatcher, - MessagingServiceType.TWILIO_TEXT.value: _twilio_sms_dispatcher, - }[message_service_type.value] + if message_service_type == MessagingServiceType.MAILGUN.value: + return _mailgun_dispatcher + if message_service_type == MessagingServiceType.TWILIO_TEXT.value: + return _twilio_sms_dispatcher + return None def _mailgun_dispatcher( @@ -235,9 +238,11 @@ def _mailgun_dispatcher( if not to: logger.error("Message failed to send. No email identity supplied.") raise MessageDispatchException("No email identity supplied.") - if not messaging_config.details: - logger.error("Message failed to send. No mailgun config details supplied.") - raise MessageDispatchException("No mailgun config details supplied.") + if not messaging_config.details or not messaging_config.secrets: + logger.error( + "Message failed to send. No mailgun config details or secrets supplied." + ) + raise MessageDispatchException("No mailgun config details or secrets supplied.") base_url = ( "https://api.mailgun.net" if messaging_config.details[MessagingServiceDetails.IS_EU_DOMAIN.value] is False @@ -280,15 +285,17 @@ def _twilio_sms_dispatcher( if not to: logger.error("Message failed to send. No phone identity supplied.") raise MessageDispatchException("No phone identity supplied.") - + if messaging_config.secrets is None: + logger.error("Message failed to send. No config secrets supplied.") + raise MessageDispatchException("No config secrets supplied.") account_sid = messaging_config.secrets[MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value] # type: ignore auth_token = messaging_config.secrets[MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value] # type: ignore messaging_service_id = messaging_config.secrets[ MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value - ] # type:ignore + ] # type: ignore sender_phone_number = messaging_config.secrets[ MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value - ] # type:ignore + ] # type: ignore client = Client(account_sid, auth_token) try: diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 7045f49a2cd..1be403c4765 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -83,11 +83,8 @@ def test_post_email_config_with_invalid_mailgun_details( auth_header = generate_auth_header([MESSAGING_CREATE_OR_UPDATE]) response = api_client.post(url, headers=auth_header, json=payload) assert 422 == response.status_code - assert json.loads(response.text)["detail"][0]["msg"] == "field required" - assert ( - json.loads(response.text)["detail"][1]["msg"] - == "extra fields not permitted" - ) + assert response.json()["detail"][0]["msg"] == "field required" + assert response.json()["detail"][1]["msg"] == "extra fields not permitted" def test_post_email_config_with_not_supported_service_type( self, @@ -188,9 +185,6 @@ def test_post_email_config_missing_detail( "key": "my_mailgun_messaging_config", "name": "mailgun", "service_type": MessagingServiceType.MAILGUN.value, - "details": { - # "domain": "my.mailgun.domain" - }, }, ) assert response.status_code == 422 From d4dfa707d8cc93bade9975a517228c56669b9623 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Fri, 4 Nov 2022 11:31:17 -0600 Subject: [PATCH 23/28] fix test, update changelog --- CHANGELOG.md | 1 + .../ops/service/messaging/message_dispatch_service.py | 10 +++++----- tests/ops/api/v1/endpoints/test_messaging_endpoints.py | 1 + 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 948a957960a..1d4e51b4114 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -42,6 +42,7 @@ The types of changes are: * Config Wizard: Added a column selector to the scan results page of the config wizard [#1590](https://github.com/ethyca/fides/pull/1590) * Config Wizard: Flow for runtime scanner option [#1640](https://github.com/ethyca/fides/pull/1640) * Access support for Twilio Conversations API [#1520](https://github.com/ethyca/fides/pull/1520) +* Message Config: Adds Twilio Email/SMS support [#1519](https://github.com/ethyca/fides/pull/1519) ### Changed diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index ffcd0a3baff..33d0c3e50b6 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -106,9 +106,9 @@ def dispatch_message( logger.info( "Retrieving appropriate dispatcher for email service: %s", messaging_service ) - dispatcher: Optional[Callable[ - [MessagingConfig, Any, Optional[str]], None - ]] = _get_dispatcher_from_config_type(message_service_type=messaging_service) + dispatcher: Optional[ + Callable[[MessagingConfig, Any, Optional[str]], None] + ] = _get_dispatcher_from_config_type(message_service_type=messaging_service) if not dispatcher: logger.error( "Dispatcher has not been implemented for message service type: %s", @@ -222,9 +222,9 @@ def _get_dispatcher_from_config_type( message_service_type: MessagingServiceType, ) -> Optional[Callable[[MessagingConfig, Any, Optional[str]], None]]: """Determines which dispatcher to use based on message service type""" - if message_service_type == MessagingServiceType.MAILGUN.value: + if message_service_type == MessagingServiceType.MAILGUN: return _mailgun_dispatcher - if message_service_type == MessagingServiceType.TWILIO_TEXT.value: + if message_service_type == MessagingServiceType.TWILIO_TEXT: return _twilio_sms_dispatcher return None diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 1be403c4765..2333bbc923a 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -185,6 +185,7 @@ def test_post_email_config_missing_detail( "key": "my_mailgun_messaging_config", "name": "mailgun", "service_type": MessagingServiceType.MAILGUN.value, + "details": None, }, ) assert response.status_code == 422 From b81957db813b3c1c9b81d1a3b0b6190a9ed54155 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Fri, 4 Nov 2022 12:10:25 -0600 Subject: [PATCH 24/28] adding validator for mailgun config details, catching that valueerror to throw custom http exception --- .../api/v1/endpoints/messaging_endpoints.py | 21 ++++++++++++++++++- .../api/ops/schemas/messaging/messaging.py | 10 +++++++++ .../v1/endpoints/test_messaging_endpoints.py | 4 ++-- .../request_runner_service_test.py | 6 +++--- 4 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py index be2f5737760..0a8e40ff029 100644 --- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -70,6 +70,16 @@ def post_config( try: return create_or_update_messaging_config(db=db, config=messaging_config) + except ValueError as e: + logger.warning( + "Create failed for messaging config %s: %s", + messaging_config.key, + Pii(str(e)), + ) + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"Config with key {messaging_config.key} failed to be added: {e}", + ) except Exception as exc: logger.warning( "Create failed for messaging config %s: %s", @@ -104,7 +114,16 @@ def patch_config_by_key( status_code=HTTP_404_NOT_FOUND, detail=f"No messaging config found with key {config_key}", ) - + except ValueError as e: + logger.warning( + "Create failed for messaging config %s: %s", + messaging_config.key, + Pii(str(e)), + ) + raise HTTPException( + status_code=HTTP_400_BAD_REQUEST, + detail=f"Config with key {messaging_config.key} failed to be added: {e}", + ) except Exception as exc: logger.warning( "Patch failed for messaging config %s: %s", diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index ebd9dd6728c..b899013b9a5 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -204,6 +204,16 @@ class Config: use_enum_values = False orm_mode = True + @root_validator + def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: + service_type: MessagingServiceType = values.get("service_type") + if service_type == MessagingServiceType.MAILGUN: + if not values.get("details"): + raise ValueError( + "Mailgun messaging config must include details" + ) + return values + class MessagingConfigResponse(BaseModel): """Messaging Config Response Schema""" diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 2333bbc923a..241ce14360a 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -188,10 +188,10 @@ def test_post_email_config_missing_detail( "details": None, }, ) - assert response.status_code == 422 + assert response.status_code == 400 errors = response.json()["detail"] assert "details" in errors[0]["loc"] - assert errors[0]["msg"] == "field required" + assert errors[0]["msg"] == "Mailgun messaging config must include details" def test_post_email_config_service_already_exists( self, diff --git a/tests/ops/service/privacy_request/request_runner_service_test.py b/tests/ops/service/privacy_request/request_runner_service_test.py index a122057a3fa..1c06c19be21 100644 --- a/tests/ops/service/privacy_request/request_runner_service_test.py +++ b/tests/ops/service/privacy_request/request_runner_service_test.py @@ -1677,9 +1677,9 @@ def test_create_and_process_erasure_request_email_connector( ) pr.delete(db=db) assert mailgun_send.called - kwargs = mailgun_send.call_args.kwargs - assert type(kwargs["messaging_config"]) == MessagingConfig - assert type(kwargs["message"]) == EmailForActionType + args = mailgun_send.call_args.args + assert type(args[0]) == MessagingConfig + assert type(args[1]) == EmailForActionType @mock.patch( "fides.api.ops.service.messaging.message_dispatch_service._mailgun_dispatcher" From aca5b77474f254f77d39120b4611d6c8a56ecd93 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Fri, 4 Nov 2022 12:12:36 -0600 Subject: [PATCH 25/28] format --- src/fides/api/ops/schemas/messaging/messaging.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/src/fides/api/ops/schemas/messaging/messaging.py b/src/fides/api/ops/schemas/messaging/messaging.py index b899013b9a5..3a4d9ae3851 100644 --- a/src/fides/api/ops/schemas/messaging/messaging.py +++ b/src/fides/api/ops/schemas/messaging/messaging.py @@ -206,12 +206,10 @@ class Config: @root_validator def validate_fields(cls, values: Dict[str, Any]) -> Dict[str, Any]: - service_type: MessagingServiceType = values.get("service_type") + service_type: MessagingServiceType = values.get("service_type") # type: ignore if service_type == MessagingServiceType.MAILGUN: if not values.get("details"): - raise ValueError( - "Mailgun messaging config must include details" - ) + raise ValueError("Mailgun messaging config must include details") return values From ca9be4cf7fc7c5e5bc2a2499484bacfee109db33 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 7 Nov 2022 10:40:01 -0600 Subject: [PATCH 26/28] update validator for mailgun details and tests --- .../ops/api/v1/endpoints/messaging_endpoints.py | 10 ---------- .../service/messaging/message_dispatch_service.py | 14 +++++++++----- .../api/v1/endpoints/test_messaging_endpoints.py | 3 +-- .../messaging/message_dispatch_service_test.py | 6 +++--- 4 files changed, 13 insertions(+), 20 deletions(-) diff --git a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py index 0a8e40ff029..009b8a5dfd3 100644 --- a/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/messaging_endpoints.py @@ -114,16 +114,6 @@ def patch_config_by_key( status_code=HTTP_404_NOT_FOUND, detail=f"No messaging config found with key {config_key}", ) - except ValueError as e: - logger.warning( - "Create failed for messaging config %s: %s", - messaging_config.key, - Pii(str(e)), - ) - raise HTTPException( - status_code=HTTP_400_BAD_REQUEST, - detail=f"Config with key {messaging_config.key} failed to be added: {e}", - ) except Exception as exc: logger.warning( "Patch failed for messaging config %s: %s", diff --git a/src/fides/api/ops/service/messaging/message_dispatch_service.py b/src/fides/api/ops/service/messaging/message_dispatch_service.py index 33d0c3e50b6..0e87d0c539e 100644 --- a/src/fides/api/ops/service/messaging/message_dispatch_service.py +++ b/src/fides/api/ops/service/messaging/message_dispatch_service.py @@ -260,7 +260,7 @@ def _mailgun_dispatcher( f"{base_url}/{messaging_config.details[MessagingServiceDetails.API_VERSION.value]}/{domain}/messages", auth=( "api", - messaging_config.secrets[MessagingServiceSecrets.MAILGUN_API_KEY.value], # type: ignore + messaging_config.secrets[MessagingServiceSecrets.MAILGUN_API_KEY.value], ), data=data, ) @@ -288,14 +288,18 @@ def _twilio_sms_dispatcher( if messaging_config.secrets is None: logger.error("Message failed to send. No config secrets supplied.") raise MessageDispatchException("No config secrets supplied.") - account_sid = messaging_config.secrets[MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value] # type: ignore - auth_token = messaging_config.secrets[MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value] # type: ignore + account_sid = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_ACCOUNT_SID.value + ] + auth_token = messaging_config.secrets[ + MessagingServiceSecrets.TWILIO_AUTH_TOKEN.value + ] messaging_service_id = messaging_config.secrets[ MessagingServiceSecrets.TWILIO_MESSAGING_SERVICE_SID.value - ] # type: ignore + ] sender_phone_number = messaging_config.secrets[ MessagingServiceSecrets.TWILIO_SENDER_PHONE_NUMBER.value - ] # type: ignore + ] client = Client(account_sid, auth_token) try: diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index 241ce14360a..fa1cbb134ee 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -185,10 +185,9 @@ def test_post_email_config_missing_detail( "key": "my_mailgun_messaging_config", "name": "mailgun", "service_type": MessagingServiceType.MAILGUN.value, - "details": None, }, ) - assert response.status_code == 400 + assert response.status_code == 422 errors = response.json()["detail"] assert "details" in errors[0]["loc"] assert errors[0]["msg"] == "Mailgun messaging config must include details" diff --git a/tests/ops/service/messaging/message_dispatch_service_test.py b/tests/ops/service/messaging/message_dispatch_service_test.py index 7a2f9da39bf..f3b9e85d30d 100644 --- a/tests/ops/service/messaging/message_dispatch_service_test.py +++ b/tests/ops/service/messaging/message_dispatch_service_test.py @@ -43,12 +43,12 @@ def test_email_dispatch_mailgun_success( ) body = '\n\n\n \n ID Code\n\n\n
\n

\n Your privacy request verification code is 2348.\n Please return to the Privacy Center and enter the code to\n continue. This code will expire in 10 minutes\n

\n
\n\n' mock_mailgun_dispatcher.assert_called_with( - messaging_config=messaging_config, - message=EmailForActionType( + messaging_config, + EmailForActionType( subject="Your one-time code", body=body, ), - to="test@email.com", + "test@email.com", ) From a7fa6a2f687f471858a93962f4f9fc9f55fb78e7 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 7 Nov 2022 11:00:18 -0600 Subject: [PATCH 27/28] remove unneeded assert --- tests/ops/api/v1/endpoints/test_messaging_endpoints.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py index fa1cbb134ee..793c6dd4fc0 100644 --- a/tests/ops/api/v1/endpoints/test_messaging_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_messaging_endpoints.py @@ -189,7 +189,6 @@ def test_post_email_config_missing_detail( ) assert response.status_code == 422 errors = response.json()["detail"] - assert "details" in errors[0]["loc"] assert errors[0]["msg"] == "Mailgun messaging config must include details" def test_post_email_config_service_already_exists( From 4667a6355f7c709fe88262ec3d6a24a4f24273c3 Mon Sep 17 00:00:00 2001 From: eastandwestwind Date: Mon, 7 Nov 2022 11:15:39 -0600 Subject: [PATCH 28/28] rerunning ci