[Backend] Add Required Notice Key to Privacy Notices (#3337)

.fides/db_dataset.yml

@@ -1573,6 +1573,9 @@ dataset:
     - name: name
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: notice_key
+      data_categories: [ system.operations ]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
     - name: description
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
@@ -1628,6 +1631,9 @@ dataset:
     - name: name
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
+    - name: notice_key
+      data_categories: [ system.operations ]
+      data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified
     - name: description
       data_categories: [system.operations]
       data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified

CHANGELOG.md

@@ -24,6 +24,7 @@ The types of changes are:
 - Restrict strack-trace logging when not in Dev mode [#3081](https://github.com/ethyca/fides/pull/3081)
 - Refactor CSS variables for `fides-js` to match brandable color palette [#3321](https://github.com/ethyca/fides/pull/3321)
 - Moved all of the dirs from `fides.api.ops` into `fides.api` [#3318](https://github.com/ethyca/fides/pull/3318)
+- Add required notice key to privacy notices [#3337](https://github.com/ethyca/fides/pull/3337)
 
 ### Added
 

docs/fides/docs/development/postman/Fides.postman_collection.json

@@ -5216,7 +5216,7 @@
 						"header": [],
 						"body": {
 							"mode": "raw",
-							"raw": "[\n   {\n      \"name\":\"Profiling\",\n      \"regions\":[\n         \"us_ca\",\n         \"us_ut\"\n      ],\n      \"description\":\"Making a decision solely by automated means.\",\n      \"consent_mechanism\":\"opt_in\",\n      \"data_uses\":[\n         \"personalize\"\n      ],\n      \"enforcement_level\":\"system_wide\",\n      \"has_gpc_flag\":false,\n      \"displayed_in_overlay\":true\n   },\n   {\n      \"name\":\"Essential\",\n      \"regions\":[\n         \"eu_de\"\n      ],\n      \"description\":\"Notify the user about data processing activities that are essential to your services' functionality. Typically consent is not required for this.\",\n      \"consent_mechanism\":\"notice_only\",\n      \"data_uses\":[\n         \"provide.service\"\n      ],\n      \"enforcement_level\":\"system_wide\",\n      \"displayed_in_overlay\":true\n   },\n   {\n      \"name\":\"Advertising\",\n      \"regions\":[\n         \"us_ca\"\n      ],\n      \"description\":\"Sample advertising notice\",\n      \"consent_mechanism\":\"opt_out\",\n      \"data_uses\":[\n         \"advertising\"\n      ],\n      \"enforcement_level\":\"system_wide\",\n      \"displayed_in_privacy_center\":true\n   }\n]",
+							"raw": "[\n   {\n      \"name\":\"Profiling\",\n      \"notice_key\": \"profiling\",\n      \"regions\":[\n         \"us_ca\",\n         \"us_ut\"\n      ],\n      \"description\":\"Making a decision solely by automated means.\",\n      \"consent_mechanism\":\"opt_in\",\n      \"data_uses\":[\n         \"personalize\"\n      ],\n      \"enforcement_level\":\"system_wide\",\n      \"has_gpc_flag\":false,\n      \"displayed_in_overlay\":true\n   },\n   {\n      \"name\":\"Essential\",\n      \"notice_key\": \"essential\",\n      \"regions\":[\n         \"eu_de\"\n      ],\n      \"description\":\"Notify the user about data processing activities that are essential to your services' functionality. Typically consent is not required for this.\",\n      \"consent_mechanism\":\"notice_only\",\n      \"data_uses\":[\n         \"provide.service\"\n      ],\n      \"enforcement_level\":\"system_wide\",\n      \"displayed_in_overlay\":true\n   },\n   {\n      \"name\":\"Advertising\",\n      \"notice_key\": \"advertising\",\n      \"regions\":[\n         \"us_ca\"\n      ],\n      \"description\":\"Sample advertising notice\",\n      \"consent_mechanism\":\"opt_out\",\n      \"data_uses\":[\n         \"advertising\"\n      ],\n      \"enforcement_level\":\"system_wide\",\n      \"displayed_in_privacy_center\":true\n   }\n]",
 							"options": {
 								"raw": {
 									"language": "json"

src/fides/api/ctl/migrations/versions/8a71872089e4_add_notice_key_to_notices.py

@@ -0,0 +1,68 @@
+"""add_notice_key_to_notices
+
+Revision ID: 8a71872089e4
+Revises: 2661f31daffb
+Create Date: 2023-05-18 19:48:33.268790
+
+"""
+import sqlalchemy as sa
+from alembic import op
+from fideslang.validation import FidesKey, FidesValidationError
+
+# revision identifiers, used by Alembic.
+from sqlalchemy import text
+from sqlalchemy.engine import ResultProxy
+
+revision = "8a71872089e4"
+down_revision = "2661f31daffb"
+branch_labels = None
+depends_on = None
+
+
+def validate_fides_key_suitability(names: ResultProxy, table_name: str) -> None:
+    for row in names:
+        name: str = row["name"].strip(" ").replace(" ", "_")
+        try:
+            FidesKey.validate(name)
+        except FidesValidationError as exc:
+            raise Exception(
+                f"Cannot auto-migrate, adjust existing {table_name} name: '{name}' to remove invalid characters: {exc}."
+            )
+
+
+def upgrade():
+    """Add new non-nullable notice_key fields to privacy notice and notice history tables
+    and automatically create existing notice keys from the notice names.
+
+    Notice keys are not unique.
+    """
+    bind = op.get_bind()
+    existing_history_names: ResultProxy = bind.execute(
+        text("select name from privacynoticehistory;")
+    )
+    validate_fides_key_suitability(existing_history_names, "privacynoticehistory")
+
+    existing_notice_names: ResultProxy = bind.execute(
+        text("select name from privacynotice;")
+    )
+    validate_fides_key_suitability(existing_notice_names, "privacynotice")
+
+    op.add_column("privacynotice", sa.Column("notice_key", sa.String(), nullable=True))
+    op.add_column(
+        "privacynoticehistory", sa.Column("notice_key", sa.String(), nullable=True)
+    )
+
+    op.execute(
+        "update privacynoticehistory set notice_key = LOWER(REPLACE(TRIM(name), ' ', '_'));"
+    )
+    op.execute(
+        "update privacynotice set notice_key = LOWER(REPLACE(TRIM(name), ' ', '_'));"
+    )
+
+    op.alter_column("privacynotice", "notice_key", nullable=False)
+    op.alter_column("privacynoticehistory", "notice_key", nullable=False)
+
+
+def downgrade():
+    op.drop_column("privacynoticehistory", "notice_key")
+    op.drop_column("privacynotice", "notice_key")

src/fides/api/models/privacy_notice.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import re
 from collections import defaultdict
 from enum import Enum
 from typing import Any, Dict, Iterable, List, Optional, Tuple, Type
@@ -149,6 +150,7 @@ class PrivacyNoticeBase:
     displayed_in_privacy_center = Column(Boolean, nullable=False, default=False)
     displayed_in_overlay = Column(Boolean, nullable=False, default=False)
     displayed_in_api = Column(Boolean, nullable=False, default=False)
+    notice_key = Column(String, nullable=False)
 
     def applies_to_system(self, system: System) -> bool:
         """Privacy Notice applies to System if a data use matches or the Privacy Notice
@@ -160,6 +162,15 @@ def applies_to_system(self, system: System) -> bool:
                     return True
         return False
 
+    @classmethod
+    def generate_notice_key(cls, name: Optional[str]) -> FidesKey:
+        """Generate a notice key from a notice name"""
+        if not isinstance(name, str):
+            raise Exception("Privacy notice keys must be generated from a string.")
+        notice_key: str = re.sub(r"\s+", "_", name.lower().strip())
+        FidesKey.validate(notice_key)
+        return notice_key
+
 
 class PrivacyNotice(PrivacyNoticeBase, Base):
     """
@@ -244,6 +255,7 @@ def update(self, db: Session, *, data: dict[str, Any]) -> PrivacyNotice:
             # and is no longer the history record 'id' column
             history_data = {
                 "name": self.name,
+                "notice_key": self.notice_key,
                 "description": self.description or None,
                 "origin": self.origin or None,
                 "regions": self.regions,

src/fides/api/schemas/privacy_notice.py

@@ -3,14 +3,12 @@
 from datetime import datetime
 from typing import Any, Dict, List, Optional
 
+from fideslang.validation import FidesKey
 from pydantic import Extra, conlist, root_validator, validator
 
-from fides.api.models.privacy_notice import (
-    ConsentMechanism,
-    EnforcementLevel,
-    PrivacyNoticeRegion,
-    UserConsentPreference,
-)
+from fides.api.models.privacy_notice import ConsentMechanism, EnforcementLevel
+from fides.api.models.privacy_notice import PrivacyNotice as PrivacyNoticeModel
+from fides.api.models.privacy_notice import PrivacyNoticeRegion, UserConsentPreference
 from fides.api.schemas.base_class import FidesSchema
 
 
@@ -23,6 +21,7 @@ class PrivacyNotice(FidesSchema):
     """
 
     name: Optional[str]
+    notice_key: Optional[FidesKey]
     description: Optional[str]
     internal_description: Optional[str]
     origin: Optional[str]
@@ -110,6 +109,18 @@ class PrivacyNoticeCreation(PrivacyNotice):
     data_uses: conlist(str, min_items=1)  # type: ignore
     enforcement_level: EnforcementLevel
 
+    @root_validator(pre=True)
+    def validate_notice_key(cls, values: Dict[str, Any]) -> Dict[str, Any]:
+        """
+        Generate the notice_key from the name if not supplied
+        """
+        if not values.get("notice_key"):
+            values["notice_key"] = PrivacyNoticeModel.generate_notice_key(
+                values.get("name")
+            )
+
+        return values
+
 
 class PrivacyNoticeWithId(PrivacyNotice):
     """

src/fides/api/service/connectors/consent_email_connector.py

@@ -103,6 +103,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]:
                                 preference=UserConsentPreference.opt_in,
                                 privacy_notice_history=PrivacyNoticeHistorySchema(
                                     name="Targeted Advertising",
+                                    notice_key="targeted_advertising",
                                     regions=["us_ca"],
                                     id="test_1",
                                     privacy_notice_id="12345",
@@ -117,6 +118,7 @@ def test_connection(self) -> Optional[ConnectionTestStatus]:
                                 preference=UserConsentPreference.opt_out,
                                 privacy_notice_history=PrivacyNoticeHistorySchema(
                                     name="Analytics",
+                                    notice_key="analytics",
                                     regions=["us_ca"],
                                     id="test_2",
                                     privacy_notice_id="67890",

tests/fixtures/application_fixtures.py

@@ -1444,6 +1444,7 @@ def privacy_notice(db: Session) -> Generator:
         db=db,
         data={
             "name": "example privacy notice",
+            "notice_key": "example_privacy_notice",
             "description": "a sample privacy notice configuration",
             "origin": "privacy_notice_template_1",
             "regions": [
@@ -1468,6 +1469,7 @@ def privacy_notice_us_ca_provide(db: Session) -> Generator:
         db=db,
         data={
             "name": "example privacy notice us_ca provide",
+            "notice_key": "example_privacy_notice_us_ca_provide",
             # no description or origin on this privacy notice to help
             # cover edge cases due to column nullability
             "regions": [PrivacyNoticeRegion.us_ca],
@@ -1532,6 +1534,7 @@ def privacy_notice_us_co_third_party_sharing(db: Session) -> Generator:
         db=db,
         data={
             "name": "example privacy notice us_co third_party_sharing",
+            "notice_key": "example_privacy_notice_us_co_third_party_sharing",
             "description": "a sample privacy notice configuration",
             "origin": "privacy_notice_template_2",
             "regions": [PrivacyNoticeRegion.us_co],
@@ -1553,6 +1556,7 @@ def privacy_notice_us_co_provide_service_operations(db: Session) -> Generator:
         db=db,
         data={
             "name": "example privacy notice us_co provide.service.operations",
+            "notice_key": "example_privacy_notice_us_co_provide.service.operations",
             "description": "a sample privacy notice configuration",
             "origin": "privacy_notice_template_2",
             "regions": [PrivacyNoticeRegion.us_co],
@@ -1574,6 +1578,7 @@ def privacy_notice_eu_fr_provide_service_frontend_only(db: Session) -> Generator
         db=db,
         data={
             "name": "example privacy notice us_co provide.service.operations",
+            "notice_key": "example_privacy_notice_us_co_provide.service.operations",
             "description": "a sample privacy notice configuration",
             "origin": "privacy_notice_template_2",
             "regions": [PrivacyNoticeRegion.eu_fr],
@@ -1594,7 +1599,8 @@ def privacy_notice_eu_cy_provide_service_frontend_only(db: Session) -> Generator
     privacy_notice = PrivacyNotice.create(
         db=db,
         data={
-            "name": "example privacy notice us_co provide.service.operations",
+            "name": "example privacy notice eu_cy provide.service.operations",
+            "notice_key": "example_privacy_notice_eu_cy_provide.service.operations",
             "description": "a sample privacy notice configuration",
             "regions": [PrivacyNoticeRegion.eu_cy],
             "consent_mechanism": ConsentMechanism.opt_out,

tests/ops/api/v1/endpoints/test_privacy_experience_config_endpoints.py

@@ -683,6 +683,7 @@ def test_opt_in_notices_must_be_delivered_via_banner(
             db=db,
             data={
                 "name": "Test Notice",
+                "notice_key": "test_notice",
                 "regions": [PrivacyNoticeRegion.us_tx],
                 "consent_mechanism": ConsentMechanism.opt_in,
                 "data_uses": ["provide"],
@@ -1474,6 +1475,7 @@ def test_opt_in_notices_must_be_delivered_via_banner_not_already_linked(
             db=db,
             data={
                 "name": "Test Notice",
+                "notice_key": "test_notice",
                 "regions": [PrivacyNoticeRegion.us_tx],
                 "consent_mechanism": ConsentMechanism.opt_in,
                 "data_uses": ["provide"],
@@ -1531,6 +1533,7 @@ def test_opt_in_notices_must_be_delivered_via_banner_already_linked(
             db=db,
             data={
                 "name": "Test Notice",
+                "notice_key": "test_notice",
                 "regions": [PrivacyNoticeRegion.us_tx],
                 "consent_mechanism": ConsentMechanism.opt_in,
                 "data_uses": ["provide"],

tests/ops/api/v1/endpoints/test_privacy_experience_endpoints.py

@@ -746,3 +746,7 @@ def test_get_privacy_experience_detail_fides_user_device_id_filter(
         assert data["privacy_notices"][0]["default_preference"] == "opt_out"
         assert data["privacy_notices"][0]["current_preference"] == "opt_in"
         assert data["privacy_notices"][0]["outdated_preference"] is None
+        assert (
+            data["privacy_notices"][0]["notice_key"]
+            == "example_privacy_notice_us_ca_provide"
+        )