Developer-Friendly TCF Experiences

CHANGELOG.md

@@ -27,6 +27,8 @@ The types of changes are:
 - Fides-js can now display preliminary TCF data [#3879](https://github.com/ethyca/fides/pull/3879)
 - Fides-js can persist TCF preferences to the backend [#3887](https://github.com/ethyca/fides/pull/3887)
 - TCF modal now supports setting legitimate interest fields [#4037](https://github.com/ethyca/fides/pull/4037)
+- TCF overlay description updates [#4051] https://github.com/ethyca/fides/pull/4151
+- Added developer-friendly TCF information under Experience meta [#4160](https://github.com/ethyca/fides/pull/4160/)
 
 ### Changed
 - Added further config options to customize the privacy center [#4090](https://github.com/ethyca/fides/pull/4090)

requirements.txt

@@ -19,6 +19,7 @@ firebase-admin==5.3.0
 GitPython==3.1.35
 httpx==0.23.1
 hvac==0.11.2
+iab-tcf==0.2.2
 importlib_resources==5.12.0
 Jinja2==3.1.2
 loguru==0.6.0

src/fides/api/api/v1/endpoints/privacy_experience_endpoints.py

@@ -34,7 +34,12 @@
     get_fides_user_device_id_provided_identity,
 )
 from fides.api.util.endpoint_utils import fides_limiter, transform_fields
-from fides.api.util.tcf_util import TCF_COMPONENT_MAPPING, TCFExperienceContents
+from fides.api.util.tcf.experience_meta import build_experience_tcf_meta
+from fides.api.util.tcf_util import (
+    TCF_COMPONENT_MAPPING,
+    TCFExperienceContents,
+    get_tcf_contents,
+)
 from fides.common.api.v1 import urn_registry as urls
 from fides.config import CONFIG
 
@@ -131,6 +136,7 @@ def privacy_experience_list(
     has_config: Optional[bool] = None,
     fides_user_device_id: Optional[str] = None,
     systems_applicable: Optional[bool] = False,
+    include_meta: Optional[bool] = False,
     request: Request,  # required for rate limiting
     response: Response,  # required for rate limiting
 ) -> AbstractPage[PrivacyExperience]:
@@ -150,6 +156,7 @@ def privacy_experience_list(
     :param has_config: If True, returns Experiences with copy. If False, returns just Experiences without copy.
     :param fides_user_device_id: Supplement the response with current saved preferences of the given user
     :param systems_applicable: Only return embedded Notices associated with systems.
+    :param include_meta: If True, returns TCF Experience meta if applicable
     :param request:
     :param response:
     :return:
@@ -205,6 +212,9 @@ def privacy_experience_list(
 
     results: List[PrivacyExperience] = []
     should_unescape: Optional[str] = request.headers.get(UNESCAPE_SAFESTR_HEADER)
+    # Builds TCF Experience Contents once here, in case multiple TCF Experiences are requested
+    base_tcf_contents: TCFExperienceContents = get_tcf_contents(db)
+
     for privacy_experience in experience_query.order_by(
         PrivacyExperience.created_at.desc()
     ):
@@ -215,6 +225,8 @@ def privacy_experience_list(
             systems_applicable=systems_applicable,
             fides_user_provided_identity=fides_user_provided_identity,
             should_unescape=should_unescape,
+            include_meta=include_meta,
+            base_tcf_contents=base_tcf_contents,
         )
 
         if content_required and not content_exists:
@@ -245,6 +257,8 @@ def embed_experience_details(
     systems_applicable: Optional[bool],
     fides_user_provided_identity: Optional[ProvidedIdentity],
     should_unescape: Optional[str],
+    include_meta: Optional[bool],
+    base_tcf_contents: TCFExperienceContents,
 ) -> bool:
     """
     Embed the contents of the PrivacyExperience at runtime. Adds Privacy Notices or TCF contents if applicable.
@@ -254,19 +268,18 @@ def embed_experience_details(
     """
     # Reset any temporary cached items just in case
     privacy_experience.privacy_notices = []
+    privacy_experience.meta = {}
     for component in TCF_COMPONENT_MAPPING:
         setattr(privacy_experience, component, [])
 
-    # Fetch the base TCF Contents
-    tcf_contents: TCFExperienceContents = privacy_experience.get_related_tcf_contents(
-        db, fides_user_provided_identity
-    )
-    has_tcf_contents: bool = any(
-        getattr(tcf_contents, component) for component in TCF_COMPONENT_MAPPING
+    # Updates Privacy Experience in-place with TCF Contents if applicable, and then returns
+    # if TCF contents exist
+    has_tcf_contents: bool = privacy_experience.update_with_tcf_contents(
+        db, base_tcf_contents, fides_user_provided_identity
     )
-    # Add fetched TCF contents to the Privacy Experience if applicable
-    for component in TCF_COMPONENT_MAPPING:
-        setattr(privacy_experience, component, getattr(tcf_contents, component))
+
+    if has_tcf_contents and include_meta:
+        privacy_experience.meta = build_experience_tcf_meta(base_tcf_contents)
 
     privacy_notices: List[
         PrivacyNotice

src/fides/api/models/privacy_experience.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+from copy import copy
 from enum import Enum
 from typing import Any, Dict, List, Optional, Set, Tuple, Type, Union
 
@@ -28,7 +29,6 @@
     TCF_COMPONENT_MAPPING,
     ConsentRecordType,
     TCFExperienceContents,
-    get_tcf_contents,
 )
 
 BANNER_CONSENT_MECHANISMS: Set[ConsentMechanism] = {
@@ -223,6 +223,8 @@ class PrivacyExperience(Base):
     tcf_features: List = []
     tcf_special_features: List = []
     tcf_systems: List = []
+    # TCF Developer-Friendly Meta added at runtime as the result of build_tc_data_for_mobile
+    meta: Dict = {}
 
     # Attribute that is cached on the PrivacyExperience object by "get_should_show_banner", calculated at runtime
     show_banner: bool
@@ -308,15 +310,23 @@ def get_related_privacy_notices(
 
         return notices
 
-    def get_related_tcf_contents(
-        self, db: Session, fides_user_provided_identity: Optional[ProvidedIdentity]
-    ) -> TCFExperienceContents:
-        """Returns the contents of a TCF experience supplemented with any previous records of
-        a user being served TCF components and/or consenting to any of the individual TCF components
+    def update_with_tcf_contents(
+        self,
+        db: Session,
+        base_tcf_contents: TCFExperienceContents,
+        fides_user_provided_identity: Optional[ProvidedIdentity],
+    ) -> bool:
+        """
+        Supplements the given TCF experience in-place with TCF contents at runtime, and returns whether
+        TCF contents exist.
+
+        The TCF experience is determined by systems in the data map as well as any previous records
+        of a user being served TCF components and/or consenting to any individual TCF components.
         """
         if self.component == ComponentType.tcf_overlay:
-            tcf_contents: TCFExperienceContents = get_tcf_contents(db)
+            tcf_contents = copy(base_tcf_contents)
 
+            # Fetch previously saved records for the current user
             for tcf_component, field_name in TCF_COMPONENT_MAPPING.items():
                 for record in getattr(tcf_contents, tcf_component):
                     cache_saved_and_served_on_consent_record(
@@ -325,8 +335,19 @@ def get_related_tcf_contents(
                         fides_user_provided_identity=fides_user_provided_identity,
                         record_type=field_name,
                     )
-            return tcf_contents
-        return TCFExperienceContents()
+
+            has_tcf_contents: bool = False
+            for component in TCF_COMPONENT_MAPPING:
+                tcf_contents_for_component = getattr(tcf_contents, component)
+                if bool(tcf_contents_for_component):
+                    has_tcf_contents = True
+                # Add TCF contents to the privacy experience where applicable
+                setattr(self, component, tcf_contents_for_component)
+
+            if has_tcf_contents:
+                return True
+
+        return False
 
     @staticmethod
     def create_default_experience_for_region(

src/fides/api/models/privacy_notice.py

@@ -84,6 +84,7 @@ class UserConsentPreference(Enum):
         ("us_wv", "us_wv"),  # west virginia
         ("us_wi", "us_wi"),  # wisconsin
         ("us_wy", "us_wy"),  # wyoming
+        ("eea", "eea"),  # european economic area
         ("be", "be"),  # belgium
         ("bg", "bg"),  # bulgaria
         ("cz", "cz"),  # czechia

src/fides/api/schemas/privacy_experience.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
 from datetime import datetime
-from typing import Any, Dict, List, Optional
+from typing import Any, Dict, List, Literal, Optional
 
 from pydantic import Extra, Field, root_validator, validator
 
@@ -182,6 +182,90 @@ class PrivacyExperienceWithId(PrivacyExperience):
     id: str
 
 
+BinaryChoice = Literal[0, 1]
+
+
+class TCMobileData(FidesSchema):
+    """Pre-parsed TC data and TC string for a CMP SDK:
+
+    https://github.com/InteractiveAdvertisingBureau/GDPR-Transparency-and-Consent-Framework/blob/master/TCFv2/IAB%20Tech%20Lab%20-%20CMP%20API%20v2.md#in-app-details
+    """
+
+    IABTCF_CmpSdkID: Optional[int] = Field(
+        description="The unsigned integer ID of CMP SDK"
+    )
+    IABTCF_CmpSdkVersion: Optional[int] = Field(
+        description="The unsigned integer version number of CMP SDK"
+    )
+    IABTCF_PolicyVersion: Optional[int] = Field(
+        description="The unsigned integer representing the version of the TCF that these consents adhere to."
+    )
+    IABTCF_gdprApplies: Optional[BinaryChoice] = Field(
+        description="1: GDPR applies in current context, 0 - GDPR does not apply in current context, None=undetermined"
+    )
+    IABTCF_PublisherCC: Optional[str] = Field(
+        default="AA", description="Two-letter ISO 3166-1 alpha-2 code"
+    )
+    IABTCF_PurposeOneTreatment: Optional[BinaryChoice] = Field(
+        description="Vendors can use this value to determine whether consent for purpose one is required. 0: "
+        "no special treatment. 1: purpose one not disclosed"
+    )
+    IABTCF_UseNonStandardTexts: Optional[BinaryChoice] = Field(
+        description="1 - CMP uses customized stack descriptions and/or modified or supplemented standard illustrations."
+        "0 - CMP did not use a non-standard stack desc. and/or modified or supplemented Illustrations"
+    )
+    IABTCF_TCString: Optional[str] = Field(description="Fully encoded TC string")
+    IABTCF_VendorConsents: Optional[str] = Field(
+        description="Binary string: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the "
+        "consent status for Vendor ID n+1; false and true respectively. eg. '1' at index 0 is consent "
+        "true for vendor ID 1"
+    )
+    IABTCF_VendorLegitimateInterests: Optional[str] = Field(
+        description="Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the "
+        "legitimate interest status for Vendor ID n+1; false and true respectively. eg. '1' at index 0 is "
+        "legitimate interest established true for vendor ID 1"
+    )
+    IABTCF_PurposeConsents: Optional[str] = Field(
+        description="Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the "
+        "consent status for purpose ID n+1; false and true respectively. eg. '1' at index 0 is consent "
+        "true for purpose ID 1"
+    )
+    IABTCF_PurposeLegitimateInterests: Optional[str] = Field(
+        description="Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates the"
+        " legitimate interest status for purpose ID n+1; false and true respectively. eg. '1' at index 0 "
+        "is legitimate interest established true for purpose ID 1"
+    )
+    IABTCF_SpecialFeaturesOptIns: Optional[str] = Field(
+        description="Binary String: The '0' or '1' at position n – where n's indexing begins at 0 – indicates "
+        "the opt-in status for special feature ID n+1; false and true respectively. eg. '1' at index 0 is "
+        "opt-in true for special feature ID 1"
+    )
+    # IABTCF_PublisherRestrictions{ID}  # TODO this field has dynamic keys.  Add when we start surfacing publisher restrictions
+    IABTCF_PublisherConsent: Optional[str] = None
+    IABTCF_PublisherLegitimateInterests: Optional[str] = None
+    IABTCF_PublisherCustomPurposesConsents: Optional[str] = None
+    IABTCF_PublisherCustomPurposesLegitimateInterests: Optional[str] = None
+
+
+class ExperienceMeta(FidesSchema):
+    """Supplements experience with developer-friendly meta information"""
+
+    version_hash: Optional[str] = Field(
+        description="A hashed value that can be compared to previously-fetched "
+        "hash values to determine if the Experience has meaningfully changed"
+    )
+    accept_all_tc_string: Optional[str] = Field(
+        description="The TC string corresponding to a user opting in to all "
+        "available options"
+    )
+    accept_all_tc_mobile_data: Optional[TCMobileData] = None
+    reject_all_tc_string: Optional[str] = Field(
+        description="The TC string corresponding to a user opting out of all "
+        "available options"
+    )
+    reject_all_tc_mobile_data: Optional[TCMobileData] = None
+
+
 class PrivacyExperienceResponse(PrivacyExperienceWithId):
     """
     An API representation of a PrivacyExperience used for response payloads
@@ -217,3 +301,4 @@ class PrivacyExperienceResponse(PrivacyExperienceWithId):
     experience_config: Optional[ExperienceConfigResponse] = Field(
         description="The Experience copy or language"
     )
+    meta: Optional[ExperienceMeta] = None

src/fides/api/schemas/privacy_preference.py

@@ -86,6 +86,10 @@ class PrivacyPreferencesRequest(FidesSchema):
 
     browser_identity: Identity
     code: Optional[SafeStr]
+    tc_string: Optional[str] = Field(
+        description="If supplied, *_preferences fields will be ignored in favor of this TC string. Saves preferences"
+        "against purposes, special features, and vendors."
+    )  # TODO: Not yet implemented
     preferences: conlist(ConsentOptionCreate, max_items=200) = []  # type: ignore
     purpose_preferences: conlist(TCFPurposeSave, max_items=200) = []  # type: ignore
     special_purpose_preferences: conlist(TCFSpecialPurposeSave, max_items=200) = []  # type: ignore

src/fides/api/util/consent_util.py

@@ -616,6 +616,7 @@ def create_default_experience_config(
 
 
 EEA_COUNTRIES: List[PrivacyNoticeRegion] = [
+    PrivacyNoticeRegion.eea,
     PrivacyNoticeRegion.be,
     PrivacyNoticeRegion.bg,
     PrivacyNoticeRegion.cz,

src/fides/api/util/tcf/experience_meta.py

@@ -0,0 +1,83 @@
+import hashlib
+import json
+from typing import Dict, List
+
+from pydantic import Extra, root_validator
+
+from fides.api.models.privacy_notice import UserConsentPreference
+from fides.api.schemas.base_class import FidesSchema
+from fides.api.schemas.privacy_experience import ExperienceMeta, TCMobileData
+from fides.api.util.tcf.tc_mobile_data import build_tc_data_for_mobile
+from fides.api.util.tcf.tc_model import TCModel, build_tc_model
+from fides.api.util.tcf_util import TCFExperienceContents
+
+
+class TCFVersionHash(FidesSchema):
+    """Minimal subset of the TCF experience details that capture when consent should be resurfaced"""
+
+    policy_version: int
+    purpose_consents: List[int]
+    purpose_legitimate_interests: List[int]
+    special_feature_optins: List[int]
+    vendor_consents: List[int]
+    vendor_legitimate_interests: List[int]
+
+    @root_validator()
+    @classmethod
+    def sort_lists(cls, values: Dict) -> Dict:
+        """Verify lists are sorted ascending for repeatability"""
+        for field, val in values.items():
+            if isinstance(val, list):
+                values[field] = sorted(val)
+        return values
+
+    class Config:
+        extra = Extra.ignore
+
+
+def _build_tcf_version_hash_model(
+    tcf_contents: TCFExperienceContents,
+) -> TCFVersionHash:
+    """Given tcf_contents, constructs the TCFVersionHash model containing
+    the raw contents to build the `version_hash` for the TCF Experience.
+
+    Builds a model that assumes the customer has *opted in* to all preferences, to
+    get the maximum possible number of attributes added to our hash for the current
+    system configuration.
+    """
+    model: TCModel = build_tc_model(tcf_contents, UserConsentPreference.opt_in)
+    return TCFVersionHash(**model.dict())
+
+
+def build_tcf_version_hash(tcf_contents: TCFExperienceContents) -> str:
+    """Returns a 12-character version hash for TCF that should only change
+    if there are updates to vendors, purposes, and special features sections or legal basis.
+
+    This hash can be used to determine if the TCF Experience needs to be resurfaced to the customer,
+    because the experience has changed in a meaningful way.
+    """
+    tcf_version_hash_model: TCFVersionHash = _build_tcf_version_hash_model(tcf_contents)
+    json_str: str = json.dumps(tcf_version_hash_model.dict(), sort_keys=True)
+    hashed_val: str = hashlib.sha256(json_str.encode()).hexdigest()
+    return hashed_val[:12]  # Shortening string for usability, collision risk is low
+
+
+def build_experience_tcf_meta(tcf_contents: TCFExperienceContents) -> Dict:
+    """Build TCF Meta information to supplement a TCF Privacy Experience at runtime"""
+    accept_all_tc_model: TCModel = build_tc_model(
+        tcf_contents, UserConsentPreference.opt_in
+    )
+    reject_all_tc_model: TCModel = build_tc_model(
+        tcf_contents, UserConsentPreference.opt_out
+    )
+
+    accept_all_mobile_data: TCMobileData = build_tc_data_for_mobile(accept_all_tc_model)
+    reject_all_mobile_data: TCMobileData = build_tc_data_for_mobile(reject_all_tc_model)
+
+    return ExperienceMeta(
+        version_hash=build_tcf_version_hash(tcf_contents),
+        accept_all_tc_string=accept_all_mobile_data.IABTCF_TCString,
+        reject_all_tc_string=reject_all_mobile_data.IABTCF_TCString,
+        accept_all_tc_mobile_data=accept_all_mobile_data,
+        reject_all_tc_mobile_data=reject_all_mobile_data,
+    ).dict()