diff --git a/.fides/db_dataset.yml b/.fides/db_dataset.yml index 422d7a6f40c..a23c29c077a 100644 --- a/.fides/db_dataset.yml +++ b/.fides/db_dataset.yml @@ -148,6 +148,10 @@ dataset: data_categories: - system.operations data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device_provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: id data_categories: - system.operations @@ -1919,6 +1923,18 @@ dataset: data_categories: - user data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: fides_user_device_provided_identity_id + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified + - name: hashed_fides_user_device + data_categories: + - system.operations + data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified - name: id data_categories: - system.operations diff --git a/CHANGELOG.md b/CHANGELOG.md index b7e102062ae..aa2018fe801 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The types of changes are: - Side navigation bar can now also have children navigation links [#3099](https://github.com/ethyca/fides/pull/3099) - Endpoints for consent reporting [#3095](https://github.com/ethyca/fides/pull/3095) - Custom fields table [#3097](https://github.com/ethyca/fides/pull/3097) +- Endpoints to save the new-style Privacy Preferences with respect to a fides user device id [#3132](https://github.com/ethyca/fides/pull/3132) ### Changed diff --git a/docs/fides/docs/development/postman/Fides.postman_collection.json b/docs/fides/docs/development/postman/Fides.postman_collection.json index ee3b21b4a1f..bf3414753c8 100644 --- a/docs/fides/docs/development/postman/Fides.postman_collection.json +++ b/docs/fides/docs/development/postman/Fides.postman_collection.json @@ -5486,6 +5486,67 @@ }, "response": [] }, + { + "name": "Save Privacy Preferences for Device Id", + "request": { + "method": "PATCH", + "header": [], + "body": { + "mode": "raw", + "raw": "{\n \"browser_identity\": {\n \"ga_client_id\": \"UA-XXXXXXXXX\",\n \"ljt_readerID\": \"test_sovrn_id\",\n \"fides_user_device_id\": \"{{fides_user_device_id}}\"\n },\n \"preferences\": [{\n \"privacy_notice_history_id\": \"{{privacy_notice_history_id}}\",\n \"preference\": \"opt_out\"\n }],\n \"request_origin\": \"privacy_center\",\n \"url_recorded\": \"example.com/privacy_center\",\n \"user_agent\": \"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24\",\n \"user_geography\": \"us_ca\"\n}", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-preferences", + "host": [ + "{{host}}" + ], + "path": [ + "privacy-preferences" + ] + } + }, + "response": [] + }, + { + "name": "Privacy Preferences by Fides User Device Id", + "protocolProfileBehavior": { + "disableBodyPruning": true + }, + "request": { + "method": "GET", + "header": [], + "body": { + "mode": "raw", + "raw": "", + "options": { + "raw": { + "language": "json" + } + } + }, + "url": { + "raw": "{{host}}/privacy-preferences?fides_user_device_id={{fides_user_device_id}}", + "host": [ + "{{host}}" + ], + "path": [ + "privacy-preferences" + ], + "query": [ + { + "key": "fides_user_device_id", + "value": "{{fides_user_device_id}}" + } + ] + } + }, + "response": [] + }, { "name": "Get Historical Preferences", "request": { @@ -5872,6 +5933,11 @@ "key": "oauth_connector_key", "value": "", "type": "string" + }, + { + "key": "fides_user_device_id", + "value": "", + "type": "string" } ] } \ No newline at end of file diff --git a/src/fides/api/ctl/migrations/versions/3842d1acac5f_adds_fides_device_id_enum.py b/src/fides/api/ctl/migrations/versions/3842d1acac5f_adds_fides_device_id_enum.py index 236941a139c..0c3451d605a 100644 --- a/src/fides/api/ctl/migrations/versions/3842d1acac5f_adds_fides_device_id_enum.py +++ b/src/fides/api/ctl/migrations/versions/3842d1acac5f_adds_fides_device_id_enum.py @@ -7,7 +7,6 @@ """ from alembic import op - # revision identifiers, used by Alembic. revision = "3842d1acac5f" down_revision = "8342453518cc" diff --git a/src/fides/api/ctl/migrations/versions/451684a726a5_save_privacy_preferences_on_device_id.py b/src/fides/api/ctl/migrations/versions/451684a726a5_save_privacy_preferences_on_device_id.py new file mode 100644 index 00000000000..906abfc2120 --- /dev/null +++ b/src/fides/api/ctl/migrations/versions/451684a726a5_save_privacy_preferences_on_device_id.py @@ -0,0 +1,110 @@ +"""save privacy preferences on device id + +Revision ID: 451684a726a5 +Revises: 48d9caacebd4 +Create Date: 2023-04-23 16:06:19.788074 + +""" +import sqlalchemy as sa +import sqlalchemy_utils +from alembic import op + +# revision identifiers, used by Alembic. +revision = "451684a726a5" +down_revision = "48d9caacebd4" +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column( + "currentprivacypreference", + sa.Column("fides_user_device_provided_identity_id", sa.String(), nullable=True), + ) + op.create_unique_constraint( + "fides_user_device_identity_privacy_notice", + "currentprivacypreference", + ["fides_user_device_provided_identity_id", "privacy_notice_id"], + ) + op.create_index( + op.f("ix_currentprivacypreference_fides_user_device_provided_identity_id"), + "currentprivacypreference", + ["fides_user_device_provided_identity_id"], + unique=False, + ) + op.create_foreign_key( + "currentprivacypreference_fides_user_device_provided_identi_fkey", + "currentprivacypreference", + "providedidentity", + ["fides_user_device_provided_identity_id"], + ["id"], + ) + op.add_column( + "privacypreferencehistory", + sa.Column( + "fides_user_device", + sqlalchemy_utils.types.encrypted.encrypted_type.StringEncryptedType(), + nullable=True, + ), + ) + op.add_column( + "privacypreferencehistory", + sa.Column("fides_user_device_provided_identity_id", sa.String(), nullable=True), + ) + op.add_column( + "privacypreferencehistory", + sa.Column("hashed_fides_user_device", sa.String(), nullable=True), + ) + op.create_index( + op.f("ix_privacypreferencehistory_fides_user_device_provided_identity_id"), + "privacypreferencehistory", + ["fides_user_device_provided_identity_id"], + unique=False, + ) + op.create_index( + op.f("ix_privacypreferencehistory_hashed_fides_user_device"), + "privacypreferencehistory", + ["hashed_fides_user_device"], + unique=False, + ) + op.create_foreign_key( + "privacypreferencehistory_fides_user_device_provided_identi_fkey", + "privacypreferencehistory", + "providedidentity", + ["fides_user_device_provided_identity_id"], + ["id"], + ) + + +def downgrade(): + op.drop_constraint( + "privacypreferencehistory_fides_user_device_provided_identi_fkey", + "privacypreferencehistory", + type_="foreignkey", + ) + op.drop_index( + op.f("ix_privacypreferencehistory_hashed_fides_user_device"), + table_name="privacypreferencehistory", + ) + op.drop_index( + op.f("ix_privacypreferencehistory_fides_user_device_provided_identity_id"), + table_name="privacypreferencehistory", + ) + op.drop_column("privacypreferencehistory", "hashed_fides_user_device") + op.drop_column("privacypreferencehistory", "fides_user_device_provided_identity_id") + op.drop_column("privacypreferencehistory", "fides_user_device") + op.drop_constraint( + "currentprivacypreference_fides_user_device_provided_identi_fkey", + "currentprivacypreference", + type_="foreignkey", + ) + op.drop_index( + op.f("ix_currentprivacypreference_fides_user_device_provided_identity_id"), + table_name="currentprivacypreference", + ) + op.drop_constraint( + "fides_user_device_identity_privacy_notice", + "currentprivacypreference", + type_="unique", + ) + op.drop_column("currentprivacypreference", "fides_user_device_provided_identity_id") diff --git a/src/fides/api/ops/api/v1/endpoints/privacy_preference_endpoints.py b/src/fides/api/ops/api/v1/endpoints/privacy_preference_endpoints.py index 132a21270a7..c170186a5cc 100644 --- a/src/fides/api/ops/api/v1/endpoints/privacy_preference_endpoints.py +++ b/src/fides/api/ops/api/v1/endpoints/privacy_preference_endpoints.py @@ -2,8 +2,9 @@ from datetime import datetime from typing import List, Optional, Tuple +from urllib.parse import urlparse -from fastapi import Depends, HTTPException +from fastapi import Depends, HTTPException, Request from fastapi.params import Security from fastapi_pagination import Page, Params from fastapi_pagination.bases import AbstractPage @@ -11,7 +12,12 @@ from loguru import logger from sqlalchemy import literal from sqlalchemy.orm import Query, Session -from starlette.status import HTTP_200_OK, HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND +from starlette.status import ( + HTTP_200_OK, + HTTP_400_BAD_REQUEST, + HTTP_403_FORBIDDEN, + HTTP_404_NOT_FOUND, +) from fides.api.ctl.database.seed import DEFAULT_CONSENT_POLICY from fides.api.ops.api.deps import get_db @@ -29,8 +35,9 @@ from fides.api.ops.api.v1.urn_registry import ( CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY, CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID, - CURRENT_PRIVACY_PREFERENCES, - HISTORICAL_PRIVACY_PREFERENCES, + CURRENT_PRIVACY_PREFERENCES_REPORT, + HISTORICAL_PRIVACY_PREFERENCES_REPORT, + PRIVACY_PREFERENCES, V1_URL_PREFIX, ) from fides.api.ops.models.privacy_notice import PrivacyNotice, PrivacyNoticeHistory @@ -39,6 +46,7 @@ PrivacyPreferenceHistory, ) from fides.api.ops.models.privacy_request import ( + ConsentRequest, PrivacyRequest, ProvidedIdentity, ProvidedIdentityType, @@ -56,7 +64,12 @@ ) from fides.api.ops.schemas.redis_cache import Identity from fides.api.ops.util.api_router import APIRouter +from fides.api.ops.util.consent_util import ( + get_fides_user_device_id_provided_identity, + get_or_create_fides_user_device_id_provided_identity, +) from fides.api.ops.util.oauth_util import verify_oauth_client +from fides.core.config import CONFIG from fides.core.config.config_proxy import ConfigProxy from fides.lib.models.fides_user import FidesUser @@ -136,12 +149,13 @@ def verify_privacy_notice_and_historical_records( def extract_identity_from_provided_identity( - identity: ProvidedIdentity, identity_type: ProvidedIdentityType + identity: Optional[ProvidedIdentity], identity_type: ProvidedIdentityType ) -> Tuple[Optional[str], Optional[str]]: """Pull the identity data off of the ProvidedIdentity given that it's the correct type""" value: Optional[str] = None hashed_value: Optional[str] = None - if identity.encrypted_value and identity.field_name == identity_type: + + if identity and identity.encrypted_value and identity.field_name == identity_type: value = identity.encrypted_value["value"] hashed_value = identity.hashed_value @@ -153,19 +167,17 @@ def extract_identity_from_provided_identity( status_code=HTTP_200_OK, response_model=List[CurrentPrivacyPreferenceSchema], ) -def save_privacy_preferences( +def save_privacy_preferences_with_verified_identity( *, consent_request_id: str, db: Session = Depends(get_db), data: PrivacyPreferencesCreateWithCode, ) -> List[CurrentPrivacyPreference]: - """Verifies the verification code and saves the user's privacy preferences if successful. + """Saves privacy preferences with respect to a verified user identity like an email or phone number + and optionally a fides user device id. - Creates a historical record for each privacy preference for record keeping and then upserts the current preference - for each privacy notice. Creates a Privacy Request linked to the historical preferences to - propagate these preferences where applicable. - - This workflow is for users saving preferences for privacy notices with a verified identity. + Creates historical records for these preferences for record keeping, and also updates current preferences. + Creates a privacy request to propagate preferences to third party systems. """ verify_privacy_notice_and_historical_records(db=db, data=data) consent_request, provided_identity = _get_consent_request_and_provided_identity( @@ -178,32 +190,72 @@ def save_privacy_preferences( status_code=HTTP_404_NOT_FOUND, detail="Provided identity missing" ) + try: + fides_user_provided_identity = ( + get_or_create_fides_user_device_id_provided_identity( + db=db, identity_data=data.browser_identity + ) + ) + except HTTPException: + fides_user_provided_identity = None + logger.info("Saving privacy preferences") + return _save_privacy_preferences_for_identities( + db=db, + consent_request=consent_request, + verified_provided_identity=provided_identity, + fides_user_provided_identity=fides_user_provided_identity, + request_data=data, + ) + + +def _save_privacy_preferences_for_identities( + db: Session, + consent_request: Optional[ConsentRequest], + verified_provided_identity: Optional[ProvidedIdentity], + fides_user_provided_identity: Optional[ProvidedIdentity], + request_data: PrivacyPreferencesCreateWithCode, +) -> List[CurrentPrivacyPreference]: + """ + Saves privacy preferences (both historical and current records) and creates a privacy request to propagate those + preferences for when we have a verified user identity (like email/phone number), just a fides user device from + the browser, or both. + """ created_historical_preferences: List[PrivacyPreferenceHistory] = [] upserted_current_preferences: List[CurrentPrivacyPreference] = [] email, hashed_email = extract_identity_from_provided_identity( - provided_identity, ProvidedIdentityType.email + verified_provided_identity, ProvidedIdentityType.email ) phone_number, hashed_phone_number = extract_identity_from_provided_identity( - provided_identity, ProvidedIdentityType.phone_number + verified_provided_identity, ProvidedIdentityType.phone_number + ) + fides_user_device_id, hashed_device_id = extract_identity_from_provided_identity( + fides_user_provided_identity, ProvidedIdentityType.fides_user_device_id ) - for privacy_preference in data.preferences: + for privacy_preference in request_data.preferences: historical_preference: PrivacyPreferenceHistory = PrivacyPreferenceHistory.create( db=db, data={ "email": email, + "fides_user_device": fides_user_device_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id + if fides_user_provided_identity + else None, "hashed_email": hashed_email, + "hashed_fides_user_device": hashed_device_id, "hashed_phone_number": hashed_phone_number, "phone_number": phone_number, "preference": privacy_preference.preference, "privacy_notice_history_id": privacy_preference.privacy_notice_history_id, - "provided_identity_id": provided_identity.id, - "request_origin": data.request_origin, - "user_agent": data.user_agent, - "user_geography": data.user_geography, - "url_recorded": data.url_recorded, + "provided_identity_id": verified_provided_identity.id + if verified_provided_identity + else None, + "request_origin": request_data.request_origin, + "user_agent": request_data.user_agent, + "user_geography": request_data.user_geography, + "url_recorded": request_data.url_recorded, }, check_name=False, ) @@ -214,12 +266,16 @@ def save_privacy_preferences( created_historical_preferences.append(historical_preference) upserted_current_preferences.append(upserted_current_preference) - identity = data.browser_identity if data.browser_identity else Identity() - setattr( - identity, - provided_identity.field_name.value, # type:ignore[attr-defined] - provided_identity.encrypted_value["value"], # type:ignore[index] - ) # Pull the information on the ProvidedIdentity for the ConsentRequest to pass along to create a PrivacyRequest + identity = ( + request_data.browser_identity if request_data.browser_identity else Identity() + ) + if verified_provided_identity: + # Pull the information on the ProvidedIdentity for the ConsentRequest to pass along to create a PrivacyRequest + setattr( + identity, + verified_provided_identity.field_name.value, # type:ignore[attr-defined] + verified_provided_identity.encrypted_value["value"], # type:ignore[index] + ) # Privacy Request needs to be created with respect to the *historical* privacy preferences privacy_request_results: BulkPostPrivacyRequests = create_privacy_request_func( @@ -228,7 +284,7 @@ def save_privacy_preferences( data=[ PrivacyRequestCreate( identity=identity, - policy_key=data.policy_key or DEFAULT_CONSENT_POLICY, + policy_key=request_data.policy_key or DEFAULT_CONSENT_POLICY, ) ], authenticated=True, @@ -241,21 +297,111 @@ def save_privacy_preferences( detail=privacy_request_results.failed[0].message, ) - consent_request.privacy_request_id = privacy_request_results.succeeded[0].id - consent_request.save(db=db) - + if consent_request: + # If we have a verified user identity, go ahead and update the associated ConsentRequest for record keeping + consent_request.privacy_request_id = privacy_request_results.succeeded[0].id + consent_request.save(db=db) return upserted_current_preferences +def verify_address(request: Request) -> None: + """Verify request is coming from approved address""" + origin = request.headers.get("Origin") or request.headers.get("Referer") + + if origin: + parsed_url = urlparse(origin) + if ( + parsed_url.scheme + "://" + parsed_url.netloc + not in CONFIG.security.cors_origins + ): + raise HTTPException( + status_code=HTTP_403_FORBIDDEN, + detail="Can't save privacy preferences from non-approved addresses", + ) + + +@router.get( + PRIVACY_PREFERENCES, + status_code=HTTP_200_OK, + response_model=Page[CurrentPrivacyPreferenceSchema], +) +def get_privacy_preferences_by_device_id( + *, + request: Request, + fides_user_device_id: str, + db: Session = Depends(get_db), + params: Params = Depends(), +) -> AbstractPage[CurrentPrivacyPreference]: + """Retrieves privacy preferences with respect to a fides user device id.""" + verify_address(request) + fides_user_provided_identity: Optional[ + ProvidedIdentity + ] = get_fides_user_device_id_provided_identity( + db=db, fides_user_device_id=fides_user_device_id + ) + + if not fides_user_provided_identity: + raise HTTPException( + status_code=HTTP_403_FORBIDDEN, + detail="Provided identity not found", + ) + + logger.info( + "Getting privacy preferences with respect to fides user provided identity" + ) + query: Query[CurrentPrivacyPreference] = ( + db.query(CurrentPrivacyPreference) + .filter( + CurrentPrivacyPreference.fides_user_device_provided_identity_id + == fides_user_provided_identity.id + ) + .order_by(CurrentPrivacyPreference.updated_at.desc()) + ) + return paginate(query, params) + + +@router.patch( + PRIVACY_PREFERENCES, + status_code=HTTP_200_OK, + response_model=List[CurrentPrivacyPreferenceSchema], +) +def save_privacy_preferences( + *, + request: Request, + db: Session = Depends(get_db), + data: PrivacyPreferencesCreateWithCode, +) -> List[CurrentPrivacyPreference]: + """Saves privacy preferences with respect to a fides user device id. + + Creates historical records for these preferences for record keeping, and also updates current preferences. + Creates a privacy request to propagate preferences to third party systems. + """ + verify_privacy_notice_and_historical_records(db=db, data=data) + verify_address(request) + + fides_user_provided_identity = get_or_create_fides_user_device_id_provided_identity( + db=db, identity_data=data.browser_identity + ) + + logger.info("Saving privacy preferences with respect to fides user device id") + return _save_privacy_preferences_for_identities( + db=db, + consent_request=None, + verified_provided_identity=None, + fides_user_provided_identity=fides_user_provided_identity, + request_data=data, + ) + + @router.get( - CURRENT_PRIVACY_PREFERENCES, + CURRENT_PRIVACY_PREFERENCES_REPORT, status_code=HTTP_200_OK, dependencies=[ Security(verify_oauth_client, scopes=[CURRENT_PRIVACY_PREFERENCE_READ]) ], response_model=Page[CurrentPrivacyPreferenceReportingSchema], ) -def get_current_privacy_preferences( +def get_current_privacy_preferences_report( *, params: Params = Depends(), db: Session = Depends(get_db), @@ -279,14 +425,14 @@ def get_current_privacy_preferences( @router.get( - HISTORICAL_PRIVACY_PREFERENCES, + HISTORICAL_PRIVACY_PREFERENCES_REPORT, status_code=HTTP_200_OK, dependencies=[ Security(verify_oauth_client, scopes=[PRIVACY_PREFERENCE_HISTORY_READ]) ], response_model=Page[ConsentReportingSchema], ) -def get_historical_consent_reporting( +def get_historical_consent_report( *, params: Params = Depends(), db: Session = Depends(get_db), @@ -303,7 +449,9 @@ def get_historical_consent_reporting( db.query( PrivacyPreferenceHistory.id, PrivacyRequest.id.label("privacy_request_id"), - PrivacyPreferenceHistory.email.label("user_id"), + PrivacyPreferenceHistory.email.label("email"), + PrivacyPreferenceHistory.phone_number.label("phone_number"), + PrivacyPreferenceHistory.fides_user_device.label("fides_user_device_id"), PrivacyPreferenceHistory.secondary_user_ids, PrivacyPreferenceHistory.created_at.label("request_timestamp"), PrivacyPreferenceHistory.request_origin.label("request_origin"), diff --git a/src/fides/api/ops/api/v1/urn_registry.py b/src/fides/api/ops/api/v1/urn_registry.py index ccbd4890f65..b62bcfd81af 100644 --- a/src/fides/api/ops/api/v1/urn_registry.py +++ b/src/fides/api/ops/api/v1/urn_registry.py @@ -21,8 +21,11 @@ CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY = ( "/consent-request/{consent_request_id}/verify-for-privacy-preferences" ) -HISTORICAL_PRIVACY_PREFERENCES = "/historical-privacy-preferences" -CURRENT_PRIVACY_PREFERENCES = "/current-privacy-preferences" +PRIVACY_PREFERENCES = "/privacy-preferences" + +# Reporting endpoints - have records for *all* users +HISTORICAL_PRIVACY_PREFERENCES_REPORT = "/historical-privacy-preferences" +CURRENT_PRIVACY_PREFERENCES_REPORT = "/current-privacy-preferences" # Oauth Client URLs diff --git a/src/fides/api/ops/models/privacy_preference.py b/src/fides/api/ops/models/privacy_preference.py index 2a977324db6..251b3047de4 100644 --- a/src/fides/api/ops/models/privacy_preference.py +++ b/src/fides/api/ops/models/privacy_preference.py @@ -14,7 +14,10 @@ from sqlalchemy_utils import StringEncryptedType from sqlalchemy_utils.types.encrypted.encrypted_type import AesGcmEngine -from fides.api.ops.common_exceptions import PrivacyNoticeHistoryNotFound +from fides.api.ops.common_exceptions import ( + IdentityNotFoundException, + PrivacyNoticeHistoryNotFound, +) from fides.api.ops.db.base_class import JSONTypeOverride from fides.api.ops.models.privacy_notice import ( PrivacyNotice, @@ -47,10 +50,13 @@ class PrivacyPreferenceHistory(Base): every time consent preferences are saved for reporting purposes. """ + # Systems capable of propagating their consent, and their status. If the preference is + # not relevant for the system, or we couldn't propagate a preference, the status is skipped affected_system_status = Column( MutableDict.as_mutable(JSONB), server_default="{}", default=dict ) created_at = Column(DateTime(timezone=True), server_default=func.now(), index=True) + # Encrypted email, for reporting email = Column( StringEncryptedType( type_in=String(), @@ -59,8 +65,26 @@ class PrivacyPreferenceHistory(Base): padding="pkcs5", ), ) - hashed_email = Column(String, index=True) # For filtering - hashed_phone_number = Column(String, index=True) # For filtering + # Encrypted fides user device id, for reporting + fides_user_device = Column( + StringEncryptedType( + type_in=String(), + key=CONFIG.security.app_encryption_key, + engine=AesGcmEngine, + padding="pkcs5", + ), + ) + # Optional FK to a fides user device id provided identity, if applicable + fides_user_device_provided_identity_id = Column( + String, ForeignKey(ProvidedIdentity.id), index=True + ) + # Hashed email, for searching + hashed_email = Column(String, index=True) + # Hashed fides user device id, for searching + hashed_fides_user_device = Column(String, index=True) + # Hashed phone number, for searching + hashed_phone_number = Column(String, index=True) + # Encrypted phone number, for reporting phone_number = Column( StringEncryptedType( type_in=String(), @@ -69,18 +93,24 @@ class PrivacyPreferenceHistory(Base): padding="pkcs5", ), ) + # Whether the user wants to opt in, opt out, or has acknowledged the notice preference = Column(EnumColumn(UserConsentPreference), nullable=False, index=True) + # The specific historical record the user consented to privacy_notice_history_id = Column( String, ForeignKey(PrivacyNoticeHistory.id), nullable=False, index=True ) + # The privacy request created to propage the preferences privacy_request_id = Column( String, ForeignKey(PrivacyRequest.id, ondelete="SET NULL"), index=True ) + # Optional FK to a verified provided identity (like email or phone), if applicable provided_identity_id = Column(String, ForeignKey(ProvidedIdentity.id), index=True) - relevant_systems = Column( - MutableList.as_mutable(ARRAY(String)) - ) # Systems whose data use match. This doesn't necessarily mean we propagate. Some may be intentionally skipped later. + + # Systems whose data use match. This doesn't necessarily mean we propagate. Some may be intentionally skipped later. + relevant_systems = Column(MutableList.as_mutable(ARRAY(String))) + # Location where we received the request request_origin = Column(EnumColumn(RequestOrigin)) + # Relevant identities are added to the report during request propagation secondary_user_ids = Column( MutableDict.as_mutable( StringEncryptedType( @@ -107,7 +137,12 @@ class PrivacyPreferenceHistory(Base): # Relationships privacy_notice_history = relationship(PrivacyNoticeHistory) privacy_request = relationship(PrivacyRequest, backref="privacy_preferences") - provided_identity = relationship(ProvidedIdentity) + provided_identity = relationship( + ProvidedIdentity, foreign_keys=[provided_identity_id] + ) + fides_user_device_provided_identity = relationship( + ProvidedIdentity, foreign_keys=[fides_user_device_provided_identity_id] + ) current_privacy_preference = relationship( # Only exists if this is the same as the Current Privacy Preference "CurrentPrivacyPreference", back_populates="privacy_preference_history", @@ -149,6 +184,8 @@ def create( check_name: bool = False, ) -> PrivacyPreferenceHistory: """Create a PrivacyPreferenceHistory record and then upsert the CurrentPrivacyPreference record. + If separate CurrentPrivacyPreferences exist for both a verified provided identity and a fides user device + id provided identity, consolidate these "current" preferences into a single record. There is only one CurrentPrivacyPreference for each PrivacyNotice/ProvidedIdentity. """ @@ -158,6 +195,13 @@ def create( if not privacy_notice_history: raise PrivacyNoticeHistoryNotFound() + if not data.get("provided_identity_id") and not data.get( + "fides_user_device_provided_identity_id" + ): + raise IdentityNotFoundException( + "Must supply a verified provided identity id or a fides_user_device_provided_identity_id" + ) + data["relevant_systems"] = privacy_notice_history.calculate_relevant_systems(db) created_privacy_preference_history = super().create( db=db, data=data, check_name=check_name @@ -169,21 +213,51 @@ def create( "privacy_notice_id": privacy_notice_history.privacy_notice_id, "privacy_notice_history_id": privacy_notice_history.id, "privacy_preference_history_id": created_privacy_preference_history.id, + "fides_user_device_provided_identity_id": created_privacy_preference_history.fides_user_device_provided_identity_id, } - existing_current_preference = ( - db.query(CurrentPrivacyPreference) - .filter( - CurrentPrivacyPreference.provided_identity_id - == created_privacy_preference_history.provided_identity_id, - CurrentPrivacyPreference.privacy_notice_id - == privacy_notice_history.privacy_notice_id, + existing_current_preference_on_provided_identity = None + # Check if there are Current Privacy Preferences saved against the ProvidedIdentity + if created_privacy_preference_history.provided_identity_id: + existing_current_preference_on_provided_identity = ( + db.query(CurrentPrivacyPreference) + .filter( + CurrentPrivacyPreference.provided_identity_id + == created_privacy_preference_history.provided_identity_id, + CurrentPrivacyPreference.privacy_notice_id + == privacy_notice_history.privacy_notice_id, + ) + .first() + ) + + # Check if there are Current Privacy Preferences saved against the Fides User Device Id Provided Identity + existing_current_preference_on_fides_user_device_provided_identity = None + if created_privacy_preference_history.fides_user_device_provided_identity_id: + existing_current_preference_on_fides_user_device_provided_identity = ( + db.query(CurrentPrivacyPreference) + .filter( + CurrentPrivacyPreference.fides_user_device_provided_identity_id + == created_privacy_preference_history.fides_user_device_provided_identity_id, + CurrentPrivacyPreference.privacy_notice_id + == privacy_notice_history.privacy_notice_id, + ) + .first() + ) + + if ( + existing_current_preference_on_provided_identity + and existing_current_preference_on_fides_user_device_provided_identity + and existing_current_preference_on_provided_identity + != existing_current_preference_on_fides_user_device_provided_identity + ): + # If both exist and were saved separately, let's delete one so we can consolidate. + # Let's consider these identities as belonging to the same user, and can save their current preferences together. + existing_current_preference_on_fides_user_device_provided_identity.delete( + db ) - .first() - ) - if existing_current_preference: - existing_current_preference.update( + if existing_current_preference_on_provided_identity: + existing_current_preference_on_provided_identity.update( db=db, data=current_privacy_preference_data ) else: @@ -209,6 +283,9 @@ class CurrentPrivacyPreference(Base): ) preference = Column(EnumColumn(UserConsentPreference), nullable=False, index=True) provided_identity_id = Column(String, ForeignKey(ProvidedIdentity.id), index=True) + fides_user_device_provided_identity_id = Column( + String, ForeignKey(ProvidedIdentity.id), index=True + ) privacy_notice_id = Column( String, ForeignKey(PrivacyNotice.id), nullable=False, index=True ) @@ -223,8 +300,20 @@ class CurrentPrivacyPreference(Base): provided_identity_id, privacy_notice_id, name="identity_privacy_notice" ) + UniqueConstraint( + fides_user_device_provided_identity_id, + privacy_notice_id, + name="fides_user_device_identity_privacy_notice", + ) + # Relationships privacy_notice_history = relationship(PrivacyNoticeHistory) privacy_preference_history = relationship( PrivacyPreferenceHistory, cascade="delete, delete-orphan", single_parent=True ) + provided_identity = relationship( + ProvidedIdentity, foreign_keys=[provided_identity_id] + ) + fides_user_device_provided_identity = relationship( + ProvidedIdentity, foreign_keys=[fides_user_device_provided_identity_id] + ) diff --git a/src/fides/api/ops/schemas/privacy_preference.py b/src/fides/api/ops/schemas/privacy_preference.py index 3ebce8df6d4..a1d5ff3b1ca 100644 --- a/src/fides/api/ops/schemas/privacy_preference.py +++ b/src/fides/api/ops/schemas/privacy_preference.py @@ -52,7 +52,11 @@ class ConsentReportingSchema(BaseSchema): privacy_request_id: Optional[str] = Field( title="The Privacy Request id created to propagate preferences" ) - user_id: Optional[str] = Field(title="Email if provided") + email: Optional[str] = Field(title="Email if applicable") + phone_number: Optional[str] = Field(title="Phone number if applicable") + fides_user_device_id: Optional[str] = Field( + title="Fides user device id if applicable" + ) secondary_user_ids: Optional[Dict] = Field(title="Other browser identifiers") request_timestamp: datetime = Field( title="Timestamp when Privacy Preference was saved." diff --git a/src/fides/api/ops/schemas/redis_cache.py b/src/fides/api/ops/schemas/redis_cache.py index 5ca17015bbd..199c1bc8208 100644 --- a/src/fides/api/ops/schemas/redis_cache.py +++ b/src/fides/api/ops/schemas/redis_cache.py @@ -26,6 +26,7 @@ class Identity(IdentityBase): email: Optional[EmailStr] = None ga_client_id: Optional[str] = None ljt_readerID: Optional[str] = None + fides_user_device_id: Optional[str] = None class Config: """Only allow phone_number, and email.""" diff --git a/src/fides/api/ops/util/consent_util.py b/src/fides/api/ops/util/consent_util.py index 9e54c5e4b8f..05defb887f5 100644 --- a/src/fides/api/ops/util/consent_util.py +++ b/src/fides/api/ops/util/consent_util.py @@ -1,6 +1,8 @@ from typing import Any, Dict, List, Optional, Tuple +from fastapi import HTTPException from sqlalchemy.orm import Session +from starlette.status import HTTP_422_UNPROCESSABLE_ENTITY from fides.api.ctl.sql_models import System # type: ignore[attr-defined] from fides.api.ops.models.connectionconfig import ConnectionConfig @@ -9,7 +11,13 @@ PrivacyPreferenceHistory, UserConsentPreference, ) -from fides.api.ops.models.privacy_request import ExecutionLogStatus, PrivacyRequest +from fides.api.ops.models.privacy_request import ( + ExecutionLogStatus, + PrivacyRequest, + ProvidedIdentity, + ProvidedIdentityType, +) +from fides.api.ops.schemas.redis_cache import Identity def filter_privacy_preferences_for_propagation( @@ -165,3 +173,60 @@ def add_errored_system_status_for_consent_reporting( connection_config.system_key, ExecutionLogStatus.error, ) + + +def get_fides_user_device_id_provided_identity( + db: Session, fides_user_device_id: Optional[str] +) -> Optional[ProvidedIdentity]: + """Look up a fides user device id that is not attached to a privacy request if it exists + + There can be many fides user device ids attached to privacy requests, but we should try to keep them + unique for consent requests. + """ + if not fides_user_device_id: + return None + + return ProvidedIdentity.filter( + db=db, + conditions=( + (ProvidedIdentity.field_name == ProvidedIdentityType.fides_user_device_id) + & ( + ProvidedIdentity.hashed_value + == ProvidedIdentity.hash_value(fides_user_device_id) + ) + & (ProvidedIdentity.privacy_request_id.is_(None)) + ), + ).first() + + +def get_or_create_fides_user_device_id_provided_identity( + db: Session, + identity_data: Identity, +) -> ProvidedIdentity: + """Gets an existing fides user device id provided identity or creates one if it doesn't exist. + Raises an error if no fides user device id is supplied. + """ + if not identity_data.fides_user_device_id: + raise HTTPException( + HTTP_422_UNPROCESSABLE_ENTITY, + detail="Fides user device id not found in identity data", + ) + + identity = get_fides_user_device_id_provided_identity( + db, identity_data.fides_user_device_id + ) + + if not identity: + identity = ProvidedIdentity.create( + db, + data={ + "privacy_request_id": None, + "field_name": ProvidedIdentityType.fides_user_device_id.value, + "hashed_value": ProvidedIdentity.hash_value( + identity_data.fides_user_device_id + ), + "encrypted_value": {"value": identity_data.fides_user_device_id}, + }, + ) + + return identity # type: ignore[return-value] diff --git a/tests/ctl/core/test_system.py b/tests/ctl/core/test_system.py index cec847fb70a..b766d29925f 100644 --- a/tests/ctl/core/test_system.py +++ b/tests/ctl/core/test_system.py @@ -34,7 +34,6 @@ def delete_server_systems(test_config: FidesConfig, systems: List[System]) -> No def test_get_system_data_uses(db, system) -> None: - assert sql_System.get_data_uses([system]) == {"advertising"} system.privacy_declarations[0].update( diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 98a211eca0f..8a25da79463 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -1453,16 +1453,26 @@ def privacy_notice_us_ca_provide(db: Session) -> Generator: def privacy_preference_history_us_ca_provide( db: Session, privacy_notice_us_ca_provide ) -> Generator: + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("test2@email.com"), + "encrypted_value": {"value": "test2@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + pref_1 = PrivacyPreferenceHistory.create( db=db, data={ "preference": "opt_in", + "provided_identity_id": provided_identity.id, "privacy_notice_history_id": privacy_notice_us_ca_provide.privacy_notice_history_id, }, check_name=False, ) yield pref_1 pref_1.delete(db) + provided_identity.delete(db) @pytest.fixture(scope="function") @@ -1523,16 +1533,26 @@ def privacy_notice_eu_fr_provide_service_frontend_only(db: Session) -> Generator def privacy_preference_history_eu_fr_provide_service_frontend_only( db: Session, privacy_notice_eu_fr_provide_service_frontend_only ) -> Generator: + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("test2@email.com"), + "encrypted_value": {"value": "test2@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + pref_1 = PrivacyPreferenceHistory.create( db=db, data={ "preference": "opt_in", + "provided_identity_id": provided_identity.id, "privacy_notice_history_id": privacy_notice_eu_fr_provide_service_frontend_only.privacy_notice_history_id, }, check_name=False, ) yield pref_1 pref_1.delete(db) + provided_identity.delete(db) @pytest.fixture(scope="function") @@ -1857,6 +1877,20 @@ def provided_identity_and_consent_request(db): consent_request.delete(db=db) +@pytest.fixture(scope="function") +def fides_user_provided_identity(db): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "fides_user_device_id", + "hashed_value": ProvidedIdentity.hash_value("FGHIJ_TEST_FIDES"), + "encrypted_value": {"value": "FGHIJ_TEST_FIDES"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + + yield provided_identity + provided_identity.delete(db=db) + + @pytest.fixture(scope="function") def executable_consent_request( db, diff --git a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py index bc06df08957..e622c389035 100644 --- a/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_consent_request_endpoints.py @@ -24,6 +24,7 @@ ProvidedIdentity, ) from fides.api.ops.schemas.messaging.messaging import MessagingServiceType +from fides.api.ops.util.consent_util import get_fides_user_device_id_provided_identity from fides.core.config import CONFIG diff --git a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py index 13e6f9b6e12..fa80373eed6 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_preference_endpoints.py @@ -15,11 +15,13 @@ from fides.api.ops.api.v1.urn_registry import ( CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY, CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID, - CURRENT_PRIVACY_PREFERENCES, - HISTORICAL_PRIVACY_PREFERENCES, + CURRENT_PRIVACY_PREFERENCES_REPORT, + HISTORICAL_PRIVACY_PREFERENCES_REPORT, + PRIVACY_PREFERENCES, V1_URL_PREFIX, ) from fides.api.ops.models.privacy_preference import ( + CurrentPrivacyPreference, PrivacyPreferenceHistory, UserConsentPreference, ) @@ -30,6 +32,7 @@ ProvidedIdentity, ) from fides.api.ops.schemas.privacy_notice import PrivacyNoticeHistorySchema +from fides.api.ops.schemas.redis_cache import Identity from fides.core.config import CONFIG from tests.conftest import generate_auth_header, generate_role_header_for_user @@ -593,6 +596,111 @@ def test_set_privacy_preferences_without_verification_required( assert not mock_verify_identity.called assert run_privacy_request_mock.called + @pytest.mark.usefixtures( + "subject_identity_verification_required", "automatically_approved" + ) + @mock.patch( + "fides.api.ops.service.privacy_request.request_runner_service.run_privacy_request.delay" + ) + def test_verify_then_set_privacy_preferences_with_additional_fides_user_device_id( + self, + run_privacy_request_mock, + provided_identity_and_consent_request, + api_client, + verification_code, + db: Session, + request_body, + privacy_notice, + system, + ): + """Verify code and then return privacy preferences + + Besides having a verified identity, we also have the fides_user_device_id from the browser + """ + request_body["browser_identity"][ + "fides_user_device_id" + ] = "test_fides_user_device_id" + _, consent_request = provided_identity_and_consent_request + consent_request.cache_identity_verification_code(verification_code) + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY.format(consent_request_id=consent_request.id)}", + json={"code": verification_code}, + ) + assert response.status_code == 200 + # Assert no existing privacy preferences exist for this identity + assert response.json() == {"items": [], "total": 0, "page": 1, "size": 50} + + response = api_client.patch( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PRIVACY_PREFERENCES_WITH_ID.format(consent_request_id=consent_request.id)}", + json=request_body, + ) + assert response.status_code == 200 + assert len(response.json()) == 1 + + response_json = response.json()[0] + created_privacy_preference_history_id = response_json[ + "privacy_preference_history_id" + ] + privacy_preference_history = ( + db.query(PrivacyPreferenceHistory) + .filter( + PrivacyPreferenceHistory.id == created_privacy_preference_history_id + ) + .first() + ) + assert response_json["preference"] == "opt_out" + + assert ( + response_json["privacy_notice_history"] + == PrivacyNoticeHistorySchema.from_orm(privacy_notice.histories[0]).dict() + ) + db.refresh(consent_request) + assert consent_request.privacy_request_id + + response = api_client.post( + f"{V1_URL_PREFIX}{CONSENT_REQUEST_PRIVACY_PREFERENCES_VERIFY.format(consent_request_id=consent_request.id)}", + json={"code": verification_code}, + ) + assert response.status_code == 200 + assert len(response.json()["items"]) == 1 + response_json = response.json()["items"][0] + assert response_json["id"] is not None + assert response_json["preference"] == "opt_out" + assert ( + response_json["privacy_notice_history"] + == PrivacyNoticeHistorySchema.from_orm(privacy_notice.histories[0]).dict() + ) + assert ( + privacy_preference_history.fides_user_device == "test_fides_user_device_id" + ) + assert ( + privacy_preference_history.hashed_fides_user_device + == ProvidedIdentity.hash_value("test_fides_user_device_id") + ) + assert ( + privacy_preference_history.fides_user_device_provided_identity_id + is not None + ) + fides_user_device_provided_identity = ( + privacy_preference_history.fides_user_device_provided_identity + ) + assert ( + fides_user_device_provided_identity + != privacy_preference_history.provided_identity + ) + assert ( + fides_user_device_provided_identity.encrypted_value["value"] + == "test_fides_user_device_id" + ) + assert ( + fides_user_device_provided_identity.hashed_value + == ProvidedIdentity.hash_value("test_fides_user_device_id") + ) + + privacy_preference_history.delete(db=db) + assert run_privacy_request_mock.called + class TestPrivacyPreferenceVerify: @pytest.fixture(scope="function") @@ -742,10 +850,250 @@ def test_consent_verify_consent_preferences( assert consent_request.identity_verified_at is not None +class TestSavePrivacyPreferencesForFidesDeviceId: + @pytest.fixture(scope="function") + def url(self) -> str: + return V1_URL_PREFIX + PRIVACY_PREFERENCES + + @pytest.fixture(scope="function") + def request_body(self, privacy_notice, consent_policy): + return { + "browser_identity": { + "ga_client_id": "test", + "fides_user_device_id": "ABCDE_TEST_FIDES", + }, + "preferences": [ + { + "privacy_notice_history_id": privacy_notice.histories[0].id, + "preference": "opt_out", + } + ], + "policy_key": consent_policy.key, + "request_origin": "privacy_center", + "url_recorded": "example.com/privacy_center", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2)...", + "user_geography": "us_ca", + } + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_save_privacy_preferences_from_outside_domain( + self, api_client, url, request_body + ): + response = api_client.patch( + url, + json=request_body, + headers={"Origin": "https://www.outside_request.com"}, + ) + assert response.status_code == 403 + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_no_fides_user_device_id_supplied(self, api_client, url, request_body): + del request_body["browser_identity"]["fides_user_device_id"] + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 422 + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_save_privacy_preferences_with_bad_notice( + self, api_client, url, request_body + ): + request_body["preferences"][0]["privacy_notice_history_id"] = "bad_history" + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 400 + + def test_save_privacy_preferences_with_respect_to_fides_user_device_id( + self, db, api_client, url, request_body, privacy_notice + ): + """Assert CurrentPrivacyPreference records were updated and PrivacyPreferenceHistory records were created for recordkeeping + with respect to the fides user device id in the request + """ + response = api_client.patch( + url, json=request_body, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 200 + response_json = response.json()[0] + assert response_json["preference"] == "opt_out" + assert ( + response_json["privacy_notice_history"]["id"] + == privacy_notice.histories[0].id + ) + + privacy_preference_history_id = response_json["privacy_preference_history_id"] + + # Fetch current privacy preference that was updated + current_preference = CurrentPrivacyPreference.get( + db, object_id=response_json["id"] + ) + # Get corresponding historical preference that was just created + privacy_preference_history = current_preference.privacy_preference_history + assert privacy_preference_history.id == privacy_preference_history_id + + fides_user_device_provided_identity = ( + privacy_preference_history.fides_user_device_provided_identity + ) + # Same fides user device identity added to both the historical and current record + assert ( + current_preference.fides_user_device_provided_identity + == fides_user_device_provided_identity + ) + assert ( + fides_user_device_provided_identity.hashed_value + == ProvidedIdentity.hash_value("ABCDE_TEST_FIDES") + ) + assert ( + fides_user_device_provided_identity.encrypted_value["value"] + == "ABCDE_TEST_FIDES" + ) + # Values also cached on the historical record for reporting + assert ( + privacy_preference_history.hashed_fides_user_device + == ProvidedIdentity.hash_value("ABCDE_TEST_FIDES") + ) # Cached here for reporting + assert ( + privacy_preference_history.fides_user_device == "ABCDE_TEST_FIDES" + ) # Cached here for reporting + + +class TestGetPrivacyPreferencesForFidesDeviceId: + @pytest.fixture(scope="function") + def url(self) -> str: + return ( + V1_URL_PREFIX + + PRIVACY_PREFERENCES + + "?fides_user_device_id=FGHIJ_TEST_FIDES" + ) + + @pytest.fixture(scope="function") + def preference_data(self, privacy_notice, consent_policy): + return { + "browser_identity": { + "ga_client_id": "test", + "fides_user_device_id": "FGHIJ_TEST_FIDES", + }, + "preferences": [ + { + "privacy_notice_history_id": privacy_notice.histories[0].id, + "preference": "opt_out", + } + ], + "policy_key": consent_policy.key, + "request_origin": "privacy_center", + "url_recorded": "example.com/privacy_center", + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2)...", + "user_geography": "us_ca", + } + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_get_privacy_preferences_from_outside_domain(self, api_client, url): + response = api_client.get( + url, + headers={"Origin": "https://www.outside_request.com"}, + ) + assert response.status_code == 403 + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_no_fides_user_device_id_supplied_as_query_param(self, api_client, url): + response = api_client.get( + V1_URL_PREFIX + PRIVACY_PREFERENCES, + headers={"Origin": "http://localhost:8080"}, + ) + assert response.status_code == 422 + + @pytest.mark.usefixtures( + "privacy_notice", + ) + def test_get_privacy_preferences_no_provided_identity_exists(self, api_client, url): + response = api_client.get(url, headers={"Origin": "http://localhost:8080"}) + assert response.status_code == 403 + + @pytest.mark.usefixtures("privacy_notice", "fides_user_provided_identity") + def test_get_privacy_preferences_with_respect_to_fides_user_device_id_none_saved( + self, api_client, url + ): + response = api_client.get(url, headers={"Origin": "http://localhost:8080"}) + assert response.status_code == 200 + assert response.json() == {"items": [], "total": 0, "page": 1, "size": 50} + + def test_get_privacy_preferences_with_respect_to_fides_user_device_id( + self, + db, + api_client, + url, + preference_data, + privacy_notice, + fides_user_provided_identity, + ): + # Make request to save some preferences first + response = api_client.patch( + url, json=preference_data, headers={"Origin": "http://localhost:8080"} + ) + assert response.status_code == 200 + + response = api_client.get(url, headers={"Origin": "http://localhost:8080"}) + assert response.status_code == 200 + assert response.json()["total"] == 1 + assert response.json()["page"] == 1 + assert response.json()["size"] == 50 + items = response.json()["items"][0] + + assert items["preference"] == "opt_out" + assert items["privacy_notice_history"]["id"] == privacy_notice.histories[0].id + + privacy_preference_history_id = items["privacy_preference_history_id"] + + # Fetch current privacy preference + current_preference = CurrentPrivacyPreference.get(db, object_id=items["id"]) + # Get corresponding historical preference + privacy_preference_history = current_preference.privacy_preference_history + assert privacy_preference_history.id == privacy_preference_history_id + + fides_user_device_provided_identity = ( + privacy_preference_history.fides_user_device_provided_identity + ) + # Same fides user device identity is on both the historical and current record + assert ( + current_preference.fides_user_device_provided_identity + == fides_user_device_provided_identity + == fides_user_provided_identity + ) + assert ( + fides_user_device_provided_identity.hashed_value + == ProvidedIdentity.hash_value("FGHIJ_TEST_FIDES") + ) + assert ( + fides_user_device_provided_identity.encrypted_value["value"] + == "FGHIJ_TEST_FIDES" + ) + # Values also cached on the historical record for reporting + assert ( + privacy_preference_history.hashed_fides_user_device + == ProvidedIdentity.hash_value("FGHIJ_TEST_FIDES") + ) # Cached here for reporting + assert ( + privacy_preference_history.fides_user_device == "FGHIJ_TEST_FIDES" + ) # Cached here for reporting + + current_preference.delete(db) + privacy_preference_history.delete(db) + + class TestHistoricalPreferences: @pytest.fixture(scope="function") def url(self) -> str: - return V1_URL_PREFIX + HISTORICAL_PRIVACY_PREFERENCES + return V1_URL_PREFIX + HISTORICAL_PRIVACY_PREFERENCES_REPORT def test_get_historical_preferences_not_authenticated( self, api_client: TestClient, url @@ -821,7 +1169,9 @@ def test_get_historical_preferences( response_body["privacy_request_id"] == privacy_request_with_consent_policy.id ) - assert response_body["user_id"] == "test@email.com" + assert response_body["email"] == "test@email.com" + assert response_body["phone_number"] is None + assert response_body["fides_user_device_id"] is None assert response_body["secondary_user_ids"] == { "ljt_readerID": "preference_history_test" } @@ -942,7 +1292,7 @@ def test_get_historical_preferences_date_filtering( class TestCurrentPrivacyPreferences: @pytest.fixture(scope="function") def url(self) -> str: - return V1_URL_PREFIX + CURRENT_PRIVACY_PREFERENCES + return V1_URL_PREFIX + CURRENT_PRIVACY_PREFERENCES_REPORT def test_get_current_preferences_not_authenticated( self, api_client: TestClient, url diff --git a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py index e27488adbca..ebc665e0a08 100644 --- a/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/ops/api/v1/endpoints/test_privacy_request_endpoints.py @@ -1317,6 +1317,7 @@ def test_get_privacy_requests_csv_format( "phone_number": TEST_PHONE, "ga_client_id": None, "ljt_readerID": None, + "fides_user_device_id": None, } assert first_row["Request Type"] == "access" assert first_row["Status"] == "approved" diff --git a/tests/ops/models/test_privacy_preference.py b/tests/ops/models/test_privacy_preference.py index eb8ac648036..f43201d3d06 100644 --- a/tests/ops/models/test_privacy_preference.py +++ b/tests/ops/models/test_privacy_preference.py @@ -1,10 +1,13 @@ import pytest -from sqlalchemy.exc import IntegrityError +from sqlalchemy.exc import InvalidRequestError from fides.api.ops.api.v1.endpoints.privacy_preference_endpoints import ( extract_identity_from_provided_identity, ) -from fides.api.ops.common_exceptions import PrivacyNoticeHistoryNotFound +from fides.api.ops.common_exceptions import ( + IdentityNotFoundException, + PrivacyNoticeHistoryNotFound, +) from fides.api.ops.models.privacy_notice import PrivacyNoticeRegion from fides.api.ops.models.privacy_preference import ( PrivacyPreferenceHistory, @@ -20,11 +23,20 @@ class TestPrivacyPreferenceHistory: def test_create_privacy_preference_min_fields(self, db, privacy_notice): + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("test@email.com"), + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + pref = PrivacyPreferenceHistory.create( db=db, data={ "preference": "opt_in", "privacy_notice_history_id": privacy_notice.histories[0].id, + "provided_identity_id": provided_identity.id, }, check_name=False, ) @@ -45,15 +57,40 @@ def test_create_privacy_preference_no_privacy_notice_history(self, db): check_name=False, ) + def test_create_privacy_preference_history_without_identity( + self, db, privacy_notice + ): + with pytest.raises(IdentityNotFoundException): + PrivacyPreferenceHistory.create( + db=db, + data={ + "preference": "opt_in", + "privacy_notice_history_id": privacy_notice.histories[0].id, + }, + check_name=False, + ) + def test_create_privacy_preference( self, db, privacy_notice, system, privacy_request ): provided_identity_data = { "privacy_request_id": None, "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("test@email.com"), "encrypted_value": {"value": "test@email.com"}, } + fides_user_provided_identity_data = { + "privacy_request_id": None, + "field_name": "fides_user_device_id", + "hashed_value": ProvidedIdentity.hash_value( + "test_fides_user_device_id_1234567" + ), + "encrypted_value": {"value": "test_fides_user_device_id_1234567"}, + } provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + fides_user_provided_identity = ProvidedIdentity.create( + db, data=fides_user_provided_identity_data + ) privacy_notice_history = privacy_notice.histories[0] @@ -63,12 +100,21 @@ def test_create_privacy_preference( phone_number, hashed_phone_number = extract_identity_from_provided_identity( provided_identity, ProvidedIdentityType.phone_number ) + ( + fides_user_device_id, + hashed_device_id, + ) = extract_identity_from_provided_identity( + fides_user_provided_identity, ProvidedIdentityType.fides_user_device_id + ) preference_history_record = PrivacyPreferenceHistory.create( db=db, data={ "email": email, + "fides_user_device": fides_user_device_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, "hashed_email": hashed_email, + "hashed_fides_user_device": hashed_device_id, "hashed_phone_number": hashed_phone_number, "phone_number": phone_number, "preference": "opt_out", @@ -84,13 +130,35 @@ def test_create_privacy_preference( ) assert preference_history_record.affected_system_status == {} assert preference_history_record.email == "test@email.com" - assert preference_history_record.hashed_email == provided_identity.hashed_value + assert ( + preference_history_record.hashed_email + == provided_identity.hashed_value + is not None + ) + assert ( + preference_history_record.fides_user_device + == fides_user_device_id + is not None + ) + assert ( + preference_history_record.hashed_fides_user_device + == hashed_device_id + is not None + ) + assert ( + preference_history_record.fides_user_device_provided_identity + == fides_user_provided_identity + ) - assert preference_history_record.phone_number is None + assert preference_history_record.email == "test@email.com" assert ( - preference_history_record.hashed_phone_number + preference_history_record.hashed_email == provided_identity.hashed_value + is not None ) + + assert preference_history_record.phone_number is None + assert preference_history_record.hashed_phone_number is None assert preference_history_record.preference == UserConsentPreference.opt_out assert ( preference_history_record.privacy_notice_history == privacy_notice_history @@ -219,3 +287,185 @@ def test_update_secondary_user_ids(self, privacy_preference_history, db): "email": "hello@example.com", "ljt_readerID": "customer-123", } + + def test_consolidate_current_privacy_preferences(self, db, privacy_notice): + """We might have privacy preferences saved just under a fides user device id in an overlay, + and then later, have privacy preferences saved both under an email and that same fides user device id + + We should consider these preferences as being for the same individual, and consolidate + them for our "current privacy preferences" + """ + + # Let's first just save a privacy preference under a fides user device id + fides_user_provided_identity_data = { + "privacy_request_id": None, + "field_name": "fides_user_device_id", + "hashed_value": ProvidedIdentity.hash_value( + "test_fides_user_device_id_1234567" + ), + "encrypted_value": {"value": "test_fides_user_device_id_1234567"}, + } + fides_user_provided_identity = ProvidedIdentity.create( + db, data=fides_user_provided_identity_data + ) + + privacy_notice_history = privacy_notice.histories[0] + ( + fides_user_device_id, + hashed_device_id, + ) = extract_identity_from_provided_identity( + fides_user_provided_identity, ProvidedIdentityType.fides_user_device_id + ) + + preference_history_record_for_device = PrivacyPreferenceHistory.create( + db=db, + data={ + "email": None, + "fides_user_device": fides_user_device_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, + "hashed_email": None, + "hashed_fides_user_device": hashed_device_id, + "hashed_phone_number": None, + "phone_number": None, + "preference": "opt_out", + "privacy_notice_history_id": privacy_notice_history.id, + "provided_identity_id": None, + "request_origin": "privacy_center", + "secondary_user_ids": {"ga_client_id": "test"}, + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24", + "user_geography": "us_ca", + "url_recorded": "example.com/privacy_center", + }, + check_name=False, + ) + + # Assert a CurrentPrivacyPreference record was created when the PrivacyPreferenceHistory was created + fides_user_device_current_preference = ( + preference_history_record_for_device.current_privacy_preference + ) + assert fides_user_device_current_preference.created_at is not None + assert fides_user_device_current_preference.updated_at is not None + assert ( + fides_user_device_current_preference.preference + == UserConsentPreference.opt_out + ) + assert fides_user_device_current_preference.provided_identity_id is None + assert ( + fides_user_device_current_preference.fides_user_device_provided_identity_id + == fides_user_provided_identity.id + ) + assert ( + fides_user_device_current_preference.privacy_notice_id == privacy_notice.id + ) + assert ( + fides_user_device_current_preference.privacy_notice_history_id + == privacy_notice_history.id + ) + assert ( + fides_user_device_current_preference.privacy_preference_history_id + == preference_history_record_for_device.id + ) + + provided_identity_data = { + "privacy_request_id": None, + "field_name": "email", + "hashed_value": ProvidedIdentity.hash_value("test@email.com"), + "encrypted_value": {"value": "test@email.com"}, + } + provided_identity = ProvidedIdentity.create(db, data=provided_identity_data) + email, hashed_email = extract_identity_from_provided_identity( + provided_identity, ProvidedIdentityType.email + ) + + preference_history_record_for_email = PrivacyPreferenceHistory.create( + db=db, + data={ + "email": email, + "fides_user_device": None, + "fides_user_device_provided_identity_id": None, + "hashed_email": hashed_email, + "hashed_fides_user_device": None, + "hashed_phone_number": None, + "phone_number": None, + "preference": "opt_in", + "privacy_notice_history_id": privacy_notice_history.id, + "provided_identity_id": provided_identity.id, + "request_origin": "privacy_center", + "secondary_user_ids": {"ga_client_id": "test"}, + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24", + "user_geography": "us_ca", + "url_recorded": "example.com/privacy_center", + }, + check_name=False, + ) + + # Assert a new CurrentPrivacyPreference record was created when the PrivacyPreferenceHistory was created with email + email_current_preference = ( + preference_history_record_for_email.current_privacy_preference + ) + assert email_current_preference.created_at is not None + assert email_current_preference.updated_at is not None + assert email_current_preference.preference == UserConsentPreference.opt_in + assert email_current_preference.provided_identity_id == provided_identity.id + assert email_current_preference.fides_user_device_provided_identity_id is None + assert email_current_preference.privacy_notice_id == privacy_notice.id + assert ( + email_current_preference.privacy_notice_history_id + == privacy_notice_history.id + ) + assert ( + email_current_preference.privacy_preference_history_id + == preference_history_record_for_email.id + ) + + # Now user saves a preference from the privacy center with verified email that also has their device id + preference_history_saved_with_both_email_and_device_id = PrivacyPreferenceHistory.create( + db=db, + data={ + "email": email, + "fides_user_device": fides_user_device_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, + "hashed_email": hashed_email, + "hashed_fides_user_device": hashed_device_id, + "hashed_phone_number": None, + "phone_number": None, + "preference": "opt_in", + "privacy_notice_history_id": privacy_notice_history.id, + "provided_identity_id": provided_identity.id, + "request_origin": "privacy_center", + "secondary_user_ids": {"ga_client_id": "test"}, + "user_agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_14_2) AppleWebKit/324.42 (KHTML, like Gecko) Chrome/108.0.0.0 Safari/425.24", + "user_geography": "us_ca", + "url_recorded": "example.com/privacy_center", + }, + check_name=False, + ) + + # Assert existing CurrentPrivacyPreference record was updated when the PrivacyPreferenceHistory was created + # and consolidated preferences for the email and the user device. The preferences for device only was deleted + current_preference = ( + preference_history_saved_with_both_email_and_device_id.current_privacy_preference + ) + assert current_preference.created_at is not None + assert current_preference.updated_at is not None + assert current_preference.preference == UserConsentPreference.opt_in + assert current_preference.provided_identity_id == provided_identity.id + assert ( + current_preference.fides_user_device_provided_identity_id + == fides_user_provided_identity.id + ) + assert current_preference.privacy_notice_id == privacy_notice.id + assert current_preference.privacy_notice_history_id == privacy_notice_history.id + assert ( + current_preference.privacy_preference_history_id + == preference_history_saved_with_both_email_and_device_id.id + ) + + assert ( + current_preference + == email_current_preference + != fides_user_device_current_preference + ) + with pytest.raises(InvalidRequestError): + # Can't refresh because this preference has been deleted, and consolidated with the other + db.refresh(fides_user_device_current_preference) diff --git a/tests/ops/util/test_consent_util.py b/tests/ops/util/test_consent_util.py index 9e9d2d4ee28..bbacebbb7a5 100644 --- a/tests/ops/util/test_consent_util.py +++ b/tests/ops/util/test_consent_util.py @@ -6,6 +6,7 @@ add_complete_system_status_for_consent_reporting, add_errored_system_status_for_consent_reporting, cache_initial_status_and_identities_for_consent_reporting, + get_fides_user_device_id_provided_identity, should_opt_in_to_service, ) @@ -23,6 +24,7 @@ def test_matching_data_use( system, privacy_request_with_consent_policy, privacy_notice, + fides_user_provided_identity, ): """ Privacy Notice Enforcement Level = "system_wide" @@ -34,6 +36,7 @@ def test_matching_data_use( data={ "preference": preference, "privacy_notice_history_id": privacy_notice.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -44,6 +47,8 @@ def test_matching_data_use( ) assert collapsed_opt_in_preference == should_opt_in + pref.delete(db) + @pytest.mark.parametrize( "preference, should_opt_in", [("opt_in", True), ("opt_out", False), ("acknowledge", None)], @@ -56,6 +61,7 @@ def test_notice_use_is_parent_of_system_use( system, privacy_notice_us_ca_provide, privacy_request_with_consent_policy, + fides_user_provided_identity, ): """ Privacy Notice Enforcement Level = "system_wide" @@ -76,6 +82,7 @@ def test_notice_use_is_parent_of_system_use( data={ "preference": preference, "privacy_notice_history_id": privacy_notice_us_ca_provide.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -85,6 +92,7 @@ def test_notice_use_is_parent_of_system_use( system, privacy_request_with_consent_policy ) assert collapsed_opt_in_preference == should_opt_in + pref.delete(db) @pytest.mark.parametrize( "preference, should_opt_in", @@ -98,6 +106,7 @@ def test_notice_use_is_child_of_system_use( system, privacy_notice_us_co_provide_service_operations, privacy_request_with_consent_policy, + fides_user_provided_identity, ): """ Privacy Notice Enforcement Level = "system_wide" @@ -115,6 +124,7 @@ def test_notice_use_is_child_of_system_use( data={ "preference": preference, "privacy_notice_history_id": privacy_notice_us_co_provide_service_operations.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -124,6 +134,7 @@ def test_notice_use_is_child_of_system_use( system, privacy_request_with_consent_policy ) assert collapsed_opt_in_preference == should_opt_in + pref.delete(db) @pytest.mark.parametrize( "preference, should_opt_in", @@ -137,6 +148,7 @@ def test_enforcement_frontend_only( system, privacy_request_with_consent_policy, privacy_notice_eu_fr_provide_service_frontend_only, + fides_user_provided_identity, ): """ Privacy Notice Enforcement Level = "frontend" @@ -148,6 +160,7 @@ def test_enforcement_frontend_only( data={ "preference": preference, "privacy_notice_history_id": privacy_notice_eu_fr_provide_service_frontend_only.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -157,6 +170,7 @@ def test_enforcement_frontend_only( system, privacy_request_with_consent_policy ) assert collapsed_opt_in_preference == should_opt_in + pref.delete(db) @pytest.mark.parametrize( "preference, should_opt_in", @@ -169,6 +183,7 @@ def test_no_system_means_no_data_use_check( db, privacy_notice_us_co_provide_service_operations, privacy_request_with_consent_policy, + fides_user_provided_identity, ): """ Privacy Notice Enforcement Level = "system_wide" @@ -180,6 +195,7 @@ def test_no_system_means_no_data_use_check( data={ "preference": preference, "privacy_notice_history_id": privacy_notice_us_co_provide_service_operations.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -189,12 +205,15 @@ def test_no_system_means_no_data_use_check( None, privacy_request_with_consent_policy ) assert collapsed_opt_in_preference == should_opt_in + pref.delete(db) def test_conflict_preferences_opt_out_wins( self, db, privacy_request_with_consent_policy, privacy_notice, + privacy_notice_us_ca_provide, + fides_user_provided_identity, ): """ Privacy Notice Enforcement Level = "system_wide" @@ -206,6 +225,7 @@ def test_conflict_preferences_opt_out_wins( data={ "preference": "opt_in", "privacy_notice_history_id": privacy_notice.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -213,7 +233,8 @@ def test_conflict_preferences_opt_out_wins( db=db, data={ "preference": "opt_out", - "privacy_notice_history_id": privacy_notice.privacy_notice_history_id, + "privacy_notice_history_id": privacy_notice_us_ca_provide.privacy_notice_history_id, + "fides_user_device_provided_identity_id": fides_user_provided_identity.id, }, check_name=False, ) @@ -227,6 +248,8 @@ def test_conflict_preferences_opt_out_wins( ) assert collapsed_opt_in_preference is False assert filtered_preferences == [pref_2] + pref_1.delete(db) + pref_2.delete(db) def test_old_workflow_preferences_saved_with_respect_to_data_use( self, @@ -414,3 +437,23 @@ def test_add_error_system_status_for_consent_reporting( connection_config.name: "skipped" } assert privacy_preference_history.secondary_user_ids is None + + +class TestGetFidesUserProvidedIdentity: + def test_no_identifier_supplied(self, db): + provided_identity = get_fides_user_device_id_provided_identity(db, None) + assert provided_identity is None + + def test_no_provided_identifier_exists(self, db): + provided_identity = get_fides_user_device_id_provided_identity( + db, "fides_user_device_id" + ) + assert provided_identity is None + + def test_get_fides_user_device_id_provided_identity( + self, db, fides_user_provided_identity + ): + provided_identity = get_fides_user_device_id_provided_identity( + db, "FGHIJ_TEST_FIDES" + ) + assert provided_identity == fides_user_provided_identity