From 04fb26dfc420d31bae89ed831282eab491ab90de Mon Sep 17 00:00:00 2001 From: Catherine Smith Date: Tue, 10 May 2022 13:56:48 -0400 Subject: [PATCH] Extracts privacy request endpoint logic into separate service for DRP (#470) * extracts privacy request endpoint logic into separate service for DRP * lint * unused import * enum formatting * return types * more lint * import DrpAction enum instead of using str * remove db call abstraction * remove import --- .../v1/endpoints/privacy_request_endpoints.py | 57 ++++++----------- src/fidesops/models/privacy_request.py | 13 ++++ src/fidesops/schemas/drp_privacy_request.py | 33 ++++++++++ .../privacy_request/request_service.py | 61 +++++++++++++++++++ src/fidesops/util/cache.py | 7 +++ 5 files changed, 131 insertions(+), 40 deletions(-) create mode 100644 src/fidesops/schemas/drp_privacy_request.py create mode 100644 src/fidesops/service/privacy_request/request_service.py diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index 4ab98c3bc..efe662a10 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -39,7 +39,7 @@ from fidesops.models.client import ClientDetail from fidesops.models.connectionconfig import ConnectionConfig from fidesops.models.datasetconfig import DatasetConfig -from fidesops.models.policy import ActionType, Policy, PolicyPreWebhook +from fidesops.models.policy import Policy, PolicyPreWebhook from fidesops.models.privacy_request import ( ExecutionLog, PrivacyRequest, @@ -47,9 +47,6 @@ ) from fidesops.schemas.dataset import CollectionAddressResponse, DryRunDatasetResponse from fidesops.schemas.external_https import PrivacyRequestResumeFormat -from fidesops.schemas.masking.masking_configuration import MaskingConfiguration -from fidesops.schemas.masking.masking_secrets import MaskingSecretCache -from fidesops.schemas.policy import Rule from fidesops.schemas.privacy_request import ( BulkPostPrivacyRequests, BulkReviewResponse, @@ -60,8 +57,11 @@ ReviewPrivacyRequestIds, DenyPrivacyRequests, ) -from fidesops.service.masking.strategy.masking_strategy_factory import get_strategy from fidesops.service.privacy_request.request_runner_service import PrivacyRequestRunner +from fidesops.service.privacy_request.request_service import ( + build_required_privacy_request_kwargs, + cache_data, +) from fidesops.task.graph_task import EMPTY_REQUEST, collect_queries from fidesops.task.task_resources import TaskResources from fidesops.util.cache import FidesopsRedis @@ -127,7 +127,7 @@ def create_privacy_request( continue logger.info(f"Finding policy with key '{privacy_request_data.policy_key}'") - policy = Policy.get_by( + policy: Optional[Policy] = Policy.get_by( db=db, field="key", value=privacy_request_data.policy_key, @@ -144,47 +144,24 @@ def create_privacy_request( failed.append(failure) continue - kwargs = { - "requested_at": privacy_request_data.requested_at, - "policy_id": policy.id, - "status": "pending", - } + kwargs = build_required_privacy_request_kwargs( + privacy_request_data.requested_at, policy.id + ) for field in optional_fields: attr = getattr(privacy_request_data, field) if attr is not None: kwargs[field] = attr try: - privacy_request = PrivacyRequest.create(db=db, data=kwargs) - - # Store identity in the cache - logger.info(f"Caching identity for privacy request {privacy_request.id}") - privacy_request.cache_identity(privacy_request_data.identity) - privacy_request.cache_encryption(privacy_request_data.encryption_key) - - # Store masking secrets in the cache - logger.info( - f"Caching masking secrets for privacy request {privacy_request.id}" - ) - erasure_rules: List[Rule] = policy.get_rules_for_action( - action_type=ActionType.erasure + privacy_request: PrivacyRequest = PrivacyRequest.create(db=db, data=kwargs) + + cache_data( + privacy_request, + policy, + privacy_request_data.identity, + privacy_request_data.encryption_key, + None, ) - unique_masking_strategies_by_name: Set[str] = set() - for rule in erasure_rules: - strategy_name: str = rule.masking_strategy["strategy"] - configuration: MaskingConfiguration = rule.masking_strategy[ - "configuration" - ] - if strategy_name in unique_masking_strategies_by_name: - continue - unique_masking_strategies_by_name.add(strategy_name) - masking_strategy = get_strategy(strategy_name, configuration) - if masking_strategy.secrets_required(): - masking_secrets: List[ - MaskingSecretCache - ] = masking_strategy.generate_secrets_for_cache() - for masking_secret in masking_secrets: - privacy_request.cache_masking_secret(masking_secret) if not config.execution.REQUIRE_MANUAL_REQUEST_APPROVAL: PrivacyRequestRunner( diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index 60730dba3..4f21a0074 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -35,6 +35,7 @@ WebhookDirection, WebhookTypes, ) +from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate from fidesops.schemas.external_https import ( SecondPartyRequestFormat, SecondPartyResponseFormat, @@ -49,6 +50,7 @@ FidesopsRedis, get_encryption_cache_key, get_masking_secret_cache_key, + get_drp_request_body_cache_key, ) from fidesops.util.oauth_util import generate_jwe @@ -178,6 +180,17 @@ def cache_identity(self, identity: PrivacyRequestIdentity) -> None: value, ) + def cache_drp_request_body(self, drp_request_body: DrpPrivacyRequestCreate) -> None: + """Sets the identity's values at their specific locations in the Fidesops app cache""" + cache: FidesopsRedis = get_cache() + drp_request_body_dict: Dict[str, Any] = dict(drp_request_body) + for key, value in drp_request_body_dict.items(): + if value is not None: + cache.set_with_autoexpire( + get_drp_request_body_cache_key(self.id, key), + value, + ) + def cache_encryption(self, encryption_key: Optional[str] = None) -> None: """Sets the encryption key in the Fidesops app cache if provided""" if not encryption_key: diff --git a/src/fidesops/schemas/drp_privacy_request.py b/src/fidesops/schemas/drp_privacy_request.py new file mode 100644 index 000000000..784df605a --- /dev/null +++ b/src/fidesops/schemas/drp_privacy_request.py @@ -0,0 +1,33 @@ +from enum import Enum +from typing import Optional, List + +from fidesops.models.policy import DrpAction +from fidesops.schemas.base_class import BaseSchema + + +class DrpMeta(BaseSchema): + """Enum to hold Drp metadata. Only version is supported at this time""" + + version: str + + +class DrpRegime(Enum): + """Enum to hold Drp Regime. Only ccpa supported at this time""" + + ccpa = "ccpa" + + +class DrpPrivacyRequestCreate(BaseSchema): + """Data required to create a DRP PrivacyRequest""" + + meta: DrpMeta + regime: Optional[DrpRegime] + exercise: DrpAction + relationships: Optional[List[str]] + identity: str + status_callback: Optional[str] + + class Config: + """Populate models with the raw value of enum fields, rather than the enum itself""" + + use_enum_values = True diff --git a/src/fidesops/service/privacy_request/request_service.py b/src/fidesops/service/privacy_request/request_service.py new file mode 100644 index 000000000..e8d9743b0 --- /dev/null +++ b/src/fidesops/service/privacy_request/request_service.py @@ -0,0 +1,61 @@ +import logging +from datetime import datetime +from typing import Optional, Any, Dict, Set, List + +from fidesops.models.policy import Policy, ActionType +from fidesops.models.privacy_request import PrivacyRequest +from fidesops.schemas.drp_privacy_request import DrpPrivacyRequestCreate +from fidesops.schemas.masking.masking_configuration import MaskingConfiguration +from fidesops.schemas.masking.masking_secrets import MaskingSecretCache +from fidesops.schemas.policy import Rule +from fidesops.schemas.redis_cache import PrivacyRequestIdentity +from fidesops.service.masking.strategy.masking_strategy_factory import get_strategy + +logger = logging.getLogger(__name__) + + +def build_required_privacy_request_kwargs( + requested_at: Optional[datetime], policy_id: str +) -> Dict[str, Any]: + """Build kwargs required for creating privacy request""" + return { + "requested_at": requested_at, + "policy_id": policy_id, + "status": "pending", + } + + +def cache_data( + privacy_request: PrivacyRequest, + policy: Policy, + identity: PrivacyRequestIdentity, + encryption_key: Optional[str], + drp_request_body: Optional[DrpPrivacyRequestCreate], +) -> None: + """Cache privacy request data""" + # Store identity and encryption key in the cache + logger.info(f"Caching identity for privacy request {privacy_request.id}") + privacy_request.cache_identity(identity) + privacy_request.cache_encryption(encryption_key) # handles None already + + # Store masking secrets in the cache + logger.info(f"Caching masking secrets for privacy request {privacy_request.id}") + erasure_rules: List[Rule] = policy.get_rules_for_action( + action_type=ActionType.erasure + ) + unique_masking_strategies_by_name: Set[str] = set() + for rule in erasure_rules: + strategy_name: str = rule.masking_strategy["strategy"] + configuration: MaskingConfiguration = rule.masking_strategy["configuration"] + if strategy_name in unique_masking_strategies_by_name: + continue + unique_masking_strategies_by_name.add(strategy_name) + masking_strategy = get_strategy(strategy_name, configuration) + if masking_strategy.secrets_required(): + masking_secrets: List[ + MaskingSecretCache + ] = masking_strategy.generate_secrets_for_cache() + for masking_secret in masking_secrets: + privacy_request.cache_masking_secret(masking_secret) + if drp_request_body: + privacy_request.cache_drp_request_body(drp_request_body) diff --git a/src/fidesops/util/cache.py b/src/fidesops/util/cache.py index c1e30a6d0..53c607e2e 100644 --- a/src/fidesops/util/cache.py +++ b/src/fidesops/util/cache.py @@ -125,6 +125,13 @@ def get_identity_cache_key(privacy_request_id: str, identity_attribute: str) -> return f"id-{privacy_request_id}-identity-{identity_attribute}" +def get_drp_request_body_cache_key( + privacy_request_id: str, identity_attribute: str +) -> str: + """Return the key at which to save this PrivacyRequest's drp request body for the passed in attribute""" + return f"id-{privacy_request_id}-drp-{identity_attribute}" + + def get_encryption_cache_key(privacy_request_id: str, encryption_attr: str) -> str: """Return the key at which to save this PrivacyRequest's encryption attribute""" return f"id-{privacy_request_id}-encryption-{encryption_attr}"