Fidesops -> Unified Fides final merge

CHANGELOG.md

@@ -25,16 +25,18 @@ The types of changes are:
 * Handle malformed tokens [#1523](https://github.com/ethyca/fides/pull/1523)
 * Remove thrown exception from getAllPrivacyRequests method [#1592](https://github.com/ethyca/fides/pull/1593)
 * Include systems without a privacy declaration on data map [#1603](https://github.com/ethyca/fides/pull/1603)
-
+* After editing a dataset, the table will stay on the previously selected collection instead of resetting to the first one. [#1511](https://github.com/ethyca/fides/pull/1511)
+* Fix redis `db_index` config issue [#1647](https://github.com/ethyca/fides/pull/1647)
 
 ### Docs
 
 * Add unlinked docs and fix any remaining broken links [#1266](https://github.com/ethyca/fides/pull/1266)
 * Update privacy center docs to include consent information [#1537](https://github.com/ethyca/fides/pull/1537)
 
-### Fixed
+### Changed 
+
+* Allow multiple masking strategies to be specified when using fides as a masking engine [#1647](https://github.com/ethyca/fides/pull/1647)
 
-* After editing a dataset, the table will stay on the previously selected collection instead of resetting to the first one. [#1511](https://github.com/ethyca/fides/pull/1511)
 
 ## [1.9.5](https://github.com/ethyca/fides/compare/1.9.4...1.9.5)
 

src/fides/api/main.py

@@ -90,7 +90,6 @@ async def dispatch_log_request(request: Request, call_next: Callable) -> Respons
     if (not path.startswith(API_PREFIX)) or (path.endswith("/health")):
         return await call_next(request)
 
-
     fides_source: Optional[str] = request.headers.get("X-Fides-Source")
     now: datetime = datetime.now(tz=timezone.utc)
     endpoint = f"{request.method}: {request.url}"

src/fides/api/ops/api/v1/endpoints/masking_endpoints.py

@@ -1,5 +1,5 @@
 import logging
-from typing import List
+from typing import Any, List
 
 from fastapi import HTTPException
 from starlette.status import HTTP_400_BAD_REQUEST, HTTP_404_NOT_FOUND
@@ -13,6 +13,7 @@
 from fides.api.ops.schemas.masking.masking_strategy_description import (
     MaskingStrategyDescription,
 )
+from fides.api.ops.schemas.policy import PolicyMaskingSpec
 from fides.api.ops.service.masking.strategy.masking_strategy import MaskingStrategy
 from fides.api.ops.util.api_router import APIRouter
 
@@ -25,17 +26,31 @@
 def mask_value(request: MaskingAPIRequest) -> MaskingAPIResponse:
     """Masks the value(s) provided using the provided masking strategy"""
     try:
-        values = request.values
-        masking_strategy = request.masking_strategy
-        strategy = MaskingStrategy.get_strategy(
-            masking_strategy.strategy, masking_strategy.configuration
-        )
-        logger.info(
-            "Starting masking of %s value(s) with strategy %s",
-            len(values),
-            masking_strategy.strategy,
-        )
-        masked_values = strategy.mask(values, None)
+        values: List[Any] = request.values
+        masked_values: List[Any] = request.values.copy()
+        masking_strategies: List[PolicyMaskingSpec] = request.masking_strategies
+
+        num_strat: int = len(masking_strategies)
+
+        if num_strat > 1:
+            logger.info(
+                "%s masking strategies requested; running in order.",
+                num_strat,
+            )
+
+        for strategy in masking_strategies:
+            masking_strategy = MaskingStrategy.get_strategy(
+                strategy.strategy, strategy.configuration
+            )
+            logger.info(
+                "Starting masking of %s value(s) with strategy %s",
+                len(values),
+                strategy.strategy,
+            )
+            masked_values = masking_strategy.mask(  # type: ignore
+                masked_values, None
+            )  # passing in masked values from previous strategy
+
         return MaskingAPIResponse(plain=values, masked_values=masked_values)
     except NoSuchStrategyException as e:
         raise HTTPException(status_code=HTTP_404_NOT_FOUND, detail=str(e))

src/fides/api/ops/schemas/masking/masking_api.py

@@ -1,6 +1,6 @@
-from typing import Any, List
+from typing import Any, Dict, List, Union
 
-from pydantic import BaseModel
+from pydantic import BaseModel, root_validator
 
 from fides.api.ops.schemas.policy import PolicyMaskingSpec
 
@@ -9,7 +9,25 @@ class MaskingAPIRequest(BaseModel):
     """The API Request for masking operations"""
 
     values: List[str]
-    masking_strategy: PolicyMaskingSpec
+    masking_strategy: Union[PolicyMaskingSpec, List[PolicyMaskingSpec]]
+    masking_strategies: List[PolicyMaskingSpec] = []
+
+    @root_validator(pre=False)
+    def build_masking_strategies(cls, values: Dict) -> Dict:
+        """
+        Update "masking_strategies" field by inspecting "masking_strategy".
+        The ability to specify one or more masking strategies was added in a
+        backwards-compatible way.  Pass in a single masking strategy, or a list
+        of masking_strategy objects under "masking_strategy", and we
+        set the "masking_strategies" value from there.
+        Masking_strategies should not be supplied directly.
+        """
+        strategy = values.get("masking_strategy")
+        values["masking_strategies"] = (
+            strategy if isinstance(strategy, list) else [strategy]
+        )
+
+        return values
 
 
 class MaskingAPIResponse(BaseModel):

src/fides/api/ops/schemas/policy.py

@@ -1,19 +1,18 @@
-from typing import Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional
 
 from fides.api.ops.models.policy import ActionType, DrpAction
 from fides.api.ops.schemas.api import BulkResponse, BulkUpdateFailed
 from fides.api.ops.schemas.base_class import BaseSchema
-from fides.api.ops.schemas.masking.masking_configuration import FormatPreservationConfig
 from fides.api.ops.schemas.shared_schemas import FidesOpsKey
 from fides.api.ops.schemas.storage.storage import StorageDestinationResponse
 from fides.api.ops.util.data_category import DataCategory
 
 
 class PolicyMaskingSpec(BaseSchema):
-    """Models the masking strategy definition int the policy document"""
+    """Models the masking strategy definition"""
 
     strategy: str
-    configuration: Dict[str, Union[str, FormatPreservationConfig]]
+    configuration: Dict[str, Any]
 
 
 class PolicyMaskingSpecResponse(BaseSchema):

src/fides/cli/commands/util.py

@@ -148,7 +148,9 @@ def up(ctx: click.Context, no_pull: bool = False, no_init: bool = False) -> None
             config_path = create_config_file(ctx=ctx, fides_directory_location=".")
             check_and_update_analytics_config(ctx=ctx, config_path=config_path)
         else:
-            echo_green("Deployment successful! Skipping CLI initialization (run 'fides init' to initialize)")
+            echo_green(
+                "Deployment successful! Skipping CLI initialization (run 'fides init' to initialize)"
+            )
         print_divider()
 
         print_deploy_success()

src/fides/ctl/core/config/redis_settings.py

@@ -37,7 +37,8 @@ def assemble_connection_url(
             # If the whole URL is provided via the config, preference that
             return v
 
-        return f"redis://{quote_plus(values.get('user', ''))}:{quote_plus(values['password'])}@{values['host']}:{values['port']}/{values.get('db_index', '')}"
+        db_index = values.get("db_index") if values.get("db_index") is not None else ""
+        return f"redis://{quote_plus(values.get('user', ''))}:{quote_plus(values.get('password', ''))}@{values.get('host', '')}:{values.get('port', '')}/{db_index}"
 
     class Config:
         env_prefix = ENV_PREFIX

tests/ops/api/v1/endpoints/test_masking_endpoints.py

@@ -43,6 +43,27 @@ def test_read_strategies(self, api_client: TestClient):
 
 
 class TestMaskValues:
+    def test_no_strategies_specified(self, api_client: TestClient):
+        value = "my_email"
+        request = {
+            "values": [value],
+        }
+
+        response = api_client.put(f"{V1_URL_PREFIX}{MASKING}", json=request)
+        assert 422 == response.status_code
+
+    def test_mask_nothing(self, api_client: TestClient):
+        request = {
+            "values": None,
+            "masking_strategy": {
+                "strategy": StringRewriteMaskingStrategy.name,
+                "configuration": {"rewrite_value": "MASKED"},
+            },
+        }
+
+        response = api_client.put(f"{V1_URL_PREFIX}{MASKING}", json=request)
+        assert 422 == response.status_code
+
     def test_mask_value_string_rewrite(self, api_client: TestClient):
         value = "check"
         rewrite_val = "mate"
@@ -210,3 +231,62 @@ def test_masking_value_null(self, api_client: TestClient):
         json_response = json.loads(response.text)
         assert value == json_response["plain"][0]
         assert json_response["masked_values"][0] is None
+
+    def test_masking_values_multiple_strategies(self, api_client: TestClient):
+        value = "check"
+        rewrite_val = "mate"
+        request = {
+            "values": [value],
+            "masking_strategy": [
+                {
+                    "strategy": StringRewriteMaskingStrategy.name,
+                    "configuration": {"rewrite_value": rewrite_val},
+                },
+                {
+                    "strategy": HashMaskingStrategy.name,
+                    "configuration": {},
+                },
+            ],
+        }
+
+        response = api_client.put(f"{V1_URL_PREFIX}{MASKING}", json=request)
+        assert 200 == response.status_code
+        assert response.json()["plain"] == ["check"]
+        assert response.json()["masked_values"] != [
+            rewrite_val
+        ], "Final value is hashed, because that was the last strategy"
+
+        switch_order = {
+            "values": [value],
+            "masking_strategy": [
+                {
+                    "strategy": HashMaskingStrategy.name,
+                    "configuration": {},
+                },
+                {
+                    "strategy": StringRewriteMaskingStrategy.name,
+                    "configuration": {"rewrite_value": rewrite_val},
+                },
+            ],
+        }
+        response = api_client.put(f"{V1_URL_PREFIX}{MASKING}", json=switch_order)
+        assert 200 == response.status_code
+        assert response.json()["plain"] == ["check"]
+        assert response.json()["masked_values"] == [
+            rewrite_val
+        ], "Final value is rewrite value, because that was the last strategy specified"
+
+    def test_flexible_config(self, api_client: TestClient):
+        """Test that this request is allowed.  Allow the configuration to be
+        very flexible so different configuration requirements by many masking strategies are supported"""
+        value = "my_email"
+        request = {
+            "values": [value],
+            "masking_strategy": {
+                "strategy": NullMaskingStrategy.name,
+                "configuration": {"test_val": {"test": [["test"]]}},
+            },
+        }
+
+        response = api_client.put(f"{V1_URL_PREFIX}{MASKING}", json=request)
+        assert 200 == response.status_code