Compare serialized data in assertion compr & improve diff report (#46)

* chore: ignore vscode settings

* test: remove non-deterministic test for unhashable set

* feat: improve assertion diff #22

fix: compare serialized snapshots, close #21

* wip: define serialize as abstract method

* chore: print full diff in verbose mode

* chore: don't collect coverage from abstract methods

* cr: simplify read raw, success style, toml

* wip: add back requirements.txt

* test: fix coverage reporting (#48)

* chore: do not report on implicitly covered lines

* wip: more explicit

* wip: all is not what it seems

* wip: run coverage without pytest-cov

* wip: fix coverage source config

* wip: fix coverage source config

Co-authored-by: Emmanuel Ogbizi <iamogbz+github@gmail.com>

.gitignore

@@ -34,6 +34,7 @@ pip-delete-this-directory.txt
 # Unit test / coverage reports
 .pytest_cache/
 .mypy_cache/
+.coverage
 
 # Environments
 .env
@@ -44,3 +45,4 @@ venv/
 ENV/
 env.bak/
 venv.bak/
+.vscode

dev-requirements.in

@@ -9,6 +9,6 @@ semver
 twine
 wheel
 py-githooks
-pytest-cov
 codecov
 isort
+coverage[toml]

dev-requirements.txt

@@ -4,50 +4,45 @@
 #
 #    pip-compile dev-requirements.in
 #
---extra-index-url https://pip-readonly:eyJ2ZXIiOiIyIiwidHlwIjoiSldUIiwiYWxnIjoiUlMyNTYiLCJraWQiOiJYaFp6N3l3eV8zSGdEdDFFa3BNTTY4bnYzTGNRd0syb1p6SFI0TjY3cXdnIn0.eyJzdWIiOiJqZnJ0QDAxYnQzbnhtNTltNGVoMGd4NHBwZjcxZzBxXC91c2Vyc1wvcGlwLXJlYWRvbmx5Iiwic2NwIjoibWVtYmVyLW9mLWdyb3Vwczpwcml2YXRlLXJlYWRlcnMgYXBpOioiLCJhdWQiOiJqZnJ0QDAxYnQzbnhtNTltNGVoMGd4NHBwZjcxZzBxIiwiaXNzIjoiamZydEAwMWJ0M254bTU5bTRlaDBneDRwcGY3MWcwcSIsImlhdCI6MTU2NDUxNDE1NSwianRpIjoiNzBkNTViYTctODY0ZC00MTAzLWIyY2EtNDYyNzIwZWVlMzhjIn0.ZlzRR-AxX24HD_5ePp3omZ0woH7yldOmab7Wq6g4-zRwWHVqvhO6vcGzuqDi2L3D_16iE6G9qLQN8Y3ggL4ykXxBXViOYhRq7ifxv8m-x13EdW2h9K2bzuIRuVLkZlxY6WKP31zgbmPcMx2N5NkjRF4q-EeNxtOOk9nd1DbP3E6mCH5VtjmQDm5nGYebO-uu_gCz5ym533j1cdX6yil5u5f6k3AaNJ6GNGeZWkZw29muAk6cGwaVDsLq-_cjIPgiQZa6fxh01d-WG3JJO2ZAOM5DzOkyDB2kuWq7WclyIL58invG1TpL4Dny7DPXDnmqfh_QqzAZiEKaaZW2IHdA5Q@thm.jfrog.io/thm/api/pypi/th-pip-prod/simple
-
 appdirs==1.4.3            # via black
-atomicwrites==1.3.0       # via pytest
 attrs==19.3.0             # via black, pytest
 black==19.10b0
 bleach==3.1.0             # via readme-renderer
-certifi==2019.9.11        # via requests
+certifi==2019.11.28       # via requests
 chardet==3.0.4            # via requests
 click==7.0                # via black, pip-tools
 codecov==2.0.15
 configparser==4.0.2       # via py-githooks
-coverage==4.5.4           # via codecov, pytest-cov
+coverage[toml]==5.0.1
 docutils==0.15.2          # via readme-renderer
-entrypoints==0.3          # via keyring
 idna==2.8                 # via requests
-importlib-metadata==0.23  # via pluggy, pytest, twine
+importlib-metadata==1.3.0  # via keyring, pluggy, pytest, twine
 invoke==1.3.0
 isort==4.3.21
-keyring==19.2.0           # via twine
-more-itertools==7.2.0     # via pytest, zipp
+keyring==20.0.0           # via twine
+more-itertools==8.0.2     # via pytest, zipp
 mypy-extensions==0.4.3    # via mypy
-mypy==0.740
+mypy==0.761
 packaging==19.2           # via pytest
 pathspec==0.6.0           # via black
-pip-tools==4.2.0
+pip-tools==4.3.0
 pkginfo==1.5.0.1          # via twine
-pluggy==0.13.0            # via pytest
+pluggy==0.13.1            # via pytest
 py-githooks==1.1.0
 py==1.8.0                 # via pytest
-pygments==2.4.2           # via readme-renderer
+pygments==2.5.2           # via readme-renderer
 pyparsing==2.4.5          # via packaging
-pytest-cov==2.8.1
-pytest==5.2.4
-pyyaml==5.1.2
+pytest==5.3.2
+pyyaml==5.2
 readme-renderer==24.0     # via twine
-regex==2019.11.1          # via black
+regex==2019.12.20         # via black
 requests-toolbelt==0.9.1  # via twine
 requests==2.22.0          # via codecov, requests-toolbelt, twine
 semver==2.9.0
 six==1.13.0               # via bleach, packaging, pip-tools, readme-renderer
-toml==0.10.0              # via black
-tqdm==4.39.0              # via twine
-twine==3.1.0
+toml==0.10.0              # via black, coverage
+tqdm==4.41.0              # via twine
+twine==3.1.1
 typed-ast==1.4.0          # via black, mypy
 typing-extensions==3.7.4.1  # via mypy
 urllib3==1.25.7           # via requests

pyproject.toml

@@ -41,3 +41,15 @@ disable = '''
 missing-function-docstring,
 missing-module-docstring,
 '''
+
+[tool.coverage.run]
+source = [
+  "./src",
+]
+
+[tool.coverage.report]
+exclude_lines = [
+  "pragma: no-cover",
+  "if TYPE_CHECKING:",
+  "@abstractmethod",
+]

requirements.txt

@@ -4,6 +4,4 @@
 #
 #    pip-compile requirements.in
 #
---extra-index-url https://pip-readonly:eyJ2ZXIiOiIyIiwidHlwIjoiSldUIiwiYWxnIjoiUlMyNTYiLCJraWQiOiJYaFp6N3l3eV8zSGdEdDFFa3BNTTY4bnYzTGNRd0syb1p6SFI0TjY3cXdnIn0.eyJzdWIiOiJqZnJ0QDAxYnQzbnhtNTltNGVoMGd4NHBwZjcxZzBxXC91c2Vyc1wvcGlwLXJlYWRvbmx5Iiwic2NwIjoibWVtYmVyLW9mLWdyb3Vwczpwcml2YXRlLXJlYWRlcnMgYXBpOioiLCJhdWQiOiJqZnJ0QDAxYnQzbnhtNTltNGVoMGd4NHBwZjcxZzBxIiwiaXNzIjoiamZydEAwMWJ0M254bTU5bTRlaDBneDRwcGY3MWcwcSIsImlhdCI6MTU2NDUxNDE1NSwianRpIjoiNzBkNTViYTctODY0ZC00MTAzLWIyY2EtNDYyNzIwZWVlMzhjIn0.ZlzRR-AxX24HD_5ePp3omZ0woH7yldOmab7Wq6g4-zRwWHVqvhO6vcGzuqDi2L3D_16iE6G9qLQN8Y3ggL4ykXxBXViOYhRq7ifxv8m-x13EdW2h9K2bzuIRuVLkZlxY6WKP31zgbmPcMx2N5NkjRF4q-EeNxtOOk9nd1DbP3E6mCH5VtjmQDm5nGYebO-uu_gCz5ym533j1cdX6yil5u5f6k3AaNJ6GNGeZWkZw29muAk6cGwaVDsLq-_cjIPgiQZa6fxh01d-WG3JJO2ZAOM5DzOkyDB2kuWq7WclyIL58invG1TpL4Dny7DPXDnmqfh_QqzAZiEKaaZW2IHdA5Q@thm.jfrog.io/thm/api/pypi/th-pip-prod/simple
-
-pyyaml==5.1.2
+pyyaml==5.2

src/syrupy/assertion.py

@@ -1,4 +1,5 @@
 from collections import namedtuple
+from itertools import zip_longest
 from typing import (
     TYPE_CHECKING,
     Any,
@@ -11,6 +12,12 @@
 )
 
 from .exceptions import SnapshotDoesNotExist
+from .terminal import (
+    error_style,
+    green,
+    red,
+    success_style,
+)
 from .types import SerializableData
 from .utils import walk_snapshot_dir
 
@@ -85,11 +92,31 @@ def assert_match(self, data: "SerializableData") -> None:
     def get_assert_diff(self, data: "SerializableData") -> List[str]:
         assertion_result = self._execution_results[self.num_executions - 1]
         snapshot_data = assertion_result.recalled
+        serialized_data = self.serializer.serialize(data)
         if snapshot_data is None:
             return ["Snapshot does not exist!"]
 
         if not assertion_result.success:
-            return [f"- {data}", f"+ {snapshot_data}"]
+            received = serialized_data.splitlines()
+            stored = snapshot_data.splitlines()
+
+            marker_stored = success_style("-")
+            marker_received = error_style("+")
+
+            diff = []
+            for received_line, stored_line in zip_longest(received, stored):
+                if received_line is None:
+                    diff.append(f"{marker_stored} {green(stored_line)}")
+                elif stored_line is None:
+                    diff.append(f"{marker_received} {red(received_line)}")
+                elif received_line != stored_line:
+                    diff.extend(
+                        [
+                            f"{marker_stored} {green(stored_line)}",
+                            f"{marker_received} {red(received_line)}",
+                        ]
+                    )
+            return diff
 
         return []
 
@@ -105,9 +132,10 @@ def __eq__(self, other: "SerializableData") -> bool:
     def _assert(self, data: "SerializableData") -> bool:
         snapshot_file = self.serializer.get_filepath(self.num_executions)
         snapshot_name = self.serializer.get_snapshot_name(self.num_executions)
+        serialized_data = self.serializer.serialize(data)
         try:
             snapshot_data = self._recall_data(index=self.num_executions)
-            matches = snapshot_data is not None and data == snapshot_data
+            matches = snapshot_data is not None and serialized_data == snapshot_data
             assertion_success = matches
             if not matches and self._update_snapshots:
                 self.serializer.create_or_update_snapshot(
@@ -122,7 +150,7 @@ def _assert(self, data: "SerializableData") -> bool:
                 file=snapshot_file,
                 name=snapshot_name,
                 recalled=snapshot_data,
-                asserted=data,
+                asserted=serialized_data,
                 success=assertion_success,
                 created=snapshot_created,
                 updated=snapshot_updated,

src/syrupy/serializers/base.py

@@ -8,6 +8,7 @@
     Callable,
     Optional,
     Set,
+    Union,
 )
 
 from syrupy.constants import SNAPSHOT_DIRNAME
@@ -26,7 +27,7 @@ def __init__(self, test_location: "TestLocation"):
     @property
     @abstractmethod
     def file_extension(self) -> str:
-        pass
+        raise NotImplementedError
 
     @property
     def test_location(self) -> "TestLocation":
@@ -47,7 +48,7 @@ def discover_snapshots(self, filepath: str) -> Set[str]:
         within the file. Snapshot name is dependent on serializer
         implementation.
         """
-        pass
+        raise NotImplementedError
 
     def read_snapshot(self, index: int) -> "SerializableData":
         """
@@ -141,7 +142,7 @@ def read_snapshot_from_file(
         """
         Read the snapshot file and get only the snapshot data for assertion
         """
-        pass
+        raise NotImplementedError
 
     @abstractmethod
     def write_snapshot_or_remove_file(
@@ -152,4 +153,12 @@ def write_snapshot_or_remove_file(
         or removes the snapshot entry if data is `None`.
         If the snapshot file will be empty remove the entire file.
         """
-        pass
+        raise NotImplementedError
+
+    @abstractmethod
+    def serialize(self, data: "SerializableData") -> Union[str, bytes]:
+        """
+        Serializes a python object / data structure into a string
+        to be used for comparison with snapshot data from disk.
+        """
+        raise NotImplementedError

src/syrupy/serializers/raw_single.py

@@ -37,6 +37,11 @@ def read_snapshot_from_file(
     ) -> "SerializableData":
         return self._read_file(snapshot_file)
 
+    def serialize(self, data: "SerializableData") -> bytes:
+        if isinstance(data, bytes):
+            return data
+        raise ValueError("Failure to serialize image data. Expected bytes.")
+
     def _read_file(self, filepath: str) -> Any:
         try:
             with open(filepath, "rb") as f:

src/syrupy/serializers/yaml.py

@@ -2,6 +2,8 @@
 from typing import (
     TYPE_CHECKING,
     Any,
+    Dict,
+    Optional,
     Set,
 )
 
@@ -21,15 +23,15 @@ def file_extension(self) -> str:
 
     def discover_snapshots(self, filepath: str) -> Set[str]:
         try:
-            return set(self._read_file(filepath).keys())
+            return set(self._read_raw_file(filepath).keys())
         except:
             return set()
 
     def read_snapshot_from_file(
         self, snapshot_file: str, snapshot_name: str
     ) -> "SerializableData":
-        snapshots = self._read_file(snapshot_file)
-        return snapshots.get(snapshot_name, {}).get("data", None)
+        raw_snapshots = self._read_raw_file(snapshot_file)
+        return raw_snapshots.get(snapshot_name, None)
 
     def write_snapshot_or_remove_file(
         self, snapshot_file: str, snapshot_name: str, data: "SerializableData"
@@ -44,13 +46,24 @@ def write_snapshot_or_remove_file(
             del snapshots[snapshot_name]
         else:
             snapshots[snapshot_name] = snapshots.get(snapshot_name, {})
-            snapshots[snapshot_name]["data"] = data
+            snapshots[snapshot_name][self._data_key] = data
 
         if snapshots:
             self._write_file(snapshot_file, snapshots)
         else:
             os.remove(snapshot_file)
 
+    def serialize(self, data: "SerializableData") -> str:
+        """
+        Returns the serialized form of 'data' to be compared
+        with the snapshot data written to disk.
+        """
+        return str(yaml.dump({self._data_key: data}, allow_unicode=True))
+
+    @property
+    def _data_key(self) -> str:
+        return "data"
+
     def _write_file(self, filepath: str, data: "SerializableData") -> None:
         """
         Writes the snapshot data into the snapshot file that be read later.
@@ -68,3 +81,25 @@ def _read_file(self, filepath: str) -> Any:
         except FileNotFoundError:
             pass
         return {}
+
+    def _read_raw_file(self, filepath: str) -> Dict[str, str]:
+        """
+        Read the raw snapshot data (str) from the snapshot file into a dict
+        of snapshot name to raw data. This does not attempt any deserialization
+        of the snapshot data.
+        """
+        snapshots = {}
+        try:
+            with open(filepath, "r") as f:
+                test_name = None
+                for line in f:
+                    indent = len(line) - len(line.lstrip(" "))
+                    if not indent:
+                        test_name = line[:-2]  # newline & colon
+                        snapshots[test_name] = ""
+                    elif test_name is not None:
+                        snapshots[test_name] += line[2:]
+        except FileNotFoundError:
+            pass
+
+        return snapshots

src/syrupy/terminal.py

@@ -19,3 +19,7 @@ def bold(text: Union[str, int]) -> str:
 
 def error_style(text: Union[str, int]) -> str:
     return bold(red(text))
+
+
+def success_style(text: Union[str, int]) -> str:
+    return bold(green(text))

tasks.py

@@ -17,7 +17,9 @@ def requirements(ctx):
     """
     Build requirements lock file
     """
-    ctx.run(f"python -m piptools compile dev-requirements.in", pty=True)
+    input_files = ["requirements.in", "dev-requirements.in"]
+    for input_file in input_files:
+        ctx.run(f"python -m piptools compile {input_file}", pty=True)
 
 
 @task
@@ -59,14 +61,17 @@ def test(ctx, coverage=False, dev=False, update_snapshots=False, verbose=False):
     """
     env = {"PYTHONPATH": "./src"} if dev else {}
     flags = {
-        "-s": verbose,
-        "--cov=./src": coverage,
+        "-s -vv": verbose,
         "--snapshot-update": update_snapshots,
     }
+    coverage_module = "coverage run -m " if coverage else ""
     test_flags = " ".join(flag for flag, enabled in flags.items() if enabled)
-    ctx.run(f"python -m pytest {test_flags} .", env=env, pty=True)
+    ctx.run(f"python -m {coverage_module}pytest {test_flags} .", env=env, pty=True)
     if coverage:
-        ctx.run("codecov", pty=True)
+        if not os.environ.get("CI"):
+            print("\nNote: Test coverage is only uploaded in CI.\n")
+        else:
+            ctx.run("codecov", pty=True)
 
 
 @task(pre=[clean])

tests/__snapshots__/test_snapshots.yaml

@@ -34,14 +34,6 @@ test_set:
     is: null
     set: null
     this: null
-test_set.1:
-  data: !!set
-    a: null
-    ? !!python/object/apply:builtins.frozenset
-    - - nested, set
-    : null
-    is: null
-    this: null
 test_simple_string:
   data: Loreeeeeem ipsum.
 test_tuples:

tests/test_image_plugin.py

@@ -19,3 +19,8 @@ def test_image(snapshot_png):
         b"RK5CYII="
     )
     assert actual == snapshot_png
+
+
+def test_raises_error_for_unserializable_data(snapshot_png):
+    with pytest.raises(ValueError):
+        assert "not a byte string" == snapshot_png

tests/test_snapshots.py

@@ -39,7 +39,6 @@ def test_dict(snapshot, actual):
 
 def test_set(snapshot):
     assert snapshot == {"this", "is", "a", "set"}
-    assert snapshot == {"this", "is", "a", frozenset({"nested, set"})}
 
 
 ExampleTuple = namedtuple("ExampleTuple", ["a", "b", "c", "d"])