Enable passing datasets to tests via command line (#439)

* Add `--ds` pytest cli option for passing in datasets

* Make ds fixture into call hook so input for tests can be modified

* Start adding cli datasets marker and tell pytest not to worry about it

* Changelog

* Work around getoption default arg not working

* Tweak cli options and marker inputs

* Start documenting

* Bit more docs

* RST isn't markdown

* Update docs/developer.rst

Co-authored-by: Stuart Mumford <stuart@cadair.com>

* Update changelog

* Rename changelog

* Update dkist/dataset/tests/test_dataset.py

Co-authored-by: Stuart Mumford <stuart@cadair.com>

* Update dkist/dataset/tests/test_dataset.py

Co-authored-by: Stuart Mumford <stuart@cadair.com>

* Undo a thing, we'll fix it another time

* Disregard last, error is in the data which is exactly the point of this PR

* Well apparently this breaks things

---------

Co-authored-by: Stuart Mumford <stuart@cadair.com>

changelog/439.trivial.rst

@@ -0,0 +1,2 @@
+Add `--ds` and `--tiled-ds` CLI options to allow passing in datasets for use with the test suite.
+The dataset given with `--ds` is passed to tests marked with `accept_cli_dataset` and those given with `--tiled-ds` are passed to tests marked with `accept_cli_tiled_dataset`.

conftest.py

@@ -1,4 +1,5 @@
 import matplotlib as mpl
+import pytest
 
 mpl.use("Agg")
 
@@ -8,3 +9,30 @@ def pytest_configure(config):
     # which will cause errors with remote_data off
     from astropy.utils.iers import IERS_Auto
     IERS_Auto.open()
+
+
+def pytest_addoption(parser):
+    parser.addoption("--ds", action="store", help="Dataset provided as a string which can be parsed by load_dataset. This will override fixtures used in tests decorated with @accept_ds_from_cli.")
+    parser.addoption("--tiled-ds", action="store", help="Tiled datasets provided as a string which can be parsed by load_dataset. These datasets will override fixtures used in tests decorated with @accept_tiled_ds_from_cli.")
+
+
+def pytest_report_header(config, start_path):
+    ds_str = config.getoption("--ds") or "None"
+    tds_str = config.getoption("--tiled-ds") or "None"
+
+    return [f"CLI dataset provided: {ds_str}",
+            f"CLI tiled dataset provided: {tds_str}"]
+
+
+@pytest.hookimpl(wrapper=True, tryfirst=True)
+def pytest_runtest_makereport(item, call):
+    report = yield
+    if report.when == "call":
+        ds = item.config.getoption("--ds")
+        tds = item.config.getoption("--tiled-ds")
+        if ds and item.get_closest_marker("accept_cli_dataset"):
+            report.nodeid += f"[{ds}]"
+        if tds and item.get_closest_marker("accept_cli_tiled_dataset"):
+            report.nodeid += f"[{tds}]"
+
+    return report

dkist/conftest.py

@@ -347,3 +347,21 @@ def visp_dataset_no_headers(tmp_path_factory):
         with open(vispdir / "test_visp_no_headers.asdf", mode="wb") as afo:
             afo.write(gfo.read())
     return load_dataset(vispdir / "test_visp_no_headers.asdf")
+
+
+@pytest.hookimpl(hookwrapper=True)
+def pytest_runtest_call(item):
+    ds = item.config.getoption("--ds")
+    tds = item.config.getoption("--tiled-ds")
+
+    # Only one of accept_cli_dataset and accept_cli_tiled_dataset should be available
+    mark = item.get_closest_marker("accept_cli_dataset") or item.get_closest_marker("accept_cli_tiled_dataset")
+    if mark:
+        # Replace either the fixture specified as the first arg of the marker, or the first fixture in the test definition
+        replace_arg = mark.args[0] if mark.args else item.fixturenames[0]
+        if ds:
+            item.funcargs[replace_arg] = load_dataset(ds)
+        if tds:
+            item.funcargs[tdsmark.args[0]] = load_dataset(tds)
+
+    yield item

dkist/dataset/tests/test_dataset.py

@@ -43,21 +43,24 @@ def test_init_missing_meta_keys(identity_gwcs):
         Dataset(data, wcs=identity_gwcs, meta={"headers": {}})
 
 
+@pytest.mark.accept_cli_dataset
 def test_repr(dataset, dataset_3d):
     r = repr(dataset)
     assert str(dataset.data) in r
     r = repr(dataset_3d)
     assert str(dataset_3d.data) in r
 
 
+@pytest.mark.accept_cli_dataset
 def test_wcs_roundtrip(dataset):
-    p = (10*u.pixel, 10*u.pixel)
+    p = [1*u.pixel] * dataset.wcs.pixel_n_dim
     w = dataset.wcs.pixel_to_world(*p)
     p2 = dataset.wcs.world_to_pixel(w)
     assert_quantity_allclose(p, p2 * u.pix)
 
 
 def test_wcs_roundtrip_3d(dataset_3d):
+    # TOO generalise this so mark.accept_cli_dataset will work, if possible
     p = (10*u.pixel, 10*u.pixel, 10*u.pixel)
     w = dataset_3d.wcs.pixel_to_world(*p)
     p2 = dataset_3d.wcs.world_to_pixel(*w) * u.pix
@@ -158,6 +161,7 @@ def test_header_slicing_single_index():
     assert (sliced.headers["DINDEX3"] == sliced_headers["DINDEX3"]).all()
 
 
+@pytest.mark.accept_cli_dataset
 def test_header_slicing_3D_slice(large_visp_dataset):
     dataset = large_visp_dataset
     idx = np.s_[:2, 10:15, 0]

dkist/dataset/tests/test_tiled_dataset.py

@@ -18,6 +18,7 @@ def test_tiled_dataset(simple_tiled_dataset, dataset):
     assert simple_tiled_dataset.shape == (2, 2)
 
 
+@pytest.mark.accept_cli_tiled_dataset
 @pytest.mark.parametrize("aslice", [np.s_[0,0],
                                     np.s_[0],
                                     np.s_[...,0],
@@ -28,6 +29,7 @@ def test_tiled_dataset_slice(simple_tiled_dataset, aslice):
     assert np.all(simple_tiled_dataset[aslice] == simple_tiled_dataset._data[aslice])
 
 
+@pytest.mark.accept_cli_tiled_dataset
 @pytest.mark.parametrize("aslice", [np.s_[0, :100, 100:200]])
 def test_tiled_dataset_slice_tiles(large_tiled_dataset, aslice):
     sliced = large_tiled_dataset.slice_tiles[aslice]
@@ -84,10 +86,13 @@ def test_tileddataset_plot(share_zscale):
     ds.plot(0, share_zscale=share_zscale)
     return plt.gcf()
 
+
+@pytest.mark.accept_cli_tiled_dataset
 def test_repr(simple_tiled_dataset):
     r = repr(simple_tiled_dataset)
     assert str(simple_tiled_dataset[0, 0].data) in r
 
 
+@pytest.mark.accept_cli_tiled_dataset
 def test_tiles_shape(simple_tiled_dataset):
     assert simple_tiled_dataset.tiles_shape == [[tile.data.shape for tile in row] for row in simple_tiled_dataset]

docs/developer.rst

@@ -92,3 +92,26 @@ Sphinx References
 When authoring documentation files each page (at minimum) should have a reference on its top title.
 This should take the form ``dkist:<section>:<page>``, so for the index page of the ``topic_guides/`` folder it is ``dkist:topic-guides:index``.
 All references should take this three element format, so a subheading in a page would be ``dkist:section:subheading``.
+
+Testing
+-------
+
+Testing with arbitrary datasets
+*******************************
+
+The DKIST Python tools allow the test suite to run on user-provided datasets.
+To enable this there are two pytest command line optionsa available, `--ds` and `--tiled-ds`.
+Each takes a string parseable by `load_dataset()`.
+Tests with the `accept_cli_dataset` or `accept_tiled_dataset` markers will take the value given for `--ds` or `--tiled-ds` respectively, load that dataset and pass it into the test in place of a fixture.
+To only run tests using the supplied datasets you can run pytest with `-m accept_cli_dataset` and/or `-m accept_cli_tiled_dataset`.
+By default the first fixture specified in the test definition is the one replaced, but if a fixture name is given to the marker that fixture will be replaced instead.
+
+For example:
+
+.. code-block:: python
+
+   @pytest.mark.accept_cli_dataset("another_dataset")
+   def test_something(dataset, another_dataset):
+       ...
+
+would usually run with the fixtures `dataset` and `another_dataset` as its inputs, but running `pytest --ds /path/to/dataset/ABCDE/` would run the test using the `dataset` fixture and the local dataset `ABCDE` instead.

pytest.ini

@@ -27,6 +27,8 @@ markers =
     online: marks this test function as needing online connectivity.
     figure: marks this test function as using hash-based Matplotlib figure verification. This mark is not meant to be directly applied, but is instead automatically applied when a test function uses the @sunpy.tests.helpers.figure_test decorator.
     benchmark: marks this test as a benchmark
+    accept_cli_dataset: marks this test as able to run with user supplied dataset as input. This should be given as a string parsable by dkist.load_dataset().
+    accept_cli_tiled_dataset: marks this test as able to run with user supplied tiled dataset as input. This should be given as a string parsable by dkist.load_dataset().
 addopts =
     --doctest-rst
     -p no:unraisableexception