AllenNeuralDynamics · dbirman · Oct 4, 2024 · Aug 20, 2024 · Aug 23, 2024 · Aug 30, 2024
diff --git a/.flake8 b/.flake8
@@ -5,3 +5,7 @@ exclude =
     build
 max-complexity = 10
 max-line-length = 120
+
+[flake8:local-plugins]
+extension =
+    PF = aind_flake8_extensions.plugin:run_ast_checks
diff --git a/docs/source/aind_data_schema.core.rst b/docs/source/aind_data_schema.core.rst
@@ -84,6 +84,14 @@ aind\_data\_schema.core.subject module
    :undoc-members:
    :show-inheritance:
 
+aind\_data\_schema.core.quality_control module
+--------------------------------------
+
+.. automodule:: aind_data_schema.core.quality_control
+   :members:
+   :undoc-members:
+   :show-inheritance:
+
 Module contents
 ---------------
 

diff --git a/docs/source/conf.py b/docs/source/conf.py
@@ -21,6 +21,7 @@
     rig,
     session,
     subject,
+    quality_control
 )
 
 dummy_object = [
@@ -33,6 +34,7 @@
     rig,
     session,
     subject,
+    quality_control
 ]  # A temporary workaround to bypass "Imported but unused" error
 
 INSTITUTE_NAME = "Allen Institute for Neural Dynamics"
@@ -55,6 +57,7 @@
     "sphinxcontrib.autodoc_pydantic",
     "sphinx.ext.napoleon",
     "sphinx_jinja",
+    "myst_parser",
 ]
 templates_path = ["_templates"]
 exclude_patterns = []
@@ -88,6 +91,7 @@
     "rig",
     "session",
     "subject",
+    "quality_control",
 ]
 rst_epilog = ""
 for diagram in diagrams_list:

diff --git a/docs/source/index.rst b/docs/source/index.rst
@@ -27,6 +27,7 @@ A given data asset will have the following JSON files:
 - :doc:`rig <rig>` or :doc:`instrument <rig>`: Metadata describing the equipment used to acquire data, including part names, serial numbers, and configuration details.
 - :doc:`session <session>` or :doc:`acquisition <acquisition>`: Metadata describing how the data was acquired
 - :doc:`processing <processing>`: Metadata describing how data has been processed and analyzed into derived data assets, including information on the software and parameters used for processing and analysis.
+- :doc:`quality_control <quality_control>`: Metadata describing how the data has been evaluated for quality control.
 
 Example
 =======
@@ -175,6 +176,7 @@ build NWB extension that easily embeds metadata not covered by the core NWB sche
    session
    acquisition
    processing
+   quality_control
 
 
 .. toctree::

diff --git a/docs/source/quality_control.md b/docs/source/quality_control.md
@@ -0,0 +1,107 @@
+# Quality control
+
+## Overview
+
+Quality control is a collection of **evaluations** based on sets of **metrics** about the data. 
+
+`QCEvaluation`s should be generated during pipelines: before raw data upload, during processing, and during analysis by researchers.
+
+Each `QualityControl`, `QCEvaluation`, and `QCMetric` includes a `aind_data_schema.quality_control.State` which is a timestamped object indicating that the Overall QC/Evaluation/Metric passes, fails, or is in a pending state waiting for manual annotation.
+
+The state of an evaluation is set automatically to the lowest of its metric's states. A single failed metric sets an entire evaluation to fail. While a single pending metric (with all other metrics passing) sets an entire evaluation to pending. An optional setting `QCEvaluation.allow_failed_metrics` allows you to ignore failures, which can be useful in situations where an evaluation is not critical for quality control.
+
+## Details
+
+**Q: What is an evaluation?**
+
+Each `QCEvaluation` should be thought of as a single aspect of the data asset, from one `Modality`, that is evaluated for quality at a specific `Stage` in data acquisition or analysis. For example, the brain moves a small amount during electrophysiology. This evaluation would be marked with `Stage:RAW` and `Modality:ECEPHYS`. Evaluations will often consist of multiple metrics, some of which can be measured and evaluated automatically, as well as qualititative metrics that need to be evaluated by a human observer.
+
+The state of an evaluation depends on the state of its metrics according to these rules:
+
+- If any metric fails the evaluation fails (except when `allow_failed_metrics=True`, see below)
+- If any metric is pending and the rest pass the evaluation is pending
+- If all metrics pass the evaluation passes
+
+There are many situations where quality control is evaluated on an aspect of the data that isn't critical to the overall experimental goals. For example, you may have a `QCEvaluation` that checks whether the temperature and humidity sensors on the rig were functional, but the overall analysis can proceed with or without the these data. In these situations set `QCEvaluation.allow_failed_metrics=True` to allow the evaluation to pass even if these sensors actually failed. This ensures that the overall `QualityControl` for the data asset can also pass, without regard to these optional elements of the data. 
+
+**Q: What is a metric?**
+
+Each `QCMetric` is a single value or set of values that can be computed, or observed, about a set of data as part of an evaluation. These can have any type. See the AIND section for special rules for annotating metrics with options.
+
+`QCMetric`s have a `Status`. The `Status` should depend directly on the `QCMetric.value`, either by a simple function: "value>5", or by a qualitative rule: "Field of view includes visual areas". The `QCMetric.description` field should be used to describe the rule used to set the status. Metrics can be evaluated multiple times, in which case the new status should be appended the `QCMetric.status_history`.
+
+Metrics should include a `QCMetric.reference`. References are intended to be publicly accessible images, figures, combined figures with multiple panels, or videos that support the metric or provide information necessary for manual annotation of a metric's status.
+
+**Q: What are the status options for metrics?**
+
+In our quality control a metric's status is always `PASS`, `PENDING` (waiting for manual annotation), or `FAIL`. `PENDING` should be used when a user must manually annotated the metric's state.
+
+We enforce this minimal set of states to prevent ambiguity and make it easier to build tools that can interpret the status of a data asset.
+
+## Details for AIND users
+
+### Uploading QC
+
+#### Preferred workflow 
+
+**Metadata**
+
+If you are building `QCEvaluation` and `QCMetric` objects prior to raw data upload or in a capsule alongside your processing or analysis, your workflow is: 
+
+```
+from aind_data_schema.core.quality_control import QualityControl
+
+# Build your QCEvaluations and metrics
+evaluations = [QCEvaluation(), ...]
+
+# Build your QualityControl object
+qc = QualityControl(evaluations=evaluations)
+
+qc.write_standard_file()
+```
+
+The indexer will pick up this file alongside the other metadata files and handle it appropriately.
+
+**References**
+
+Each `QCMetric` you build should have an attached reference. Our preference is that you post these images to [FigURL](https://github.com/flatironinstitute/figurl/blob/main/doc/intro.md) and put the generated URL into the reference.
+
+We recommend that you write PNG files for images and static multi-panel figures, MP4 files for videos, and Altair charts for interactive figures.
+
+#### Alternate workflows
+
+**Metadata**
+
+We'll post documentation on how to append `QCEvaluations` to pre-existing quality_control.json files, via DocDB using the `aind-data-access-api`, in the future.
+
+**References**
+
+You can also place the references in the data asset itself, to do this include the relative path "qc_images/your_image.png" to that asset inside of the results folder.
+
+### QC Portal
+
+The QC Portal is a browser application that allows users to view and interact with the AIND QC metadata and to annotate ``PENDING`` metrics with qualitative evaluations. The portal is maintained by scientific computing, reach out to us with any questions or concerns.
+
+The portal works by pulling the metadata object from the Document Database (DocDB). When you make changes to metrics or add notes the **submit** button will be enabled, submitting pushes your updates to the DocDB along with a timestamp and your name.
+
+**Q: When does the state get set for the QCEvaluation and QualityControl objects?**
+
+The QC portal automatically calls ``QualityControl.evaluate_status()`` whenever you submit changes to the metrics. This first evaluates the individual `QCEvaluation` objects, and then evaluates the overall status.
+
+**Q: How do reference URLs get pulled into the QC Portal?**
+
+Each metric is associated with a reference figure, image, or video. The QC portal can interpret four ways of setting the reference field:
+
+- Provide a relative path to a file in this data asset's S3 bucket
+- Provide a url to a FigURL figure
+- Provide the name of one of the interactive plots, e.g. "ecephys-drift-map"
+
+<!-- There are many situations where it's helpful to be able to "swipe" between two images. If you have two identical images separated by a ';' the portal will allow users to swipe between them. For example, you might show snippets of the raw electrophysiology raster with detected spikes overlaid on the swipe. -->
+
+**Q: I saw fancy things like dropdowns in the QC Portal, how do I do that?**
+
+By default the QC portal displays dictionaries as tables where the values can be edited. We also support a few special cases to allow a bit more flexibility or to constrain the actions that manual annotators can take. Install the `aind-qcportal-schema` package and set the `value` field to the corresponding pydantic object to use these. 
+
+### Multi-session QC
+
+[Details coming soon, this is under discussion]
diff --git a/examples/aibs_smartspim_instrument.json b/examples/aibs_smartspim_instrument.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/instrument.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "instrument_id": "440_SmartSPIM2_20231004",
    "modification_date": "2023-10-04",
    "instrument_type": "SmartSPIM",

diff --git a/examples/aibs_smartspim_procedures.json b/examples/aibs_smartspim_procedures.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/procedures.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.1.1",
    "subject_id": "651286",
    "subject_procedures": [
       {

diff --git a/examples/aind_smartspim_instrument.json b/examples/aind_smartspim_instrument.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/instrument.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "instrument_id": "440_SmartSPIM1_20231004",
    "modification_date": "2023-10-04",
    "instrument_type": "SmartSPIM",

diff --git a/examples/bergamo_ophys_session.json b/examples/bergamo_ophys_session.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "protocol_id": [],
    "experimenter_full_name": [
       "John Doe"

diff --git a/examples/data_description.json b/examples/data_description.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/data_description.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "license": "CC-BY-4.0",
    "platform": {
       "name": "Electrophysiology platform",

diff --git a/examples/ephys_rig.json b/examples/ephys_rig.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "rig_id": "323_EPHYS1_20231003",
    "modification_date": "2023-10-03",
    "mouse_platform": {

diff --git a/examples/ephys_session.json b/examples/ephys_session.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/session.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "protocol_id": [],
    "experimenter_full_name": [
       "Max Quibble",

diff --git a/examples/exaspim_acquisition.json b/examples/exaspim_acquisition.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/acquisition.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "protocol_id": [],
    "experimenter_full_name": [
       "###"

diff --git a/examples/exaspim_instrument.json b/examples/exaspim_instrument.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/instrument.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "instrument_id": "440_exaSPIM1-20231004",
    "modification_date": "2023-10-04",
    "instrument_type": "exaSPIM",

diff --git a/examples/fip_behavior_rig.json b/examples/fip_behavior_rig.json
@@ -1,6 +1,6 @@
 {
    "describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/rig.py",
-   "schema_version": "1.0.0",
+   "schema_version": "1.0.1",
    "rig_id": "447_FIP-Behavior_20000101",
    "modification_date": "2000-01-01",
    "mouse_platform": {
@@ -43,7 +43,7 @@
          "reward_spouts": [
             {
                "device_type": "Reward spout",
-               "name": "Janelia_Lick_Detector Left",
+               "name": "Left spout",
                "serial_number": null,
                "manufacturer": null,
                "model": null,
@@ -66,12 +66,30 @@
                   "additional_settings": {},
                   "notes": null
                },
-               "lick_sensor": null,
+               "lick_sensor": {
+                  "device_type": "Lick detector",
+                  "name": "Janelia_Lick_Detector Left",
+                  "serial_number": null,
+                  "manufacturer": {
+                     "name": "Janelia Research Campus",
+                     "abbreviation": "Janelia",
+                     "registry": {
+                        "name": "Research Organization Registry",
+                        "abbreviation": "ROR"
+                     },
+                     "registry_identifier": "013sk6x84"
+                  },
+                  "model": null,
+                  "path_to_cad": null,
+                  "port_index": null,
+                  "additional_settings": {},
+                  "notes": null
+               },
                "lick_sensor_type": "Capacitive"
             },
             {
                "device_type": "Reward spout",
-               "name": "Janelia_Lick_Detector Right",
+               "name": "Right spout",
                "serial_number": null,
                "manufacturer": null,
                "model": null,
@@ -94,7 +112,25 @@
                   "additional_settings": {},
                   "notes": null
                },
-               "lick_sensor": null,
+               "lick_sensor": {
+                  "device_type": "Lick detector",
+                  "name": "Janelia_Lick_Detector Right",
+                  "serial_number": null,
+                  "manufacturer": {
+                     "name": "Janelia Research Campus",
+                     "abbreviation": "Janelia",
+                     "registry": {
+                        "name": "Research Organization Registry",
+                        "abbreviation": "ROR"
+                     },
+                     "registry_identifier": "013sk6x84"
+                  },
+                  "model": null,
+                  "path_to_cad": null,
+                  "port_index": null,
+                  "additional_settings": {},
+                  "notes": null
+               },
                "lick_sensor_type": "Capacitive"
             }
          ]

diff --git a/examples/fip_behavior_rig.py b/examples/fip_behavior_rig.py
@@ -98,17 +98,27 @@
         d.RewardDelivery(
             reward_spouts=[
                 d.RewardSpout(
-                    name="Janelia_Lick_Detector Left",
+                    name="Left spout",
                     side=d.SpoutSide.LEFT,
                     spout_diameter=1.2,
                     solenoid_valve=d.Device(device_type="Solenoid", name="Solenoid Left"),
+                    lick_sensor=d.Device(
+                        name="Janelia_Lick_Detector Left",
+                        device_type="Lick detector",
+                        manufacturer=d.Organization.JANELIA,
+                    ),
                     lick_sensor_type=d.LickSensorType("Capacitive"),
                 ),
                 d.RewardSpout(
-                    name="Janelia_Lick_Detector Right",
+                    name="Right spout",
                     side=d.SpoutSide.RIGHT,
                     spout_diameter=1.2,
                     solenoid_valve=d.Device(device_type="Solenoid", name="Solenoid Right"),
+                    lick_sensor=d.Device(
+                        name="Janelia_Lick_Detector Right",
+                        device_type="Lick detector",
+                        manufacturer=d.Organization.JANELIA,
+                    ),
                     lick_sensor_type=d.LickSensorType("Capacitive"),
                 ),
             ],