Skip to content

Commit

Permalink
1072 qc automate state properties in qualitycontrol and evaluation ba…
Browse files Browse the repository at this point in the history 
…sed on metric states (#1074)

* refactor: changing references -> reference

* feat: add an overall evaluator name

* feat: adding QCStatus and lists of status for versioning

* chore: lint

* tests: updating tests for QCStatus

* feat: add QCStatus list to metrics

* feat: adding a function to automate setting the evaluation_status field

Coverage not at 100% yet
Also replaced date with datetime in QCStatus

* test: test coverage for overall_status

* test: add coverage for properties

* refactor: date -> datetime

* chore: lint

* doc: missing docstrings

* tests: remove unnecessary test

* feat: auto-set status when never previously evaluated, and tests

* fix: fixing an issue with private attributes

You can't do what I wanted (hide the _evaluation_status list) because pydantic doesn't serialize the field. So we have to keep the history exposed but we'll have to encourage people not to access it directly

* chore: example and lint

* fix: removing default_factory, trying to allow timestamp pass through

* test: fixing examples test timestamp issue

* chore: lint

* refactor: datetime -> AwareDatetime and remove default

* tests: add a test that confirms that order is preserved

* fix: missing timezones for example

* feat: add option to allow failures in metrics

* feat: add a convenience function to get metrics that are failing

* tests: update qc.json example

* feat: ensure metric_status_history not empty with validator

* chore: docstring

* test: improving examples

* chore: lint

* feat: add asset_id field to qc

* tests: fix examples

* tests: fix examples

* docs: extended desc for allowing failed metrics

* feat: removing prefixes to clean up fields

* feat: remove property prefixes

* chore: lint

* chore/tests: lint and fix examples

* tests: fix refactor issues in tests

* docs: fix typo

* chore: unused import snuck in?

* fix: allow none to return from failed_metrics

* ci: fix for python 3.8 failures

* fix: remove the confusing "Automated" evaluator

* tests: fix qc example

* refactor: removing redundant status_history from QualityControl and QCEvaluation

* fix: forgot to remove redundant fields

* tests: fix examples
  • Loading branch information
@dbirman
dbirman authored Oct 3, 2024
1 parent cb97a80 commit e880ccd
Show file tree
Hide file tree
Showing 5 changed files with 620 additions and 134 deletions.
169 changes: 113 additions & 56 deletions examples/quality_control.json
View file
Original file line number Diff line number Diff line change
@@ -1,118 +1,175 @@
{
"describedBy": "https://github.com/AllenNeuralDynamics/aind-data-schema/main/src/aind_data_schema/core/quality_control.py",
"schema_version": "1.0.0",
"overall_status": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22"
}
],
"evaluations": [
{
"evaluation_modality": {
"modality": {
"name": "Extracellular electrophysiology",
"abbreviation": "ecephys"
},
"evaluation_stage": "Processing",
"evaluation_name": "Drift map",
"evaluation_description": "Qualitative check that drift map shows minimal movement",
"qc_metrics": [
"stage": "Raw data",
"name": "Drift map",
"description": "Qualitative check that drift map shows minimal movement",
"metrics": [
{
"name": "Probe A drift",
"value": "High",
"value": {
"value": "",
"options": [
"Low",
"Medium",
"High"
],
"status": [
"Pass",
"Fail",
"Fail"
],
"type": "dropdown"
},
"description": null,
"reference": "ecephys-drift-map"
"reference": "ecephys-drift-map",
"status_history": [
{
"evaluator": "",
"status": "Pending",
"timestamp": "2022-11-22T00:00:00Z"
}
]
},
{
"name": "Probe B drift",
"value": "Low",
"value": {
"value": "",
"options": [
"Drift visible in entire session",
"Drift visible in part of session",
"Sudden movement event"
],
"status": [
"Fail",
"Pass",
"Fail"
],
"type": "checkbox"
},
"description": null,
"reference": "ecephys-drift-map"
"reference": "ecephys-drift-map",
"status_history": [
{
"evaluator": "",
"status": "Pending",
"timestamp": "2022-11-22T00:00:00Z"
}
]
},
{
"name": "Probe C drift",
"value": "Low",
"description": null,
"reference": "ecephys-drift-map"
}
],
"evaluation_status": [
{
"evaluator": "Fred Flintstone",
"status": "Fail",
"timestamp": "2022-11-22"
"reference": "ecephys-drift-map",
"status_history": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22T00:00:00Z"
}
]
}
],
"notes": "Manually annotated: failed due to high drift on probe A"
"notes": "",
"allow_failed_metrics": false
},
{
"evaluation_modality": {
"modality": {
"name": "Behavior videos",
"abbreviation": "behavior-videos"
},
"evaluation_stage": "Raw data",
"evaluation_name": "Video frame count check",
"evaluation_description": null,
"qc_metrics": [
"stage": "Raw data",
"name": "Video frame count check",
"description": null,
"metrics": [
{
"name": "video_1_num_frames",
"value": 662,
"description": null,
"reference": null
"reference": null,
"status_history": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22T00:00:00Z"
}
]
},
{
"name": "video_2_num_frames",
"value": 662,
"description": null,
"reference": null
"reference": null,
"status_history": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22T00:00:00Z"
}
]
}
],
"evaluation_status": [
{
"evaluator": "Fred Flintstone",
"status": "Fail",
"timestamp": "2022-11-22"
}
],
"notes": "Pass when video_1_num_frames==video_2_num_frames"
"notes": "Pass when video_1_num_frames==video_2_num_frames",
"allow_failed_metrics": false
},
{
"evaluation_modality": {
"modality": {
"name": "Extracellular electrophysiology",
"abbreviation": "ecephys"
},
"evaluation_stage": "Raw data",
"evaluation_name": "Probes present",
"evaluation_description": null,
"qc_metrics": [
"stage": "Raw data",
"name": "Probes present",
"description": null,
"metrics": [
{
"name": "ProbeA_success",
"value": true,
"description": null,
"reference": null
"reference": null,
"status_history": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22T00:00:00Z"
}
]
},
{
"name": "ProbeB_success",
"value": true,
"description": null,
"reference": null
"reference": null,
"status_history": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22T00:00:00Z"
}
]
},
{
"name": "ProbeC_success",
"value": true,
"description": null,
"reference": null
}
],
"evaluation_status": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22"
"reference": null,
"status_history": [
{
"evaluator": "Automated",
"status": "Pass",
"timestamp": "2022-11-22T00:00:00Z"
}
]
}
],
"notes": null
"notes": null,
"allow_failed_metrics": false
}
],
"notes": null
Expand Down
91 changes: 63 additions & 28 deletions examples/quality_control.py
View file
Original file line number Diff line number Diff line change
@@ -1,51 +1,86 @@
"""Example quality control processing"""

from datetime import date
from datetime import datetime, timezone

from aind_data_schema_models.modalities import Modality

from aind_data_schema.core.quality_control import QCEvaluation, QualityControl, QCMetric, Stage, Status, QCStatus

t = date(2022, 11, 22)
t = datetime(2022, 11, 22, 0, 0, 0, tzinfo=timezone.utc)

s = QCStatus(evaluator="Automated", status=Status.PASS, timestamp=t)
sp = QCStatus(evaluator="", status=Status.PENDING, timestamp=t)

# Example of how to use a dictionary to provide options a metric
drift_value_with_options = {
"value": "",
"options": ["Low", "Medium", "High"],
"status": [
"Pass",
"Fail",
"Fail",
], # when set, this field will be used to automatically parse the status, blank forces manual update
"type": "dropdown", # other type options: "checkbox"
}

# Example of how to use a dictionary to provide multiple checkable flags, some of which will fail the metric
drift_value_with_flags = {
"value": "",
"options": ["Drift visible in entire session", "Drift visible in part of session", "Sudden movement event"],
"status": ["Fail", "Pass", "Fail"],
"type": "checkbox",
}

eval0 = QCEvaluation(
evaluation_name="Drift map",
evaluation_description="Qualitative check that drift map shows minimal movement",
evaluation_modality=Modality.ECEPHYS,
evaluation_stage=Stage.PROCESSING,
evaluation_status=[QCStatus(evaluator="Fred Flintstone", timestamp=t, status=Status.FAIL)],
qc_metrics=[
QCMetric(name="Probe A drift", value="High", reference="ecephys-drift-map"),
QCMetric(name="Probe B drift", value="Low", reference="ecephys-drift-map"),
QCMetric(name="Probe C drift", value="Low", reference="ecephys-drift-map"),
name="Drift map",
description="Qualitative check that drift map shows minimal movement",
modality=Modality.ECEPHYS,
stage=Stage.RAW,
metrics=[
QCMetric(
name="Probe A drift",
value=drift_value_with_options,
reference="ecephys-drift-map",
status_history=[sp],
),
QCMetric(
name="Probe B drift",
value=drift_value_with_flags,
reference="ecephys-drift-map",
status_history=[sp],
),
QCMetric(name="Probe C drift", value="Low", reference="ecephys-drift-map", status_history=[s]),
],
notes="Manually annotated: failed due to high drift on probe A",
notes="",
)

eval1 = QCEvaluation(
evaluation_name="Video frame count check",
evaluation_modality=Modality.BEHAVIOR_VIDEOS,
evaluation_stage=Stage.RAW,
evaluation_status=[QCStatus(evaluator="Fred Flintstone", timestamp=t, status=Status.FAIL)],
qc_metrics=[QCMetric(name="video_1_num_frames", value=662), QCMetric(name="video_2_num_frames", value=662)],
name="Video frame count check",
modality=Modality.BEHAVIOR_VIDEOS,
stage=Stage.RAW,
metrics=[
QCMetric(name="video_1_num_frames", value=662, status_history=[s]),
QCMetric(name="video_2_num_frames", value=662, status_history=[s]),
],
notes="Pass when video_1_num_frames==video_2_num_frames",
)

eval2 = QCEvaluation(
evaluation_name="Probes present",
evaluation_modality=Modality.ECEPHYS,
evaluation_stage=Stage.RAW,
evaluation_status=[QCStatus(evaluator="Automated", timestamp=t, status=Status.PASS)],
qc_metrics=[
QCMetric(name="ProbeA_success", value=True),
QCMetric(name="ProbeB_success", value=True),
QCMetric(name="ProbeC_success", value=True),
name="Probes present",
modality=Modality.ECEPHYS,
stage=Stage.RAW,
metrics=[
QCMetric(name="ProbeA_success", value=True, status_history=[s]),
QCMetric(name="ProbeB_success", value=True, status_history=[s]),
QCMetric(name="ProbeC_success", value=True, status_history=[s]),
],
)

q = QualityControl(
overall_status=[QCStatus(evaluator="Automated", timestamp=t, status=Status.PASS)], evaluations=[eval0, eval1, eval2]
)
q = QualityControl(evaluations=[eval0, eval1, eval2])

# This is a special call that needs to be made to populate the .overall_status and .evaluation_status properties
# Note that the timestamp is set here because of how examples testing works, in general you should not set the
# timestamp manually

serialized = q.model_dump_json()
deserialized = QualityControl.model_validate_json(serialized)
Expand Down
Loading

0 comments on commit e880ccd

Please sign in to comment.