diff --git a/python/lsst/pipe/tasks/calibrateImage.py b/python/lsst/pipe/tasks/calibrateImage.py index c9064837b..06732364b 100644 --- a/python/lsst/pipe/tasks/calibrateImage.py +++ b/python/lsst/pipe/tasks/calibrateImage.py @@ -23,10 +23,13 @@ "AllCentroidsFlaggedError"] import numpy as np +from astropy.time import Time +import requests +import os import lsst.afw.table as afwTable import lsst.afw.image as afwImage -from lsst.ip.diffim.utils import evaluateMaskFraction +from lsst.ip.diffim.utils import evaluateMaskFraction, populate_sattle_visit_cache import lsst.meas.algorithms import lsst.meas.algorithms.installGaussianPsf import lsst.meas.algorithms.measureApCorr @@ -384,6 +387,21 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali doc="If True, include astrometric errors in the output catalog.", ) + run_sattle = pexConfig.Field( + dtype=bool, + default=False, + doc="If True, the sattle service will populate a cache for later use " + "in ip_diffim.detectAndMeasure alert verification." + ) + + sattle_historical = pexConfig.Field( + dtype=bool, + default=False, + doc="If re-running a pipeline that requires sattle, this should be set" + "to True. This will populate sattle's cache with the historic data" + "closest in time to the exposure.", + ) + def setDefaults(self): super().setDefaults() @@ -565,6 +583,12 @@ def validate(self): "doApplyFlatBackgroundRatio=True if do_illumination_correction=True." ) + if self.config.run_sattle: + self.sattle_uri = os.getenv("SATTLE_URI") + if not self.sattle_uri: + raise ValueError("Sattle uri environment variable not set.") + + class CalibrateImageTask(pipeBase.PipelineTask): """Compute the PSF, aperture corrections, astrometric and photometric @@ -901,6 +925,18 @@ def run( result.applied_photo_calib = photo_calib else: result.applied_photo_calib = None + + if self.config.run_sattle: + # send boresight and timing information to sattle so the cache + # is populated by the time we reach ip_diffim detectAndMeasure. + try: + populate_sattle_visit_cache(result.exposure.getInfo().getVisitInfo(), + historical=self.config.sattle_historical) + self.log.info('Successfully loaded sattle visit cache') + except Exception as e: + # log but do not stop processing if sattle caching fails + self.log.error(e) + return result def _apply_illumination_correction(self, exposure, background_flat, illumination_correction): diff --git a/tests/test_calibrateImage.py b/tests/test_calibrateImage.py index cf02eb167..211cd4513 100644 --- a/tests/test_calibrateImage.py +++ b/tests/test_calibrateImage.py @@ -22,6 +22,7 @@ import unittest from unittest import mock import tempfile +import json as json_package import astropy.units as u from astropy.coordinates import SkyCoord @@ -120,6 +121,7 @@ def setUp(self): # We don't have many test points, so can't match on complicated shapes. self.config.astrometry.sourceSelector["science"].flags.good = [] self.config.astrometry.matcher.numPointsForShape = 3 + self.config.run_sattle = False # ApFlux has more noise than PsfFlux (the latter unrealistically small # in this test data), so we need to do magnitude rejection at higher # sigma, otherwise we can lose otherwise good sources. @@ -584,6 +586,65 @@ def test_calibrate_image_illumcorr(self): self.assertIn(key, result.exposure.metadata) self.assertEqual(result.exposure.metadata[key], True) + @staticmethod + def _mocked_requests_put(url, json=None): + data = json + + class MockResponse: + def __init__(self, json_data, status_code, text): + self.content = json_package.dumps(json_data) + self.status_code = status_code + self.text = text + + def json(self): + return self.json_data + + # sattle returns an error code + if data['visit_id'] == 42: + return MockResponse(None, 500, "general error") + elif data['visit_id'] == 99: + return MockResponse(None, 200, + "Success") + + return MockResponse(None, 404, "Nothing here") + + def test_fail_on_sattle_miconfiguration(self): + """Test for failure if sattle is requested without appropriate configurations. + """ + self.config.run_sattle = True + calibrate = CalibrateImageTask(config=self.config) + calibrate.astrometry.setRefObjLoader(self.ref_loader) + calibrate.photometry.match.setRefObjLoader(self.ref_loader) + with self.assertRaises(RuntimeError): + calibrate.run(exposures=self.exposure) + + @mock.patch('lsst.pipe.tasks.calibrateImage.requests.put', side_effect=_mocked_requests_put) + def test_warn_on_sattle_failure(self, mock_put): + """Test for a warning when sattle returns status codes other than 200. + """ + self.config.run_sattle = True + self.config.sattle_port = '9999' + self.config.sattle_host = 'fake_host' + self.exposure.info.id = 42 + calibrate = CalibrateImageTask(config=self.config) + calibrate.astrometry.setRefObjLoader(self.ref_loader) + calibrate.photometry.match.setRefObjLoader(self.ref_loader) + with self.assertWarns(Warning): + calibrate.run(exposures=self.exposure) + + @mock.patch('lsst.pipe.tasks.calibrateImage.requests.put', side_effect=_mocked_requests_put) + def test_sattle(self, mock_put): + """Test that run() returns reasonable values to be butler put. + """ + self.config.run_sattle = True + self.config.sattle_port = '9999' + self.config.sattle_host = 'fake_host' + self.exposure.info.id = 99 + calibrate = CalibrateImageTask(config=self.config) + calibrate.astrometry.setRefObjLoader(self.ref_loader) + calibrate.photometry.match.setRefObjLoader(self.ref_loader) + calibrate.run(exposures=self.exposure) + class CalibrateImageTaskRunQuantumTests(lsst.utils.tests.TestCase): """Tests of ``CalibrateImageTask.runQuantum``, which need a test butler,