From 5034f2277b295569e418608a34c688768a8a4a0d Mon Sep 17 00:00:00 2001 From: Brianna Smart Date: Wed, 2 Jul 2025 10:04:14 -0700 Subject: [PATCH 1/2] Add sattle service to pipe_task --- python/lsst/pipe/tasks/calibrateImage.py | 60 +++++++++++++++++++++++ tests/test_calibrateImage.py | 61 ++++++++++++++++++++++++ 2 files changed, 121 insertions(+) diff --git a/python/lsst/pipe/tasks/calibrateImage.py b/python/lsst/pipe/tasks/calibrateImage.py index c9064837b..8dff3ba95 100644 --- a/python/lsst/pipe/tasks/calibrateImage.py +++ b/python/lsst/pipe/tasks/calibrateImage.py @@ -23,6 +23,9 @@ "AllCentroidsFlaggedError"] import numpy as np +from astropy.time import Time +import requests +import os import lsst.afw.table as afwTable import lsst.afw.image as afwImage @@ -384,6 +387,35 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali doc="If True, include astrometric errors in the output catalog.", ) + run_sattle = pexConfig.Field( + dtype=bool, + default=True, + doc="If True, sattle service will populate catalog for use in " + "ip_diffim.detectAndMeasure alert verification." + ) + + sattle_historical = sattle_host = pexConfig.Field( + dtype=bool, + default=False, + doc="If re-running a pipeline that requires sattle, this should be set" + "to True. This will population sattles cache with the historic data" + "closest in time to the exposure.", + ) + + sattle_host = pexConfig.Field( + dtype=str, + default=os.getenv("SATTLE_HOST"), + doc="The host name for the sattle API.", + optional=True, + ) + + sattle_port = pexConfig.Field( + dtype=int, + default=os.getenv("SATTLE_PORT"), + doc="Port for the sattle API.", + optional=True, + ) + def setDefaults(self): super().setDefaults() @@ -901,6 +933,34 @@ def run( result.applied_photo_calib = photo_calib else: result.applied_photo_calib = None + + if self.config.run_sattle: + if not self.config.sattle_host or not self.config.sattle_port: + raise RuntimeError("Sattle filtering is on but service endpoints not set.") + + visit_id = result.exposure.getInfo().getVisitInfo().id + visit_date = Time(result.exposure.getInfo().getVisitInfo().getDate().toPython()).jd + exposure_time_days = result.exposure.getInfo().getVisitInfo().getExposureTime() / 86400.0 + exposure_end_jd = visit_date + exposure_time_days / 2.0 + exposure_start_jd = visit_date - exposure_time_days / 2.0 + boresight_ra = result.exposure.getInfo().getVisitInfo().boresightRaDec[ + 0].asDegrees() + boresight_dec = result.exposure.getInfo().getVisitInfo().boresightRaDec[ + 1].asDegrees() + + r = requests.put(f'{self.config.sattle_host}:{self.config.sattle_port}' + f'/visit_cache', json={"visit_id": visit_id, + "exposure_start_mjd": exposure_start_jd, + "exposure_end_mjd": exposure_end_jd, + "boresight_ra": boresight_ra, + "boresight_dec": boresight_dec, + "historical": self.config.sattle_historical}) + + if r.status_code != 200: + self.log.warning(f'Sattle cache returned {r.status_code}: {r.text}') + else: + self.log.info('Successfully loaded sattle visit cache') + return result def _apply_illumination_correction(self, exposure, background_flat, illumination_correction): diff --git a/tests/test_calibrateImage.py b/tests/test_calibrateImage.py index cf02eb167..634a7a0f0 100644 --- a/tests/test_calibrateImage.py +++ b/tests/test_calibrateImage.py @@ -22,6 +22,7 @@ import unittest from unittest import mock import tempfile +import json as json_package import astropy.units as u from astropy.coordinates import SkyCoord @@ -120,6 +121,7 @@ def setUp(self): # We don't have many test points, so can't match on complicated shapes. self.config.astrometry.sourceSelector["science"].flags.good = [] self.config.astrometry.matcher.numPointsForShape = 3 + self.config.run_sattle = False # ApFlux has more noise than PsfFlux (the latter unrealistically small # in this test data), so we need to do magnitude rejection at higher # sigma, otherwise we can lose otherwise good sources. @@ -584,6 +586,65 @@ def test_calibrate_image_illumcorr(self): self.assertIn(key, result.exposure.metadata) self.assertEqual(result.exposure.metadata[key], True) + @staticmethod + def _mocked_requests_put(url, json=None): + data = json + + class MockResponse: + def __init__(self, json_data, status_code, text): + self.content = json_package.dumps(json_data) + self.status_code = status_code + self.text = text + + def json(self): + return self.json_data + + # sattle returns an error code + if data['visit_id'] == 42: + return MockResponse(None, 500, "general error") + elif data['visit_id'] == 99: + return MockResponse(None, 200, + "Success") + + return MockResponse(None, 404, "Nothing here") + + def test_fail_on_sattle_miconfiguration(self): + """Test for failure if sattle is requested without appropriate configurations. + """ + self.config.run_sattle = True + calibrate = CalibrateImageTask(config=self.config) + calibrate.astrometry.setRefObjLoader(self.ref_loader) + calibrate.photometry.match.setRefObjLoader(self.ref_loader) + with self.assertRaises(RuntimeError): + calibrate.run(exposures=self.exposure) + + @mock.patch('lsst.pipe.tasks.calibrateImage.requests.put', side_effect=_mocked_requests_put) + def test_warn_on_sattle_failure(self, mock_put): + """Test for a warning when sattle returns status codes other than 200. + """ + self.config.run_sattle = True + self.config.sattle_port = 9999 + self.config.sattle_host = 'fake_host' + self.exposure.info.id = 42 + calibrate = CalibrateImageTask(config=self.config) + calibrate.astrometry.setRefObjLoader(self.ref_loader) + calibrate.photometry.match.setRefObjLoader(self.ref_loader) + with self.assertWarns(Warning): + calibrate.run(exposures=self.exposure) + + @mock.patch('lsst.pipe.tasks.calibrateImage.requests.put', side_effect=_mocked_requests_put) + def test_sattle(self, mock_put): + """Test that run() returns reasonable values to be butler put. + """ + self.config.run_sattle = True + self.config.sattle_port = 9999 + self.config.sattle_host = 'fake_host' + self.exposure.info.id = 99 + calibrate = CalibrateImageTask(config=self.config) + calibrate.astrometry.setRefObjLoader(self.ref_loader) + calibrate.photometry.match.setRefObjLoader(self.ref_loader) + calibrate.run(exposures=self.exposure) + class CalibrateImageTaskRunQuantumTests(lsst.utils.tests.TestCase): """Tests of ``CalibrateImageTask.runQuantum``, which need a test butler, From 7eebb386bfd5ae980abcf6973aae97f61c979dc3 Mon Sep 17 00:00:00 2001 From: Eric Bellm Date: Sun, 13 Jul 2025 22:44:53 -0700 Subject: [PATCH 2/2] sattle should default false --- python/lsst/pipe/tasks/calibrateImage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/pipe/tasks/calibrateImage.py b/python/lsst/pipe/tasks/calibrateImage.py index 8dff3ba95..225ab620d 100644 --- a/python/lsst/pipe/tasks/calibrateImage.py +++ b/python/lsst/pipe/tasks/calibrateImage.py @@ -389,7 +389,7 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali run_sattle = pexConfig.Field( dtype=bool, - default=True, + default=False, doc="If True, sattle service will populate catalog for use in " "ip_diffim.detectAndMeasure alert verification." )