diff --git a/python/lsst/pipe/tasks/calibrateImage.py b/python/lsst/pipe/tasks/calibrateImage.py index a399eb0b2..d64ecc14d 100644 --- a/python/lsst/pipe/tasks/calibrateImage.py +++ b/python/lsst/pipe/tasks/calibrateImage.py @@ -55,7 +55,7 @@ class AllCentroidsFlaggedError(pipeBase.AlgorithmError): """ def __init__(self, n_sources, psf_shape_ixx, psf_shape_iyy, psf_shape_ixy, psf_size): msg = (f"All source centroids (out of {n_sources}) flagged during PSF fitting. " - "Original image PSF is likely unuseable; best-fit PSF shape parameters: " + "Original image PSF is likely unusable; best-fit PSF shape parameters: " f"Ixx={psf_shape_ixx}, Iyy={psf_shape_iyy}, Ixy={psf_shape_ixy}, size={psf_size}" ) super().__init__(msg) @@ -284,7 +284,7 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali ) psf_subtract_background = pexConfig.ConfigurableField( target=lsst.meas.algorithms.SubtractBackgroundTask, - doc="Task to perform intial background subtraction, before first detection pass.", + doc="Task to perform initial background subtraction, before first detection pass.", ) psf_detection = pexConfig.ConfigurableField( target=lsst.meas.algorithms.SourceDetectionTask, @@ -337,7 +337,7 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali star_background_peak_fraction = pexConfig.Field( dtype=float, default=0.01, - doc="The minimum number of footprints in the detection mask for star_background measuremen " + doc="The minimum number of footprints in the detection mask for star_background measurement. " "gets set to the maximum of this fraction of the detected peaks and the value set in " "config.star_background_min_footprints. If the number of footprints is less than the " "current minimum set, the detection threshold is iteratively increased until the " @@ -494,7 +494,7 @@ class CalibrateImageConfig(pipeBase.PipelineTaskConfig, pipelineConnections=Cali def setDefaults(self): super().setDefaults() - # Use a very broad PSF here, to throughly reject CRs. + # Use a very broad PSF here, to thoroughly reject CRs. # TODO investigation: a large initial psf guess may make stars look # like CRs for very good seeing images. self.install_simple_psf.fwhm = 4 @@ -534,7 +534,7 @@ def setDefaults(self): self.psf_measure_psf.psfDeterminer["psfex"].photometricFluxField = \ "base_CircularApertureFlux_12_0_instFlux" - # No extendeness information available: we need the aperture + # No extendedness information available: we need the aperture # corrections to determine that. self.measure_aperture_correction.sourceSelector["science"].doUnresolved = False self.measure_aperture_correction.sourceSelector["science"].flags.good = ["calib_psf_used"] @@ -785,7 +785,7 @@ def __init__(self, initial_stars_schema=None, **kwargs): # The final catalog will have calibrated flux columns, which we add to # the init-output schema by calibrating our zero-length catalog with an - # arbitrary dummy PhotoCalib. We also use this schema to initialze + # arbitrary dummy PhotoCalib. We also use this schema to initialize # the stars catalog in order to ensure it's the same even when we hit # an error (and write partial outputs) before calibrating the catalog # - note that calibrateCatalog will happily reuse existing output @@ -973,6 +973,7 @@ def run( result.exposure.detector.getId()) result.background = None + result.background_to_photometric_ratio = None summary_stat_catalog = None # Some exposure components are set to initial placeholder objects # while we try to bootstrap them. If we fail before we fit for them, @@ -988,10 +989,9 @@ def run( illumination_correction, ) - result.psf_stars_footprints, result.background, _, adaptive_det_res_struct = self._compute_psf( - result.exposure, + result.psf_stars_footprints, _, adaptive_det_res_struct = self._compute_psf( + result, id_generator, - background_to_photometric_ratio=result.background_to_photometric_ratio, ) have_fit_psf = True @@ -1014,7 +1014,7 @@ def run( self._measure_aperture_correction(result.exposure, result.psf_stars_footprints) result.psf_stars = result.psf_stars_footprints.asAstropy() # Run astrometry using PSF candidate stars. - # Update "the psf_stars" source cooordinates with the current wcs. + # Update "the psf_stars" source coordinates with the current wcs. afwTable.updateSourceCoords( result.exposure.wcs, sourceList=result.psf_stars_footprints, @@ -1055,7 +1055,7 @@ def run( self._match_psf_stars(result.psf_stars_footprints, result.stars_footprints, psfSigma=psfSigma) - # Update the "stars" source cooordinates with the current wcs. + # Update the "stars" source coordinates with the current wcs. afwTable.updateSourceCoords( result.exposure.wcs, sourceList=result.stars_footprints, @@ -1203,7 +1203,7 @@ def _apply_illumination_correction(self, exposure, background_flat, illumination return background_to_photometric_ratio - def _compute_psf(self, exposure, id_generator, background_to_photometric_ratio=None): + def _compute_psf(self, result, id_generator): """Find bright sources detected on an exposure and fit a PSF model to them, repairing likely cosmic rays before detection. @@ -1212,22 +1212,35 @@ def _compute_psf(self, exposure, id_generator, background_to_photometric_ratio=N Parameters ---------- - exposure : `lsst.afw.image.Exposure` - Exposure to detect and measure bright stars on. + result : `lsst.pipe.base.Struct` + Result struct that is modified to allow saving of partial outputs + for some failure conditions. Should contain at least the following + attributes: + + - exposure : `lsst.afw.image.Exposure` + Exposure to detect and measure bright stars on. + - background : `lsst.afw.math.BackgroundList` | `None` + Background that was fit to the exposure during detection. + - background_to_photometric_ratio : `lsst.afw.image.Image` | `None` + Image to convert photometric-flattened image to + background-flattened image. id_generator : `lsst.meas.base.IdGenerator` Object that generates source IDs and provides random seeds. - background_to_photometric_ratio : `lsst.afw.image.Image`, optional - Image to convert photometric-flattened image to - background-flattened image. Returns ------- sources : `lsst.afw.table.SourceCatalog` Catalog of detected bright sources. - background : `lsst.afw.math.BackgroundList` - Background that was fit to the exposure during detection. cell_set : `lsst.afw.math.SpatialCellSet` PSF candidates returned by the psf determiner. + adaptive_det_res_struct : `lsst.pipe.base.Struct` + Result struct from the adaptive threshold detection. + + Notes + ----- + This method modifies the exposure, background and + background_to_photometric_ratio attributes of the result struct + in-place. """ def log_psf(msg, addToMetadata=False): """Log the parameters of the psf and background, with a prepended @@ -1242,11 +1255,11 @@ def log_psf(msg, addToMetadata=False): Whether to add the final psf sigma value to the task metadata (the default is False). """ - position = exposure.psf.getAveragePosition() - sigma = exposure.psf.computeShape(position).getDeterminantRadius() - dimensions = exposure.psf.computeImage(position).getDimensions() - if background is not None: - median_background = np.median(background.getImage().array) + position = result.exposure.psf.getAveragePosition() + sigma = result.exposure.psf.computeShape(position).getDeterminantRadius() + dimensions = result.exposure.psf.computeImage(position).getDimensions() + if result.background is not None: + median_background = np.median(result.background.getImage().array) else: median_background = 0.0 self.log.info("%s sigma=%0.4f, dimensions=%s; median background=%0.2f", @@ -1254,16 +1267,16 @@ def log_psf(msg, addToMetadata=False): if addToMetadata: self.metadata["final_psf_sigma"] = sigma - self.log.info("First pass detection with Guassian PSF FWHM=%s pixels", + self.log.info("First pass detection with Gaussian PSF FWHM=%s pixels", self.config.install_simple_psf.fwhm) - self.install_simple_psf.run(exposure=exposure) + self.install_simple_psf.run(exposure=result.exposure) - background = self.psf_subtract_background.run( - exposure=exposure, - backgroundToPhotometricRatio=background_to_photometric_ratio, + result.background = self.psf_subtract_background.run( + exposure=result.exposure, + backgroundToPhotometricRatio=result.background_to_photometric_ratio, ).background log_psf("Initial PSF:") - self.psf_repair.run(exposure=exposure, keepCRs=True) + self.psf_repair.run(exposure=result.exposure, keepCRs=True) table = afwTable.SourceTable.make(self.psf_schema, id_generator.make_table_id_factory()) if not self.config.do_adaptive_threshold_detection: @@ -1272,15 +1285,15 @@ def log_psf(msg, addToMetadata=False): # measurement uses the most accurate background-subtraction. detections = self.psf_detection.run( table=table, - exposure=exposure, - background=background, - backgroundToPhotometricRatio=background_to_photometric_ratio, + exposure=result.exposure, + background=result.background, + backgroundToPhotometricRatio=result.background_to_photometric_ratio, ) else: initialThreshold = self.config.psf_detection.thresholdValue initialThresholdMultiplier = self.config.psf_detection.includeThresholdMultiplier adaptive_det_res_struct = self.psf_adaptive_threshold_detection.run( - table, exposure, + table, result.exposure, initialThreshold=initialThreshold, initialThresholdMultiplier=initialThresholdMultiplier, ) @@ -1290,8 +1303,8 @@ def log_psf(msg, addToMetadata=False): self.metadata["initial_psf_negative_footprint_count"] = detections.numNeg self.metadata["initial_psf_positive_peak_count"] = detections.numPosPeaks self.metadata["initial_psf_negative_peak_count"] = detections.numNegPeaks - self.psf_source_measurement.run(detections.sources, exposure) - psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources) + self.psf_source_measurement.run(detections.sources, result.exposure) + psf_result = self.psf_measure_psf.run(exposure=result.exposure, sources=detections.sources) # This 2nd round of PSF fitting has been deemed unnecessary (and # sometimes even causing harm), so it is being skipped for the @@ -1302,7 +1315,7 @@ def log_psf(msg, addToMetadata=False): # Replace the initial PSF with something simpler for the second # repair/detect/measure/measure_psf step: this can help it # converge. - self.install_simple_psf.run(exposure=exposure) + self.install_simple_psf.run(exposure=result.exposure) log_psf("Rerunning with simple PSF:") # TODO investigation: Should we only re-run repair here, to use the @@ -1312,18 +1325,17 @@ def log_psf(msg, addToMetadata=False): # for the post-psf_measure_psf step, since we only want to do # PsfFlux and GaussianFlux *after* we have a PSF? Maybe that's not # relevant once DM-39203 is merged? - self.psf_repair.run(exposure=exposure, keepCRs=True) + self.psf_repair.run(exposure=result.exposure, keepCRs=True) # Re-estimate the background during this detection step, so that # measurement uses the most accurate background-subtraction. detections = self.psf_detection.run( table=table, - exposure=exposure, - background=background, - backgroundToPhotometricRatio=background_to_photometric_ratio, + exposure=result.exposure, + background=result.background, + backgroundToPhotometricRatio=result.background_to_photometric_ratio, ) - self.psf_source_measurement.run(detections.sources, exposure) - psf_result = self.psf_measure_psf.run(exposure=exposure, sources=detections.sources) - + self.psf_source_measurement.run(detections.sources, result.exposure) + psf_result = self.psf_measure_psf.run(exposure=result.exposure, sources=detections.sources) self.metadata["simple_psf_positive_footprint_count"] = detections.numPos self.metadata["simple_psf_negative_footprint_count"] = detections.numNeg self.metadata["simple_psf_positive_peak_count"] = detections.numPosPeaks @@ -1331,13 +1343,13 @@ def log_psf(msg, addToMetadata=False): log_psf("Final PSF:", addToMetadata=True) # Final repair with final PSF, removing cosmic rays this time. - self.psf_repair.run(exposure=exposure) + self.psf_repair.run(exposure=result.exposure) # Final measurement with the CRs removed. - self.psf_source_measurement.run(detections.sources, exposure) + self.psf_source_measurement.run(detections.sources, result.exposure) # PSF is set on exposure; candidates are returned to use for # calibration flux normalization and aperture corrections. - return detections.sources, background, psf_result.cellSet, adaptive_det_res_struct + return detections.sources, psf_result.cellSet, adaptive_det_res_struct def _measure_aperture_correction(self, exposure, bright_sources): """Measure and set the ApCorrMap on the Exposure, using @@ -1851,7 +1863,7 @@ def _remeasure_star_background(self, result, background_to_photometric_ratio=Non result : `lsst.pipe.base.Struct` The modified result Struct with the new background subtracted. """ - # Restore the previously measured backgroud and remeasure it + # Restore the previously measured background and remeasure it # using an adaptive threshold detection iteration to ensure a # "Goldilocks Zone" for the fraction of detected pixels. backgroundOrig = result.background.clone() @@ -1992,7 +2004,7 @@ def _remeasure_star_background(self, result, background_to_photometric_ratio=Non bad_mask_planes) self.log.info("nIter = %d, thresh = %.2f: Fraction of pixels marked as DETECTED or " "DETECTED_NEGATIVE in star_background_detection = %.3f " - "(max is %.3f; min is %.3f) nFooprint = %d (current min is %d)", + "(max is %.3f; min is %.3f) nFootprint = %d (current min is %d)", nIter, starBackgroundDetectionConfig.thresholdValue, detected_fraction, maxDetFracForFinalBg, minDetFracForFinalBg, nFootprintTemp, minFootprints) diff --git a/tests/test_calibrateImage.py b/tests/test_calibrateImage.py index 4c819ec5c..dc27016fe 100644 --- a/tests/test_calibrateImage.py +++ b/tests/test_calibrateImage.py @@ -43,6 +43,7 @@ import lsst.meas.extensions.psfex import lsst.meas.base import lsst.meas.base.tests +import lsst.pipe.base as pipeBase import lsst.pipe.base.testUtils from lsst.pipe.tasks.calibrateImage import CalibrateImageTask, \ NoPsfStarsToStarsMatchError, AllCentroidsFlaggedError @@ -104,7 +105,7 @@ def setUp(self): # self.truth_exposure.variance.array[10, 10] = 100000/noise # Copy the truth exposure, because CalibrateImage modifies the input. - # Post-ISR ccds only contain: initial WCS, VisitInfo, filter + # Post-ISR images only contain: initial WCS, VisitInfo, filter self.exposure = afwImage.ExposureF(self.truth_exposure, deep=True) self.exposure.setWcs(self.truth_exposure.wcs) self.exposure.info.setVisitInfo(self.truth_exposure.visitInfo) @@ -113,6 +114,12 @@ def setUp(self): self.exposure.setFilter(lsst.afw.image.FilterLabel(physical='truth', band="truth")) self.exposure.metadata["LSST ISR FLAT APPLIED"] = True + # Set up a basic results struct to hold exposure attribute data + self.attributes = pipeBase.Struct() + self.attributes.exposure = self.exposure + self.attributes.background = None + self.attributes.background_to_photometric_ratio = None + # Test-specific configuration: self.config = CalibrateImageTask.ConfigClass() # We don't have many sources, so have to fit simpler models. @@ -140,7 +147,7 @@ def setUp(self): # Make a realistic id generator so that output catalog ids are useful. # NOTE: The id generator is used to seed the noise replacer during # measurement, so changes to values here can have subtle effects on - # the centroids and fluxes mesaured on the image, which might cause + # the centroids and fluxes measured on the image, which might cause # tests to fail. data_id = lsst.daf.butler.DataCoordinate.standardize( instrument="I", @@ -203,13 +210,14 @@ def _check_run(self, calibrate, result, expect_calibrated_pixels: bool = True, # PhotoCalib comparison is very approximate because we are basing this # comparison on just 2-3 stars. self.assertFloatsAlmostEqual(photo_calib.getCalibrationMean(), self.photo_calib, rtol=1e-2) - # Should have calibrated flux/magnitudes in the afw and astropy catalogs + # Should have calibrated flux/mags in the afw and astropy catalogs self.assertIn("slot_PsfFlux_flux", result.stars_footprints.schema) self.assertIn("slot_PsfFlux_mag", result.stars_footprints.schema) self.assertEqual(result.stars["slot_PsfFlux_flux"].unit, u.nJy) self.assertEqual(result.stars["slot_PsfFlux_mag"].unit, u.ABmag) - # Should have detected all S/N >= 10 sources plus 2 sky sources, whether 1 or 2 snaps. + # Should have detected all S/N >= 10 sources plus 2 sky sources, + # whether 1 or 2 snaps. self.assertEqual(len(result.stars), 6) # Did the psf flags get propagated from the psf_stars catalog? self.assertEqual(result.stars["calib_psf_used"].sum(), 3) @@ -243,7 +251,7 @@ def test_run(self): self._check_run(calibrate, result) - def test_run_adaptive_threshold_deteection(self): + def test_run_adaptive_threshold_detection(self): """Test that run() runs with adaptive threshold detection turned on. """ config = copy.copy(self.config) @@ -343,21 +351,21 @@ def test_run_no_astrom_errors(self): def test_compute_psf(self): """Test that our brightest sources are found by _compute_psf(), - that a PSF is assigned to the expopsure. + that a PSF is assigned to the exposure. """ calibrate = CalibrateImageTask(config=self.config) - psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator) + psf_stars, candidates, _ = calibrate._compute_psf(self.attributes, self.id_generator) # Catalog ids should be very large from this id generator. self.assertTrue(all(psf_stars['id'] > 1000000000)) # Background should have 3 elements: initial subtraction, and two from # re-estimation during the two detection passes. - self.assertEqual(len(background), 3) + self.assertEqual(len(self.attributes.background), 3) # Only the point-sources with S/N > 50 should be in this output. self.assertEqual(psf_stars["calib_psf_used"].sum(), 3) - # Sort in order of brightness, to easily compare with expected positions. + # Sort in brightness order, to easily compare with expected positions. psf_stars.sort(psf_stars.getPsfFluxSlot().getMeasKey()) for record, flux, center in zip(psf_stars[::-1], self.fluxes, self.centroids[self.fluxes > 50]): self.assertFloatsAlmostEqual(record.getX(), center[0], rtol=0.01) @@ -418,14 +426,14 @@ def test_measure_aperture_correction(self): exposure. """ calibrate = CalibrateImageTask(config=self.config) - psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator) + psf_stars, candidates, _ = calibrate._compute_psf(self.attributes, self.id_generator) # First check that the exposure doesn't have an ApCorrMap. self.assertIsNone(self.exposure.apCorrMap) calibrate._measure_aperture_correction(self.exposure, psf_stars) self.assertIsInstance(self.exposure.apCorrMap, afwImage.ApCorrMap) - # We know that there are 2 fields from the normalization, plus more from - # other configured plugins. + # We know that there are 2 fields from the normalization, plus more + # from other configured plugins. self.assertGreater(len(self.exposure.apCorrMap), 2) def test_find_stars(self): @@ -433,23 +441,23 @@ def test_find_stars(self): in the image and returns them in the output catalog. """ calibrate = CalibrateImageTask(config=self.config) - psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator) + psf_stars, candidates, _ = calibrate._compute_psf(self.attributes, self.id_generator) calibrate._measure_aperture_correction(self.exposure, psf_stars) - stars = calibrate._find_stars(self.exposure, background, self.id_generator) + stars = calibrate._find_stars(self.exposure, self.attributes.background, self.id_generator) # Catalog ids should be very large from this id generator. self.assertTrue(all(stars['id'] > 1000000000)) # Background should have 4 elements: 3 from compute_psf and one from # re-estimation during source detection. - self.assertEqual(len(background), 4) + self.assertEqual(len(self.attributes.background), 4) # Only 5 psf-like sources with S/N>10 should be in the output catalog, # plus two sky sources. self.assertEqual(len(stars), 6) self.assertTrue(stars.isContiguous()) - # Sort in order of brightness, to easily compare with expected positions. + # Sort in brightness order, to easily compare with expected positions. stars.sort(stars.getPsfFluxSlot().getMeasKey()) for record, flux, center in zip(stars[::-1], self.fluxes, self.centroids[self.fluxes > 50]): self.assertFloatsAlmostEqual(record.getX(), center[0], rtol=0.01) @@ -461,9 +469,9 @@ def test_astrometry(self): """ calibrate = CalibrateImageTask(config=self.config) calibrate.astrometry.setRefObjLoader(self.ref_loader) - psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator) + psf_stars, candidates, _ = calibrate._compute_psf(self.attributes, self.id_generator) calibrate._measure_aperture_correction(self.exposure, psf_stars) - stars = calibrate._find_stars(self.exposure, background, self.id_generator) + stars = calibrate._find_stars(self.exposure, self.attributes.background, self.id_generator) calibrate._fit_astrometry(self.exposure, stars) @@ -481,13 +489,13 @@ def test_photometry(self): calibrate = CalibrateImageTask(config=self.config) calibrate.astrometry.setRefObjLoader(self.ref_loader) calibrate.photometry.match.setRefObjLoader(self.ref_loader) - psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator) + psf_stars, candidates, _ = calibrate._compute_psf(self.attributes, self.id_generator) calibrate._measure_aperture_correction(self.exposure, psf_stars) - stars = calibrate._find_stars(self.exposure, background, self.id_generator) + stars = calibrate._find_stars(self.exposure, self.attributes.background, self.id_generator) calibrate._fit_astrometry(self.exposure, stars) stars, matches, meta, photoCalib = calibrate._fit_photometry(self.exposure, stars) - calibrate._apply_photometry(self.exposure, background) + calibrate._apply_photometry(self.exposure, self.attributes.background) # NOTE: With this test data, PhotoCalTask returns calibrationErr==0, # so we can't check that the photoCal error has been set. @@ -495,7 +503,7 @@ def test_photometry(self): # The exposure should be calibrated by the applied photoCalib, # and the background should be calibrated to match. uncalibrated = self.exposure.image.clone() - uncalibrated += background.getImage() + uncalibrated += self.attributes.background.getImage() uncalibrated /= self.photo_calib self.assertFloatsAlmostEqual(uncalibrated.array, self.truth_exposure.image.array, rtol=1e-2) # PhotoCalib on the exposure must be identically 1. @@ -522,9 +530,9 @@ def test_match_psf_stars(self): and candidates. """ calibrate = CalibrateImageTask(config=self.config) - psf_stars, background, candidates, _ = calibrate._compute_psf(self.exposure, self.id_generator) + psf_stars, candidates, _ = calibrate._compute_psf(self.attributes, self.id_generator) calibrate._measure_aperture_correction(self.exposure, psf_stars) - stars = calibrate._find_stars(self.exposure, background, self.id_generator) + stars = calibrate._find_stars(self.exposure, self.attributes.background, self.id_generator) # There should be no psf-related flags set at first. self.assertEqual(stars["calib_psf_candidate"].sum(), 0) @@ -542,7 +550,7 @@ def test_match_psf_stars(self): calibrate._match_psf_stars(psf_stars, stars) - # Check that the three brightest stars have the psf flags transfered + # Check that the three brightest stars have the psf flags transferred # from the psf_stars catalog by sorting in order of brightness. stars.sort(stars.getPsfFluxSlot().getMeasKey()) # sort() above leaves the catalog non-contiguous. @@ -654,8 +662,9 @@ def test_run_with_diffraction_spike_mask(self): self._check_run(calibrate, result) @mock.patch.dict(os.environ, {"SATTLE_URI_BASE": ""}) - def test_fail_on_sattle_miconfiguration(self): - """Test for failure if sattle is requested without appropriate configurations. + def test_fail_on_sattle_misconfiguration(self): + """Test for failure if sattle is requested without appropriate + configurations. """ self.config.run_sattle = True with self.assertRaises(pexConfig.FieldValidationError): @@ -663,7 +672,8 @@ def test_fail_on_sattle_miconfiguration(self): @mock.patch.dict(os.environ, {"SATTLE_URI_BASE": "fake_host:1234"}) def test_continue_on_sattle_failure(self): - """Processing should continue when sattle returns status codes other than 200. + """Processing should continue when sattle returns status codes other + than 200. """ response = MockResponse({}, 500, "internal sattle error") @@ -677,7 +687,8 @@ def test_continue_on_sattle_failure(self): @mock.patch.dict(os.environ, {"SATTLE_URI_BASE": "fake_host:1234"}) def test_sattle(self): - """Test for successful completion when sattle call returns successfully. + """Test for successful completion when sattle call returns + successfully. """ response = MockResponse({}, 200, "success")