From 570b315e55f085c8f2d3c03e65142bc76dbe8b4e Mon Sep 17 00:00:00 2001 From: Bryce Kalmbach Date: Sun, 19 May 2024 06:07:12 +0000 Subject: [PATCH 1/3] Use the binaryFlagArray to inform which zernikes raw values to keep in combineZernikesSigmaClipTask. --- python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py b/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py index c3d857f6..2989ed63 100644 --- a/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py +++ b/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py @@ -81,6 +81,6 @@ def combineZernikes(self, zernikeArray): np.isnan(sigArray[:, : self.maxZernClip]), axis=1 ).astype(int) # Identify which rows to use when calculating final mean - keepIdx = ~np.any(np.isnan(sigArray), axis=1) + keepIdx = ~np.array(binaryFlagArray, dtype=bool) return np.mean(zernikeArray[keepIdx], axis=0), binaryFlagArray From 02a32c9930c65c091f463990120879d7af76ca41 Mon Sep 17 00:00:00 2001 From: Bryce Kalmbach Date: Sun, 19 May 2024 08:07:36 +0000 Subject: [PATCH 2/3] Update tests to discern whether flags and mean use the same indices. --- tests/task/test_combineZernikesSigmaClipTask.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/task/test_combineZernikesSigmaClipTask.py b/tests/task/test_combineZernikesSigmaClipTask.py index 8e2ee18f..72cc9c4d 100644 --- a/tests/task/test_combineZernikesSigmaClipTask.py +++ b/tests/task/test_combineZernikesSigmaClipTask.py @@ -71,7 +71,7 @@ def testCombineZernikes(self): # Test that zernikes higher than maxZernClip don't remove # a row from the final averaging zernikeArray[0, 3:] += 100.0 - zernikeArray[51, 3:] -= 100.0 + zernikeArray[49, 3:] -= 100.0 # Revert the change in the 100th row from previous test trueFlags[100] = 1 combinedZernikes, testFlags = self.task.combineZernikes(zernikeArray) @@ -80,13 +80,15 @@ def testCombineZernikes(self): self.assertTrue(isinstance(testFlags[0], numbers.Integral)) # Test that changing the maxZernClip parameter does change - # if a row is removed from the final result + # whether a row is removed from the final result + zernikeArray[50, 3:] += 100.0 + zernikeArray[51, 3:] -= 100.0 self.config.maxZernClip = 5 self.task = CombineZernikesSigmaClipTask(config=self.config) combinedZernikes, testFlags = self.task.combineZernikes(zernikeArray) np.testing.assert_array_equal(np.ones(10) * 2.0, combinedZernikes) trueFlags[0] = 1 - trueFlags[51] = 1 + trueFlags[49:52] = 1 np.testing.assert_array_equal(trueFlags, testFlags) self.assertTrue(isinstance(testFlags[0], numbers.Integral)) From b9506c677c437395e5c1206398c35565c4a51e1d Mon Sep 17 00:00:00 2001 From: Bryce Kalmbach Date: Mon, 20 May 2024 21:54:28 +0000 Subject: [PATCH 3/3] Add while loop to reduce maxZernClip by 1 if no donuts pass sigma clipping within first maxZernClip values. --- doc/versionHistory.rst | 9 +++++++ .../wep/task/combineZernikesSigmaClipTask.py | 25 ++++++++++++++++--- .../task/test_combineZernikesSigmaClipTask.py | 18 +++++++++++++ 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/doc/versionHistory.rst b/doc/versionHistory.rst index 7ba087a1..1fb38eda 100644 --- a/doc/versionHistory.rst +++ b/doc/versionHistory.rst @@ -6,6 +6,15 @@ Version History ################## +.. _lsst.ts.wep-9.5.5: + +------------- +9.5.5 +------------- + +* Correct indices used to calculate Zernike average. +* Update tests to discern whether flags and mean use the same indices. + .. _lsst.ts.wep-9.5.4: ------------- diff --git a/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py b/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py index 2989ed63..8a034fe3 100644 --- a/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py +++ b/python/lsst/ts/wep/task/combineZernikesSigmaClipTask.py @@ -77,10 +77,29 @@ def combineZernikes(self, zernikeArray): # Create a binary flag array that indicates # donuts have outlier values. This array is 1 if # it has any outlier values. - binaryFlagArray = np.any( - np.isnan(sigArray[:, : self.maxZernClip]), axis=1 - ).astype(int) + # If all available donuts have a clipped value in the + # first maxZernClip coefficients then reduce maxZernClip by 1 + # until we get one that passes. + numRejected = len(sigArray) + effMaxZernClip = self.maxZernClip + 1 + + while numRejected == len(sigArray): + effMaxZernClip -= 1 + binaryFlagArray = np.any( + np.isnan(sigArray[:, :effMaxZernClip]), axis=1 + ).astype(int) + numRejected = np.sum(binaryFlagArray) # Identify which rows to use when calculating final mean keepIdx = ~np.array(binaryFlagArray, dtype=bool) + self.log.info( + f"MaxZernClip config: {self.maxZernClip}. MaxZernClip used: {effMaxZernClip}." + ) + if effMaxZernClip < self.maxZernClip: + self.log.warning( + f"EffMaxZernClip ({effMaxZernClip}) was less than MaxZernClip config ({self.maxZernClip})." + ) + self.metadata["maxZernClip"] = self.maxZernClip + self.metadata["effMaxZernClip"] = effMaxZernClip + return np.mean(zernikeArray[keepIdx], axis=0), binaryFlagArray diff --git a/tests/task/test_combineZernikesSigmaClipTask.py b/tests/task/test_combineZernikesSigmaClipTask.py index 72cc9c4d..29777620 100644 --- a/tests/task/test_combineZernikesSigmaClipTask.py +++ b/tests/task/test_combineZernikesSigmaClipTask.py @@ -92,6 +92,24 @@ def testCombineZernikes(self): np.testing.assert_array_equal(trueFlags, testFlags) self.assertTrue(isinstance(testFlags[0], numbers.Integral)) + def testCombineZernikesEffectiveMaxZernClip(self): + testWhileZernikeArray = np.ones((3, 10)) + testWhileZernikeArray[0, 4] = 3 + testWhileZernikeArray[1, 3] = 3 + testWhileZernikeArray[2, 2] = 3 + + # Test that changing the maxZernClip parameter does change + # whether a row is removed from the final result + self.config.maxZernClip = 6 + self.task = CombineZernikesSigmaClipTask(config=self.config) + combinedZernikes, testFlags = self.task.combineZernikes(testWhileZernikeArray) + np.testing.assert_array_equal(testWhileZernikeArray[0], combinedZernikes) + self.assertEqual(self.task.metadata["maxZernClip"], 6) + self.assertEqual(self.task.metadata["effMaxZernClip"], 4) + trueFlags = np.array([0, 1, 1]) + np.testing.assert_array_equal(trueFlags, testFlags) + self.assertTrue(isinstance(testFlags[0], numbers.Integral)) + def testTaskRun(self): zernikeArray, trueFlags = self.prepareTestData() combinedZernikesStruct = self.task.run(zernikeArray)