Update tests to use new FgcmBuildFromIsolatedStarsTask.

tests/config/fgcmBuildFromIsolatedStarsHsc.py

@@ -0,0 +1,33 @@
+import os
+from lsst.obs.hsc.hscFilters import HSC_FILTER_DEFINITIONS
+
+# This override is to be consistent with the old tests
+config.referenceCCD = 13
+# The filterMap and bands are for the small subset of bands used in the tests
+config.physicalFilterMap = {'HSC-G': 'g', 'HSC-R': 'r', 'HSC-I': 'i'}
+config.requiredBands = ['r', 'i']
+config.primaryBands = ['i']
+# The coarseNside is set appropriate to the area of the test data
+config.coarseNside = 64
+# We have only a few visits, so checkpointing is more frequent
+config.nVisitsPerCheckpoint = 5
+# The tests are done with only the brightest reference stars
+config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.minimum = 50.0
+# The test data do not have persisted backgrounds, so don't use them
+config.doModelErrorsWithBackground = False
+
+config.minPerBand = 2
+config.connections.ref_cat = "ps1_pv3_3pi_20170110"
+# This is set low enough to ensure that this code path is triggered.
+config.densityCutMaxPerPixel = 200
+
+configDir = os.path.join(os.path.dirname(__file__))
+config.physicalFilterMap = HSC_FILTER_DEFINITIONS.physical_to_band
+config.doSubtractLocalBackground = True
+config.sourceSelector["science"].flags.bad.append("localBackground_flag")
+config.fgcmLoadReferenceCatalog.load(os.path.join(configDir, 'filterMap.py'))
+config.fgcmLoadReferenceCatalog.applyColorTerms = True
+config.fgcmLoadReferenceCatalog.colorterms.load(os.path.join(configDir, 'colorterms.py'))
+config.fgcmLoadReferenceCatalog.referenceSelector.doSignalToNoise = True
+config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.fluxField = 'i_flux'
+config.fgcmLoadReferenceCatalog.referenceSelector.signalToNoise.errField = 'i_fluxErr'

tests/config/fgcmBuildStarsTableHsc.py

@@ -26,6 +26,7 @@
 configDir = os.path.join(os.path.dirname(__file__))
 config.physicalFilterMap = HSC_FILTER_DEFINITIONS.physical_to_band
 config.doSubtractLocalBackground = True
+config.sourceSelector["science"].flags.bad.append("localBackground_flag")
 config.fgcmLoadReferenceCatalog.load(os.path.join(configDir, 'filterMap.py'))
 config.fgcmLoadReferenceCatalog.applyColorTerms = True
 config.fgcmLoadReferenceCatalog.colorterms.load(os.path.join(configDir, 'colorterms.py'))

tests/fgcmcalTestBase.py

@@ -119,7 +119,6 @@ def _runPipeline(self, repo, pipelineFile, queryString='',
         butler = SimplePipelineExecutor.prep_butler(repo,
                                                     inputs=inputCollections,
                                                     output=outputCollection)
-
         pipeline = Pipeline.fromFile(pipelineFile)
         for taskName, fileList in configFiles.items():
             for fileName in fileList:
@@ -278,6 +277,58 @@ def _testFgcmBuildStarsTable(self, instName, testName, queryString, visits, nSta
                              instrument=instName)
         self.assertEqual(len(starObs), nObs)
 
+    def _testFgcmBuildFromIsolatedStars(self, instName, testName, queryString, visits, nStar, nObs):
+        """Test running of FgcmBuildFromIsolatedStarsTask.
+
+        Parameters
+        ----------
+        instName : `str`
+            Short name of the instrument
+        testName : `str`
+            Base name of the test collection
+        queryString : `str`
+            Query to send to the pipetask.
+        visits : `list`
+            List of visits to calibrate
+        nStar : `int`
+            Number of stars expected
+        nObs : `int`
+            Number of observations of stars expected
+        """
+        instCamel = instName.title()
+
+        configFiles = {'fgcmBuildFromIsolatedStars': [
+            os.path.join(ROOT,
+                         'config',
+                         f'fgcmBuildFromIsolatedStars{instCamel}.py')
+        ]}
+        outputCollection = f'{instName}/{testName}/buildstars'
+
+        self._runPipeline(self.repo,
+                          os.path.join(ROOT,
+                                       'pipelines',
+                                       'fgcmBuildFromIsolatedStars%s.yaml' % (instCamel)),
+                          configFiles=configFiles,
+                          inputCollections=[f'{instName}/{testName}/lut',
+                                            'refcats/gen2'],
+                          outputCollection=outputCollection,
+                          queryString=queryString,
+                          registerDatasetTypes=True)
+
+        butler = dafButler.Butler(self.repo)
+
+        visitCat = butler.get('fgcmVisitCatalog', collections=[outputCollection],
+                              instrument=instName)
+        self.assertEqual(len(visits), len(visitCat))
+
+        starIds = butler.get('fgcm_star_ids', collections=[outputCollection],
+                             instrument=instName)
+        self.assertEqual(len(starIds), nStar)
+
+        starObs = butler.get('fgcm_star_observations', collections=[outputCollection],
+                             instrument=instName)
+        self.assertEqual(len(starObs), nObs)
+
     def _testFgcmFitCycle(self, instName, testName, cycleNumber,
                           nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots,
                           skipChecks=False, extraConfig=None):
@@ -440,11 +491,6 @@ def _testFgcmOutputProducts(self, instName, testName,
         rawStars = butler.get('fgcmStandardStars' + config.connections.cycleNumber,
                               collections=[inputCollection], instrument=instName)
 
-        candRatio = (rawStars['npsfcand'][:, 0].astype(np.float64)
-                     / rawStars['ntotal'][:, 0].astype(np.float64))
-        self.assertFloatsAlmostEqual(candRatio.min(), 0.0)
-        self.assertFloatsAlmostEqual(candRatio.max(), 1.0)
-
         # Test the fgcm_photoCalib output
         zptCat = butler.get('fgcmZeropoints' + config.connections.cycleNumber,
                             collections=[inputCollection], instrument=instName)
@@ -609,9 +655,11 @@ def _testFgcmMultiFit(self, instName, testName, queryString, visits, zpOffsets):
         """
         instCamel = instName.title()
 
-        configFiles = {'fgcmBuildStarsTable': [os.path.join(ROOT,
-                                                            'config',
-                                                            f'fgcmBuildStarsTable{instCamel}.py')],
+        configFiles = {'fgcmBuildFromIsolatedStars': [
+            os.path.join(ROOT,
+                         'config',
+                         f'fgcmBuildFromIsolatedStars{instCamel}.py'
+                         )],
                        'fgcmFitCycle': [os.path.join(ROOT,
                                                      'config',
                                                      f'fgcmFitCycle{instCamel}.py')],

tests/pipelines/fgcmBuildFromIsolatedStarsHsc.yaml

@@ -0,0 +1,5 @@
+description: Build fgcm input stars
+instrument: lsst.obs.subaru.HyperSuprimeCam
+tasks:
+  fgcmBuildFromIsolatedStars:
+    class: lsst.fgcmcal.fgcmBuildFromIsolatedStars.FgcmBuildFromIsolatedStarsTask

tests/pipelines/fgcmFullPipelineHsc.yaml

@@ -1,8 +1,8 @@
 description: Full fgcmcal pipeline
 instrument: lsst.obs.subaru.HyperSuprimeCam
 tasks:
-  fgcmBuildStarsTable:
-    class: lsst.fgcmcal.fgcmBuildStarsTable.FgcmBuildStarsTableTask
+  fgcmBuildFromIsolatedStars:
+    class: lsst.fgcmcal.fgcmBuildFromIsolatedStars.FgcmBuildFromIsolatedStarsTask
   fgcmFitCycle:
     class: lsst.fgcmcal.fgcmFitCycle.FgcmFitCycleTask
     config:

tests/test_fgcmcal_hsc.py

@@ -29,8 +29,8 @@
 import tempfile
 import numpy as np
 
-import matplotlib
-matplotlib.use("Agg")
+# import matplotlib
+# matplotlib.use("Agg")
 
 import lsst.utils  # noqa: E402
 import lsst.pipe.tasks  # noqa: E402
@@ -67,7 +67,12 @@ def setUpClass(cls):
                               os.path.join(cls.dataDir, 'hsc/repo'),
                               os.path.join(cls.dataDir, 'hsc', 'exports.yaml'))
 
-    def test_fgcmcalPipeline(self):
+    def test_fgcmcalPipelineBuildFromTable(self):
+        """Test running the full pipeline, using older association code.
+
+        This test uses the FgcmBuildStarsFromTableTask instead of the new
+        FgcmBuildFromIsolatedStarsTask.
+        """
         # Set numpy seed for stability
         np.random.seed(seed=1000)
 
@@ -99,6 +104,73 @@ def test_fgcmcalPipeline(self):
         nStdStars = 235
         nPlots = 47
 
+        # We need an extra config file to turn off parquet format.
+        extraConfigFile = os.path.join(self.testDir, "turn_off_parquet.py")
+        with open(extraConfigFile, "w") as f:
+            f.write("config.useParquetCatalogFormat = False\n")
+
+        self._testFgcmFitCycle(instName, testName,
+                               0, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True,
+                               extraConfig=extraConfigFile)
+        self._testFgcmFitCycle(instName, testName,
+                               1, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True,
+                               extraConfig=extraConfigFile)
+
+        # We need to create an extra config file to turn on "sub-ccd gray" for testing.
+        # We also want to exercise the code path setting useExposureReferenceOffset = False.
+        extraConfigFile = os.path.join(self.testDir, "cycle03_patch_config.py")
+        with open(extraConfigFile, "w") as f:
+            f.write("config.useParquetCatalogFormat = False\n")
+            f.write("config.isFinalCycle = True\n")
+            f.write("config.ccdGraySubCcdDict = {'g': True, 'r': True, 'i': True}\n")
+            f.write("config.useExposureReferenceOffset = False")
+
+        self._testFgcmFitCycle(instName, testName,
+                               2, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots,
+                               extraConfig=extraConfigFile)
+
+    def test_fgcmcalPipeline(self):
+        """Test running the full pipeline, using new isolated star association code.
+
+        This test uses the FgcmBuildFromIsolatedStarsTask instead of the old
+        FgcmBuildStarsFromTableTask.
+        """
+        # Set numpy seed for stability
+        np.random.seed(seed=1000)
+
+        instName = 'HSC'
+        testName = 'testfgcmcalpipe'
+
+        nBand = 3
+        i0Std = np.array(I0STD)
+        i10Std = np.array(I10STD)
+        i0Recon = np.array(I0RECON)
+        i10Recon = np.array(I10RECON)
+
+        self._testFgcmMakeLut(instName, testName,
+                              nBand, i0Std, i0Recon, i10Std, i10Recon)
+
+        visits = [34648, 34690, 34714, 34674, 34670, 36140, 35892, 36192, 36260, 36236]
+
+        nStar = 295
+        nObs = 1808
+
+        self._testFgcmBuildFromIsolatedStars(
+            instName,
+            testName,
+            "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I')",
+            visits,
+            nStar,
+            nObs,
+        )
+
+        nZp = 1120
+        nGoodZp = 27
+        nOkZp = 27
+        nBadZp = 1093
+        nStdStars = 227
+        nPlots = 47
+
         self._testFgcmFitCycle(instName, testName,
                                0, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots, skipChecks=True)
         self._testFgcmFitCycle(instName, testName,
@@ -116,8 +188,8 @@ def test_fgcmcalPipeline(self):
                                2, nZp, nGoodZp, nOkZp, nBadZp, nStdStars, nPlots,
                                extraConfig=extraConfigFile)
 
-        zpOffsets = np.array([-0.0008051003096625209,
-                              0.0072303167544305325])
+        zpOffsets = np.array([-0.001461041159927845,
+                              0.005496968515217304])
 
         self._testFgcmOutputProducts(instName, testName,
                                      zpOffsets, 36236, 87, 'i', 1)
@@ -142,8 +214,8 @@ def test_fgcmcalMultipleFitPipeline(self):
 
         # These are slightly different from above due to the configuration change
         # mid-way in the separate fits.
-        zpOffsets = np.array([-0.0006988655077293515,
-                              0.004102597013115883])
+        zpOffsets = np.array([-0.0027262186631560326,
+                              0.004101278726011515])
 
         self._testFgcmMultiFit(instName, testName,
                                "physical_filter IN ('HSC-G', 'HSC-R', 'HSC-I')",