Merge pull request #116 from lsst/tickets/DM-42285

DM-42285: Update for use with future numpy (1.25+) and tighten down loose ends.

python/lsst/fgcmcal/fgcmBuildFromIsolatedStars.py

@@ -35,7 +35,7 @@
 from smatch.matcher import Matcher
 from astropy.table import Table, vstack
 
-from fgcm.fgcmUtilities import objFlagDict
+from fgcm.fgcmUtilities import objFlagDict, histogram_rev_sorted
 
 import lsst.pex.config as pexConfig
 import lsst.pipe.base as pipeBase
@@ -633,8 +633,9 @@ def _density_downsample(self, fgcm_obj, star_obs):
         ipnest = hpg.angle_to_pixel(self.config.densityCutNside, fgcm_obj["ra"], fgcm_obj["dec"])
         # Use the esutil.stat.histogram function to pull out the histogram of
         # grouped pixels, and the rev_indices which describes which inputs
-        # are grouped together.
-        hist, rev_indices = esutil.stat.histogram(ipnest, rev=True)
+        # are grouped together.  The fgcm histogram_rev_sorted shim
+        # ensures that the indices are properly sorted.
+        hist, rev_indices = histogram_rev_sorted(ipnest)
 
         obj_use = np.ones(len(fgcm_obj), dtype=bool)
 
@@ -717,7 +718,7 @@ def _associate_reference_stars(self, lookup_table_handle, visit_cat, fgcm_obj):
 
         # Split into healpix pixels for more efficient matching.
         ipnest = hpg.angle_to_pixel(self.config.coarseNside, fgcm_obj["ra"], fgcm_obj["dec"])
-        hpix, revpix = esutil.stat.histogram(ipnest, rev=True)
+        hpix, revpix = histogram_rev_sorted(ipnest)
 
         pixel_cats = []
 
@@ -810,7 +811,7 @@ def _compute_delta_aper_summary_statistics(self, visit_cat, star_obs):
         a, b = esutil.numpy_util.match(visit_cat["visit"], star_obs["visit"][ok])
         visit_index[b] = a
 
-        h, rev = esutil.stat.histogram(visit_index, rev=True)
+        h, rev = histogram_rev_sorted(visit_index)
 
         visit_cat["deltaAper"] = -9999.0
         h_use, = np.where(h >= 3)

python/lsst/fgcmcal/fgcmCalibrateTractBase.py

@@ -321,9 +321,9 @@ def run(self, handleDict, tract,
                 self.log.info("  Band %s, repeatability: %.2f mmag" % (band, rep))
 
             # Check for convergence
-            if np.alltrue((previousReservedRawRepeatability
-                           - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
-                          < self.config.convergenceTolerance):
+            if np.all((previousReservedRawRepeatability
+                       - fgcmFitCycle.fgcmPars.compReservedRawRepeatability)
+                      < self.config.convergenceTolerance):
                 self.log.info("Raw repeatability has converged after cycle number %d." % (cycleNumber))
                 converged = True
             else:

python/lsst/fgcmcal/fgcmFitCycle.py

@@ -1785,7 +1785,7 @@ def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
         rec = parCat.addNew()
 
         # info section
-        rec['nCcd'] = parInfo['NCCD']
+        rec['nCcd'] = parInfo['NCCD'][0]
         rec['lutFilterNames'] = lutFilterNameString
         rec['fitBands'] = fitBandString
         # note these are not currently supported here.
@@ -1814,7 +1814,7 @@ def _makeParCatalog(self, parSchema, parInfo, pars, parSuperStarFlat,
                     'compEpsilonCcdMap', 'compEpsilonCcdNStarMap', 'compExpRefOffset']
 
         for scalarName in scalarNames:
-            rec[scalarName] = pars[scalarName.upper()]
+            rec[scalarName] = pars[scalarName.upper()][0]
 
         for arrName in arrNames:
             rec[arrName][:] = np.atleast_1d(pars[0][arrName.upper()])[:]

python/lsst/fgcmcal/fgcmOutputProducts.py

@@ -35,7 +35,6 @@
 
 import numpy as np
 import hpgeom as hpg
-import esutil
 from astropy import units
 
 import lsst.daf.base as dafBase
@@ -561,7 +560,7 @@ def _computeReferenceOffsets(self, stdCat, lutCat, physicalFilterMap, bands):
             stdCat['coord_dec'],
             degrees=False,
         )
-        h, rev = esutil.stat.histogram(ipring, rev=True)
+        h, rev = fgcm.fgcmUtilities.histogram_rev_sorted(ipring)
 
         gdpix, = np.where(h >= self.config.referencePixelizationMinStars)
 

tests/fgcmcalTestBase.py

@@ -566,7 +566,7 @@ def _testFgcmOutputProducts(self, instName, testName,
                 photoCalMeanCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'])
                 photoCalMags[i] = testCal.instFluxToMagnitude(rec['slot_CalibFlux_instFlux'],
                                                               rec.getCentroid())
-                zptMeanCalMags[i] = fgcmZpt - 2.5*np.log10(rec['slot_CalibFlux_instFlux'])
+                zptMeanCalMags[i] = fgcmZpt[0] - 2.5*np.log10(rec['slot_CalibFlux_instFlux'])
 
             # These should be very close but some tiny differences because the fgcm value
             # is defined at the center of the bbox, and the photoCal is the mean over the box

tests/test_fgcmcal_hsc.py

@@ -102,7 +102,7 @@ def test_fgcmcalPipelineBuildFromTable(self):
         nGoodZp = 27
         nOkZp = 27
         nBadZp = 1093
-        nStdStars = 235
+        nStdStars = 237
         nPlots = 47
 
         # We need an extra config file to turn off parquet format.
@@ -239,7 +239,7 @@ def test_fgcmcalTractPipeline(self):
                               nBand, i0Std, i0Recon, i10Std, i10Recon)
 
         rawRepeatability = np.array([0.0,
-                                     0.005631336514834158,
+                                     0.003440500079097844,
                                      0.004095912225403857])
         filterNCalibMap = {'HSC-R': 12,
                            'HSC-I': 15}