DM-16704: Add fgcmcal-tract mode to fgcmcal

fgcm/_version.py

@@ -1,5 +1,5 @@
 from __future__ import division, absolute_import, print_function
 
-__version__ = '2.1.2'
+__version__ = '2.2.0'
 
 __version_info__ = __version__.split('.')

fgcm/fgcmApertureCorrection.py

@@ -35,7 +35,7 @@ class FgcmApertureCorrection(object):
     """
     def __init__(self,fgcmConfig,fgcmPars,fgcmGray):
         self.fgcmLog = fgcmConfig.fgcmLog
-        self.fgcmLog.info('Initializing FgcmApertureCorrection')
+        self.fgcmLog.debug('Initializing FgcmApertureCorrection')
 
         self.fgcmPars = fgcmPars
 
@@ -47,6 +47,7 @@ def __init__(self,fgcmConfig,fgcmPars,fgcmGray):
         self.illegalValue = fgcmConfig.illegalValue
         self.plotPath = fgcmConfig.plotPath
         self.outfileBaseWithCycle = fgcmConfig.outfileBaseWithCycle
+        self.quietMode = fgcmConfig.quietMode
 
     def computeApertureCorrections(self):
         """
@@ -61,8 +62,9 @@ def computeApertureCorrections(self):
             return
 
         startTime=time.time()
-        self.fgcmLog.info('Computing aperture corrections with %d bins' %
-                         (self.aperCorrFitNBins))
+        if not self.quietMode:
+            self.fgcmLog.info('Computing aperture corrections with %d bins' %
+                              (self.aperCorrFitNBins))
 
         # need to make a local copy since we're modifying
         expGray = snmm.getArray(self.fgcmGray.expGrayHandle)
@@ -73,7 +75,7 @@ def computeApertureCorrections(self):
         # first, remove any previous correction if necessary...
         if (np.max(self.fgcmPars.compAperCorrRange[1,:]) >
             np.min(self.fgcmPars.compAperCorrRange[0,:])) :
-            self.fgcmLog.info('Removing old aperture corrections')
+            self.fgcmLog.debug('Removing old aperture corrections')
 
             expSeeingVariableClipped = np.clip(self.fgcmPars.expSeeingVariable,
                                                self.fgcmPars.compAperCorrRange[0,self.fgcmPars.expBandIndex],
@@ -213,5 +215,6 @@ def computeApertureCorrections(self):
         ## MAYBE: modify ccd gray and exp gray?
         ##  could rely on the iterations taking care of this.
 
-        self.fgcmLog.info('Computed aperture corrections in %.2f seconds.' %
-                         (time.time() - startTime))
+        if not self.quietMode:
+            self.fgcmLog.info('Computed aperture corrections in %.2f seconds.' %
+                              (time.time() - startTime))

fgcm/fgcmBrightObs.py

@@ -54,7 +54,7 @@ def __init__(self,fgcmConfig,fgcmPars,fgcmStars,fgcmLUT):
 
         self.fgcmLog = fgcmConfig.fgcmLog
 
-        self.fgcmLog.info('Initializing FgcmBrightObs')
+        self.fgcmLog.debug('Initializing FgcmBrightObs')
 
         # need fgcmPars because it tracks good exposures
         self.fgcmPars = fgcmPars
@@ -67,6 +67,7 @@ def __init__(self,fgcmConfig,fgcmPars,fgcmStars,fgcmLUT):
         self.nCore = fgcmConfig.nCore
         self.nStarPerRun = fgcmConfig.nStarPerRun
         self.bandFitIndex = fgcmConfig.bandFitIndex
+        self.quietMode = fgcmConfig.quietMode
 
         if (fgcmConfig.useSedLUT and self.fgcmLUT.hasSedLUT):
             self.useSedLUT = True
@@ -89,7 +90,8 @@ def brightestObsMeanMag(self,debug=False,computeSEDSlopes=False):
             raise ValueError("Must run FgcmChisq to compute magStd before FgcmBrightObs")
 
         startTime=time.time()
-        self.fgcmLog.info('Selecting good stars from Bright Observations')
+        if not self.quietMode:
+            self.fgcmLog.info('Selecting good stars from Bright Observations')
 
         self.debug = debug
         self.computeSEDSlopes = computeSEDSlopes
@@ -112,17 +114,20 @@ def brightestObsMeanMag(self,debug=False,computeSEDSlopes=False):
         obsFlag = snmm.getArray(self.fgcmStars.obsFlagHandle)
 
         preStartTime=time.time()
-        self.fgcmLog.info('Pre-matching stars and observations...')
+        if not self.quietMode:
+            self.fgcmLog.info('Pre-matching stars and observations...')
 
         goodStarsSub, goodObs = self.fgcmStars.getGoodObsIndices(goodStars)
 
-        self.fgcmLog.info('Pre-matching done in %.1f sec.' %
-                         (time.time() - preStartTime))
+        if not self.quietMode:
+            self.fgcmLog.info('Pre-matching done in %.1f sec.' %
+                              (time.time() - preStartTime))
 
         if (self.debug):
             self._worker((goodStars,goodObs))
         else:
-            self.fgcmLog.info('Running BrightObs on %d cores' % (self.nCore))
+            if not self.quietMode:
+                self.fgcmLog.info('Running BrightObs on %d cores' % (self.nCore))
 
             # split goodStars into a list of arrays of roughly equal size
 
@@ -149,8 +154,8 @@ def brightestObsMeanMag(self,debug=False,computeSEDSlopes=False):
             # reverse sort so the longest running go first
             workerList.sort(key=lambda elt:elt[1].size, reverse=True)
 
-            self.fgcmLog.info('Using %d sections (%.1f seconds)' %
-                             (nSections,time.time() - prepStartTime))
+            self.fgcmLog.debug('Using %d sections (%.1f seconds)' %
+                               (nSections,time.time() - prepStartTime))
 
             # make a pool
             pool = Pool(processes=self.nCore)
@@ -159,8 +164,9 @@ def brightestObsMeanMag(self,debug=False,computeSEDSlopes=False):
             pool.join()
 
 
-        self.fgcmLog.info('Finished BrightObs in %.2f seconds.' %
-                         (time.time() - startTime))
+        if not self.quietMode:
+            self.fgcmLog.info('Finished BrightObs in %.2f seconds.' %
+                              (time.time() - startTime))
 
 
     def _worker(self,goodStarsAndObs):

fgcm/fgcmChisq.py

@@ -59,8 +59,7 @@ def __init__(self,fgcmConfig,fgcmPars,fgcmStars,fgcmLUT):
 
         self.fgcmLog = fgcmConfig.fgcmLog
 
-        #self.fgcmLog.log('INFO','Initializing FgcmChisq')
-        self.fgcmLog.info('Initializing FgcmChisq')
+        self.fgcmLog.debug('Initializing FgcmChisq')
 
         # does this need to be shm'd?
         self.fgcmPars = fgcmPars
@@ -84,6 +83,7 @@ def __init__(self,fgcmConfig,fgcmPars,fgcmStars,fgcmLUT):
         self.useRefStarsWithInstrument = fgcmConfig.useRefStarsWithInstrument
         self.instrumentParsPerBand = fgcmConfig.instrumentParsPerBand
         self.saveParsForDebugging = fgcmConfig.saveParsForDebugging
+        self.quietMode = fgcmConfig.quietMode
 
         self.outfileBaseWithCycle = fgcmConfig.outfileBaseWithCycle
 
@@ -101,7 +101,8 @@ def __init__(self,fgcmConfig,fgcmPars,fgcmStars,fgcmLUT):
 
         # this is the default number of parameters
         self.nActualFitPars = self.fgcmPars.nFitPars
-        self.fgcmLog.info('Default: fit %d parameters.' % (self.nActualFitPars))
+        if not self.quietMode:
+            self.fgcmLog.info('Default: fit %d parameters.' % (self.nActualFitPars))
 
         self.clearMatchCache()
 
@@ -266,7 +267,8 @@ def __call__(self,fitParams,fitterUnits=False,computeDerivatives=False,computeSE
 
         goodStars = self.fgcmStars.getGoodStarIndices(includeReserve=self.includeReserve)
 
-        self.fgcmLog.info('Found %d good stars for chisq' % (goodStars.size))
+        if not self.quietMode:
+            self.fgcmLog.info('Found %d good stars for chisq' % (goodStars.size))
 
         if (goodStars.size == 0):
             raise RuntimeError("No good stars to fit!")
@@ -282,7 +284,7 @@ def __call__(self,fitParams,fitterUnits=False,computeDerivatives=False,computeSE
 
         if (self.useMatchCache and self.matchesCached) :
             # we have already done the matching
-            self.fgcmLog.info('Retrieving cached matches')
+            self.fgcmLog.debug('Retrieving cached matches')
             goodObs = self.goodObs
             goodStarsSub = self.goodStarsSub
         else:
@@ -297,11 +299,11 @@ def __call__(self,fitParams,fitterUnits=False,computeDerivatives=False,computeSE
 
             goodStarsSub, goodObs = self.fgcmStars.getGoodObsIndices(goodStars, expFlag=expFlag)
 
-            self.fgcmLog.info('Pre-matching done in %.1f sec.' %
-                             (time.time() - preStartTime))
+            self.fgcmLog.debug('Pre-matching done in %.1f sec.' %
+                               (time.time() - preStartTime))
 
             if (self.useMatchCache) :
-                self.fgcmLog.info('Caching matches for next iteration')
+                self.fgcmLog.debug('Caching matches for next iteration')
                 self.matchesCached = True
                 self.goodObs = goodObs
                 self.goodStarsSub = goodStarsSub
@@ -371,10 +373,10 @@ def __call__(self,fitParams,fitterUnits=False,computeDerivatives=False,computeSE
             # reverse sort so the longest running go first
             workerList.sort(key=lambda elt:elt[1].size, reverse=True)
 
-            self.fgcmLog.info('Using %d sections (%.1f seconds)' %
-                              (nSections,time.time()-prepStartTime))
+            self.fgcmLog.debug('Using %d sections (%.1f seconds)' %
+                               (nSections,time.time()-prepStartTime))
 
-            self.fgcmLog.info('Running chisq on %d cores' % (self.nCore))
+            self.fgcmLog.debug('Running chisq on %d cores' % (self.nCore))
 
             # make a pool
             pool = Pool(processes=self.nCore)
@@ -412,7 +414,8 @@ def __call__(self,fitParams,fitterUnits=False,computeDerivatives=False,computeSE
                                     (partialSums[3*self.fgcmPars.nFitPars:
                                                      4*self.fgcmPars.nFitPars] > 0))
                 self.nActualFitPars = nonZero.size
-                self.fgcmLog.info('Actually fit %d parameters.' % (self.nActualFitPars))
+                if not self.quietMode:
+                    self.fgcmLog.info('Actually fit %d parameters.' % (self.nActualFitPars))
 
             fitDOF = partialSums[-3] + partialSums[-1] - float(self.nActualFitPars)
 
@@ -488,8 +491,9 @@ def __call__(self,fitParams,fitterUnits=False,computeDerivatives=False,computeSE
         for key in self.totalHandleDict.keys():
             snmm.freeArray(self.totalHandleDict[key])
 
-        self.fgcmLog.info('Chisq computation took %.2f seconds.' %
-                         (time.time() - startTime))
+        if not self.quietMode:
+            self.fgcmLog.info('Chisq computation took %.2f seconds.' %
+                              (time.time() - startTime))
 
         self.fgcmStars.magStdComputed = True
         if (self.allExposures):

fgcm/fgcmComputeStepUnits.py

@@ -44,7 +44,7 @@ class FgcmComputeStepUnits(object):
     def __init__(self, fgcmConfig, fgcmPars, fgcmStars, fgcmLUT):
         self.fgcmLog = fgcmConfig.fgcmLog
 
-        self.fgcmLog.info('Initializing FgcmChisq')
+        self.fgcmLog.debug('Initializing FgcmComputeStepUnits')
 
         # does this need to be shm'd?
         self.fgcmPars = fgcmPars
@@ -69,6 +69,7 @@ def __init__(self, fgcmConfig, fgcmPars, fgcmStars, fgcmLUT):
         self.stepUnitReference = fgcmConfig.stepUnitReference
         self.fitGradientTolerance = fgcmConfig.fitGradientTolerance
         self.saveParsForDebugging = fgcmConfig.saveParsForDebugging
+        self.quietMode = fgcmConfig.quietMode
 
         self.outfileBaseWithCycle = fgcmConfig.outfileBaseWithCycle
 
@@ -101,7 +102,8 @@ def run(self, fitParams):
         self.fgcmPars.parsToExposures()
 
         goodStars = self.fgcmStars.getGoodStarIndices(includeReserve=False)
-        self.fgcmLog.info('Found %d good stars for step units' % (goodStars.size))
+        if not self.quietMode:
+            self.fgcmLog.info('Found %d good stars for step units' % (goodStars.size))
 
         if (goodStars.size == 0):
             raise RuntimeError("No good stars to fit!")
@@ -111,7 +113,8 @@ def run(self, fitParams):
 
         expFlag = self.fgcmPars.expFlag
 
-        goodStarsSub, goodObs = self.fgcmStars.getGoodObsIndices(goodStars, expFlag=expFlag)
+        goodStarsSub, goodObs = self.fgcmStars.getGoodObsIndices(goodStars, expFlag=expFlag,
+                                                                 checkBadMag=True)
 
         self.nSums = 1 # nobs
         # 0: nFitPars -> derivative for step calculation
@@ -297,8 +300,13 @@ def run(self, fitParams):
 
             pyfits.writeto('%s_stepUnits3.fits' % (self.outfileBaseWithCycle), tempCat, overwrite=True)
 
-        self.fgcmLog.info('Step size computation took %.2f seconds.' %
-                          (time.time() - startTime))
+        # free shared arrays
+        for key in self.totalHandleDict.keys():
+            snmm.freeArray(self.totalHandleDict[key])
+
+        if not self.quietMode:
+            self.fgcmLog.info('Step size computation took %.2f seconds.' %
+                              (time.time() - startTime))
 
     def _stepWorker(self, goodStarsAndObs):
         """