DM-37309: Nominally bring NightReport channel online

python/lsst/summit/utils/__init__.py

@@ -21,6 +21,7 @@
 
 from .bestEffort import *
 from .spectrumExaminer import *
+from .nightReport import *
 from .imageExaminer import *
 from .quickLook import *
 from .butlerUtils import *

python/lsst/summit/utils/astrometry/anet.py

@@ -127,11 +127,13 @@ def __init__(self,
                  checkInParallel=True,
                  timeout=300,
                  binary='solve-field',
+                 fluxSlot='base_CircularApertureFlux_3_0_instFlux',
                  ):
         self.indexFilePath = indexFilePath
         self.checkInParallel = checkInParallel
         self.timeout = timeout
         self.binary = binary
+        self.fluxSlot = fluxSlot
         if not shutil.which(binary):
             raise RuntimeError(f"Could not find {binary} in path, please install 'solve-field' and either"
                                " put it on your PATH or specify the full path to it in the 'binary' argument")
@@ -180,7 +182,7 @@ def _writeFitsTable(self, sourceCat):
         filename : `str`
             The filename to which the catalog was written.
         """
-        fluxArray = sourceCat.columns.getGaussianInstFlux()
+        fluxArray = sourceCat[self.fluxSlot]
         fluxFinite = np.logical_and(np.isfinite(fluxArray), fluxArray > 0)
         fluxArray = fluxArray[fluxFinite]
         indices = np.argsort(fluxArray)
@@ -192,6 +194,7 @@ def _writeFitsTable(self, sourceCat):
         x = fits.Column(name='X', format='D', array=xArray)
         y = fits.Column(name='Y', format='D', array=yArray)
         flux = fits.Column(name='FLUX', format='D', array=fluxArray)
+        print(f' of which {len(fluxArray)} made it into the fit')
         hdu = fits.BinTableHDU.from_columns([flux, x, y])
 
         filename = tempfile.mktemp(suffix='.fits')
@@ -233,6 +236,7 @@ def run(self, exp, sourceCat, isWideField, *, percentageScaleError=10, radius=No
             raise ValueError("No WCS in exposure")
 
         configFile = self._writeConfigFile(wide=isWideField)
+        print(f'Fitting image with {len(sourceCat)} sources', end='')
         fitsFile = self._writeFitsTable(sourceCat)
 
         plateScale = wcs.getPixelScale().asArcseconds()
@@ -257,9 +261,11 @@ def run(self, exp, sourceCat, isWideField, *, percentageScaleError=10, radius=No
                f"--scale-low {scaleMin:.3f} "  # the scale range
                f"--scale-high {scaleMax:.3f} "  # the scale range
                f"--scale-units arcsecperpix "
-               "--crpix-center "  # the CRPIX is always the center of the image
+               f"--crpix-x {wcs.getPixelOrigin()[0]} "  # set the pixel origin
+               f"--crpix-y {wcs.getPixelOrigin()[1]} "  # set the pixel origin
                f"--config {configFile} "
                f"-D {tempDir} "
+               "--no-plots "  # don't make plots
                "--overwrite "  # shouldn't matter as we're using temp files
                )
 

python/lsst/summit/utils/astrometry/plotting.py

@@ -24,6 +24,7 @@
 import matplotlib.pyplot as plt
 from astropy.coordinates import Angle
 import astropy.units as u
+import copy
 
 from lsst.obs.lsst.translators.latiss import AUXTEL_LOCATION
 from .. import quickSmooth
@@ -32,7 +33,7 @@
 # TODO: Add some of Craig's nice overlay stuff here
 
 def plot(exp, icSrc=None, filteredSources=None, saveAs=None,
-         clipMin=1, clipMax=1000000):
+         clipMin=1, clipMax=1000000, doSmooth=True):
     """Plot an exposure, overlaying the selected sources and compass arrows.
 
     Plots the exposure on a logNorm scale, with the brightest sources, as
@@ -55,15 +56,18 @@ def plot(exp, icSrc=None, filteredSources=None, saveAs=None,
         Clip values in the image below this value to ``clipMin``.
     clipMax : `float`
         Clip values in the image above this value to ``clipMax``.
+    doSmooth : `bool`, optional
+        Smooth the image slightly to improve the visability of low SNR sources?
     """
-    plt.figure(figsize=(16, 16))
-    arr = exp.image.array
-    arr = np.clip(arr, clipMin, clipMax)
-    arr = quickSmooth(arr)  # smooth slightly to help visualize
-    plt.imshow(np.arcsinh(arr)/10,
+    fig = plt.figure(figsize=(16, 16))
+    data = copy.deepcopy(exp.image.array)
+    data = np.clip(data, clipMin, clipMax)
+    if doSmooth:
+        data = quickSmooth(data)  # smooth slightly to help visualize
+    plt.imshow(np.arcsinh(data)/10,
                interpolation='None', cmap='gray', origin='lower')
 
-    height, width = exp.image.array.shape
+    height, width = data.shape
     leftFraction = .15  # fraction into the image to start the N/E compass
     rightFraction = .225  # fraction into the image to start the az/el compass
     fontsize = 20  # for the compass labels
@@ -136,3 +140,7 @@ def plot(exp, icSrc=None, filteredSources=None, saveAs=None,
     if saveAs:
         plt.savefig(saveAs)
     plt.show()
+
+    plt.gcf().clear()
+    del fig
+    del data

python/lsst/summit/utils/astrometry/utils.py

@@ -147,11 +147,27 @@ def getAverageElFromHeader(header):
     return (elStart + elEnd) / 2
 
 
+def patchHeader(header):
+    """This is a TEMPORARY function to patch some info into the headers.
+    """
+    if header.get('SECPIX') == '3.11':
+        # the narrow camera currently is wrong about its place scale by of ~2.2
+        header['SECPIX'] = '1.44'
+        # update the boresight locations until this goes into the header
+        # service
+        header['CRPIX1'] = 1898.10
+        header['CRPIX2'] = 998.47
+    if header.get('SECPIX') == '8.64':
+        # update the boresight locations until this goes into the header
+        # service
+        header['CRPIX1'] = 1560.85
+        header['CRPIX2'] = 1257.15
+    return header
+
+
 def genericCameraHeaderToWcs(exp):
     header = exp.getMetadata().toDict()
-    width, height = exp.image.array.shape
-    header['CRPIX1'] = width/2
-    header['CRPIX2'] = height/2
+    header = patchHeader(header)
 
     header['CTYPE1'] = 'RA---TAN-SIP'
     header['CTYPE2'] = 'DEC--TAN-SIP'
@@ -243,17 +259,30 @@ def runCharactierizeImage(exp, snr, minPix):
     charConfig.doApCorr = False
     charConfig.doDeblend = False
     charConfig.repair.doCosmicRay = False
-    charConfig.repair.doInterpolate = True
+
     charConfig.detection.minPixels = minPix
     charConfig.detection.thresholdValue = snr
+    charConfig.detection.includeThresholdMultiplier = 1
+
+    # fit background with the most simple thing possible as we don't need
+    # much sophistication here. weighting=False is *required* for very
+    # large binSizes.
+    charConfig.background.algorithm = 'CONSTANT'
+    charConfig.background.approxOrderX = 0
+    charConfig.background.approxOrderY = -1
+    charConfig.background.binSize = max(exp.getWidth(), exp.getHeight())
+    charConfig.background.weighting = False
+
+    # set this to use all the same minimal settings as those above
+    charConfig.detection.background = charConfig.background
 
     charTask = CharacterizeImageTask(config=charConfig)
 
     charResult = charTask.run(exp)
     return charResult
 
 
-def filterSourceCatOnBrightest(catalog, brightFraction, minSources=15,
+def filterSourceCatOnBrightest(catalog, brightFraction, minSources=15, maxSources=200,
                                flux_field="base_CircularApertureFlux_3_0_instFlux"):
     """Filter a sourceCat on the brightness, leaving only the top fraction.
 
@@ -270,6 +299,8 @@ def filterSourceCatOnBrightest(catalog, brightFraction, minSources=15,
         Return this fraction of the brightest sources.
     minSources : `int`, optional
         Always return at least this many sources.
+    maxSources : `int`, optional
+        Never return more than this many sources.
     flux_field : `str`, optional
         Name of flux field to filter on.
 
@@ -280,6 +311,10 @@ def filterSourceCatOnBrightest(catalog, brightFraction, minSources=15,
     """
     assert minSources > 0
     assert brightFraction > 0 and brightFraction <= 1
+    if not maxSources >= minSources:
+        raise ValueError('maxSources must be greater than or equal to minSources, got '
+                         f'{maxSources=}, {minSources=}')
+
     maxFlux = np.nanmax(catalog[flux_field])
     result = catalog.subset(catalog[flux_field] > maxFlux * 0.001)
 
@@ -290,4 +325,4 @@ def filterSourceCatOnBrightest(catalog, brightFraction, minSources=15,
     result.sort(item.key)
     result = result.copy(deep=True)  # make it memory contiguous
     end = int(np.ceil(len(result)*brightFraction))
-    return result[-max(end, minSources):]
+    return result[-min(maxSources, max(end, minSources)):]

python/lsst/summit/utils/bestEffort.py

@@ -217,8 +217,8 @@ def getExposure(self, expIdOrDataId, extraIsrOptions={}, skipCosmics=False, forc
         isrConfig.doWrite = False  # this task writes separately, no need for this
         isrConfig.doSaturation = True  # saturation very important for roundness measurement in qfm
         isrConfig.doSaturationInterpolation = True
-        isrConfig.overscan.leadingColumnsToSkip = 5
         isrConfig.overscan.fitType = 'MEDIAN_PER_ROW'
+        isrConfig.overscan.doParallelOverscan = True
 
         # apply general overrides
         self._applyConfigOverrides(isrConfig, self.defaultExtraIsrOptions)

python/lsst/summit/utils/butlerUtils.py

@@ -91,7 +91,7 @@ def getLatissDefaultCollections():
         site = ''
 
     if site == 'tucson':
-        collections.append("LATISS-test-data-TTS")
+        collections.append("LATISS-test-data")
         return collections
     if site == 'summit':
         collections.append("LATISS-test-data")