tickets/dm-33992 Support generating Pipeline to include in DRP pipe

.gitignore

@@ -12,6 +12,11 @@ config.log
 # Built by sconsUtils
 version.py
 bin/
+pipelines/generic_coadd_plots.yaml
+pipelines/generic_visit_plots.yaml
+pipelines/analysis_drp_plots.yaml
+pipelines/LSSTCam-imSim/dc2_coadd_plots.yaml
+pipelines/LSSTCam-imSim/analysis_drp_plots.yaml
 
 # Pytest
 tests/.tests

SConstruct

@@ -1,4 +1,10 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
+
 # Python-only package
-scripts.BasicSConstruct("analysis_drp", disableCc=True, noCfgFile=True)
+scripts.BasicSConstruct(
+    "analysis_drp",
+    disableCc=True,
+    noCfgFile=True,
+    defaultTargets=scripts.DEFAULT_TARGETS + ("pipelines",),
+)

pipelines/LSSTCam-imSim/analysis_drp_plots.in

@@ -0,0 +1,6 @@
+imports:
+  - "$ANALYSIS_DRP_DIR/pipelines/LSSTCam-imSim/dc2_coadd_plots.yaml"
+  - "$ANALYSIS_DRP_DIR/pipelines/generic_visit_plots.yaml"
+
+description: |
+  analysis drp plot pipeline specialized for imsim

pipelines/LSSTCam-imSim/dc2_coadd_plots.in

@@ -0,0 +1,7 @@
+imports:
+  - "$ANALYSIS_DRP_DIR/pipelines/LSSTCam-imSim/redgal_truth_plot_pipeline.yaml"
+  - "$ANALYSIS_DRP_DIR/pipelines/generic_coadd_plots.yaml"
+  - "$ANALYSIS_DRP_DIR/pipelines/LSSTCam-imSim/coaddQAPlots_matched_truth.yaml"
+
+description: |
+  Coadd analysis drp plots specialized for LSSTCam

pipelines/SConscript

@@ -0,0 +1,87 @@
+from lsst.sconsUtils.state import env, targets
+from lsst.sconsUtils.utils import libraryLoaderEnvironment
+
+
+generic_coadd = env.Command(
+    "generic_coadd_plots.yaml",
+    "generic_coadd_plots.in",
+    " ".join(
+        [
+            libraryLoaderEnvironment(),
+            "python",
+            "-m",
+            "lsst.analysis.drp.pipelineBuilder",
+            "$SOURCE",
+            "analysis_coadd_plots",
+        ]
+    ),
+)
+
+generic_visit = env.Command(
+    "generic_visit_plots.yaml",
+    "generic_visit_plots.in",
+    " ".join(
+        [
+            libraryLoaderEnvironment(),
+            "python",
+            "-m",
+            "lsst.analysis.drp.pipelineBuilder",
+            "$SOURCE",
+            "analysis_visit_plots",
+        ]
+    ),
+)
+
+analysis_pipeline = env.Command(
+    "analysis_drp_plots.yaml",
+    "analysis_drp_plots.in",
+    " ".join(
+        [
+            libraryLoaderEnvironment(),
+            "python",
+            "-m",
+            "lsst.analysis.drp.pipelineBuilder",
+            "$SOURCE",
+        ]
+    ),
+)
+env.Depends(analysis_pipeline, ("generic_visit_plots.yaml", "generic_coadd_plots.yaml"))
+
+# Imsim commands
+imsim_coadd = env.Command(
+    "LSSTCam-imSim/dc2_coadd_plots.yaml",
+    "LSSTCam-imSim/dc2_coadd_plots.in",
+    " ".join(
+        [
+            libraryLoaderEnvironment(),
+            "python",
+            "-m",
+            "lsst.analysis.drp.pipelineBuilder",
+            "$SOURCE",
+            "imsim_analysis_coadd_plots",
+        ]
+    ),
+)
+env.Depends(imsim_coadd, "analysis_drp_plots.yaml")
+
+imsim_pipeline = env.Command(
+    "LSSTCam-imSim/analysis_drp_plots.yaml",
+    "LSSTCam-imSim/analysis_drp_plots.in",
+    " ".join(
+        [
+            libraryLoaderEnvironment(),
+            "python",
+            "-m",
+            "lsst.analysis.drp.pipelineBuilder",
+            "$SOURCE",
+        ]
+    ),
+)
+env.Depends(imsim_pipeline, "LSSTCam-imSim/dc2_coadd_plots.yaml")
+
+
+targets.setdefault("pipelines", []).extend(
+    [generic_coadd, generic_visit, analysis_pipeline, imsim_coadd, imsim_pipeline]
+)
+
+env.AlwaysBuild(targets["pipelines"])

pipelines/analysis_drp_plots.in

@@ -0,0 +1,7 @@
+imports:
+  - "$ANALYSIS_DRP_DIR/pipelines/generic_visit_plots.yaml"
+  - "$ANALYSIS_DRP_DIR/pipelines/generic_coadd_plots.yaml"
+
+description: |
+  Collection of Analysis DRP plots that are suitable to run on collections
+  produced by a generic DRP processing pipeline

pipelines/coaddQAPlots.yaml

@@ -1,6 +1,6 @@
 description: Make coadd plots for QA
 parameters:
-  ApFluxName: "i_ap09Flux"
+  CoaddApFluxName: "i_ap09Flux"
   band: i
 tasks:
   # Make a plot showing the difference between the aperture 12 magnitude and the PSF magnitude.
@@ -9,7 +9,7 @@ tasks:
     class: lsst.analysis.drp.skyPlot.SkyPlotTask
     config:
       axisActions.zAction: MagDiff
-      axisActions.zAction.col1: parameters.ApFluxName
+      axisActions.zAction.col1: parameters.CoaddApFluxName
       axisActions.zAction.col2: "i_psfFlux"
       connections.plotName: CircAp12_sub_PSF_meas_stars_i
       selectorActions.flagSelector: CoaddPlotFlagSelector

pipelines/generic_coadd_plots.in

@@ -0,0 +1,6 @@
+imports:
+  - "$ANALYSIS_DRP_DIR/pipelines/coaddQAPlots.yaml"
+  - "$ANALYSIS_DRP_DIR/pipelines/stellarLocusPlots.yaml"
+
+description: |
+  Coadd level analysis drp plots

pipelines/generic_visit_plots.in

@@ -0,0 +1,6 @@
+imports:
+  - "$ANALYSIS_DRP_DIR/pipelines/visitQAEllipPlots.yaml"
+  - "$ANALYSIS_DRP_DIR/pipelines/visitQAPlots.yaml"
+
+description: |
+  Visit level analysis drp plots

pipelines/visitQAPlots.yaml

@@ -6,7 +6,7 @@ description: |
     the coadd level tables, which have band_ prepended to their column names, and the visit level
     tables which do not.
 parameters:
-  ApFluxName: "ap12Flux"
+  VisitApFluxName: "ap12Flux"
   band: i
 tasks:
   # Make a plot showing the difference between the aperture 12 magnitude and the PS magnitude.
@@ -15,7 +15,7 @@ tasks:
     class: lsst.analysis.drp.skyPlotVisit.SkyPlotVisitTask
     config:
       axisActions.zAction: MagDiff
-      axisActions.zAction.col1: parameters.ApFluxName
+      axisActions.zAction.col1: parameters.VisitApFluxName
       axisActions.zAction.col2: "psfFlux"
       connections.plotName: CircAp12_sub_PS_meas_stars
       selectorActions.flagSelector: VisitPlotFlagSelector

python/lsst/analysis/drp/histPlot.py

@@ -13,6 +13,7 @@
 import lsst.pex.config as pexConfig
 
 from .plotUtils import generateSummaryStats, parsePlotInfo, addPlotInfo
+from . import calcFunctors  # noqa
 
 
 class HistPlotTaskConnections(pipeBase.PipelineTaskConnections,

python/lsst/analysis/drp/pipelineBuilder.py

@@ -0,0 +1,154 @@
+# This file is part of analysis_drp.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from typing import Iterable, Mapping, Optional, Sequence
+
+from lsst.resources import ResourcePath
+from lsst.pipe.base.pipelineIR import PipelineIR, LabeledSubset
+
+
+class PlotPipelineBuilder:
+    """Class to support building a complete analysis drp pipeline from an input
+    file.
+    """
+
+    def __init__(self):
+        # Setup the command line argument parser that will be used to build
+        # the pipeline
+        self.parser = argparse.ArgumentParser()
+        self.parser.add_argument("pipeline", help="input pipeline to process")
+        self.parser.add_argument(
+            "subsetName",
+            nargs="?",
+            help="name of the subset which will contain all the tasks defined in the input file, leave "
+            "blank if no subset is to be created",
+        )
+
+    def getSubsets(self) -> Mapping[str, tuple[Optional[str], Iterable[str]]]:
+        """Allows defining subsets based on pattern matching labels.
+
+        In order to define a subset, it should be added to the mapping returned
+        by this method. The key will be the label used for the subset, and the
+        value is a list of regular expressions that will be used to match the
+        labels in the input pipeline.
+
+        For instance returning:
+        >>> {'axisAxis': ["ScatterPlot_.*", ".*HistPlot.*", "SpecificPlot"]}
+
+        Will create a subset named axisAxis, and will contain any label with
+        "ScatterPlot_" at the beginning of its label, "HistPlot" anywhere in
+        the label, and exactly the label "SpecificPlot".
+        """
+        return {}
+
+    def run(self, args: Sequence[str]) -> None:
+        """Main entry point for actually building a pipeline. This is
+        responsible for parsing the supplied args (likely coming from the
+        command line), and then calling the build method. The command expects
+        that args contain the script name in the 0th index.
+        """
+        parsedArgs = self.parser.parse_args(args[1:])
+        resource = ResourcePath(parsedArgs.pipeline)
+
+        # validate that the supplied files are considered inputs and not
+        # finalized pipelines
+        if resource.getExtension() != ".in":
+            raise ValueError("This builder can only process pipeline files that end in .in")
+        self.build(resource, parsedArgs.subsetName)
+
+    @property
+    def excludeSet(self) -> set[str]:
+        """Returns a list of labels which should be ignored for some reason"""
+        return set()
+
+    def build(self, pipeline: ResourcePath, subsetName: Optional[str] = None) -> None:
+        """Builds a pipeline out of the input proto-pipeline supplied to run.
+
+        Subsets are added to the output pipeline that contain all the plots
+        (except plots contained in the exclude set), and any other subsets
+        defined within the builder's ``getSubsets`` method.
+
+        Parameters
+        ----------
+        pipeline : `ResourcePath`
+            The `ResourcePath` that points to an input proto-pipeline that is
+            to be transformed into a final pipeline.
+
+        subsetName : `str`, optional
+            If supplied, a subset will be created containing all the tasks in
+            the input proto-pipeline file with the supplied name. Defaults to
+            `None`.
+
+        Raises
+        ------
+        ValueError
+            Raised if a subset is already declared with a name returned from
+            the ``getGroups`` method.
+            Raised if a subset is already declared with a name that conflicts
+            with the subset name for all analysis plots.
+        """
+        inputPipeline = PipelineIR.from_uri(pipeline)
+        excludes = self.excludeSet
+        for name, (description, subset_exps) in self.getSubsets():
+            tmpTaskSet = set()
+            for exp in subset_exps:
+                pattern = re.compile(exp)
+                for task in inputPipeline.tasks:
+                    if task in excludes:
+                        continue
+                    if pattern.match(task):
+                        tmpTaskSet.add(task)
+            if tmpTaskSet:
+                if name in inputPipeline.labeled_subsets:
+                    raise ValueError(
+                        f"Conflicting subset name: {name} is already " "declared as a labeled subset"
+                    )
+                inputPipeline.labeled_subsets[name] = LabeledSubset(
+                    label=name, subset=tmpTaskSet, description=description
+                )
+        # need to exclude any labels that should not be in the pipeline as
+        # defined in the excluded labels set
+        for label in self.excludeSet:
+            inputPipeline.tasks.pop(label)
+
+        # add a generic labeled subset with all tasks in it
+        if subsetName:
+            if subsetName in inputPipeline.labeled_subsets:
+                raise ValueError(
+                    f"Labeled Subset {subsetName} was already declared in input "
+                    "pipeline, please ensure this label is not defined in any "
+                    "input pipelines"
+                )
+            inputPipeline.labeled_subsets[subsetName] = LabeledSubset(
+                label=subsetName,
+                subset=set(inputPipeline.tasks) - self.excludeSet,
+                description=None,
+            )
+
+        inputPipeline.write_to_uri(pipeline.updatedExtension("yaml"))
+
+
+if __name__ == "__main__":
+    PlotPipelineBuilder().run(sys.argv)

python/lsst/analysis/drp/scatterPlot.py

@@ -250,7 +250,11 @@ def run(self, catPlot, dataId, runName, skymap, tableName, bands, plotName):
         sumStats = {} if skymap is None else generateSummaryStats(
             plotDf, self.config.axisLabels["y"], skymap, plotInfo)
         # Make the plot
-        fig = self.scatterPlotWithTwoHists(plotDf, plotInfo, sumStats)
+        try:
+            fig = self.scatterPlotWithTwoHists(plotDf, plotInfo, sumStats)
+        except Exception:
+            # This is a workaround until scatterPlotWithTwoHists works properly
+            fig = plt.figure(dpi=300)
 
         return pipeBase.Struct(scatterPlot=fig)
 
@@ -385,12 +389,15 @@ def scatterPlotWithTwoHists(self, catPlot, plotInfo, sumStats, yLims=False, xLim
                            sourceTypeMapper["galaxies"]),
                           (xsStars.values, ysStars.values, "midnightblue", newBlues,
                            sourceTypeMapper["stars"])]
-        if np.any(catPlot["sourceType"] == sourceTypeMapper["unknowns"]):
+        elif np.any(catPlot["sourceType"] == sourceTypeMapper["unknowns"]):
             unknowns = (catPlot["sourceType"] == sourceTypeMapper["unknowns"])
             toPlotList = [(catPlot.loc[unknowns, xCol].values, catPlot.loc[unknowns, yCol].values,
-                           "green", sourceTypeMapper["unknowns"])]
-        if np.any(catPlot["sourceType"] == sourceTypeMapper["all"]):
-            toPlotList = [(catPlot[xCol].values, catPlot[yCol].values, "purple", sourceTypeMapper["all"])]
+                           "green", None, sourceTypeMapper["unknowns"])]
+        elif np.any(catPlot["sourceType"] == sourceTypeMapper["all"]):
+            toPlotList = [(catPlot[xCol].values, catPlot[yCol].values, "purple", None,
+                           sourceTypeMapper["all"])]
+        else:
+            toPlotList = []
 
         for (j, (xs, ys, color, cmap, sourceType)) in enumerate(toPlotList):
             if len(xs) < 2:

python/lsst/analysis/drp/skyPlot.py

@@ -377,6 +377,11 @@ def skyPlot(self, catPlot, plotInfo, sumStats):
                 maxRa = np.max(xs)
                 minDec = np.min(ys)
                 maxDec = np.max(ys)
+                # Avoid identical end points which causes problems in binning
+                if minRa == maxRa:
+                    maxRa += 1e-5  # There is no reason to pick this number in particular
+                if minDec == maxDec:
+                    maxDec += 1e-5  # There is no reason to pick this number in particular
             med = np.median(colorVals)
             mad = sigmaMad(colorVals)
             vmin = med - 2*mad