Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-15534 #64

Merged
merged 2 commits into from
Sep 7, 2018
Merged

DM-15534 #64

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
10ea4a0
Enable --reuse deblendCoaddSources
yalsayyad Sep 6, 2018
b6aadc1
Remove unused import and fix linelength
yalsayyad Sep 6, 2018
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
34 changes: 24 additions & 10 deletions python/lsst/pipe/drivers/multiBandDriver.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@
MeasureMergedCoaddSourcesTask,
MergeMeasurementsTask,)
from lsst.ctrl.pool.parallel import BatchPoolTask
from lsst.ctrl.pool.pool import Pool, abortOnError, NODE
from lsst.ctrl.pool.pool import Pool, abortOnError
from lsst.meas.base.references import MultiBandReferencesTask
from lsst.meas.base.forcedPhotCoadd import ForcedPhotCoaddTask
from lsst.pipe.drivers.utils import getDataRef, TractDataIdContainer
Expand Down Expand Up @@ -206,7 +206,7 @@ def _makeArgumentParser(cls, *args, **kwargs):
parser.add_id_argument("--id", "deepCoadd", help="data ID, e.g. --id tract=12345 patch=1,2",
ContainerClass=TractDataIdContainer)
parser.addReuseOption(["detectCoaddSources", "mergeCoaddDetections", "measureCoaddSources",
"mergeCoaddMeasurements", "forcedPhotCoadd"])
"mergeCoaddMeasurements", "forcedPhotCoadd", "deblendCoaddSources"])
return parser

@classmethod
Expand Down Expand Up @@ -262,7 +262,8 @@ def runDataRef(self, patchRefList):
for patchRef in patchRefList:
if ("detectCoaddSources" in self.reuse and
patchRef.datasetExists(self.config.coaddName + "Coadd_calexp", write=True)):
self.log.info("Skipping detectCoaddSources for %s; output already exists." % patchRef.dataId)
self.log.info("Skipping detectCoaddSources for %s; output already exists." %
patchRef.dataId)
continue
if not patchRef.datasetExists(self.config.coaddName + "Coadd"):
self.log.debug("Not processing %s; required input %sCoadd missing." %
Expand All @@ -274,7 +275,8 @@ def runDataRef(self, patchRefList):

patchRefList = [patchRef for patchRef in patchRefList if
patchRef.datasetExists(self.config.coaddName + "Coadd_calexp") and
patchRef.datasetExists(self.config.coaddName + "Coadd_det", write=self.config.doDetection)]
patchRef.datasetExists(self.config.coaddName + "Coadd_det",
write=self.config.doDetection)]
dataIdList = [patchRef.dataId for patchRef in patchRefList]

# Group by patch
Expand Down Expand Up @@ -416,19 +418,31 @@ def runDeblendMerged(self, cache, dataIdList):
dataId in dataIdList]
reprocessing = False # Does this patch require reprocessing?
if ("deblendCoaddSources" in self.reuse and
dataRef.datasetExists(self.config.coaddName + self.measurementInput, write=True)):
all([dataRef.datasetExists(self.config.coaddName + "Coadd_" + self.measurementInput,
write=True) for dataRef in dataRefList])):
if not self.config.reprocessing:
self.log.info("Skipping deblendCoaddSources for %s; output already exists" % dataIdList)
return False

catalog = dataRefList[0].get(self.config.coaddName + self.measurementInput)
bigFlag = catalog["deblend.parent-too-big"]
# Footprints are the same every band, therefore we can check just one
catalog = dataRefList[0].get(self.config.coaddName + "Coadd_" + self.measurementInput)
bigFlag = catalog["deblend_parentTooBig"]
# Footprints marked too large by the previous deblender run
numOldBig = bigFlag.sum()
if numOldBig == 0:
self.log.info("No large footprints in %s" %
(dataRef.dataId,))
self.log.info("No large footprints in %s" % (dataRefList[0].dataId))
return False
numNewBig = sum((self.deblendCoaddSources.isLargeFootprint(src.getFootprint()) for

# This if-statement can be removed after DM-15662
if self.config.deblendCoaddSources.simultaneous:
deblender = self.deblendCoaddSources.multiBandDeblend
else:
deblender = self.deblendCoaddSources.singleBandDeblend

# isLargeFootprint() can potentially return False for a source that is marked
# too big in the catalog, because of "new"/different deblender configs.
# numNewBig is the number of footprints that *will* be too big if reprocessed
numNewBig = sum((deblender.isLargeFootprint(src.getFootprint()) for
src in catalog[bigFlag]))
if numNewBig == numOldBig:
self.log.info("All %d formerly large footprints continue to be large in %s" %
Expand Down