Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-37865: Remove unused parameters from deferred dataset handle get calls #752

Merged
merged 4 commits into from
Feb 7, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
59 changes: 1 addition & 58 deletions python/lsst/pipe/tasks/coaddBase.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.

__all__ = ["CoaddBaseTask", "getSkyInfo", "makeSkyInfo"]
__all__ = ["CoaddBaseTask", "makeSkyInfo"]

import lsst.pex.config as pexConfig
import lsst.afw.image as afwImage
Expand Down Expand Up @@ -140,33 +140,6 @@ def __init__(self, **kwargs):
self.makeSubtask("select")
self.makeSubtask("inputRecorder")

def getSkyInfo(self, patchRef):
"""Use getSkyinfo to return the skyMap, tract and patch information, wcs and the outer bbox
of the patch.

Parameters
----------
patchRef : `Unknown`
Data reference for sky map. Must include keys "tract" and "patch".

Returns
-------
getSkyInfo : `lsst.pipe.base.Struct`
Sky Info as a struct with attributes:

``skyMap``
sky map (`lsst.skyMap.SkyMap`).
``tractInfo``
Information for chosen tract of sky map (`lsst.skymap.TractInfo`).
``patchInfo``
Information about chosen patch of tract (`lsst.skymap.PatchInfo`).
``wcs``
WCS of tract (`lsst.afw.image.SkyWcs`).
``bbox``
Outer bbox of patch, as an geom Box2I (`lsst.afw.geom.Box2I`).
"""
return getSkyInfo(coaddName=self.config.coaddName, patchRef=patchRef)

def getTempExpDatasetName(self, warpType="direct"):
"""Return warp name for given warpType and task config

Expand All @@ -187,36 +160,6 @@ def getBadPixelMask(self):
return afwImage.Mask.getPlaneBitMask(self.config.badMaskPlanes)


def getSkyInfo(coaddName, patchRef):
"""Return the SkyMap, tract and patch information, wcs, and outer bbox of the patch to be coadded.

Parameters
----------
coaddName : `str`
Coadd name; typically one of deep or goodSeeing.
patchRef : `Unknown`
Data reference for sky map. Must include keys "tract" and "patch".

Returns
-------
makeSkyInfo : `lsst.pipe.base.Struct`
pipe_base Struct with attributes:

``skyMap``
Sky map (`lsst.skyMap.SkyMap`).
``tractInfo``
Information for chosen tract of sky map (`lsst.skyMap.TractInfo`).
``patchInfo``
Information about chosen patch of tract (`lsst.skyMap.PatchInfo`).
``wcs``
WCS of tract (`lsst.afw.image.SkyWcs`).
``bbox``
Outer bbox of patch, as an geom Box2I (`lsst.afw.geom.Box2I`).
"""
skyMap = patchRef.get(coaddName + "Coadd_skyMap")
return makeSkyInfo(skyMap, patchRef.dataId["tract"], patchRef.dataId["patch"])


def makeSkyInfo(skyMap, tractId, patchId):
"""Constructs SkyInfo used by coaddition tasks for multiple
patchId formats.
Expand Down
4 changes: 1 addition & 3 deletions python/lsst/pipe/tasks/drpAssociationPipe.py
Original file line number Diff line number Diff line change
Expand Up @@ -166,9 +166,7 @@ def run(self,

diaSourceHistory = []
for catRef in diaSourceTables:
cat = catRef.get(
datasetType=self.config.connections.diaSourceTables,
immediate=True)
cat = catRef.get()

isInTractPatch = self._trimToPatch(cat,
innerPatchBox,
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/pipe/tasks/extended_psf.py
Original file line number Diff line number Diff line change
Expand Up @@ -354,7 +354,7 @@ def run(self, bss_ref_list, region_name=None):
self.log.info('Building extended PSF from stamps extracted from %d detector images%s',
len(bss_ref_list), region_message)
# read in example set of full stamps
example_bss = bss_ref_list[0].get(datasetType="brightStarStamps", immediate=True)
example_bss = bss_ref_list[0].get()
example_stamp = example_bss[0].stamp_im
# create model image
ext_psf = afwImage.MaskedImageF(example_stamp.getBBox())
Expand All @@ -372,7 +372,7 @@ def run(self, bss_ref_list, region_name=None):
for jbbox, bbox in enumerate(sub_bboxes):
all_stars = None
for bss_ref in bss_ref_list:
read_stars = bss_ref.get(datasetType="brightStarStamps", parameters={'bbox': bbox})
read_stars = bss_ref.get(parameters={'bbox': bbox})
if self.config.do_mag_cut:
read_stars = read_stars.selectByMag(magMax=self.config.mag_limit)
if all_stars:
Expand Down
8 changes: 4 additions & 4 deletions python/lsst/pipe/tasks/makeWarp.py
Original file line number Diff line number Diff line change
Expand Up @@ -439,8 +439,8 @@ def run(self, calExpList, ccdIdList, skyInfo, visitId=0, dataIdList=None, **kwar
List of data references for calexps that (may)
overlap the patch of interest.
skyInfo : `lsst.pipe.base.Struct`
Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch.
Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
geometric information about the patch.
visitId : `int`
Integer identifier for visit, for the table that will
produce the CoaddPsf.
Expand Down Expand Up @@ -738,8 +738,8 @@ def _prepareEmptyExposure(skyInfo):
Parameters
----------
skyInfo : `lsst.pipe.base.Struct`
Struct from CoaddBaseTask.getSkyInfo() with geometric
information about the patch.
Struct from `~lsst.pipe.base.coaddBase.makeSkyInfo()` with
geometric information about the patch.

Returns
-------
Expand Down
6 changes: 3 additions & 3 deletions python/lsst/pipe/tasks/matchBackgrounds.py
Original file line number Diff line number Diff line change
Expand Up @@ -236,7 +236,7 @@ def run(self, expRefList, expDatasetType, imageScalerList=None, refExpDataRef=No
if refInd is not None and refInd not in refIndSet:
raise RuntimeError("Internal error: selected reference %s not found in expRefList")

refExposure = refExpDataRef.get(expDatasetType, immediate=True)
refExposure = refExpDataRef.get()
if refImageScaler is not None:
refMI = refExposure.getMaskedImage()
refImageScaler.scaleMaskedImage(refMI)
Expand All @@ -258,7 +258,7 @@ def run(self, expRefList, expDatasetType, imageScalerList=None, refExpDataRef=No
else:
self.log.info("Matching background of %s to %s", toMatchRef.dataId, refExpDataRef.dataId)
try:
toMatchExposure = toMatchRef.get(expDatasetType, immediate=True)
toMatchExposure = toMatchRef.get()
if imageScaler is not None:
toMatchMI = toMatchExposure.getMaskedImage()
imageScaler.scaleMaskedImage(toMatchMI)
Expand Down Expand Up @@ -326,7 +326,7 @@ def selectRefExposure(self, expRefList, imageScalerList, expDatasetType):
(len(expRefList), len(imageScalerList)))

for expRef, imageScaler in zip(expRefList, imageScalerList):
exposure = expRef.get(expDatasetType, immediate=True)
exposure = expRef.get()
maskedImage = exposure.getMaskedImage()
if imageScaler is not None:
try:
Expand Down
6 changes: 2 additions & 4 deletions python/lsst/pipe/tasks/matchFakes.py
Original file line number Diff line number Diff line change
Expand Up @@ -212,12 +212,10 @@ def composeFakeCat(self, fakeCats, skyMap):
quantum.
"""
if len(fakeCats) == 1:
return fakeCats[0].get(
datasetType=self.config.connections.fakeCats)
return fakeCats[0].get()
outputCat = []
for fakeCatRef in fakeCats:
cat = fakeCatRef.get(
datasetType=self.config.connections.fakeCats)
cat = fakeCatRef.get()
tractId = fakeCatRef.dataId["tract"]
# Make sure all data is within the inner part of the tract.
outputCat.append(cat[
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/pipe/tasks/multiBand.py
Original file line number Diff line number Diff line change
Expand Up @@ -539,7 +539,7 @@ def __init__(self, butler=None, schema=None, peakSchema=None, refObjLoader=None,
schema = initInputs['inputSchema'].schema
if schema is None:
assert butler is not None, "Neither butler nor schema is defined"
schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema", immediate=True).schema
schema = butler.get(self.config.coaddName + self.inputCatalog + "_schema").schema
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Wait, are tasks ever initialized with a butler in gen3?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was wondering. The code is clearly never called because if it was called it would break... I wasn't brave enough to remove the butler parameter completely from the 8 places that use it but maybe I should.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think I want to get into that on this ticket though.

self.schemaMapper = afwTable.SchemaMapper(schema)
self.schemaMapper.addMinimalSchema(schema)
self.schema = self.schemaMapper.getOutputSchema()
Expand Down
18 changes: 6 additions & 12 deletions python/lsst/pipe/tasks/processCcdWithFakes.py
Original file line number Diff line number Diff line change
Expand Up @@ -337,16 +337,14 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
externalSkyWcsCatalog = None
for externalSkyWcsCatalogRef in externalSkyWcsCatalogList:
if externalSkyWcsCatalogRef.dataId["tract"] == tractId:
externalSkyWcsCatalog = externalSkyWcsCatalogRef.get(
datasetType=self.config.connections.externalSkyWcsTractCatalog)
externalSkyWcsCatalog = externalSkyWcsCatalogRef.get()
break
if externalSkyWcsCatalog is None:
usedTract = externalSkyWcsCatalogList[-1].dataId["tract"]
self.log.warn(
f"Warning, external SkyWcs for tract {tractId} not found. Using tract {usedTract} "
"instead.")
externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get(
datasetType=self.config.connections.externalSkyWcsTractCatalog)
externalSkyWcsCatalog = externalSkyWcsCatalogList[-1].get()
row = externalSkyWcsCatalog.find(detectorId)
if row is None:
self.log.info("No %s external tract sky WCS for exposure %s so cannot insert fake "
Expand All @@ -373,16 +371,14 @@ def runQuantum(self, butlerQC, inputRefs, outputRefs):
externalPhotoCalibCatalog = None
for externalPhotoCalibCatalogRef in externalPhotoCalibCatalogList:
if externalPhotoCalibCatalogRef.dataId["tract"] == tractId:
externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get(
datasetType=self.config.connections.externalPhotoCalibTractCatalog)
externalPhotoCalibCatalog = externalPhotoCalibCatalogRef.get()
break
if externalPhotoCalibCatalog is None:
usedTract = externalPhotoCalibCatalogList[-1].dataId["tract"]
self.log.warn(
f"Warning, external PhotoCalib for tract {tractId} not found. Using tract {usedTract} "
"instead.")
externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get(
datasetType=self.config.connections.externalPhotoCalibTractCatalog)
externalPhotoCalibCatalog = externalPhotoCalibCatalogList[-1].get()
row = externalPhotoCalibCatalog.find(detectorId)
if row is None:
self.log.info("No %s external tract photoCalib for exposure %s so cannot insert fake "
Expand Down Expand Up @@ -484,12 +480,10 @@ def composeFakeCat(self, fakeCats, skyMap):
quantum.
"""
if len(fakeCats) == 1:
return fakeCats[0].get(
datasetType=self.config.connections.fakeCats)
return fakeCats[0].get()
outputCat = []
for fakeCatRef in fakeCats:
cat = fakeCatRef.get(
datasetType=self.config.connections.fakeCats)
cat = fakeCatRef.get()
tractId = fakeCatRef.dataId["tract"]
# Make sure all data is within the inner part of the tract.
outputCat.append(cat[
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/pipe/tasks/propagateVisitFlags.py
Original file line number Diff line number Diff line change
Expand Up @@ -223,7 +223,7 @@ def processCcd(ccdSources, wcsUpdate):
v = ccdRecord.get(visitKey)
c = ccdRecord.get(ccdKey)
dataId = {"visit": int(v), self.config.ccdName: int(c)}
ccdSources = butler.get("src", dataId=dataId, immediate=True)
ccdSources = butler.get("src", dataId=dataId)
processCcd(ccdSources, ccdRecord.getWcs())

# Apply threshold
Expand Down
5 changes: 1 addition & 4 deletions tests/assembleCoaddTestUtils.py
Original file line number Diff line number Diff line change
Expand Up @@ -71,7 +71,7 @@ def __init__(self, exposure, coaddName='deep', patch=42, tract=0, visit=100):
self.patch = patch
self.visit = visit

def get(self, bbox=None, component=None, parameters=None, **kwargs):
def get(self, bbox=None, component=None, parameters=None):
"""Retrieve the specified dataset using the API of the Gen 3 Butler.

Parameters
Expand All @@ -83,9 +83,6 @@ def get(self, bbox=None, component=None, parameters=None, **kwargs):
parameters : `dict`, optional
If supplied, use the parameters to modify the exposure,
typically by taking a subset.
**kwargs
Additional keyword arguments such as `immediate=True` that would
control internal butler behavior.

Returns
-------
Expand Down
21 changes: 3 additions & 18 deletions tests/surveyPropertyMapsTestUtils.py
Original file line number Diff line number Diff line change
Expand Up @@ -195,15 +195,9 @@ def __init__(self, visit_summary, visit):
self.visit_summary = visit_summary
self.visit = visit

def get(self, **kwargs):
def get(self):
"""Retrieve the specified dataset using the API of the Gen3 Butler.

Parameters
----------
**kwargs :
Additional keyword arguments such as `immediate=True` that would
control internal butler behavior.

Returns
-------
visit_summary : `lsst.afw.table.ExposureCatalog`
Expand Down Expand Up @@ -232,17 +226,14 @@ def __init__(self, exposure, coaddName="deep", patch=0, tract=0):
self.tract = tract
self.patch = patch

def get(self, component=None, **kwargs):
def get(self, component=None):
"""Retrieve the specified dataset using the API of the Gen 3 Butler.

Parameters
----------
component : `str`, optional
If supplied, return the named metadata of the exposure. Allowed
components are "photoCalib" or "coaddInputs".
**kwargs
Additional keyword arguments such as `immediate=True` that would
control internal butler behavior.

Returns
-------
Expand Down Expand Up @@ -276,16 +267,10 @@ def __init__(self, input_map, patch=0, tract=0):
self.tract = tract
self.patch = patch

def get(self, **kwargs):
def get(self):
"""
Retrieve the specified dataset using the API of the Gen 3 Butler.

Parameters
----------
**kwargs
Additional keyword arguments such as `immediate=True` that would
control internal butler behavior.

Returns
-------
input_map : `healsparse.HealSparseMap`
Expand Down