Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-8212: Add support for reading metadata, length and schema of a catalog #11

Merged
merged 2 commits into from
Nov 14, 2016
Merged

DM-8212: Add support for reading metadata, length and schema of a catalog #11

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
b8a6266
CameraMapper: simplify setting of dataset mappings
PaulPrice Nov 14, 2016
d33c0b1
CameraMapper: add extra dataset features for catalogs
PaulPrice Nov 14, 2016
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
187 changes: 111 additions & 76 deletions python/lsst/obs/base/cameraMapper.py
View file
Open in desktop
Expand Up @@ -35,6 +35,7 @@
import lsst.daf.base as dafBase
import lsst.afw.geom as afwGeom
import lsst.afw.image as afwImage
import lsst.afw.table as afwTable
import lsst.afw.cameraGeom as afwCameraGeom
import lsst.log as lsstLog
import lsst.pex.policy as pexPolicy
Expand Down Expand Up @@ -260,6 +261,55 @@ def __init__(self, policy, repositoryDir,
else:
calibRegistry = None

# Dict of valid keys and their value types
self.keyDict = dict()

self._initMappings(policy, root, calibRoot, calibRegistry, provided=None)

# Camera geometry
self.cameraDataLocation = None # path to camera geometry config file
self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir)

# Defect registry and root
self.defectRegistry = None
if 'defects' in policy:
self.defectPath = os.path.join(repositoryDir, policy['defects'])
defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)

# Filter translation table
self.filters = None

# Skytile policy
self.skypolicy = policy['skytiles']

# verify that the class variable packageName is set before attempting
# to instantiate an instance
if self.packageName is None:
raise ValueError('class variable packageName must not be None')

self.makeRawVisitInfo = self.MakeRawVisitInfoClass(log=self.log)

def _initMappings(self, policy, root=None, calibRoot=None, calibRegistry=None, provided=None):
"""Initialize mappings

For each of the dataset types that we want to be able to read, there are
methods that can be created to support them:
* map_<dataset> : determine the path for dataset
* std_<dataset> : standardize the retrieved dataset
* bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery)
* query_<dataset> : query the registry

Besides the dataset types explicitly listed in the policy, we create
additional, derived datasets for additional conveniences, e.g., reading
the header of an image, retrieving only the size of a catalog.

@param policy (Policy) Policy with per-camera defaults already merged
@param root (string) Root directory for data
@param calibRoot (string) Root directory for calibrations
@param calibRegistry (string) Path to registry with calibrations' metadata
@param provided (list of strings) Keys provided by the mapper
"""
# Sub-dictionaries (for exposure/calibration/dataset types)
imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
"obs_base", "ImageMappingDictionary.paf", "policy"))
Expand All @@ -270,9 +320,6 @@ def __init__(self, policy, repositoryDir,
dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
"obs_base", "DatasetMappingDictionary.paf", "policy"))

# Dict of valid keys and their value types
self.keyDict = dict()

# Mappings
mappingList = (
("images", imgMappingPolicy, ImageMapping),
Expand Down Expand Up @@ -338,80 +385,68 @@ def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
return mapping.standardize(mapper, item, dataId)
setattr(self, "std_" + datasetType, stdClosure)

mapFunc = "map_" + datasetType + "_filename"
bypassFunc = "bypass_" + datasetType + "_filename"
if not hasattr(self, mapFunc):
setattr(self, mapFunc, getattr(self, "map_" + datasetType))
if not hasattr(self, bypassFunc):
setattr(self, bypassFunc,
lambda datasetType, pythonType, location, dataId: location.getLocations())

# Set up metadata versions
if name == "exposures" or name == "images":
expFunc = "map_" + datasetType # Function name to map exposure
mdFunc = expFunc + "_md" # Function name to map metadata
bypassFunc = "bypass_" + datasetType + "_md" # Func name to bypass daf_persistence
if not hasattr(self, mdFunc):
setattr(self, mdFunc, getattr(self, expFunc))
if not hasattr(self, bypassFunc):
setattr(self, bypassFunc,
lambda datasetType, pythonType, location, dataId:
afwImage.readMetadata(location.getLocations()[0]))
if not hasattr(self, "query_" + datasetType + "_md"):
setattr(self, "query_" + datasetType + "_md",
getattr(self, "query_" + datasetType))

subFunc = expFunc + "_sub" # Function name to map subimage
if not hasattr(self, subFunc):
def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self),
mapping=mapping):
subId = dataId.copy()
del subId['bbox']
loc = mapping.map(mapper, subId, write)
bbox = dataId['bbox']
llcX = bbox.getMinX()
llcY = bbox.getMinY()
width = bbox.getWidth()
height = bbox.getHeight()
loc.additionalData.set('llcX', llcX)
loc.additionalData.set('llcY', llcY)
loc.additionalData.set('width', width)
loc.additionalData.set('height', height)
if 'imageOrigin' in dataId:
loc.additionalData.set('imageOrigin',
dataId['imageOrigin'])
return loc
setattr(self, subFunc, mapSubClosure)
if not hasattr(self, "query_" + datasetType + "_sub"):
def querySubClosure(key, format, dataId, mapping=mapping):
subId = dataId.copy()
del subId['bbox']
return mapping.lookup(format, subId)
setattr(self, "query_" + datasetType + "_sub", querySubClosure)

# Camera geometry
self.cameraDataLocation = None # path to camera geometry config file
self.camera = self._makeCamera(policy=policy, repositoryDir=repositoryDir)

# Defect registry and root
self.defectRegistry = None
if 'defects' in policy:
self.defectPath = os.path.join(repositoryDir, policy['defects'])
defectRegistryLocation = os.path.join(self.defectPath, "defectRegistry.sqlite3")
self.defectRegistry = dafPersist.Registry.create(defectRegistryLocation)

# Filter translation table
self.filters = None
def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I feel like this whole subPolicy block that you've cleanedup/added (basically most of the contents of this inner for loop) should be pulled into a separate _addPolicies method. __init__ is really long as it stands, and this could help break it up (I didn't even see that this was all in a double-for loop!).

"""Set convenience methods on CameraMapper"""
mapName = "map_" + datasetType + "_" + suffix
bypassName = "bypass_" + datasetType + "_" + suffix
queryName = "query_" + datasetType + "_" + suffix
if not hasattr(self, mapName):
setattr(self, mapName, mapImpl or getattr(self, "map_" + datasetType))
if not hasattr(self, bypassName):
if bypassImpl is None and hasattr(self, "bypass_" + datasetType):
bypassImpl = getattr(self, "bypass_" + datasetType)
if bypassImpl is not None:
setattr(self, bypassName, bypassImpl)
if not hasattr(self, queryName):
setattr(self, queryName, queryImpl or getattr(self, "query_" + datasetType))

# Filename of dataset
setMethods("filename", bypassImpl=lambda datasetType, pythonType, location, dataId:
location.getLocations())

# Metadata from FITS file
if subPolicy["storage"] == "FitsStorage": # a FITS image
setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwImage.readMetadata(location.getLocations()[0]))
if subPolicy["storage"] == "FitsCatalogStorage": # a FITS catalog
setMethods("md", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwImage.readMetadata(location.getLocations()[0], 2))

# Sub-images
if subPolicy["storage"] == "FitsStorage":
def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
subId = dataId.copy()
del subId['bbox']
loc = mapping.map(mapper, subId, write)
bbox = dataId['bbox']
llcX = bbox.getMinX()
llcY = bbox.getMinY()
width = bbox.getWidth()
height = bbox.getHeight()
loc.additionalData.set('llcX', llcX)
loc.additionalData.set('llcY', llcY)
loc.additionalData.set('width', width)
loc.additionalData.set('height', height)
if 'imageOrigin' in dataId:
loc.additionalData.set('imageOrigin',
dataId['imageOrigin'])
return loc
def querySubClosure(key, format, dataId, mapping=mapping):
subId = dataId.copy()
del subId['bbox']
return mapping.lookup(format, subId)
setMethods("sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)

if subPolicy["storage"] == "FitsCatalogStorage":
# Length of catalog
setMethods("len", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwImage.readMetadata(location.getLocations()[0], 2).get("NAXIS2"))

# Schema of catalog
if not datasetType.endswith("_schema") and datasetType + "_schema" not in datasets:
setMethods("schema", bypassImpl=lambda datasetType, pythonType, location, dataId:
afwTable.Schema.readFits(location.getLocations()[0]))

# Skytile policy
self.skypolicy = policy['skytiles']

# verify that the class variable packageName is set before attempting
# to instantiate an instance
if self.packageName is None:
raise ValueError('class variable packageName must not be None')

self.makeRawVisitInfo = self.MakeRawVisitInfoClass(log=self.log)

def _computeCcdExposureId(self, dataId):
"""Compute the 64-bit (long) identifier for a CCD exposure.
Expand Down
10 changes: 5 additions & 5 deletions tests/MinMapper2.paf
View file
Open in desktop
Expand Up @@ -64,11 +64,11 @@ calibrations: {
}
}
datasets: {
src: {
template: "src-v%(visit)d-c%(ccd)02d.boost"
python: "lsst.afw.detection.PersistableSourceVector"
persistable: "PersistableSourceVector"
storage: "BoostStorage"
someCatalog: {
template: "catalog-v%(visit)d-c%(ccd)02d.fits"
python: "lsst.afw.table.BaseCatalog"
persistable: "BasesCatalog"
storage: "FitsCatalogStorage"
tables: "raw"
}
}
33 changes: 30 additions & 3 deletions tests/testCameraMapper.py
View file
Open in desktop
Expand Up @@ -29,6 +29,7 @@

import lsst.utils.tests
import lsst.afw.geom as afwGeom
import lsst.afw.table as afwTable
import lsst.daf.persistence as dafPersist
import lsst.obs.base
from lsst.utils import getPackageDir
Expand Down Expand Up @@ -163,10 +164,11 @@ def tearDown(self):
def testGetDatasetTypes(self):
expectedTypes = BaseMapper().getDatasetTypes()
# Add the expected additional types to what the base class provides
expectedTypes.extend(["flat", "flat_filename", "raw", "raw_md",
"raw_filename", "raw_sub",
expectedTypes.extend(["flat", "flat_md", "flat_filename", "flat_sub",
"raw", "raw_md", "raw_filename", "raw_sub",
"some", "some_filename", "some_md", "some_sub",
"src", "src_filename",
"someCatalog", "someCatalog_md", "someCatalog_filename",
"someCatalog_len", "someCatalog_schema",
"other_sub", "other_filename", "other_md", "other",
"someGz", "someGz_filename", "someFz", "someFz_filename", "someGz_md",
"someFz_sub", "someFz_md", "someGz_sub"
Expand Down Expand Up @@ -209,6 +211,31 @@ def testSubMap(self):
'ccd = 13\nheight = 400\nimageOrigin = "PARENT"\n'
'llcX = 200\nllcY = 100\nwidth = 300\n')

def testCatalogExtras(self):
butler = dafPersist.Butler(mapper=self.mapper)
schema = afwTable.Schema()
aa = schema.addField("a", type=int, doc="a")
bb = schema.addField("b", type=float, doc="b")
catalog = lsst.afw.table.BaseCatalog(schema)
row = catalog.addNew()
row.set(aa, 12345)
row.set(bb, 1.2345)
size = len(catalog)
dataId = dict(visit=123, ccd=45)
filename = butler.get("someCatalog_filename", dataId)[0]
butler.put(catalog, "someCatalog", dataId)
try:
self.assertTrue(os.path.exists(filename))
self.assertEqual(butler.get("someCatalog_schema", dataId), schema)
self.assertEqual(butler.get("someCatalog_len", dataId), size)
header = butler.get("someCatalog_md", dataId)
self.assertEqual(header.get("NAXIS2"), size)
finally:
try:
os.remove(filename)
except OSError as exc:
print("Warning: could not remove file %r: %s" % (filename, exc))

def testImage(self):
loc = self.mapper.map("some", dict(ccd=35))
expectedLocations = [os.path.join(testDir, "bar-35.fits")]
Expand Down