Skip to content

Commit

Permalink
Update assembler interface to know its own storage class
Browse files Browse the repository at this point in the history 
Most of the Assembler methods needed to know the storage class.
Therefore StorageClass now returns an assembler with an instance
of itself attached. This simplifies the interface.
  • Loading branch information
@timj
timj committed Feb 20, 2018
1 parent 89e6cbb commit 75fb57f
Show file tree
Hide file tree
Showing 7 changed files with 49 additions and 59 deletions.
47 changes: 18 additions & 29 deletions python/lsst/daf/butler/assemblers/exposureAssembler.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,9 @@ class ExposureAssembler(CompositeAssembler):
EXPOSURE_INFO_COMPONENTS = set(("apCorrMap", "coaddInputs", "calib", "metadata",
"filter", "transmissionCurve", "visitInfo"))

@classmethod
def _groupRequestedComponents(cls, storageClass):
def _groupRequestedComponents(self):
"""Group requested components into top level and ExposureInfo.
Parameters
----------
storageClass : `StorageClass`
Place to retrieve list of requested components.
Returns
-------
expComps : `dict`
Expand All @@ -56,15 +50,15 @@ def _groupRequestedComponents(cls, storageClass):
There are components defined in the storage class that are not
expected by this assembler.
"""
requested = set(storageClass.components.keys())
requested = set(self.storageClass.components.keys())

# Check that we are requesting something that we support
unknown = requested - (cls.EXPOSURE_COMPONENTS | cls.EXPOSURE_INFO_COMPONENTS)
unknown = requested - (self.EXPOSURE_COMPONENTS | self.EXPOSURE_INFO_COMPONENTS)
if unknown:
raise ValueError("Asking for unrecognized component: {}".format(unknown))

expItems = requested & cls.EXPOSURE_COMPONENTS
expInfoItems = requested & cls.EXPOSURE_INFO_COMPONENTS
expItems = requested & self.EXPOSURE_COMPONENTS
expInfoItems = requested & self.EXPOSURE_INFO_COMPONENTS
return expItems, expInfoItems

def getComponent(self, composite, componentName):
Expand Down Expand Up @@ -100,33 +94,30 @@ def getComponent(self, composite, componentName):
raise AttributeError("Do not know how to retrieve component {} from {}".format(componentName,
type(composite)))

def getValidComponents(self, composite, storageClass):
def getValidComponents(self, composite):
"""Extract all non-None components from a composite.
Parameters
----------
composite : `object`
Composite from which to extract components.
storageClass : `StorageClass`
`StorageClass` associated with this composite object.
If None, it is assumed this is not to be treated as a composite.
Returns
-------
comps : `dict`
Non-None components extracted from the composite, indexed by the component
name as derived from the `StorageClass`.
name as derived from the `self.storageClass`.
"""
# For Exposure we call the generic version twice: once for top level
# components, and again for ExposureInfo.
expItems, expInfoItems = self._groupRequestedComponents(storageClass)
expItems, expInfoItems = self._groupRequestedComponents()

components = super().getValidComponents(composite, storageClass)
infoComps = super().getValidComponents(composite.getInfo(), storageClass)
components = super().getValidComponents(composite)
infoComps = super().getValidComponents(composite.getInfo())
components.update(infoComps)
return components

def disassemble(self, composite, storageClass):
def disassemble(self, composite):
"""Disassemble an afw Exposure.
This implementation attempts to extract components from the parent
Expand All @@ -137,13 +128,11 @@ def disassemble(self, composite, storageClass):
----------
composite : `lsst.afw.Exposure`
`Exposure` composite object consisting of components to be extracted.
storageClass : `StorageClass`
`StorageClass` associated with the parent, with defined components.
Returns
-------
components : `dict`
`dict` with keys matching the components defined in `storageClass`
`dict` with keys matching the components defined in `self.storageClass`
and values being `DatasetComponent` instances describing the component.
Raises
Expand All @@ -152,20 +141,20 @@ def disassemble(self, composite, storageClass):
A requested component can not be found in the parent using generic
lookups.
TypeError
The parent object does not match the supplied `StorageClass`.
The parent object does not match the supplied `self.storageClass`.
"""
if not storageClass.validateInstance(composite):
if not self.storageClass.validateInstance(composite):
raise TypeError("Unexpected type mismatch between parent and StorageClass"
" ({} != {})".format(type(composite), storageClass.pytype))
" ({} != {})".format(type(composite), self.storageClass.pytype))

# Only look for components that are defined by the StorageClass
components = {}
expItems, expInfoItems = self._groupRequestedComponents(storageClass)
expItems, expInfoItems = self._groupRequestedComponents()

fromExposure = super().disassemble(composite, storageClass, subset=expItems)
fromExposure = super().disassemble(composite, subset=expItems)
components.update(fromExposure)

fromExposureInfo = super().disassemble(composite, storageClass,
fromExposureInfo = super().disassemble(composite,
subset=expInfoItems, override=composite.getInfo())
components.update(fromExposureInfo)

Expand Down
47 changes: 23 additions & 24 deletions python/lsst/daf/butler/core/composites.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,8 +48,15 @@ def __init__(self, name, storageClass, component):
class CompositeAssembler:
"""Class for providing assembler and disassembler support for composites.
Parameters
----------
storageClass : `StorageClass`
`StorageClass` to be used with this assembler.
"""

def __init__(self, storageClass):
self.storageClass = storageClass

@staticmethod
def _attrNames(componentName, getter=True):
"""Return list of suitable attribute names to attempt to use.
Expand Down Expand Up @@ -78,7 +85,7 @@ def _attrNames(componentName, getter=True):
capitalized = "{}{}{}".format(root, first, tail)
return (componentName, "{}_{}".format(root, componentName), capitalized)

def assemble(self, storageClass, components, pytype=None):
def assemble(self, components, pytype=None):
"""Construct an object from components based on storageClass.
This generic implementation assumes that instances of objects
Expand All @@ -87,9 +94,6 @@ def assemble(self, storageClass, components, pytype=None):
Parameters
----------
storageClass : `StorageClass`
`StorageClass` describing the entity to be created from the
components.
components : `dict`
Collection of components from which to assemble a new composite
object. Keys correspond to composite names in the `StorageClass`.
Expand All @@ -112,10 +116,10 @@ def assemble(self, storageClass, components, pytype=None):
if pytype is not None:
cls = pytype
else:
cls = storageClass.pytype
cls = self.storageClass.pytype

# Check that the storage class components are consistent
understood = set(storageClass.components)
understood = set(self.storageClass.components)
requested = set(components.keys())
unknown = requested - understood
if unknown:
Expand Down Expand Up @@ -151,26 +155,23 @@ def assemble(self, storageClass, components, pytype=None):

return obj

def getValidComponents(self, composite, storageClass):
def getValidComponents(self, composite):
"""Extract all non-None components from a composite.
Parameters
----------
composite : `object`
Composite from which to extract components.
storageClass : `StorageClass`
`StorageClass` associated with this composite object.
If None, it is assumed this is not to be treated as a composite.
Returns
-------
comps : `dict`
Non-None components extracted from the composite, indexed by the
component name as derived from the `StorageClass`.
component name as derived from the `self.storageClass`.
"""
components = {}
if storageClass is not None and storageClass.components:
for c in storageClass.components:
if self.storageClass is not None and self.storageClass.components:
for c in self.storageClass.components:
if isinstance(composite, collections.Mapping):
comp = composite[c]
else:
Expand Down Expand Up @@ -219,7 +220,7 @@ def getComponent(self, composite, componentName):
raise AttributeError("Unable to get component {}".format(componentName))
return component

def disassemble(self, composite, storageClass, subset=None, override=None):
def disassemble(self, composite, subset=None, override=None):
"""Generic implementation of a disassembler.
This implementation attempts to extract components from the parent
Expand All @@ -230,11 +231,9 @@ def disassemble(self, composite, storageClass, subset=None, override=None):
----------
composite : `object`
Parent composite object consisting of components to be extracted.
storageClass : `StorageClass`
`StorageClass` associated with the parent, with defined components.
subset : `iterable`, optional
Iterable containing subset of components to extract from composite.
Must be a subset of those defined in `storageClass`.
Must be a subset of those defined in `self.storageClass`.
override : `object`, optional
Object to use for disassembly instead of parent. This can be useful
when called from subclasses that have composites in a hierarchy.
Expand All @@ -244,24 +243,24 @@ def disassemble(self, composite, storageClass, subset=None, override=None):
components : `dict`
`dict` with keys matching the components defined in `storageClass`
and values being `DatasetComponent` instances describing the
component. Returns None if this is not a composite `StorageClass`.
component. Returns None if this is not a composite `self.storageClass`.
Raises
------
ValueError
A requested component can not be found in the parent using generic
lookups.
TypeError
The parent object does not match the supplied `StorageClass`.
The parent object does not match the supplied `self.storageClass`.
"""
if storageClass.components is None:
if self.storageClass.components is None:
return

if not storageClass.validateInstance(composite):
if not self.storageClass.validateInstance(composite):
raise TypeError("Unexpected type mismatch between parent and StorageClass"
" ({} != {})".format(type(composite), storageClass.pytype))
" ({} != {})".format(type(composite), self.storageClass.pytype))

requested = set(storageClass.components.keys())
requested = set(self.storageClass.components)

if subset is not None:
subset = set(subset)
Expand All @@ -286,7 +285,7 @@ def disassemble(self, composite, storageClass, subset=None, override=None):
# If we found a match store it in the results dict and remove
# it from the list of components we are still looking for.
if component is not None:
components[c] = DatasetComponent(c, storageClass.components[c], component)
components[c] = DatasetComponent(c, self.storageClass.components[c], component)
requested.remove(c)

if requested:
Expand Down
6 changes: 4 additions & 2 deletions python/lsst/daf/butler/core/storageClass.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -93,9 +93,11 @@ def assembler(self):
Returns
-------
assembler : `CompositeAssembler`
Instance of the assembler associated with this `StorageClass`
Instance of the assembler associated with this `StorageClass`.
Assembler is constructed with a new instance of this
`StorageClass`.
"""
return self.assemblerClass()
return self.assemblerClass(storageClass=type(self)())

def validateInstance(self, instance):
"""Check that the supplied instance has the expected Python type
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/datastores/posixDatastore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,7 +163,7 @@ def put(self, inMemoryDataset, storageClass, storageHint, typeName=None):
# and also has components
if storageClass.assemblerClass.disassemble is not None and storageClass.components:
compUris = {}
components = storageClass.assembler().disassemble(inMemoryDataset, storageClass)
components = storageClass.assembler().disassemble(inMemoryDataset)
if components:
for comp, info in components.items():
compTypeName = typeName
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/formatters/fileFormatter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ def write(self, inMemoryDataset, fileDescriptor):

# Get the list of valid components so we can build URIs
storageClass = fileDescriptor.storageClass
components = storageClass.assembler().getValidComponents(inMemoryDataset, storageClass)
components = storageClass.assembler().getValidComponents(inMemoryDataset)

return (fileDescriptor.location.uri,
{c: fileDescriptor.location.componentUri(c) for c in components})
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/formatters/jsonFormatter.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -95,5 +95,5 @@ def _coerceType(self, inMemoryDataset, storageClass, pytype=None):
Object of expected type `pytype`.
"""
if not hasattr(builtins, pytype.__name__):
inMemoryDataset = storageClass.assembler().assemble(storageClass, inMemoryDataset, pytype=pytype)
inMemoryDataset = storageClass.assembler().assemble(inMemoryDataset, pytype=pytype)
return inMemoryDataset
2 changes: 1 addition & 1 deletion tests/test_posixDatastore.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,7 @@ def testCompositePutGet(self):
components[c] = datastore.get(u, storageClass=storageClass.components[c], parameters=None)

# combine them into a new metrics composite object
metricsOut = storageClass.assembler().assemble(storageClass, components)
metricsOut = storageClass.assembler().assemble(components)
self.assertEqualMetrics(metrics, metricsOut)

def testRemove(self):
Expand Down

0 comments on commit 75fb57f

Please sign in to comment.