Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

DM-17611: performance optimizations for data ID manipulations #126

Merged
merged 7 commits into from
Feb 2, 2019
Merged

DM-17611: performance optimizations for data ID manipulations #126

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
95650e6
Reduce unnecessary DimensionNameSet construction.
TallJimbo Jan 30, 2019
14c5828
Compute the full set of DimensionElement dependencies up-front.
TallJimbo Jan 30, 2019
2799326
Cache DimensionSet.links.
TallJimbo Jan 30, 2019
7454fb2
Move assertion after short-cut in recursion.
TallJimbo Jan 30, 2019
aae512a
Make universe DimensionGraphs their own subclass.
TallJimbo Jan 30, 2019
5187f0e
Add cache to DimensionGraph construction.
TallJimbo Jan 30, 2019
d4bffbf
Drop row-wide data ID expansion from selectDimensions.
TallJimbo Jan 30, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
9 changes: 4 additions & 5 deletions python/lsst/daf/butler/core/dimensions/dataId.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,7 +297,7 @@ def __new__(cls, dataId=None, *, dimensions=None, dimension=None, universe=None,

assert isinstance(dimensions, DimensionGraph), "should be set by earlier logic"

allDimensions = DimensionGraph(dimensions.universe, dimensions=dimensions, implied=True)
allDimensions = dimensions.universe.extract(dimensions=dimensions, implied=True)

if isinstance(dataId, DataId):

Expand All @@ -315,8 +315,7 @@ def hasLinkValueChanged(linkName):
)

if changedLinkValues:
constantDimensions = DimensionGraph(
dimensions.universe,
constantDimensions = dimensions.universe.extract(
dimensions=[d for d in dimensions if d.links().isdisjoint(changedLinkValues)],
implied=True
)
Expand Down Expand Up @@ -395,7 +394,7 @@ def __init__(self, dataId=None, *, dimensions=None, dimension=None, universe=Non
for linkName in missing:
# Didn't get enough key-value pairs to identify all dimensions
# from the links; look in entries for those.
for element in self.dimensions().withLink(linkName):
for element in self.dimensions().universe.withLink(linkName):
try:
self._linkValues[linkName] = self.entries[element][linkName]
break
Expand All @@ -416,7 +415,7 @@ def __init__(self, dataId=None, *, dimensions=None, dimension=None, universe=Non
def addLinksToEntries(items):
for linkName, linkValue in items:
try:
associated = self.dimensions(implied=True).withLink(linkName)
associated = self.dimensions().universe.withLink(linkName)
except KeyError:
# This isn't a link. If an explicit dimension was
# provided, assume these fields are metadata for that
Expand Down
3 changes: 2 additions & 1 deletion python/lsst/daf/butler/core/dimensions/elements.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -69,6 +69,7 @@ def __init__(self, universe, name, hasRegion, links, required, implied, doc):
self._doc = doc
self._requiredDependencies = DimensionSet(universe, required, expand=True, implied=False)
self._impliedDependencies = DimensionSet(universe, implied, expand=False)
self._allDependencies = self._requiredDependencies | self._impliedDependencies
# Compute _primaryKeys dict, used to back primaryKeys property.
expandedLinks = set(self._localLinks)
for dimension in self.dependencies(implied=False):
Expand Down Expand Up @@ -148,7 +149,7 @@ def dependencies(self, required=True, implied=False):
"""
if required:
if implied:
return self._requiredDependencies | self._impliedDependencies
return self._allDependencies
else:
return self._requiredDependencies
elif implied:
Expand Down