Skip to content

Commit

Permalink
Write dimension version info to export yaml
Browse files Browse the repository at this point in the history
And use that version on import.
  • Loading branch information
timj committed May 27, 2022
1 parent c6f08fb commit 4164d9c
Show file tree
Hide file tree
Showing 4 changed files with 44 additions and 4 deletions.
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -2044,7 +2044,7 @@ def export(
filename = os.path.join(directory, filename)
BackendClass = get_class_of(self._config["repo_transfer_formats"][format]["export"])
with open(filename, "w") as stream:
backend = BackendClass(stream)
backend = BackendClass(stream, universe=self.registry.dimensions)
try:
helper = RepoExportContext(
self.registry, self.datastore, backend=backend, directory=directory, transfer=transfer
Expand Down
26 changes: 25 additions & 1 deletion python/lsst/daf/butler/core/dimensions/_universe.py
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,30 @@ def __new__(

return self

@property
def version(self) -> int:
"""The version number of this universe.
Returns
-------
version : `int`
An integer representing the version number of this universe.
Uniquely defined when combined with the `namespace`.
"""
return self._version

@property
def namespace(self) -> str:
"""The namespace associated with this universe.
Returns
-------
namespace : `str`
The namespace. When combined with the `version` can uniquely
define this universe.
"""
return self._namespace

def __repr__(self) -> str:
return f"DimensionUniverse({self._version}, {self._namespace})"

Expand Down Expand Up @@ -495,4 +519,4 @@ def __deepcopy__(self, memo: dict) -> DimensionUniverse:

_version: int

_namespace: Optional[str]
_namespace: str
15 changes: 13 additions & 2 deletions python/lsst/daf/butler/transfers/_yaml.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@
Datastore,
DimensionElement,
DimensionRecord,
DimensionUniverse,
FileDataset,
Timespan,
)
Expand Down Expand Up @@ -94,8 +95,9 @@ class YamlRepoExportBackend(RepoExportBackend):
A writeable file-like object.
"""

def __init__(self, stream: IO):
def __init__(self, stream: IO, universe: DimensionUniverse):
self.stream = stream
self.universe = universe
self.data: List[Dict[str, Any]] = []

def saveDimensionData(self, element: DimensionElement, *data: DimensionRecord) -> None:
Expand Down Expand Up @@ -195,6 +197,8 @@ def finish(self) -> None:
{
"description": "Butler Data Repository Export",
"version": str(EXPORT_FORMAT_VERSION),
"universe_version": self.universe.version,
"universe_namespace": self.universe.namespace,
"data": self.data,
},
stream=self.stream,
Expand Down Expand Up @@ -248,13 +252,20 @@ def __init__(self, stream: IO, registry: Registry):
self.refsByFileId: Dict[DatasetId, DatasetRef] = {}
self.registry: Registry = registry

universe_version = wrapper.get("universe_version", 0)
universe_namespace = wrapper.get("universe_namespace", "daf_butler")

# If this is data exported before the reorganization of visits
# and visit systems and that new schema is in use, some filtering
# will be needed. The entry in the visit dimension record will be
# silently dropped when visit is created but the
# visit_system_membership must be constructed.
migrate_visit_system = False
if fileVersion < VersionTuple(1, 0, 2) and "visit_system_membership" in self.registry.dimensions:
if (
universe_version < 2
and universe_namespace == "daf_butler"
and "visit_system_membership" in self.registry.dimensions
):
migrate_visit_system = True

datasetData = []
Expand Down
5 changes: 5 additions & 0 deletions tests/test_dimensions.py
Original file line number Diff line number Diff line change
Expand Up @@ -128,6 +128,11 @@ def testConfigPresent(self):
config = self.universe.dimensionConfig
self.assertIsInstance(config, DimensionConfig)

def testVersion(self):
self.assertEqual(self.universe.namespace, "daf_butler")
# Test was added starting at version 2.
self.assertGreaterEqual(self.universe.version, 2)

def testConfigRead(self):
self.assertEqual(
set(self.universe.getStaticDimensions().names),
Expand Down

0 comments on commit 4164d9c

Please sign in to comment.