Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DM-26683: Make exporting dimension data friendlier #375

Merged
merged 10 commits into from
Sep 17, 2020
1 change: 1 addition & 0 deletions python/lsst/daf/butler/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,4 +11,5 @@
from ._butlerConfig import *
from ._deferredDatasetHandle import *
from ._butler import *
from .transfers import YamlRepoExportBackend, YamlRepoImportBackend
from .version import *
12 changes: 6 additions & 6 deletions python/lsst/daf/butler/_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@
DatasetType,
Datastore,
FileDataset,
RepoExport,
StorageClassFactory,
ValidationError,
)
Expand All @@ -72,6 +71,7 @@
from ._butlerConfig import ButlerConfig
from .registry import Registry, RegistryConfig, CollectionType
from .registry.wildcards import CollectionSearch
from .transfers import RepoExportContext

log = logging.getLogger(__name__)

Expand Down Expand Up @@ -1205,12 +1205,12 @@ def ingest(self, *datasets: FileDataset, transfer: Optional[str] = "auto", run:
def export(self, *, directory: Optional[str] = None,
filename: Optional[str] = None,
format: Optional[str] = None,
transfer: Optional[str] = None) -> ContextManager[RepoExport]:
transfer: Optional[str] = None) -> ContextManager[RepoExportContext]:
"""Export datasets from the repository represented by this `Butler`.

This method is a context manager that returns a helper object
(`RepoExport`) that is used to indicate what information from the
repository should be exported.
(`RepoExportContext`) that is used to indicate what information from
the repository should be exported.

Parameters
----------
Expand Down Expand Up @@ -1266,8 +1266,8 @@ def export(self, *, directory: Optional[str] = None,
with open(filename, 'w') as stream:
backend = BackendClass(stream)
try:
helper = RepoExport(self.registry, self.datastore, backend=backend,
directory=directory, transfer=transfer)
helper = RepoExportContext(self.registry, self.datastore, backend=backend,
directory=directory, transfer=transfer)
yield helper
except BaseException:
raise
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/_butlerConfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,10 +37,10 @@
Config,
DatastoreConfig,
DimensionConfig,
RepoTransferFormatConfig,
StorageClassConfig,
)
from .registry import RegistryConfig
from .transfers import RepoTransferFormatConfig

CONFIG_COMPONENT_CLASSES = (RegistryConfig, StorageClassConfig,
DatastoreConfig, DimensionConfig,
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,6 @@
from .storageClassDelegate import *
from .storedFileInfo import *
from .dimensions import *
from .repoTransfers import *
from .fileDataset import *
from . import time_utils
from .timespan import *
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/core/datastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -55,12 +55,12 @@
from .exceptions import ValidationError, DatasetTypeNotSupportedError
from .constraints import Constraints
from .storageClass import StorageClassFactory
from .fileDataset import FileDataset

if TYPE_CHECKING:
from ..registry.interfaces import DatastoreRegistryBridgeManager
from .datasets import DatasetRef, DatasetType
from .configSupport import LookupKey
from .repoTransfers import FileDataset
from .storageClass import StorageClass
from ._butlerUri import ButlerURI

Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/core/dimensions/universe.py
Original file line number Diff line number Diff line change
Expand Up @@ -158,7 +158,7 @@ def __new__(cls, config: Optional[Config] = None) -> DimensionUniverse:
return self

def __repr__(self) -> str:
return f"DimensionUniverse({self})"
return f"DimensionUniverse({self._version})"

def __getitem__(self, name: str) -> DimensionElement:
return self._elements[name]
Expand Down
68 changes: 68 additions & 0 deletions python/lsst/daf/butler/core/fileDataset.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,68 @@
# This file is part of daf_butler.
#
# Developed for the LSST Data Management System.
# This product includes software developed by the LSST Project
# (http://www.lsst.org).
# See the COPYRIGHT file at the top-level directory of this distribution
# for details of code ownership.
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

__all__ = ["FileDataset"]

from dataclasses import dataclass
from typing import (
List,
Optional,
Union,
)


from .datasets import DatasetRef
from .formatter import FormatterParameter


@dataclass
class FileDataset:
"""A struct that represents a dataset exported to a file.
"""
__slots__ = ("refs", "path", "formatter")

refs: List[DatasetRef]
"""Registry information about the dataset. (`list` of `DatasetRef`).
"""

path: str
"""Path to the dataset (`str`).

If the dataset was exported with ``transfer=None`` (i.e. in-place),
this is relative to the datastore root (only datastores that have a
well-defined root in the local filesystem can be expected to support
in-place exports). Otherwise this is relative to the directory passed
to `Datastore.export`.
"""

formatter: Optional[FormatterParameter]
"""A `Formatter` class or fully-qualified name.
"""

def __init__(self, path: str, refs: Union[DatasetRef, List[DatasetRef]], *,
formatter: Optional[FormatterParameter] = None):
self.path = path
if isinstance(refs, DatasetRef):
refs = [refs]
self.refs = refs
self.formatter = formatter