Skip to content

Commit

Permalink
Replace Butler.putDirect with put()
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Mar 6, 2023
1 parent 903b1e5 commit be9bdca
Show file tree
Hide file tree
Showing 4 changed files with 69 additions and 22 deletions.
33 changes: 21 additions & 12 deletions python/lsst/daf/butler/_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -1148,22 +1148,16 @@ def _findDatasetRef(
return ref

@transactional
def putDirect(self, obj: Any, ref: DatasetRef) -> DatasetRef:
def putDirect(self, obj: Any, ref: DatasetRef, /) -> DatasetRef:
# Docstring inherited.
(imported_ref,) = self.registry._importDatasets(
[ref],
expand=True,
)
if imported_ref.id != ref.getCheckedId():
raise RuntimeError("This registry configuration does not support putDirect.")
self.datastore.put(obj, ref)
return ref
return self.put(obj, ref)

@transactional
def put(
self,
obj: Any,
datasetRefOrType: Union[DatasetRef, DatasetType, str],
/,
dataId: Optional[DataId] = None,
*,
run: Optional[str] = None,
Expand All @@ -1177,18 +1171,19 @@ def put(
The dataset.
datasetRefOrType : `DatasetRef`, `DatasetType`, or `str`
When `DatasetRef` is provided, ``dataId`` should be `None`.
Otherwise the `DatasetType` or name thereof.
Otherwise the `DatasetType` or name thereof. If a fully resolved
`DatasetRef` is given the run and ID are used directly.
dataId : `dict` or `DataCoordinate`
A `dict` of `Dimension` link name, value pairs that label the
`DatasetRef` within a Collection. When `None`, a `DatasetRef`
should be provided as the second argument.
run : `str`, optional
The name of the run the dataset should be added to, overriding
``self.run``.
``self.run``. Not used if a resolved `DatasetRef` is provided.
**kwargs
Additional keyword arguments used to augment or construct a
`DataCoordinate`. See `DataCoordinate.standardize`
parameters.
parameters. Not used if a resolve `DatasetRef` is provided.
Returns
-------
Expand All @@ -1201,6 +1196,18 @@ def put(
TypeError
Raised if the butler is read-only or if no run has been provided.
"""
if isinstance(datasetRefOrType, DatasetRef) and datasetRefOrType.id is not None:
# This is a direct put of predefined DatasetRef.
log.debug("Butler put direct: %s", datasetRefOrType)
(imported_ref,) = self.registry._importDatasets(
[datasetRefOrType],
expand=True,
)
if imported_ref.id != datasetRefOrType.getCheckedId():
raise RuntimeError("This registry configuration does not support putDirect.")
self.datastore.put(obj, datasetRefOrType)
return datasetRefOrType

log.debug("Butler put: %s, dataId=%s, run=%s", datasetRefOrType, dataId, run)
if not self.isWriteable():
raise TypeError("Butler is read-only.")
Expand Down Expand Up @@ -1463,6 +1470,7 @@ def get(
def getURIs(
self,
datasetRefOrType: Union[DatasetRef, DatasetType, str],
/,
dataId: Optional[DataId] = None,
*,
predict: bool = False,
Expand Down Expand Up @@ -1519,6 +1527,7 @@ def getURIs(
def getURI(
self,
datasetRefOrType: Union[DatasetRef, DatasetType, str],
/,
dataId: Optional[DataId] = None,
*,
predict: bool = False,
Expand Down
40 changes: 39 additions & 1 deletion python/lsst/daf/butler/_limited_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,46 @@ def isWriteable(self) -> bool:
"""Return `True` if this `Butler` supports write operations."""
raise NotImplementedError()

@deprecated(
reason="Butler.put() now behaves like Butler.putDirect() when given a DatasetRef."
" Please use Butler.put(). Will be removed after v27.0.",
version="v26.0",
category=FutureWarning,
)
def putDirect(self, obj: Any, ref: DatasetRef, /) -> DatasetRef:
"""Store a dataset that already has a UUID and ``RUN`` collection.
Parameters
----------
obj : `object`
The dataset.
ref : `DatasetRef`
Resolved reference for a not-yet-stored dataset.
Returns
-------
ref : `DatasetRef`
The same as the given, for convenience and symmetry with
`Butler.put`.
Raises
------
TypeError
Raised if the butler is read-only.
AmbiguousDatasetError
Raised if ``ref.id is None``, i.e. the reference is unresolved.
Notes
-----
Whether this method inserts the given dataset into a ``Registry`` is
implementation defined (some `LimitedButler` subclasses do not have a
`Registry`), but it always adds the dataset to a `Datastore`, and the
given ``ref.id`` and ``ref.run`` are always preserved.
"""
return self.put(obj, ref)

@abstractmethod
def putDirect(self, obj: Any, ref: DatasetRef) -> DatasetRef:
def put(self, obj: Any, ref: DatasetRef, /) -> DatasetRef:
"""Store a dataset that already has a UUID and ``RUN`` collection.
Parameters
Expand Down
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/_quantum_backed.py
Original file line number Diff line number Diff line change
Expand Up @@ -468,7 +468,7 @@ def dimensions(self) -> DimensionUniverse:
# Docstring inherited.
return self._dimensions

def putDirect(self, obj: Any, ref: DatasetRef) -> DatasetRef:
def put(self, obj: Any, ref: DatasetRef, /) -> DatasetRef:
# Docstring inherited.
if ref.id not in self._predicted_outputs:
raise RuntimeError("Cannot `put` dataset that was not predicted as an output.")
Expand Down
16 changes: 8 additions & 8 deletions tests/test_quantumBackedButler.py
Original file line number Diff line number Diff line change
Expand Up @@ -165,7 +165,7 @@ def _test_factory(self, qbb: QuantumBackedButler) -> None:
self.assertEqual(qbb._actual_inputs, set())
self.assertEqual(qbb._actual_output_refs, set())

def test_getPutDirect(self) -> None:
def test_getput(self) -> None:
"""Test for getDirect/putDirect methods"""

quantum = self.make_quantum()
Expand All @@ -190,7 +190,7 @@ def test_getPutDirect(self) -> None:

# Write all expected outputs.
for ref in self.output_refs:
qbb.putDirect({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
qbb.put({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)

# Must be able to read them back
for ref in self.output_refs:
Expand Down Expand Up @@ -284,7 +284,7 @@ def test_pruneDatasets(self) -> None:

# Write all expected outputs.
for ref in self.output_refs:
qbb.putDirect({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
qbb.put({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)

# Must be able to read them back
for ref in self.output_refs:
Expand All @@ -307,7 +307,7 @@ def test_pruneDatasets(self) -> None:
data = qbb.get(ref)

# can store it again
qbb.putDirect({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
qbb.put({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
self.assertTrue(qbb.datasetExistsDirect(ref))

# Purge completely.
Expand All @@ -316,7 +316,7 @@ def test_pruneDatasets(self) -> None:
self.assertFalse(qbb.datasetExistsDirect(ref))
with self.assertRaises(FileNotFoundError):
data = qbb.get(ref)
qbb.putDirect({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
qbb.put({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
self.assertTrue(qbb.datasetExistsDirect(ref))

def test_extract_provenance_data(self) -> None:
Expand All @@ -333,7 +333,7 @@ def test_extract_provenance_data(self) -> None:
for ref in self.init_inputs_refs:
qbb.get(ref)
for ref in self.output_refs:
qbb.putDirect({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
qbb.put({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)

provenance1 = qbb.extract_provenance_data()
prov_json = provenance1.json()
Expand Down Expand Up @@ -378,12 +378,12 @@ def test_collect_and_transfer(self) -> None:
for ref in self.init_inputs_refs:
qbb1.get(ref)
for ref in self.output_refs:
qbb1.putDirect({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)
qbb1.put({"data": cast(int, ref.dataId["detector"]) ** 2}, ref)

for ref in self.output_refs:
qbb2.get(ref)
for ref in self.output_refs2:
qbb2.putDirect({"data": cast(int, ref.dataId["detector"]) ** 3}, ref)
qbb2.put({"data": cast(int, ref.dataId["detector"]) ** 3}, ref)

QuantumProvenanceData.collect_and_transfer(
self.butler,
Expand Down

0 comments on commit be9bdca

Please sign in to comment.