Skip to content

Commit

Permalink
Add Datastore._set_trust_mode
Browse files Browse the repository at this point in the history
Method makes it easier for quantum backed butler and tests to change
the flag.
  • Loading branch information
timj committed Jan 26, 2024
1 parent 87728a7 commit 6142e59
Show file tree
Hide file tree
Showing 5 changed files with 29 additions and 12 deletions.
4 changes: 1 addition & 3 deletions python/lsst/daf/butler/_quantum_backed.py
Original file line number Diff line number Diff line change
Expand Up @@ -357,9 +357,7 @@ def _initialize(
# TODO: We need to inform `Datastore` here that it needs to support
# predictive reads; This only really works for file datastore but
# we need to try everything in case there is a chained datastore.
for this_datastore in getattr(datastore, "datastores", [datastore]):
if hasattr(this_datastore, "trustGetRequest"):
this_datastore.trustGetRequest = True
datastore._set_trust_mode(True)

if datastore_records is not None:
datastore.import_records(datastore_records)
Expand Down
18 changes: 18 additions & 0 deletions python/lsst/daf/butler/datastore/_datastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -476,6 +476,24 @@ def transaction(self) -> Iterator[DatastoreTransaction]:
self._transaction.commit()
self._transaction = self._transaction.parent

def _set_trust_mode(self, mode: bool) -> None:
"""Set the trust mode for this datastore.
Parameters
----------
mode : `bool`
If `True`, get requests will be attempted even if the datastore
does not know about the dataset.
Notes
-----
This is a private method to indicate that trust mode might be a
transitory property that we do not want to make fully public. For now
only a `~lsst.daf.butler.datastores.FileDatastore` understands this
concept. By default this method does nothing.
"""
return

@abstractmethod
def knows(self, ref: DatasetRef) -> bool:
"""Check if the dataset is known to the datastore.
Expand Down
4 changes: 4 additions & 0 deletions python/lsst/daf/butler/datastores/chainedDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -264,6 +264,10 @@ def __str__(self) -> str:
chainName = ", ".join(str(ds) for ds in self.datastores)
return chainName

def _set_trust_mode(self, mode: bool) -> None:
for datastore in self.datastores:
datastore._set_trust_mode(mode)

def knows(self, ref: DatasetRef) -> bool:
"""Check if the dataset is known to any of the datastores.
Expand Down
3 changes: 3 additions & 0 deletions python/lsst/daf/butler/datastores/fileDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -359,6 +359,9 @@ def roots(self) -> dict[str, ResourcePath | None]:
# Docstring inherited.
return {self.name: self.root}

def _set_trust_mode(self, mode: bool) -> None:
self.trustGetRequest = mode

def _artifact_exists(self, location: Location) -> bool:
"""Check that an artifact exists in this datastore at the specified
location.
Expand Down
12 changes: 3 additions & 9 deletions tests/test_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,7 @@ def mock_s3(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-untyped-def]
if TYPE_CHECKING:
import types

from lsst.daf.butler import Datastore, DimensionGroup, Registry, StorageClass
from lsst.daf.butler import DimensionGroup, Registry, StorageClass

TESTDIR = os.path.abspath(os.path.dirname(__file__))

Expand Down Expand Up @@ -2114,12 +2114,6 @@ def testTransferUuidToUuid(self) -> None:
self.create_butlers()
self.assertButlerTransfers()

def _enable_trust(self, datastore: Datastore) -> None:
datastores = getattr(datastore, "datastores", [datastore])
for this_datastore in datastores:
if hasattr(this_datastore, "trustGetRequest"):
this_datastore.trustGetRequest = True

def testTransferMissing(self) -> None:
"""Test transfers where datastore records are missing.
Expand All @@ -2128,7 +2122,7 @@ def testTransferMissing(self) -> None:
self.create_butlers()

# Configure the source butler to allow trust.
self._enable_trust(self.source_butler._datastore)
self.source_butler._datastore._set_trust_mode(True)

self.assertButlerTransfers(purge=True)

Expand All @@ -2140,7 +2134,7 @@ def testTransferMissingDisassembly(self) -> None:
self.create_butlers()

# Configure the source butler to allow trust.
self._enable_trust(self.source_butler._datastore)
self.source_butler._datastore._set_trust_mode(True)

# Test disassembly.
self.assertButlerTransfers(purge=True, storageClassName="StructuredComposite")
Expand Down

0 comments on commit 6142e59

Please sign in to comment.