Skip to content

Commit

Permalink
Add skip datastore to ButlerConfig and add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
timj committed Jul 20, 2023
1 parent b55c3dc commit 1879ecc
Show file tree
Hide file tree
Showing 3 changed files with 57 additions and 1 deletion.
2 changes: 1 addition & 1 deletion python/lsst/daf/butler/_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -222,7 +222,7 @@ def __init__(
self.storageClasses = butler.storageClasses
self._config: ButlerConfig = butler._config
else:
self._config = ButlerConfig(config, searchPaths=searchPaths)
self._config = ButlerConfig(config, searchPaths=searchPaths, skip_datastore=skip_datastore)
try:
if "root" in self._config:
butlerRoot = self._config["root"]
Expand Down
7 changes: 7 additions & 0 deletions python/lsst/daf/butler/_butlerConfig.py
Original file line number Diff line number Diff line change
Expand Up @@ -61,12 +61,15 @@ class ButlerConfig(Config):
than those read from the environment in
`ConfigSubset.defaultSearchPaths()`. They are only read if ``other``
refers to a configuration file or directory.
skip_datastore : `bool`, optional
If `True` remove the datastore configuration.
"""

def __init__(
self,
other: ResourcePathExpression | Config | None = None,
searchPaths: Sequence[ResourcePathExpression] | None = None,
skip_datastore: bool = False,
):
self.configDir: ResourcePath | None = None

Expand Down Expand Up @@ -158,6 +161,10 @@ def __init__(
# Only send the parent config if the child
# config component is present (otherwise it assumes that the
# keys from other components are part of the child)
if skip_datastore and configClass is DatastoreConfig:
del butlerConfig[configClass.component]
continue

localOverrides = None
if configClass.component in butlerConfig:
localOverrides = butlerConfig
Expand Down
49 changes: 49 additions & 0 deletions tests/test_butler.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,7 @@ def mock_s3(*args: Any, **kwargs: Any) -> Any: # type: ignore[no-untyped-def]
FileDataset,
FileTemplate,
FileTemplateValidationError,
NullDatastore,
StorageClassFactory,
ValidationError,
script,
Expand Down Expand Up @@ -2332,6 +2333,54 @@ class ChainedDatastoreTransfers(PosixDatastoreTransfers):
configFile = os.path.join(TESTDIR, "config/basic/butler-chained.yaml")


class NullDatastoreTestCase(unittest.TestCase):
"""Test that we can fall back to a null datastore."""

# Need a good config to create the repo.
configFile = os.path.join(TESTDIR, "config/basic/butler.yaml")

@classmethod
def setUpClass(cls) -> None:
cls.storageClassFactory = StorageClassFactory()
cls.storageClassFactory.addFromConfig(cls.configFile)

def setUp(self) -> None:
"""Create a new butler root for each test."""
self.root = makeTestTempDir(TESTDIR)
Butler.makeRepo(self.root, config=Config(self.configFile))

def tearDown(self) -> None:
removeTestTempDir(self.root)

def test_fallback(self) -> None:
# Read the butler config and mess with the datastore section.
bad_config = Config(os.path.join(self.root, "butler.yaml"))
bad_config["datastore", "cls"] = "lsst.not.a.datastore.Datastore"

with self.assertRaises(RuntimeError):
Butler(bad_config)

butler = Butler(bad_config, writeable=True, skip_datastore=True)
self.assertIsInstance(butler._datastore, NullDatastore)

# Check that registry is working.
butler.registry.registerRun("MYRUN")
collections = butler.registry.queryCollections(...)
self.assertIn("MYRUN", set(collections))

# Create a ref.
dimensions = butler.dimensions.extract([])
storageClass = self.storageClassFactory.getStorageClass("StructuredDataDict")
datasetTypeName = "metric"
datasetType = DatasetType(datasetTypeName, dimensions, storageClass)
butler.registry.registerDatasetType(datasetType)
ref = DatasetRef(datasetType, {}, run="MYRUN")

# Check that datastore will complain.
with self.assertRaises(NotImplementedError):
butler.get(ref)


def setup_module(module: types.ModuleType) -> None:
"""Set up the module for pytest."""
clean_environment()
Expand Down

0 comments on commit 1879ecc

Please sign in to comment.