Skip to content

Commit

Permalink
Add explicit test for datastore.export
Browse files Browse the repository at this point in the history
This required we change the exception type in some failure
modes to match that raise by butler.export.
  • Loading branch information
timj committed Oct 11, 2022
1 parent 1388ee2 commit 373ec5c
Show file tree
Hide file tree
Showing 3 changed files with 30 additions and 9 deletions.
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/datastores/chainedDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -957,10 +957,10 @@ def export(
) -> Iterable[FileDataset]:
# Docstring inherited from Datastore.export.
if transfer is not None and directory is None:
raise RuntimeError(f"Cannot export using transfer mode {transfer} with no export directory given")
raise TypeError(f"Cannot export using transfer mode {transfer} with no export directory given")

if transfer == "move":
raise RuntimeError("Can not export by moving files out of datastore.")
raise TypeError("Can not export by moving files out of datastore.")

# Exporting from a chain has the potential for a dataset to be
# in one or more of the datastores in the chain. We only need one
Expand Down
4 changes: 2 additions & 2 deletions python/lsst/daf/butler/datastores/fileDatastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -2704,10 +2704,10 @@ def export(
) -> Iterable[FileDataset]:
# Docstring inherited from Datastore.export.
if transfer is not None and directory is None:
raise RuntimeError(f"Cannot export using transfer mode {transfer} with no export directory given")
raise TypeError(f"Cannot export using transfer mode {transfer} with no export directory given")

if transfer == "move":
raise RuntimeError("Can not export by moving files out of datastore.")
raise TypeError("Can not export by moving files out of datastore.")
elif transfer == "direct":
# For an export, treat this as equivalent to None. We do not
# want an import to risk using absolute URIs to datasets owned
Expand Down
31 changes: 26 additions & 5 deletions tests/test_datastore.py
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.

from __future__ import annotations

import os
import shutil
import sys
Expand All @@ -33,8 +35,10 @@
import yaml
from lsst.daf.butler import (
Config,
DatasetRef,
DatasetRefURIs,
DatasetTypeNotSupportedError,
Datastore,
DatastoreCacheManager,
DatastoreCacheManagerConfig,
DatastoreConfig,
Expand Down Expand Up @@ -817,10 +821,8 @@ def testIngestSymlinkOfSymlink(self):
# since it will get the same file name in store
datastore.remove(ref)

def testExportImportRecords(self):
"""Test for export_records and import_records methods."""

datastore = self.makeDatastore("test_datastore")
def _populate_export_datastore(self, name: str) -> tuple[Datastore, list[DatasetRef]]:
datastore = self.makeDatastore(name)

# For now only the FileDatastore can be used for this test.
# ChainedDatastore that only includes InMemoryDatastores have to be
Expand All @@ -837,11 +839,18 @@ def testExportImportRecords(self):

refs = []
for visit in (2048, 2049, 2050):
dataId = {"instrument": "dummy", "visit": visit, "physical_filter": "Uprime"}
dataId = FakeDataCoordinate.from_dict(
{"instrument": "dummy", "visit": visit, "physical_filter": "Uprime"}
)
ref = self.makeDatasetRef("metric", dimensions, sc, dataId, conform=False)
print("REF IS ", repr(ref))
datastore.put(metrics, ref)
refs.append(ref)
return datastore, refs

def testExportImportRecords(self):
"""Test for export_records and import_records methods."""
datastore, refs = self._populate_export_datastore("test_datastore")
for exported_refs in (refs, refs[1:]):
n_refs = len(exported_refs)
records = datastore.export_records(exported_refs)
Expand All @@ -865,6 +874,18 @@ def testExportImportRecords(self):
data = datastore2.get(refs[2])
self.assertIsNotNone(data)

def testExport(self):
datastore, refs = self._populate_export_datastore("test_datastore")

datasets = list(datastore.export(refs, transfer=None))
self.assertEqual(len(datasets), 3)

with self.assertRaises(TypeError):
list(datastore.export(refs))

with self.assertRaises(TypeError):
list(datastore.export(refs, directory="exportDir", transfer="move"))


class PosixDatastoreTestCase(DatastoreTests, unittest.TestCase):
"""PosixDatastore specialization"""
Expand Down

0 comments on commit 373ec5c

Please sign in to comment.