Enable butler tests using new datastore

lsst · Mar 29, 2018 · f92de54 · f92de54
1 parent 9fff796
commit f92de54
Show file tree

Hide file tree

Showing 5 changed files with 42 additions and 12 deletions.
diff --git a/python/lsst/daf/butler/core/fileTemplates.py b/python/lsst/daf/butler/core/fileTemplates.py
@@ -181,8 +181,8 @@ def format(self, dataId, datasetType=None, component=None):
 
         # Complain if we were meant to use a component
         if component is not None and not usedComponent:
-            raise KeyError("Component {} specified but template {} did not use it".format(component,
-                                                                                          self.template))
+            raise KeyError("Component '{}' specified but template {} did not use it".format(component,
+                                                                                            self.template))
 
         # Since this is known to be a path, normalize it in case some double
         # slashes have crept in

diff --git a/python/lsst/daf/butler/core/storageClass.py b/python/lsst/daf/butler/core/storageClass.py
@@ -153,9 +153,12 @@ def makeNewStorageClass(name, pytype=None, components=None, assembler=None):
     newtype : `StorageClass`
         Newly created Python type.
     """
+    if components is None:
+        components = {}
+
     clsargs = {"name": name,
                "_pytypeName": pytype,
-               "components": components}
+               "_components": components}
     # if the assembler is not None also set it and clear the default assembler
     if assembler is not None:
         clsargs["_assemblerClassName"] = assembler

diff --git a/tests/config/basic/butler.yaml b/tests/config/basic/butler.yaml
@@ -4,9 +4,10 @@ datastore:
   root: ./butler_test_repository
   create: true
   templates:
-    default: "{datasetType}/{tract:?}/{patch:?}/{filter:?}/{visit:?}"
+    default: "{datasetType}/{tract:?}/{patch:?}/{filter:?}/{camera:?}_{visit:?}"
     calexp: "{datasetType}.{component:?}/{datasetType}_v{visit}_f{filter}_{component:?}"
     metric: "{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{filter}_{component:?}"
+    test_metric_comp: "{datasetType}.{component:?}/{datasetType}_v{visit:08d}_f{camera}_{component:?}"
   formatters:
     StructuredDataDictYaml: lsst.daf.butler.formatters.yamlFormatter.YamlFormatter
     StructuredDataListYaml: lsst.daf.butler.formatters.yamlFormatter.YamlFormatter

diff --git a/tests/test_butler.py b/tests/test_butler.py
@@ -45,6 +45,17 @@ class ButlerTestCase(lsst.utils.tests.TestCase):
     """Test for Butler.
     """
 
+    @staticmethod
+    def registerDatasetTypes(datasetTypeName, dataUnits, storageClass, registry):
+        """Bulk register DatasetTypes
+        """
+        datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
+        registry.registerDatasetType(datasetType)
+
+        for comp, sc in storageClass.components.items():
+            compType = DatasetType("{}.{}".format(datasetTypeName, comp), dataUnits, sc)
+            registry.registerDatasetType(compType)
+
     @classmethod
     def setUpClass(cls):
         cls.testDir = os.path.dirname(__file__)
@@ -64,12 +75,7 @@ def testBasicPutGet(self):
         datasetTypeName = "test_metric"
         dataUnits = ("camera", "visit")
         storageClass = self.storageClassFactory.getStorageClass("StructuredData")
-        datasetType = DatasetType(datasetTypeName, dataUnits, storageClass)
-        butler.registry.registerDatasetType(datasetType)
-
-        for comp, sc in storageClass.components.items():
-            compType = DatasetType("{}.{}".format(datasetTypeName, comp), dataUnits, sc)
-            butler.registry.registerDatasetType(compType)
+        self.registerDatasetTypes(datasetTypeName, dataUnits, storageClass, butler.registry)
 
         # Create and store a dataset
         metric = makeExampleMetrics()
@@ -83,6 +89,26 @@ def testBasicPutGet(self):
         metricOut = butler.get(datasetTypeName, dataId)
         self.assertEqual(metric, metricOut)
 
+    def testCompositePutGet(self):
+        butler = Butler(self.configFile)
+        # Create and register a DatasetType
+        datasetTypeName = "test_metric_comp"
+        dataUnits = ("camera", "visit")
+        storageClass = self.storageClassFactory.getStorageClass("StructuredComposite")
+        self.registerDatasetTypes(datasetTypeName, dataUnits, storageClass, butler.registry)
+
+        # Create and store a dataset
+        metric = makeExampleMetrics()
+        dataId = {"camera": "DummyCamComp", "visit": 423}
+        ref = butler.put(metric, datasetTypeName, dataId)
+        self.assertIsInstance(ref, DatasetRef)
+        # Test getDirect (does not work yet)
+        # metricOut = butler.getDirect(ref)
+        # self.assertEqual(metric, metricOut)
+        # Test get
+        # metricOut = butler.get(datasetTypeName, dataId)
+        # self.assertEqual(metric, metricOut)
+
 
 class MemoryTester(lsst.utils.tests.MemoryTestCase):
     pass

diff --git a/tests/test_storageClass.py b/tests/test_storageClass.py
@@ -46,7 +46,7 @@ def testCreation(self):
         sc = newclass()
         self.assertIsInstance(sc, storageClass.StorageClass)
         self.assertEqual(sc.name, className)
-        self.assertIsNone(sc.components)
+        self.assertFalse(sc.components)
 
         # Test the caching by using private class attribute
         self.assertIsNone(newclass._pytype)
@@ -69,7 +69,7 @@ def testRegistry(self):
         sc = factory.getStorageClass(className)
         self.assertIsInstance(sc, storageClass.StorageClass)
         self.assertEqual(sc.name, className)
-        self.assertIsNone(sc.components)
+        self.assertFalse(sc.components)
         self.assertEqual(sc.pytype, PythonType)