Composite put and get with butler mostly work

python/lsst/daf/butler/butler.py

@@ -136,9 +136,22 @@ def getDirect(self, ref):
         obj : `object`
             The dataset.
         """
-        # Currently a direct pass-through to `Datastore.get` but this should
-        # change for composites.
-        return self.datastore.get(ref)
+        print("Ref: {}".format(ref))
+
+        # if the ref exists in the store we return it directly
+        if self.datastore.exists(ref):
+            return self.datastore.get(ref)
+        elif ref.components:
+            # Reconstruct the composite
+            components = {}
+            for compName, compRef in ref.components.items():
+                components[compName] = self.datastore.get(compRef)
+
+            # Assemble the components
+            return ref.datasetType.storageClass.assembler().assemble(components)
+        else:
+            # single entity in datastore
+            raise ValueError("Unable to locate ref {} in datastore {}".format(ref.id, self.datastore.name))
 
     def get(self, datasetType, dataId):
         """Retrieve a stored dataset.

python/lsst/daf/butler/core/datasets.py

@@ -209,3 +209,10 @@ def assembler(self):
         Read-only; update via `Registry.setAssembler()`.
         """
         return self._assembler
+
+    def __str__(self):
+        comp = ""
+        if self.components:
+            comp = ", components=[" + ", ".join(self.components) + "]"
+        return "DatasetRef({}, id={}, dataId={} {})".format(self.datasetType.name,
+                                                            self.id, self.dataId, comp)

python/lsst/daf/butler/core/datastore.py

@@ -71,6 +71,22 @@ def __init__(self, config, registry):
         self.registry = registry
         self.name = "ABCDataStore"
 
+    @abstractmethod
+    def exists(self, datasetRef):
+        """Check if the dataset exists in the datastore.
+
+        Parameters
+        ----------
+        datasetRef : `DatasetRef`
+            Reference to the required dataset.
+
+        Returns
+        -------
+        exists : `bool`
+            `True` if the entity exists in the `Datastore`.
+        """
+        raise NotImplementedError("Must be implemented by subclass")
+
     @abstractmethod
     def get(self, datasetRef, parameters=None):
         """Load an `InMemoryDataset` from the store.

python/lsst/daf/butler/core/mappingFactory.py

@@ -107,7 +107,6 @@ def placeInRegistry(self, registryKey, typeName):
         if keyString in self._registry:
             # Compare the class strings since dynamic classes can be the
             # same thing but be different.
-            print("Duplicate: {} vs {}".format(self._registry[keyString], typeName))
             if str(self._registry[keyString]) == str(typeName):
                 return
 

python/lsst/daf/butler/datastores/posixDatastore.py

@@ -140,6 +140,29 @@ def getStoredFileInfo(self, ref):
         storageClass = self.storageClassFactory.getStorageClass(storageClass)
         return StoredFileInfo(formatter_name, path, storageClass)
 
+    def exists(self, ref):
+        """Check if the dataset exists in the datastore.
+
+        Parameters
+        ----------
+        ref : `DatasetRef`
+            Reference to the required dataset.
+
+        Returns
+        -------
+        exists : `bool`
+            `True` if the entity exists in the `Datastore`.
+        """
+        # Get the file information (this will fail if no file)
+        try:
+            storedFileInfo = self.getStoredFileInfo(ref)
+        except KeyError:
+            return False
+
+        # Use the path to determine the location
+        location = self.locationFactory.fromPath(storedFileInfo.path)
+        return os.path.exists(location.path)
+
     def get(self, ref, parameters=None):
         """Load an InMemoryDataset from the store.
 
@@ -242,7 +265,7 @@ def put(self, inMemoryDataset, ref):
         path = formatter.write(inMemoryDataset, FileDescriptor(location,
                                                                storageClass=storageClass))
 
-        print("Wrote something to {}..{}".format(os.getcwd(), path))
+        print("Wrote {} to {}..{}".format(ref.id, os.getcwd(), path))
 
         # Create Storage information in the registry
         ospath = os.path.join(self.root, path)

python/lsst/daf/butler/registries/sqlRegistry.py

@@ -748,9 +748,7 @@ def find(self, collection, datasetType, dataId):
                     dataIdExpression))).fetchone()
         # TODO update unit values and add Run, Quantum and assembler?
         if result is not None:
-            return DatasetRef(datasetType=datasetType,
-                              dataId=dataId,
-                              id=result['dataset_id'])
+            return self.getDataset(result['dataset_id'])
         else:
             return None
 

tests/test_butler.py

@@ -89,6 +89,10 @@ def testBasicPutGet(self):
         metricOut = butler.get(datasetTypeName, dataId)
         self.assertEqual(metric, metricOut)
 
+        # Get a component
+        summary = butler.get("{}.{}".format(datasetTypeName, "summary"), dataId)
+        self.assertEqual(summary, metric.summary)
+
     def testCompositePutGet(self):
         butler = Butler(self.configFile)
         # Create and register a DatasetType
@@ -102,12 +106,16 @@ def testCompositePutGet(self):
         dataId = {"camera": "DummyCamComp", "visit": 423}
         ref = butler.put(metric, datasetTypeName, dataId)
         self.assertIsInstance(ref, DatasetRef)
-        # Test getDirect (does not work yet)
-        # metricOut = butler.getDirect(ref)
-        # self.assertEqual(metric, metricOut)
+        # Test getDirect
+        metricOut = butler.getDirect(ref)
+        self.assertEqual(metric, metricOut)
         # Test get
-        # metricOut = butler.get(datasetTypeName, dataId)
-        # self.assertEqual(metric, metricOut)
+        metricOut = butler.get(datasetTypeName, dataId)
+        self.assertEqual(metric, metricOut)
+
+        # Get a component
+        summary = butler.get("{}.{}".format(datasetTypeName, "summary"), dataId)
+        self.assertEqual(summary, metric.summary)
 
 
 class MemoryTester(lsst.utils.tests.MemoryTestCase):