Merge pull request #51 from lsst/tickets/DM-14760

DM-14760: Merge StorageFileInfo into StorageInfo

config/schema.yaml

@@ -1554,7 +1554,7 @@ schema:
 
     DatasetStorage:
       doc: >
-        A table that provides information on how a Dataset is stored in one
+        A table that provides information on whether a Dataset is stored in one
         or more Datastores.  The presence or absence of a record in this
         table itself indicates whether the Dataset is present in that
         Datastore.
@@ -1572,14 +1572,6 @@ schema:
         nullable: false
         doc: >
           Name of the Datastore this entry corresponds to.
-      - name: checksum
-        type: string
-        doc: >
-          Checksum (e.g. md5) of the stored dataset, if applicable.
-      - name: size
-        type: int
-        doc: >
-          Total size of the stored dataset in bytes, if applicable.
       foreignKeys:
       -
         src: dataset_id

python/lsst/daf/butler/core/__init__.py

@@ -23,7 +23,6 @@
 from .run import *
 from .schema import *
 from .storageClass import *
-from .storageInfo import *
 from .storedFileInfo import *
 from .dataUnit import *
 from .databaseDict import *

python/lsst/daf/butler/core/config.py

@@ -444,77 +444,77 @@ def dumpToFile(self, path):
 
     @staticmethod
     def overrideParameters(configType, config, full, toUpdate=None, toCopy=None):
-            """Generic helper function for overriding specific config parameters
-
-            Allows for named parameters to be set to new values in bulk, and
-            for other values to be set by copying from a reference config.
-
-            Assumes that the supplied config is compatible with ```configType``
-            and will attach the updated values to the supplied config by
-            looking for the related component key.  It is assumed that
-            ``config`` and ``full`` are from the same part of the
-            configuration hierarchy.
-
-            Parameters
-            ----------
-            configType : `ConfigSubset`
-                Config type to use to extract relevant items from ``config``.
-            config : `Config`
-                A `Config` to update. Only the subset understood by
-                the supplied `ConfigSubset` will be modified. Default values
-                will not be inserted and the content will not be validated
-                since mandatory keys are allowed to be missing until
-                populated later by merging.
-            full : `Config`
-                A complete config with all defaults expanded that can be
-                converted to a ``configType``. Read-only and will not be
-                modified by this method. Values are read from here if
-                ``toCopy`` is defined.
-
-                Repository-specific options that should not be obtained
-                from defaults when Butler instances are constructed
-                should be copied from `full` to `Config`.
-            toUpdate : `dict`, optional
-                A `dict` defining the keys to update and the new value to use.
-                The keys and values can be any supported by `Config`
-                assignment.
-            toCopy : `tuple`, optional
-                `tuple` of keys whose values should be copied from ``full``
-                into ``config``.
-
-            Raises
-            ------
-            ValueError
-                Neither ``toUpdate`` not ``toCopy`` were defined.
-            """
-            if toUpdate is None and toCopy is None:
-                raise ValueError("One of toUpdate or toCopy parameters must be set.")
-
-            # If this is a parent configuration then we need to ensure that
-            # the supplied config has the relevant component key in it.
-            # If this is a parent configuration we add in the stub entry
-            # so that the ConfigSubset constructor will do the right thing.
-            # We check full for this since that is guaranteed to be complete.
-            if configType.component in full and configType.component not in config:
-                config[configType.component] = {}
-
-            # Extract the part of the config we wish to update
-            localConfig = configType(config, mergeDefaults=False, validate=False)
-
-            if toUpdate:
-                for key, value in toUpdate.items():
-                    localConfig[key] = value
-
-            if toCopy:
-                localFullConfig = configType(full, mergeDefaults=False)
-                for key in toCopy:
-                    localConfig[key] = localFullConfig[key]
-
-            # Reattach to parent if this is a child config
-            if configType.component in config:
-                config[configType.component] = localConfig
-            else:
-                config.update(localConfig)
+        """Generic helper function for overriding specific config parameters
+
+        Allows for named parameters to be set to new values in bulk, and
+        for other values to be set by copying from a reference config.
+
+        Assumes that the supplied config is compatible with ```configType``
+        and will attach the updated values to the supplied config by
+        looking for the related component key.  It is assumed that
+        ``config`` and ``full`` are from the same part of the
+        configuration hierarchy.
+
+        Parameters
+        ----------
+        configType : `ConfigSubset`
+            Config type to use to extract relevant items from ``config``.
+        config : `Config`
+            A `Config` to update. Only the subset understood by
+            the supplied `ConfigSubset` will be modified. Default values
+            will not be inserted and the content will not be validated
+            since mandatory keys are allowed to be missing until
+            populated later by merging.
+        full : `Config`
+            A complete config with all defaults expanded that can be
+            converted to a ``configType``. Read-only and will not be
+            modified by this method. Values are read from here if
+            ``toCopy`` is defined.
+
+            Repository-specific options that should not be obtained
+            from defaults when Butler instances are constructed
+            should be copied from `full` to `Config`.
+        toUpdate : `dict`, optional
+            A `dict` defining the keys to update and the new value to use.
+            The keys and values can be any supported by `Config`
+            assignment.
+        toCopy : `tuple`, optional
+            `tuple` of keys whose values should be copied from ``full``
+            into ``config``.
+
+        Raises
+        ------
+        ValueError
+            Neither ``toUpdate`` not ``toCopy`` were defined.
+        """
+        if toUpdate is None and toCopy is None:
+            raise ValueError("One of toUpdate or toCopy parameters must be set.")
+
+        # If this is a parent configuration then we need to ensure that
+        # the supplied config has the relevant component key in it.
+        # If this is a parent configuration we add in the stub entry
+        # so that the ConfigSubset constructor will do the right thing.
+        # We check full for this since that is guaranteed to be complete.
+        if configType.component in full and configType.component not in config:
+            config[configType.component] = {}
+
+        # Extract the part of the config we wish to update
+        localConfig = configType(config, mergeDefaults=False, validate=False)
+
+        if toUpdate:
+            for key, value in toUpdate.items():
+                localConfig[key] = value
+
+        if toCopy:
+            localFullConfig = configType(full, mergeDefaults=False)
+            for key in toCopy:
+                localConfig[key] = localFullConfig[key]
+
+        # Reattach to parent if this is a child config
+        if configType.component in config:
+            config[configType.component] = localConfig
+        else:
+            config.update(localConfig)
 
 
 class ConfigSubset(Config):

python/lsst/daf/butler/core/registry.py

@@ -383,24 +383,8 @@ def disassociate(self, collection, refs, remove=True):
 
     @abstractmethod
     @transactional
-    def addStorageInfo(self, ref, storageInfo):
-        """Add storage information for a given dataset.
-
-        Typically used by `Datastore`.
-
-        Parameters
-        ----------
-        ref : `DatasetRef`
-            A reference to the dataset for which to add storage information.
-        storageInfo : `StorageInfo`
-            Storage information about the dataset.
-        """
-        raise NotImplementedError("Must be implemented by subclass")
-
-    @abstractmethod
-    @transactional
-    def updateStorageInfo(self, ref, datastoreName, storageInfo):
-        """Update storage information for a given dataset.
+    def addDatasetLocation(self, ref, datastoreName):
+        """Add datastore name locating a given dataset.
 
         Typically used by `Datastore`.
 
@@ -409,41 +393,34 @@ def updateStorageInfo(self, ref, datastoreName, storageInfo):
         ref : `DatasetRef`
             A reference to the dataset for which to add storage information.
         datastoreName : `str`
-            What datastore association to update.
-        storageInfo : `StorageInfo`
-            Storage information about the dataset.
+            Name of the datastore holding this dataset.
         """
         raise NotImplementedError("Must be implemented by subclass")
 
     @abstractmethod
-    def getStorageInfo(self, ref, datastoreName):
-        """Retrieve storage information for a given dataset.
+    def getDatasetLocations(self, ref):
+        """Retrieve datastore locations for a given dataset.
 
         Typically used by `Datastore`.
 
         Parameters
         ----------
         ref : `DatasetRef`
-            A reference to the dataset for which to add storage information.
-        datastoreName : `str`
-            What datastore association to update.
+            A reference to the dataset for which to retrieve storage
+            information.
 
         Returns
         -------
-        info : `StorageInfo`
-            Storage information about the dataset.
-
-        Raises
-        ------
-        KeyError
-            The requested Dataset does not exist.
+        datastores : `set` of `str`
+            All the matching datastores holding this dataset. Empty set
+            if the dataset does not exist anywhere.
         """
         raise NotImplementedError("Must be implemented by subclass")
 
     @abstractmethod
     @transactional
-    def removeStorageInfo(self, datastoreName, ref):
-        """Remove storage information associated with this dataset.
+    def removeDatasetLocation(self, datastoreName, ref):
+        """Remove datastore location associated with this dataset.
 
         Typically used by `Datastore` when a dataset is removed.
 

python/lsst/daf/butler/core/storedFileInfo.py

@@ -47,9 +47,9 @@ class StoredFileInfo:
     """
 
     __eq__ = slotValuesAreEqual
-    __slots__ = ("_formatter", "_path", "_storageClass")
+    __slots__ = ("_formatter", "_path", "_storageClass", "_checksum", "_size")
 
-    def __init__(self, formatter, path, storageClass):
+    def __init__(self, formatter, path, storageClass, checksum=None, size=None):
         assert isinstance(formatter, str) or isinstance(formatter, Formatter)
         if isinstance(formatter, Formatter):
             formatter = formatter.name()
@@ -58,6 +58,10 @@ def __init__(self, formatter, path, storageClass):
         self._path = path
         assert isinstance(storageClass, StorageClass)
         self._storageClass = storageClass
+        assert checksum is None or isinstance(checksum, str)
+        self._checksum = checksum
+        assert size is None or isinstance(size, int)
+        self._size = size
 
     @property
     def formatter(self):
@@ -76,3 +80,19 @@ def storageClass(self):
         """StorageClass used (`StorageClass`).
         """
         return self._storageClass
+
+    @property
+    def checksum(self):
+        """Checksum (`str`).
+        """
+        return self._checksum
+
+    @property
+    def size(self):
+        """Size of stored object in bytes (`int`).
+        """
+        return self._size
+
+    def __repr__(self):
+        return f'{type(self).__qualname__}(path="{self.path}", formatter="{self.formatter}"' \
+            f' size={self.size}, checksum="{self.checksum}", storageClass="{self.storageClass.name}")'