Merge pull request #260 from lsst/tickets/DM-25327

DM-25327: Add new PackagesFormatter

python/lsst/obs/base/cli/opt/instrument.py

@@ -1,4 +1,4 @@
-# This file is part of daf_butler.
+# This file is part of obs_base.
 #
 # Developed for the LSST Data Management System.
 # This product includes software developed by the LSST Project

python/lsst/obs/base/formatters/fitsExposure.py

@@ -75,6 +75,7 @@ class FitsExposureFormatter(Formatter):
             variance: *default
 
     """
+    supportedExtensions = frozenset({".fits", ".fits.gz", ".fits.fz"})
     extension = ".fits"
     _metadata = None
     supportedWriteParameters = frozenset({"recipe"})

python/lsst/obs/base/formatters/fitsGeneric.py

@@ -1,4 +1,4 @@
-# This file is part of daf_butler.
+# This file is part of obs_base.
 #
 # Developed for the LSST Data Management System.
 # This product includes software developed by the LSST Project

python/lsst/obs/base/formatters/packages.py

@@ -0,0 +1,123 @@
+# This file is part of obs_base.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ("PackagesFormatter", )
+
+import os.path
+
+from lsst.daf.butler.formatters.file import FileFormatter
+
+
+class PackagesFormatter(FileFormatter):
+    """Interface for reading and writing objects that support the standard
+    afw I/O readFits/writeFits methods.
+    """
+    supportedWriteParameters = frozenset({"format"})
+    supportedExtensions = frozenset({".yaml", ".pickle", ".pkl"})
+
+    @property
+    def extension(self) -> str:
+        # Default to YAML but allow configuration via write parameter
+        format = self.writeParameters.get("format", "yaml")
+        if format == "yaml":
+            return ".yaml"
+        elif format == "pickle":
+            return ".pickle"
+        raise RuntimeError(f"Requested file format '{format}' is not supported for Packages")
+
+    def _readFile(self, path, pytype):
+        """Read a file from the path in FITS format.
+
+        Parameters
+        ----------
+        path : `str`
+            Path to use to open the file.
+        pytype : `type`
+            Class to use to read the serialized file.
+
+        Returns
+        -------
+        data : `object`
+            Instance of class ``pytype`` read from serialized file. None
+            if the file could not be opened.
+        """
+        if not os.path.exists(path):
+            return None
+
+        return pytype.read(path)
+
+    def _fromBytes(self, serializedDataset, pytype=None):
+        """Read the bytes object as a python object.
+
+        Parameters
+        ----------
+        serializedDataset : `bytes`
+            Bytes object to unserialize.
+        pytype : `type`
+            The Python type to be instantiated. Required.
+
+        Returns
+        -------
+        inMemoryDataset : `object`
+            The requested data as an object, or None if the string could
+            not be read.
+        """
+        # The format can not come from the formatter configuration
+        # because the current configuration has no connection to how
+        # the data were stored.
+        format = "yaml" if serializedDataset.startswith(b"!<lsst.base.Packages>") else "pickle"
+        return pytype.fromBytes(serializedDataset, format)
+
+    def _writeFile(self, inMemoryDataset):
+        """Write the in memory dataset to file on disk.
+
+        Parameters
+        ----------
+        inMemoryDataset : `object`
+            Object to serialize.
+
+        Raises
+        ------
+        Exception
+            The file could not be written.
+        """
+        inMemoryDataset.write(self.fileDescriptor.location.path)
+
+    def _toBytes(self, inMemoryDataset):
+        """Write the in memory dataset to a bytestring.
+
+        Parameters
+        ----------
+        inMemoryDataset : `object`
+            Object to serialize
+
+        Returns
+        -------
+        serializedDataset : `bytes`
+            YAML string encoded to bytes.
+
+        Raises
+        ------
+        Exception
+            The object could not be serialized.
+        """
+        format = "yaml" if self.extension == ".yaml" else "pickle"
+        return inMemoryDataset.toBytes(format)

tests/test_butlerFits.py

@@ -33,6 +33,7 @@
 import lsst.afw.image
 from lsst.afw.image import LOCAL
 from lsst.geom import Box2I, Point2I
+from lsst.base import Packages
 from lsst.daf.base import PropertyList, PropertySet
 
 from lsst.daf.butler import Config
@@ -109,6 +110,7 @@ def setUpClass(cls):
         # And some dataset types that have no dimensions for easy testing
         for datasetTypeName, storageClassName in (("ps", "PropertySet"),
                                                   ("pl", "PropertyList"),
+                                                  ("pkg", "Packages")
                                                   ):
             storageClass = cls.storageClassFactory.getStorageClass(storageClassName)
             addDatasetType(cls.creatorButler, datasetTypeName, {}, storageClass)
@@ -179,6 +181,9 @@ def testFundamentalTypes(self) -> None:
         pl.setComment("B", "A string comment")
         self.runFundamentalTypeTest("pl", pl)
 
+        pkg = Packages.fromSystem()
+        self.runFundamentalTypeTest("pkg", pkg)
+
     def testFitsCatalog(self) -> None:
         catalog = self.makeExampleCatalog()
         dataId = {"visit": 42, "instrument": "DummyCam", "physical_filter": "d-r"}