Merge branch 'tickets/DM-27356' into master

.github/workflows/build.yaml

@@ -34,7 +34,9 @@ jobs:
         run: pip install cheroot wsgidav
 
       - name: Install dependencies
-        run: pip install -r requirements.txt
+        run: |
+          python -m pip install pip==20.2.4
+          pip install -r requirements.txt
 
       # We have two cores so we can speed up the testing with xdist
       - name: Install pytest packages

python/lsst/daf/butler/configs/datastores/formatters.yaml

@@ -62,3 +62,5 @@ BrighterFatterKernel: lsst.daf.butler.formatters.pickle.PickleFormatter
 StructuredDataDict: lsst.daf.butler.formatters.yaml.YamlFormatter
 Filter: lsst.obs.base.formatters.filter.FilterFormatter
 Stamps: lsst.obs.base.formatters.fitsGeneric.FitsGenericFormatter
+AstropyTable: lsst.daf.butler.formatters.astropyTable.AstropyTableFormatter
+AstropyQTable: lsst.daf.butler.formatters.astropyTable.AstropyTableFormatter

python/lsst/daf/butler/configs/storageClasses.yaml

@@ -171,3 +171,7 @@ storageClasses:
     pytype: lsst.verify.Measurement
   Stamps:
     pytype: lsst.meas.algorithms.Stamps
+  AstropyTable:
+    pytype: astropy.table.Table
+  AstropyQTable:
+    pytype: astropy.table.QTable

python/lsst/daf/butler/formatters/astropyTable.py

@@ -0,0 +1,90 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+__all__ = ("AstropyTableFormatter", )
+
+from typing import (
+    Any,
+    Optional,
+    Type,
+)
+import os.path
+
+from .file import FileFormatter
+
+
+class AstropyTableFormatter(FileFormatter):
+    """Interface for reading and writing astropy.Table objects
+    in either ECSV or FITS format
+    """
+    supportedWriteParameters = frozenset({"format"})
+    # Ideally we'd also support fits, but that doesn't
+    # round trip string columns correctly, so things
+    # need to be fixed up on read.
+    supportedExtensions = frozenset({".ecsv", })
+
+    @property
+    def extension(self) -> str:  # type: ignore
+        # Typing is ignored above since this is a property and the
+        # parent class has a class attribute
+
+        # Default to ECSV but allow configuration via write parameter
+        format = self.writeParameters.get("format", "ecsv")
+        if format == "ecsv":
+            return ".ecsv"
+        # Other supported formats can be added here
+        raise RuntimeError(f"Requested file format '{format}' is not supported for Table")
+
+    def _readFile(self, path: str, pytype: Optional[Type[Any]] = None) -> Any:
+        """Read a file from the path in a supported format format.
+
+        Parameters
+        ----------
+        path : `str`
+            Path to use to open the file.
+        pytype : `type`
+            Class to use to read the serialized file.
+
+        Returns
+        -------
+        data : `object`
+            Instance of class ``pytype`` read from serialized file. None
+            if the file could not be opened.
+        """
+        if not os.path.exists(path) or pytype is None:
+            return None
+
+        return pytype.read(path)
+
+    def _writeFile(self, inMemoryDataset: Any) -> None:
+        """Write the in memory dataset to file on disk.
+
+        Parameters
+        ----------
+        inMemoryDataset : `object`
+            Object to serialize.
+
+        Raises
+        ------
+        Exception
+            The file could not be written.
+        """
+        inMemoryDataset.write(self.fileDescriptor.location.path)

tests/test_astropyTableFormatter.py

@@ -0,0 +1,69 @@
+# This file is part of daf_butler.
+#
+# Developed for the LSST Data Management System.
+# This product includes software developed by the LSST Project
+# (http://www.lsst.org).
+# See the COPYRIGHT file at the top-level directory of this distribution
+# for details of code ownership.
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+"""Tests for AstropyTableFormatter.
+"""
+
+import unittest
+import tempfile
+import os
+import shutil
+import numpy
+
+from astropy.table import Table
+
+from lsst.daf.butler import Butler, DatasetType
+
+TESTDIR = os.path.abspath(os.path.dirname(__file__))
+
+
+class AstropyTableFormatterTestCase(unittest.TestCase):
+    """Test for AstropyTableFormatter.
+    """
+
+    def setUp(self):
+        self.root = tempfile.mkdtemp(dir=TESTDIR)
+        Butler.makeRepo(self.root)
+        ints = [1, 2, 3]
+        names = ['one', 'two', 'three']
+        transcendentals = [3.14, 2.718, 0.643]
+        self.table = Table([ints, names, transcendentals],
+                           names=['ints', 'names', 'transcendentals'])
+
+    def tearDown(self):
+        if os.path.exists(self.root):
+            shutil.rmtree(self.root, ignore_errors=True)
+        del self.table
+
+    def testAstropyTableFormatter(self):
+        butler = Butler(self.root, run="testrun")
+        datasetType = DatasetType("table", [], "AstropyTable",
+                                  universe=butler.registry.dimensions)
+        butler.registry.registerDatasetType(datasetType)
+        ref = butler.put(self.table, datasetType)
+        uri = butler.getURI(ref)
+        self.assertEqual(uri.getExtension(), '.ecsv')
+        table = butler.get('table')
+        self.assertTrue(numpy.all(table == self.table))
+
+
+if __name__ == "__main__":
+    unittest.main()