Merge pull request #588 from lsst/tickets/DM-28716

DM-28716: Add pybind wrapper to afw archive that handles adding set of HDUs

include/lsst/afw/table/io/python.h

@@ -47,8 +47,9 @@ namespace python {
  * This allows your class to be used without importing lsst.afw.table in Python.
  *
  * Use as follows:
- * - When declaring the pybind11 class that wraps your Class do *not* list
- *   table::io::PersistableFacade<Class> and table::io::Persistable as base classes.
+ * - When declaring the pybind11 class that wraps your Class in afw, do *not* list
+ *   table::io::PersistableFacade<Class> and table::io::Persistable as base classes
+ *   unless you have to, and in that case be prepared for possible trouble.
  * - Call this function to wrap the methods that make your object persistable.
  */
 template <typename Class, typename... Args>

python/lsst/afw/geom/_transform.cc

@@ -32,6 +32,7 @@
 #include "ndarray/pybind11.h"
 
 #include "lsst/afw/table/io/python.h"
+#include "lsst/afw/table/io/Persistable.h"
 #include "lsst/afw/geom/Endpoint.h"
 #include "lsst/afw/geom/Transform.h"
 
@@ -78,7 +79,7 @@ void declareTransform(lsst::utils::python::WrapperCollection &wrappers) {
 
     std::string const pyClassName = Class::getShortClassName();
     wrappers.wrapType(
-            py::class_<Class, std::shared_ptr<Class>>(wrappers.module, pyClassName.c_str()),
+            py::class_<Class, std::shared_ptr<Class>, table::io::Persistable>(wrappers.module, pyClassName.c_str()),
             [](auto &mod, auto &cls) {
                 std::string const pyClassName = Class::getShortClassName();
                 cls.def(py::init<ast::FrameSet const &, bool>(), "frameSet"_a, "simplify"_a = true);

python/lsst/afw/table/io/SConscript

@@ -3,5 +3,5 @@ from lsst.sconsUtils import scripts
 scripts.BasicSConscript.pybind11(
     ['_io', ],
     # Need .cc here or Scons will look for the wrong file type
-    extraSrc={'_io': ['_fits.cc', '_persistable.cc'], },
+    extraSrc={'_io': ['_fits.cc', '_persistable.cc', '_inputArchive.cc', '_outputArchive.cc'], },
     addUnderscore=False)

python/lsst/afw/table/io/_inputArchive.cc

@@ -0,0 +1,49 @@
+/*
+ * Developed for the LSST Data Management System.
+ * This product includes software developed by the LSST Project
+ * (https://www.lsst.org).
+ * See the COPYRIGHT file at the top-level directory of this distribution
+ * for details of code ownership.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "pybind11/pybind11.h"
+
+#include "lsst/utils/python.h"
+
+#include "lsst/afw/table/io/InputArchive.h"
+#include "lsst/afw/fits.h"
+
+namespace py = pybind11;
+using namespace py::literals;
+
+namespace lsst {
+namespace afw {
+namespace table {
+namespace io {
+
+using PyInputArchive = py::class_<InputArchive, std::shared_ptr<InputArchive>>;
+
+void wrapInputArchive(utils::python::WrapperCollection &wrappers) {
+    wrappers.wrapType(PyInputArchive(wrappers.module, "InputArchive"), [](auto &mod, auto &cls) {
+        cls.def("get", &InputArchive::get<Persistable>);
+        cls.def("readFits", &InputArchive::readFits);
+    });
+}
+
+}  // namespace io
+}  // namespace table
+}  // namespace afw
+}  // namespace lsst

python/lsst/afw/table/io/_io.cc

@@ -37,11 +37,15 @@ using utils::python::WrapperCollection;
 
 void wrapFits(WrapperCollection&);
 void wrapPersistable(WrapperCollection&);
+void wrapInputArchive(WrapperCollection&);
+void wrapOutputArchive(WrapperCollection&);
 
 PYBIND11_MODULE(_io, mod) {
     WrapperCollection wrappers(mod, "lsst.afw.table.io");
     wrapPersistable(wrappers);
     wrapFits(wrappers);
+    wrapInputArchive(wrappers);
+    wrapOutputArchive(wrappers);
     wrappers.finish();
 }
 

python/lsst/afw/table/io/_outputArchive.cc

@@ -0,0 +1,54 @@
+/*
+ * Developed for the LSST Data Management System.
+ * This product includes software developed by the LSST Project
+ * (https://www.lsst.org).
+ * See the COPYRIGHT file at the top-level directory of this distribution
+ * for details of code ownership.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ */
+
+#include "pybind11/pybind11.h"
+
+#include "lsst/utils/python.h"
+
+#include "lsst/afw/table/io/OutputArchive.h"
+#include "lsst/afw/fits.h"
+#include "lsst/afw/table/io/Persistable.h"
+
+namespace py = pybind11;
+using namespace py::literals;
+
+namespace lsst {
+namespace afw {
+namespace table {
+namespace io {
+
+using PyOutputArchive = py::class_<OutputArchive, std::shared_ptr<OutputArchive>>;
+
+void wrapOutputArchive(utils::python::WrapperCollection &wrappers) {
+    wrappers.wrapType(PyOutputArchive(wrappers.module, "OutputArchive"), [](auto &mod, auto &cls) {
+        cls.def(py::init<>());
+        cls.def("put",
+                py::overload_cast<std::shared_ptr<Persistable const>, bool>(&OutputArchive::put),
+                "obj"_a, "permissive"_a=false
+                );
+        cls.def("writeFits", &OutputArchive::writeFits);
+    });
+}
+
+}  // namespace io
+}  // namespace table
+}  // namespace afw
+}  // namespace lsst

src/image/MaskedImage.cc

@@ -454,6 +454,7 @@ void processPlaneMetadata(std::shared_ptr<daf::base::PropertySet const> metadata
     }
     hdr->set("INHERIT", true);
     hdr->set("EXTTYPE", exttype);
+    hdr->set("EXTNAME", exttype);
 }
 
 }  // namespace

src/table/io/OutputArchive.cc

@@ -99,6 +99,8 @@ class OutputArchive::Impl {
         } catch (pex::exceptions::NotFoundError &) {
             iter->getTable()->getMetadata()->add("AR_NAME", name, "Class name for objects stored here");
         }
+        // Also add an EXTNAME. The most recent AR_NAME given will be used.
+        iter->getTable()->getMetadata()->set("EXTNAME", name);
         indexRecord->set(indexKeys.row0, iter->size());
         indexRecord->set(indexKeys.catArchive, catArchive);
         iter->insert(iter->end(), catalog.begin(), catalog.end(), false);
@@ -142,6 +144,7 @@ class OutputArchive::Impl {
     Impl() : _nextId(1), _map(), _index(ArchiveIndexSchema::get().schema) {
         std::shared_ptr<daf::base::PropertyList> metadata(new daf::base::PropertyList());
         metadata->set("EXTTYPE", "ARCHIVE_INDEX");
+        metadata->set("EXTNAME", "ARCHIVE_INDEX");
         metadata->set("AR_CATN", 0, "# of this catalog relative to the start of this archive");
         _index.getTable()->setMetadata(metadata);
     }

tests/test_readers.py

@@ -23,6 +23,7 @@
 
 import os
 import numpy as np
+import astropy.io.fits
 
 import lsst.utils.tests
 from lsst.daf.base import PropertyList
@@ -179,6 +180,17 @@ def checkExposureFitsReader(self, exposureIn, fileName, dtypesOut):
         reader = ExposureFitsReader(fileName)
         self.assertIn('EXPINFO_V', reader.readMetadata().toDict(), "metadata is automatically versioned")
         reader.readMetadata().remove('EXPINFO_V')
+        # ensure EXTNAMEs can be read and make sense
+        extnames = set(('PRIMARY', 'IMAGE', 'MASK', 'VARIANCE', 'ARCHIVE_INDEX',
+                        'Detector', 'TransformMap', 'TransformPoint2ToPoint2',
+                        'FilterLabel', 'SkyWcs', 'ApCorrMap', 'PhotoCalib',
+                        'ChebyshevBoundedField', 'CoaddInputs', 'GaussianPsf',
+                        'Polygon', 'VisitInfo'))
+        with astropy.io.fits.open(fileName) as astropyReadFile:
+            for hdu in astropyReadFile:
+                self.assertIn(hdu.name, extnames)
+        self.assertIn('EXTNAME', reader.readMetadata().toDict(), "EXTNAME is added upon writing")
+        reader.readMetadata().remove('EXTNAME')
         self.assertGreaterEqual(reader.readSerializationVersion(), 0)
         self.assertEqual(exposureIn.getMetadata().toDict(), reader.readMetadata().toDict())
         self.assertWcsAlmostEqualOverBBox(exposureIn.getWcs(), reader.readWcs(), self.bbox,