DM-8467: Wrap lsst_distrib with pybind11

.gitignore

@@ -8,8 +8,6 @@ _build*
 *.so
 *.cfgc
 *.pyc
-*_wrap.cc
-*Lib.py
 doc/html
 doc/*.tag
 doc/*.inc

include/lsst/daf/persistence/python/readProxy.h

@@ -0,0 +1,78 @@
+// -*- lsst-c++ -*-
+
+/* 
+ * LSST Data Management System
+ * Copyright 2008, 2009, 2010 LSST Corporation.
+ * 
+ * This product includes software developed by the
+ * LSST Project (http://www.lsst.org/).
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ * 
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ * 
+ * You should have received a copy of the LSST License Statement and 
+ * the GNU General Public License along with this program.  If not, 
+ * see <http://www.lsstcorp.org/LegalNotices/>.
+ */
+
+#ifndef LSST_DAF_PERSISTENCE_PYTHON_READPROXY_H
+#define LSST_DAF_PERSISTENCE_PYTHON_READPROXY_H
+
+#include "pybind11/pybind11.h"
+
+namespace lsst {
+namespace daf {
+namespace persistence {
+namespace python {
+
+/** @class lsst::daf::persistence::python::ReadProxyBase
+  * @brief Base class for lazy-loading proxy.
+  *
+  * This class exists purely as a base class for ReadProxy in Python.
+  *
+  * @ingroup daf_persistence
+  */
+
+class ReadProxyBase {
+public:
+    pybind11::object subject;
+};
+
+/** Register a type for (optional) lazy load.
+  * 
+  * When called in the pybind11 wrapper this function adds an implict conversion
+  * from ReadProxy to the registered type.
+  * 
+  * @tparam OutputType Type to register (e.g. DateTime)
+  */
+
+template <typename OutputType> void register_proxy() {
+    namespace py = pybind11;
+    auto implicit_caster = [](PyObject *obj, PyTypeObject *type) -> PyObject * {
+        if (!py::detail::make_caster<lsst::daf::persistence::python::ReadProxyBase>().load(obj, false)) {
+            return nullptr;
+        }
+        PyObject *result = PyObject_GetAttrString(obj, "__subject__");
+        if (result == nullptr) {
+            PyErr_Clear();  // needed to fall through to next conversion
+        }
+        return result;
+    };
+
+    if (auto tinfo = py::detail::get_type_info(typeid(OutputType))) {
+        tinfo->implicit_conversions.push_back(implicit_caster);
+    } else {
+        py::pybind11_fail("register_proxy: Unable to find type " + py::type_id<OutputType>());
+    }
+}
+
+}}}}
+
+#endif

python/lsst/daf/persistence/SConscript

@@ -1,3 +1,3 @@
 # -*- python -*-
 from lsst.sconsUtils import scripts
-scripts.BasicSConscript.python(['persistenceLib'])
+scripts.BasicSConscript.pybind11(['storage/storage', 'logicalLocation', 'persistence', 'dbAuth', 'dbStorage'], addUnderscore=False)

python/lsst/daf/persistence/__init__.py

@@ -22,11 +22,20 @@
 
 """Python interface to lsst::daf::persistence classes
 """
+from __future__ import absolute_import
+
+StorageList = list
+
+from .logicalLocation import *
+from .persistence import *
+from .storage import *
+from .dbAuth import *
+from .dbStorage import *
+
 from .utils import *
 from .genericAssembler import *
 from .registries import *
 from .fsScanner import *
-from .persistenceLib import *
 from .butlerExceptions import *
 from .policy import *
 from .registries import *
@@ -36,11 +45,11 @@
 from .butlerSubset import *
 from .access import *
 from .repositoryCfg import *
-from .storage import *
 from .posixStorage import *
 from .mapper import *
 from .repositoryMapper import *
 from .repository import *
 from .butler import *
 from .butlerFactory import *
 from .version import *
+

python/lsst/daf/persistence/butlerLocation.py

@@ -208,7 +208,8 @@ def __repr__(self):
 
     def __init__(self, pythonType, cppType, storageName, locationList, dataId, mapper, storage=None,
                  usedDataId=None, datasetType=None):
-        self.pythonType = pythonType
+        # pythonType is sometimes unicode with Python 2 and pybind11; this breaks the interpreter
+        self.pythonType = str(pythonType) if isinstance(pythonType, basestring) else pythonType
         self.cppType = cppType
         self.storageName = storageName
         self.mapper = mapper

python/lsst/daf/persistence/dbAuth.cc

@@ -0,0 +1,28 @@
+#include "pybind11/pybind11.h"
+
+#include "lsst/daf/persistence/DbAuth.h"
+
+namespace py = pybind11;
+
+namespace lsst {
+namespace daf {
+namespace persistence {
+
+PYBIND11_PLUGIN(dbAuth) {
+    py::module mod("dbAuth");
+
+    py::class_<DbAuth> cls(mod, "DbAuth");
+
+    cls.def_static("setPolicy", &DbAuth::setPolicy);
+    cls.def_static("resetPolicy", &DbAuth::resetPolicy);
+    cls.def_static("available", &DbAuth::available);
+    cls.def_static("authString", &DbAuth::authString);
+    cls.def_static("username", &DbAuth::username);
+    cls.def_static("password", &DbAuth::password);
+
+    return mod.ptr();
+}
+
+}  // persistence
+}  // daf
+}  // lsst

python/lsst/daf/persistence/dbStorage.cc

@@ -0,0 +1,76 @@
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+#include "lsst/daf/persistence/Storage.h"
+#include "lsst/daf/persistence/DbStorage.h"
+#include "lsst/daf/persistence/LogicalLocation.h"
+
+namespace py = pybind11;
+using namespace pybind11::literals;
+
+namespace lsst {
+namespace daf {
+namespace persistence {
+namespace {
+
+template <typename T, typename C>
+void declareParams(C& cls, const std::string& suffix) {
+    cls.def(("setColumn" + suffix).c_str(), &DbStorage::setColumn<T>);
+    cls.def(("condParam" + suffix).c_str(), &DbStorage::condParam<T>);
+    cls.def(("outParam" + suffix).c_str(), &DbStorage::outParam<T>);
+    cls.def(("getColumnByPos" + suffix).c_str(), &DbStorage::getColumnByPos<T>);
+}
+
+}  // <anonymous>
+
+PYBIND11_PLUGIN(dbStorage) {
+    py::module mod("dbStorage");
+
+    py::class_<DbStorage, std::shared_ptr<DbStorage>, Storage> cls(mod, "DbStorage");
+
+    /* Constructors */
+    cls.def(py::init<>());
+
+    /* Member functions */
+    cls.def("setPolicy", &DbStorage::setPolicy);
+    cls.def("setPersistLocation", &DbStorage::setPersistLocation);
+    cls.def("setRetrieveLocation", &DbStorage::setRetrieveLocation);
+    cls.def("startTransaction", &DbStorage::startTransaction);
+    cls.def("endTransaction", &DbStorage::endTransaction);
+    cls.def("createTableFromTemplate", &DbStorage::createTableFromTemplate, "tableName"_a, "templateName"_a,
+            "mayAlreadyExist"_a = false);
+    cls.def("dropTable", &DbStorage::dropTable);
+    cls.def("truncateTable", &DbStorage::truncateTable);
+    cls.def("executeSql", &DbStorage::executeSql);
+    cls.def("setTableForInsert", &DbStorage::setTableForInsert);
+    cls.def("setColumnToNull", &DbStorage::setColumnToNull);
+    cls.def("insertRow", &DbStorage::insertRow);
+    cls.def("setTableForQuery", &DbStorage::setTableForQuery, "tableName"_a, "isExpr"_a = false);
+    cls.def("setTableListForQuery", &DbStorage::setTableListForQuery);
+    cls.def("outColumn", &DbStorage::outColumn, "columnName"_a, "isExpr"_a = false);
+    cls.def("orderBy", &DbStorage::orderBy);
+    cls.def("groupBy", &DbStorage::groupBy);
+    cls.def("setQueryWhere", &DbStorage::setQueryWhere);
+    cls.def("query", &DbStorage::query);
+    cls.def("next", &DbStorage::next);
+    cls.def("__next__", &DbStorage::next);
+    cls.def("columnIsNull", &DbStorage::columnIsNull);
+    cls.def("finishQuery", &DbStorage::finishQuery);
+
+    /* Templated member functions */
+    declareParams<char>(cls, "Char");
+    declareParams<short>(cls, "Short");
+    declareParams<int>(cls, "Int");
+    declareParams<long>(cls, "Long");
+    declareParams<long long>(cls, "Int64");
+    declareParams<float>(cls, "Float");
+    declareParams<double>(cls, "Double");
+    declareParams<std::string>(cls, "String");
+    declareParams<bool>(cls, "Bool");
+
+    return mod.ptr();
+}
+
+}  // persistence
+}  // daf
+}  // lsst

python/lsst/daf/persistence/logicalLocation.cc

@@ -0,0 +1,28 @@
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+#include "lsst/daf/base/PropertySet.h"
+#include "lsst/daf/persistence/LogicalLocation.h"
+
+namespace py = pybind11;
+
+namespace lsst {
+namespace daf {
+namespace persistence {
+
+PYBIND11_PLUGIN(logicalLocation) {
+    py::module mod("logicalLocation");
+
+    py::class_<LogicalLocation> cls(mod, "LogicalLocation");
+
+    cls.def(py::init<std::string const&>());
+    cls.def(py::init<std::string const&, CONST_PTR(dafBase::PropertySet)>());
+    cls.def("locString", &LogicalLocation::locString);
+    cls.def_static("setLocationMap", LogicalLocation::setLocationMap);
+
+    return mod.ptr();
+}
+
+}  // persistence
+}  // daf
+}  // lsst

python/lsst/daf/persistence/persistence.cc

@@ -0,0 +1,41 @@
+#include "pybind11/pybind11.h"
+#include "pybind11/stl.h"
+
+#include "lsst/daf/base/Citizen.h"
+#include "lsst/daf/base/PropertySet.h"
+#include "lsst/daf/persistence/Persistence.h"
+#include "lsst/daf/persistence/Storage.h"
+#include "lsst/daf/persistence/LogicalLocation.h"
+
+#include "lsst/daf/persistence/python/readProxy.h"
+
+namespace py = pybind11;
+
+namespace lsst {
+namespace daf {
+namespace persistence {
+
+PYBIND11_PLUGIN(persistence) {
+    py::module::import("lsst.daf.base");
+
+    py::module mod("persistence");
+
+    py::class_<python::ReadProxyBase, std::shared_ptr<python::ReadProxyBase>>(mod, "ReadProxyBase")
+            .def(py::init<>())
+            .def_readwrite("subject", &python::ReadProxyBase::subject);
+
+    py::class_<Persistence, std::shared_ptr<Persistence>, base::Citizen> clsPersistence(mod, "Persistence");
+
+    clsPersistence.def("getPersistStorage", &Persistence::getPersistStorage);
+    clsPersistence.def("getRetrieveStorage", &Persistence::getRetrieveStorage);
+    clsPersistence.def("persist", &Persistence::persist);
+    clsPersistence.def("retrieve", &Persistence::retrieve);
+    clsPersistence.def("unsafeRetrieve", &Persistence::unsafeRetrieve);
+    clsPersistence.def_static("getPersistence", &Persistence::getPersistence);
+
+    return mod.ptr();
+}
+
+}  // persistence
+}  // daf
+}  // lsst