feat: Python bindings for the Examples (#845)

See: https://indico.cern.ch/event/1024261/#3-python-bindings-for-acts-exa

Right now, this PR also contains changes that were made to make the bindings work better / be less awkward in a few places. I'll split those up and make individual PRs for them.

Note: A large fraction of "Files changed" is the bundled copy of `pybind11`.

I extracted the changes I made along the way, but which don't directly depend on the python changes themselves into a series of PRs.

Will be followed up with a number of PRs adding additional example scripts, python based automated tests.

.github/workflows/builds.yml

@@ -45,6 +45,7 @@ jobs:
           -DCMAKE_CXX_FLAGS=-Werror
           -DCMAKE_INSTALL_PREFIX="${INSTALL_DIR}"
           -DACTS_BUILD_EVERYTHING=on
+          -DACTS_BUILD_EXAMPLES_PYTHON_BINDINGS=ON
           -DACTS_LOG_FAILURE_THRESHOLD=WARNING
       - name: Build
         run: ${SETUP} && cmake --build build -- 

CMakeLists.txt

@@ -50,6 +50,7 @@ option(ACTS_BUILD_EXAMPLES_GEANT4 "Build Geant4-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_HEPMC3 "Build HepMC3-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_PYTHIA8 "Build Pythia8-based code in the examples" OFF)
 option(ACTS_BUILD_EXAMPLES_SCRIPTS "Build Analysis applications in the examples" OFF)
+option(ACTS_BUILD_EXAMPLES_PYTHON_BINDINGS "Build python bindings for the examples" OFF)
 # test related options
 option(ACTS_BUILD_BENCHMARKS "Build benchmarks" OFF)
 option(ACTS_BUILD_INTEGRATIONTESTS "Build integration tests" OFF)
@@ -172,6 +173,9 @@ endmacro()
 # the same package twice. This avoids having complex if/else trees to sort out
 # when a particular package is actually needed.
 # plugin dependencies
+if(ACTS_BUILD_EXAMPLES_PYTHON_BINDINGS)
+    add_subdirectory(thirdparty/pybind11)
+endif()
 if(ACTS_BUILD_PLUGIN_AUTODIFF)
   if(ACTS_USE_SYSTEM_AUTODIFF)
     find_package(autodiff ${_acts_autodiff_version} CONFIG REQUIRED)

Core/include/Acts/TrackFitting/KalmanFitter.hpp

@@ -1192,7 +1192,9 @@ class KalmanFitter {
     }
 
     if (!kalmanResult.result.ok()) {
-      ACTS_ERROR("KalmanFilter failed: " << kalmanResult.result.error());
+      ACTS_ERROR("KalmanFilter failed: "
+                 << kalmanResult.result.error() << ", "
+                 << kalmanResult.result.error().message());
       return kalmanResult.result.error();
     }
 

Examples/CMakeLists.txt

@@ -4,4 +4,4 @@ add_subdirectory(Framework)
 add_subdirectory(Io)
 add_subdirectory(Run)
 add_subdirectory_if(Scripts ACTS_BUILD_EXAMPLES_SCRIPTS)
-
+add_subdirectory_if(Python ACTS_BUILD_EXAMPLES_PYTHON_BINDINGS)

Examples/Detectors/TGeoDetector/include/ActsExamples/TGeoDetector/TGeoDetector.hpp

@@ -53,6 +53,9 @@ struct TGeoDetector : public ActsExamples::IBaseDetector {
       LayerTriplet(T value)
           : negative{value}, central{value}, positive{value} {}
 
+      LayerTriplet(T _negative, T _central, T _positive)
+          : negative{_negative}, central{_central}, positive{_positive} {}
+
       T negative;
       T central;
       T positive;

Examples/Framework/include/ActsExamples/Framework/WhiteBoard.hpp

@@ -51,6 +51,8 @@ class WhiteBoard {
   template <typename T>
   const T& get(const std::string& name) const;
 
+  bool exists(const std::string& name) const;
+
  private:
   // type-erased value holder for move-constructible types
   struct IHolder {
@@ -104,3 +106,7 @@ inline const T& ActsExamples::WhiteBoard::get(const std::string& name) const {
   ACTS_VERBOSE("Retrieved object '" << name << "'");
   return reinterpret_cast<const HolderT<T>*>(holder)->value;
 }
+
+inline bool ActsExamples::WhiteBoard::exists(const std::string& name) const {
+  return m_store.find(name) != m_store.end();
+}

Examples/Python/CMakeLists.txt

@@ -0,0 +1,117 @@
+add_library(ActsPythonUtilities INTERFACE)
+target_include_directories(ActsPythonUtilities INTERFACE include)
+
+set(_python_dir "${CMAKE_BINARY_DIR}/python")
+file(MAKE_DIRECTORY ${_python_dir})
+file(MAKE_DIRECTORY ${_python_dir}/acts)
+
+pybind11_add_module(ActsPythonBindings 
+  src/ModuleEntry.cpp
+  src/Base.cpp
+  src/Detector.cpp
+  src/Material.cpp
+  src/Geometry.cpp
+  src/ExampleAlgorithms.cpp
+  src/MagneticField.cpp
+  src/Output.cpp
+  src/Input.cpp
+  src/Propagation.cpp
+  src/Generators.cpp
+  src/TruthTracking.cpp
+  src/TrackFitting.cpp
+  src/TrackFinding.cpp
+  src/Vertexing.cpp
+)
+install(TARGETS ActsPythonBindings DESTINATION .)
+
+set_target_properties(ActsPythonBindings PROPERTIES INSTALL_RPATH "\$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
+set_target_properties(ActsPythonBindings PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${_python_dir}/acts)
+
+target_link_libraries(ActsPythonBindings PUBLIC 
+  ActsCore 
+  ActsPythonUtilities
+  ActsExamplesFramework
+  ActsExamplesDetectorsCommon
+  ActsExamplesDetectorGeneric
+  ActsExamplesDetectorContextual
+  ActsExamplesDetectorTGeo
+  ActsExamplesMagneticField
+  ActsExamplesIoRoot
+  ActsExamplesIoNuclearInteractions
+  ActsExamplesIoCsv
+  ActsExamplesIoObj
+  ActsExamplesIoJson
+  ActsExamplesIoPerformance
+  ActsExamplesGenerators
+  ActsExamplesTrackFinding
+  ActsExamplesTrackFitting
+  ActsExamplesVertexing
+  ActsExamplesFatras
+  ActsExamplesPrinters
+)
+
+set(py_files 
+  __init__.py 
+  examples/__init__.py
+  _adapter.py
+)
+
+if(ACTS_BUILD_PLUGIN_JSON)
+  target_link_libraries(ActsPythonBindings PUBLIC ActsPluginJson)
+  target_sources(ActsPythonBindings PRIVATE src/Json.cpp)
+else()
+  target_sources(ActsPythonBindings PRIVATE src/JsonStub.cpp)
+endif()
+
+if(ACTS_BUILD_PLUGIN_DD4HEP AND ACTS_BUILD_EXAMPLES_DD4HEP)
+  pybind11_add_module(ActsPythonBindingsDD4hep src/DD4hepComponent.cpp)
+  target_link_libraries(ActsPythonBindingsDD4hep PUBLIC 
+    ActsPythonUtilities
+    ActsExamplesDetectorDD4hep)
+  add_dependencies(ActsPythonBindings ActsPythonBindingsDD4hep)
+
+  install(TARGETS ActsPythonBindingsDD4hep DESTINATION .)
+  set_target_properties(ActsPythonBindingsDD4hep PROPERTIES INSTALL_RPATH "\$ORIGIN/${CMAKE_INSTALL_LIBDIR}")
+  set_target_properties(ActsPythonBindingsDD4hep PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${_python_dir}/acts)
+  list(APPEND py_files examples/dd4hep.py)
+endif()
+
+if(ACTS_BUILD_EXAMPLES_PYTHIA8)
+  target_link_libraries(ActsPythonBindings PUBLIC ActsExamplesGeneratorsPythia8)
+  target_sources(ActsPythonBindings PRIVATE src/Pythia8.cpp)
+else()
+  target_sources(ActsPythonBindings PRIVATE src/Pythia8Stub.cpp)
+endif()
+
+if(ACTS_BUILD_PLUGIN_DIGITIZATION AND ACTS_BUILD_PLUGIN_IDENTIFICATION)
+  target_link_libraries(ActsPythonBindings PUBLIC ActsExamplesDigitization)
+  target_sources(ActsPythonBindings PRIVATE src/Digitization.cpp)
+else()
+  target_sources(ActsPythonBindings PRIVATE src/DigitizationStub.cpp)
+endif()
+
+
+if(ACTS_BUILD_EXAMPLES_HEPMC3)
+  target_link_libraries(ActsPythonBindings PUBLIC ActsExamplesHepMC3)
+  target_sources(ActsPythonBindings PRIVATE src/HepMC3.cpp)
+  list(APPEND py_files examples/hepmc3.py)
+else()
+  target_sources(ActsPythonBindings PRIVATE src/HepMC3Stub.cpp)
+endif()
+
+
+add_custom_target(ActsPythonGlueCode)
+configure_file(setup.sh.in ${_python_dir}/setup.sh)
+
+
+foreach(f ${py_files})
+set(_target ${_python_dir}/acts/${f})
+get_filename_component(_dir ${_target} DIRECTORY)
+file(MAKE_DIRECTORY ${_dir})
+
+file(CREATE_LINK ${CMAKE_CURRENT_SOURCE_DIR}/python/acts/${f} ${_target})
+endforeach()
+
+add_dependencies(ActsPythonBindings ActsPythonGlueCode)
+
+

Examples/Python/include/Acts/Plugins/Python/Utilities.hpp

@@ -0,0 +1,68 @@
+// This file is part of the Acts project.
+//
+// Copyright (C) 2021 CERN for the benefit of the Acts project
+//
+// This Source Code Form is subject to the terms of the Mozilla Public
+// License, v. 2.0. If a copy of the MPL was not distributed with this
+// file, You can obtain one at http://mozilla.org/MPL/2.0/.
+
+#pragma once
+
+#include <string>
+#include <unordered_map>
+
+#include <pybind11/pybind11.h>
+
+namespace Acts::Python {
+
+struct Context {
+  std::unordered_map<std::string, pybind11::module_*> modules;
+
+  pybind11::module_& get(const std::string& name) { return *modules.at(name); }
+
+  template <typename... Args,
+            typename = std::enable_if_t<sizeof...(Args) >= 2> >
+  auto get(Args&&... args) {
+    return std::make_tuple((*modules.at(args))...);
+  }
+};
+
+template <typename T, typename Ur, typename Ut>
+void pythonRangeProperty(T& obj, const std::string& name, Ur Ut::*begin,
+                         Ur Ut::*end) {
+  obj.def_property(
+      name.c_str(),
+      [=](Ut& self) {
+        return std::pair{self.*begin, self.*end};
+      },
+      [=](Ut& self, std::pair<Ur, Ur> p) {
+        self.*begin = p.first;
+        self.*end = p.second;
+      });
+}
+
+inline void patchClassesWithConfig(pybind11::module_& m) {
+  pybind11::module::import("acts._adapter").attr("_patch_config")(m);
+}
+
+template <typename T>
+void patchKwargsConstructor(T& c) {
+  pybind11::module::import("acts._adapter").attr("_patchKwargsConstructor")(c);
+}
+
+#define ACTS_PYTHON_MEMBER(name) \
+  _binding_instance.def_readwrite(#name, &_struct_type::name)
+
+#define ACTS_PYTHON_STRUCT_BEGIN(obj, cls) \
+  {                                        \
+    auto& _binding_instance = obj;         \
+    using _struct_type = cls;              \
+    do {                                   \
+    } while (0)
+
+#define ACTS_PYTHON_STRUCT_END() \
+  }                              \
+  do {                           \
+  } while (0)
+
+}  // namespace Acts::Python

Examples/Python/python/acts/__init__.py

@@ -0,0 +1,48 @@
+from pathlib import Path
+from typing import Union
+
+
+from .ActsPythonBindings import *
+from .ActsPythonBindings import __version__
+from . import ActsPythonBindings
+from ._adapter import _patch_config
+
+
+def Propagator(stepper, navigator):
+    for prefix in ("Eigen", "Atlas", "StraightLine"):
+        _stepper = getattr(ActsPythonBindings, f"{prefix}Stepper")
+        if isinstance(stepper, _stepper):
+            return getattr(ActsPythonBindings._propagator, f"{prefix}Propagator")(
+                stepper, navigator
+            )
+    raise TypeError(f"Unknown stepper {type(stepper).__name__}")
+
+
+_patch_config(ActsPythonBindings)
+
+
+@staticmethod
+def _decoratorFromFile(file: Union[str, Path], **kwargs):
+    if isinstance(file, str):
+        file = Path(file)
+
+    kwargs.setdefault("level", ActsPythonBindings.logging.INFO)
+
+    if file.suffix in (".json", ".cbor"):
+        c = ActsPythonBindings.MaterialMapJsonConverter.Config()
+        for k in kwargs.keys():
+            if hasattr(c, k):
+                setattr(c, k, kwargs.pop(k))
+
+        return ActsPythonBindings.JsonMaterialDecorator(
+            jFileName=str(file), rConfig=c, **kwargs
+        )
+    elif file.suffix == ".root":
+        return ActsPythonBindings._examples.RootMaterialDecorator(
+            fileName=str(file), **kwargs
+        )
+    else:
+        raise ValueError(f"Unknown file type {file.suffix}")
+
+
+ActsPythonBindings.IMaterialDecorator.fromFile = _decoratorFromFile