Merge pull request #3938 from vicentebolea/add-new-python-api

python: Implement Python high level API in python
ornladios · Dec 29, 2023 · 3f91c41 · 3f91c41
2 parents 13723e8 + ab3ab37
commit 3f91c41
Show file tree

Hide file tree

Showing 75 changed files with 3,295 additions and 2,305 deletions.
diff --git a/flake8.cfg → .flake8 b/flake8.cfg → .flake8
diff --git a/.github/workflows/everything.yml b/.github/workflows/everything.yml
@@ -90,9 +90,15 @@ jobs:
       - name: CXX
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-clang-format.sh
-      - name: Python
+      - name: Python flake8
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-flake8.sh
+      - name: Python pylint
+        working-directory: source
+        run: ../gha/scripts/ci/scripts/run-pylint.sh
+      - name: Python black
+        working-directory: source
+        run: ../gha/scripts/ci/scripts/run-black.sh
       - name: Shell
         working-directory: source
         run: ../gha/scripts/ci/scripts/run-shellcheck.sh

diff --git a/.gitignore b/.gitignore
@@ -40,3 +40,7 @@ docs/pyvenv.cfg
 # Visual Studio
 .vs/
 CMakeSettings.json
+
+# Python wheels stuff
+
+*.egg-info/
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -329,6 +329,13 @@ add_subdirectory(source)
 #------------------------------------------------------------------------------#
 add_subdirectory(bindings)
 
+#------------------------------------------------------------------------------#
+# Language Libraries
+#------------------------------------------------------------------------------#
+if(ADIOS2_HAVE_Python)
+  add_subdirectory(python)
+endif()
+
 #------------------------------------------------------------------------------#
 # Plugins
 #------------------------------------------------------------------------------#
@@ -342,6 +349,7 @@ if(ADIOS2_BUILD_EXAMPLES)
   add_subdirectory(examples)
 endif()
 
+
 #------------------------------------------------------------------------------#
 # Testing
 #------------------------------------------------------------------------------#

diff --git a/bindings/Python/CMakeLists.txt b/bindings/Python/CMakeLists.txt
@@ -7,14 +7,12 @@ Python_add_library(adios2_py MODULE
   py11Engine.cpp
   py11Operator.cpp
   py11Query.cpp
-  py11File.cpp py11File.tcc
   py11glue.cpp
 )
-target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2${ADIOS2_LIBRARY_SUFFIX}")
+target_compile_definitions(adios2_py PRIVATE "ADIOS2_PYTHON_MODULE_NAME=adios2_bindings${ADIOS2_LIBRARY_SUFFIX}")
 if(ADIOS2_HAVE_MPI)
   target_sources(adios2_py PRIVATE
     py11ADIOSMPI.cpp
-    py11FileMPI.cpp
     py11IOMPI.cpp
   )
   set(maybe_adios2_cxx11_mpi adios2_cxx11_mpi)
@@ -29,44 +27,42 @@ target_link_libraries(adios2_py PRIVATE
   ${maybe_adios2_cxx11_mpi} adios2_cxx11
   ${maybe_adios2_core_mpi} adios2_core
   adios2::thirdparty::pybind11
-  ${maybe_mpi4py} Python::NumPy
+  ${maybe_mpi4py}
   )
 configure_file(
   ${CMAKE_CURRENT_SOURCE_DIR}/__init__.py.in
-  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
+  ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
   @ONLY
 )
 
 set_target_properties(adios2_py PROPERTIES
   CXX_VISIBILITY_PRESET hidden
-  OUTPUT_NAME adios2${ADIOS2_LIBRARY_SUFFIX}
-  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
-  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2
+  OUTPUT_NAME adios2_bindings${ADIOS2_LIBRARY_SUFFIX}
+  ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  LIBRARY_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  RUNTIME_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
+  COMPILE_PDB_OUTPUT_DIRECTORY ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings
 )
 
-string(REGEX REPLACE "[^/]+" ".." relative_base "${CMAKE_INSTALL_PYTHONDIR}/adios2")
-if(CMAKE_SYSTEM_NAME MATCHES "Linux")
-  if (NOT ADIOS2_USE_PIP)
-    set_target_properties(adios2_py PROPERTIES
-      INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
-    )
-  endif()
-endif()
-
-set(install_location ${CMAKE_INSTALL_PYTHONDIR})
+set(install_location "${CMAKE_INSTALL_PYTHONDIR}/adios2")
 if (ADIOS2_USE_PIP)
   set(install_location ${CMAKE_INSTALL_LIBDIR})
 endif()
 
+string(REGEX REPLACE "[^/]+" ".." relative_base "${install_location}/bindings")
+if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+  set_property(TARGET adios2_py APPEND PROPERTY
+    INSTALL_RPATH "$ORIGIN/${relative_base}/${CMAKE_INSTALL_LIBDIR}"
+  )
+endif()
+
 install(TARGETS adios2_py
-  DESTINATION ${install_location}
+  DESTINATION ${install_location}/bindings
   COMPONENT adios2_python-python
 )
-install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/__init__.py
-  DESTINATION ${install_location}
+install(FILES ${CMAKE_PYTHON_OUTPUT_DIRECTORY}/adios2/bindings/__init__.py
+  DESTINATION ${install_location}/bindings
   COMPONENT adios2_python-python
 )
 install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/test

diff --git a/bindings/Python/__init__.py.in b/bindings/Python/__init__.py.in
@@ -1,3 +1,3 @@
-from .adios2@ADIOS2_LIBRARY_SUFFIX@ import *
+from .adios2_bindings@ADIOS2_LIBRARY_SUFFIX@ import *
 
 __version__ = "@ADIOS2_VERSION@"
diff --git a/bindings/Python/py11Engine.cpp b/bindings/Python/py11Engine.cpp
@@ -171,6 +171,12 @@ void Engine::EndStep()
     m_Engine->EndStep();
 }
 
+bool Engine::BetweenStepPairs() const
+{
+    helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::EndStep");
+    return m_Engine->BetweenStepPairs();
+}
+
 void Engine::Flush(const int transportIndex)
 {
     helper::CheckForNullptr(m_Engine, "for engine, in call to Engine::Flush");
@@ -243,20 +249,37 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
         {
             std::map<std::string, std::string> info_map;
             std::stringstream start_ss;
-            std::cout << "Info loop" << std::endl;
-            for (size_t i = 0; i < (size_t)minBlocksInfo->Dims; ++i)
+            if (info.Start == nullptr)
             {
-                if (i != 0)
-                    start_ss << ",";
-                start_ss << info.Start[i];
+                start_ss << "0";
+            }
+            else
+            {
+                for (size_t i = 0; i < (size_t)minBlocksInfo->Dims; ++i)
+                {
+                    if (i)
+                    {
+                        start_ss << ",";
+                    }
+                    start_ss << info.Start[i];
+                }
             }
             info_map["Start"] = start_ss.str();
             std::stringstream count_ss;
-            for (size_t i = 0; i < (size_t)minBlocksInfo->Dims; ++i)
+            if (info.Count == nullptr)
+            {
+                count_ss << "0";
+            }
+            else
             {
-                if (i != 0)
-                    count_ss << ",";
-                count_ss << info.Count[i];
+                for (size_t i = 0; i < (size_t)minBlocksInfo->Dims; ++i)
+                {
+                    if (i)
+                    {
+                        count_ss << ",";
+                    }
+                    count_ss << info.Count[i];
+                }
             }
             info_map["Count"] = count_ss.str();
             info_map["WriterID"] = std::to_string(info.WriterID);
@@ -317,6 +340,7 @@ std::vector<std::map<std::string, std::string>> Engine::BlocksInfo(std::string &
             info_map["IsReverseDims"] = minBlocksInfo->IsReverseDims ? "True" : "False";
             rv.push_back(info_map);
         }
+        delete minBlocksInfo;
         return rv;
     }
     // Use the macro incantation to call the right instantiation of

diff --git a/bindings/Python/py11Engine.h b/bindings/Python/py11Engine.h
@@ -60,6 +60,12 @@ class Engine
 
     void EndStep();
 
+    /**
+     * Returns current status information for each engine.
+     * @return if between BeginStep/EndStep() pair
+     */
+    bool BetweenStepPairs() const;
+
     void Flush(const int transportIndex = -1);
 
     void Close(const int transportIndex = -1);