Caliper integration with new PFA (Clang 18)

.github/workflows/github-actions.yml

@@ -9,49 +9,76 @@ jobs:
     runs-on: ubuntu-24.04
     strategy:
       matrix:
-        config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only]
+        config: [boilerplate, release, debug_cuda_only, debug_mpi_only, debug_threads_only, boilerplate_caliper]
 
         include:
           - config: boilerplate
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: ON
             PERFFLOWASPECT_WITH_MULTITHREADS: ON
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: release
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: ON
             PERFFLOWASPECT_WITH_MULTITHREADS: ON
             CMAKE_BUILD_TYPE: Release
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: debug_cuda_only
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: OFF
             PERFFLOWASPECT_WITH_MULTITHREADS: OFF
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: debug_mpi_only
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: ON
             PERFFLOWASPECT_WITH_MULTITHREADS: OFF
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
           - config: debug_threads_only
             PERFFLOWASPECT_WITH_CUDA: OFF
             PERFFLOWASPECT_WITH_MPI: OFF
             PERFFLOWASPECT_WITH_MULTITHREADS: ON
             CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: OFF
 
+          - config: boilerplate_caliper
+            PERFFLOWASPECT_WITH_CUDA: OFF
+            PERFFLOWASPECT_WITH_MPI: OFF
+            PERFFLOWASPECT_WITH_MULTITHREADS: OFF
+            CMAKE_BUILD_TYPE: Debug
+            PERFFLOWASPECT_WITH_CALIPER: ON
+
     steps:
       # Checkout PerfFlowAspect repository under $GITHUB_WORKSPACE
       - uses: actions/checkout@v2
 
       - name: Setup Build Env
         run: |
           sudo apt-get update
-          sudo apt install clang-18 llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich
+          sudo apt install clang-18 llvm-dev libjansson-dev libssl-dev bison flex make cmake mpich pybind11-dev
           clang++ --version
 
+      - name: Clone Caliper
+        uses: actions/checkout@v2
+        with:
+          repository: LLNL/Caliper
+          path: Caliper
+
+      - name: Build Caliper
+        working-directory: Caliper
+        run: |
+          mkdir build && mkdir install
+          cd build
+          cmake -DWITH_PYTHON_BINDINGS=On -DCMAKE_INSTALL_PREFIX=../install ../
+          make VERBOSE=1
+          make install
+
       - name: Compile check
         run: |
           cd src/c
@@ -62,6 +89,7 @@ jobs:
           export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CUDA=${{matrix.PERFFLOWASPECT_WITH_CUDA}}"
           export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MPI=${{matrix.PERFFLOWASPECT_WITH_MPI}}"
           export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_MULTITHREADS=${{matrix.PERFFLOWASPECT_WITH_MULTITHREADS}}"
+          export CMAKE_OPTS="${CMAKE_OPTS} -DPERFFLOWASPECT_WITH_CALIPER=${{matrix.PERFFLOWASPECT_WITH_CALIPER}} -Dcaliper_DIR=/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install"
           echo -e ${CMAKE_OPTS}
           cmake ${CMAKE_OPTS} ..
           # build
@@ -97,6 +125,8 @@ jobs:
           cd src/python
           python -m pip install -r requirements.txt
           python -m pip list
+          # Export path for caliper
+          export PYTHONPATH="/home/runner/work/PerfFlowAspect/PerfFlowAspect/Caliper/install/lib/${{ matrix.python-version }}/site-packages:$PYTHONPATH"
 
       - name: Run Python Smoke Tests
         run: |

docs/BasicTutorial.rst

@@ -103,6 +103,8 @@ variable. Separate multiple variables with a colon as follows:
 +----------------------+--------------------------------------------------------------+---------------+---------------------------------+
 | log-format           | Dump JSON events in array or object format                   | Array         | Array, Object                   |
 +----------------------+--------------------------------------------------------------+---------------+---------------------------------+
+| caliper-enable       | Toggle Caliper data collection on/off (for Python only).     | False         | True, False                     |
++----------------------+--------------------------------------------------------------+---------------+---------------------------------+
 
 **********************************************
  Visualization of PerfFlowAspect Output Files

docs/CaliperIntegration.rst

@@ -0,0 +1,128 @@
+..
+   # Copyright 2021 Lawrence Livermore National Security, LLC and other
+   # PerfFlowAspect Project Developers. See the top-level LICENSE file for
+   # details.
+   #
+   # SPDX-License-Identifier: LGPL-3.0
+
+#####################
+ Caliper Integration
+#####################
+
+PerfFlowAspect can be built with Caliper to leverage collection of additional
+performance data, such as hardware performance counters on CPUs and GPU
+measurements on NVIDIA GPUs. `Caliper <https://github.com/llnl/caliper>`_ is an
+instrumentation and performance annotation library.
+
+PerfFlowAspect supports the Caliper integration for both C/C++ and Python
+codebases. This enables a single PerfFlowAspect annotation to generate both
+`.pfw` trace files and `.cali` files, so users do not need any additional
+annotations to obtain Caliper data.
+
+A Caliper install is required before building PerfFlowAspect in this case.
+Caliper can be configured at runtime, as shown in examples below.
+
+*************
+ C/C++ Build
+*************
+
+Caliper needs to be is built with the same `clang` compiler chain as
+PerfFlowAspect. These can be specified with the `-DCMAKE_C_COMPILER` and
+`-DCMAKE_CXX_COMPILER`.
+
+.. code:: bash
+
+   cmake -DCMAKE_C_COMPILER=<path-to-clang18-compiler> \
+         -DCMAKE_C_COMPILER=<path-to-clang18++-compiler> \
+         -DWITH_PYTHON_BINDINGS=On \
+         -Dcaliper_DIR=<path-to-install>/share/lib/caliper ../
+
+***************
+ C/C++ Example
+***************
+
+.. code:: bash
+
+   CALI_CONFIG=runtime-report ./smoketest
+   CALI_CONFIG=runtime-report,output=test.cali ./smoketest
+
+.. code:: bash
+
+   Path        Min time/rank Max time/rank Avg time/rank Time %
+   _Z3fooRKSs       0.004527      0.004527      0.004527 45.778137
+     _Z3barv        0.004511      0.004511      0.004511 45.616341
+       _Z3basv      0.000079      0.000079      0.000079  0.798867
+
+.. code:: bash
+
+   CALI_PAPI_COUNTERS=PAPI_TOT_CYC,PAPI_L2_DCM CALI_SERVICES_ENABLE=event,trace,papi,report ./smoketest
+
+.. code:: bash
+
+   event.begin#annotation papi.PAPI_TOT_CYC papi.PAPI_L2_DCM annotation                 region.count event.end#annotation
+   _Z3fooRKSs                        118289              679
+   _Z3barv                           200050              765 _Z3fooRKSs
+   _Z3basv                           115098              352 _Z3fooRKSs/_Z3barv
+                                      66564              242 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv
+                                     117061              385 _Z3fooRKSs/_Z3barv                    1 _Z3barv
+                                      93592              206 _Z3fooRKSs                            1 _Z3fooRKSs
+   _Z3fooRKSs                        146308              332
+   _Z3barv                            87811              255 _Z3fooRKSs
+   _Z3basv                            84904              244 _Z3fooRKSs/_Z3barv
+                                      34547               66 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv
+                                      82540              168 _Z3fooRKSs/_Z3barv                    1 _Z3barv
+                                      80711              144 _Z3fooRKSs                            1 _Z3fooRKSs
+   _Z3fooRKSs                        127765              183
+   _Z3barv                            85440              241 _Z3fooRKSs
+   _Z3basv                            82100              250 _Z3fooRKSs/_Z3barv
+                                      33969               67 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv
+                                      81511              161 _Z3fooRKSs/_Z3barv                    1 _Z3barv
+                                      77498              128 _Z3fooRKSs                            1 _Z3fooRKSs
+   _Z3fooRKSs                        119853              164
+   _Z3barv                            83285              227 _Z3fooRKSs
+   _Z3basv                            82702              297 _Z3fooRKSs/_Z3barv
+                                      34170               78 _Z3fooRKSs/_Z3barv/_Z3basv            1 _Z3basv
+                                      81589              149 _Z3fooRKSs/_Z3barv                    1 _Z3barv
+                                      78920              119 _Z3fooRKSs                            1 _Z3fooRKSs
+
+.. code:: bash
+
+   PERFFLOW_OPTIONS='cpu-mem-usage=True:log-event=compact' CALI_CONFIG="load(time_exclusive.json),spot" ./smoketest
+
+.. code:: bash
+
+   Path        Min time/rank Max time/rank Avg time/rank Total time spot.channel
+   _Z3fooRKSs       0.018068      0.018068      0.018068   0.018068 regionprofile
+     _Z3barv        0.009124      0.009124      0.009124   0.009124 regionprofile
+       _Z3basv      0.000074      0.000074      0.000074   0.000074 regionprofile
+
+****************
+ Python Example
+****************
+
+For Python applications, Caliper can be enabled by setting the `caliper-enable`
+option. In this case, `PYTHONPATH` needs to be set to include the `pycaliper`
+package from Caliper installation.
+
+.. code:: bash
+
+   export PYTHONPATH=<path-to-caliper-install>/lib/python<X.Y>/site-packages
+   PERFFLOW_OPTIONS="caliper-enable=True" CALI_CONFIG=runtime-report ./smoketest.py
+
+   Inside main
+   foo
+   bar
+   bas
+   foo
+   bar
+   bas
+   foo
+   bar
+   bas
+   foo
+   bar
+   bas
+   Path                                   Time (E) Time (I) Time % (E) Time % (I)
+   <function foo at 0x155546a24540>       0.004692 0.009423  47.547246  95.486397
+      <function bar at 0x155546a24400>    0.004659 0.004731  47.213609  47.939151
+         <function bas at 0x15554732de40> 0.000072 0.000072   0.725542   0.725542

docs/index.rst

@@ -85,6 +85,7 @@ uniformity as to how performance is measured and controlled.
 
    BuildingPerfFlowAspect
    Annotations
+   CaliperIntegration
    UpcomingFeatures
 
 .. toctree::

src/c/CMakeLists.txt

@@ -2,6 +2,9 @@ cmake_minimum_required(VERSION 3.12)
 
 project(PerfFlowAspect VERSION "0.1.0")
 
+# Higher-level build options.
+option(PERFFLOWASPECT_WITH_CALIPER "Build with Caliper support" OFF)
+
 # Fail if using Clang < 18.0
 if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang")
     if (CMAKE_CXX_COMPILER_VERSION VERSION_LESS 18.0)

src/c/cmake/Setup3rdParty.cmake

@@ -19,3 +19,19 @@ include(cmake/thirdparty/FindOpenSSL.cmake)
 if(PERFFLOWASPECT_WITH_MULTITHREADS)
     include(cmake/thirdparty/FindThreads.cmake)
 endif()
+
+if(PERFFLOWASPECT_WITH_CALIPER)
+    # first Check for CALIPER_DIR
+    if(NOT caliper_DIR)
+        MESSAGE(FATAL_ERROR "Caliper support needs explicit caliper_DIR")
+    endif()
+
+    if(caliper_DIR)
+        message(STATUS "${caliper_DIR}")
+        include(cmake/thirdparty/FindCaliper.cmake)
+    endif()
+
+    if(CALIPER_FOUND)
+        add_definitions(-DPERFFLOWASPECT_WITH_CALIPER)
+    endif()
+endif()

src/c/cmake/thirdparty/FindCaliper.cmake

@@ -0,0 +1,15 @@
+message(STATUS "Looking for Caliper in: ${caliper_DIR}")
+
+find_package(caliper REQUIRED
+             PATHS ${caliper_DIR}/share/cmake/caliper
+             NO_DEFAULT_PATH
+             NO_CMAKE_ENVIRONMENT_PATH
+             NO_CMAKE_PATH
+             NO_SYSTEM_ENVIRONMENT_PATH
+             NO_CMAKE_SYSTEM_PATH)
+
+message(STATUS "FOUND Caliper: ${caliper_DIR}")
+
+#set(ADIAK_FOUND TRUE)
+set(CALIPER_FOUND TRUE)
+set(PERFFLOWASPECT_CALIPER_ENABLED TRUE)

src/c/config/CMakeLists.txt

@@ -42,6 +42,8 @@ configure_package_config_file(
 install(FILES
     ${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config.cmake
     ${CMAKE_CURRENT_BINARY_DIR}/perfflowaspect-config-version.cmake
+    perfflowaspect_setup_deps.cmake
+    perfflowaspect_setup_targets.cmake
     DESTINATION ${PERFFLOWASPECT_INSTALL_CMAKE_MODULE_DIR})
 
 # Create pkg-config .pc file

src/c/config/perfflowaspect-config.cmake.in

@@ -3,6 +3,9 @@ if (NOT PERFFLOWASPECT_CONFIG_LOADED)
   set(PERFFLOWASPECT_DIR     "@CMAKE_INSTALL_PREFIX@")
   set(PERFFLOWASPECT_LIB_DIR "@CMAKE_INSTALL_PREFIX@/@CMAKE_INSTALL_LIBDIR@")
 
+  set(PERFFLOWASPECT_CALIPER_ENABLED "@PERFFLOWASPECT_CALIPER_ENABLED@")
+  set(PERFFLOWASPECT_CALIPER_DIR  "@caliper_DIR@")
+
   include(CMakeFindDependencyMacro)
 
   find_dependency(OpenSSL REQUIRED)

src/c/config/perfflowaspect_setup_deps.cmake

@@ -0,0 +1,17 @@
+include(CMakeFindDependencyMacro)
+
+if (NOT caliper_DIR)
+    set(caliper_DIR ${PERFFLOWASPECT_CALIPER_DIR})
+endif()
+
+if(caliper_DIR)
+    if(NOT PerfFlowAspect_FIND_QUIETLY)
+        message(STATUS "PerfFlowAspect was built with Caliper Support")
+        message(STATUS "Looking for Caliper at: ${caliper_DIR}/share/cmake/caliper")
+    endif()
+
+    # find caliper
+    find_package(caliper REQUIRED
+                 NO_DEFAULT_PATH
+                 PATHS ${caliper_DIR}/share/cmake/caliper)
+endif()

src/c/config/perfflowaspect_setup_targets.cmake

@@ -0,0 +1,6 @@
+# create convenience target that bundles all reg perfflowaspect deps
+add_library(perfflowaspect::perfflowaspect INTERFACE IMPORTED)
+
+set_property(TARGET perfflowaspect::perfflowaspect
+             PROPERTY INTERFACE_LINK_LIBRARIES
+             perfflowaspect)

src/c/host-configs/tioga-4.18.0-x86_64-rocm@6.3.1-amdclang@18.0.0.cmake

@@ -10,3 +10,8 @@
 
 set(CMAKE_C_COMPILER "/opt/rocm-6.3.1/lib/llvm/bin/amdclang" CACHE PATH "")
 set(CMAKE_CXX_COMPILER "/opt/rocm-6.3.1/lib/llvm/bin/amdclang++" CACHE PATH "")
+
+# To enable fine-grained profiling with Caliper, utilize the below options.
+# Caliper should be built with the same compilers as the PerfFlowAspect.
+# set(PERFFLOWASPECT_WITH_CALIPER ON CACHE BOOL "") 
+# set(caliper_DIR "path-to-caliper-install-directory" CACHE PATH "") 

src/c/host-configs/tuolumne-4.18.0-x86_64-rocm@6.2.1-amdclang@18.0.0.cmake

@@ -10,3 +10,8 @@
 
 set(CMAKE_C_COMPILER "/opt/rocm-6.2.1/lib/llvm/bin/amdclang" CACHE PATH "")
 set(CMAKE_CXX_COMPILER "/opt/rocm-6.2.1/lib/llvm/bin/amdclang++" CACHE PATH "")
+
+# To enable fine-grained profiling with Caliper, utilize the below options.
+# Caliper should be built with the same compilers as the PerfFlowAspect.
+# set(PERFFLOWASPECT_WITH_CALIPER ON CACHE BOOL "") 
+# set(caliper_DIR "path-to-caliper-install-directory" CACHE PATH "") 

src/c/runtime/CMakeLists.txt

@@ -14,12 +14,20 @@ set(perfflow_runtime_sources
 
 include_directories(${JANSSON_INCLUDE_DIRS})
 
+if(PERFFLOWASPECT_WITH_CALIPER)
+    include_directories(${caliper_INCLUDE_DIR})
+endif()
+
 add_library(perfflow_runtime SHARED
             ${perfflow_runtime_sources}
             ${perfflow_runtime_headers}
 )
 
-target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
+if(PERFFLOWASPECT_WITH_CALIPER)
+    target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto caliper)
+else()
+    target_link_libraries(perfflow_runtime ${JANSSON_LIBRARY} OpenSSL::SSL OpenSSL::Crypto)
+endif()
 
 install(TARGETS perfflow_runtime
         EXPORT perfflow_export