Roctracer flush activity fix + perfetto.cfg (#317)

* Fix roctracer_flush_activity

- invoke roctracer_flush_activity() before disabling domains

* create comp::roctracer::flush()

- real issue was the global state when roctracer_flush_activity() was called

* formatting

* Update lib/omnitrace/library/components/roctracer.hpp

- provide definition of comp::roctracer::flush when OMNITRACE_USE_ROCTRACER is not defined

* omnitrace.cfg -> perfetto.cfg

- rename provided perfetto config file (omnitrace.cfg) to perfetto.cfg to avoid confusion

* Update lib/core

- gpu.hpp: defines for OMNITRACE_USE_{HIP,ROCTRACER,ROCPROFILER,ROCM_SMI}
- gpu.cpp
  - include core/hip_runtime.hpp
  - fix serialization of hipDeviceProp_t
- add hip_runtime.hpp
  -  ensure proper inclusion of hip_runtime.h
- add rccl.hpp
  - ensure proper inclusion of rccl.h

* Update lib/omnitrace/library

- rcclp.cpp
  - update includes for rccl
- roctracer.hpp
  - update includes for hip_runtime
- components/comm_data.hpp
  - update includes for rccl
- components/rcclp.hpp
  - update includes for rccl

* Update bin/omnitrace-avail/avail.cpp

- update includes for hip_runtime

* Update examples/rccl/CMakeLists.txt

- fix find_package for rccl when CI enabled

* Update CMakeLists.txt

- set cmake policy CMP0135 to NEW for cmake >= 3.24
  - Enable DOWNLOAD_EXTRACT_TIMESTAMP with ExternalProject_Add + URL download method

* Update timemory submodule

* Update pybind11 submodule

* Update pybind11 submodule

* Update lib/core/rccl.hpp

- include rccl.h only if OMNITRACE_USE_RCCL > 0

* Update lib/core/{gpu,hip_runtime}.hpp

* Update lib/core/gpu.cpp

- reintroduce some ppdefs

* Update lib/core/gpu.cpp

- fix ifdef on OMNITRACE_HIP_VERSION

* Update lib/core/gpu.cpp

- fix static assert for OMNITRACE_HIP_VERSION_MINOR when HIP version 4.x or older (unreliable minor versions)

* Update lib/core/gpu.cpp

- fix ifdef on OMNITRACE_HIP_VERSION

* Update lib/core/config.cpp

- disable OMNITRACE_PERFETTO_COMBINE_TRACES by default

* Update lib/core/perfetto.cpp

- if unable to open perfetto temp file, return the ReadTraceBlocking()

* Update lib/core/config.*

- flush tmpfile before closing

CMakeLists.txt

@@ -71,6 +71,10 @@ set(CMAKE_POSITION_INDEPENDENT_CODE
     ON
     CACHE BOOL "Build position independent code")
 
+if(CMAKE_VERSION VERSION_GREATER_EQUAL 3.24)
+    cmake_policy(SET CMP0135 NEW)
+endif()
+
 if("${CMAKE_BUILD_TYPE}" STREQUAL "")
     set(CMAKE_BUILD_TYPE
         Release
@@ -366,8 +370,8 @@ if(NOT OMNITRACE_USE_ROCPROFILER)
 endif()
 
 configure_file(
-    ${PROJECT_SOURCE_DIR}/omnitrace.cfg
-    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/omnitrace.cfg
+    ${PROJECT_SOURCE_DIR}/perfetto.cfg
+    ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/perfetto.cfg
     COPYONLY)
 
 configure_file(
@@ -381,7 +385,7 @@ configure_file(
 
 install(
     FILES ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/setup-env.sh
-          ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/omnitrace.cfg
+          ${PROJECT_BINARY_DIR}/${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}/perfetto.cfg
     DESTINATION ${CMAKE_INSTALL_DATAROOTDIR}/${PROJECT_NAME}
     COMPONENT setup)
 

README.md

@@ -325,7 +325,7 @@ Enable `traced` and `perfetto` in the background:
 ```shell
 pkill traced
 traced --background
-perfetto --out ./omnitrace-perfetto.proto --txt -c ${OMNITRACE_ROOT}/share/omnitrace.cfg --background
+perfetto --out ./omnitrace-perfetto.proto --txt -c ${OMNITRACE_ROOT}/share/perfetto.cfg --background
 ```
 
 > ***NOTE: if the perfetto tools were installed by omnitrace, replace `traced` with `omnitrace-perfetto-traced` and***

examples/rccl/CMakeLists.txt

@@ -31,7 +31,10 @@ if(NOT hip_FOUND)
     return()
 endif()
 
-if("${CMAKE_PROJECT_NAME}" STREQUAL "omnitrace" AND "$ENV{OMNITRACE_CI}")
+if("${CMAKE_PROJECT_NAME}" STREQUAL "omnitrace"
+   AND ("$ENV{OMNITRACE_CI}"
+        OR OMNITRACE_CI
+        OR OMNITRACE_BUILD_CI))
     find_package(rccl QUIET) # avoid generating warning in CI
 else()
     find_package(rccl)

omnitrace.cfg → perfetto.cfg

@@ -1,6 +1,4 @@
-# perfetto --out OUTPUT_FILE --txt -c omnitrace.cfg
-# 5 minute trace, but can be stopped prematurely.
-duration_ms: 300000
+# perfetto --out OUTPUT_FILE --txt -c perfetto.cfg
 write_into_file: true
 
 # One buffer allocated within the central tracing binary for the entire trace,

source/bin/omnitrace-avail/avail.cpp

@@ -33,6 +33,7 @@
 #include "api.hpp"
 #include "core/config.hpp"
 #include "core/gpu.hpp"
+#include "core/hip_runtime.hpp"
 #include "library/rocprofiler.hpp"
 
 #include <timemory/components.hpp>
@@ -62,12 +63,6 @@
 #include <utility>
 #include <vector>
 
-#if defined(OMNITRACE_USE_HIP) && OMNITRACE_USE_HIP > 0
-#    include <hip/hip_runtime.h>
-#elif !defined(OMNITRACE_USE_HIP)
-#    define OMNITRACE_USE_HIP 0
-#endif
-
 #if defined(TIMEMORY_UNIX)
 #    include <sys/ioctl.h>  // ioctl() and TIOCGWINSZ
 #    include <unistd.h>     // for STDOUT_FILENO

source/docs/runtime.md

@@ -205,7 +205,7 @@ OMNITRACE_KOKKOSP_KERNEL_LOGGER                    = false
 OMNITRACE_PAPI_EVENTS                              = PAPI_TOT_CYC
 OMNITRACE_PERFETTO_BACKEND                         = inprocess
 OMNITRACE_PERFETTO_BUFFER_SIZE_KB                  = 1024000
-OMNITRACE_PERFETTO_COMBINE_TRACES                  = true
+OMNITRACE_PERFETTO_COMBINE_TRACES                  = false
 OMNITRACE_PERFETTO_FILE                            = perfetto-trace.proto
 OMNITRACE_PERFETTO_FILL_POLICY                     = discard
 OMNITRACE_PERFETTO_SHMEM_SIZE_HINT_KB              = 4096

source/lib/core/CMakeLists.txt

@@ -29,10 +29,12 @@ set(core_headers
     ${CMAKE_CURRENT_LIST_DIR}/dynamic_library.hpp
     ${CMAKE_CURRENT_LIST_DIR}/exception.hpp
     ${CMAKE_CURRENT_LIST_DIR}/gpu.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/hip_runtime.hpp
     ${CMAKE_CURRENT_LIST_DIR}/locking.hpp
     ${CMAKE_CURRENT_LIST_DIR}/mproc.hpp
     ${CMAKE_CURRENT_LIST_DIR}/perf.hpp
     ${CMAKE_CURRENT_LIST_DIR}/perfetto.hpp
+    ${CMAKE_CURRENT_LIST_DIR}/rccl.hpp
     ${CMAKE_CURRENT_LIST_DIR}/redirect.hpp
     ${CMAKE_CURRENT_LIST_DIR}/state.hpp
     ${CMAKE_CURRENT_LIST_DIR}/timemory.hpp

source/lib/core/config.cpp

@@ -31,6 +31,7 @@
 #include "perfetto.hpp"
 #include "utility.hpp"
 
+#include <asm-generic/errno-base.h>
 #include <timemory/backends/capability.hpp>
 #include <timemory/backends/dmp.hpp>
 #include <timemory/backends/mpi.hpp>
@@ -650,8 +651,7 @@ configure_settings(bool _init)
     OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_PERFETTO_COMBINE_TRACES",
                              "Combine Perfetto traces. If not explicitly set, it will "
                              "default to the value of OMNITRACE_COLLAPSE_PROCESSES",
-                             _config->get<bool>("collapse_processes"), "perfetto", "data",
-                             "advanced");
+                             false, "perfetto", "data", "advanced");
 
     OMNITRACE_CONFIG_SETTING(
         bool, "OMNITRACE_PERFETTO_ROCTRACER_PER_STREAM",
@@ -2527,9 +2527,34 @@ tmp_file::fopen(const char* _mode)
     return (file != nullptr && fd > 0);
 }
 
+bool
+tmp_file::flush()
+{
+    if(stream.is_open())
+    {
+        stream.flush();
+    }
+    else if(file != nullptr)
+    {
+        int _ret = fflush(file);
+        int _cnt = 0;
+        while(_ret == EAGAIN || _ret == EINTR)
+        {
+            std::this_thread::sleep_for(std::chrono::milliseconds{ 100 });
+            _ret = fflush(file);
+            if(++_cnt > 10) break;
+        }
+        return (_ret == 0);
+    }
+
+    return true;
+}
+
 bool
 tmp_file::close()
 {
+    flush();
+
     if(stream.is_open())
     {
         stream.close();

source/lib/core/config.hpp

@@ -394,6 +394,7 @@ struct tmp_file
 
     bool open(std::ios::openmode = std::ios::binary | std::ios::in | std::ios::out);
     bool fopen(const char* = "r+");
+    bool flush();
     bool close();
     bool remove();