Skip to content

Commit

Permalink
Remove critical trace usage in source code
Browse files Browse the repository at this point in the history
  • Loading branch information
jrmadsen committed Jan 11, 2024
1 parent cfa62a0 commit 465c403
Show file tree
Hide file tree
Showing 16 changed files with 20 additions and 519 deletions.
1 change: 0 additions & 1 deletion source/bin/omnitrace-causal/impl.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -200,7 +200,6 @@ get_initial_environment()
update_env(_env, "OMNITRACE_TRACE", false);
update_env(_env, "OMNITRACE_PROFILE", false);
update_env(_env, "OMNITRACE_USE_PROCESS_SAMPLING", false);
update_env(_env, "OMNITRACE_CRITICAL_TRACE", false);
update_env(_env, "OMNITRACE_THREAD_POOL_SIZE",
get_env<int>("OMNITRACE_THREAD_POOL_SIZE", 0));
update_env(_env, "OMNITRACE_LAUNCHER", "omnitrace-causal");
Expand Down
1 change: 0 additions & 1 deletion source/lib/core/argparse.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1225,7 +1225,6 @@ add_core_arguments(parser_t& _parser, parser_data& _data)
add_group_arguments(_parser, "perfetto", _data, true);
add_group_arguments(_parser, "timemory", _data, true);
add_group_arguments(_parser, "rocm", _data, true);
add_group_arguments(_parser, "critical_trace", _data, true);

_parser.start_group("MISCELLANEOUS OPTIONS", "");

Expand Down
6 changes: 0 additions & 6 deletions source/lib/core/categories.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -110,9 +110,6 @@ OMNITRACE_DEFINE_CATEGORY(category, mpi, OMNITRACE_CATEGORY_MPI, "mpi", "MPI reg
OMNITRACE_DEFINE_CATEGORY(category, ompt, OMNITRACE_CATEGORY_OMPT, "ompt", "OpenMP tools regions")
OMNITRACE_DEFINE_CATEGORY(category, process_sampling, OMNITRACE_CATEGORY_PROCESS_SAMPLING, "process_sampling", "Process-level data")
OMNITRACE_DEFINE_CATEGORY(category, comm_data, OMNITRACE_CATEGORY_COMM_DATA, "comm_data", "MPI/RCCL counters for tracking amount of data sent or received")
OMNITRACE_DEFINE_CATEGORY(category, critical_trace, OMNITRACE_CATEGORY_CRITICAL_TRACE, "critical-trace", "Critical trace data")
OMNITRACE_DEFINE_CATEGORY(category, host_critical_trace, OMNITRACE_CATEGORY_HOST_CRITICAL_TRACE, "host-critical-trace", "Host-side critical trace data")
OMNITRACE_DEFINE_CATEGORY(category, device_critical_trace, OMNITRACE_CATEGORY_DEVICE_CRITICAL_TRACE, "device-critical-trace", "Device-side critical trace data")
OMNITRACE_DEFINE_CATEGORY(category, causal, OMNITRACE_CATEGORY_CAUSAL, "causal", "Causal profiling data")
OMNITRACE_DEFINE_CATEGORY(category, cpu_freq, OMNITRACE_CATEGORY_CPU_FREQ, "cpu_frequency", "CPU frequency (collected in background thread)")
OMNITRACE_DEFINE_CATEGORY(category, process_page, OMNITRACE_CATEGORY_PROCESS_PAGE, "process_page_fault", "Memory page faults in process (collected in background thread)")
Expand Down Expand Up @@ -174,9 +171,6 @@ using name = perfetto_category<Tp...>;
OMNITRACE_PERFETTO_CATEGORY(category::sampling), \
OMNITRACE_PERFETTO_CATEGORY(category::process_sampling), \
OMNITRACE_PERFETTO_CATEGORY(category::comm_data), \
OMNITRACE_PERFETTO_CATEGORY(category::critical_trace), \
OMNITRACE_PERFETTO_CATEGORY(category::host_critical_trace), \
OMNITRACE_PERFETTO_CATEGORY(category::device_critical_trace), \
OMNITRACE_PERFETTO_CATEGORY(category::causal), \
OMNITRACE_PERFETTO_CATEGORY(category::cpu_freq), \
OMNITRACE_PERFETTO_CATEGORY(category::process_page), \
Expand Down
75 changes: 0 additions & 75 deletions source/lib/core/config.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -520,10 +520,6 @@ configure_settings(bool _init)
_backend, "perfetto")
->set_choices({ "inprocess", "system", "all" });

OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_CRITICAL_TRACE",
"Enable generation of the critical trace", false, "backend",
"critical_trace");

OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_TRACE_THREAD_LOCKS",
"Enable tracing calls to pthread_mutex_lock, "
"pthread_mutex_unlock, pthread_mutex_trylock",
Expand Down Expand Up @@ -647,15 +643,6 @@ configure_settings(bool _init)
"is collected on every available device",
"", "rocprofiler", "rocm", "hardware_counters");

OMNITRACE_CONFIG_SETTING(bool, "OMNITRACE_CRITICAL_TRACE_DEBUG",
"Enable debugging for critical trace", _omnitrace_debug,
"debugging", "critical_trace", "advanced");

OMNITRACE_CONFIG_SETTING(
bool, "OMNITRACE_CRITICAL_TRACE_SERIALIZE_NAMES",
"Include names in serialization of critical trace (mainly for debugging)",
_omnitrace_debug, "debugging", "critical_trace", "advanced");

OMNITRACE_CONFIG_SETTING(size_t, "OMNITRACE_PERFETTO_SHMEM_SIZE_HINT_KB",
"Hint for shared-memory buffer size in perfetto (in KB)",
size_t{ 4096 }, "perfetto", "data", "advanced");
Expand Down Expand Up @@ -721,21 +708,6 @@ configure_settings(bool _init)
1),
"parallelism", "advanced");

OMNITRACE_CONFIG_EXT_SETTING(int64_t, "OMNITRACE_CRITICAL_TRACE_COUNT",
"Number of critical trace to export (0 == all)",
int64_t{ 0 }, "critical_trace",
"omnitrace-critical-trace", "advanced");

OMNITRACE_CONFIG_SETTING(uint64_t, "OMNITRACE_CRITICAL_TRACE_BUFFER_COUNT",
"Number of critical trace records to store in thread-local "
"memory before submitting to shared buffer",
uint64_t{ 2000 }, "critical_trace", "advanced");

OMNITRACE_CONFIG_EXT_SETTING(
int64_t, "OMNITRACE_CRITICAL_TRACE_PER_ROW",
"How many critical traces per row in perfetto (0 == all in one row)",
int64_t{ 0 }, "critical_trace", "omnitrace-critical-trace", "advanced");

OMNITRACE_CONFIG_SETTING(
std::string, "OMNITRACE_TIMEMORY_COMPONENTS",
"List of components to collect via timemory (see `omnitrace-avail -C`)",
Expand Down Expand Up @@ -1157,14 +1129,12 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
_set("OMNITRACE_USE_OMPT", false);
_set("OMNITRACE_USE_SAMPLING", false);
_set("OMNITRACE_USE_PROCESS_SAMPLING", false);
_set("OMNITRACE_CRITICAL_TRACE", false);
}
else if(get_mode() == Mode::Causal)
{
_set("OMNITRACE_USE_CAUSAL", true);
_set("OMNITRACE_TRACE", false);
_set("OMNITRACE_PROFILE", false);
_set("OMNITRACE_CRITICAL_TRACE", false);
_set("OMNITRACE_USE_SAMPLING", false);
_set("OMNITRACE_USE_PROCESS_SAMPLING", false);
}
Expand Down Expand Up @@ -1223,7 +1193,6 @@ configure_mode_settings(const std::shared_ptr<settings>& _config)
_set("OMNITRACE_USE_SAMPLING", false);
_set("OMNITRACE_USE_PROCESS_SAMPLING", false);
_set("OMNITRACE_USE_CODE_COVERAGE", false);
_set("OMNITRACE_CRITICAL_TRACE", false);
set_setting_value("OMNITRACE_TIMEMORY_COMPONENTS", std::string{});
set_setting_value("OMNITRACE_PAPI_EVENTS", std::string{});
}
Expand Down Expand Up @@ -1404,7 +1373,6 @@ configure_disabled_settings(const std::shared_ptr<settings>& _config)
_handle_use_option("OMNITRACE_USE_ROCM_SMI", "rocm_smi");
_handle_use_option("OMNITRACE_USE_ROCTRACER", "roctracer");
_handle_use_option("OMNITRACE_USE_ROCPROFILER", "rocprofiler");
_handle_use_option("OMNITRACE_CRITICAL_TRACE", "critical_trace");

#if !defined(OMNITRACE_USE_ROCTRACER) || OMNITRACE_USE_ROCTRACER == 0
_config->find("OMNITRACE_USE_ROCTRACER")->second->set_hidden(true);
Expand Down Expand Up @@ -1971,13 +1939,6 @@ get_use_mpip()
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
}

bool&
get_use_critical_trace()
{
static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE");
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
}

bool
get_use_kokkosp()
{
Expand Down Expand Up @@ -2024,20 +1985,6 @@ get_num_threads_hint()
return static_cast<tim::tsettings<size_t>&>(*_v->second).get();
}

bool
get_critical_trace_debug()
{
static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_DEBUG");
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
}

bool
get_critical_trace_serialize_names()
{
static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_SERIALIZE_NAMES");
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
}

bool
get_sampling_keep_internal()
{
Expand Down Expand Up @@ -2094,13 +2041,6 @@ get_trace_hsa_activity()
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
}

int64_t
get_critical_trace_per_row()
{
static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_PER_ROW");
return static_cast<tim::tsettings<int64_t>&>(*_v->second).get();
}

size_t
get_perfetto_shmem_size_hint()
{
Expand Down Expand Up @@ -2210,14 +2150,6 @@ get_perfetto_annotations()
return static_cast<tim::tsettings<bool>&>(*_v->second).get();
}

uint64_t
get_critical_trace_update_freq()
{
static uint64_t _v =
get_config()->get<uint64_t>("OMNITRACE_CRITICAL_TRACE_BUFFER_COUNT");
return _v;
}

uint64_t
get_thread_pool_size()
{
Expand Down Expand Up @@ -2389,13 +2321,6 @@ get_sampling_allocator_size()
return std::max<size_t>(static_cast<tim::tsettings<size_t>&>(*_v->second).get(), 1);
}

int64_t
get_critical_trace_count()
{
static auto _v = get_config()->find("OMNITRACE_CRITICAL_TRACE_COUNT");
return static_cast<tim::tsettings<int64_t>&>(*_v->second).get();
}

double
get_process_sampling_freq()
{
Expand Down
18 changes: 0 additions & 18 deletions source/lib/core/config.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -218,9 +218,6 @@ get_use_pid();
bool&
get_use_mpip();

bool&
get_use_critical_trace() OMNITRACE_HOT;

bool
get_use_kokkosp();

Expand Down Expand Up @@ -251,12 +248,6 @@ get_trace_hsa_api();
bool
get_trace_hsa_activity();

bool
get_critical_trace_debug();

bool
get_critical_trace_serialize_names();

size_t
get_perfetto_shmem_size_hint();

Expand All @@ -278,9 +269,6 @@ get_disabled_categories();
bool
get_perfetto_annotations() OMNITRACE_HOT;

uint64_t
get_critical_trace_update_freq();

uint64_t
get_thread_pool_size();

Expand All @@ -297,9 +285,6 @@ get_perfetto_output_filename();
bool
get_perfetto_roctracer_per_stream() OMNITRACE_HOT;

int64_t
get_critical_trace_count();

double
get_trace_delay();

Expand Down Expand Up @@ -360,9 +345,6 @@ get_process_sampling_duration();
std::string
get_sampling_gpus();

int64_t
get_critical_trace_per_row();

bool
get_trace_thread_locks();

Expand Down
9 changes: 0 additions & 9 deletions source/lib/core/debug.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,9 +67,6 @@ get_debug_tid() OMNITRACE_HOT;

bool
get_debug_pid() OMNITRACE_HOT;

bool
get_critical_trace_debug() OMNITRACE_HOT;
} // namespace config

namespace debug
Expand Down Expand Up @@ -560,12 +557,6 @@ as_hex<void*>(void*, size_t);
#define OMNITRACE_BASIC_DEBUG_F(...) \
OMNITRACE_CONDITIONAL_BASIC_PRINT_F(::omnitrace::get_debug_env(), __VA_ARGS__)

#define OMNITRACE_CT_DEBUG(...) \
OMNITRACE_CONDITIONAL_PRINT(::omnitrace::get_critical_trace_debug(), __VA_ARGS__)

#define OMNITRACE_CT_DEBUG_F(...) \
OMNITRACE_CONDITIONAL_PRINT_F(::omnitrace::get_critical_trace_debug(), __VA_ARGS__)

//--------------------------------------------------------------------------------------//
//
// Verbose macros
Expand Down
2 changes: 1 addition & 1 deletion source/lib/core/exception.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -50,14 +50,14 @@ template <typename... Args>
auto
get_backtrace(Args... _arg)
{
consume_args(_arg...);
auto _bt = std::stringstream{};
if constexpr(sizeof...(Args) > 0)
{
((_bt << _arg), ...) << "\n";
}
tim::unwind::detailed_backtrace<2>(_bt, true);
return strdup(_bt.str().c_str());
consume_args(_arg...);
}
} // namespace

Expand Down
3 changes: 0 additions & 3 deletions source/lib/omnitrace-user/omnitrace/categories.h
Original file line number Diff line number Diff line change
Expand Up @@ -63,9 +63,6 @@ extern "C"
OMNITRACE_CATEGORY_OMPT,
OMNITRACE_CATEGORY_PROCESS_SAMPLING,
OMNITRACE_CATEGORY_COMM_DATA,
OMNITRACE_CATEGORY_CRITICAL_TRACE,
OMNITRACE_CATEGORY_HOST_CRITICAL_TRACE,
OMNITRACE_CATEGORY_DEVICE_CRITICAL_TRACE,
OMNITRACE_CATEGORY_CAUSAL,
OMNITRACE_CATEGORY_CPU_FREQ,
OMNITRACE_CATEGORY_PROCESS_PAGE,
Expand Down
52 changes: 0 additions & 52 deletions source/lib/omnitrace/library.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,6 @@
#include "library/components/pthread_gotcha.hpp"
#include "library/components/rocprofiler.hpp"
#include "library/coverage.hpp"
#include "library/critical_trace.hpp"
#include "library/ompt.hpp"
#include "library/process_sampler.hpp"
#include "library/ptl.hpp"
Expand Down Expand Up @@ -203,9 +202,6 @@ ensure_finalization(bool _static_init = false)
return scope::destructor{ []() { omnitrace_finalize_hidden(); } };
}

using Device = critical_trace::Device;
using Phase = critical_trace::Phase;

template <typename... Tp>
struct fini_bundle
{
Expand Down Expand Up @@ -402,11 +398,6 @@ omnitrace_init_library_hidden()
if(_debug_init) config::set_setting_value("OMNITRACE_DEBUG", _debug_value);
} };

// below will effectively do:
// get_cpu_cid_stack(0)->emplace_back(-1);
// plus query some env variables
add_critical_trace<Device::CPU, Phase::NONE>(0, -1, 0, 0, 0, 0, 0, 0, 0, -1, 0);

tim::trait::runtime_enabled<comp::roctracer>::set(get_use_roctracer());
tim::trait::runtime_enabled<comp::roctracer_data>::set(get_use_roctracer() &&
get_use_timemory());
Expand Down Expand Up @@ -920,55 +911,12 @@ omnitrace_finalize_hidden(void)
causal::finish_experimenting();
}

if(get_use_critical_trace() || (get_use_rocm_smi() && get_use_roctracer()))
{
OMNITRACE_VERBOSE_F(1, "Generating the critical trace...\n");
for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
{
using critical_trace_hash_data =
thread_data<critical_trace::hash_ids, critical_trace::id>;

if(i < critical_trace_hash_data::get()->size() &&
critical_trace_hash_data::get()->at(i))
{
OMNITRACE_DEBUG_F("Copying the hash id data for thread %zu...\n", i);
critical_trace::add_hash_id(*critical_trace_hash_data::get()->at(i));
}
}

for(size_t i = 0; i < thread_info::get_peak_num_threads(); ++i)
{
using critical_trace_chain_data = thread_data<critical_trace::call_chain>;

if(i < critical_trace_chain_data::get()->size() &&
critical_trace_chain_data::get()->at(i))
{
OMNITRACE_DEBUG_F(
"Updating the critical trace call-chains for thread %zu...\n", i);
critical_trace::update(i); // launch update task
}
}

OMNITRACE_VERBOSE_F(1, "Waiting on critical trace updates...\n");
tasking::join();
}

if(get_use_process_sampling())
{
OMNITRACE_VERBOSE_F(1, "Post-processing the system-level samples...\n");
process_sampler::post_process();
}

if(get_use_critical_trace())
{
// launch compute task
OMNITRACE_VERBOSE_F(1, "Launching critical trace compute task...\n");
critical_trace::compute();

OMNITRACE_VERBOSE_F(1, "Waiting on critical trace computation...\n");
tasking::join();
}

// shutdown tasking before timemory is finalized, especially the roctracer thread-pool
OMNITRACE_VERBOSE_F(1, "Shutting down thread-pools...\n");
tasking::shutdown();
Expand Down
Loading

0 comments on commit 465c403

Please sign in to comment.