Skip to content

Commit

Permalink
Merge pull request #22574 from fwyzard/NVProfilerService_module_prefe…
Browse files Browse the repository at this point in the history 
…tching

NVProfilerService: improve handling of module prefetching
  • Loading branch information
@cmsbuild
cmsbuild committed Mar 13, 2018
2 parents 445faaf + 4cc9fa2 commit 8142cc2
Showing 1 changed file with 44 additions and 31 deletions.
75 changes: 44 additions & 31 deletions HeterogeneousCore/CUDAServices/plugins/NVProfilerService.cc
View file
Expand Up @@ -65,6 +65,7 @@ namespace {
return nvtxDomainRangePushEx(domain, &eventAttrib);
}

__attribute__((unused))
nvtxRangeId_t nvtxDomainRangeStart(nvtxDomainHandle_t domain, const char* message) {
nvtxEventAttributes_t eventAttrib = { 0 };
eventAttrib.version = NVTX_VERSION;
Expand Down Expand Up @@ -109,10 +110,12 @@ namespace {
enum {
nvtxBlack = 0x00000000,
nvtxRed = 0x00ff0000,
nvtxDarkGreen = 0x0000c000,
nvtxDarkGreen = 0x00009900,
nvtxGreen = 0x0000ff00,
nvtxLightGreen = 0x00ccffcc,
nvtxBlue = 0x000000ff,
nvtxAmber = 0x00ffbf00,
nvtxLightAmber = 0x00fff2cc,
nvtxWhite = 0x00ffffff
};
}
Expand Down Expand Up @@ -269,6 +272,7 @@ class NVProfilerService {
bool highlight(std::string const&);

std::vector<std::string> highlightModules_;
bool showModulePrefetching_;

unsigned int concurrentStreams_;
std::vector<nvtxRangeId_t> event_; // per-stream event ranges
Expand Down Expand Up @@ -318,6 +322,7 @@ class NVProfilerService {

NVProfilerService::NVProfilerService(edm::ParameterSet const & config, edm::ActivityRegistry & registry) :
highlightModules_(config.getUntrackedParameter<std::vector<std::string>>("highlightModules")),
showModulePrefetching_(config.getUntrackedParameter<bool>("showModulePrefetching")),
concurrentStreams_(0),
domains_(this)
{
Expand Down Expand Up @@ -481,9 +486,13 @@ void
NVProfilerService::fillDescriptions(edm::ConfigurationDescriptions & descriptions) {
edm::ParameterSetDescription desc;
desc.addUntracked<std::vector<std::string>>("highlightModules", {})->setComment("");
desc.addUntracked<bool>("showDelayedModules", true)->setComment("");
desc.addUntracked<bool>("showModulePrefetching", false)->setComment("Show the stack of dependencies that requested to run a module.");
descriptions.add("NVProfilerService", desc);
descriptions.setComment("This Service provides CMSSW-aware annotations to nvprof/nvvm.");
descriptions.setComment(R"(This Service provides CMSSW-aware annotations to nvprof/nvvm.
Notes:
- the option '--cpu-profiling on' currently results in cmsRun being stuck at the beginning of the job.
- the option '--cpu-thread-tracing on' is not compatible with jemalloc, and should only be used with cmsRunGlibC.)");
}

void
Expand Down Expand Up @@ -584,7 +593,7 @@ NVProfilerService::preModuleBeginStream(edm::StreamContext const& sc, edm::Modul
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -603,7 +612,7 @@ NVProfilerService::preModuleEndStream(edm::StreamContext const& sc, edm::ModuleC
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand Down Expand Up @@ -727,21 +736,25 @@ NVProfilerService::postPathEvent(edm::StreamContext const& sc, edm::PathContext

void
NVProfilerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
auto const & label = mcc.moduleDescription()->moduleLabel();
auto const & msg = label + " prefetching";
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
if (showModulePrefetching_) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
auto const & label = mcc.moduleDescription()->moduleLabel();
auto const & msg = label + " prefetching";
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxLightAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxLightGreen);
}
}

void
NVProfilerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
nvtxDomainRangeEnd(stream_domain(sid), stream_modules_[sid][mid]);
if (showModulePrefetching_) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
nvtxDomainRangeEnd(stream_domain(sid), stream_modules_[sid][mid]);
}
}

void
Expand All @@ -753,7 +766,7 @@ NVProfilerService::preModuleConstruction(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -770,7 +783,7 @@ NVProfilerService::preModuleBeginJob(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -787,7 +800,7 @@ NVProfilerService::preModuleEndJob(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -804,7 +817,7 @@ NVProfilerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCalli
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxGreen);;
}

void
Expand All @@ -824,7 +837,7 @@ NVProfilerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::M
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxGreen);;
*/
}

Expand All @@ -847,7 +860,7 @@ NVProfilerService::preEventReadFromSource(edm::StreamContext const& sc, edm::Mod
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
*/
}

Expand All @@ -869,7 +882,7 @@ NVProfilerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::Mo
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -888,7 +901,7 @@ NVProfilerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::Modu
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -907,7 +920,7 @@ NVProfilerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::M
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -926,7 +939,7 @@ NVProfilerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::Mod
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -944,7 +957,7 @@ NVProfilerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::Mo
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -961,7 +974,7 @@ NVProfilerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::Modu
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -978,7 +991,7 @@ NVProfilerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::M
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -995,7 +1008,7 @@ NVProfilerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::Mod
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -1013,7 +1026,7 @@ NVProfilerService::preSourceConstruction(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand Down

0 comments on commit 8142cc2

Please sign in to comment.