Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

NVProfilerService: improve handling of module prefetching #22574

Merged
merged 1 commit into from Mar 13, 2018
Merged

NVProfilerService: improve handling of module prefetching #22574

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
75 changes: 44 additions & 31 deletions HeterogeneousCore/CUDAServices/plugins/NVProfilerService.cc
View file
Open in desktop
Expand Up @@ -65,6 +65,7 @@ namespace {
return nvtxDomainRangePushEx(domain, &eventAttrib);
}

__attribute__((unused))
nvtxRangeId_t nvtxDomainRangeStart(nvtxDomainHandle_t domain, const char* message) {
nvtxEventAttributes_t eventAttrib = { 0 };
eventAttrib.version = NVTX_VERSION;
Expand Down Expand Up @@ -109,10 +110,12 @@ namespace {
enum {
nvtxBlack = 0x00000000,
nvtxRed = 0x00ff0000,
nvtxDarkGreen = 0x0000c000,
nvtxDarkGreen = 0x00009900,
nvtxGreen = 0x0000ff00,
nvtxLightGreen = 0x00ccffcc,
nvtxBlue = 0x000000ff,
nvtxAmber = 0x00ffbf00,
nvtxLightAmber = 0x00fff2cc,
nvtxWhite = 0x00ffffff
};
}
Expand Down Expand Up @@ -269,6 +272,7 @@ class NVProfilerService {
bool highlight(std::string const&);

std::vector<std::string> highlightModules_;
bool showModulePrefetching_;

unsigned int concurrentStreams_;
std::vector<nvtxRangeId_t> event_; // per-stream event ranges
Expand Down Expand Up @@ -318,6 +322,7 @@ class NVProfilerService {

NVProfilerService::NVProfilerService(edm::ParameterSet const & config, edm::ActivityRegistry & registry) :
highlightModules_(config.getUntrackedParameter<std::vector<std::string>>("highlightModules")),
showModulePrefetching_(config.getUntrackedParameter<bool>("showModulePrefetching")),
concurrentStreams_(0),
domains_(this)
{
Expand Down Expand Up @@ -481,9 +486,13 @@ void
NVProfilerService::fillDescriptions(edm::ConfigurationDescriptions & descriptions) {
edm::ParameterSetDescription desc;
desc.addUntracked<std::vector<std::string>>("highlightModules", {})->setComment("");
desc.addUntracked<bool>("showDelayedModules", true)->setComment("");
desc.addUntracked<bool>("showModulePrefetching", false)->setComment("Show the stack of dependencies that requested to run a module.");
descriptions.add("NVProfilerService", desc);
descriptions.setComment("This Service provides CMSSW-aware annotations to nvprof/nvvm.");
descriptions.setComment(R"(This Service provides CMSSW-aware annotations to nvprof/nvvm.

Notes:
- the option '--cpu-profiling on' currently results in cmsRun being stuck at the beginning of the job.
- the option '--cpu-thread-tracing on' is not compatible with jemalloc, and should only be used with cmsRunGlibC.)");
}

void
Expand Down Expand Up @@ -584,7 +593,7 @@ NVProfilerService::preModuleBeginStream(edm::StreamContext const& sc, edm::Modul
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -603,7 +612,7 @@ NVProfilerService::preModuleEndStream(edm::StreamContext const& sc, edm::ModuleC
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand Down Expand Up @@ -727,21 +736,25 @@ NVProfilerService::postPathEvent(edm::StreamContext const& sc, edm::PathContext

void
NVProfilerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
auto const & label = mcc.moduleDescription()->moduleLabel();
auto const & msg = label + " prefetching";
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
if (showModulePrefetching_) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
auto const & label = mcc.moduleDescription()->moduleLabel();
auto const & msg = label + " prefetching";
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxLightAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxLightGreen);
}
}

void
NVProfilerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
nvtxDomainRangeEnd(stream_domain(sid), stream_modules_[sid][mid]);
if (showModulePrefetching_) {
auto sid = sc.streamID();
auto mid = mcc.moduleDescription()->id();
nvtxDomainRangeEnd(stream_domain(sid), stream_modules_[sid][mid]);
}
}

void
Expand All @@ -753,7 +766,7 @@ NVProfilerService::preModuleConstruction(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -770,7 +783,7 @@ NVProfilerService::preModuleBeginJob(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -787,7 +800,7 @@ NVProfilerService::preModuleEndJob(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -804,7 +817,7 @@ NVProfilerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCalli
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxGreen);;
}

void
Expand All @@ -824,7 +837,7 @@ NVProfilerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::M
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxGreen);;
*/
}

Expand All @@ -847,7 +860,7 @@ NVProfilerService::preEventReadFromSource(edm::StreamContext const& sc, edm::Mod
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
*/
}

Expand All @@ -869,7 +882,7 @@ NVProfilerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::Mo
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -888,7 +901,7 @@ NVProfilerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::Modu
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -907,7 +920,7 @@ NVProfilerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::M
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -926,7 +939,7 @@ NVProfilerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::Mod
if (highlight(label))
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
else
stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -944,7 +957,7 @@ NVProfilerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::Mo
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -961,7 +974,7 @@ NVProfilerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::Modu
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -978,7 +991,7 @@ NVProfilerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::M
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -995,7 +1008,7 @@ NVProfilerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::Mod
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand All @@ -1013,7 +1026,7 @@ NVProfilerService::preSourceConstruction(edm::ModuleDescription const& desc) {
if (highlight(label))
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
else
global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
}

void
Expand Down