diff --git a/HeterogeneousCore/CUDAServices/plugins/NVProfilerService.cc b/HeterogeneousCore/CUDAServices/plugins/NVProfilerService.cc
index b7dcd34244c3e..5e946681c878c 100644
--- a/HeterogeneousCore/CUDAServices/plugins/NVProfilerService.cc
+++ b/HeterogeneousCore/CUDAServices/plugins/NVProfilerService.cc
@@ -65,6 +65,7 @@ namespace {
     return nvtxDomainRangePushEx(domain, &eventAttrib);
   }
 
+  __attribute__((unused))
   nvtxRangeId_t nvtxDomainRangeStart(nvtxDomainHandle_t domain, const char* message) {
     nvtxEventAttributes_t eventAttrib = { 0 };
     eventAttrib.version         = NVTX_VERSION;
@@ -109,10 +110,12 @@ namespace {
   enum {
     nvtxBlack       = 0x00000000,
     nvtxRed         = 0x00ff0000,
-    nvtxDarkGreen   = 0x0000c000,
+    nvtxDarkGreen   = 0x00009900,
     nvtxGreen       = 0x0000ff00,
+    nvtxLightGreen  = 0x00ccffcc,
     nvtxBlue        = 0x000000ff,
     nvtxAmber       = 0x00ffbf00,
+    nvtxLightAmber  = 0x00fff2cc,
     nvtxWhite       = 0x00ffffff
   };
 }
@@ -269,6 +272,7 @@ class NVProfilerService {
   bool highlight(std::string const&);
 
   std::vector<std::string> highlightModules_;
+  bool                     showModulePrefetching_;
 
   unsigned int concurrentStreams_;
   std::vector<nvtxRangeId_t>              event_;               // per-stream event ranges
@@ -318,6 +322,7 @@ class NVProfilerService {
 
 NVProfilerService::NVProfilerService(edm::ParameterSet const & config, edm::ActivityRegistry & registry) :
   highlightModules_(config.getUntrackedParameter<std::vector<std::string>>("highlightModules")),
+  showModulePrefetching_(config.getUntrackedParameter<bool>("showModulePrefetching")),
   concurrentStreams_(0),
   domains_(this)
 {
@@ -481,9 +486,13 @@ void
 NVProfilerService::fillDescriptions(edm::ConfigurationDescriptions & descriptions) {
   edm::ParameterSetDescription desc;
   desc.addUntracked<std::vector<std::string>>("highlightModules", {})->setComment("");
-  desc.addUntracked<bool>("showDelayedModules", true)->setComment("");
+  desc.addUntracked<bool>("showModulePrefetching", false)->setComment("Show the stack of dependencies that requested to run a module.");
   descriptions.add("NVProfilerService", desc);
-  descriptions.setComment("This Service provides CMSSW-aware annotations to nvprof/nvvm.");
+  descriptions.setComment(R"(This Service provides CMSSW-aware annotations to nvprof/nvvm.
+
+Notes:
+  - the option '--cpu-profiling on' currently results in cmsRun being stuck at the beginning of the job.
+  - the option '--cpu-thread-tracing on' is not compatible with jemalloc, and should only be used with cmsRunGlibC.)");
 }
 
 void
@@ -584,7 +593,7 @@ NVProfilerService::preModuleBeginStream(edm::StreamContext const& sc, edm::Modul
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -603,7 +612,7 @@ NVProfilerService::preModuleEndStream(edm::StreamContext const& sc, edm::ModuleC
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -727,21 +736,25 @@ NVProfilerService::postPathEvent(edm::StreamContext const& sc, edm::PathContext
 
 void
 NVProfilerService::preModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
-  auto sid = sc.streamID();
-  auto mid = mcc.moduleDescription()->id();
-  auto const & label = mcc.moduleDescription()->moduleLabel();
-  auto const & msg = label + " prefetching";
-  if (highlight(label))
-    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
-  else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
+  if (showModulePrefetching_) {
+    auto sid = sc.streamID();
+    auto mid = mcc.moduleDescription()->id();
+    auto const & label = mcc.moduleDescription()->moduleLabel();
+    auto const & msg = label + " prefetching";
+    if (highlight(label))
+      stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxLightAmber);
+    else
+      stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxLightGreen);
+  }
 }
 
 void
 NVProfilerService::postModuleEventPrefetching(edm::StreamContext const& sc, edm::ModuleCallingContext const& mcc) {
-  auto sid = sc.streamID();
-  auto mid = mcc.moduleDescription()->id();
-  nvtxDomainRangeEnd(stream_domain(sid), stream_modules_[sid][mid]);
+  if (showModulePrefetching_) {
+    auto sid = sc.streamID();
+    auto mid = mcc.moduleDescription()->id();
+    nvtxDomainRangeEnd(stream_domain(sid), stream_modules_[sid][mid]);
+  }
 }
 
 void
@@ -753,7 +766,7 @@ NVProfilerService::preModuleConstruction(edm::ModuleDescription const& desc) {
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -770,7 +783,7 @@ NVProfilerService::preModuleBeginJob(edm::ModuleDescription const& desc) {
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -787,7 +800,7 @@ NVProfilerService::preModuleEndJob(edm::ModuleDescription const& desc) {
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -804,7 +817,7 @@ NVProfilerService::preModuleEvent(edm::StreamContext const& sc, edm::ModuleCalli
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxGreen);;
 }
 
 void
@@ -824,7 +837,7 @@ NVProfilerService::preModuleEventDelayedGet(edm::StreamContext const& sc, edm::M
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), label.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), label.c_str(), nvtxGreen);;
   */
 }
 
@@ -847,7 +860,7 @@ NVProfilerService::preEventReadFromSource(edm::StreamContext const& sc, edm::Mod
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
   */
 }
 
@@ -869,7 +882,7 @@ NVProfilerService::preModuleStreamBeginRun(edm::StreamContext const& sc, edm::Mo
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -888,7 +901,7 @@ NVProfilerService::preModuleStreamEndRun(edm::StreamContext const& sc, edm::Modu
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -907,7 +920,7 @@ NVProfilerService::preModuleStreamBeginLumi(edm::StreamContext const& sc, edm::M
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -926,7 +939,7 @@ NVProfilerService::preModuleStreamEndLumi(edm::StreamContext const& sc, edm::Mod
   if (highlight(label))
     stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxAmber);
   else
-    stream_modules_[sid][mid] = nvtxDomainRangeStart(stream_domain(sid), msg.c_str());
+    stream_modules_[sid][mid] = nvtxDomainRangeStartColor(stream_domain(sid), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -944,7 +957,7 @@ NVProfilerService::preModuleGlobalBeginRun(edm::GlobalContext const& gc, edm::Mo
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -961,7 +974,7 @@ NVProfilerService::preModuleGlobalEndRun(edm::GlobalContext const& gc, edm::Modu
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -978,7 +991,7 @@ NVProfilerService::preModuleGlobalBeginLumi(edm::GlobalContext const& gc, edm::M
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -995,7 +1008,7 @@ NVProfilerService::preModuleGlobalEndLumi(edm::GlobalContext const& gc, edm::Mod
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void
@@ -1013,7 +1026,7 @@ NVProfilerService::preSourceConstruction(edm::ModuleDescription const& desc) {
   if (highlight(label))
     global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxAmber);
   else
-    global_modules_[mid] = nvtxDomainRangeStart(global_domain(), msg.c_str());
+    global_modules_[mid] = nvtxDomainRangeStartColor(global_domain(), msg.c_str(), nvtxGreen);;
 }
 
 void