Create new ECAL DQM GpuTask to monitor and compare CPU and GPU generated ECAL Rec Hits

DQM/EcalMonitorTasks/interface/Collections.h

@@ -41,6 +41,10 @@ namespace ecaldqm {
     kEEBasicCluster,
     kEBSuperCluster,
     kEESuperCluster,
+    kEBCpuRecHit,
+    kEECpuRecHit,
+    kEBGpuRecHit,
+    kEEGpuRecHit,
     nCollections
   };
 
@@ -78,7 +82,11 @@ namespace ecaldqm {
                                                     "EBBasicCluster",
                                                     "EEBasicCluster",
                                                     "EBSuperCluster",
-                                                    "EESuperCluster"};
+                                                    "EESuperCluster",
+                                                    "EBCpuRecHit",
+                                                    "EECpuRecHit",
+                                                    "EBGpuRecHit",
+                                                    "EEGpuRecHit"};
 
 }  // namespace ecaldqm
 

DQM/EcalMonitorTasks/interface/GpuTask.h

@@ -0,0 +1,56 @@
+#ifndef DQM_EcalMonitorTasks_GpuTask_H
+#define DQM_EcalMonitorTasks_GpuTask_H
+
+#include "DQWorkerTask.h"
+
+#include "DataFormats/EcalRecHit/interface/EcalRecHitCollections.h"
+
+namespace ecaldqm {
+
+  class GpuTask : public DQWorkerTask {
+  public:
+    GpuTask();
+    ~GpuTask() override {}
+
+    void addDependencies(DependencySet&) override;
+
+    bool filterRunType(short const*) override;
+
+    void beginEvent(edm::Event const&, edm::EventSetup const&, bool const&, bool&) override;
+    bool analyze(void const*, Collections) override;
+
+    void runOnCpuRecHits(EcalRecHitCollection const&, Collections);
+    void runOnGpuRecHits(EcalRecHitCollection const&, Collections);
+
+  private:
+    void setParams(edm::ParameterSet const&) override;
+
+    bool runOnGpu_;
+    EcalRecHitCollection const* EBCpuRecHits_;
+    EcalRecHitCollection const* EECpuRecHits_;
+  };
+
+  inline bool GpuTask::analyze(void const* _p, Collections _collection) {
+    switch (_collection) {
+      case kEBCpuRecHit:
+      case kEECpuRecHit:
+        if (_p)
+          runOnCpuRecHits(*static_cast<EcalRecHitCollection const*>(_p), _collection);
+        return true;
+        break;
+      case kEBGpuRecHit:
+      case kEEGpuRecHit:
+        if (_p && runOnGpu_)
+          runOnGpuRecHits(*static_cast<EcalRecHitCollection const*>(_p), _collection);
+        return runOnGpu_;
+        break;
+      default:
+        break;
+    }
+
+    return false;
+  }
+
+}  // namespace ecaldqm
+
+#endif

DQM/EcalMonitorTasks/plugins/EcalDQMonitorTask2.cc

@@ -119,6 +119,10 @@ void EcalDQMonitorTask::formSchedule(std::vector<ecaldqm::Collections> const& _p
       case ecaldqm::kEBReducedRecHit:
       case ecaldqm::kEERecHit:
       case ecaldqm::kEEReducedRecHit:
+      case ecaldqm::kEBCpuRecHit:
+      case ecaldqm::kEECpuRecHit:
+      case ecaldqm::kEBGpuRecHit:
+      case ecaldqm::kEEGpuRecHit:
         collectionTokens_[*colItr] = edm::EDGetToken(consumes<EcalRecHitCollection>(tag));
         sch.first = &EcalDQMonitorTask::runOnCollection<EcalRecHitCollection>;
         break;

DQM/EcalMonitorTasks/python/CollectionTags_cfi.py

@@ -44,5 +44,9 @@
     EBBasicCluster = cms.untracked.InputTag("particleFlowClusterECAL"),
     EEBasicCluster = cms.untracked.InputTag("particleFlowClusterECAL"),
     EBSuperCluster = cms.untracked.InputTag("particleFlowSuperClusterECAL", "particleFlowSuperClusterECALBarrel"),
-    EESuperCluster = cms.untracked.InputTag("particleFlowSuperClusterECAL", "particleFlowSuperClusterECALEndcapWithPreshower")
+    EESuperCluster = cms.untracked.InputTag("particleFlowSuperClusterECAL", "particleFlowSuperClusterECALEndcapWithPreshower"),
+    EBCpuRecHit = cms.untracked.InputTag("ecalRecHit@cpu", "EcalRecHitsEB"),
+    EECpuRecHit = cms.untracked.InputTag("ecalRecHit@cpu", "EcalRecHitsEE"),
+    EBGpuRecHit = cms.untracked.InputTag("ecalRecHit@cuda", "EcalRecHitsEB"),
+    EEGpuRecHit = cms.untracked.InputTag("ecalRecHit@cuda", "EcalRecHitsEE")
 )

DQM/EcalMonitorTasks/python/EcalMonitorTask_cfi.py

@@ -14,6 +14,7 @@
 from DQM.EcalMonitorTasks.SelectiveReadoutTask_cfi import ecalSelectiveReadoutTask
 from DQM.EcalMonitorTasks.TimingTask_cfi import ecalTimingTask
 from DQM.EcalMonitorTasks.TrigPrimTask_cfi import ecalTrigPrimTask
+from DQM.EcalMonitorTasks.GpuTask_cfi import ecalGpuTask
 
 from DQMServices.Core.DQMEDAnalyzer import DQMEDAnalyzer
 ecalMonitorTask = DQMEDAnalyzer('EcalDQMonitorTask',
@@ -28,7 +29,8 @@
         "RawDataTask",
         "RecoSummaryTask",
         "TimingTask",
-        "TrigPrimTask"
+        "TrigPrimTask",
+        "GpuTask"
     ),
     # task parameters (included from indivitual cfis)
     workerParameters =  cms.untracked.PSet(
@@ -41,7 +43,8 @@
         RecoSummaryTask = ecalRecoSummaryTask,
         SelectiveReadoutTask = ecalSelectiveReadoutTask,
         TimingTask = ecalTimingTask,
-        TrigPrimTask = ecalTrigPrimTask
+        TrigPrimTask = ecalTrigPrimTask,
+        GpuTask = ecalGpuTask
     ),
     commonParameters = ecalCommonParams,
     collectionTags = ecalDQMCollectionTags,

DQM/EcalMonitorTasks/python/GpuTask_cfi.py

@@ -0,0 +1,130 @@
+import FWCore.ParameterSet.Config as cms
+
+nHitsBins = 100
+nHitsMax = 5000
+
+energyBins = 100
+energyMax = 2.0
+
+timeBins = 100
+timeMax = 12.5
+
+flagsBins = 40
+flagsMax = 1500
+
+deltaBins = 101
+delta = 0.2
+
+ecalGpuTask = cms.untracked.PSet(
+    params = cms.untracked.PSet(
+        runOnGpu = cms.untracked.bool(False)
+    ),
+    MEs = cms.untracked.PSet(
+        # CPU Plots
+        RecHitCpu = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT number of rec hits cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(nHitsMax),
+                nbins = cms.untracked.int32(nHitsBins),
+                low = cms.untracked.double(0),
+                title = cms.untracked.string('Rec Hits per Event')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('Number of total CPU Rec Hits per Event')
+        ),
+        RecHitCpuEnergy = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT rec hit energy cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(energyMax),
+                nbins = cms.untracked.int32(energyBins),
+                low = cms.untracked.double(0),
+                title = cms.untracked.string('Energy (Gev)')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('CPU Rec Hit Energy (GeV)')
+        ),
+        RecHitCpuTime = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT rec hit time cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(timeMax),
+                nbins = cms.untracked.int32(timeBins),
+                low = cms.untracked.double(-timeMax),
+                title = cms.untracked.string('Time (ns)')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('CPU Rec Hit Time')
+        ),
+        RecHitCpuFlags = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT rec hit flags cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(flagsMax),
+                nbins = cms.untracked.int32(flagsBins),
+                low = cms.untracked.double(0),
+                title = cms.untracked.string('Flags')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('CPU Rec Hit Flags')
+        ),
+        # GPU-CPU Difference Plots
+        RecHitGpuCpu = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT number of rec hits diff gpu-cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(delta*nHitsMax),
+                nbins = cms.untracked.int32(deltaBins),
+                low = cms.untracked.double(-delta*nHitsMax),
+                title = cms.untracked.string('GPU-CPU Rec Hits per Event')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('GPU-CPU difference of number of total Rec Hits per Event')
+        ),
+        RecHitGpuCpuEnergy = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT rec hit energy diff gpu-cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(delta*energyMax),
+                nbins = cms.untracked.int32(deltaBins),
+                low = cms.untracked.double(-delta*energyMax),
+                title = cms.untracked.string('GPU-CPU Energy (GeV)')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('GPU-CPU difference of Rec Hit Energy (GeV)')
+        ),
+        RecHitGpuCpuTime = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT rec hit time diff gpu-cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(delta*timeMax),
+                nbins = cms.untracked.int32(deltaBins),
+                low = cms.untracked.double(-delta*timeMax),
+                title = cms.untracked.string('GPU-CPU Time (ns)')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('GPU-CPU difference of Rec Hit Time')
+        ),
+        RecHitGpuCpuFlags = cms.untracked.PSet(
+            path = cms.untracked.string('%(subdet)s/%(prefix)sGpuTask/%(prefix)sGT rec hit flags diff gpu-cpu'),
+            kind = cms.untracked.string('TH1F'),
+            otype = cms.untracked.string('Ecal2P'),
+            xaxis = cms.untracked.PSet(
+                high = cms.untracked.double(delta*flagsMax),
+                nbins = cms.untracked.int32(deltaBins),
+                low = cms.untracked.double(-delta*flagsMax),
+                title = cms.untracked.string('GPU-CPU Flags')
+            ),
+            btype = cms.untracked.string('User'),
+            description = cms.untracked.string('GPU-CPU differnece of Rec Hit Flags')
+        )
+    )
+)

DQM/EcalMonitorTasks/src/GpuTask.cc

@@ -0,0 +1,126 @@
+#include "DQM/EcalMonitorTasks/interface/GpuTask.h"
+
+#include "DQM/EcalCommon/interface/EcalDQMCommonUtils.h"
+
+#include "FWCore/ParameterSet/interface/ParameterSet.h"
+#include "FWCore/MessageLogger/interface/MessageLogger.h"
+
+#include "DataFormats/EcalRawData/interface/EcalDCCHeaderBlock.h"
+
+namespace ecaldqm {
+  GpuTask::GpuTask() : DQWorkerTask(), runOnGpu_(false), EBCpuRecHits_(nullptr), EECpuRecHits_(nullptr) {}
+
+  void GpuTask::addDependencies(DependencySet& _dependencies) {
+    // Ensure we run on CpuRecHits before GpuRecHits
+    if (runOnGpu_) {
+      _dependencies.push_back(Dependency(kEBGpuRecHit, kEBCpuRecHit));
+      _dependencies.push_back(Dependency(kEEGpuRecHit, kEECpuRecHit));
+    }
+  }
+
+  void GpuTask::setParams(edm::ParameterSet const& _params) {
+    runOnGpu_ = _params.getUntrackedParameter<bool>("runOnGpu");
+    if (!runOnGpu_) {
+      MEs_.erase(std::string("RecHitGpuCpu"));
+      MEs_.erase(std::string("RecHitGpuCpuEnergy"));
+      MEs_.erase(std::string("RecHitGpuCpuTime"));
+      MEs_.erase(std::string("RecHitGpuCpuFlags"));
+    }
+  }
+
+  bool GpuTask::filterRunType(short const* _runType) {
+    for (unsigned iFED(0); iFED != ecaldqm::nDCC; iFED++) {
+      if (_runType[iFED] == EcalDCCHeaderBlock::COSMIC || _runType[iFED] == EcalDCCHeaderBlock::MTCC ||
+          _runType[iFED] == EcalDCCHeaderBlock::COSMICS_GLOBAL ||
+          _runType[iFED] == EcalDCCHeaderBlock::PHYSICS_GLOBAL || _runType[iFED] == EcalDCCHeaderBlock::COSMICS_LOCAL ||
+          _runType[iFED] == EcalDCCHeaderBlock::PHYSICS_LOCAL)
+        return true;
+    }
+
+    return false;
+  }
+
+  void GpuTask::beginEvent(edm::Event const&, edm::EventSetup const&, bool const&, bool&) {
+    EBCpuRecHits_ = nullptr;
+    EECpuRecHits_ = nullptr;
+  }
+
+  void GpuTask::runOnCpuRecHits(EcalRecHitCollection const& _hits, Collections _collection) {
+    MESet& meRecHitCpu(MEs_.at("RecHitCpu"));
+    MESet& meRecHitCpuEnergy(MEs_.at("RecHitCpuEnergy"));
+    MESet& meRecHitCpuTime(MEs_.at("RecHitCpuTime"));
+    MESet& meRecHitCpuFlags(MEs_.at("RecHitCpuFlags"));
+
+    int iSubdet(_collection == kEBCpuRecHit ? EcalBarrel : EcalEndcap);
+
+    // Save CpuRecHits for comparison with GpuRecHits
+    if (iSubdet == EcalBarrel)
+      EBCpuRecHits_ = &_hits;
+    else
+      EECpuRecHits_ = &_hits;
+
+    unsigned nCpuHits(_hits.size());
+    meRecHitCpu.fill(getEcalDQMSetupObjects(), iSubdet, nCpuHits);
+
+    for (EcalRecHitCollection::const_iterator hitItr(_hits.begin()); hitItr != _hits.end(); ++hitItr) {
+      float cpuEnergy(hitItr->energy());
+      if (cpuEnergy < 0.)
+        continue;
+
+      float cpuTime(hitItr->time());
+      uint32_t cpuFlags(hitItr->flagsBits());
+
+      meRecHitCpuEnergy.fill(getEcalDQMSetupObjects(), iSubdet, cpuEnergy);
+      meRecHitCpuTime.fill(getEcalDQMSetupObjects(), iSubdet, cpuTime);
+      meRecHitCpuFlags.fill(getEcalDQMSetupObjects(), iSubdet, cpuFlags);
+    }
+  }
+
+  // Should always run after GpuTask::runOnGpuRecHits()
+  void GpuTask::runOnGpuRecHits(EcalRecHitCollection const& _gpuHits, Collections _collection) {
+    MESet& meRecHitGpuCpu(MEs_.at("RecHitGpuCpu"));
+    MESet& meRecHitGpuCpuEnergy(MEs_.at("RecHitGpuCpuEnergy"));
+    MESet& meRecHitGpuCpuTime(MEs_.at("RecHitGpuCpuTime"));
+    MESet& meRecHitGpuCpuFlags(MEs_.at("RecHitGpuCpuFlags"));
+
+    int iSubdet(_collection == kEBGpuRecHit ? EcalBarrel : EcalEndcap);
+
+    // Get CpuRecHits saved from GpuTask::runOnCpuRecHits() for this event
+    // Note: _gpuHits is a collection and cpuHits is a pointer to a collection
+    EcalRecHitCollection const* cpuHits = (iSubdet == EcalBarrel) ? EBCpuRecHits_ : EECpuRecHits_;
+    if (!cpuHits) {
+      edm::LogWarning("EcalDQM") << "GpuTask: Did not find " << ((iSubdet == EcalBarrel) ? "EB" : "EE")
+                                 << "CpuRecHits Collection. Aborting runOnGpuRecHits\n";
+      return;
+    }
+
+    unsigned nGpuHits(_gpuHits.size());
+    unsigned nCpuHits(cpuHits->size());
+    meRecHitGpuCpu.fill(getEcalDQMSetupObjects(), iSubdet, nGpuHits - nCpuHits);
+
+    for (EcalRecHitCollection::const_iterator gpuItr(_gpuHits.begin()); gpuItr != _gpuHits.end(); ++gpuItr) {
+      // Find CpuRecHit with matching DetId
+      DetId gpuId(gpuItr->detid());
+      EcalRecHitCollection::const_iterator cpuItr(cpuHits->find(gpuId));
+      if (cpuItr == cpuHits->end()) {
+        edm::LogWarning("EcalDQM") << "GpuTask: Did not find DetId " << gpuId.rawId() << " in a CPU collection\n";
+        continue;
+      }
+
+      float gpuEnergy(gpuItr->energy());
+      float cpuEnergy(cpuItr->energy());
+
+      float gpuTime(gpuItr->time());
+      float cpuTime(cpuItr->time());
+
+      uint32_t gpuFlags(gpuItr->flagsBits());
+      uint32_t cpuFlags(cpuItr->flagsBits());
+
+      meRecHitGpuCpuEnergy.fill(getEcalDQMSetupObjects(), iSubdet, gpuEnergy - cpuEnergy);
+      meRecHitGpuCpuTime.fill(getEcalDQMSetupObjects(), iSubdet, gpuTime - cpuTime);
+      meRecHitGpuCpuFlags.fill(getEcalDQMSetupObjects(), iSubdet, gpuFlags - cpuFlags);
+    }
+  }
+
+  DEFINE_ECALDQM_WORKER(GpuTask);
+}  // namespace ecaldqm