From 4b0cb835452f94463bbb801ce364622c8f52c402 Mon Sep 17 00:00:00 2001 From: maria Date: Mon, 9 Nov 2020 13:29:43 +0100 Subject: [PATCH] Add HCAL analyser for GPU vs CPU comparison (#567) --- .../HCALGPUAnalyzer/plugins/BuildFile.xml | 11 + .../plugins/HCALGPUAnalyzer.cc | 312 ++++++++++++++++++ .../HCALGPUAnalyzer/python/ConfFile_fig.py | 27 ++ .../HCALGPUAnalyzer/python/hcal_gpu.py | 153 +++++++++ .../python/sourceFromRaw_cff.py | 51 +++ 5 files changed, 554 insertions(+) create mode 100644 ComparisonPlots/HCALGPUAnalyzer/plugins/BuildFile.xml create mode 100644 ComparisonPlots/HCALGPUAnalyzer/plugins/HCALGPUAnalyzer.cc create mode 100644 ComparisonPlots/HCALGPUAnalyzer/python/ConfFile_fig.py create mode 100644 ComparisonPlots/HCALGPUAnalyzer/python/hcal_gpu.py create mode 100644 ComparisonPlots/HCALGPUAnalyzer/python/sourceFromRaw_cff.py diff --git a/ComparisonPlots/HCALGPUAnalyzer/plugins/BuildFile.xml b/ComparisonPlots/HCALGPUAnalyzer/plugins/BuildFile.xml new file mode 100644 index 0000000000000..677a3d96e203e --- /dev/null +++ b/ComparisonPlots/HCALGPUAnalyzer/plugins/BuildFile.xml @@ -0,0 +1,11 @@ + + + + + + + + + + + diff --git a/ComparisonPlots/HCALGPUAnalyzer/plugins/HCALGPUAnalyzer.cc b/ComparisonPlots/HCALGPUAnalyzer/plugins/HCALGPUAnalyzer.cc new file mode 100644 index 0000000000000..89642d4a7a85d --- /dev/null +++ b/ComparisonPlots/HCALGPUAnalyzer/plugins/HCALGPUAnalyzer.cc @@ -0,0 +1,312 @@ +// -*- C++ -*- +// +// Package: ComparisonPlots/HCALGPUAnalyzer +// Class: HCALGPUAnalyzer +// +/**\class HCALGPUAnalyzer HCALGPUAnalyzer.cc ComparisonPlots/HCALGPUAnalyzer/plugins/HCALGPUAnalyzer.cc + + Description: [one line class summary] + + Implementation: + [Notes on implementation] +*/ +// +// Original Author: Mariarosaria D'Alfonso +// Created: Mon, 17 Dec 2018 16:22:58 GMT +// +// + +// system include files +#include +#include +#include +#include +using namespace std; + +// user include files +#include "FWCore/Framework/interface/Frameworkfwd.h" +#include "FWCore/Framework/interface/one/EDAnalyzer.h" + +#include "FWCore/Framework/interface/Event.h" +#include "FWCore/Framework/interface/MakerMacros.h" + +#include "FWCore/ParameterSet/interface/ParameterSet.h" +#include "FWCore/Utilities/interface/InputTag.h" +#include "FWCore/MessageLogger/interface/MessageLogger.h" +#include "FWCore/ServiceRegistry/interface/Service.h" +#include "CommonTools/UtilAlgos/interface/TFileService.h" + +#include "DataFormats/HcalRecHit/interface/HBHERecHit.h" +#include "DataFormats/HcalRecHit/interface/HcalRecHitCollections.h" +#include "DataFormats/HcalDetId/interface/HcalDetId.h" + +#include "SimDataFormats/CaloHit/interface/PCaloHit.h" +#include "SimDataFormats/CaloHit/interface/PCaloHitContainer.h" + +#include "SimCalorimetry/HcalSimAlgos/interface/HcalSimParameterMap.h" + +#include "TH2F.h" + +// +// class declaration +// + +class HCALGPUAnalyzer : public edm::one::EDAnalyzer { +public: + explicit HCALGPUAnalyzer(const edm::ParameterSet &); + ~HCALGPUAnalyzer(); + + static void fillDescriptions(edm::ConfigurationDescriptions &descriptions); + +private: + virtual void beginJob() override; + virtual void analyze(const edm::Event &, const edm::EventSetup &) override; + virtual void endJob() override; + + // ----------member data --------------------------- + // void ClearVariables(); + + // some variables for storing information + double Method0Energy, Method0EnergyGPU; + double RecHitEnergy, RecHitEnergyGPU; + double RecHitTime, RecHitTimeGPU; + double iEta, iEtaGPU; + double iPhi, iPhiGPU; + int depth, depthGPU; + + TH2F *hEnergy_2dMahi; + TH2F *hEnergy_2dM0; + TH2F *hTime_2dMahi; + + TH2F *Unmatched; + TH2F *Matched; + TH1F *hEnergy_cpu; + TH1F *hEnergy_gpu; + TH1F *hEnergy_cpugpu; + TH1F *hEnergy_cpugpu_rel; + TH1F *hEnergyM0_cpu; + TH1F *hEnergyM0_gpu; + TH1F *hTime_cpu; + TH1F *hTime_gpu; + + // create the output file + edm::Service FileService; + // create the token to retrieve hit information + edm::EDGetTokenT hRhToken; + edm::EDGetTokenT hRhTokenGPU; +}; + +// +// constants, enums and typedefs +// + +// +// static data member definitions +// + +// +// constructors and destructor +// +HCALGPUAnalyzer::HCALGPUAnalyzer(const edm::ParameterSet &iConfig) { + usesResource("TFileService"); + + hRhToken = consumes(iConfig.getUntrackedParameter("HBHERecHits", "hbheprereco")); + hRhTokenGPU = consumes( + iConfig.getUntrackedParameter("HBHERecHits", "hcalCPURecHitsProducer:recHitsLegacyHBHE")); + + // + + hEnergy_2dM0 = FileService->make("hEnergy_2dM0", "hEnergy_2dM0", 1000, 0., 100., 1000, 0., 100.); + hEnergy_2dM0->GetXaxis()->SetTitle("Cpu M0 Energy"); + hEnergy_2dM0->GetYaxis()->SetTitle("GPU M0 Energy"); + + hEnergy_2dMahi = FileService->make("hEnergy_2dMahi", "hEnergy_2dMahi", 1000, 0., 100., 1000, 0., 100.); + hEnergy_2dMahi->GetXaxis()->SetTitle("CPU Energy"); + hEnergy_2dMahi->GetYaxis()->SetTitle("GPU Energy"); + + hTime_2dMahi = FileService->make("hTime_2dMahi", "hTime_2dMahi", 250, -12.5, 12.5, 250, -12.5, 12.5); + hTime_2dMahi->GetXaxis()->SetTitle("Mahi Time CPU"); + hTime_2dMahi->GetYaxis()->SetTitle("Mahi Time GPU"); + + // + + hEnergyM0_cpu = FileService->make("hEnergyM0_cpu", "hEnergyM0_cpu", 100, 0., 100.); + hEnergyM0_cpu->GetXaxis()->SetTitle("CPU Energy"); + + hEnergy_cpu = FileService->make("hEnergy_cpu", "hEnergy_cpu", 50, 0., 50.); + hEnergy_cpu->GetXaxis()->SetTitle("CPU Energy"); + + hEnergy_gpu = FileService->make("hEnergy_gpu", "hEnergy_gpu", 50, 0., 50.); + hEnergy_gpu->GetXaxis()->SetTitle("GPU Energy"); + + // + + hEnergy_cpugpu = FileService->make("hEnergy_cpugpu", "hEnergy_cpugpu", 500, -2.5, 2.5); + hEnergy_cpugpu->GetXaxis()->SetTitle("GPU Energy - CPU Energy [GeV]"); + hEnergy_cpugpu->GetYaxis()->SetTitle("# RecHits"); + + hEnergy_cpugpu_rel = + FileService->make("hEnergy_cpugpu_rel", "hEnergy_cpugpu_rel ( E > 0.005 GeV)", 500, -2.5, 2.5); + hEnergy_cpugpu_rel->GetXaxis()->SetTitle("(GPU Energy - CPU Energy) / CPU energy"); + hEnergy_cpugpu_rel->GetYaxis()->SetTitle("# RecHits"); + + // + + hTime_cpu = FileService->make("hTime_cpu", "hTime_cpu", 50, -25., 25.); + hTime_cpu->GetXaxis()->SetTitle("CPU Time"); + + hTime_gpu = FileService->make("hTime_gpu", "hTime_gpu", 50, -25., 25.); + hTime_gpu->GetXaxis()->SetTitle("GPU Time"); + + Unmatched = FileService->make("Unmatched", "Unmatched (eta,phi)", 100, -50., 50., 85, 0., 85.); + Matched = FileService->make("Matched", "Matched (eta,phi)", 100, -50., 50., 85, 0., 85.); + + //now do what ever initialization is needed +} + +HCALGPUAnalyzer::~HCALGPUAnalyzer() { + // do anything here that needs to be done at desctruction time + // (e.g. close files, deallocate resources etc.) +} + +// +// member functions +// + +// ------------ method called for each event ------------ +void HCALGPUAnalyzer::analyze(const edm::Event &iEvent, const edm::EventSetup &iSetup) { + using namespace edm; + + // Read events + Handle hRecHits; + iEvent.getByToken(hRhToken, hRecHits); + + Handle hRecHitsGPU; + iEvent.getByToken(hRhTokenGPU, hRecHitsGPU); + + // Loop over all rechits in one event + for (int i = 0; i < (int)hRecHits->size(); i++) { + // get ID information for the reconstructed hit + HcalDetId detID_rh = (*hRecHits)[i].id().rawId(); + + // ID information can get us detector coordinates + depth = (*hRecHits)[i].id().depth(); + iEta = detID_rh.ieta(); + iPhi = detID_rh.iphi(); + + // get some variables + Method0Energy = (*hRecHits)[i].eraw(); + RecHitEnergy = (*hRecHits)[i].energy(); + RecHitTime = (*hRecHits)[i].time(); + + hEnergy_cpu->Fill(RecHitEnergy); + hTime_cpu->Fill(RecHitTime); + + /* + cout << "Run " << i << ": "; + cout << "Method0Energy: " << Method0Energy; + cout << "RecHitEnergy: " << RecHitEnergy; + cout << "depth: " << depth; + cout << "iEta: " << iEta; + cout << "iPhi: " << iPhi; + cout << "RecHitTime" << RecHitTime; + */ + } + + for (int i = 0; i < (int)hRecHitsGPU->size(); i++) { + // get ID information for the reconstructed hit + HcalDetId detID_rh = (*hRecHitsGPU)[i].id().rawId(); + + // ID information can get us detector coordinates + depthGPU = (*hRecHitsGPU)[i].id().depth(); + iEtaGPU = detID_rh.ieta(); + iPhiGPU = detID_rh.iphi(); + + // get some variables + Method0EnergyGPU = (*hRecHitsGPU)[i].eraw(); + RecHitEnergyGPU = (*hRecHitsGPU)[i].energy(); + RecHitTimeGPU = (*hRecHitsGPU)[i].time(); + + hEnergy_gpu->Fill(RecHitEnergyGPU); + hTime_gpu->Fill(RecHitTimeGPU); + + /* + cout << "Run " << i << ": "; + cout << "Method0Energy: " << Method0EnergyGPU; + cout << "RecHitEnergy: " << RecHitEnergyGPU; + cout << "depth: " << depthGPU; + cout << "iEta: " << iEtaGPU; + cout << "iPhi: " << iPhiGPU; + cout << "RecHitTime" << RecHitTimeGPU; + */ + } + + // Loop over all rechits in one event + for (int i = 0; i < (int)hRecHits->size(); i++) { + HcalDetId detID_rh = (*hRecHits)[i].id().rawId(); + + bool unmatched = true; + // cout << "--------------------------------------------------------" << endl; + + for (int j = 0; j < (int)hRecHitsGPU->size(); j++) { + HcalDetId detID_gpu = (*hRecHitsGPU)[j].id().rawId(); + + if ((detID_rh == detID_gpu)) { + /* + cout << "Mtime(cpu)" << (*hRecHits)[i].time() << endl; + cout << " Mtime(gpu)" << (*hRecHitsGPU)[j].time() << endl; + + cout << "M0E(cpu)" << (*hRecHits)[i].eraw() << endl; + cout << " M0E(gpu)" << (*hRecHitsGPU)[j].eraw() << endl; + */ + + auto relValue = ((*hRecHitsGPU)[j].energy() - (*hRecHits)[i].energy()) / (*hRecHits)[i].energy(); + + hEnergy_2dM0->Fill((*hRecHits)[i].eraw(), (*hRecHitsGPU)[j].eraw()); + hEnergy_2dMahi->Fill((*hRecHits)[i].energy(), (*hRecHitsGPU)[j].energy()); + hEnergy_cpugpu->Fill((*hRecHitsGPU)[j].energy() - (*hRecHits)[i].energy()); + if ((*hRecHits)[i].energy() > 0.005) + hEnergy_cpugpu_rel->Fill(relValue); + hTime_2dMahi->Fill((*hRecHits)[i].time(), (*hRecHitsGPU)[j].time()); + + /* + if((relValue < - 0.9) and ((*hRecHits)[i].energy()>0.005)) { + cout << "----------------------------------"<< endl; + cout << " detID = " << detID_rh.rawId() << endl; + cout << "ME(cpu)" << (*hRecHits)[i].energy() << endl; + cout << " ME(gpu)" << (*hRecHitsGPU)[j].energy() << endl; + } + */ + + Matched->Fill(detID_rh.ieta(), detID_rh.iphi()); + + unmatched = false; + } + } + + /// + + if (unmatched) { + Unmatched->Fill(detID_rh.ieta(), detID_rh.iphi()); + // cout << " recHit not matched =" << detID_rh << " E(raw)=" << (*hRecHits)[i].eraw() << " E=" << (*hRecHits)[i].energy() << endl; + } + } +} + +// ------------ method called once each job just before starting event loop ------------ +void HCALGPUAnalyzer::beginJob() {} + +// ------------ method called once each job just after ending the event loop ------------ +void HCALGPUAnalyzer::endJob() {} + +// ------------ method fills 'descriptions' with the allowed parameters for the module ------------ +void HCALGPUAnalyzer::fillDescriptions(edm::ConfigurationDescriptions &descriptions) { + //The following says we do not know what parameters are allowed so do no validation + // Please change this to state exactly what you do use, even if it is no parameters + edm::ParameterSetDescription desc; + desc.setUnknown(); + descriptions.addDefault(desc); +} + +//define this as a plug-in +DEFINE_FWK_MODULE(HCALGPUAnalyzer); diff --git a/ComparisonPlots/HCALGPUAnalyzer/python/ConfFile_fig.py b/ComparisonPlots/HCALGPUAnalyzer/python/ConfFile_fig.py new file mode 100644 index 0000000000000..0a34c722546a6 --- /dev/null +++ b/ComparisonPlots/HCALGPUAnalyzer/python/ConfFile_fig.py @@ -0,0 +1,27 @@ +import FWCore.ParameterSet.Config as cms + +process = cms.Process("Demo") + +process.load("FWCore.MessageService.MessageLogger_cfi") +process.options = cms.untracked.PSet (wantSummary = cms.untracked.bool(False)) + +process.load('Configuration.StandardSequences.GeometryRecoDB_cff') +process.load("Configuration.StandardSequences.FrontierConditions_GlobalTag_cff") +#process.GlobalTag.globaltag = '101X_dataRun2_Prompt_v11' +process.GlobalTag.globaltag = '101X_dataRun2_HLT_frozen_v10' + +process.maxEvents = cms.untracked.PSet( input = cms.untracked.int32(-1) ) +process.MessageLogger.cerr.FwkReport.reportEvery = 500 + +process.source = cms.Source("PoolSource", + fileNames = cms.untracked.vstring( + + 'file:$CMSSW_BASE/src/ComparisonPlots/HCALGPUAnalyzer/python/test_bothCPUGPU_NEW.root' + ) +) + +process.comparisonPlots = cms.EDAnalyzer('HCALGPUAnalyzer') + +process.TFileService = cms.Service('TFileService', fileName = cms.string('test_both.root') ) + +process.p = cms.Path(process.comparisonPlots) diff --git a/ComparisonPlots/HCALGPUAnalyzer/python/hcal_gpu.py b/ComparisonPlots/HCALGPUAnalyzer/python/hcal_gpu.py new file mode 100644 index 0000000000000..2c22727740894 --- /dev/null +++ b/ComparisonPlots/HCALGPUAnalyzer/python/hcal_gpu.py @@ -0,0 +1,153 @@ +import FWCore.ParameterSet.Config as cms + +from Configuration.StandardSequences.Eras import eras +#from Configuration.ProcessModifiers.gpu_cff import gpu + +process = cms.Process('RECOgpu', eras.Run2_2018) + +# import of standard configurations +process.load('Configuration.StandardSequences.Services_cff') +process.load('FWCore.MessageService.MessageLogger_cfi') +process.load('HeterogeneousCore.CUDAServices.CUDAService_cfi') + +process.load('Configuration.StandardSequences.FrontierConditions_GlobalTag_cff') +process.GlobalTag.globaltag = '101X_dataRun2_HLT_frozen_v10' + +process.maxEvents = cms.untracked.PSet( + input = cms.untracked.int32(1000) +) + +### load data using the DAQ source (Run2 data) +import sys, os, inspect +sys.path.append(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))) +process.load('sourceFromRaw_cff') + +#----------------------------------------- +# CMSSW/Hcal non-DQM Related Module import +#----------------------------------------- + +process.load('Configuration.StandardSequences.GeometryRecoDB_cff') +process.load("RecoLocalCalo.Configuration.hcalLocalReco_cff") +process.load("EventFilter.HcalRawToDigi.HcalRawToDigi_cfi") +process.load("RecoLuminosity.LumiProducer.bunchSpacingProducer_cfi") + +process.hcalDigis.InputLabel = cms.InputTag("rawDataCollector") + +#----------------------------------------- +# CMSSW/Hcal GPU related files +#----------------------------------------- + +process.load("RecoLocalCalo.HcalRecProducers.hbheRecHitProducerGPUTask_cff") +process.load("RecoLocalCalo.HcalRecProducers.hcalCPURecHitsProducer_cfi") +process.hcalCPURecHitsProducer.recHitsM0LabelIn = cms.InputTag("hbheRecHitProducerGPU","") +process.hcalCPURecHitsProducer.recHitsM0LabelOut = cms.string("") + +#----------------------------------------- +# Temporary customization (things not implemented on the GPU) +#----------------------------------------- + +## the one below is taken directly from the DB, regard M0 +#process.hbheprereco.algorithm.__setattr__('correctForPhaseContainment', cms.bool(False)) + +## do always 8 pulse +process.hbheprereco.algorithm.__setattr__('chiSqSwitch', cms.double(-1)) + +## to match hard coded setting (will be fixed on CPU) +process.hbheprereco.algorithm.__setattr__('nMaxItersMin',cms.int32(50)) + +#----------------------------------------- +# Final Custmization for Run3 +#----------------------------------------- + +# we will not run arrival Time at HLT +process.hbheprereco.algorithm.__setattr__('calculateArrivalTime',cms.bool(False)) + +## we do not need this +process.hbheprereco.algorithm.__setattr__('applyLegacyHBMCorrection', cms.bool(False)) + +# we only runMahi at HLT +process.hbheprereco.algorithm.__setattr__('useM3',cms.bool(False)) + +# we will not have the HPD noise flags in Run3, as will be all siPM +process.hbheprereco.setLegacyFlagsQIE8 = cms.bool(False) +process.hbheprereco.setNegativeFlagsQIE8 = cms.bool(False) +process.hbheprereco.setNoiseFlagsQIE8 = cms.bool(False) +process.hbheprereco.setPulseShapeFlagsQIE8 = cms.bool(False) + +# for testing M0 only +##process.hbheprereco.algorithm.__setattr__('useMahi',cms.bool(False)) + +#----------------------------------------- +# OUTPUT +#----------------------------------------- + +process.out = cms.OutputModule( + "PoolOutputModule", +### fileName = cms.untracked.string("test_bothCPUGPU_DEFAULT.root") + fileName = cms.untracked.string("test_bothCPUGPU_NEW.root") +) + +#process.out = cms.OutputModule("AsciiOutputModule", +# outputCommands = cms.untracked.vstring( +# 'keep *_*_*_*', +# ), +# verbosity = cms.untracked.uint32(0) +#) + +process.hltGetRaw = cms.EDAnalyzer( "HLTGetRaw", + RawDataCollection = cms.InputTag( "rawDataCollector" ) +) + +process.input = cms.Path( process.hltGetRaw ) + + +process.finalize = cms.EndPath(process.out) + +process.bunchSpacing = cms.Path( + process.bunchSpacingProducer +) + +#----------------------------------------- +# gpu test +#----------------------------------------- + +process.digiPathCPU = cms.Path( + process.hcalDigis +) + +process.recoPathCPU = cms.Path( + process.hbheprereco +) + +#--------------- + +## hcalCPUDigisProducer <-- this convert the GPU digi on cpu (for dqm) +process.recoPathGPU = cms.Path( + process.hbheRecHitProducerGPUSequence + * process.hcalCPURecHitsProducer +) + +#--------------- + +process.schedule = cms.Schedule( + process.input, + process.digiPathCPU, + process.recoPathCPU, + process.recoPathGPU, + process.finalize +) + +process.options = cms.untracked.PSet( + numberOfThreads = cms.untracked.uint32(8), + numberOfStreams = cms.untracked.uint32(8), + SkipEvent = cms.untracked.vstring('ProductNotFound'), + wantSummary = cms.untracked.bool(True) +) + +# report CUDAService messages +process.MessageLogger.cerr.FwkReport.reportEvery = 100 +process.MessageLogger.categories.append("CUDAService") + +#dumpFile = open("dump.py", "w") +#dumpFile.write(process.dumpPython()) +#dumpFile.close() diff --git a/ComparisonPlots/HCALGPUAnalyzer/python/sourceFromRaw_cff.py b/ComparisonPlots/HCALGPUAnalyzer/python/sourceFromRaw_cff.py new file mode 100644 index 0000000000000..91e4806c885fa --- /dev/null +++ b/ComparisonPlots/HCALGPUAnalyzer/python/sourceFromRaw_cff.py @@ -0,0 +1,51 @@ +import FWCore.ParameterSet.Config as cms + +# input +FastMonitoringService = cms.Service( "FastMonitoringService", + filePerFwkStream = cms.untracked.bool( False ), + fastMonIntervals = cms.untracked.uint32( 2 ), + sleepTime = cms.untracked.int32( 1 ) +) + +EvFDaqDirector = cms.Service( "EvFDaqDirector", + runNumber = cms.untracked.uint32( 321177 ), + + baseDir = cms.untracked.string( "tmp" ), + buBaseDir = cms.untracked.string( "tmp" ), + + useFileBroker = cms.untracked.bool( False ), + fileBrokerKeepAlive = cms.untracked.bool( True ), + fileBrokerPort = cms.untracked.string( "8080" ), + fileBrokerUseLocalLock = cms.untracked.bool( True ), + fuLockPollInterval = cms.untracked.uint32( 2000 ), + + requireTransfersPSet = cms.untracked.bool( False ), + selectedTransferMode = cms.untracked.string( "" ), + mergingPset = cms.untracked.string( "" ), + + outputAdler32Recheck = cms.untracked.bool( False ), +) + +source = cms.Source( "FedRawDataInputSource", + runNumber = cms.untracked.uint32( 321177 ), + getLSFromFilename = cms.untracked.bool(True), + testModeNoBuilderUnit = cms.untracked.bool(False), + verifyAdler32 = cms.untracked.bool( True ), + verifyChecksum = cms.untracked.bool( True ), + useL1EventID = cms.untracked.bool( False ), # True + alwaysStartFromfirstLS = cms.untracked.uint32( 0 ), + + eventChunkBlock = cms.untracked.uint32( 240 ), # 32 + eventChunkSize = cms.untracked.uint32( 240), # 32 + maxBufferedFiles = cms.untracked.uint32( 8 ), # 2 + numBuffers = cms.untracked.uint32( 8 ), # 2 + + fileListMode = cms.untracked.bool( True ), # False + fileNames = cms.untracked.vstring( + '/data/store/dalfonso/run321177_ls0142_index000000.raw', + '/data/store/dalfonso/run321177_ls0142_index000001.raw', + '/data/store/dalfonso/run321177_ls0142_index000002.raw', + '/data/store/dalfonso/run321177_ls0142_index000003.raw', + '/data/store/dalfonso/run321177_ls0142_index000004.raw', + ), +)