Merge pull request #17208 from schneiml/phase1pixeldqm-post-christmas…

…-version

Phase1 Pixel DQM PixelMaps, new TrackEfficiencies, removed Harvesting, new Plots and various Bugfixes

DQM/Integration/python/clients/pixel_up_dqm_sourceclient-live_cfg.py

@@ -313,7 +313,7 @@
 process.SiPixelPhase1RawDataNErrors.enabled = True
 process.SiPixelPhase1RawDataNErrors.specs = VPSet(
   Specification().groupBy("PXForward/PXBlade").groupBy("PXForward", "EXTEND_X").save(),
-  Specification().groupBy("FEDChannel").groupBy("", "EXTEND_X").save()
+  Specification().groupBy("LinkInFed").groupBy("", "EXTEND_X").save()
 )
 
 process.SiPixelPhase1RawDataTypeNErrors.enabled = True

DQM/Integration/python/config/fileinputsource_cfi.py

@@ -69,7 +69,7 @@
   readFiles = cms.untracked.vstring()
   secFiles = cms.untracked.vstring()
   # this outputs all results, which can be a lot...
-  read, sec = filesFromDASQuery("file dataset=%s" % dataset, option=" --limit 10000 ")
+  read, sec = filesFromDASQuery("file run=%d dataset=%s" % (options.runNumber, dataset), option=" --limit 10000 ")
   readFiles.extend(read)
   secFiles.extend(sec)
 

DQM/SiPixelPhase1Clusters/interface/SiPixelPhase1Clusters.h

@@ -15,13 +15,18 @@ class SiPixelPhase1Clusters : public SiPixelPhase1Base {
   enum {
     CHARGE,
     SIZE,
+    SIZEX,
+    SIZEY,
     NCLUSTERS,
+    NCLUSTERSINCLUSIVE,
     EVENTRATE,
     POSITION_B,
     POSITION_F,
     POSITION_XZ,
     POSITION_YZ,
-    SIZE_VS_ETA
+    SIZE_VS_ETA,
+    READOUT_CHARGE,
+    READOUT_NCLUSTERS
   };
 
   public:

DQM/SiPixelPhase1Clusters/python/SiPixelPhase1Clusters_cfi.py

@@ -8,9 +8,11 @@
   xlabel = "Charge (electrons)",
 
   specs = VPSet(
-    StandardSpecification2DProfile,
+    #StandardSpecification2DProfile,
+    StandardSpecificationPixelmapProfile,
     StandardSpecificationTrend,
-    StandardSpecifications1D
+    StandardSpecifications1D,
+    StandardSpecificationTrend2D
   )
 )
 
@@ -20,37 +22,85 @@
   range_min = 0, range_max = 30, range_nbins = 30,
   xlabel = "size[pixels]",
   specs = VPSet(
-    StandardSpecification2DProfile,
+    #StandardSpecification2DProfile,
+    StandardSpecificationPixelmapProfile,
     StandardSpecificationTrend,
-    StandardSpecifications1D
+    StandardSpecifications1D,
+    StandardSpecificationTrend2D
   )
 )
 
+SiPixelPhase1ClustersSizeX = DefaultHistoDigiCluster.clone(
+  name = "sizeX",
+  title = "Cluster Size in X",
+  range_min = 0, range_max = 30, range_nbins = 30,
+  xlabel = "size[pixels]",
+  specs = VPSet(
+    #StandardSpecification2DProfile,
+    #StandardSpecificationPixelmapProfile,
+    #StandardSpecificationTrend,
+    StandardSpecifications1D,
+    #StandardSpecificationTrend2D
+  )
+)
+
+SiPixelPhase1ClustersSizeY = DefaultHistoDigiCluster.clone(
+  name = "sizeY",
+  title = "Cluster Size in Y",
+  range_min = 0, range_max = 30, range_nbins = 30,
+  xlabel = "size[pixels]",
+  specs = VPSet(
+    #StandardSpecification2DProfile,
+    #StandardSpecificationPixelmapProfile,
+    #StandardSpecificationTrend,
+    StandardSpecifications1D,
+    #StandardSpecificationTrend2D
+  )
+)
+
+
 SiPixelPhase1ClustersNClusters = DefaultHistoDigiCluster.clone(
   name = "clusters",
   title = "Clusters",
   range_min = 0, range_max = 10, range_nbins = 10,
   xlabel = "clusters",
   dimensions = 0,
   specs = VPSet(
+    StandardSpecificationOccupancy,
     StandardSpecification2DProfile_Num,
     StandardSpecificationTrend_Num,
-    StandardSpecifications1D_Num
+    StandardSpecifications1D_Num,
   )
 )
 
+
+SiPixelPhase1ClustersNClustersInclusive = DefaultHistoDigiCluster.clone(
+  name = "clusters",
+  title = "Clusters",
+  range_min = 0, range_max = 2000, range_nbins = 200,
+  xlabel = "clusters",
+  dimensions = 0,
+  specs = VPSet(
+    StandardSpecificationInclusive_Num
+  )
+)
+
+
 SiPixelPhase1ClustersEventrate = DefaultHistoDigiCluster.clone(
-  name = "bigfpixclustereventrate",
-  title = "Number of Events with > 180 FPIX clusters",
-  xlabel = "Lumisection",
+  name = "clustereventrate",
+  title = "Number of Events with clusters",
   ylabel = "#Events",
   dimensions = 0,
   specs = VPSet(
     Specification().groupBy("Lumisection")
+                   .groupBy("", "EXTEND_X").save(),
+    Specification().groupBy("BX")
                    .groupBy("", "EXTEND_X").save()
-  )
+    )
+
 )
 
+
 SiPixelPhase1ClustersPositionB = DefaultHistoDigiCluster.clone(
   name = "clusterposition_zphi",
   title = "Cluster Positions",
@@ -115,16 +165,58 @@
   )
 )
 
+SiPixelPhase1ClustersReadoutCharge = DefaultHistoReadout.clone(
+  name = "charge",
+  title = "Cluster Charge",
+  range_min = 0, range_max = 200e3, range_nbins = 200,
+  xlabel = "Charge (electrons)",
+  specs = VPSet(
+    Specification(PerReadout).groupBy("PXBarrel/Shell/Sector").save(),
+    Specification(PerReadout).groupBy("PXForward/HalfCylinder").save(),
+
+    Specification(PerReadout).groupBy("PXBarrel/Shell/Sector/OnlineBlock")
+                             .groupBy("PXBarrel/Shell/Sector", "EXTEND_Y").save(),
+    Specification(PerReadout).groupBy("PXForward/HalfCylinder/OnlineBlock")
+                             .groupBy("PXForward/HalfCylinder", "EXTEND_Y").save(),
+  )
+)
+
+SiPixelPhase1ClustersReadoutNClusters = DefaultHistoReadout.clone(
+  name = "clusters",
+  title = "Clusters",
+  range_min = 0, range_max = 10, range_nbins = 10,
+  xlabel = "clusters",
+  dimensions = 0,
+  specs = VPSet(
+    Specification(PerReadout).groupBy("PXBarrel/Shell/Sector/DetId/Event").reduce("COUNT")
+                             .groupBy("PXBarrel/Shell/Sector").save(),
+    Specification(PerReadout).groupBy("PXForward/HalfCylinder/DetId/Event").reduce("COUNT")
+                             .groupBy("PXForward/HalfCylinder").save(),
+
+    Specification(PerReadout).groupBy("PXBarrel/Shell/Sector/DetId/Event").reduce("COUNT")
+                             .groupBy("PXBarrel/Shell/Sector/Lumisection").reduce("MEAN")
+                             .groupBy("PXBarrel/Shell/Sector", "EXTEND_X").save(),
+    Specification(PerReadout).groupBy("PXForward/HalfCylinder/DetId/Event").reduce("COUNT")
+                             .groupBy("PXForward/HalfCylinder/Lumisection").reduce("MEAN")
+                             .groupBy("PXForward/HalfCylinder", "EXTEND_X").save(),
+  )
+)
+
 SiPixelPhase1ClustersConf = cms.VPSet(
   SiPixelPhase1ClustersCharge,
   SiPixelPhase1ClustersSize,
+  SiPixelPhase1ClustersSizeX,
+  SiPixelPhase1ClustersSizeY,
   SiPixelPhase1ClustersNClusters,
+  SiPixelPhase1ClustersNClustersInclusive,
   SiPixelPhase1ClustersEventrate,
   SiPixelPhase1ClustersPositionB,
   SiPixelPhase1ClustersPositionF,
   SiPixelPhase1ClustersPositionXZ,
   SiPixelPhase1ClustersPositionYZ,
-  SiPixelPhase1ClustersSizeVsEta
+  SiPixelPhase1ClustersSizeVsEta,
+  SiPixelPhase1ClustersReadoutCharge,
+  SiPixelPhase1ClustersReadoutNClusters
 )
 
 

DQM/SiPixelPhase1Clusters/src/SiPixelPhase1Clusters.cc

@@ -29,13 +29,12 @@ void SiPixelPhase1Clusters::analyze(const edm::Event& iEvent, const edm::EventSe
   iEvent.getByToken(srcToken_, input);
   if (!input.isValid()) return;
 
+  bool hasClusters=false;  
+
   edm::ESHandle<TrackerGeometry> tracker;
   iSetup.get<TrackerDigiGeometryRecord>().get(tracker);
   assert(tracker.isValid());
 
-  auto forward = geometryInterface.intern("PXForward");
-  auto nforward = 0;
-
   edmNew::DetSetVector<SiPixelCluster>::const_iterator it;
   for (it = input->begin(); it != input->end(); ++it) {
     auto id = DetId(it->detId());
@@ -44,10 +43,18 @@ void SiPixelPhase1Clusters::analyze(const edm::Event& iEvent, const edm::EventSe
     const PixelTopology& topol = theGeomDet->specificTopology();
 
     for(SiPixelCluster const& cluster : *it) {
-      histo[CHARGE].fill(double(cluster.charge()), id, &iEvent);
-      histo[SIZE  ].fill(double(cluster.size()  ), id, &iEvent);
-      if (cluster.size() > 1)
-        histo[NCLUSTERS].fill(id, &iEvent);
+      int row = cluster.x()-0.5, col = cluster.y()-0.5;
+      histo[READOUT_CHARGE].fill(double(cluster.charge()), id, &iEvent, col, row);
+      histo[CHARGE].fill(double(cluster.charge()), id, &iEvent, col, row);
+      histo[SIZE  ].fill(double(cluster.size()  ), id, &iEvent, col, row);
+      histo[SIZEX  ].fill(double(cluster.sizeX()  ), id, &iEvent, col, row);
+      histo[SIZEY  ].fill(double(cluster.sizeY()  ), id, &iEvent, col, row);
+      histo[NCLUSTERS].fill(id, &iEvent, col, row);
+      histo[NCLUSTERSINCLUSIVE].fill(id, &iEvent);
+      hasClusters=true;
+      if (cluster.size()>1){
+        histo[READOUT_NCLUSTERS].fill(id, &iEvent);
+      }
 
       LocalPoint clustlp = topol.localPosition(MeasurementPoint(cluster.x(), cluster.y()));
       GlobalPoint clustgp = theGeomDet->surface().toGlobal(clustlp);
@@ -57,14 +64,16 @@ void SiPixelPhase1Clusters::analyze(const edm::Event& iEvent, const edm::EventSe
       histo[POSITION_YZ].fill(clustgp.y(),   clustgp.z(),     id, &iEvent);
       histo[SIZE_VS_ETA].fill(clustgp.eta(), cluster.sizeY(), id, &iEvent);
 
-      if (geometryInterface.extract(forward, id) != GeometryInterface::UNDEFINED)
-        nforward++;
     }
   }
 
-  if (nforward > 180) 
-    histo[EVENTRATE].fill(DetId(0), &iEvent);
+
+  if (hasClusters) histo[EVENTRATE].fill(DetId(0), &iEvent);
+
   histo[NCLUSTERS].executePerEventHarvesting(&iEvent);
+  histo[READOUT_NCLUSTERS].executePerEventHarvesting(&iEvent);
+  histo[NCLUSTERSINCLUSIVE].executePerEventHarvesting(&iEvent);
+
 }
 
 DEFINE_FWK_MODULE(SiPixelPhase1Clusters);

DQM/SiPixelPhase1Common/README.md

@@ -81,8 +81,6 @@ The framework provides a few simple commands to describe histograms using this m
 - The `reduce` command reduces the right-hand side histograms into histograms of a lower dimensionality. Typically, e.g. for the mean, this is a 0D-histogram, which means a single bin with the respective value. The quantity to extract is specified by the parameter.
 - The `save`command tells the framework that we are happy with the histograms that are now in the table and that they should be saved to the output file. Proper names are inferred from the column names and steps specified before.
 
-The framework also allows a `custom` command that passes the current table state to custom code, to compute things that the framework does not allow itself. However, this should be used carefully, since such code has to be heavily dependent on internal data structures.
-
 Since the framework can see where quantities go during the computation of derived histograms, it can also automatically keep track of the axis labels and ranges and set them to useful, consistent values automatically. For technical reasons the framework is not able to execute every conceivable specification, but it is able to handle all relevant cases in an efficient way. Since the specification is very abstract, it gives the framework a lot of freedom to implement it in the most efficient way possible.
 
 
@@ -224,7 +222,7 @@ This is an unspectacular wrapper around histogram-like things. It is used as the
 
 This is the base class that all plugins should derive from. It instantiates `HistogramManager`s from config, sets up a `GeometryInterface`, and calls the `book` method of the `HistogramManager`s. If you need anything special, you can also do this yourself ad ignore the base class.
 
-The same file declares the `SiPixelPhase1Harvester`, which is very similar to the base class but is an `DQMEDHarvester` and calls the harvesting methods. Usually this does not need to be modified, it is enough instantiate a copy and set the configuration to be the same as for the analyzer. If you want too use a `custom` step, derive from this class and call the `setCustomHandler` on the HistogramManager before the actual harvesting starts.
+The same file declares the `SiPixelPhase1Harvester`, which is very similar to the base class but is an `DQMEDHarvester` and calls the harvesting methods. Usually this does not need to be modified, it is enough instantiate a copy and set the configuration to be the same as for the analyzer. 
 
 ### SummationSpecification
 
@@ -295,7 +293,3 @@ The fill function asks the `GeometryInterface` for the column values for the mod
 For the harvesting, first the internal table structure has to be repopulated (remember, we are in a completely new process now). To do this, `loadFromDQMStore` mirrors the booking process (massively stripped down and not actually sharing code) and `get`s the MEs from the `DQMStore`. So, in sum there are 3 places where step1 commands are interpreted: in booking, in filling, and in loading. Make sure to keep them in sync when anything changes. 
 
 Afterwards, a classical interpreter loop runs over the step2 commands and calls methods to actually execute them, by updating the table structure. This is mostly straight-forward code. Booking happens as the execution progresses, and all metadata is tracked in the `TH1` fields (labels, range).
-
-In the `custom` command, a custom handler function (that has to be set beforehand) is called. This is again a lambda (using inheritance would be possible as well, but it tends to be a mess in C++), which gets passed the internal table state. It is explicitly allowed to just save a copy of that state for later use, which may be necessary to compute quantities that depend on multiple other quantities (e.g. efficiencies). Later, a `custom` step on a different HistogramManager (possibly one that did not record any data in step1) can take the saved tables, create derived quantities and put them into the histograms in its own table, which the HistogramManager already booked with consistent names (also doing further summation, if specified).
-
-Alternativey, the `custom` handler gets passed the `IBooker` and `IGetter` objects from the `DQMStore`, so things which cannot be booked using the framework can be handled manually. But then you also need to handle folder- and object names your self; best is to use the path names of the MEs you find in the table and derive new ME names from the existing ME names.

DQM/SiPixelPhase1Common/interface/GeometryInterface.h

@@ -131,6 +131,8 @@ class GeometryInterface {
   // edm::ParameterSet& iConfig);
   void loadFromTopology(edm::EventSetup const& iSetup,
                         const edm::ParameterSet& iConfig);
+  void loadFromSiPixelCoordinates(edm::EventSetup const& iSetup,
+                        const edm::ParameterSet& iConfig);
   void loadTimebased(edm::EventSetup const& iSetup,
                      const edm::ParameterSet& iConfig);
   void loadModuleLevel(edm::EventSetup const& iSetup,

DQM/SiPixelPhase1Common/interface/HistogramManager.h

@@ -52,17 +52,10 @@ class HistogramManager {
   void executeHarvesting(DQMStore::IBooker& iBooker, DQMStore::IGetter& iGetter);
 
   typedef std::map<GeometryInterface::Values, AbstractHistogram> Table;
-  // Set a handler to be called when a custom() step is hit. This can do 
-  // arbitrary things to the histogram Table, including copying it for later 
-  // use. Using such saved tables form other HistogramManagers, e.g. 
-  // efficiencies can be computed here.
-  template<typename FUNC>
-  void setCustomHandler(FUNC handler) {customHandler = handler; };
 
 private:
   const edm::ParameterSet& iConfig;
   GeometryInterface& geometryInterface;
-  std::function<void(SummationStep const& step, Table& t, DQMStore::IBooker& iBooker, DQMStore::IGetter& iGetter)> customHandler;
 
   std::vector<SummationSpecification> specs;
   std::vector<Table> tables;