Merge pull request #32683 from b-fontana/SplitModules

Heterogeneous HGCAL RecHit Calibration

CUDADataFormats/HGCal/BuildFile.xml

@@ -0,0 +1,8 @@
+<use name="FWCore/Utilities"/>
+<use name="Geometry/HGCalGeometry"/>
+<use name="HeterogeneousCore/CUDACore"/>
+<use name="cuda"/>
+
+<export>
+    <lib name="1"/>
+</export>

CUDADataFormats/HGCal/interface/ConstHGCRecHitSoA.h

@@ -0,0 +1,16 @@
+#ifndef CUDADataFormats_HGCal_ConstHGCRecHitSoA_h
+#define CUDADataFormats_HGCal_ConstHGCRecHitSoA_h
+
+#include <cstdint>
+
+class ConstHGCRecHitSoA {  //const version of the HGCRecHit class (data in the event should be immutable)
+public:
+  float const *energy_;       //calibrated energy of the rechit
+  float const *time_;         //time jitter of the UncalibRecHit
+  float const *timeError_;    //time resolution
+  uint32_t const *id_;        //rechit detId
+  uint32_t const *flagBits_;  //rechit flags describing its status (DataFormats/HGCRecHit/interface/HGCRecHit.h)
+  uint8_t const *son_;        //signal over noise
+};
+
+#endif  //CUDADataFormats_HGCal_ConstHGCRecHitSoA_h

CUDADataFormats/HGCal/interface/HGCConditions.h

@@ -0,0 +1,203 @@
+#ifndef CUDADataFormats_HGCal_HGCConditions_h
+#define CUDADataFormats_HGCal_HGCConditions_h
+
+#include <cstdint>
+#include <cstddef>
+#include <array>
+#include <vector>
+
+class HeterogeneousHGCSiliconDetId {
+public:
+  constexpr HeterogeneousHGCSiliconDetId(uint32_t id) : id_(id) {}
+  constexpr std::int32_t type() { return (id_ >> kHGCalTypeOffset) & kHGCalTypeMask; }
+  constexpr std::int32_t zside() { return (((id_ >> kHGCalZsideOffset) & kHGCalZsideMask) ? -1 : 1); }
+  constexpr std::int32_t layer() { return (id_ >> kHGCalLayerOffset) & kHGCalLayerMask; }
+  constexpr std::int32_t waferUAbs() { return (id_ >> kHGCalWaferUOffset) & kHGCalWaferUMask; }
+  constexpr std::int32_t waferVAbs() { return (id_ >> kHGCalWaferVOffset) & kHGCalWaferVMask; }
+  constexpr std::int32_t waferU() {
+    return (((id_ >> kHGCalWaferUSignOffset) & kHGCalWaferUSignMask) ? -waferUAbs() : waferUAbs());
+  }
+  constexpr std::int32_t waferV() {
+    return (((id_ >> kHGCalWaferVSignOffset) & kHGCalWaferVSignMask) ? -waferVAbs() : waferVAbs());
+  }
+  constexpr std::int32_t waferX() { return (-2 * waferU() + waferV()); }
+  constexpr std::int32_t waferY() { return (2 * waferV()); }
+  constexpr std::int32_t cellU() { return (id_ >> kHGCalCellUOffset) & kHGCalCellUMask; }
+  constexpr std::int32_t cellV() { return (id_ >> kHGCalCellVOffset) & kHGCalCellVMask; }
+  constexpr std::int32_t nCellsSide() { return (type() == HGCalFine) ? HGCalFineN : HGCalCoarseN; }
+  constexpr std::int32_t cellX() {
+    const std::int32_t N = nCellsSide();
+    return (3 * (cellV() - N) + 2);
+  }
+  constexpr std::int32_t cellY() {
+    const std::int32_t N = nCellsSide();
+    return (2 * cellU() - (N + cellV()));
+  }
+
+private:
+  std::uint32_t id_;
+  enum waferType { HGCalFine = 0, HGCalCoarseThin = 1, HGCalCoarseThick = 2 };
+  static constexpr std::int32_t HGCalFineN = 12;
+  static constexpr std::int32_t HGCalCoarseN = 8;
+  static constexpr std::int32_t kHGCalCellUOffset = 0;
+  static constexpr std::int32_t kHGCalCellUMask = 0x1F;
+  static constexpr std::int32_t kHGCalCellVOffset = 5;
+  static constexpr std::int32_t kHGCalCellVMask = 0x1F;
+  static constexpr std::int32_t kHGCalWaferUOffset = 10;
+  static constexpr std::int32_t kHGCalWaferUMask = 0xF;
+  static constexpr std::int32_t kHGCalWaferUSignOffset = 14;
+  static constexpr std::int32_t kHGCalWaferUSignMask = 0x1;
+  static constexpr std::int32_t kHGCalWaferVOffset = 15;
+  static constexpr std::int32_t kHGCalWaferVMask = 0xF;
+  static constexpr std::int32_t kHGCalWaferVSignOffset = 19;
+  static constexpr std::int32_t kHGCalWaferVSignMask = 0x1;
+  static constexpr std::int32_t kHGCalLayerOffset = 20;
+  static constexpr std::int32_t kHGCalLayerMask = 0x1F;
+  static constexpr std::int32_t kHGCalZsideOffset = 25;
+  static constexpr std::int32_t kHGCalZsideMask = 0x1;
+  static constexpr std::int32_t kHGCalTypeOffset = 26;
+  static constexpr std::int32_t kHGCalTypeMask = 0x3;
+};
+
+class HeterogeneousHGCScintillatorDetId {
+public:
+  constexpr HeterogeneousHGCScintillatorDetId(uint32_t id) : id_(id) {}
+  constexpr std::int32_t type() { return (id_ >> kHGCalTypeOffset) & kHGCalTypeMask; }
+  constexpr std::int32_t zside() const { return (((id_ >> kHGCalZsideOffset) & kHGCalZsideMask) ? -1 : 1); }
+  constexpr std::int32_t layer() const { return (id_ >> kHGCalLayerOffset) & kHGCalLayerMask; }
+
+private:
+  std::uint32_t id_;
+  static constexpr std::uint32_t kHGCalPhiOffset = 0;
+  static constexpr std::uint32_t kHGCalPhiMask = 0x1FF;
+  static constexpr std::uint32_t kHGCalRadiusOffset = 9;
+  static constexpr std::uint32_t kHGCalRadiusMask = 0xFF;
+  static constexpr std::uint32_t kHGCalLayerOffset = 17;
+  static constexpr std::uint32_t kHGCalLayerMask = 0x1F;
+  static constexpr std::uint32_t kHGCalTriggerOffset = 22;
+  static constexpr std::uint32_t kHGCalTriggerMask = 0x1;
+  static constexpr std::uint32_t kHGCalZsideOffset = 25;
+  static constexpr std::uint32_t kHGCalZsideMask = 0x1;
+  static constexpr std::uint32_t kHGCalTypeOffset = 26;
+  static constexpr std::uint32_t kHGCalTypeMask = 0x3;
+};
+
+namespace hgcal_conditions {
+  namespace parameters {
+    enum class HeterogeneousHGCalEEParametersType { Double, Int32_t };
+    enum class HeterogeneousHGCalHEFParametersType { Double, Int32_t };
+    enum class HeterogeneousHGCalHEBParametersType { Double, Int32_t };
+
+    const std::array<HeterogeneousHGCalEEParametersType, 5> typesEE = {{HeterogeneousHGCalEEParametersType::Double,
+                                                                        HeterogeneousHGCalEEParametersType::Double,
+                                                                        HeterogeneousHGCalEEParametersType::Double,
+                                                                        HeterogeneousHGCalEEParametersType::Double,
+                                                                        HeterogeneousHGCalEEParametersType::Int32_t}};
+
+    const std::array<HeterogeneousHGCalHEFParametersType, 5> typesHEF = {
+        {HeterogeneousHGCalHEFParametersType::Double,
+         HeterogeneousHGCalHEFParametersType::Double,
+         HeterogeneousHGCalHEFParametersType::Double,
+         HeterogeneousHGCalHEFParametersType::Double,
+         HeterogeneousHGCalHEFParametersType::Int32_t}};
+
+    const std::array<HeterogeneousHGCalHEBParametersType, 2> typesHEB = {
+        {HeterogeneousHGCalHEBParametersType::Double, HeterogeneousHGCalHEBParametersType::Int32_t}};
+
+    class HeterogeneousHGCalEEParameters {
+    public:
+      //indexed by cell number
+      double *cellFineX_;
+      double *cellFineY_;
+      double *cellCoarseX_;
+      double *cellCoarseY_;
+      //index by wafer number
+      std::int32_t *waferTypeL_;
+    };
+    class HeterogeneousHGCalHEFParameters {
+    public:
+      //indexed by cell number
+      double *cellFineX_;
+      double *cellFineY_;
+      double *cellCoarseX_;
+      double *cellCoarseY_;
+      //index by wafer number
+      std::int32_t *waferTypeL_;
+    };
+    class HeterogeneousHGCalHEBParameters {
+    public:
+      double *testD_;
+      std::int32_t *testI_;
+    };
+
+  }  //namespace parameters
+
+  namespace positions {
+
+    enum class HeterogeneousHGCalPositionsType { Float, Int32_t, Uint32_t };
+
+    const std::vector<HeterogeneousHGCalPositionsType> types = {HeterogeneousHGCalPositionsType::Float,
+                                                                HeterogeneousHGCalPositionsType::Float,
+                                                                HeterogeneousHGCalPositionsType::Float,
+                                                                HeterogeneousHGCalPositionsType::Int32_t,
+                                                                HeterogeneousHGCalPositionsType::Int32_t,
+                                                                HeterogeneousHGCalPositionsType::Int32_t,
+                                                                HeterogeneousHGCalPositionsType::Uint32_t};
+
+    struct HGCalPositionsMapping {
+      std::vector<float> zLayer;                    //z position per layer
+      std::vector<std::int32_t> nCellsLayer;        //#cells per layer
+      std::vector<std::int32_t> nCellsWaferUChunk;  //#cells per U wafer (each in turn including all V wafers)
+      std::vector<std::int32_t> nCellsHexagon;      //#cells per V wafer
+      std::vector<std::uint32_t> detid;
+      //variables required for calculating the positions (x,y) from the detid in the GPU
+      float waferSize;
+      float sensorSeparation;
+      //variables required for the mapping of detid -> cell in the geometry
+      std::int32_t firstLayer;
+      std::int32_t lastLayer;
+      std::int32_t waferMax;
+      std::int32_t waferMin;
+    };
+
+    struct HeterogeneousHGCalPositionsMapping {
+      //the x, y and z positions will not be filled in the CPU
+      float *x;
+      float *y;
+      float *zLayer;
+      std::int32_t *nCellsLayer;
+      std::int32_t *nCellsWaferUChunk;
+      std::int32_t *nCellsHexagon;
+      std::uint32_t *detid;
+      //variables required for calculating the positions (x,y) from the detid in the GPU
+      float waferSize;
+      float sensorSeparation;
+      //variables required for the mapping of detid -> cell in the geometry
+      std::int32_t firstLayer;
+      std::int32_t lastLayer;
+      std::int32_t waferMax;
+      std::int32_t waferMin;
+    };
+
+  }  //namespace positions
+
+  struct HeterogeneousEEConditionsESProduct {
+    parameters::HeterogeneousHGCalEEParameters params;
+  };
+  struct HeterogeneousHEFConditionsESProduct {
+    parameters::HeterogeneousHGCalHEFParameters params;
+    //positions::HeterogeneousHGCalPositionsMapping posmap;
+    //size_t nelems_posmap;
+  };
+  struct HeterogeneousHEBConditionsESProduct {
+    parameters::HeterogeneousHGCalHEBParameters params;
+  };
+
+  struct HeterogeneousHEFCellPositionsConditionsESProduct {
+    positions::HeterogeneousHGCalPositionsMapping posmap;
+    std::size_t nelems_posmap;
+  };
+
+}  // namespace hgcal_conditions
+
+#endif  //CUDADataFormats_HGCal_HGCConditions_h

CUDADataFormats/HGCal/interface/HGCRecHitCPUProduct.h

@@ -0,0 +1,65 @@
+#ifndef CUDADAtaFormats_HGCal_HGCRecHitCPUProduct_H
+#define CUDADAtaFormats_HGCal_HGCRecHitCPUProduct_H
+
+#include <cassert>
+#include <numeric>
+
+#include "HeterogeneousCore/CUDAUtilities/interface/host_unique_ptr.h"
+#include "CUDADataFormats/HGCal/interface/HGCRecHitSoA.h"
+#include "CUDADataFormats/HGCal/interface/ConstHGCRecHitSoA.h"
+#include "CUDADataFormats/HGCal/interface/HGCUncalibRecHitSoA.h"
+
+class HGCRecHitCPUProduct {
+public:
+  HGCRecHitCPUProduct() = default;
+  explicit HGCRecHitCPUProduct(uint32_t nhits, const cudaStream_t &stream) : nhits_(nhits) {
+    size_tot_ = std::accumulate(sizes_.begin(), sizes_.end(), 0);  //this might be done at compile time
+    pad_ = ((nhits - 1) / 32 + 1) * 32;                            //align to warp boundary (assumption: warpSize = 32)
+    mem_ = cms::cuda::make_host_unique<std::byte[]>(pad_ * size_tot_, stream);
+  }
+  ~HGCRecHitCPUProduct() = default;
+
+  HGCRecHitCPUProduct(const HGCRecHitCPUProduct &) = delete;
+  HGCRecHitCPUProduct &operator=(const HGCRecHitCPUProduct &) = delete;
+  HGCRecHitCPUProduct(HGCRecHitCPUProduct &&) = default;
+  HGCRecHitCPUProduct &operator=(HGCRecHitCPUProduct &&) = default;
+
+  HGCRecHitSoA get() {
+    HGCRecHitSoA soa;
+    soa.energy_ = reinterpret_cast<float *>(mem_.get());
+    soa.time_ = soa.energy_ + pad_;
+    soa.timeError_ = soa.time_ + pad_;
+    soa.id_ = reinterpret_cast<uint32_t *>(soa.timeError_ + pad_);
+    soa.flagBits_ = soa.id_ + pad_;
+    soa.son_ = reinterpret_cast<uint8_t *>(soa.flagBits_ + pad_);
+    soa.nbytes_ = size_tot_;
+    soa.nhits_ = nhits_;
+    soa.pad_ = pad_;
+    return soa;
+  }
+  ConstHGCRecHitSoA get() const {
+    ConstHGCRecHitSoA soa;
+    soa.energy_ = reinterpret_cast<float const *>(mem_.get());
+    soa.time_ = soa.energy_ + pad_;
+    soa.timeError_ = soa.time_ + pad_;
+    soa.id_ = reinterpret_cast<uint32_t const *>(soa.timeError_ + pad_);
+    soa.flagBits_ = soa.id_ + pad_;
+    soa.son_ = reinterpret_cast<uint8_t const *>(soa.flagBits_ + pad_);
+    return soa;
+  }
+  uint32_t nHits() const { return nhits_; }
+  uint32_t pad() const { return pad_; }
+  uint32_t nBytes() const { return size_tot_; }
+
+private:
+  cms::cuda::host::unique_ptr<std::byte[]> mem_;
+  static constexpr std::array<int, memory::npointers::ntypes_hgcrechits_soa> sizes_ = {
+      {memory::npointers::float_hgcrechits_soa * sizeof(float),
+       memory::npointers::uint32_hgcrechits_soa * sizeof(uint32_t),
+       memory::npointers::uint8_hgcrechits_soa * sizeof(uint8_t)}};
+  uint32_t pad_;
+  uint32_t nhits_;
+  uint32_t size_tot_;
+};
+
+#endif  //CUDADAtaFormats_HGCal_HGCRecHitCPUProduct_H

CUDADataFormats/HGCal/interface/HGCRecHitGPUProduct.h

@@ -0,0 +1,69 @@
+#ifndef CUDADAtaFormats_HGCal_HGCRecHitGPUProduct_H
+#define CUDADAtaFormats_HGCal_HGCRecHitGPUProduct_H
+
+#include <cassert>
+#include <numeric>
+
+#include "HeterogeneousCore/CUDAUtilities/interface/device_unique_ptr.h"
+#include "CUDADataFormats/HGCal/interface/HGCRecHitSoA.h"
+#include "CUDADataFormats/HGCal/interface/ConstHGCRecHitSoA.h"
+#include "CUDADataFormats/HGCal/interface/HGCUncalibRecHitSoA.h"
+
+class HGCRecHitGPUProduct {
+public:
+  HGCRecHitGPUProduct() = default;
+  explicit HGCRecHitGPUProduct(uint32_t nhits, const cudaStream_t &stream) : nhits_(nhits) {
+    size_tot_ = std::accumulate(sizes_.begin(), sizes_.end(), 0);  //this might be done at compile time
+    pad_ = ((nhits - 1) / 32 + 1) * 32;                            //align to warp boundary (assumption: warpSize = 32)
+    mem_ = cms::cuda::make_device_unique<std::byte[]>(pad_ * size_tot_, stream);
+  }
+  ~HGCRecHitGPUProduct() = default;
+
+  HGCRecHitGPUProduct(const HGCRecHitGPUProduct &) = delete;
+  HGCRecHitGPUProduct &operator=(const HGCRecHitGPUProduct &) = delete;
+  HGCRecHitGPUProduct(HGCRecHitGPUProduct &&) = default;
+  HGCRecHitGPUProduct &operator=(HGCRecHitGPUProduct &&) = default;
+
+  HGCRecHitSoA get() {
+    HGCRecHitSoA soa;
+    soa.energy_ = reinterpret_cast<float *>(mem_.get());
+    soa.time_ = soa.energy_ + pad_;
+    soa.timeError_ = soa.time_ + pad_;
+    soa.id_ = reinterpret_cast<uint32_t *>(soa.timeError_ + pad_);
+    soa.flagBits_ = soa.id_ + pad_;
+    soa.son_ = reinterpret_cast<uint8_t *>(soa.flagBits_ + pad_);
+    soa.nbytes_ = size_tot_;
+    soa.nhits_ = nhits_;
+    soa.pad_ = pad_;
+    return soa;
+  }
+  ConstHGCRecHitSoA get() const {
+    ConstHGCRecHitSoA soa;
+    soa.energy_ = reinterpret_cast<float const *>(mem_.get());
+    soa.time_ = soa.energy_ + pad_;
+    soa.timeError_ = soa.time_ + pad_;
+    soa.id_ = reinterpret_cast<uint32_t const *>(soa.timeError_ + pad_);
+    soa.flagBits_ = soa.id_ + pad_;
+    soa.son_ = reinterpret_cast<uint8_t const *>(soa.flagBits_ + pad_);
+    return soa;
+  }
+
+  //number of hits stored in the SoA
+  uint32_t nHits() const { return nhits_; }
+  //pad of memory block (used for warp alignment, slighlty larger than 'nhits_')
+  uint32_t pad() const { return pad_; }
+  //number of bytes of the SoA
+  uint32_t nBytes() const { return size_tot_; }
+
+private:
+  cms::cuda::device::unique_ptr<std::byte[]> mem_;
+  static constexpr std::array<int, memory::npointers::ntypes_hgcrechits_soa> sizes_ = {
+      {memory::npointers::float_hgcrechits_soa * sizeof(float),
+       memory::npointers::uint32_hgcrechits_soa * sizeof(uint32_t),
+       memory::npointers::uint8_hgcrechits_soa * sizeof(uint8_t)}};
+  uint32_t pad_;
+  uint32_t nhits_;
+  uint32_t size_tot_;
+};
+
+#endif  //CUDADAtaFormats_HGCal_HGCRecHitGPUProduct_H

CUDADataFormats/HGCal/interface/HGCRecHitSoA.h

@@ -0,0 +1,29 @@
+#ifndef CUDADataFormats_HGCal_HGCRecHitSoA_h
+#define CUDADataFormats_HGCal_HGCRecHitSoA_h
+
+#include <cstdint>
+
+class HGCRecHitSoA {
+public:
+  float *energy_;       //calibrated energy of the rechit
+  float *time_;         //time jitter of the UncalibRecHit
+  float *timeError_;    //time resolution
+  uint32_t *id_;        //rechit detId
+  uint32_t *flagBits_;  //rechit flags describing its status (DataFormats/HGCRecHit/interface/HGCRecHit.h)
+  uint8_t *son_;        //signal over noise
+
+  uint32_t nbytes_;  //number of bytes of the SoA
+  uint32_t nhits_;   //number of hits stored in the SoA
+  uint32_t pad_;     //pad of memory block (used for warp alignment, slightly larger than 'nhits_')
+};
+
+namespace memory {
+  namespace npointers {
+    constexpr unsigned float_hgcrechits_soa = 3;   //number of float pointers in the rechits SoA
+    constexpr unsigned uint32_hgcrechits_soa = 2;  //number of uint32_t pointers in the rechits SoA
+    constexpr unsigned uint8_hgcrechits_soa = 1;   //number of uint8_t pointers in the rechits SoA
+    constexpr unsigned ntypes_hgcrechits_soa = 3;  //number of different pointer types in the rechits SoA
+  }                                                // namespace npointers
+}  // namespace memory
+
+#endif  //CUDADataFormats_HGCal_HGCRecHitSoA_h