Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

ECAL GPU unpacker: adapt the buffer size to the ECAL FEDs size [12.3.x] #38205

Merged
merged 2 commits into from Jun 9, 2022
Merged

ECAL GPU unpacker: adapt the buffer size to the ECAL FEDs size [12.3.x] #38205

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
7d865d1
Adapt the buffer size to the ECAL FEDs size
fwyzard Jun 2, 2022
7ffb562
Remove the EcalRawToDigi.maxFedSize parameter from the HLT menu
fwyzard Jun 2, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
7 changes: 0 additions & 7 deletions EventFilter/EcalRawToDigi/plugins/DeclsForKernels.h
View file
Open in desktop
Expand Up @@ -15,13 +15,6 @@
namespace ecal {
namespace raw {

constexpr auto empty_event_size = EMPTYEVENTSIZE;
constexpr uint32_t nfeds_max = 54;
constexpr uint32_t nbytes_per_fed_max = 41616; // max FED size in full readout mode
// DCC header and trailer: 10 words (64bit),
// TCC block: 18 words, SR block 6 words,
// (25 channels per tower * 3 words + 1 header word) * 68 towers

struct InputDataCPU {
cms::cuda::host::unique_ptr<unsigned char[]> data;
cms::cuda::host::unique_ptr<uint32_t[]> offsets;
Expand Down
49 changes: 27 additions & 22 deletions EventFilter/EcalRawToDigi/plugins/EcalRawToDigiGPU.cc
View file
Open in desktop
Expand Up @@ -18,7 +18,7 @@
class EcalRawToDigiGPU : public edm::stream::EDProducer<edm::ExternalWork> {
public:
explicit EcalRawToDigiGPU(edm::ParameterSet const& ps);
~EcalRawToDigiGPU() override;
~EcalRawToDigiGPU() override = default;
static void fillDescriptions(edm::ConfigurationDescriptions&);

private:
Expand All @@ -33,7 +33,6 @@ class EcalRawToDigiGPU : public edm::stream::EDProducer<edm::ExternalWork> {

cms::cuda::ContextState cudaState_;

const uint32_t maxFedSize_;
std::vector<int> fedsToUnpack_;

ecal::raw::ConfigurationParameters config_;
Expand All @@ -45,7 +44,6 @@ void EcalRawToDigiGPU::fillDescriptions(edm::ConfigurationDescriptions& confDesc
edm::ParameterSetDescription desc;

desc.add<edm::InputTag>("InputLabel", edm::InputTag("rawDataCollector"));
desc.add<uint32_t>("maxFedSize", ecal::raw::nbytes_per_fed_max);
std::vector<int> feds(54);
for (uint32_t i = 0; i < 54; ++i)
feds[i] = i + 601;
Expand All @@ -64,14 +62,11 @@ EcalRawToDigiGPU::EcalRawToDigiGPU(const edm::ParameterSet& ps)
digisEBToken_{produces<OutputProduct>(ps.getParameter<std::string>("digisLabelEB"))},
digisEEToken_{produces<OutputProduct>(ps.getParameter<std::string>("digisLabelEE"))},
eMappingToken_{esConsumes<ecal::raw::ElectronicsMappingGPU, EcalMappingElectronicsRcd>()},
maxFedSize_{ps.getParameter<uint32_t>("maxFedSize")},
fedsToUnpack_{ps.getParameter<std::vector<int>>("FEDs")} {
config_.maxChannelsEB = ps.getParameter<uint32_t>("maxChannelsEB");
config_.maxChannelsEE = ps.getParameter<uint32_t>("maxChannelsEE");
}

EcalRawToDigiGPU::~EcalRawToDigiGPU() {}

void EcalRawToDigiGPU::acquire(edm::Event const& event,
edm::EventSetup const& setup,
edm::WaitingTaskWithArenaHolder holder) {
Expand All @@ -92,17 +87,30 @@ void EcalRawToDigiGPU::acquire(edm::Event const& event,
// scratch
ecal::raw::ScratchDataGPU scratchGPU = {cms::cuda::make_device_unique<uint32_t[]>(2, ctx.stream())};

// make a first iteration over the FEDs to compute the total buffer size
uint32_t size = 0;
uint32_t feds = 0;
for (auto const& fed : fedsToUnpack_) {
auto const& data = rawDataHandle->FEDData(fed);
auto const nbytes = data.size();

// skip empty FEDs
if (nbytes < globalFieds::EMPTYEVENTSIZE)
continue;

size += nbytes;
++feds;
}

// input cpu data
ecal::raw::InputDataCPU inputCPU = {
cms::cuda::make_host_unique<unsigned char[]>(ecal::raw::nfeds_max * maxFedSize_, ctx.stream()),
cms::cuda::make_host_unique<uint32_t[]>(ecal::raw::nfeds_max, ctx.stream()),
cms::cuda::make_host_unique<int[]>(ecal::raw::nfeds_max, ctx.stream())};
ecal::raw::InputDataCPU inputCPU = {cms::cuda::make_host_unique<unsigned char[]>(size, ctx.stream()),
cms::cuda::make_host_unique<uint32_t[]>(feds, ctx.stream()),
cms::cuda::make_host_unique<int[]>(feds, ctx.stream())};

// input data gpu
ecal::raw::InputDataGPU inputGPU = {
cms::cuda::make_device_unique<unsigned char[]>(ecal::raw::nfeds_max * maxFedSize_, ctx.stream()),
cms::cuda::make_device_unique<uint32_t[]>(ecal::raw::nfeds_max, ctx.stream()),
cms::cuda::make_device_unique<int[]>(ecal::raw::nfeds_max, ctx.stream())};
ecal::raw::InputDataGPU inputGPU = {cms::cuda::make_device_unique<unsigned char[]>(size, ctx.stream()),
cms::cuda::make_device_unique<uint32_t[]>(feds, ctx.stream()),
cms::cuda::make_device_unique<int[]>(feds, ctx.stream())};

// output cpu
outputCPU_ = {cms::cuda::make_host_unique<uint32_t[]>(2, ctx.stream())};
Expand All @@ -113,20 +121,15 @@ void EcalRawToDigiGPU::acquire(edm::Event const& event,
// output gpu
outputGPU_.allocate(config_, ctx.stream());

// iterate over feds
// TODO: another idea
// - loop over all feds to unpack and enqueue cuda memcpy
// - accumulate the sizes
// - after the loop launch cuda memcpy for sizes
// - enqueue the kernel
// iterate over FEDs to fill the cpu buffer
uint32_t currentCummOffset = 0;
uint32_t counter = 0;
for (auto const& fed : fedsToUnpack_) {
auto const& data = rawDataHandle->FEDData(fed);
auto const nbytes = data.size();

// skip empty feds
if (nbytes < ecal::raw::empty_event_size)
// skip empty FEDs
if (nbytes < globalFieds::EMPTYEVENTSIZE)
continue;

// copy raw data into plain buffer
Expand All @@ -139,6 +142,8 @@ void EcalRawToDigiGPU::acquire(edm::Event const& event,
currentCummOffset += nbytes;
++counter;
}
assert(currentCummOffset == size);
assert(counter == feds);

// unpack if at least one FED has data
if (counter > 0) {
Expand Down
11 changes: 11 additions & 0 deletions HLTrigger/Configuration/python/customizeHLTforCMSSW.py
View file
Open in desktop
Expand Up @@ -189,12 +189,23 @@ def customiseForOffline(process):
return process


# ECAL GPU unpacker: adapt the buffer size to the ECAL FEDs size (#38202)
# remove the EcalRawToDigi.maxFedSize parameter from the menu
def customizeHLTfor38202(process):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should be 38205, but I don't object, it's clear enough.

for producer in producers_by_type(process, "EcalRawToDigiGPU"):
if hasattr(producer, "maxFedSize"):
delattr(producer, "maxFedSize")

return process


# CMSSW version specific customizations
def customizeHLTforCMSSW(process, menuType="GRun"):

process = customiseForOffline(process)

# add call to action function in proper order: newest last!
# process = customiseFor12718(process)
process = customizeHLTfor38202(process)

return process