From 92c713b426c1d972aa34c4c945aec3846d197143 Mon Sep 17 00:00:00 2001 From: Srecko Morovic Date: Thu, 9 Sep 2021 15:34:17 +0200 Subject: [PATCH 1/3] * use a shared file lock instead of exclusive lock. Only EoLS takes exclusive lock, so it is ensured to be written in the same order as notified from the file broker service. Creation of EoLS was moved out of the shared section. Check of directory existing was moved out of the critical section * silence warnings of TCDS FED mismatch when using test TCDS range * set file locking monitoring state at the correct location --- .../Utilities/interface/AuxiliaryMakers.h | 3 +- .../Utilities/interface/EvFDaqDirector.h | 3 +- .../interface/FedRawDataInputSource.h | 2 +- EventFilter/Utilities/src/AuxiliaryMakers.cc | 5 +- EventFilter/Utilities/src/EvFDaqDirector.cc | 58 ++++++++++++++----- .../Utilities/src/FedRawDataInputSource.cc | 14 +++-- 6 files changed, 61 insertions(+), 24 deletions(-) diff --git a/EventFilter/Utilities/interface/AuxiliaryMakers.h b/EventFilter/Utilities/interface/AuxiliaryMakers.h index 00d15bc0081b1..39cee97ed042f 100644 --- a/EventFilter/Utilities/interface/AuxiliaryMakers.h +++ b/EventFilter/Utilities/interface/AuxiliaryMakers.h @@ -12,7 +12,8 @@ namespace evf { bool isRealData, const edm::EventAuxiliary::ExperimentType&, const std::string& processGUID, - bool verifyLumiSection); + bool verifyLumiSection, + bool suppressWarning); } } // namespace evf #endif diff --git a/EventFilter/Utilities/interface/EvFDaqDirector.h b/EventFilter/Utilities/interface/EvFDaqDirector.h index fbc0881ead2d4..b38ded4451689 100644 --- a/EventFilter/Utilities/interface/EvFDaqDirector.h +++ b/EventFilter/Utilities/interface/EvFDaqDirector.h @@ -127,7 +127,8 @@ namespace evf { void createBoLSFile(const uint32_t lumiSection, bool checkIfExists) const; void createLumiSectionFiles(const uint32_t lumiSection, const uint32_t currentLumiSection, - bool doCreateBoLS = true); + bool doCreateBoLS, + bool doCreateEoLS); static int parseFRDFileHeader(std::string const& rawSourcePath, int& rawFd, uint16_t& rawHeaderSize, diff --git a/EventFilter/Utilities/interface/FedRawDataInputSource.h b/EventFilter/Utilities/interface/FedRawDataInputSource.h index b68c98ca9a624..005e103daa8e8 100644 --- a/EventFilter/Utilities/interface/FedRawDataInputSource.h +++ b/EventFilter/Utilities/interface/FedRawDataInputSource.h @@ -60,7 +60,7 @@ class FedRawDataInputSource : public edm::RawInputSource { void maybeOpenNewLumiSection(const uint32_t lumiSection); evf::EvFDaqDirector::FileStatus nextEvent(); evf::EvFDaqDirector::FileStatus getNextEvent(); - edm::Timestamp fillFEDRawDataCollection(FEDRawDataCollection&); + edm::Timestamp fillFEDRawDataCollection(FEDRawDataCollection& rawData, bool& tcdsInRange); void readSupervisor(); void readWorker(unsigned int tid); diff --git a/EventFilter/Utilities/src/AuxiliaryMakers.cc b/EventFilter/Utilities/src/AuxiliaryMakers.cc index a11ccd229fa09..5886700681e86 100644 --- a/EventFilter/Utilities/src/AuxiliaryMakers.cc +++ b/EventFilter/Utilities/src/AuxiliaryMakers.cc @@ -12,7 +12,8 @@ namespace evf { bool isRealData, const edm::EventAuxiliary::ExperimentType& eventType, const std::string& processGUID, - bool verifyLumiSection) { + bool verifyLumiSection, + bool suppressWarning) { edm::EventID eventId(runNumber, // check that runnumber from record is consistent lumiSection, tcds->header.eventNumber); @@ -29,7 +30,7 @@ namespace evf { const uint64_t orbitnr = ((uint64_t)tcds->header.orbitHigh << 16) | tcds->header.orbitLow; const uint32_t recordLumiSection = tcds->header.lumiSection; - if (isRealData) { + if (isRealData && !suppressWarning) { //warnings are disabled for generated data if (verifyLumiSection && recordLumiSection != lumiSection) edm::LogWarning("AuxiliaryMakers") diff --git a/EventFilter/Utilities/src/EvFDaqDirector.cc b/EventFilter/Utilities/src/EvFDaqDirector.cc index 5943bee8c0c48..a9a75016b913f 100644 --- a/EventFilter/Utilities/src/EvFDaqDirector.cc +++ b/EventFilter/Utilities/src/EvFDaqDirector.cc @@ -942,14 +942,17 @@ namespace evf { void EvFDaqDirector::createLumiSectionFiles(const uint32_t lumiSection, const uint32_t currentLumiSection, - bool doCreateBoLS) { + bool doCreateBoLS, + bool doCreateEoLS) { if (currentLumiSection > 0) { const std::string fuEoLS = getEoLSFilePathOnFU(currentLumiSection); struct stat buf; bool found = (stat(fuEoLS.c_str(), &buf) == 0); if (!found) { - int eol_fd = open(fuEoLS.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); - close(eol_fd); + if (doCreateEoLS) { + int eol_fd = open(fuEoLS.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + close(eol_fd); + } if (doCreateBoLS) createBoLSFile(lumiSection, false); } @@ -1677,8 +1680,10 @@ namespace evf { serverError = true; } } + break; } + } catch (std::exception const& e) { edm::LogWarning("EvFDaqDirector") << "Exception in socket handling"; serverError = true; @@ -1704,6 +1709,7 @@ namespace evf { fileStatus = noFile; sleep(1); //back-off if error detected } + return fileStatus; } @@ -1760,7 +1766,7 @@ namespace evf { //local lock to force index json and EoLS files to appear in order if (fileBrokerUseLocalLock_) - lockFULocal2(); + lockFULocal(); int maxLS = stopFileLS < 0 ? -1 : std::max(stopFileLS, (int)currentLumiSection); bool rawHeader = false; @@ -1770,22 +1776,22 @@ namespace evf { if (serverError) { //do not update anything if (fileBrokerUseLocalLock_) - unlockFULocal2(); + unlockFULocal(); return noFile; + } - //handle creation of EoLS and BoLS files if lumisection has changed + //handle creation of BoLS files if lumisection has changed if (currentLumiSection == 0) { - if (fileStatus == runEnded) { - createLumiSectionFiles(closedServerLS, 0); - createLumiSectionFiles(serverLS, closedServerLS, false); // +1 - } else - createLumiSectionFiles(serverLS, 0); + if (fileStatus == runEnded) + createLumiSectionFiles(closedServerLS, 0, true, false); + else + createLumiSectionFiles(serverLS, 0, true, false); } else { - //loop over and create any EoLS files missing if (closedServerLS >= currentLumiSection) { + //only BoLS files for (uint32_t i = std::max(currentLumiSection, 1U); i <= closedServerLS; i++) - createLumiSectionFiles(i + 1, i); + createLumiSectionFiles(i + 1, i, true, false); } } @@ -1803,6 +1809,11 @@ namespace evf { close(rawFd); rawFd = -1; } + + //can unlock because all files have been created locally + if (fileBrokerUseLocalLock_) + unlockFULocal(); + if (!fileFound) { //catch condition where directory got deleted fileStatus = noFile; @@ -1813,9 +1824,26 @@ namespace evf { } } - //can unlock because all files have been created locally - if (fileBrokerUseLocalLock_) + //handle creation of EoLS files if lumisection has changed, this needs to be locked exclusively + //so that EoLS files can not appear locally before index files + if (currentLumiSection == 0) { + lockFULocal2(); + if (fileStatus == runEnded) { + createLumiSectionFiles(closedServerLS, 0, false, true); + createLumiSectionFiles(serverLS, closedServerLS, false, true); // +1 + } else { + createLumiSectionFiles(serverLS, 0, false, true); + } unlockFULocal2(); + } else { + if (closedServerLS >= currentLumiSection) { + //lock exclusive to create EoLS files + lockFULocal2(); + for (uint32_t i = std::max(currentLumiSection, 1U); i <= closedServerLS; i++) + createLumiSectionFiles(i + 1, i, false, true); + unlockFULocal2(); + } + } if (fileStatus == runEnded) ls = std::max(currentLumiSection, serverLS); diff --git a/EventFilter/Utilities/src/FedRawDataInputSource.cc b/EventFilter/Utilities/src/FedRawDataInputSource.cc index e39363529cbaa..1da0211999e73 100644 --- a/EventFilter/Utilities/src/FedRawDataInputSource.cc +++ b/EventFilter/Utilities/src/FedRawDataInputSource.cc @@ -613,7 +613,8 @@ inline evf::EvFDaqDirector::FileStatus FedRawDataInputSource::getNextEvent() { void FedRawDataInputSource::read(edm::EventPrincipal& eventPrincipal) { setMonState(inReadEvent); std::unique_ptr rawData(new FEDRawDataCollection); - edm::Timestamp tstamp = fillFEDRawDataCollection(*rawData); + bool tcdsInRange; + edm::Timestamp tstamp = fillFEDRawDataCollection(*rawData, tcdsInRange); if (useL1EventID_) { eventID_ = edm::EventID(eventRunNumber_, currentLumiSection_, L1EventID_); @@ -639,7 +640,8 @@ void FedRawDataInputSource::read(edm::EventPrincipal& eventPrincipal) { event_->isRealData(), static_cast(fedHeader.triggerType()), processGUID(), - !fileListLoopMode_); + !fileListLoopMode_, + !tcdsInRange); aux.setProcessHistoryID(processHistoryID_); makeEvent(eventPrincipal, aux); } @@ -684,7 +686,7 @@ void FedRawDataInputSource::read(edm::EventPrincipal& eventPrincipal) { return; } -edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollection& rawData) { +edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollection& rawData, bool &tcdsInRange) { edm::TimeValue_t time; timeval stv; gettimeofday(&stv, nullptr); @@ -696,6 +698,7 @@ edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollect unsigned char* event = (unsigned char*)event_->payload(); GTPEventID_ = 0; tcds_pointer_ = nullptr; + tcdsInRange = false; uint16_t selectedTCDSFed = 0; while (eventSize > 0) { assert(eventSize >= FEDTrailer::length); @@ -712,6 +715,9 @@ edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollect if (!selectedTCDSFed) { selectedTCDSFed = fedId; tcds_pointer_ = event + eventSize; + if (fedId >= FEDNumbering::MINTCDSuTCAFEDID && fedId <= FEDNumbering::MAXTCDSuTCAFEDID) { + tcdsInRange = true; + } } else throw cms::Exception("FedRawDataInputSource::fillFEDRawDataCollection") << "Second TCDS FED ID " << fedId << " found. First ID: " << selectedTCDSFed; @@ -810,7 +816,6 @@ void FedRawDataInputSource::readSupervisor() { if (fms_) { setMonStateSup(inSupBusy); fms_->startedLookingForFile(); - setMonStateSup(inSupLockPolling); } evf::EvFDaqDirector::FileStatus status = evf::EvFDaqDirector::noFile; @@ -830,6 +835,7 @@ void FedRawDataInputSource::readSupervisor() { assert(rawFd == -1); uint64_t thisLockWaitTimeUs = 0.; + setMonStateSup(inSupLockPolling); if (fileListMode_) { //return LS if LS not set, otherwise return file status = getFile(ls, nextFile, fileSizeIndex, thisLockWaitTimeUs); From 969d7820a0ee0f235464b5292303a5b2461da9a8 Mon Sep 17 00:00:00 2001 From: Srecko Morovic Date: Thu, 9 Sep 2021 19:19:53 +0200 Subject: [PATCH 2/3] implement filling random FED data in the fake raw data generator --- .../Utilities/plugins/DaqFakeReader.cc | 32 +++++++++++++++++++ EventFilter/Utilities/plugins/DaqFakeReader.h | 4 +++ EventFilter/Utilities/test/LocalRunBUFU.sh | 2 +- EventFilter/Utilities/test/startBU.py | 1 + 4 files changed, 38 insertions(+), 1 deletion(-) diff --git a/EventFilter/Utilities/plugins/DaqFakeReader.cc b/EventFilter/Utilities/plugins/DaqFakeReader.cc index 7bc931d1658a2..f1ed5cb8d56c4 100644 --- a/EventFilter/Utilities/plugins/DaqFakeReader.cc +++ b/EventFilter/Utilities/plugins/DaqFakeReader.cc @@ -20,6 +20,8 @@ #include #include #include +#include +#include using namespace std; using namespace edm; @@ -33,6 +35,7 @@ DaqFakeReader::DaqFakeReader(const edm::ParameterSet& pset) : runNum(1), eventNum(1), empty_events(pset.getUntrackedParameter("emptyEvents", false)), + fillRandom_(pset.getUntrackedParameter("fillRandom", false)), meansize(pset.getUntrackedParameter("meanSize", 1024)), width(pset.getUntrackedParameter("width", 1024)), injected_errors_per_million_events(pset.getUntrackedParameter("injectErrPpm", 0)), @@ -43,6 +46,11 @@ DaqFakeReader::DaqFakeReader(const edm::ParameterSet& pset) if (tcdsFEDID_ < FEDNumbering::MINTCDSuTCAFEDID) throw cms::Exception("DaqFakeReader::DaqFakeReader") << " TCDS FED ID lower than " << FEDNumbering::MINTCDSuTCAFEDID; + if (fillRandom_) { + //intialize random seed + auto time_count = static_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + srand(time_count & 0xffffffff); + } produces(); } @@ -102,6 +110,18 @@ void DaqFakeReader::fillFEDs( // Allocate space for header+trailer+payload feddata.resize(size + 16); + if (fillRandom_) { + //fill FED with random values + size_t size_ui = size - size % sizeof(unsigned int); + for (size_t i=0; i < size_ui; i += sizeof(unsigned int)) { + *((unsigned int*)(feddata.data() + i)) = (unsigned int)rand(); + } + //remainder + for (size_t i = size_ui; i < size; i++) { + *(feddata.data() + i) = rand() & 0xff; + } + } + // Generate header FEDHeader::set(feddata.data(), 1, // Trigger type @@ -164,3 +184,15 @@ void DaqFakeReader::beginLuminosityBlock(LuminosityBlock const& iL, EventSetup c std::cout << "DaqFakeReader begin Lumi " << iL.luminosityBlock() << std::endl; fakeLs_ = iL.luminosityBlock(); } + +void DaqFakeReader::fillDescriptions(edm::ConfigurationDescriptions& descriptions) { + edm::ParameterSetDescription desc; + desc.setComment("Injector of generated raw FED data for DAQ testing"); + desc.addUntracked("emptyEvents", false); + desc.addUntracked("fillRandom", false); + desc.addUntracked("meanSize", 1024); + desc.addUntracked("width", 1024); + desc.addUntracked("injectErrPpm", 1024); + desc.addUntracked("tcdsFEDID", 1024); + descriptions.add("DaqFakeReader", desc); +} diff --git a/EventFilter/Utilities/plugins/DaqFakeReader.h b/EventFilter/Utilities/plugins/DaqFakeReader.h index 94d9d6d79bd4b..33152b790c686 100644 --- a/EventFilter/Utilities/plugins/DaqFakeReader.h +++ b/EventFilter/Utilities/plugins/DaqFakeReader.h @@ -7,6 +7,7 @@ * \author N. Amapane - CERN */ +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" #include "FWCore/Framework/interface/one/EDProducer.h" #include "FWCore/Framework/interface/Event.h" #include "FWCore/Framework/interface/EventSetup.h" @@ -33,6 +34,8 @@ class DaqFakeReader : public edm::one::EDProducer<> { void produce(edm::Event&, edm::EventSetup const&) override; + static void fillDescriptions(edm::ConfigurationDescriptions& descriptions); + private: // // private member functions @@ -48,6 +51,7 @@ class DaqFakeReader : public edm::one::EDProducer<> { edm::RunNumber_t runNum; edm::EventNumber_t eventNum; bool empty_events; + bool fillRandom_; unsigned int meansize; // in bytes unsigned int width; unsigned int injected_errors_per_million_events; diff --git a/EventFilter/Utilities/test/LocalRunBUFU.sh b/EventFilter/Utilities/test/LocalRunBUFU.sh index 156521aca2c45..8497d1cce955e 100755 --- a/EventFilter/Utilities/test/LocalRunBUFU.sh +++ b/EventFilter/Utilities/test/LocalRunBUFU.sh @@ -19,7 +19,7 @@ RC=0 P=$$ PREFIX=results_${USER}${P} OUTDIR=${LOCAL_TMP_DIR}/${PREFIX} - +echo "Output will be temporarily written to ${OUTDIR}" mkdir ${OUTDIR} cp ${SCRIPTDIR}/startBU.py ${OUTDIR} diff --git a/EventFilter/Utilities/test/startBU.py b/EventFilter/Utilities/test/startBU.py index c827a6e5d3022..c4825646fbaaa 100644 --- a/EventFilter/Utilities/test/startBU.py +++ b/EventFilter/Utilities/test/startBU.py @@ -121,6 +121,7 @@ defaultQualifier = cms.untracked.int32(0)) process.s = cms.EDProducer("DaqFakeReader", + fillRandom = cms.untracked.bool(True), meanSize = cms.untracked.uint32(options.fedMeanSize), width = cms.untracked.uint32(int(math.ceil(options.fedMeanSize/2.))), tcdsFEDID = cms.untracked.uint32(1024), From 6ee7254007d12a3a46b92645d9df03aa890e7347 Mon Sep 17 00:00:00 2001 From: Srecko Morovic Date: Thu, 9 Sep 2021 20:15:24 +0200 Subject: [PATCH 3/3] code-format --- EventFilter/Utilities/plugins/DaqFakeReader.cc | 5 +++-- EventFilter/Utilities/src/EvFDaqDirector.cc | 4 ++-- EventFilter/Utilities/src/FedRawDataInputSource.cc | 4 ++-- 3 files changed, 7 insertions(+), 6 deletions(-) diff --git a/EventFilter/Utilities/plugins/DaqFakeReader.cc b/EventFilter/Utilities/plugins/DaqFakeReader.cc index f1ed5cb8d56c4..60fd1ace513ea 100644 --- a/EventFilter/Utilities/plugins/DaqFakeReader.cc +++ b/EventFilter/Utilities/plugins/DaqFakeReader.cc @@ -48,7 +48,8 @@ DaqFakeReader::DaqFakeReader(const edm::ParameterSet& pset) << " TCDS FED ID lower than " << FEDNumbering::MINTCDSuTCAFEDID; if (fillRandom_) { //intialize random seed - auto time_count = static_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); + auto time_count = + static_cast(std::chrono::high_resolution_clock::now().time_since_epoch().count()); srand(time_count & 0xffffffff); } produces(); @@ -113,7 +114,7 @@ void DaqFakeReader::fillFEDs( if (fillRandom_) { //fill FED with random values size_t size_ui = size - size % sizeof(unsigned int); - for (size_t i=0; i < size_ui; i += sizeof(unsigned int)) { + for (size_t i = 0; i < size_ui; i += sizeof(unsigned int)) { *((unsigned int*)(feddata.data() + i)) = (unsigned int)rand(); } //remainder diff --git a/EventFilter/Utilities/src/EvFDaqDirector.cc b/EventFilter/Utilities/src/EvFDaqDirector.cc index a9a75016b913f..810db0c5c4735 100644 --- a/EventFilter/Utilities/src/EvFDaqDirector.cc +++ b/EventFilter/Utilities/src/EvFDaqDirector.cc @@ -950,7 +950,8 @@ namespace evf { bool found = (stat(fuEoLS.c_str(), &buf) == 0); if (!found) { if (doCreateEoLS) { - int eol_fd = open(fuEoLS.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); + int eol_fd = + open(fuEoLS.c_str(), O_RDWR | O_CREAT, S_IRUSR | S_IWUSR | S_IRGRP | S_IWGRP | S_IROTH | S_IWOTH); close(eol_fd); } if (doCreateBoLS) @@ -1778,7 +1779,6 @@ namespace evf { if (fileBrokerUseLocalLock_) unlockFULocal(); return noFile; - } //handle creation of BoLS files if lumisection has changed diff --git a/EventFilter/Utilities/src/FedRawDataInputSource.cc b/EventFilter/Utilities/src/FedRawDataInputSource.cc index 1da0211999e73..2854629157da7 100644 --- a/EventFilter/Utilities/src/FedRawDataInputSource.cc +++ b/EventFilter/Utilities/src/FedRawDataInputSource.cc @@ -686,7 +686,7 @@ void FedRawDataInputSource::read(edm::EventPrincipal& eventPrincipal) { return; } -edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollection& rawData, bool &tcdsInRange) { +edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollection& rawData, bool& tcdsInRange) { edm::TimeValue_t time; timeval stv; gettimeofday(&stv, nullptr); @@ -716,7 +716,7 @@ edm::Timestamp FedRawDataInputSource::fillFEDRawDataCollection(FEDRawDataCollect selectedTCDSFed = fedId; tcds_pointer_ = event + eventSize; if (fedId >= FEDNumbering::MINTCDSuTCAFEDID && fedId <= FEDNumbering::MAXTCDSuTCAFEDID) { - tcdsInRange = true; + tcdsInRange = true; } } else throw cms::Exception("FedRawDataInputSource::fillFEDRawDataCollection")