Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Monitoring JSON changes in DAQ #12136

Merged
merged 3 commits into from Nov 3, 2015
Merged

Monitoring JSON changes in DAQ #12136

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f427b6b
new (json) format for module and path legend json file
smorovic Sep 4, 2015
134d1bc
outputmodules entry in json legend
smorovic Sep 28, 2015
b9ccbe9
*adding HLTErrorEvents field to output JSON (to be used for filling S…
smorovic Oct 26, 2015
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
4 changes: 3 additions & 1 deletion DQMServices/Components/src/DQMFileSaver.cc
View file
Open in desktop
Expand Up @@ -291,7 +291,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, c
}
// The availability test of the FastMonitoringService was done in the ctor.
bpt::ptree data;
bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32, transferDestination;
bpt::ptree processedEvents, acceptedEvents, errorEvents, bitmask, fileList, fileSize, inputFiles, fileAdler32, transferDestination, hltErrorEvents;

processedEvents.put("", nProcessed); // Processed events
acceptedEvents.put("", nProcessed); // Accepted events, same as processed for our purposes
Expand All @@ -303,6 +303,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, c
inputFiles.put("", ""); // We do not care about input files!
fileAdler32.put("", -1); // placeholder to match output json definition
transferDestination.put("", transferDestinationStr); // SM Transfer destination field
hltErrorEvents.put("", 0); // Error events

data.push_back(std::make_pair("", processedEvents));
data.push_back(std::make_pair("", acceptedEvents));
Expand All @@ -313,6 +314,7 @@ DQMFileSaver::fillJson(int run, int lumi, const std::string& dataFilePathName, c
data.push_back(std::make_pair("", inputFiles));
data.push_back(std::make_pair("", fileAdler32));
data.push_back(std::make_pair("", transferDestination));
data.push_back(std::make_pair("", hltErrorEvents));

pt.add_child("data", data);

Expand Down
7 changes: 5 additions & 2 deletions EventFilter/Utilities/interface/FastMonitoringService.h
View file
Open in desktop
Expand Up @@ -112,8 +112,8 @@ namespace evf{
FastMonitoringService(const edm::ParameterSet&,edm::ActivityRegistry&);
~FastMonitoringService();

std::string makePathLegenda();
std::string makeModuleLegenda();
std::string makePathLegendaJson();
std::string makeModuleLegendaJson();

void preallocate(edm::service::SystemBounds const&);
void jobFailure();
Expand Down Expand Up @@ -258,8 +258,11 @@ namespace evf{
std::atomic<unsigned long> totalEventsProcessed_;

std::string moduleLegendFile_;
std::string moduleLegendFileJson_;
std::string pathLegendFile_;
std::string pathLegendFileJson_;
bool pathLegendWritten_ = false;
unsigned int nOutputModules_ =0;

std::atomic<bool> monInit_;
bool exception_detected_ = false;
Expand Down
5 changes: 5 additions & 0 deletions EventFilter/Utilities/plugins/RecoEventOutputModuleForFU.h
View file
Open in desktop
Expand Up @@ -61,6 +61,7 @@ namespace evf {
StringJ inputFiles_;
IntJ fileAdler32_;
StringJ transferDestination_;
IntJ hltErrorEvents_;
boost::shared_ptr<FastMonitor> jsonMonitor_;
evf::FastMonitoringService *fms_;
DataPointDefinition outJsonDef_;
Expand All @@ -85,6 +86,7 @@ namespace evf {
inputFiles_(),
fileAdler32_(1),
transferDestination_(),
hltErrorEvents_(0),
outBuf_(new unsigned char[1024*1024])
{
std::string baseRunDir = edm::Service<evf::EvFDaqDirector>()->baseRunDir();
Expand Down Expand Up @@ -122,6 +124,7 @@ namespace evf {
inputFiles_.setName("InputFiles");
fileAdler32_.setName("FileAdler32");
transferDestination_.setName("TransferDestination");
hltErrorEvents_.setName("HLTErrorEvents");

outJsonDef_.setDefaultGroup("data");
outJsonDef_.addLegendItem("Processed","integer",DataPointDefinition::SUM);
Expand All @@ -133,6 +136,7 @@ namespace evf {
outJsonDef_.addLegendItem("InputFiles","string",DataPointDefinition::CAT);
outJsonDef_.addLegendItem("FileAdler32","integer",DataPointDefinition::ADLER32);
outJsonDef_.addLegendItem("TransferDestination","string",DataPointDefinition::SAME);
outJsonDef_.addLegendItem("HLTErrorEvents","integer",DataPointDefinition::SUM);
std::stringstream tmpss,ss;
tmpss << baseRunDir << "/open/" << "output_" << getpid() << ".jsd";
ss << baseRunDir << "/" << "output_" << getpid() << ".jsd";
Expand Down Expand Up @@ -161,6 +165,7 @@ namespace evf {
jsonMonitor_->registerGlobalMonitorable(&inputFiles_,false);
jsonMonitor_->registerGlobalMonitorable(&fileAdler32_,false);
jsonMonitor_->registerGlobalMonitorable(&transferDestination_,false);
jsonMonitor_->registerGlobalMonitorable(&hltErrorEvents_,false);
jsonMonitor_->commit(nullptr);

}
Expand Down
5 changes: 5 additions & 0 deletions EventFilter/Utilities/plugins/output.jsd
View file
Open in desktop
Expand Up @@ -44,6 +44,11 @@
"name" : "TransferDestination",
"operation" : "same",
"type" : "string"
},
{
"name" : "HLTErrorEvents",
"operation" : "sum",
"type" : "integer"
}

]
Expand Down
62 changes: 41 additions & 21 deletions EventFilter/Utilities/src/FastMonitoringService.cc
View file
Open in desktop
Expand Up @@ -20,6 +20,10 @@

constexpr double throughputFactor() {return (1000000)/double(1024*1024);}

#define NRESERVEDMODULES 33
#define NSPECIALMODULES 7
#define NRESERVEDPATHS 1

namespace evf{

const std::string FastMonitoringService::macroStateNames[FastMonitoringThread::MCOUNT] =
Expand All @@ -32,7 +36,7 @@ namespace evf{
FastMonitoringService::FastMonitoringService(const edm::ParameterSet& iPS,
edm::ActivityRegistry& reg) :
MicroStateService(iPS,reg)
,encModule_(33)
,encModule_(NRESERVEDMODULES)
,nStreams_(0)//until initialized
,sleepTime_(iPS.getUntrackedParameter<int>("sleepTime", 1))
,fastMonIntervals_(iPS.getUntrackedParameter<unsigned int>("fastMonIntervals", 1))
Expand Down Expand Up @@ -80,26 +84,34 @@ namespace evf{
}


std::string FastMonitoringService::makePathLegenda(){
//taken from first stream
std::ostringstream ost;
for(int i = 0;
i < encPath_[0].current_;
i++)
ost<<i<<"="<<*((std::string *)(encPath_[0].decode(i)))<<" ";
return ost.str();
std::string FastMonitoringService::makePathLegendaJson() {
Json::Value legendaVector(Json::arrayValue);
for(int i = 0; i < encPath_[0].current_; i++)
legendaVector.append(Json::Value(*((std::string *)(encPath_[0].decode(i)))));
Json::Value valReserved(NRESERVEDPATHS);
Json::Value pathLegend;
pathLegend["names"]=legendaVector;
pathLegend["reserved"]=valReserved;
Json::StyledWriter writer;
return writer.write(pathLegend);
}


std::string FastMonitoringService::makeModuleLegenda()
{
std::ostringstream ost;
std::string FastMonitoringService::makeModuleLegendaJson(){
Json::Value legendaVector(Json::arrayValue);
for(int i = 0; i < encModule_.current_; i++)
ost<<i<<"="<<((const edm::ModuleDescription *)(encModule_.decode(i)))->moduleLabel()<<" ";
return ost.str();
legendaVector.append(Json::Value(((const edm::ModuleDescription *)(encModule_.decode(i)))->moduleLabel()));
Json::Value valReserved(NRESERVEDMODULES);
Json::Value valSpecial(NSPECIALMODULES);
Json::Value valOutputModules(nOutputModules_);
Json::Value moduleLegend;
moduleLegend["names"]=legendaVector;
moduleLegend["reserved"]=valReserved;
moduleLegend["special"]=valSpecial;
moduleLegend["output"]=valOutputModules;
Json::StyledWriter writer;
return writer.write(moduleLegend);
}


void FastMonitoringService::preallocate(edm::service::SystemBounds const & bounds)
{

Expand Down Expand Up @@ -132,12 +144,18 @@ namespace evf{
fastPath_ = fast.string();

std::ostringstream moduleLegFile;
std::ostringstream moduleLegFileJson;
moduleLegFile << "microstatelegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".leg";
moduleLegFileJson << "microstatelegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".jsn";
moduleLegendFile_ = (workingDirectory_/moduleLegFile.str()).string();
moduleLegendFileJson_ = (workingDirectory_/moduleLegFileJson.str()).string();

std::ostringstream pathLegFile;
std::ostringstream pathLegFileJson;
pathLegFile << "pathlegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".leg";
pathLegendFile_ = (workingDirectory_/pathLegFile.str()).string();
pathLegFileJson << "pathlegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".jsn";
pathLegendFileJson_ = (workingDirectory_/pathLegFileJson.str()).string();

LogDebug("FastMonitoringService") << "Initializing FastMonitor with microstate def path -: "
<< microstateDefPath_;
Expand Down Expand Up @@ -266,16 +284,18 @@ namespace evf{
//build a map of modules keyed by their module description address
//here we need to treat output modules in a special way so they can be easily singled out
if(desc.moduleName() == "Stream" || desc.moduleName() == "ShmStreamConsumer" || desc.moduleName() == "EvFOutputModule" ||
desc.moduleName() == "EventStreamFileWriter" || desc.moduleName() == "PoolOutputModule")
desc.moduleName() == "EventStreamFileWriter" || desc.moduleName() == "PoolOutputModule") {
encModule_.updateReserved((void*)&desc);
nOutputModules_++;
}
else
encModule_.update((void*)&desc);
}

void FastMonitoringService::postBeginJob()
{
std::string && moduleLegStr = makeModuleLegenda();
FileIO::writeStringToFile(moduleLegendFile_, moduleLegStr);
std::string && moduleLegStrJson = makeModuleLegendaJson();
FileIO::writeStringToFile(moduleLegendFileJson_, moduleLegStrJson);

macrostate_ = FastMonitoringThread::sJobReady;

Expand Down Expand Up @@ -464,8 +484,8 @@ namespace evf{
collectedPathList_[sc.streamID()]->store(true,std::memory_order_seq_cst);
fmt_.m_data.ministateBins_=encPath_[sc.streamID()].vecsize();
if (!pathLegendWritten_) {
std::string pathLegendStr = makePathLegenda();
FileIO::writeStringToFile(pathLegendFile_, pathLegendStr);
std::string pathLegendStrJson = makePathLegendaJson();
FileIO::writeStringToFile(pathLegendFileJson_, pathLegendStrJson);
pathLegendWritten_=true;
}
}
Expand Down
3 changes: 3 additions & 0 deletions HLTrigger/JSONMonitoring/plugins/TriggerJSONMonitoring.cc
View file
Open in desktop
Expand Up @@ -649,6 +649,7 @@ TriggerJSONMonitoring::globalEndLuminosityBlockSummary(const edm::LuminosityBloc
IntJ daqJsnAccepted = daqJsnProcessed;
IntJ daqJsnErrorEvents = 0;
IntJ daqJsnRetCodeMask = 0;
IntJ daqJsnHLTErrorEvents = 0;

//write out HLT metadata jsn
Json::Value hltDaqJsn;
Expand All @@ -664,6 +665,7 @@ TriggerJSONMonitoring::globalEndLuminosityBlockSummary(const edm::LuminosityBloc
hltDaqJsn[DataPoint::DATA].append(hltJsnInputFiles.value());
hltDaqJsn[DataPoint::DATA].append(hltJsnFileAdler32);
hltDaqJsn[DataPoint::DATA].append(iSummary->streamHLTDestination);
hltDaqJsn[DataPoint::DATA].append((unsigned int)daqJsnHLTErrorEvents.value());

result = writer.write(hltDaqJsn);

Expand All @@ -689,6 +691,7 @@ TriggerJSONMonitoring::globalEndLuminosityBlockSummary(const edm::LuminosityBloc
l1DaqJsn[DataPoint::DATA].append(l1JsnInputFiles.value());
l1DaqJsn[DataPoint::DATA].append(l1JsnFileAdler32);
l1DaqJsn[DataPoint::DATA].append(iSummary->streamL1Destination);
l1DaqJsn[DataPoint::DATA].append((unsigned int)daqJsnHLTErrorEvents.value());

result = writer.write(l1DaqJsn);

Expand Down