Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Bugfixes and log cleanup in EventFilter/Utilities #3883

Merged
merged 7 commits into from May 15, 2014
Merged

Bugfixes and log cleanup in EventFilter/Utilities #3883

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
7c963cf
fixing hang in input source on termination via SIGINT (part1)
smorovic Apr 30, 2014
d1ee2bf
SIGINT shutdown fixes (part 2)
smorovic Apr 30, 2014
bc0e089
remove logs which are filling up log files when no input
smorovic May 7, 2014
2d48dd5
* removed debugging from json_value.cpp
smorovic May 9, 2014
20ae718
no FU-local lock file created in BU emulator mode
smorovic May 13, 2014
58c83d3
converting some logs to LogDebug
smorovic May 15, 2014
448ecbb
Merge branch 'CMSSW_7_1_X' into bugfixes
smorovic May 15, 2014
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
70 changes: 35 additions & 35 deletions EventFilter/Utilities/plugins/EvFDaqDirector.cc
View file
Open in desktop
Expand Up @@ -99,7 +99,7 @@ namespace evf {
// check if base dir exists or create it accordingly
int retval = mkdir(base_dir_.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
if (retval != 0 && errno != EEXIST) {
throw cms::Exception("DaqDirector") << " Error checking for base dir "
throw cms::Exception("DaqDirector") << " Error checking for base dir -: "
<< base_dir_ << " mkdir error:" << strerror(errno);
}

Expand All @@ -108,26 +108,26 @@ namespace evf {
retval = mkdir(run_dir_.c_str(),
S_IRWXU | S_IRWXG | S_IROTH | S_IRWXO | S_IXOTH);
if (retval != 0 && errno != EEXIST) {
throw cms::Exception("DaqDirector") << " Error creating run dir "
throw cms::Exception("DaqDirector") << " Error creating run dir -: "
<< run_dir_ << " mkdir error:" << strerror(errno);
}
//create fu-local.lock in run dir
std::string fulocal_lock_ = run_dir_+"/"+"fu-local.lock";
fulocal_rwlock_fd_ = open(fulocal_lock_.c_str(), O_RDWR | O_CREAT, S_IRWXU | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH);//O_RDWR?
if (fulocal_rwlock_fd_==-1)
throw cms::Exception("DaqDirector") << " Error creating/opening a local lock file " << fulocal_lock_.c_str() << " : " << strerror(errno);
chmod(fulocal_lock_.c_str(),0777);
fsync(fulocal_rwlock_fd_);
//open second fd for another input source thread
fulocal_rwlock_fd2_ = open(fulocal_lock_.c_str(), O_RDWR, S_IRWXU | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH);//O_RDWR?
if (fulocal_rwlock_fd2_==-1)
throw cms::Exception("DaqDirector") << " Error opening a local lock file " << fulocal_lock_.c_str() << " : " << strerror(errno);

//create fu-local.lock in run dir
if (!directorBu_) {
std::string fulocal_lock_ = run_dir_+"/"+"fu-local.lock";
fulocal_rwlock_fd_ = open(fulocal_lock_.c_str(), O_RDWR | O_CREAT, S_IRWXU | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH);//O_RDWR?
if (fulocal_rwlock_fd_==-1)
throw cms::Exception("DaqDirector") << " Error creating/opening a local lock file -: " << fulocal_lock_.c_str() << " : " << strerror(errno);
chmod(fulocal_lock_.c_str(),0777);
fsync(fulocal_rwlock_fd_);
//open second fd for another input source thread
fulocal_rwlock_fd2_ = open(fulocal_lock_.c_str(), O_RDWR, S_IRWXU | S_IWGRP | S_IRGRP | S_IWOTH | S_IROTH);//O_RDWR?
if (fulocal_rwlock_fd2_==-1)
throw cms::Exception("DaqDirector") << " Error opening a local lock file -: " << fulocal_lock_.c_str() << " : " << strerror(errno);
}

//bu_run_dir: for FU, for which the base dir is local and the BU is remote, it is expected to be there
//for BU, it is created at this point


if (directorBu_)
{
bu_run_dir_ = base_dir_ + "/" + run_string_;
Expand All @@ -139,14 +139,14 @@ namespace evf {
S_IRWXU | S_IRWXG | S_IRWXO);
if (retval != 0 && errno != EEXIST) {
throw cms::Exception("DaqDirector")
<< " Error creating bu run dir " << bu_run_dir_
<< " Error creating bu run dir -: " << bu_run_dir_
<< " mkdir error:" << strerror(errno) << "\n";
}
bu_run_open_dir_ = bu_run_dir_ + "/open";
retval = mkdir(bu_run_open_dir_.c_str(),
S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
if (retval != 0 && errno != EEXIST) {
throw cms::Exception("DaqDirector") << " Error creating bu run open dir "
throw cms::Exception("DaqDirector") << " Error creating bu run open dir -: "
<< bu_run_open_dir_ << " mkdir error:" << strerror(errno)
<< "\n";
}
Expand All @@ -155,14 +155,14 @@ namespace evf {
bu_writelock_fd_ = open(bulockfile.c_str(),
O_WRONLY | O_CREAT | O_TRUNC, S_IRWXU);
if (bu_writelock_fd_ == -1)
edm::LogWarning("EvFDaqDirector") << "problem with creating filedesc for buwritelock "
edm::LogWarning("EvFDaqDirector") << "problem with creating filedesc for buwritelock -: "
<< strerror(errno);
else
edm::LogInfo("EvFDaqDirector") << "creating filedesc for buwritelock "
edm::LogInfo("EvFDaqDirector") << "creating filedesc for buwritelock -: "
<< bu_writelock_fd_;
bu_w_lock_stream = fdopen(bu_writelock_fd_, "w");
if (bu_w_lock_stream == 0)
edm::LogWarning("EvFDaqDirector")<< "Error creating write lock stream " << strerror(errno);
edm::LogWarning("EvFDaqDirector")<< "Error creating write lock stream -: " << strerror(errno);

// BU INITIALIZES LOCK FILE
// FU LOCK FILE OPEN
Expand All @@ -177,7 +177,7 @@ namespace evf {

retval = mkdir(bu_base_dir_.c_str(), S_IRWXU | S_IRWXG | S_IROTH | S_IXOTH);
if (retval != 0 && errno != EEXIST) {
throw cms::Exception("DaqDirector") << " Error checking for bu base dir "
throw cms::Exception("DaqDirector") << " Error checking for bu base dir -: "
<< bu_base_dir_ << " mkdir error:" << strerror(errno) << "\n";
}

Expand Down Expand Up @@ -228,8 +228,8 @@ namespace evf {

// check if the requested run is the latest one - issue a warning if it isn't
if (dirManager_.findHighestRunDir() != run_dir_) {
edm::LogWarning("EvFDaqDirector") << "DaqDirector Warning checking run dir "
<< run_dir_ << " this is not the highest run "
edm::LogWarning("EvFDaqDirector") << "WARNING - checking run dir -: "
<< run_dir_ << ". This is not the highest run "
<< dirManager_.findHighestRunDir();
}
}
Expand Down Expand Up @@ -287,7 +287,7 @@ namespace evf {
void EvFDaqDirector::removeFile(std::string filename) {
int retval = remove(filename.c_str());
if (retval != 0)
edm::LogError("EvFDaqDirector") << "Could not remove used file " << filename << " error "
edm::LogError("EvFDaqDirector") << "Could not remove used file -: " << filename << ". error = "
<< strerror(errno);
}

Expand Down Expand Up @@ -357,11 +357,11 @@ namespace evf {
fileStatus = newFile;

if (testModeNoBuilderUnit_)
edm::LogInfo("EvFDaqDirector") << "Written to file: " << readLs << ":"
edm::LogInfo("EvFDaqDirector") << "Written to file -: " << readLs << ":"
<< readIndex + 1 << " --> " << readLs + 2
<< ":" << readIndex + 1;
else
edm::LogInfo("EvFDaqDirector") << "Written to file: " << readLs << ":"
LogDebug("EvFDaqDirector") << "Written to file -: " << readLs << ":"
<< readIndex + 1;

} else
Expand Down Expand Up @@ -390,12 +390,12 @@ namespace evf {
if (retvalu==-1) edm::LogError("EvFDaqDirector") << "Error unlocking the fu.lock " << strerror(errno);

#ifdef DEBUG
edm::LogInfo("EvFDaqDirector") << "Waited during lock:" << locked_period;
edm::LogDebug("EvFDaqDirector") << "Waited during lock -: " << locked_period << " seconds";
#endif

if ( fileStatus == noFile ) {
struct stat buf;
edm::LogInfo("EvFDaqDirector") << " looking for EoR file: " << getEoRFilePath().c_str();
//edm::LogInfo("EvFDaqDirector") << " looking for EoR file: " << getEoRFilePath().c_str();
if ( stat(getEoRFilePath().c_str(), &buf) == 0 )
fileStatus = runEnded;
}
Expand All @@ -410,7 +410,7 @@ namespace evf {
try {
fms_ = (FastMonitoringService *) (edm::Service<evf::MicroStateService>().operator->());
} catch (...) {
edm::LogError("EvFDaqDirector") <<" FastMonitoringService not found ";
edm::LogError("EvFDaqDirector") <<" FastMonitoringService not found";
}
}
if (fms_) fms_->accumulateFileSize(ls, previousFileSize_);
Expand Down Expand Up @@ -457,10 +457,10 @@ namespace evf {
std::string destEol = getEoLSFilePathOnBU(startingLumi+2);

std::string cpCmd = "cp " + sourceEol + " " + destEol;
edm::LogInfo("EvFDaqDirector") << " testmode: Running copy cmd = " << cpCmd;
edm::LogInfo("EvFDaqDirector") << " testmode: Running copy cmd -: " << cpCmd;
int rc = system(cpCmd.c_str());
if (rc != 0) {
edm::LogError("EvFDaqDirector") << " testmode: COPY EOL FAILED!!!!!: " << cpCmd;
edm::LogError("EvFDaqDirector") << " testmode: COPY EOL FAILED!!!!! -: " << cpCmd;
}
}

Expand Down Expand Up @@ -497,10 +497,10 @@ namespace evf {
fu_readwritelock_fd_ = open(fulockfile.c_str(), O_RDWR, S_IRWXU);
}
if (fu_readwritelock_fd_ == -1)
edm::LogError("EvFDaqDirector") << "problem with creating filedesc for fuwritelock "
<< strerror(errno);
edm::LogError("EvFDaqDirector") << "problem with creating filedesc for fuwritelock -: " << fulockfile.c_str()
<< " create:" << create << " error:" << strerror(errno);
else
edm::LogInfo("EvFDaqDirector") << "creating filedesc for fureadwritelock "
LogDebug("EvFDaqDirector") << "creating filedesc for fureadwritelock -: "
<< fu_readwritelock_fd_;

fu_rw_lock_stream = fdopen(fu_readwritelock_fd_, "r+");
Expand All @@ -514,7 +514,7 @@ namespace evf {
edm::LogError("EvFDaqDirector") << "problem with creating filedesc for datamerge "
<< strerror(errno);
else
edm::LogInfo("EvFDaqDirector") << "creating filedesc for datamerge "
LogDebug("EvFDaqDirector") << "creating filedesc for datamerge -: "
<< data_readwrite_fd_;
fcntl(data_readwrite_fd_, F_SETLKW, &data_rw_flk);

Expand Down
36 changes: 21 additions & 15 deletions EventFilter/Utilities/plugins/FastMonitoringService.cc
View file
Open in desktop
Expand Up @@ -13,6 +13,7 @@
#include "EventFilter/Utilities/plugins/EvFDaqDirector.h"
#include "EventFilter/Utilities/interface/FileIO.h"
#include "FWCore/MessageLogger/interface/MessageLogger.h"
#include "FWCore/Utilities/interface/UnixSignalHandlers.h"

#include "FWCore/ServiceRegistry/interface/ModuleCallingContext.h"
#include "DataFormats/Provenance/interface/ModuleDescription.h"
Expand Down Expand Up @@ -68,7 +69,6 @@ namespace evf{
reg.watchPreModuleEvent(this,&FastMonitoringService::preModuleEvent);//should be stream
reg.watchPostModuleEvent(this,&FastMonitoringService::postModuleEvent);//

edm::LogInfo("FastMonitoringService") << "Constructed";
}


Expand Down Expand Up @@ -105,18 +105,18 @@ namespace evf{

if (edm::Service<evf::EvFDaqDirector>().operator->()==nullptr)
{
throw cms::Exception("FastMonitoringService") << "ERROR: EvFDaqDirector is not present";
throw cms::Exception("FastMonitoringService") << "EvFDaqDirector is not present";

}
boost::filesystem::path runDirectory(edm::Service<evf::EvFDaqDirector>()->findCurrentRunDir());
workingDirectory_ = runDirectory_ = runDirectory;
workingDirectory_ /= "mon";

if ( !boost::filesystem::is_directory(workingDirectory_)) {
edm::LogInfo("FastMonitoringService") << "<MON> DIR NOT FOUND! Trying to create " << workingDirectory_.string() ;
LogDebug("FastMonitoringService") << "<MON> DIR NOT FOUND! Trying to create -: " << workingDirectory_.string() ;
boost::filesystem::create_directories(workingDirectory_);
if ( !boost::filesystem::is_directory(workingDirectory_))
edm::LogWarning("FastMonitoringService") << "Unable to create <MON> DIR " << workingDirectory_.string()
edm::LogWarning("FastMonitoringService") << "Unable to create <MON> DIR -: " << workingDirectory_.string()
<< ". No monitoring data will be written.";
}

Expand All @@ -135,8 +135,8 @@ namespace evf{
pathLegFile << "pathlegend_pid" << std::setfill('0') << std::setw(5) << getpid() << ".leg";
pathLegendFile_ = (workingDirectory_/pathLegFile.str()).string();

edm::LogInfo("FastMonitoringService") << "preallocate: initializing FastMonitor with microstate def path: "
<< microstateDefPath_ << " " << FastMonitoringThread::MCOUNT << " ";
LogDebug("FastMonitoringService") << "Initializing FastMonitor with microstate def path -: "
<< microstateDefPath_;
//<< encPath_.current_ + 1 << " " << encModule_.current_ + 1

nStreams_=bounds.maxNumberOfStreams();
Expand Down Expand Up @@ -251,7 +251,6 @@ namespace evf{

void FastMonitoringService::preGlobalBeginLumi(edm::GlobalContext const& gc)
{
edm::LogInfo("FastMonitoringService") << gc.luminosityBlockID().luminosityBlock();

timeval lumiStartTime;
gettimeofday(&lumiStartTime, 0);
Expand All @@ -277,8 +276,8 @@ namespace evf{
void FastMonitoringService::preGlobalEndLumi(edm::GlobalContext const& gc)
{
unsigned int lumi = gc.luminosityBlockID().luminosityBlock();
edm::LogInfo("FastMonitoringService") << "FastMonitoringService: LUMI: "
<< lumi << " ended! Writing JSON information...";
LogDebug("FastMonitoringService") << "Lumi ended. Writing JSON information. LUMI -: "
<< lumi;
timeval lumiStopTime;
gettimeofday(&lumiStopTime, 0);

Expand Down Expand Up @@ -313,18 +312,27 @@ namespace evf{
{
auto itr = sourceEventsReport_.find(lumi);
if (itr==sourceEventsReport_.end()) {
throw cms::Exception("FastMonitoringService") << "ERROR: SOURCE did not send update for lumi block " << lumi;
//do not throw exception in case of signal termination
if (edm::shutdown_flag) {
edm::LogInfo("FastMonitoringService") << "Run interrupted. Skip writing EoL information -: "
<< processedEventsPerLumi_[lumi] << " events were processed in LUMI " << lumi;
//this will prevent output modules from producing json file for possibly incomplete lumi
processedEventsPerLumi_[lumi];
return;
}
throw cms::Exception("FastMonitoringService") << "SOURCE did not send update for lumi block. LUMI -:" << lumi;
}
else {
if (itr->second!=processedEventsPerLumi_[lumi]) {
throw cms::Exception("FastMonitoringService") << " ERROR: MISMATCH with SOURCE update for lumi block" << lumi
throw cms::Exception("FastMonitoringService") << "MISMATCH with SOURCE update. LUMI -: "
<< lumi
<< ", events(processed):" << processedEventsPerLumi_[lumi]
<< " events(source):" << itr->second;
}
sourceEventsReport_.erase(itr);
}
}
edm::LogInfo("FastMonitoringService") << ">>> >>> FastMonitoringService: processed event count for this lumi = "
edm::LogInfo("FastMonitoringService") << "Statistics for lumisection -: lumi = " << lumi << " events = "
<< lumiProcessedJptr->value() << " time = " << usecondsForLumi/1000000
<< " size = " << accuSize << " thr = " << throughput;
delete lumiProcessedJptr;
Expand All @@ -338,8 +346,6 @@ namespace evf{

void FastMonitoringService::postGlobalEndLumi(edm::GlobalContext const& gc)
{
edm::LogInfo("FastMonitoringService") << "FastMonitoringService: Post-global-end LUMI: "
<< gc.luminosityBlockID().luminosityBlock();
//mark closed lumis (still keep map entries until next one)
lastGlobalLumisClosed_.push(gc.luminosityBlockID().luminosityBlock());
}
Expand Down Expand Up @@ -542,7 +548,7 @@ namespace evf{
return proc;
}
else {
throw cms::Exception("FastMonitoringService") << "ERROR: output module wants deleted info ";
throw cms::Exception("FastMonitoringService") << "output module wants already deleted LS event count for LUMI -: "<<lumi;
return 0;
}
}
Expand Down