Merge pull request #2406 from deguio/skipBadFiles_FwkIO_elegant_70X

DQMServices -- skipBadFiles flag implemented

DQMServices/FwkIO/plugins/DQMRootSource.cc

@@ -359,7 +359,7 @@ class DQMRootSource : public edm::InputSource
       void logFileAction(char const* msg, char const* fileName) const;
 
       void readNextItemType();
-      void setupFile(unsigned int iIndex);
+      bool setupFile(unsigned int iIndex);
       void readElements();
 
       const DQMRootSource& operator=(const DQMRootSource&); // stop default
@@ -386,6 +386,7 @@ class DQMRootSource : public edm::InputSource
       unsigned int m_lastSeenRun2;
       unsigned int m_lastSeenLumi2;
       unsigned int m_filterOnRun;
+      bool m_skipBadFiles;
       bool m_justOpenedFileSoNeedToGenerateRunTransition;
       bool m_shouldReadMEs;
       std::set<MonitorElement*> m_lumiElements;
@@ -411,6 +412,8 @@ DQMRootSource::fillDescriptions(edm::ConfigurationDescriptions& descriptions) {
     ->setComment("Names of files to be processed.");
   desc.addUntracked<unsigned int>("filterOnRun",0)
     ->setComment("Just limit the process to the selected run.");
+  desc.addUntracked<bool>("skipBadFiles",false)
+    ->setComment("Skip the file is it is not valid");
   desc.addUntracked<std::string>("overrideCatalog",std::string())
     ->setComment("An alternate file catalog to use instead of the standard site one.");
   descriptions.addDefault(desc);
@@ -433,6 +436,7 @@ DQMRootSource::DQMRootSource(edm::ParameterSet const& iPSet, const edm::InputSou
   m_lastSeenRun2(0),
   m_lastSeenLumi2(0),
   m_filterOnRun(iPSet.getUntrackedParameter<unsigned int>("filterOnRun", 0)),
+  m_skipBadFiles(iPSet.getUntrackedParameter<bool>("skipBadFiles", false)),
   m_justOpenedFileSoNeedToGenerateRunTransition(false),
   m_shouldReadMEs(true)
 {
@@ -620,8 +624,15 @@ DQMRootSource::readLuminosityBlock_( edm::LuminosityBlockPrincipal& lbCache)
 std::unique_ptr<edm::FileBlock>
 DQMRootSource::readFile_() {
   //std::cout <<"readFile_"<<std::endl;
-  setupFile(m_fileIndex);
-  ++m_fileIndex;
+  auto const numFiles = m_catalog.fileNames().size();
+  while(m_fileIndex < numFiles && not setupFile(m_fileIndex++)) {}
+
+  if(m_file.get() == nullptr) {
+    //last file in list was bad
+    m_nextItemType = edm::InputSource::IsStop;
+    return std::unique_ptr<edm::FileBlock>(new edm::FileBlock);
+  }
+
   readNextItemType();
 
   edm::Service<edm::JobReport> jr;
@@ -640,6 +651,7 @@ DQMRootSource::readFile_() {
 
 void
 DQMRootSource::closeFile_() {
+  if(m_file.get()==nullptr) { return; }
   edm::Service<edm::JobReport> jr;
   jr->inputFileClosed(edm::InputType::Primary, m_jrToken);
 }
@@ -745,7 +757,7 @@ void DQMRootSource::readNextItemType()
   }
 }
 
-void 
+bool
 DQMRootSource::setupFile(unsigned int iIndex)
 {
   if(m_file.get() != 0 && iIndex > 0) {
@@ -754,37 +766,49 @@ DQMRootSource::setupFile(unsigned int iIndex)
   }
   logFileAction("  Initiating request to open file ", m_catalog.fileNames()[iIndex].c_str());
   m_presentlyOpenFileIndex = iIndex;
+  m_file.reset();
+  std::auto_ptr<TFile> newFile;
   try {
-    m_file = std::auto_ptr<TFile>(TFile::Open(m_catalog.fileNames()[iIndex].c_str()));
+    newFile = std::auto_ptr<TFile>(TFile::Open(m_catalog.fileNames()[iIndex].c_str()));
   } catch(cms::Exception const& e) {
-    edm::Exception ex(edm::errors::FileOpenError,"",e);
-    ex.addContext("Opening DQM Root file");
-    ex <<"\nInput file " << m_catalog.fileNames()[iIndex] << " was not found, could not be opened, or is corrupted.\n";
-    throw ex;
+    if(!m_skipBadFiles) {
+      edm::Exception ex(edm::errors::FileOpenError,"",e);
+      ex.addContext("Opening DQM Root file");
+      ex <<"\nInput file " << m_catalog.fileNames()[iIndex] << " was not found, could not be opened, or is corrupted.\n";
+      throw ex;
+    }
+    return 0;
   }
-  if(not m_file->IsZombie()) {  
+  if(not newFile->IsZombie()) {  
     logFileAction("  Successfully opened file ", m_catalog.fileNames()[iIndex].c_str());
   } else {
-    edm::Exception ex(edm::errors::FileOpenError);
-    ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" could not be opened.\n";
-    ex.addContext("Opening DQM Root file");
-    throw ex;
+    if(!m_skipBadFiles) {
+      edm::Exception ex(edm::errors::FileOpenError);
+      ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" could not be opened.\n";
+      ex.addContext("Opening DQM Root file");
+      throw ex;
+    }
+    return 0;
   }
   //Check file format version, which is encoded in the Title of the TFile
-  if(0 != strcmp(m_file->GetTitle(),"1")) {
+  if(0 != strcmp(newFile->GetTitle(),"1")) {
     edm::Exception ex(edm::errors::FileReadError);
     ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" does not appear to be a DQM Root file.\n";
   }
 
   //Get meta Data
-  TDirectory* metaDir = m_file->GetDirectory(kMetaDataDirectoryAbsolute);
+  TDirectory* metaDir = newFile->GetDirectory(kMetaDataDirectoryAbsolute);
   if(0==metaDir) {
-    edm::Exception ex(edm::errors::FileReadError);
-    ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" appears to be corrupted since it does not have the proper internal structure.\n"
-      " Check to see if the file was closed properly.\n";    
-    ex.addContext("Opening DQM Root file");
-    throw ex;    
+    if(!m_skipBadFiles) {
+      edm::Exception ex(edm::errors::FileReadError);
+      ex<<"Input file "<<m_catalog.fileNames()[iIndex].c_str() <<" appears to be corrupted since it does not have the proper internal structure.\n"
+	" Check to see if the file was closed properly.\n";    
+      ex.addContext("Opening DQM Root file");
+      throw ex;    
+    }
+    else {return 0;}
   }
+  m_file = newFile; //passed all tests so now we want to use this file
   TTree* parameterSetTree = dynamic_cast<TTree*>(metaDir->Get(kParameterSetTree));
   assert(0!=parameterSetTree);
 
@@ -970,6 +994,8 @@ DQMRootSource::setupFile(unsigned int iIndex)
   }
   //After a file open, the framework expects to see a new 'IsRun'
   m_justOpenedFileSoNeedToGenerateRunTransition=true;
+
+  return 1;
 }
 
 void