Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Secondary input source improvements/fixes #9091

Merged
merged 1 commit into from May 16, 2015
Merged

Secondary input source improvements/fixes #9091

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
69 changes: 57 additions & 12 deletions IOPool/Input/src/RootInputFileSequence.cc
View file
Open in desktop
Expand Up @@ -58,7 +58,7 @@ namespace edm {
// yet, so the ParameterSet does not get validated yet. As soon as all the modules with a SecSource
// have defined descriptions, the defaults in the getUntrackedParameterSet function calls can
// and should be deleted from the code.
initialNumberOfEventsToSkip_(inputType == InputType::Primary ? pset.getUntrackedParameter<unsigned int>("skipEvents", 0U) : 0U),
initialNumberOfEventsToSkip_(inputType != InputType::SecondaryFile ? pset.getUntrackedParameter<unsigned int>("skipEvents", 0U) : 0U),
noEventSort_(inputType == InputType::Primary ? pset.getUntrackedParameter<bool>("noEventSort", true) : false),
skipBadFiles_(pset.getUntrackedParameter<bool>("skipBadFiles", false)),
bypassVersionCheck_(pset.getUntrackedParameter<bool>("bypassVersionCheck", false)),
Expand Down Expand Up @@ -91,8 +91,9 @@ namespace edm {
// For the secondary input source we do not stage in,
// and we randomly choose the first file to open.
// We cannot use the random number service yet.
unsigned int seed = getpid();
seed = (seed << 16) + seed;
std::ifstream f("/dev/urandom");
unsigned int seed;
f.read(reinterpret_cast<char*>(&seed), sizeof(seed));
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@bbockelm Is this how you expected it to work?

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yup - that's going to be as good as we can make things.

std::default_random_engine dre(seed);
size_t count = fileIterEnd_ - fileIterBegin_;
std::uniform_int_distribution<int> distribution(0, count - 1);
Expand Down Expand Up @@ -122,7 +123,7 @@ namespace edm {
}
if(rootFile_) {
productRegistryUpdate().updateFromInput(rootFile_->productRegistry()->productList());
if(initialNumberOfEventsToSkip_ != 0) {
if(inputType == InputType::Primary && initialNumberOfEventsToSkip_ != 0) {
skipEvents(initialNumberOfEventsToSkip_);
}
}
Expand Down Expand Up @@ -700,25 +701,47 @@ namespace edm {
productSelectorRules_ = ProductSelectorRules(pset, "inputCommands", "InputSource");
}

void
RootInputFileSequence::skipEntries(unsigned int offset) {
while(offset > 0) {
while(offset > 0 && rootFile_->nextEventEntry()) {
--offset;
}
if(offset > 0) {
++fileIter_;
if(fileIter_ == fileIterEnd_) {
fileIter_ = fileIterBegin_;
}
initFile(false);
assert(rootFile_);
rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
}
}
}

bool
RootInputFileSequence::readOneSequential(EventPrincipal& cache, size_t& fileNameHash) {
skipBadFiles_ = false;
if(fileIter_ == fileIterEnd_ || !rootFile_) {
if(firstFile_) {
firstFile_ = false;
if(fileIterEnd_ == fileIterBegin_) {
throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequential(): no input files specified for secondary input source.\n";
}
fileIter_ = fileIterBegin_;
initFile(false);
if(fileIter_ != fileIterBegin_) {
fileIter_ = fileIterBegin_;
initFile(false);
}
assert(rootFile_);
rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
skipEntries(initialNumberOfEventsToSkip_);
}
assert(rootFile_);
rootFile_->nextEventEntry();
bool found = rootFile_->readCurrentEvent(cache);
if(!found) {
++fileIter_;
if(fileIter_ == fileIterEnd_) {
return false;
fileIter_ = fileIterBegin_;
}
initFile(false);
assert(rootFile_);
Expand All @@ -731,11 +754,29 @@ namespace edm {

bool
RootInputFileSequence::readOneSequentialWithID(EventPrincipal& cache, size_t& fileNameHash, LuminosityBlockID const& id) {
if(fileIterEnd_ == fileIterBegin_) {
throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequentialWithID(): no input files specified for secondary input source.\n";
}
skipBadFiles_ = false;
if(fileIter_ == fileIterEnd_ || !rootFile_ ||
if(firstFile_) {
firstFile_ = false;
if(fileIterEnd_ == fileIterBegin_) {
throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSequential(): no input files specified for secondary input source.\n";
}
if(fileIter_ != fileIterBegin_) {
fileIter_ = fileIterBegin_;
initFile(false);
}
assert(rootFile_);
rootFile_->setAtEventEntry(IndexIntoFile::invalidEntry);
int offset = initialNumberOfEventsToSkip_;
while(offset > 0) {
bool found = readOneSequentialWithID(cache, fileNameHash, id);
if(!found) {
return false;
}
--offset;
}
}
assert(rootFile_);
if(fileIter_ == fileIterEnd_ ||
rootFile_->indexIntoFileIter().run() != id.run() ||
rootFile_->indexIntoFileIter().lumi() != id.luminosityBlock()) {
bool found = skipToItem(id.run(), id.luminosityBlock(), 0, 0, false);
Expand All @@ -761,6 +802,7 @@ namespace edm {

void
RootInputFileSequence::readOneSpecified(EventPrincipal& cache, size_t& fileNameHash, SecondaryEventIDAndFileInfo const& idx) {
firstFile_ = false;
if(fileIterEnd_ == fileIterBegin_) {
throw Exception(errors::Configuration) << "RootInputFileSequence::readOneSpecified(): no input files specified for secondary input source.\n";
}
Expand All @@ -779,10 +821,12 @@ namespace edm {
if(fileNameHash == 0U) {
fileNameHash = lfnHash_;
}
std::cerr << "Sequential event: " << cache.id() << std::endl;
}

void
RootInputFileSequence::readOneRandom(EventPrincipal& cache, size_t& fileNameHash, CLHEP::HepRandomEngine* engine) {
firstFile_ = false;
if(fileIterEnd_ == fileIterBegin_) {
throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandom(): no input files specified for secondary input source.\n";
}
Expand Down Expand Up @@ -820,6 +864,7 @@ namespace edm {

bool
RootInputFileSequence::readOneRandomWithID(EventPrincipal& cache, size_t& fileNameHash, LuminosityBlockID const& id, CLHEP::HepRandomEngine* engine) {
firstFile_ = false;
if(fileIterEnd_ == fileIterBegin_) {
throw Exception(errors::Configuration) << "RootInputFileSequence::readOneRandomWithID(): no input files specified for secondary input source.\n";
}
Expand Down
1 change: 1 addition & 0 deletions IOPool/Input/src/RootInputFileSequence.h
View file
Open in desktop
Expand Up @@ -60,6 +60,7 @@ namespace edm {
void endJob();
InputSource::ItemType getNextItemType(RunNumber_t& run, LuminosityBlockNumber_t& lumi, EventNumber_t& event);
bool containedInCurrentFile(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event) const;
void skipEntries(unsigned int offset);
bool skipEvents(int offset);
bool goToEvent(EventID const& eventID);
bool skipToItem(RunNumber_t run, LuminosityBlockNumber_t lumi, EventNumber_t event, size_t fileNameHash = 0U, bool currentFileFirst = true);
Expand Down
2 changes: 1 addition & 1 deletion IOPool/SecondaryInput/test/SecondaryProducer.cc
View file
Open in desktop
Expand Up @@ -57,7 +57,7 @@ namespace edm {
lumiSpecified_(pset.getUntrackedParameter<bool>("lumiSpecified", false)),
firstEvent_(true),
firstLoop_(true),
expectedEventNumber_(1) {
expectedEventNumber_(sequential_ ? pset.getParameterSet("input").getUntrackedParameter<unsigned int>("skipEvents", 0) + 1 : 1) {
processConfiguration_->setParameterSetID(ParameterSet::emptyParameterSetID());
processConfiguration_->setProcessConfigurationID();

Expand Down
1 change: 1 addition & 0 deletions IOPool/SecondaryInput/test/SecondarySeqInLumiInputTest_cfg.py
View file
Open in desktop
Expand Up @@ -15,6 +15,7 @@
sequential = cms.untracked.bool(True),
lumiSpecified = cms.untracked.bool(True),
input = cms.SecSource("PoolSource",
skipEvents = cms.untracked.uint32(3),
fileNames = cms.untracked.vstring('file:SecondaryInputTest2.root')
)
)
Expand Down
1 change: 1 addition & 0 deletions IOPool/SecondaryInput/test/SecondarySeqInputTest_cfg.py
View file
Open in desktop
Expand Up @@ -14,6 +14,7 @@
process.Thing = cms.EDProducer("SecondaryProducer",
sequential = cms.untracked.bool(True),
input = cms.SecSource("PoolSource",
skipEvents = cms.untracked.uint32(5),
fileNames = cms.untracked.vstring('file:SecondaryInputTest2.root')
)
)
Expand Down