New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Utilities and fixes for HipPy alignment #25874
Merged
Merged
Changes from all commits
Commits
Show all changes
8 commits
Select commit
Hold shift + click to select a range
81a3470
fix default value
hroskes 188898f
fix error handling
hroskes 041cac9
easy mechanism for excluding bad files
hroskes 8cb730e
fix exclusion of all rereco files
hroskes 0d1ea98
fix typo
hroskes 4adf40b
__file__ doesn't exist in cmsRun
hroskes 46b6201
indentation and get rid of LSF stuff
hroskes c1bac5e
from __future__ import print_function
hroskes File filter
Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
127 changes: 127 additions & 0 deletions
127
Alignment/HIPAlignmentAlgorithm/scripts/hippyaddtobaddatafiles.py
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Original file line number | Diff line number | Diff line change |
---|---|---|
@@ -0,0 +1,127 @@ | ||
#!/usr/bin/env python | ||
|
||
from __future__ import print_function | ||
|
||
import argparse, contextlib, os, re, shutil, subprocess, tempfile, time | ||
|
||
if __name__ == "__main__": | ||
def abspath(path): | ||
if not os.path.exists(path): raise ValueError(path+" does not exist") | ||
return os.path.abspath(path) | ||
|
||
p = argparse.ArgumentParser() | ||
p.add_argument("cfgfile", type=abspath) | ||
p.add_argument("baddatafileslist", nargs="?", default=None) | ||
args = p.parse_args() | ||
|
||
def runcfg(cfgfile, badfilelist): | ||
try: | ||
subprocess.check_output(["cmsRun", cfgfile], stderr=subprocess.STDOUT) | ||
except subprocess.CalledProcessError as e: | ||
if "FallbackFileOpenError" in e.output: | ||
output = e.output.split("An exception of category 'FallbackFileOpenError' occurred while")[1] | ||
filename = re.search("Failed to open the file '[^']*(/store/.*[.]root)", output).group(1) | ||
with OneAtATime(badfilelist+".tmp", 2) as f: | ||
with open(badfilelist) as f: | ||
contents = set(f.read().split()) | ||
if filename in contents: | ||
raise RuntimeError(filename+"\nis already in\n"+badfilelist+"\n\nExiting to avoid an infinite loop. Maybe you have this running on the same cfg file multiple times?") | ||
contents.add(filename) | ||
contents = sorted(contents) | ||
with open(badfilelist, "w") as f: | ||
f.write("\n".join(contents)+"\n") | ||
print("found and added a bad file:\n"+filename) | ||
else: | ||
raise | ||
return runcfg(cfgfile, badfilelist) | ||
print("all files left are good") | ||
|
||
@contextlib.contextmanager | ||
def cd(newdir): | ||
"""http://stackoverflow.com/a/24176022/5228524""" | ||
prevdir = os.getcwd() | ||
os.chdir(os.path.expanduser(newdir)) | ||
try: | ||
yield | ||
finally: | ||
os.chdir(prevdir) | ||
|
||
def cdtemp(): return cd(tempfile.mkdtemp()) | ||
|
||
class KeepWhileOpenFile(object): | ||
def __init__(self, name, message=None): | ||
self.filename = name | ||
self.__message = message | ||
self.pwd = os.getcwd() | ||
self.fd = self.f = None | ||
self.bool = False | ||
|
||
@property | ||
def wouldbevalid(self): | ||
if self: return True | ||
with self: | ||
return bool(self) | ||
|
||
def __open(self): | ||
self.fd = os.open(self.filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY) | ||
|
||
def __enter__(self): | ||
with cd(self.pwd): | ||
try: | ||
self.__open() | ||
except OSError: | ||
return None | ||
|
||
self.f = os.fdopen(self.fd, 'w') | ||
|
||
try: | ||
if self.__message is not None: | ||
self.f.write(self.__message+"\n") | ||
except IOError: | ||
pass | ||
try: | ||
self.f.close() | ||
except IOError: | ||
pass | ||
self.bool = True | ||
return True | ||
|
||
def __exit__(self, *args): | ||
if self: | ||
try: | ||
with cd(self.pwd): | ||
os.remove(self.filename) | ||
except OSError: | ||
pass #ignore it | ||
self.fd = self.f = None | ||
self.bool = False | ||
|
||
def __nonzero__(self): | ||
return self.bool | ||
|
||
class OneAtATime(KeepWhileOpenFile): | ||
def __init__(self, name, delay, message=None, printmessage=None, task="doing this"): | ||
super(OneAtATime, self).__init__(name, message=message) | ||
self.delay = delay | ||
if printmessage is None: | ||
printmessage = "Another process is already {task}! Waiting {delay} seconds." | ||
printmessage = printmessage.format(delay=delay, task=task) | ||
self.__printmessage = printmessage | ||
|
||
def __enter__(self): | ||
while True: | ||
result = super(OneAtATime, self).__enter__() | ||
if result: | ||
return result | ||
print(self.__printmessage) | ||
time.sleep(self.delay) | ||
|
||
if __name__ == "__main__": | ||
with cdtemp(): | ||
shutil.copy(args.cfgfile, ".") | ||
|
||
badfilelist = args.badfilelist | ||
if badfilelist is None: | ||
badfilelist = os.path.join(os.path.dirname(cfgfile, "../../../run/DataFiles/baddatafiles.txt")) | ||
|
||
runcfg(os.path.basename(args.cfgfile), args.badfilelist) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@hroskes Could you take consistent indentation rule in the whole script?