Utilities and fixes for HipPy alignment #25874
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,158 @@ | ||
| #!/usr/bin/env python | ||
|
|
||
| import argparse, contextlib, os, re, shutil, subprocess, tempfile, time | ||
|
|
||
| if __name__ == "__main__": | ||
| def abspath(path): | ||
| if not os.path.exists(path): raise ValueError(path+" does not exist") | ||
| return os.path.abspath(path) | ||
|
|
||
| p = argparse.ArgumentParser() | ||
| p.add_argument("cfgfile", type=abspath) | ||
| p.add_argument("baddatafileslist", nargs="?", default=None) | ||
| args = p.parse_args() | ||
|
|
||
| def runcfg(cfgfile, badfilelist): | ||
| try: | ||
| subprocess.check_output(["cmsRun", cfgfile], stderr=subprocess.STDOUT) | ||
| except subprocess.CalledProcessError as e: | ||
| if "FallbackFileOpenError" in e.output: | ||
| output = e.output.split("An exception of category 'FallbackFileOpenError' occurred while")[1] | ||
| filename = re.search("Failed to open the file '[^']*(/store/.*[.]root)", output).group(1) | ||
| with OneAtATime(badfilelist+".tmp", 2) as f: | ||
| with open(badfilelist) as f: | ||
| contents = set(f.read().split()) | ||
| if filename in contents: | ||
| raise RuntimeError(filename+"\nis already in\n"+badfilelist+"\n\nExiting to avoid an infinite loop. Maybe you have this running on the same cfg file multiple times?") | ||
| contents.add(filename) | ||
| contents = sorted(contents) | ||
| with open(badfilelist, "w") as f: | ||
| f.write("\n".join(contents)+"\n") | ||
| print "found and added a bad file:\n"+filename | ||
|
There was a problem hiding this comment. @hroskes could you please make this python3 compliant adding parentheses to the print statements and |
||
| else: | ||
| raise | ||
| return runcfg(cfgfile, badfilelist) | ||
| print "all files left are good" | ||
|
|
||
| @contextlib.contextmanager | ||
| def cd(newdir): | ||
| """http://stackoverflow.com/a/24176022/5228524""" | ||
| prevdir = os.getcwd() | ||
| os.chdir(os.path.expanduser(newdir)) | ||
| try: | ||
| yield | ||
| finally: | ||
| os.chdir(prevdir) | ||
|
|
||
| def cdtemp(): return cd(tempfile.mkdtemp()) | ||
|
|
||
| class KeepWhileOpenFile(object): | ||
| def __init__(self, name, message=None): | ||
| self.filename = name | ||
| if message is None: message = LSB_JOBID | ||
| self.__message = message | ||
| self.pwd = os.getcwd() | ||
| self.fd = self.f = None | ||
| self.bool = False | ||
|
|
||
| @property | ||
| def wouldbevalid(self): | ||
| if self: return True | ||
| with self: | ||
| return bool(self) | ||
|
|
||
| def __open(self): | ||
| self.fd = os.open(self.filename, os.O_CREAT | os.O_EXCL | os.O_WRONLY) | ||
|
|
||
| def __enter__(self): | ||
| with cd(self.pwd): | ||
| try: | ||
| self.__open() | ||
| except OSError: | ||
| if self.__message == LSB_JOBID: | ||
| try: | ||
| with open(self.filename) as f: | ||
| oldjobid = int(f.read()) | ||
| except IOError: | ||
| try: | ||
| self.__open() | ||
| except OSError: | ||
| return None | ||
| except ValueError: | ||
| return None | ||
| else: | ||
| if jobexists(oldjobid): | ||
| return None | ||
| else: | ||
| try: | ||
| os.remove(self.filename) | ||
| except OSError: #another job removed it already | ||
| pass | ||
| try: | ||
| self.__open() | ||
| except OSError: | ||
| return None | ||
| else: | ||
| return None | ||
|
|
||
| self.f = os.fdopen(self.fd, 'w') | ||
|
|
||
| if self.__message == LSB_JOBID: | ||
| self.__message = self.__message() | ||
| try: | ||
| if self.__message is not None: | ||
| self.f.write(self.__message+"\n") | ||
| except IOError: | ||
| pass | ||
| try: | ||
| self.f.close() | ||
| except IOError: | ||
| pass | ||
| self.bool = True | ||
| return True | ||
|
|
||
| def __exit__(self, *args): | ||
| if self: | ||
| try: | ||
| with cd(self.pwd): | ||
| os.remove(self.filename) | ||
| except OSError: | ||
| pass #ignore it | ||
| self.fd = self.f = None | ||
| self.bool = False | ||
|
|
||
| def __nonzero__(self): | ||
| return self.bool | ||
|
|
||
| class OneAtATime(KeepWhileOpenFile): | ||
| def __init__(self, name, delay, message=None, printmessage=None, task="doing this"): | ||
| super(OneAtATime, self).__init__(name, message=message) | ||
| self.delay = delay | ||
| if printmessage is None: | ||
| printmessage = "Another process is already {task}! Waiting {delay} seconds." | ||
| printmessage = printmessage.format(delay=delay, task=task) | ||
| self.__printmessage = printmessage | ||
|
|
||
| def __enter__(self): | ||
| while True: | ||
| result = super(OneAtATime, self).__enter__() | ||
| if result: | ||
| return result | ||
| print self.__printmessage | ||
| time.sleep(self.delay) | ||
|
|
||
| def LSB_JOBID(): | ||
| return os.environ.get("LSB_JOBID", None) | ||
|
|
||
| def jobexists(jobid): | ||
| return "is not found" not in subprocess.check_output(["bjobs", str(jobid)]) | ||
|
|
||
| if __name__ == "__main__": | ||
| with cdtemp(): | ||
| shutil.copy(args.cfgfile, ".") | ||
|
|
||
| badfilelist = args.badfilelist | ||
| if badfilelist is None: | ||
| badfilelist = os.path.join(os.path.dirname(cfgfile, "../../../run/DataFiles/baddatafiles.txt") | ||
|
|
||
| runcfg(os.path.basename(args.cfgfile), args.badfilelist) | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
@hroskes Could you take consistent indentation rule in the whole script?