Permalink
Browse files

Merge remote-tracking branch 'origin/master'

Made creation of output file optional
(please see bigjob_agent.conf for details)

Conflicts:
	bigjob/bigjob_manager.py
  • Loading branch information...
2 parents 8b3e8f0 + 17e0805 commit 0cb5ca6901ccf93b639273701a38f5484aa7477a @drelu drelu committed Apr 28, 2012
Showing with 45 additions and 15 deletions.
  1. +7 −5 README.md
  2. +13 −5 bigjob/bigjob_agent.py
  3. +15 −4 bigjob/bigjob_manager.py
  4. +5 −1 bigjob/pbsssh.py
  5. +5 −0 bigjob_agent.conf
View
@@ -12,16 +12,18 @@ Web Page & Mailing List
Web page: https://github.com/saga-project/BigJob/wiki
-Mailing list: Use https://mail.cct.lsu.edu/mailman/listinfo/bigjob-users to subscribe to bigjob-users mailing list. The mailing list archives are located at: https://mail.cct.lsu.edu/pipermail/bigjob-users/.
+Mailing list:
+bigjob-user@googlegroups.com: http://groups.google.com/group/bigjob-users
+bigjob-devel@googlegroups.com: http://groups.google.com/group/bigjob-devel
Installation
-------------
-$ curl https://svn.cct.lsu.edu/repos/saga-projects/applications/bigjob/trunk/generic/bootstrap/bigjob-bootstrap.py -o bigjob-bootstrap.py
-$ python bigjob-bootstrap.py $HOME/.bigjob/python/
-$ . $HOME/.bigjob/python/bin/activate
+ $ curl https://raw.github.com/saga-project/BigJob/master/bootstrap/bigjob-bootstrap.py -o bigjob-bootstrap.py
+ $ python bigjob-bootstrap.py $HOME/.bigjob/python/
+ $ . $HOME/.bigjob/python/bin/activate
-See http://faust.cct.lsu.edu/trac/bigjob for further information. The Wiki includes among many things information on how to setup BigJob on various machines, e.g. on XSEDE (Ranger, Kraken) or FutureGrid.
+See https://github.com/saga-project/BigJob/wiki/ for further information. The Wiki includes among many things information on how to setup BigJob on various machines, e.g. on XSEDE (Ranger, Kraken) or FutureGrid.
Configuration of Coordination Backend
View
@@ -76,6 +76,11 @@ def __init__(self, args):
self.CPR = default_dict["cpr"]
self.SHELL=default_dict["shell"]
self.MPIRUN=default_dict["mpirun"]
+ self.OUTPUT_TAR=False
+ if default_dict.has_key("create_output_tar"):
+ self.OUTPUT_TAR=eval(default_dict["create_output_tar"])
+ logger.debug("Create output tar: %r", self.OUTPUT_TAR)
+
self.LAUNCH_METHOD=self.__get_launch_method(default_dict["launch_method"])
logging.debug("Launch Method: " + self.LAUNCH_METHOD + " mpi: " + self.MPIRUN + " shell: " + self.SHELL)
@@ -617,12 +622,15 @@ def monitor_jobs(self):
def update_output_file(self):
- output_file_name = "output-" + self.id + ".tar.gz"
- logger.debug("Update output file: " + output_file_name)
- output = subprocess.Popen('tar --exclude=*.brg --exclude=*.bmf --exclude=*tmp* --exclude=*.bif --exclude=*.fa --exclude=*.fastq --exclude=bfast --exclude=output*.tar.gz -czf ' + output_file_name + ' *',
+ if self.OUTPUT_TAR==True:
+ output_file_name = "output-" + self.id + ".tar.gz"
+ logger.debug("Update output file: " + output_file_name)
+ output = subprocess.Popen('tar --exclude=*.brg --exclude=*.bmf --exclude=*tmp* --exclude=*.bif --exclude=*.fa --exclude=*.fastq --exclude=bfast --exclude=output*.tar.gz -czf ' + output_file_name + ' *',
cwd="..", shell=True)
- output.wait()
- logger.debug("Files: " + str(os.listdir(".")))
+ output.wait()
+ logger.debug("Files: " + str(os.listdir(".")))
+ else:
+ logger.debug("Create NO output.tar. Enable output.tar file creation in bigjob_agent.conf")
def print_job(self, job_url):
View
@@ -228,7 +228,8 @@ def start_pilot_job(self,
# Determine whether target machine use gsissh or ssh to logon.
- self.launch_method = self.__get_launch_method(lrms_saga_url.host, lrms_saga_url.username)
+ logger.debug("Detect launch method for: " + lrms_saga_url.host)
+ self.launch_method = self.__get_launch_method(lrms_saga_url.host,lrms_saga_url.username)
##############################################################################
# File Management and Stage-In
@@ -723,18 +724,28 @@ def __stage_files(self, filetransfers, target_url):
def __get_launch_method(self, hostname, user=None):
""" returns desired execution method: ssh, aprun """
if user == None: user = self.__discover_ssh_user(hostname)
- if user!=None: logger.debug("discovered user: " + user)
+ host = ""
+ if user!=None and user!="":
+ logger.debug("discovered user: " + user)
+ host = user + "@" + hostname
+ else:
+ host = hostname
gsissh_available = False
try:
- gsissh_available = (subprocess.call("gsissh "+ user + "@" + hostname+" /bin/date", shell=True,stdout=subprocess.PIPE,stderr=subprocess.PIPE)==0)
+ cmd = "gsissh " + host + " /bin/date"
+ logger.debug("Execute: " + cmd)
+ gsissh_available = (subprocess.call(cmd, shell=True)==0)
except:
pass
ssh_available = False
try:
- ssh_available = (subprocess.call("ssh "+ user + "@" + hostname+" /bin/date", shell=True, stdout=subprocess.PIPE,stderr=subprocess.PIPE)==0)
+ cmd = "ssh " + host + " /bin/date"
+ logger.debug("Execute: " + cmd)
+ ssh_available = (subprocess.call(cmd, shell=True)==0)
except:
pass
+ logger.debug("SSH avail: %r GSISSH avail: %r"%(ssh_available, gsissh_available))
launch_method = "ssh"
if ssh_available == False and gsissh_available == True:
View
@@ -76,7 +76,11 @@ def __init__(self, bootstrap_script, launch_method, queue, project, lrms_saga_ur
qsub_file.write("\\n")
qsub_file.write("python -c XX" + textwrap.dedent(\"\"%s\"\") + "XX")
qsub_file.close()
-os.system( "qsub " + qsub_file_name)
+
+cmd = "qsub " + qsub_file_name
+if os.path.exists(os.path.expanduser("~/.bashrc")):
+ cmd = ". ~/.bashrc; " + cmd
+os.system(cmd)
""") % (str(ppn),str(nodes),str(nodes),str(ppn),str(queue),str(queue),str(project),str(project),str(walltime_pbs), bj_working_directory, bj_working_directory, str(self.working_directory), bootstrap_script)
### escaping characters
self.bootstrap_script = self.bootstrap_script.replace("\"","\\\"")
View
@@ -9,3 +9,8 @@ mpirun = mpirun
# Default launch method is ssh
# Future support for aprun (e.g. for Kraken)
launch_method = ssh
+
+# if True a output-bj-<id>.tar.gz file with all files created in the BJ sandbox
+# directory is created.
+# This is intented for cases where no global file system is available (e.g. on OSG)
+create_output_tar=False

0 comments on commit 0cb5ca6

Please sign in to comment.