Permalink
Browse files

launcher updates for PBS

  • Loading branch information...
1 parent f36800d commit 8a8cbf5ddf834db3211ce078ef864df2608c3a40 @minrk minrk committed Mar 10, 2011
@@ -115,7 +115,11 @@
# environment variables would be setup, etc. This string is interpolated using
# the Itpl module in IPython.external. Basically, you can use ${n} for the
# number of engine and ${cluster_dir} for the cluster_dir.
-# c.PBSControllerLauncher.batch_template = """"""
+# c.PBSControllerLauncher.batch_template = """
+# #PBS -l nprocs=$n
+#
+# ipcontrollerz --cluster-dir $cluster_dir
+# """
# The name of the instantiated batch script that will actually be used to
# submit the job. This will be written to the cluster directory.
@@ -135,7 +139,11 @@
# environment variables would be setup, etc. This string is interpolated using
# the Itpl module in IPython.external. Basically, you can use ${n} for the
# number of engine and ${cluster_dir} for the cluster_dir.
-# c.PBSEngineSetLauncher.batch_template = """"""
+# c.PBSEngineSetLauncher.batch_template = """
+# #PBS -l nprocs=$n
+#
+# ipenginez --cluster-dir $cluster_dir$s
+# """
# The name of the instantiated batch script that will actually be used to
# submit the job. This will be written to the cluster directory.
@@ -123,7 +123,7 @@ def _add_arguments(self):
"""List all available clusters, by cluster directory, that can
be found in the current working directly or in the ipython
directory. Cluster directories are named using the convention
- 'cluster_<profile>'."""
+ 'clusterz_<profile>'."""
)
# The "create" subcommand parser
@@ -136,13 +136,13 @@ def _add_arguments(self):
"""Create an ipython cluster directory by its profile name or
cluster directory path. Cluster directories contain
configuration, log and security related files and are named
- using the convention 'cluster_<profile>'. By default they are
+ using the convention 'clusterz_<profile>'. By default they are
located in your ipython directory. Once created, you will
probably need to edit the configuration files in the cluster
directory to configure your cluster. Most users will create a
cluster directory by profile name,
'ipclusterz create -p mycluster', which will put the directory
- in '<ipython_dir>/cluster_mycluster'.
+ in '<ipython_dir>/clusterz_mycluster'.
"""
)
paa = parser_create.add_argument
@@ -162,7 +162,7 @@ def _add_arguments(self):
"""Start an ipython cluster by its profile name or cluster
directory. Cluster directories contain configuration, log and
security related files and are named using the convention
- 'cluster_<profile>' and should be creating using the 'start'
+ 'clusterz_<profile>' and should be creating using the 'start'
subcommand of 'ipcluster'. If your cluster directory is in
the cwd or the ipython directory, you can simply refer to it
using its profile name, 'ipclusterz start -n 4 -p <profile>`,
@@ -200,7 +200,7 @@ def _add_arguments(self):
description=
"""Stop a running ipython cluster by its profile name or cluster
directory. Cluster directories are named using the convention
- 'cluster_<profile>'. If your cluster directory is in
+ 'clusterz_<profile>'. If your cluster directory is in
the cwd or the ipython directory, you can simply refer to it
using its profile name, 'ipclusterz stop -p <profile>`, otherwise
use the '--cluster-dir' option.
@@ -223,7 +223,7 @@ def _add_arguments(self):
by profile name or cluster directory.
Cluster directories contain configuration, log and
security related files and are named using the convention
- 'cluster_<profile>' and should be creating using the 'start'
+ 'clusterz_<profile>' and should be creating using the 'start'
subcommand of 'ipcluster'. If your cluster directory is in
the cwd or the ipython directory, you can simply refer to it
using its profile name, 'ipclusterz engines -n 4 -p <profile>`,
@@ -322,7 +322,7 @@ def list_cluster_dirs(self):
files = os.listdir(path)
for f in files:
full_path = os.path.join(path, f)
- if os.path.isdir(full_path) and f.startswith('cluster_'):
+ if os.path.isdir(full_path) and f.startswith('clusterz_'):
profile = full_path.split('_')[-1]
start_cmd = 'ipclusterz start -p %s -n 4' % profile
print start_cmd + " ==> " + full_path
@@ -57,7 +57,7 @@
clients. The controller needs to be started before the engines and can be
configured using command line options or using a cluster directory. Cluster
directories contain config, log and security files and are usually located in
-your ipython directory and named as "cluster_<profile>". See the --profile
+your ipython directory and named as "clusterz_<profile>". See the --profile
and --cluster-dir options for details.
"""
@@ -64,7 +64,7 @@ class SimpleStruct:
and controller. A controller needs to be started before the engines. The
engine can be configured using command line options or using a cluster
directory. Cluster directories contain config, log and security files and are
-usually located in your ipython directory and named as "cluster_<profile>".
+usually located in your ipython directory and named as "clusterz_<profile>".
See the --profile and --cluster-dir options for details.
"""
@@ -39,7 +39,7 @@
by registering a `zmq.log.handlers.PUBHandler` with the `logging` module. The
logger can be configured using command line options or using a cluster
directory. Cluster directories contain config, log and security files and are
-usually located in your ipython directory and named as "cluster_<profile>".
+usually located in your ipython directory and named as "clusterz_<profile>".
See the --profile and --cluster-dir options for details.
"""
@@ -44,7 +44,7 @@ def check_output(*args, **kwargs):
from IPython.external import Itpl
# from IPython.config.configurable import Configurable
-from IPython.utils.traitlets import Str, Int, List, Unicode, Instance
+from IPython.utils.traitlets import Str, Int, List, Unicode, Dict, Instance
from IPython.utils.path import get_ipython_module_path
from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
@@ -738,13 +738,18 @@ class BatchSystemLauncher(BaseLauncher):
batch_file_name = Unicode(u'batch_script', config=True)
# The full path to the instantiated batch script.
batch_file = Unicode(u'')
+ # the format dict used with batch_template:
+ context = Dict()
+
+ def find_args(self):
+ return [self.submit_command]
+
def __init__(self, work_dir=u'.', config=None, **kwargs):
super(BatchSystemLauncher, self).__init__(
work_dir=work_dir, config=config, **kwargs
)
self.batch_file = os.path.join(self.work_dir, self.batch_file_name)
- self.context = {}
def parse_job_id(self, output):
"""Take the output of the submit command and return the job id."""
@@ -766,8 +771,13 @@ def write_batch_script(self, n):
f.write(script_as_string)
f.close()
- def start(self, n):
+ def start(self, n, cluster_dir):
"""Start n copies of the process using a batch system."""
+ # Here we save profile and cluster_dir in the context so they
+ # can be used in the batch script template as ${profile} and
+ # ${cluster_dir}
+ self.context['cluster_dir'] = cluster_dir
+ self.cluster_dir = unicode(cluster_dir)
self.write_batch_script(n)
output = check_output([self.submit_command, self.batch_file], env=os.environ, stdout=STDOUT)
job_id = self.parse_job_id(output)
@@ -798,13 +808,8 @@ class PBSControllerLauncher(PBSLauncher):
def start(self, cluster_dir):
"""Start the controller by profile or cluster_dir."""
- # Here we save profile and cluster_dir in the context so they
- # can be used in the batch script template as ${profile} and
- # ${cluster_dir}
- self.context['cluster_dir'] = cluster_dir
- self.cluster_dir = unicode(cluster_dir)
self.log.info("Starting PBSControllerLauncher: %r" % self.args)
- return super(PBSControllerLauncher, self).start(1)
+ return super(PBSControllerLauncher, self).start(1, cluster_dir)
class PBSEngineSetLauncher(PBSLauncher):
@@ -813,10 +818,8 @@ class PBSEngineSetLauncher(PBSLauncher):
def start(self, n, cluster_dir):
"""Start n engines by profile or cluster_dir."""
- self.program_args.extend(['--cluster-dir', cluster_dir])
- self.cluster_dir = unicode(cluster_dir)
self.log.info('Starting PBSEngineSetLauncher: %r' % self.args)
- return super(PBSEngineSetLauncher, self).start(n)
+ return super(PBSEngineSetLauncher, self).start(n, cluster_dir)
#-----------------------------------------------------------------------------

0 comments on commit 8a8cbf5

Please sign in to comment.