Permalink
Browse files

Merge PR #795 (cluster-id and launcher cleanup)

closes gh-795
  • Loading branch information...
2 parents 391deab + 7b3d060 commit 3994edb75897eb8a5e19fcb6191cd0d704eb7376 @minrk minrk committed Sep 20, 2011
@@ -207,8 +207,9 @@ def c(s):
for parent in cls.mro():
# only include parents that are not base classes
# and are not the class itself
- if issubclass(parent, Configurable) and \
- not parent in (Configurable, SingletonConfigurable, cls):
+ # and have some configurable traits to inherit
+ if parent is not cls and issubclass(parent, Configurable) and \
+ parent.class_traits(config=True):
parents.append(parent)
if parents:
@@ -75,6 +75,7 @@ def __init__(self, app):
'log-to-file' : 'BaseParallelApplication.log_to_file',
'clean-logs' : 'BaseParallelApplication.clean_logs',
'log-url' : 'BaseParallelApplication.log_url',
+ 'cluster-id' : 'BaseParallelApplication.cluster_id',
})
base_flags = {
@@ -116,6 +117,22 @@ def _work_dir_changed(self, name, old, new):
log_url = Unicode('', config=True,
help="The ZMQ URL of the iplogger to aggregate logging.")
+ cluster_id = Unicode('', config=True,
+ help="""String id to add to runtime files, to prevent name collisions when
+ using multiple clusters with a single profile simultaneously.
+
+ When set, files will be named like: 'ipcontroller-<cluster_id>-engine.json'
+
+ Since this is text inserted into filenames, typical recommendations apply:
+ Simple character strings are ideal, and spaces are not recommended (but should
+ generally work).
+ """
+ )
+ def _cluster_id_changed(self, name, old, new):
+ self.name = self.__class__.name
+ if new:
+ self.name += '-%s'%new
+
def _config_files_default(self):
return ['ipcontroller_config.py', 'ipengine_config.py', 'ipcluster_config.py']
@@ -275,36 +275,37 @@ def initialize(self, argv=None):
self.init_launchers()
def init_launchers(self):
- self.engine_launcher = self.build_launcher(self.engine_launcher_class)
+ self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
self.engine_launcher.on_stop(lambda r: self.loop.stop())
def init_signal(self):
# Setup signals
signal.signal(signal.SIGINT, self.sigint_handler)
- def build_launcher(self, clsname):
+ def build_launcher(self, clsname, kind=None):
"""import and instantiate a Launcher based on importstring"""
if '.' not in clsname:
# not a module, presume it's the raw name in apps.launcher
+ if kind and kind not in clsname:
+ # doesn't match necessary full class name, assume it's
+ # just 'PBS' or 'MPIExec' prefix:
+ clsname = clsname + kind + 'Launcher'
clsname = 'IPython.parallel.apps.launcher.'+clsname
- # print repr(clsname)
try:
klass = import_item(clsname)
except (ImportError, KeyError):
self.log.fatal("Could not import launcher class: %r"%clsname)
self.exit(1)
launcher = klass(
- work_dir=u'.', config=self.config, log=self.log
+ work_dir=u'.', config=self.config, log=self.log,
+ profile_dir=self.profile_dir.location, cluster_id=self.cluster_id,
)
return launcher
def start_engines(self):
self.log.info("Starting %i engines"%self.n)
- self.engine_launcher.start(
- self.n,
- self.profile_dir.location
- )
+ self.engine_launcher.start(self.n)
def stop_engines(self):
self.log.info("Stopping Engines...")
@@ -424,14 +425,12 @@ def _classes_default(self,):
aliases = Dict(start_aliases)
def init_launchers(self):
- self.controller_launcher = self.build_launcher(self.controller_launcher_class)
- self.engine_launcher = self.build_launcher(self.engine_launcher_class)
+ self.controller_launcher = self.build_launcher(self.controller_launcher_class, 'Controller')
+ self.engine_launcher = self.build_launcher(self.engine_launcher_class, 'EngineSet')
self.controller_launcher.on_stop(self.stop_launchers)
def start_controller(self):
- self.controller_launcher.start(
- self.profile_dir.location
- )
+ self.controller_launcher.start()
def stop_controller(self):
# self.log.info("In stop_controller")
@@ -174,7 +174,18 @@ class IPControllerApp(BaseParallelApplication):
use_threads = Bool(False, config=True,
help='Use threads instead of processes for the schedulers',
- )
+ )
+
+ engine_json_file = Unicode('ipcontroller-engine.json', config=True,
+ help="JSON filename where engine connection info will be stored.")
+ client_json_file = Unicode('ipcontroller-client.json', config=True,
+ help="JSON filename where client connection info will be stored.")
+
+ def _cluster_id_changed(self, name, old, new):
+ super(IPControllerApp, self)._cluster_id_changed(name, old, new)
+ self.engine_json_file = "%s-engine.json" % self.name
+ self.client_json_file = "%s-client.json" % self.name
+
# internal
children = List()
@@ -215,7 +226,7 @@ def load_config_from_json(self):
"""load config from existing json connector files."""
c = self.config
# load from engine config
- with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-engine.json')) as f:
+ with open(os.path.join(self.profile_dir.security_dir, self.engine_json_file)) as f:
cfg = json.loads(f.read())
key = c.Session.key = asbytes(cfg['exec_key'])
xport,addr = cfg['url'].split('://')
@@ -227,7 +238,7 @@ def load_config_from_json(self):
if not self.engine_ssh_server:
self.engine_ssh_server = cfg['ssh']
# load client config
- with open(os.path.join(self.profile_dir.security_dir, 'ipcontroller-client.json')) as f:
+ with open(os.path.join(self.profile_dir.security_dir, self.client_json_file)) as f:
cfg = json.loads(f.read())
assert key == cfg['exec_key'], "exec_key mismatch between engine and client keys"
xport,addr = cfg['url'].split('://')
@@ -277,11 +288,11 @@ def init_hub(self):
'url' : "%s://%s:%s"%(f.client_transport, f.client_ip, f.regport),
'location' : self.location
}
- self.save_connection_dict('ipcontroller-client.json', cdict)
+ self.save_connection_dict(self.client_json_file, cdict)
edict = cdict
edict['url']="%s://%s:%s"%((f.client_transport, f.client_ip, f.regport))
edict['ssh'] = self.engine_ssh_server
- self.save_connection_dict('ipcontroller-engine.json', edict)
+ self.save_connection_dict(self.engine_json_file, edict)
#
def init_schedulers(self):
@@ -93,7 +93,7 @@ class MPI(Configurable):
help='How to enable MPI (mpi4py, pytrilinos, or empty string to disable).'
)
- def _on_use_changed(self, old, new):
+ def _use_changed(self, name, old, new):
# load default init script if it's not set
if not self.init_script:
self.init_script = self.default_inits.get(new, '')
@@ -135,8 +135,8 @@ def _on_use_changed(self, old, new):
class IPEngineApp(BaseParallelApplication):
- name = Unicode(u'ipengine')
- description = Unicode(_description)
+ name = 'ipengine'
+ description = _description
examples = _examples
config_file_name = Unicode(default_config_file_name)
classes = List([ProfileDir, Session, EngineFactory, Kernel, MPI])
@@ -158,7 +158,15 @@ class IPEngineApp(BaseParallelApplication):
controller and engine are started at the same time and it
may take a moment for the controller to write the connector files.""")
- url_file_name = Unicode(u'ipcontroller-engine.json')
+ url_file_name = Unicode(u'ipcontroller-engine.json', config=True)
+
+ def _cluster_id_changed(self, name, old, new):
+ if new:
+ base = 'ipcontroller-%s' % new
+ else:
+ base = 'ipcontroller'
+ self.url_file_name = "%s-engine.json" % base
+
log_url = Unicode('', config=True,
help="""The URL for the iploggerapp instance, for forwarding
logging to a central location.""")
Oops, something went wrong.

0 comments on commit 3994edb

Please sign in to comment.