Permalink
Browse files

organize IPython.parallel into subpackages

  • Loading branch information...
1 parent e950e62 commit b9f548064ed3d808990145f1adcc99221f701863 @minrk minrk committed Apr 7, 2011
Showing with 261 additions and 625 deletions.
  1. +3 −3 IPython/config/default/ipcluster_config.py
  2. +4 −4 IPython/config/default/ipcontroller_config.py
  3. +1 −1 IPython/external/ssh/tunnel.py
  4. +8 −7 IPython/parallel/__init__.py
  5. 0 IPython/parallel/apps/__init__.py
  6. 0 IPython/parallel/{ → apps}/clusterdir.py
  7. +1 −1 IPython/parallel/{ → apps}/ipclusterapp.py
  8. +5 −4 IPython/parallel/{ → apps}/ipcontrollerapp.py
  9. +4 −3 IPython/parallel/{ → apps}/ipengineapp.py
  10. +1 −1 IPython/parallel/{ → apps}/iploggerapp.py
  11. +4 −4 IPython/parallel/{ → apps}/launcher.py
  12. +1 −1 IPython/parallel/{ → apps}/logwatcher.py
  13. 0 IPython/parallel/{ → apps}/winhpcjob.py
  14. 0 IPython/parallel/client/__init__.py
  15. +1 −1 IPython/parallel/{ → client}/asyncresult.py
  16. +5 −4 IPython/parallel/{ → client}/client.py
  17. 0 IPython/parallel/{ → client}/map.py
  18. 0 IPython/parallel/{ → client}/remotefunction.py
  19. +5 −8 IPython/parallel/{ → client}/view.py
  20. 0 IPython/parallel/controller/__init__.py
  21. +1 −2 IPython/parallel/{ → controller}/controller.py
  22. +3 −3 IPython/parallel/{ → controller}/dependency.py
  23. 0 IPython/parallel/{ → controller}/dictdb.py
  24. +2 −3 IPython/parallel/{ → controller}/heartmonitor.py
  25. +6 −7 IPython/parallel/{ → controller}/hub.py
  26. 0 IPython/parallel/{ → controller}/mongodb.py
  27. +4 −4 IPython/parallel/{ → controller}/scheduler.py
  28. +1 −1 IPython/parallel/{ → controller}/sqlitedb.py
  29. 0 IPython/parallel/engine/__init__.py
  30. +6 −5 IPython/parallel/{ → engine}/engine.py
  31. +1 −1 IPython/parallel/{ → engine}/kernelstarter.py
  32. +4 −70 IPython/parallel/{ → engine}/streamkernel.py
  33. +0 −125 IPython/parallel/entry_point.py
  34. +1 −1 IPython/parallel/factory.py
  35. +0 −97 IPython/parallel/ipcluster.py
  36. +0 −101 IPython/parallel/remotenamespace.py
  37. 0 IPython/parallel/scheduler/__init__.py
  38. +1 −1 IPython/parallel/scripts/ipcluster
  39. +1 −1 IPython/parallel/scripts/ipcontroller
  40. +1 −1 IPython/parallel/scripts/ipengine
  41. +1 −1 IPython/parallel/scripts/iplogger
  42. +0 −106 IPython/parallel/taskthread.py
  43. +3 −3 IPython/parallel/tests/__init__.py
  44. +1 −5 IPython/parallel/tests/clienttest.py
  45. +3 −3 IPython/parallel/tests/test_client.py
  46. +7 −7 IPython/parallel/tests/test_dependency.py
  47. +2 −2 IPython/parallel/tests/test_view.py
  48. +108 −0 IPython/parallel/util.py
  49. +4 −1 IPython/testing/iptest.py
  50. +1 −1 IPython/utils/pickleutil.py
  51. +1 −1 docs/source/parallel/parallel_details.txt
  52. +1 −1 docs/source/parallel/parallel_intro.txt
  53. +2 −2 docs/source/parallel/parallel_multiengine.txt
  54. +6 −6 docs/source/parallel/parallel_process.txt
  55. +6 −6 docs/source/parallel/parallel_task.txt
  56. +31 −7 docs/source/parallel/parallel_transition.txt
  57. +2 −2 docs/source/parallel/parallel_winhpc.txt
  58. +5 −5 setup.py
  59. +2 −1 setupbase.py
@@ -23,16 +23,16 @@
# - PBSControllerLauncher
# - SGEControllerLauncher
# - WindowsHPCControllerLauncher
-# c.Global.controller_launcher = 'IPython.parallel.launcher.LocalControllerLauncher'
-# c.Global.controller_launcher = 'IPython.parallel.launcher.PBSControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.LocalControllerLauncher'
+# c.Global.controller_launcher = 'IPython.parallel.apps.launcher.PBSControllerLauncher'
# Options are:
# - LocalEngineSetLauncher
# - MPIExecEngineSetLauncher
# - PBSEngineSetLauncher
# - SGEEngineSetLauncher
# - WindowsHPCEngineSetLauncher
-# c.Global.engine_launcher = 'IPython.parallel.launcher.LocalEngineSetLauncher'
+# c.Global.engine_launcher = 'IPython.parallel.apps.launcher.LocalEngineSetLauncher'
#-----------------------------------------------------------------------------
# Global configuration
@@ -89,7 +89,7 @@
# Which class to use for the db backend. Currently supported are DictDB (the
# default), and MongoDB. Uncomment this line to enable MongoDB, which will
# slow-down the Hub's responsiveness, but also reduce its memory footprint.
-# c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
# The heartbeat ping frequency. This is the frequency (in ms) at which the
# Hub pings engines for heartbeats. This determines how quickly the Hub
@@ -144,11 +144,11 @@
# ----- in-memory configuration --------
# this line restores the default behavior: in-memory storage of all results.
-# c.HubFactory.db_class = 'IPython.parallel.dictdb.DictDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.dictdb.DictDB'
# ----- sqlite configuration --------
# use this line to activate sqlite:
-# c.HubFactory.db_class = 'IPython.parallel.sqlitedb.SQLiteDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.sqlitedb.SQLiteDB'
# You can specify the name of the db-file. By default, this will be located
# in the active cluster_dir, e.g. ~/.ipython/clusterz_default/tasks.db
@@ -165,7 +165,7 @@
# ----- mongodb configuration --------
# use this line to activate mongodb:
-# c.HubFactory.db_class = 'IPython.parallel.mongodb.MongoDB'
+# c.HubFactory.db_class = 'IPython.parallel.controller.mongodb.MongoDB'
# You can specify the args and kwargs pymongo will use when creating the Connection.
# For more information on what these options might be, see pymongo documentation.
@@ -34,7 +34,7 @@
except ImportError:
pexpect = None
-from IPython.parallel.entry_point import select_random_ports
+from IPython.parallel.util import select_random_ports
#-----------------------------------------------------------------------------
# Code
@@ -12,14 +12,15 @@
import zmq
-if zmq.__version__ < '2.1.3':
- raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.3, you appear to have %s"%zmq.__version__)
+if zmq.__version__ < '2.1.4':
+ raise ImportError("IPython.parallel requires pyzmq/0MQ >= 2.1.4, you appear to have %s"%zmq.__version__)
-from .asyncresult import *
-from .client import Client
-from .dependency import *
-from .remotefunction import *
-from .view import *
from IPython.utils.pickleutil import Reference
+from .client.asyncresult import *
+from .client.client import Client
+from .client.remotefunction import *
+from .client.view import *
+from .controller.dependency import *
+
No changes.
@@ -26,7 +26,7 @@
from IPython.external.argparse import ArgumentParser, SUPPRESS
from IPython.utils.importstring import import_item
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
ApplicationWithClusterDir, ClusterDirConfigLoader,
ClusterDirError, PIDFileError
)
@@ -30,16 +30,17 @@
from zmq.utils import jsonapi as json
from IPython.config.loader import Config
+
from IPython.parallel import factory
-from IPython.parallel.controller import ControllerFactory
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
ApplicationWithClusterDir,
ClusterDirConfigLoader
)
from IPython.parallel.util import disambiguate_ip_address, split_url
# from IPython.kernel.fcutil import FCServiceFactory, FURLError
from IPython.utils.traitlets import Instance, Unicode
+from IPython.parallel.controller.controller import ControllerFactory
#-----------------------------------------------------------------------------
@@ -117,11 +118,11 @@ def _add_arguments(self):
## Hub Config:
paa('--mongodb',
dest='HubFactory.db_class', action='store_const',
- const='IPython.parallel.mongodb.MongoDB',
+ const='IPython.parallel.controller.mongodb.MongoDB',
help='Use MongoDB for task storage [default: in-memory]')
paa('--sqlite',
dest='HubFactory.db_class', action='store_const',
- const='IPython.parallel.sqlitedb.SQLiteDB',
+ const='IPython.parallel.controller.sqlitedb.SQLiteDB',
help='Use SQLite3 for DB task storage [default: in-memory]')
paa('--hb',
type=int, dest='HubFactory.hb', nargs=2,
@@ -22,16 +22,17 @@
import zmq
from zmq.eventloop import ioloop
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
ApplicationWithClusterDir,
ClusterDirConfigLoader
)
from IPython.zmq.log import EnginePUBHandler
from IPython.parallel import factory
-from IPython.parallel.engine import EngineFactory
-from IPython.parallel.streamkernel import Kernel
+from IPython.parallel.engine.engine import EngineFactory
+from IPython.parallel.engine.streamkernel import Kernel
from IPython.parallel.util import disambiguate_url
+
from IPython.utils.importstring import import_item
@@ -20,7 +20,7 @@
import zmq
-from IPython.parallel.clusterdir import (
+from .clusterdir import (
ApplicationWithClusterDir,
ClusterDirConfigLoader
)
@@ -46,7 +46,7 @@ def check_output(*args, **kwargs):
from IPython.utils.path import get_ipython_module_path
from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
# load winhpcjob only on Windows
try:
@@ -64,15 +64,15 @@ def check_output(*args, **kwargs):
ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path(
- 'IPython.parallel.ipclusterapp'
+ 'IPython.parallel.apps.ipclusterapp'
))
ipengine_cmd_argv = pycmd2argv(get_ipython_module_path(
- 'IPython.parallel.ipengineapp'
+ 'IPython.parallel.apps.ipengineapp'
))
ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path(
- 'IPython.parallel.ipcontrollerapp'
+ 'IPython.parallel.apps.ipcontrollerapp'
))
#-----------------------------------------------------------------------------
@@ -21,7 +21,7 @@
from IPython.utils.traitlets import Int, Str, Instance, List
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
#-----------------------------------------------------------------------------
# Classes
File renamed without changes.
No changes.
@@ -15,7 +15,7 @@
from zmq import MessageTracker
from IPython.external.decorator import decorator
-from . import error
+from IPython.parallel import error
#-----------------------------------------------------------------------------
# Classes
@@ -29,11 +29,12 @@
from IPython.external.decorator import decorator
from IPython.external.ssh import tunnel
-from . import error
-from . import util
-from . import streamsession as ss
+from IPython.parallel import error
+from IPython.parallel import streamsession as ss
+from IPython.parallel import util
+
from .asyncresult import AsyncResult, AsyncHubResult
-from .clusterdir import ClusterDir, ClusterDirError
+from IPython.parallel.apps.clusterdir import ClusterDir, ClusterDirError
from .view import DirectView, LoadBalancedView
#--------------------------------------------------------------------------
File renamed without changes.
@@ -23,10 +23,11 @@
from IPython.external.decorator import decorator
+from IPython.parallel import util
+from IPython.parallel.controller.dependency import Dependency, dependent
+
from . import map as Map
-from . import util
from .asyncresult import AsyncResult, AsyncMapResult
-from .dependency import Dependency, dependent
from .remotefunction import ParallelFunction, parallel, remote
#-----------------------------------------------------------------------------
@@ -68,6 +69,7 @@ def spin_after(f, self, *args, **kwargs):
# Classes
#-----------------------------------------------------------------------------
+@testdec.skip_doctest
class View(HasTraits):
"""Base View class for more convenint apply(f,*args,**kwargs) syntax via attributes.
@@ -105,7 +107,7 @@ class View(HasTraits):
history=List()
outstanding = Set()
results = Dict()
- client = Instance('IPython.parallel.client.Client')
+ client = Instance('IPython.parallel.Client')
_socket = Instance('zmq.Socket')
_flag_names = List(['targets', 'block', 'track'])
@@ -386,11 +388,6 @@ def importer(self):
"""sync_imports(local=True) as a property.
See sync_imports for details.
-
- In [10]: with v.importer:
- ....: import numpy
- ....:
- importing numpy on engine(s)
"""
return self.sync_imports(True)
No changes.
@@ -14,7 +14,6 @@
#-----------------------------------------------------------------------------
from __future__ import print_function
-import logging
from multiprocessing import Process
import zmq
@@ -23,7 +22,7 @@
from IPython.utils.importstring import import_item
from IPython.utils.traitlets import Int, CStr, Instance, List, Bool
-from .entry_point import signal_children
+from IPython.parallel.util import signal_children
from .hub import Hub, HubFactory
from .scheduler import launch_scheduler
@@ -8,9 +8,9 @@
from types import ModuleType
-from .asyncresult import AsyncResult
-from .error import UnmetDependency
-from .util import interactive
+from IPython.parallel.client.asyncresult import AsyncResult
+from IPython.parallel.error import UnmetDependency
+from IPython.parallel.util import interactive
class depend(object):
"""Dependency decorator, for use with tasks.
File renamed without changes.
@@ -12,15 +12,14 @@
from __future__ import print_function
import time
-import logging
import uuid
import zmq
-from zmq.devices import ProcessDevice,ThreadDevice
+from zmq.devices import ProcessDevice, ThreadDevice
from zmq.eventloop import ioloop, zmqstream
from IPython.utils.traitlets import Set, Instance, CFloat, Bool
-from .factory import LoggingFactory
+from IPython.parallel.factory import LoggingFactory
class Heart(object):
"""A basic heart object for responding to a HeartMonitor.
@@ -27,12 +27,11 @@
from IPython.utils.importstring import import_item
from IPython.utils.traitlets import HasTraits, Instance, Int, CStr, Str, Dict, Set, List, Bool
-from .entry_point import select_random_ports
-from .factory import RegistrationFactory, LoggingFactory
+from IPython.parallel import error
+from IPython.parallel.factory import RegistrationFactory, LoggingFactory
+from IPython.parallel.util import select_random_ports, validate_url_container, ISO8601
-from . import error
from .heartmonitor import HeartMonitor
-from .util import validate_url_container, ISO8601
#-----------------------------------------------------------------------------
# Code
@@ -160,11 +159,11 @@ def _notifier_port_default(self):
monitor_url = CStr('')
- db_class = CStr('IPython.parallel.dictdb.DictDB', config=True)
+ db_class = CStr('IPython.parallel.controller.dictdb.DictDB', config=True)
# not configurable
- db = Instance('IPython.parallel.dictdb.BaseDB')
- heartmonitor = Instance('IPython.parallel.heartmonitor.HeartMonitor')
+ db = Instance('IPython.parallel.controller.dictdb.BaseDB')
+ heartmonitor = Instance('IPython.parallel.controller.heartmonitor.HeartMonitor')
subconstructors = List()
_constructed = Bool(False)
@@ -36,11 +36,11 @@
from IPython.external.decorator import decorator
from IPython.utils.traitlets import Instance, Dict, List, Set
-from . import error
-from .dependency import Dependency
-from .entry_point import connect_logger, local_logger
-from .factory import SessionFactory
+from IPython.parallel import error
+from IPython.parallel.factory import SessionFactory
+from IPython.parallel.util import connect_logger, local_logger
+from .dependency import Dependency
@decorator
def logged(f,self,*args,**kwargs):
@@ -17,7 +17,7 @@
from IPython.utils.traitlets import CUnicode, CStr, Instance, List
from .dictdb import BaseDB
-from .util import ISO8601
+from IPython.parallel.util import ISO8601
#-----------------------------------------------------------------------------
# SQLite operators, adapters, and converters
No changes.
@@ -22,11 +22,12 @@
from IPython.utils.traitlets import Instance, Str, Dict, Int, Type, CFloat
# from IPython.utils.localinterfaces import LOCALHOST
-from . import heartmonitor
-from .factory import RegistrationFactory
+from IPython.parallel.controller.heartmonitor import Heart
+from IPython.parallel.factory import RegistrationFactory
+from IPython.parallel.streamsession import Message
+from IPython.parallel.util import disambiguate_url
+
from .streamkernel import Kernel
-from .streamsession import Message
-from .util import disambiguate_url
class EngineFactory(RegistrationFactory):
"""IPython engine"""
@@ -129,7 +130,7 @@ def complete_registration(self, msg):
loop=loop, user_ns = self.user_ns, logname=self.log.name)
self.kernel.start()
hb_addrs = [ disambiguate_url(addr, self.location) for addr in hb_addrs ]
- heart = heartmonitor.Heart(*map(str, hb_addrs), heart_id=identity)
+ heart = Heart(*map(str, hb_addrs), heart_id=identity)
# ioloop.DelayedCallback(heart.start, 1000, self.loop).start()
heart.start()
Oops, something went wrong.

0 comments on commit b9f5480

Please sign in to comment.