Merge branch 'devel'

.travis.yml

@@ -32,6 +32,8 @@ before_script:
   - curl https://bootstrap.pypa.io/get-pip.py > get-pip.py
   - pypy get-pip.py || pypy get-pip.py --user
   - pypy -m pip install coverage || pypy -m pip install --user coverage
+  - pypy -m pip install unittest2 || pypy -m pip install --user unittest2
+  - pypy setup.py install || pypy setup.py install --user
   - export COVERAGE_PROCESS_START=$(pwd)/.coveragerc
   - echo $TRAVIS_PYTHON_VERSION && which python && which pypy && python -c 'import os, sys; print(os.path.basename(sys.executable))'
 script:

README.rst

@@ -73,7 +73,7 @@ TwinObjects
 The real power of :py:mod:`cpy2py` are Twins - objects living in one
 twinterpreter and being represented by proxies in any other interpeter.
 Using twins, you can seamlessly split your application across multiple
-twins.
+twinterpreters.
 
 You create twins by inheriting from
 :py:class:`cpy2py.TwinObject` instead of :py:class:`object` and
@@ -136,7 +136,6 @@ needs.
 Note that if python is run with the `-O` flag, several logging calls are
 skipped entirely to improve performance.
 
-
 For small scale debugging, one can set the environment variable
 :envvar:`CPY2PY_DEBUG`. If it is defined and not empty, logging output
 is written to `stderr`. In addition, if it names a valid :py:mod:`logging`
@@ -179,7 +178,7 @@ Features
 
   * Pure python, no dependencies means perfect portability.
 
-  * Any interpeter compatible with python 2.6 to 3.5 is supported.
+  * Any interpreter compatible with python 2.6 to 3.7 is supported.
 
   * Virtual Environments work out of the box.
 
@@ -190,12 +189,13 @@ Gotchas/Limitations
 
 * Importing functions and classes from `__main__` may fail if the module can only be imported via its path.
 
-* Calls across interpreters are blocking and not threadsafe.
+* By default, calls across interpreters are blocking and not threadsafe.
   If recursion switches between twinterpreters, :py:class:`cpy2py.TwinMaster` must use the ``'async'`` kernel.
 
 * Module level settings are not synchronized.
   For example, configuration of :py:mod:`logging` is not applied to twinterpreters.
-  Use :py:class:`~cpy2py.twinterpreter.group_state.TwinGroupState` for initialisation, or write modules aware of twinterpreters.
+  Use :py:class:`~cpy2py.twinterpreter.group_state.TwinGroupState` for initialisation,
+  write modules aware of twinterpreters, or use immutable module-level initializers.
 
 * A :py:mod:`weakref` to objects only takes local references into account, not cross-interpreter references.
 

cpy2py/__init__.py

@@ -28,6 +28,8 @@
 
 * :py:class:`~cpy2py.proxy.proxy_object.TwinObject` lets you create objects across interpreters.
 
+* :py:class:`~cpy2py.proxy.proxy_object.twinfunction` lets you run functions in a specific interpreter.
+
 * :py:mod:`~cpy2py.twinterpreter.kernel_state` exposes all meta information you need.
 """
 import logging as _logging
@@ -52,4 +54,4 @@
 else:
     _base_logger.addHandler(_NullHandler())
 
-__all__ = ['TwinObject', 'TwinMaster', 'kernel_state', '__version__', 'localmethod']
+__all__ = ['TwinObject', 'TwinMaster', 'kernel_state', '__version__', 'localmethod', 'twinfunction']

cpy2py/ipyc/ipyc_socket.py

@@ -103,7 +103,9 @@ def connector(self):
         return self.__class__, (), {'family': int(self.family), 'address': self.address, 'is_master': False}
 
     def __repr__(self):
-        return '%s(family=%r, address=%r, is_master=%s)' % (self.__class__.__name__, self.family, self.address, self.is_master)
+        return '%s(family=%r, address=%r, is_master=%s)' % (
+            self.__class__.__name__, self.family, self.address, self.is_master
+        )
 
 
 class BufferedSocketFile(object):

cpy2py/kernel/kernel_requesthandler.py

@@ -99,7 +99,9 @@ def serve_request(self, request_id, directive):
         # run request
         try:
             if __debug__:
-                self._logger.warning('<%s> [%s] Directive %s', kernel_state.TWIN_ID, self.peer_id, E_SYMBOL[directive_type])
+                self._logger.warning(
+                    '<%s> [%s] Directive %s', kernel_state.TWIN_ID, self.peer_id, E_SYMBOL[directive_type]
+                )
             response = directive_method(directive_body)
         # catch internal errors to reraise them
         except CPy2PyException:

cpy2py/kernel/kernel_single.py

@@ -19,13 +19,13 @@
 operates *strictly* singlethreaded - no more than one request may be
 served at a time. This makes recursion across twinterpreters impossible.
 """
+from __future__ import print_function
 import sys
 import os
 import time
 import logging
 import threading
 
-from cpy2py.utility.compat import pickle
 from cpy2py.kernel import kernel_state
 
 from cpy2py.utility.exceptions import format_exception
@@ -37,11 +37,9 @@
 
 def _connect_ipyc(ipyc, pickle_protocol):
     """Connect pickle/unpickle trackers to a duplex IPyC"""
-    pickler = pickle.Pickler(ipyc.writer, pickle_protocol)
-    pickler.persistent_id = proxy_tracker.persistent_twin_id
+    pickler = proxy_tracker.twin_pickler(ipyc.writer, pickle_protocol)
     send = pickler.dump
-    unpickler = pickle.Unpickler(ipyc.reader)
-    unpickler.persistent_load = proxy_tracker.persistent_twin_load
+    unpickler = proxy_tracker.twin_unpickler(ipyc.reader)
     recv = unpickler.load
     return send, recv
 
@@ -81,21 +79,32 @@ def run(self):
         assert self._terminate.is_set(), 'Kernel already active'
         self._terminate.clear()
         exit_code = 1
-        self._logger.warning('<%s> [%s] Starting %s @ %s', kernel_state.TWIN_ID, self.peer_id, self.__class__.__name__, time.asctime())
+        self._logger.warning(
+            '<%s> [%s] Starting %s @ %s', kernel_state.TWIN_ID, self.peer_id, self.__class__.__name__, time.asctime()
+        )
         try:
             self._serve_requests()
         except StopTwinterpreter as err:
+            # actively shutting down
             self._logger.critical('<%s> [%s] TWIN KERNEL TERMINATED: %s', kernel_state.TWIN_ID, self.peer_id, err)
             exit_code = err.exit_code
         # cPickle may raise EOFError by itself
         except (ipyc_exceptions.IPyCTerminated, EOFError) as err:
+            # regular shutdown by master
             self._logger.critical('<%s> [%s] TWIN KERNEL RELEASED: %s', kernel_state.TWIN_ID, self.peer_id, err)
             exit_code = 0
         except Exception as err:  # pylint: disable=broad-except
+            # unexpected shutdown
+            # provide extended traceback if requested
             self._logger.critical(
                 '<%s> [%s] TWIN KERNEL INTERNAL EXCEPTION: %s', kernel_state.TWIN_ID, self.peer_id, err
             )
             format_exception(self._logger, 3)
+            # emulate regular python exit
+            import traceback
+            exit_code = 1
+            traceback.print_exc(file=sys.stderr)
+            print('TwinError: unhandled exception in', kernel_state.TWIN_ID, file=sys.stderr)
         finally:
             self._terminate.set()
             self._logger.critical('<%s> [%s] TWIN KERNEL SHUTDOWN: %d', kernel_state.TWIN_ID, self.peer_id, exit_code)

cpy2py/meta.py

@@ -14,4 +14,4 @@
 """
 Meta Information on the project
 """
-__version__ = "0.17.0"
+__version__ = "0.17.1"

cpy2py/twinterpreter/bootstrap.py

@@ -155,7 +155,11 @@ def bootstrap_kernel():
     # TwinMaster will run the finalizers directly
     run_initializer(settings.initializer)
     logging.getLogger('__cpy2py__.kernel.%s_to_%s.bootstrap' % (kernel_state.TWIN_ID, settings.peer_id)).warning(
-        '<%s> [%s] %s.bootstrap_kernel deploying cpy2py %s', kernel_state.TWIN_ID, settings.peer_id, 'cpy2py.twinterpreter', cpy2py_version
+        '<%s> [%s] %s.bootstrap_kernel deploying cpy2py %s',
+        kernel_state.TWIN_ID,
+        settings.peer_id,
+        'cpy2py.twinterpreter',
+        cpy2py_version
     )
     exit_code = run_kernel(
         kernel=load_kernel(settings.kernel),
@@ -165,7 +169,11 @@ def bootstrap_kernel():
         ipyc_pkl_protocol=settings.ipyc_pkl_protocol
     )
     logging.getLogger('__cpy2py__.kernel.%s_to_%s.bootstrap' % (kernel_state.TWIN_ID, settings.peer_id)).warning(
-        '<%s> [%s] %s.bootstrap_kernel exiting with %s', kernel_state.TWIN_ID, settings.peer_id, 'cpy2py.twinterpreter', exit_code
+        '<%s> [%s] %s.bootstrap_kernel exiting with %s',
+        kernel_state.TWIN_ID,
+        settings.peer_id,
+        'cpy2py.twinterpreter',
+        exit_code
     )
     sys.exit(exit_code)
 

cpy2py/twinterpreter/twin_def.py

@@ -94,14 +94,24 @@ def _resolve_kernel_arg(self, kernel_arg):
         return client, server
 
     def spawn(self, cli_args=None, env=None):
+        """
+        Spawn the Twinterpreter process
+
+        :param cli_args: command line arguments to pass to process
+        :type cli_args: list[str] or None
+        :param env: environment to pass to process
+        :type env: dict or None
+        :returns: the spawned process
+        :rtype: :py:class:`subprocess.Popen`
+        """
         _spawn_args = []
         if isinstance(self.executable, stringabc):
             # bare interpreter - /foo/bar/python
             _spawn_args.append(self.executable)
         else:
             # invoked interpreter - [ssh foo@bar python] or [which python]
             _spawn_args.extend(self.executable)
-        if cli_args:
+        if cli_args is not None:
             _spawn_args.extend(cli_args)
         env = {} if env is None else env
         return subprocess.Popen(
@@ -114,6 +124,14 @@ def spawn(self, cli_args=None, env=None):
             env=env,
         )
 
+    def __repr__(self):
+        return '%s(executable=%r, twinterpreter_id=%r, kernel=%r)' % (
+            self.__class__.__name__,
+            self.executable,
+            self.twinterpreter_id,
+            self.kernel
+        )
+
     def __eq__(self, other):
         try:
             return self.executable == other.executable\

cpy2py/twinterpreter/twin_exceptions.py

@@ -0,0 +1,25 @@
+# - # Copyright 2016 Max Fischer
+# - #
+# - # Licensed under the Apache License, Version 2.0 (the "License");
+# - # you may not use this file except in compliance with the License.
+# - # You may obtain a copy of the License at
+# - #
+# - #     http://www.apache.org/licenses/LICENSE-2.0
+# - #
+# - # Unless required by applicable law or agreed to in writing, software
+# - # distributed under the License is distributed on an "AS IS" BASIS,
+# - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# - # See the License for the specific language governing permissions and
+# - # limitations under the License.
+# pylint: disable=too-many-ancestors,non-parent-init-called,super-init-not-called
+import cpy2py.utility.exceptions
+
+
+class TwinterpreterException(cpy2py.utility.exceptions.CPy2PyException):
+    """Exceptions relating to Twinterpreter"""
+    pass
+
+
+class TwinterpreterProcessError(TwinterpreterException):
+    """Error relating to a Twinterpreter process"""
+    pass

cpy2py/twinterpreter/twin_main.py

@@ -172,7 +172,7 @@ def __getstate__(self):
         }
 
     def __setstate__(self, state):  # pragma: no cover bootstrap
-        self.__dict__ = state
+        self.__dict__.update(state)
         self._logger = logging.getLogger('__cpy2py__.main.%s' % kernel_state.TWIN_ID)
 
     def bootstrap(self):  # pragma: no cover bootstrap

cpy2py/twinterpreter/twin_master.py

@@ -11,6 +11,7 @@
 # - # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # - # See the License for the specific language governing permissions and
 # - # limitations under the License.
+from __future__ import absolute_import
 import os
 import errno
 import threading
@@ -23,6 +24,7 @@
 
 from .twin_def import TwinDef
 from .twin_main import MainDef
+from . import twin_exceptions
 
 
 class TwinMaster(object):
@@ -129,6 +131,11 @@ def start(self):
                 cli_args=self._twin_args(my_client_ipyc=my_client_ipyc, my_server_ipyc=my_server_ipyc),
                 env=self._twin_env()
             )
+            time.sleep(0.1)  # sleep while child initializes
+            if self._process.poll() is not None:
+                raise twin_exceptions.TwinterpreterProcessError(
+                    'Twinterpreter process failed at start with %s' % self._process.poll()
+                )
             self._kernel_client = self.twin_def.kernel_client(
                 self.twin_def.twinterpreter_id,
                 ipyc=my_client_ipyc,

cpy2py/utility/compat.py

@@ -14,24 +14,26 @@
 """
 Compatibility for different python versions/interpeters
 """
-# pylint: disable=invalid-name,undefined-variable
+# pylint: disable=invalid-name,undefined-variable,redefined-builtin
 import sys
 import logging as _logging
 import subprocess as _subprocess
 
 PY3 = sys.version_info[0] == 3
 
 # pickle
-if PY3:
-    import pickle
-else:
+try:
     import cPickle as pickle
+except ImportError:
+    import pickle
 
 # range/xrange
-try:
-    rangex = xrange
-except NameError:
-    rangex = range
+if sys.version_info < (3, 3):
+    import backports.range  # py2.X requires range backport
+    range = backports.range.range
+else:
+    import builtins
+    range = builtins.range
 
 # NullHandler
 try:
@@ -101,7 +103,7 @@ def str_to_bytes(bstr):
 
 __all__ = [
     'pickle',
-    'rangex',
+    'range',
     'NullHandler',
     'check_output',
     'stringabc',

cpy2py/utility/thread_tools.py

@@ -27,7 +27,7 @@
 except ImportError:
     from _thread import error as thread_error
 
-from .compat import stringabc, inf, intern_str, unicode_str, long_int, rangex
+from .compat import stringabc, inf, intern_str, unicode_str, long_int, range
 from .exceptions import CPy2PyException
 
 
@@ -400,7 +400,7 @@ def put(self, item):
         """Put a single item in the queue"""
         with self._queue_mutex:
             self._queue_content.append(item)
-            for w_idx in rangex(len(self._waiters)):
+            for w_idx in range(len(self._waiters)):
                 try:
                     self._waiters[w_idx].release()
                 except (ThreadError, RuntimeError, thread_error):

cpy2py/utility/utils.py

@@ -13,7 +13,7 @@
 # - # limitations under the License.
 import random
 
-from .compat import rangex
+from .compat import range
 
 
 _UPPERCASE_ORD = (ord('A'), ord('Z'))
@@ -24,5 +24,5 @@ def random_str(length=16, upper_chars=0.5):
     return ''.join(
         chr(random.randint(*_UPPERCASE_ORD) if random.random() < upper_chars else random.randint(*_LOWERCASE_ORD))
         for _ in
-        rangex(length)
+        range(length)
     )

cpy2py_benchmark/bm_overhead.py

@@ -19,7 +19,7 @@
 import os
 
 from cpy2py import TwinMaster
-from cpy2py.utility.compat import rangex
+from cpy2py.utility.compat import range
 from cpy2py.utility.proc_tools import get_executable_path
 import argparse
 
@@ -98,7 +98,7 @@ def time_overhead(executable, count, total, call, reply, kernel):
     # prepare kernel
     twinterpreter = TwinMaster(executable=executable, twinterpreter_id='other', kernel=kernel)
     twinterpreter.start()
-    for _ in rangex(count):
+    for _ in range(count):
         # test
         start_time = time.time()
         twin_time = twinterpreter.execute(time.time)
@@ -159,7 +159,7 @@ def main():
             results[name][interpreter] = {}
             for kernel, kname in kernels:
                 total, call, reply = TimingVector(), TimingVector(), TimingVector()
-                for idx in rangex(tries):
+                for idx in range(tries):
                     sys.stdout.write(' '.join((
                         # what
                         'Test',