Merge pull request #61 from ccordoba12/fix-multiprocessing

PR: Patch multiprocessing to make it work when all variables are removed

.circleci/install.sh

@@ -2,7 +2,7 @@
 
 export TRAVIS_OS_NAME="linux"
 export CONDA_DEPENDENCIES_FLAGS="--quiet"
-export CONDA_DEPENDENCIES="ipykernel cloudpickle nomkl numpy pandas scipy pytest pytest-cov"
+export CONDA_DEPENDENCIES="ipykernel cloudpickle nomkl numpy pandas scipy pytest pytest-cov mock"
 export PIP_DEPENDENCIES="codecov wurlitzer"
 
 echo -e "PYTHON = $PYTHON_VERSION \n============"

.circleci/run_tests.sh

@@ -1,7 +1,9 @@
 #!/bin/bash
 
-export PATH="$HOME/miniconda/bin:$PATH"
-source activate test
+source $HOME/miniconda/etc/profile.d/conda.sh
+conda activate test
+
+pip install -e .
 
 pytest -x -vv --cov=spyder_kernels spyder_kernels
 

appveyor.yml

@@ -10,8 +10,7 @@ environment:
     PIP_DEPENDENCIES_FLAGS: "-q"
     CONDA_DEPENDENCIES_FLAGS: "--quiet"
     CONDA_DEPENDENCIES: >
-      ipykernel cloudpickle numpy pandas scipy pytest pytest-cov
-    PIP_DEPENDENCIES: "wurlitzer"
+      ipykernel cloudpickle numpy pandas scipy pytest pytest-cov mock
 
   matrix:
     - PYTHON_VERSION: "2.7"
@@ -35,6 +34,7 @@ install:
   - "powershell ci-helpers/appveyor/install-miniconda.ps1"
   - "SET PATH=%PYTHON%;%PYTHON%\\Scripts;%PATH%"
   - "activate test"
+  - "pip install -e ."
 
 build: false
 

spyder_kernels/console/tests/test_console_kernel.py

@@ -10,13 +10,16 @@
 """
 
 # Standard library imports
+import ast
 import os
 import os.path as osp
 
 # Test imports
+from ipykernel.tests.test_embed_kernel import setup_kernel
 import pytest
 
 # Local imports
+from spyder_kernels.py3compat import PY3, to_text_string
 from spyder_kernels.console.kernel import PICKLE_PROTOCOL
 from spyder_kernels.utils.iofuncs import iofunctions
 from spyder_kernels.utils.test_utils import get_kernel, get_log_text
@@ -28,6 +31,7 @@
 # Constants
 # =============================================================================
 FILES_PATH = os.path.dirname(os.path.realpath(__file__))
+TIMEOUT = 15
 
 
 # =============================================================================
@@ -301,5 +305,66 @@ def test_output_from_c_libraries(kernel, capsys):
     assert captured.out == "Hello from C\n"
 
 
+def test_cwd_in_sys_path():
+    """
+    Test that cwd stays as the first element in sys.path after the
+    kernel has started.
+    """
+    # Command to start the kernel
+    cmd = "from spyder_kernels.console import start; start.main()"
+
+    with setup_kernel(cmd) as client:
+        msg_id = client.execute("import sys; sys_path = sys.path",
+                                user_expressions={'output':'sys_path'})
+        reply = client.get_shell_msg(block=True, timeout=TIMEOUT)
+
+        # Transform value obtained through user_expressions
+        user_expressions = reply['content']['user_expressions']
+        str_value = user_expressions['output']['data']['text/plain']
+        value = ast.literal_eval(str_value)
+
+        # Assert the first value of sys_path is an empty string
+        assert value[0] == ''
+
+
+@pytest.mark.skipif(not (os.name == 'nt' and PY3),
+                    reason="Only meant for Windows and Python 3")
+def test_multiprocessing(tmpdir):
+    """
+    Test that multiprocessing works on Windows and Python 3.
+    """
+    # Command to start the kernel
+    cmd = "from spyder_kernels.console import start; start.main()"
+
+    with setup_kernel(cmd) as client:
+        # Remove all variables
+        client.execute("%reset -f")
+        client.get_shell_msg(block=True, timeout=TIMEOUT)
+
+        # Write multiprocessing code to a file
+        code = """
+from multiprocessing import Pool
+
+def f(x):
+    return x*x
+
+if __name__ == '__main__':
+    with Pool(5) as p:
+        result = p.map(f, [1, 2, 3])
+"""
+        p = tmpdir.join("mp-test.py")
+        p.write(code)
+
+        # Run code
+        client.execute("runfile(r'{}')".format(to_text_string(p)))
+        client.get_shell_msg(block=True, timeout=TIMEOUT)
+
+        # Verify that the `result` variable is defined
+        client.inspect('result')
+        msg = client.get_shell_msg(block=True, timeout=TIMEOUT)
+        content = msg['content']
+        assert content['found']
+
+
 if __name__ == "__main__":
     pytest.main()

spyder_kernels/customize/spydercustomize.py

@@ -279,6 +279,35 @@ def __init__(self, *args, **kwargs):
     pass
 
 
+# =============================================================================
+# Multiprocessing adjustments
+# =============================================================================
+# This patch is only needed on Windows and Python 3
+if os.name == 'nt' and not PY2:
+    # This could fail with changes in Python itself, so we protect it
+    # with a try/except
+    try:
+        import multiprocessing.spawn
+        _old_preparation_data = multiprocessing.spawn.get_preparation_data
+
+        def _patched_preparation_data(name):
+            """
+            Patched get_preparation_data to work when all variables are
+            removed before execution.
+            """
+            try:
+                return _old_preparation_data(name)
+            except AttributeError:
+                main_module = sys.modules['__main__']
+                # Any string for __spec__ does the job
+                main_module.__spec__ = ''
+                return _old_preparation_data(name)
+
+        multiprocessing.spawn.get_preparation_data = _patched_preparation_data
+    except Exception:
+        pass
+
+
 #==============================================================================
 # Pdb adjustments
 #==============================================================================