diff --git a/external-deps/spyder-kernels/.github/workflows/linux-pip-tests.yml b/external-deps/spyder-kernels/.github/workflows/linux-pip-tests.yml index 6b79e05b615..e88b9e6f1a6 100644 --- a/external-deps/spyder-kernels/.github/workflows/linux-pip-tests.yml +++ b/external-deps/spyder-kernels/.github/workflows/linux-pip-tests.yml @@ -26,7 +26,7 @@ jobs: strategy: fail-fast: false matrix: - PYTHON_VERSION: ['2.7', '3.8', '3.9', '3.10'] + PYTHON_VERSION: ['3.8', '3.9', '3.10'] timeout-minutes: 20 steps: - name: Checkout branch @@ -46,10 +46,6 @@ jobs: shell: bash -l {0} run: | pip install -e .[test] - # Zict >2.0.0 is not compatible with Python 3 - if [ "$PYTHON_VERSION" = "2.7" ]; then - pip install zict==2.0.0 - fi - name: Show environment information shell: bash -l {0} run: | @@ -58,9 +54,9 @@ jobs: - name: Run tests shell: bash -l {0} run: | - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/external-deps/spyder-kernels/.github/workflows/linux-tests.yml b/external-deps/spyder-kernels/.github/workflows/linux-tests.yml index ff6f449a7f0..f09003084dc 100644 --- a/external-deps/spyder-kernels/.github/workflows/linux-tests.yml +++ b/external-deps/spyder-kernels/.github/workflows/linux-tests.yml @@ -62,9 +62,9 @@ jobs: - name: Run tests shell: bash -l {0} run: | - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/external-deps/spyder-kernels/.gitrepo b/external-deps/spyder-kernels/.gitrepo index 7242b76b593..d26a9a79a88 100644 --- a/external-deps/spyder-kernels/.gitrepo +++ b/external-deps/spyder-kernels/.gitrepo @@ -4,9 +4,9 @@ ; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme ; [subrepo] - remote = https://github.com/spyder-ide/spyder-kernels.git - branch = 2.x - commit = 3b3765cea8ffdb039f9c33119f9688ebb17c0281 - parent = 34676f45e8828e2bd1a01c4e326dadf5ca8605f4 + remote = https://github.com/mrclary/spyder-kernels.git + branch = mpl390 + commit = 335583b37f50d322e7dd4ae51e1c0cfb5318bc2b + parent = ea3c29a472a4c65a7fa1336394f94fe7f00fe179 method = merge - cmdver = 0.4.3 + cmdver = 0.4.6 diff --git a/external-deps/spyder-kernels/CHANGELOG.md b/external-deps/spyder-kernels/CHANGELOG.md index 0bb2216b6b6..851f7c2a8e4 100644 --- a/external-deps/spyder-kernels/CHANGELOG.md +++ b/external-deps/spyder-kernels/CHANGELOG.md @@ -1,17 +1,121 @@ # History of changes -## Version 2.5.1 (2024-02-28) +## Version 3.0.0b6 (2024-05-15) + +### Issues Closed + +* [Issue 457](https://github.com/spyder-ide/spyder-kernels/issues/457) - Detecting the interactive backend started to fail on Mac in master ([PR 486](https://github.com/spyder-ide/spyder-kernels/pull/486) by [@ccordoba12](https://github.com/ccordoba12)) + +In this release 1 issue was closed. ### Pull Requests Merged -* [PR 479](https://github.com/spyder-ide/spyder-kernels/pull/479) - PR: Fix hangs with Maplotlib interactive backends, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 486](https://github.com/spyder-ide/spyder-kernels/pull/486) - PR: Run `test_get_interactive_backend` again on Mac, by [@ccordoba12](https://github.com/ccordoba12) ([457](https://github.com/spyder-ide/spyder-kernels/issues/457)) +* [PR 485](https://github.com/spyder-ide/spyder-kernels/pull/485) - PR: Fix Matplotlib interactive backend detection, by [@ccordoba12](https://github.com/ccordoba12) + +In this release 2 pull requests were closed. + +---- + +## Version 3.0.0b5 (2024-04-23) + +### Pull Requests Merged + +* [PR 481](https://github.com/spyder-ide/spyder-kernels/pull/481) - PR: Allow magic to edit locals while debugging, by [@impact27](https://github.com/impact27) +* [PR 480](https://github.com/spyder-ide/spyder-kernels/pull/480) - PR: Save faulthandler files under `xdg_data_home/spyder` on Linux, by [@ccordoba12](https://github.com/ccordoba12) + +In this release 2 pull requests were closed. + +---- + +## Version 3.0.0b4 (2024-02-08) + +### Pull Requests Merged + +* [PR 477](https://github.com/spyder-ide/spyder-kernels/pull/477) - PR: Handle new Inline backend options `fontsize` and `bottom`, by [@jitseniesen](https://github.com/jitseniesen) In this release 1 pull request was closed. +---- + +## Version 3.0.0b3 (2023-12-18) + +### Pull Requests Merged + +* [PR 476](https://github.com/spyder-ide/spyder-kernels/pull/476) - PR: Send back pickling error correctly, by [@impact27](https://github.com/impact27) +* [PR 467](https://github.com/spyder-ide/spyder-kernels/pull/467) - PR: Fix index when skipping hidden frames (Debugger), by [@impact27](https://github.com/impact27) +* [PR 466](https://github.com/spyder-ide/spyder-kernels/pull/466) - PR: Simplify kernel configuration, by [@impact27](https://github.com/impact27) + +In this release 3 pull requests were closed. + +---- + +## Version 3.0.0b2 (2023-08-22) + +### Pull Requests Merged + +* [PR 465](https://github.com/spyder-ide/spyder-kernels/pull/465) - Save temporary file in test to temporary location, by [@juliangilbey](https://github.com/juliangilbey) +* [PR 464](https://github.com/spyder-ide/spyder-kernels/pull/464) - Remove locals inspection, by [@impact27](https://github.com/impact27) +* [PR 460](https://github.com/spyder-ide/spyder-kernels/pull/460) - PR: Add a global filter flag to settings, by [@jsbautista](https://github.com/jsbautista) +* [PR 445](https://github.com/spyder-ide/spyder-kernels/pull/445) - PR: Add `exitdb` command and some speed optimizations to the debugger, by [@impact27](https://github.com/impact27) +* [PR 429](https://github.com/spyder-ide/spyder-kernels/pull/429) - PR: Add a comm handler decorator, by [@impact27](https://github.com/impact27) +* [PR 411](https://github.com/spyder-ide/spyder-kernels/pull/411) - PR: Remove `set_debug_state` and `do_where` calls, by [@impact27](https://github.com/impact27) + +In this release 6 pull requests were closed. + ---- +## Version 3.0.0b1 (2023-06-14) + +### Issues Closed + +* [Issue 425](https://github.com/spyder-ide/spyder-kernels/issues/425) - Possible minor issues related to post mortem debugging ([PR 444](https://github.com/spyder-ide/spyder-kernels/pull/444) by [@impact27](https://github.com/impact27)) +* [Issue 340](https://github.com/spyder-ide/spyder-kernels/issues/340) - Drop support for Python 2 ([PR 341](https://github.com/spyder-ide/spyder-kernels/pull/341) by [@impact27](https://github.com/impact27)) + +In this release 2 issues were closed. + +### Pull Requests Merged + +* [PR 456](https://github.com/spyder-ide/spyder-kernels/pull/456) - PR: Remove unnecessary code for old IPykernel versions in our tests, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 453](https://github.com/spyder-ide/spyder-kernels/pull/453) - PR: Move code that loads and saves HDF5 and DICOM files from Spyder, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 447](https://github.com/spyder-ide/spyder-kernels/pull/447) - PR: Create magics for run|debug file|cell, by [@impact27](https://github.com/impact27) +* [PR 446](https://github.com/spyder-ide/spyder-kernels/pull/446) - PR: Make call to interrupt children processes work for IPykernel greater than 6.21.2, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 444](https://github.com/spyder-ide/spyder-kernels/pull/444) - PR: Fix post mortem for debugging, by [@impact27](https://github.com/impact27) ([425](https://github.com/spyder-ide/spyder-kernels/issues/425)) +* [PR 443](https://github.com/spyder-ide/spyder-kernels/pull/443) - PR: Fix small typo in Readme, by [@davidbrochart](https://github.com/davidbrochart) +* [PR 437](https://github.com/spyder-ide/spyder-kernels/pull/437) - PR: Remove imports from __future__, by [@oscargus](https://github.com/oscargus) +* [PR 421](https://github.com/spyder-ide/spyder-kernels/pull/421) - PR: Update variable explorer from the kernel, by [@impact27](https://github.com/impact27) +* [PR 417](https://github.com/spyder-ide/spyder-kernels/pull/417) - PR: Fix error in `globalsfilter`, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 412](https://github.com/spyder-ide/spyder-kernels/pull/412) - PR: Use temporary file for faulthandler and make sure iopub is open for comms, by [@impact27](https://github.com/impact27) +* [PR 409](https://github.com/spyder-ide/spyder-kernels/pull/409) - PR: Add Python executable as part of kernel info, by [@impact27](https://github.com/impact27) +* [PR 408](https://github.com/spyder-ide/spyder-kernels/pull/408) - PR: Expose package version in SpyderShell, by [@impact27](https://github.com/impact27) +* [PR 403](https://github.com/spyder-ide/spyder-kernels/pull/403) - PR: Make debugger faster by avoiding unnecessary comm messages, by [@impact27](https://github.com/impact27) +* [PR 401](https://github.com/spyder-ide/spyder-kernels/pull/401) - PR: Wait for connection file to be written, by [@impact27](https://github.com/impact27) +* [PR 400](https://github.com/spyder-ide/spyder-kernels/pull/400) - PR: Use `execute_interactive` to print errors during tests, by [@impact27](https://github.com/impact27) +* [PR 397](https://github.com/spyder-ide/spyder-kernels/pull/397) - PR: Remove Python 2 code introduced when merging PR #395, by [@impact27](https://github.com/impact27) +* [PR 396](https://github.com/spyder-ide/spyder-kernels/pull/396) - PR: Add handlers to interrupt executions and enter the debugger after that, by [@impact27](https://github.com/impact27) +* [PR 390](https://github.com/spyder-ide/spyder-kernels/pull/390) - PR: Filter comm socket thread, by [@impact27](https://github.com/impact27) +* [PR 387](https://github.com/spyder-ide/spyder-kernels/pull/387) - PR: Minor changes to finish the migration to Python 3, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 366](https://github.com/spyder-ide/spyder-kernels/pull/366) - PR: Print warning when using `global` in an empty namespace, by [@impact27](https://github.com/impact27) +* [PR 341](https://github.com/spyder-ide/spyder-kernels/pull/341) - PR: Drop support for Python 2, by [@impact27](https://github.com/impact27) ([340](https://github.com/spyder-ide/spyder-kernels/issues/340)) +* [PR 339](https://github.com/spyder-ide/spyder-kernels/pull/339) - PR: Use control channel for comms, by [@impact27](https://github.com/impact27) +* [PR 286](https://github.com/spyder-ide/spyder-kernels/pull/286) - PR: Improve and refactor the way we run and debug code, by [@impact27](https://github.com/impact27) +* [PR 257](https://github.com/spyder-ide/spyder-kernels/pull/257) - PR: Notify frontend of Matplotlib backend change, by [@impact27](https://github.com/impact27) +* [PR 171](https://github.com/spyder-ide/spyder-kernels/pull/171) - PR: Publish Pdb stack frames to Spyder, by [@impact27](https://github.com/impact27) + +In this release 25 pull requests were closed. + +---- + +## Version 2.5.1 (2024-02-28) + +### Pull Requests Merged + +* [PR 479](https://github.com/spyder-ide/spyder-kernels/pull/479) - PR: Fix hangs with Maplotlib interactive backends, by [@ccordoba12](https://github.com/ccordoba12) + +---- + ## Version 2.5.0 (2023-11-06) ### New features @@ -31,9 +135,7 @@ In this release 1 pull request was closed. In this release 6 pull requests were closed. - ---- - +---- ## Version 2.4.4 (2023-06-29) @@ -52,10 +154,8 @@ In this release 1 issue was closed. In this release 4 pull requests were closed. - ---- - ## Version 2.4.3 (2023-04-02) ### Issues Closed @@ -76,10 +176,8 @@ In this release 1 issue was closed. In this release 7 pull requests were closed. - ---- - ## Version 2.4.2 (2023-01-17) ### Issues Closed @@ -95,10 +193,8 @@ In this release 1 issue was closed. In this release 2 pull requests were closed. - ---- - ## Version 2.4.1 (2022-12-29) ### Issues Closed @@ -113,10 +209,8 @@ In this release 1 issue was closed. In this release 1 pull request was closed. - ---- - ## Version 2.4.0 (2022-11-02) ### New features @@ -137,10 +231,8 @@ In this release 1 pull request was closed. In this release 8 pull requests were closed. - ---- - ## Version 2.3.3 (2022-08-28) ### Issues Closed @@ -156,10 +248,8 @@ In this release 1 issue was closed. In this release 2 pull requests were closed. - ---- - ## Version 2.3.2 (2022-07-06) ### Issues Closed @@ -181,10 +271,8 @@ In this release 1 issue was closed. In this release 8 pull requests were closed. - ---- - ## Version 2.3.1 (2022-05-21) ### Pull Requests Merged @@ -195,10 +283,8 @@ In this release 8 pull requests were closed. In this release 3 pull requests were closed. - --- - ## Version 2.3.0 (2022-03-30) ### New features @@ -230,10 +316,8 @@ In this release 4 issues were closed. In this release 10 pull requests were closed. - ---- - ## Version 2.2.1 (2022-01-13) ### Issues Closed @@ -266,10 +350,8 @@ In this release 4 issues were closed. In this release 16 pull requests were closed. - ---- - ## Version 2.2.0 (2021-11-22) ### New features @@ -301,10 +383,8 @@ In this release 3 issues were closed. In this release 10 pull requests were closed. - ---- - ## Version 2.1.3 (2021-10-02) ### Pull Requests Merged @@ -313,10 +393,8 @@ In this release 10 pull requests were closed. In this release 1 pull request was closed. - ---- - ## Version 2.1.2 (2021-09-28) ### Pull Requests Merged @@ -326,10 +404,8 @@ In this release 1 pull request was closed. In this release 2 pull requests were closed. - ---- - ## Version 2.1.1 (2021-09-01) ### Pull Requests Merged @@ -341,10 +417,8 @@ In this release 2 pull requests were closed. In this release 4 pull requests were closed. - ---- - ## Version 2.1.0 (2021-07-31) ### New features @@ -369,10 +443,8 @@ In this release 1 issue was closed. In this release 5 pull requests were closed. - ---- - ## Version 2.0.5 (2021-07-03) ### Pull Requests Merged @@ -381,10 +453,8 @@ In this release 5 pull requests were closed. In this release 1 pull request was closed. - ---- - ## Version 2.0.4 (2021-06-10) ### Issues Closed @@ -405,10 +475,8 @@ In this release 1 issue was closed. In this release 7 pull requests were closed. - ---- - ## Version 2.0.3 (2021-05-15) ### Pull Requests Merged @@ -417,10 +485,8 @@ In this release 7 pull requests were closed. In this release 1 pull request was closed. - ---- - ## Version 2.0.2 (2021-05-02) ### Pull Requests Merged @@ -431,18 +497,14 @@ In this release 1 pull request was closed. In this release 3 pull requests were closed. - ---- - ## Version 2.0.1 (2021-04-02) * This release also contains all fixes present in version 1.10.3 - ---- - ## Version 2.0.0 (2021-04-01) ### New features @@ -456,10 +518,8 @@ In this release 3 pull requests were closed. In this release 2 pull requests were closed. - ---- - ## Version 1.10.3 (2021-04-02) ### Pull Requests Merged @@ -468,10 +528,8 @@ In this release 2 pull requests were closed. In this release 1 pull request was closed. - ---- - ## Version 1.10.2 (2021-02-21) ### Pull Requests Merged @@ -484,10 +542,8 @@ In this release 1 pull request was closed. In this release 5 pull requests were closed. - ---- - ## Version 1.10.1 (2020-12-18) ### Issues Closed @@ -511,10 +567,8 @@ In this release 2 issues were closed. In this release 9 pull requests were closed. - ---- - ## Version 1.10.0 (2020-11-08) ### New features diff --git a/external-deps/spyder-kernels/README.md b/external-deps/spyder-kernels/README.md index 410e2fd51b5..b6307c10e96 100644 --- a/external-deps/spyder-kernels/README.md +++ b/external-deps/spyder-kernels/README.md @@ -8,7 +8,7 @@ Package that provides Jupyter kernels for use with the consoles of Spyder, the Scientific Python Development Environment. -These kernels can launched either through Spyder itself or in an independent +These kernels can be launched either through Spyder itself or in an independent Python session, and allow for interactive or file-based execution of Python code inside Spyder. diff --git a/external-deps/spyder-kernels/requirements/posix.txt b/external-deps/spyder-kernels/requirements/posix.txt index fc3b9f66612..0c22fa5774f 100644 --- a/external-deps/spyder-kernels/requirements/posix.txt +++ b/external-deps/spyder-kernels/requirements/posix.txt @@ -4,3 +4,4 @@ ipython>=8.12.2,<9 jupyter_client>=7.4.9,<9 pyzmq>=24.0.0 wurlitzer>=1.0.3 +pyxdg>=0.26 diff --git a/external-deps/spyder-kernels/requirements/python-27.txt b/external-deps/spyder-kernels/requirements/python-27.txt deleted file mode 100644 index c0d798825c3..00000000000 --- a/external-deps/spyder-kernels/requirements/python-27.txt +++ /dev/null @@ -1,11 +0,0 @@ -decorator<5 -backports.functools_lru_cache -cloudpickle -ipykernel>=4.5,<5 -jupyter_client>=5.3.4,<6 -pyzmq>=17,<20 -wurlitzer>=1.0.3 -# To avoid an error with conda -click =7 -# To avoid a problem with zict -zict <2.1.0 diff --git a/external-deps/spyder-kernels/requirements/tests.txt b/external-deps/spyder-kernels/requirements/tests.txt index 0bd4e273732..42ef0167b39 100644 --- a/external-deps/spyder-kernels/requirements/tests.txt +++ b/external-deps/spyder-kernels/requirements/tests.txt @@ -11,3 +11,5 @@ scipy xarray pillow django +h5py +pydicom diff --git a/external-deps/spyder-kernels/setup.cfg b/external-deps/spyder-kernels/setup.cfg deleted file mode 100644 index e606cea4f59..00000000000 --- a/external-deps/spyder-kernels/setup.cfg +++ /dev/null @@ -1,3 +0,0 @@ -[bdist_wheel] -# Code is written to work on both Python 2 and Python 3. -universal=1 diff --git a/external-deps/spyder-kernels/setup.py b/external-deps/spyder-kernels/setup.py index e2873c98e4f..79689b46055 100644 --- a/external-deps/spyder-kernels/setup.py +++ b/external-deps/spyder-kernels/setup.py @@ -36,21 +36,14 @@ def get_version(module='spyder_kernels'): REQUIREMENTS = [ - 'decorator<5; python_version<"3"', - 'backports.functools-lru-cache; python_version<"3"', 'cloudpickle', - 'ipykernel>=4.5,<5; python_version<"3"', - 'ipykernel>=6.16.1,<6.17; python_version<"3.8"', - 'ipykernel>=6.29.3,<7; python_version>="3.8"', - 'ipython<6; python_version<"3"', - 'ipython>=7.31.1,<8; python_version<"3.8"', + 'ipykernel>=6.29.3,<7', 'ipython>=8.12.2,<8.13; python_version=="3.8"', 'ipython>=8.13.0,<9,!=8.17.1; python_version>"3.8"', - 'jupyter-client>=5.3.4,<6; python_version<"3"', - 'jupyter-client>=7.4.9,<9; python_version>="3"', - 'pyzmq>=17,<20; python_version<"3"', - 'pyzmq>=24.0.0; python_version>="3"', + 'jupyter-client>=7.4.9,<9', + 'pyzmq>=24.0.0', 'wurlitzer>=1.0.3;platform_system!="Windows"', + 'pyxdg>=0.26;platform_system=="Linux"', ] TEST_REQUIREMENTS = [ @@ -67,6 +60,8 @@ def get_version(module='spyder_kernels'): 'xarray', 'pillow', 'django', + 'h5py', + 'pydicom' ] setup( @@ -85,6 +80,7 @@ def get_version(module='spyder_kernels'): install_requires=REQUIREMENTS, extras_require={'test': TEST_REQUIREMENTS}, include_package_data=True, + python_requires='>=3.8', classifiers=[ 'Development Status :: 5 - Production/Stable', 'Framework :: Jupyter', @@ -92,10 +88,7 @@ def get_version(module='spyder_kernels'): 'Intended Audience :: Developers', 'License :: OSI Approved :: MIT License', 'Operating System :: OS Independent', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', - 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: 3.8', 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', diff --git a/external-deps/spyder-kernels/spyder_kernels/_version.py b/external-deps/spyder-kernels/spyder_kernels/_version.py index f4932aeef12..788cf5f4779 100644 --- a/external-deps/spyder-kernels/spyder_kernels/_version.py +++ b/external-deps/spyder-kernels/spyder_kernels/_version.py @@ -8,5 +8,5 @@ """Version File.""" -VERSION_INFO = (2, 6, 0, 'dev0') +VERSION_INFO = (3, 0, 0, 'dev0') __version__ = '.'.join(map(str, VERSION_INFO)) diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/commbase.py b/external-deps/spyder-kernels/spyder_kernels/comms/commbase.py index b4dea83b1e7..20e3a8a3dca 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/commbase.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/commbase.py @@ -50,8 +50,6 @@ 'call_name': The function name (mostly for debugging) } """ -from __future__ import print_function - import cloudpickle import pickle import logging @@ -59,13 +57,10 @@ import uuid import traceback -from spyder_kernels.py3compat import PY2, PY3 - logger = logging.getLogger(__name__) -# To be able to get and set variables between Python 2 and 3 -DEFAULT_PICKLE_PROTOCOL = 2 +DEFAULT_PICKLE_PROTOCOL = 4 # Max timeout (in secs) for blocking calls TIMEOUT = 3 @@ -132,7 +127,7 @@ def comm_excepthook(type, value, tb): sys.excepthook = comm_excepthook -class CommBase(object): +class CommBase: """ Class with the necessary attributes and methods to handle communications between a kernel and a frontend. @@ -154,8 +149,6 @@ def __init__(self): 'remote_call', self._handle_remote_call) self._register_message_handler( 'remote_call_reply', self._handle_remote_call_reply) - self.register_call_handler('_set_pickle_protocol', - self._set_pickle_protocol) def get_comm_id_list(self, comm_id=None): """Get a list of comms id.""" @@ -182,18 +175,6 @@ def is_open(self, comm_id=None): return len(self._comms) > 0 return comm_id in self._comms - def is_ready(self, comm_id=None): - """ - Check to see if the other side replied. - - The check is made with _set_pickle_protocol as this is the first call - made. If comm_id is not specified, check all comms. - """ - id_list = self.get_comm_id_list(comm_id) - if len(id_list) == 0: - return False - return all([self._comms[cid]['status'] == 'ready' for cid in id_list]) - def register_call_handler(self, call_name, handler): """ Register a remote call handler. @@ -203,15 +184,21 @@ def register_call_handler(self, call_name, handler): call_name : str The name of the called function. handler : callback - A function to handle the request, or `None` to unregister - `call_name`. + A function to handle the request. """ - if not handler: - self._remote_call_handlers.pop(call_name, None) - return - self._remote_call_handlers[call_name] = handler + def unregister_call_handler(self, call_name): + """ + Unegister a remote call handler. + + Parameters + ---------- + call_name : str + The name of the called function. + """ + self._remote_call_handlers.pop(call_name, None) + def remote_call(self, comm_id=None, callback=None, **settings): """Get a handler for remote calls.""" return RemoteCallFactory(self, comm_id, callback, **settings) @@ -252,7 +239,6 @@ def _set_pickle_protocol(self, protocol): """Set the pickle protocol used to send data.""" protocol = min(protocol, pickle.HIGHEST_PROTOCOL) self._comms[self.calling_comm_id]['pickle_protocol'] = protocol - self._comms[self.calling_comm_id]['status'] = 'ready' @property def _comm_name(self): @@ -309,15 +295,7 @@ def _comm_message(self, msg): # Load the buffer. Only one is supported. try: - if PY3: - # https://docs.python.org/3/library/pickle.html#pickle.loads - # Using encoding='latin1' is required for unpickling - # NumPy arrays and instances of datetime, date and time - # pickled by Python 2. - buffer = cloudpickle.loads(msg['buffers'][0], - encoding='latin-1') - else: - buffer = cloudpickle.loads(msg['buffers'][0]) + buffer = cloudpickle.loads(msg['buffers'][0]) except Exception as e: logger.debug( "Exception in cloudpickle.loads : %s" % str(e)) @@ -339,6 +317,10 @@ def _handle_remote_call(self, msg, buffer): """Handle a remote call.""" msg_dict = msg['content'] self.on_incoming_call(msg_dict) + if msg['content'].get('is_error', False): + # could not open the pickle + self._set_call_return_value(msg, buffer, is_error=True) + return try: return_value = self._remote_callback( msg_dict['call_name'], @@ -404,18 +386,20 @@ def on_incoming_call(self, call_dict): if "pickle_highest_protocol" in call_dict: self._set_pickle_protocol(call_dict["pickle_highest_protocol"]) - def _get_call_return_value(self, call_dict, call_data, comm_id): + def _send_call(self, call_dict, call_data, comm_id): + """Send call.""" + call_dict = self.on_outgoing_call(call_dict) + self._send_message( + 'remote_call', content=call_dict, data=call_data, + comm_id=comm_id) + + def _get_call_return_value(self, call_dict, comm_id): """ Send a remote call and return the reply. If settings['blocking'] == True, this will wait for a reply and return the replied value. """ - call_dict = self.on_outgoing_call(call_dict) - self._send_message( - 'remote_call', content=call_dict, data=call_data, - comm_id=comm_id) - settings = call_dict['settings'] blocking = 'blocking' in settings and settings['blocking'] @@ -432,7 +416,7 @@ def _get_call_return_value(self, call_dict, call_data, comm_id): else: timeout = TIMEOUT - self._wait_reply(call_id, call_name, timeout) + self._wait_reply(comm_id, call_id, call_name, timeout) reply = self._reply_inbox.pop(call_id) @@ -441,7 +425,7 @@ def _get_call_return_value(self, call_dict, call_data, comm_id): return reply['value'] - def _wait_reply(self, call_id, call_name, timeout): + def _wait_reply(self, comm_id, call_id, call_name, timeout): """ Wait for the other side reply. """ @@ -496,7 +480,7 @@ def _sync_error(self, error_wrapper): error_wrapper.raise_error() -class RemoteCallFactory(object): +class RemoteCallFactory: """Class to create `RemoteCall`s.""" def __init__(self, comms_wrapper, comm_id, callback, **settings): @@ -554,5 +538,6 @@ def __call__(self, *args, **kwargs): logger.debug("Call to unconnected comm: %s" % self._name) return self._comms_wrapper._register_call(call_dict, self._callback) + self._comms_wrapper._send_call(call_dict, call_data, self._comm_id) return self._comms_wrapper._get_call_return_value( - call_dict, call_data, self._comm_id) + call_dict, self._comm_id) diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/decorators.py b/external-deps/spyder-kernels/spyder_kernels/comms/decorators.py new file mode 100644 index 00000000000..b2633404a07 --- /dev/null +++ b/external-deps/spyder-kernels/spyder_kernels/comms/decorators.py @@ -0,0 +1,27 @@ +# -*- coding: utf-8 -*- +# +# Copyright © Spyder Project Contributors +# Licensed under the terms of the MIT License +# (see spyder/__init__.py for details) + +""" +Comms decorators. +""" + + +def comm_handler(fun): + """Decorator to mark comm handler methods.""" + fun._is_comm_handler = True + return fun + + +def register_comm_handlers(instance, frontend_comm): + """ + Registers an instance whose methods have been marked with comm_handler. + """ + for method_name in instance.__class__.__dict__: + method = getattr(instance, method_name) + if hasattr(method, '_is_comm_handler'): + frontend_comm.register_call_handler( + method_name, method) + diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py index edf4adb8193..9eaf2ee1f43 100644 --- a/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py +++ b/external-deps/spyder-kernels/spyder_kernels/comms/frontendcomm.py @@ -9,33 +9,16 @@ - Implements _wait_reply, so blocking calls can be made. """ -import pickle -import socket +import asyncio import sys import threading import time from IPython.core.getipython import get_ipython -from jupyter_client.localinterfaces import localhost -from tornado import ioloop import zmq from spyder_kernels.comms.commbase import CommBase, CommError -from spyder_kernels.py3compat import TimeoutError, PY2 - - -if PY2: - import thread - - -def get_free_port(): - """Find a free port on the local machine.""" - sock = socket.socket() - sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, b'\0' * 8) - sock.bind((localhost(), 0)) - port = sock.getsockname()[1] - sock.close() - return port +from spyder_kernels.comms.utils import WriteContext def frontend_request(blocking, timeout=None): @@ -63,44 +46,9 @@ def __init__(self, kernel): self.kernel = kernel self.kernel.comm_manager.register_target( self._comm_name, self._comm_open) - - self.comm_port = None - self.register_call_handler('_send_comm_config', - self._send_comm_config) - - self.comm_lock = threading.RLock() - - # self.kernel.parent is IPKernelApp unless we are in tests - if self.kernel.parent: - # Create a new socket - self.context = zmq.Context() - self.comm_socket = self.context.socket(zmq.ROUTER) - self.comm_socket.linger = 1000 - - self.comm_port = get_free_port() - - self.comm_port = self.kernel.parent._bind_socket( - self.comm_socket, self.comm_port) - if hasattr(zmq, 'ROUTER_HANDOVER'): - # Set router-handover to workaround zeromq reconnect problems - # in certain rare circumstances. - # See ipython/ipykernel#270 and zeromq/libzmq#2892 - self.comm_socket.router_handover = 1 - - self.comm_thread_close = threading.Event() - self.comm_socket_thread = threading.Thread(target=self.poll_thread) - self.comm_socket_thread.start() - - # Patch parent.close . This function only exists in Python 3. - if not PY2: - parent_close = self.kernel.parent.close - - def close(): - """Close comm_socket_thread.""" - self.close_thread() - parent_close() - - self.kernel.parent.close = close + self.comm_lock = threading.Lock() + self._cached_messages = {} + self._pending_comms = {} def close(self, comm_id=None): """Close the comm and notify the other side.""" @@ -112,21 +60,6 @@ def _send_message(self, *args, **kwargs): with self.comm_lock: return super(FrontendComm, self)._send_message(*args, **kwargs) - def close_thread(self): - """Close comm.""" - self.comm_thread_close.set() - self.comm_socket.close() - self.context.term() - self.comm_socket_thread.join() - - def poll_thread(self): - """Receive messages from comm socket.""" - if not PY2: - # Create an event loop for the handlers. - ioloop.IOLoop().initialize() - while not self.comm_thread_close.is_set(): - self.poll_one() - def poll_one(self): """Receive one message from comm socket.""" out_stream = None @@ -135,7 +68,8 @@ def poll_one(self): # use the regular shell stream. out_stream = self.kernel.shell_streams[0] try: - ident, msg = self.kernel.session.recv(self.comm_socket, 0) + ident, msg = self.kernel.session.recv( + self.kernel.parent.control_socket, 0) except zmq.error.ContextTerminated: return except Exception: @@ -143,32 +77,12 @@ def poll_one(self): return msg_type = msg['header']['msg_type'] - if msg_type == 'shutdown_request': - self.comm_thread_close.set() - self._comm_close(msg) + handler = self.kernel.control_handlers.get(msg_type, None) + if handler is None: + self.kernel.log.warning("Unknown message type: %r", msg_type) return - - handler = self.kernel.shell_handlers.get(msg_type, None) try: - if handler is None: - self.kernel.log.warning("Unknown message type: %r", msg_type) - return - if PY2: - handler(out_stream, ident, msg) - return - - import asyncio - - if (getattr(asyncio, 'run', False) and - asyncio.iscoroutinefunction(handler)): - # This is needed for ipykernel 6+ - asyncio.run(handler(out_stream, ident, msg)) - else: - # This is required for Python 3.6, which doesn't have - # asyncio.run or ipykernel versions less than 6. The - # nice thing is that ipykernel 6, which requires - # asyncio, doesn't support Python 3.6. - handler(out_stream, ident, msg) + asyncio.run(handler(out_stream, ident, msg)) except Exception: self.kernel.log.error( "Exception in message handler:", exc_info=True) @@ -180,13 +94,14 @@ def poll_one(self): out_stream.flush(zmq.POLLOUT) def remote_call(self, comm_id=None, blocking=False, callback=None, - timeout=None): + timeout=None, display_error=False): """Get a handler for remote calls.""" return super(FrontendComm, self).remote_call( blocking=blocking, comm_id=comm_id, callback=callback, - timeout=timeout) + timeout=timeout, + display_error=display_error) def wait_until(self, condition, timeout=None): """Wait until condition is met. Returns False if timeout.""" @@ -196,7 +111,7 @@ def wait_until(self, condition, timeout=None): while not condition(): if timeout is not None and time.time() > t_start + timeout: return False - if threading.current_thread() is self.comm_socket_thread: + if threading.current_thread() is self.kernel.parent.control_thread: # Wait for a reply on the comm channel. self.poll_one() else: @@ -204,15 +119,52 @@ def wait_until(self, condition, timeout=None): time.sleep(0.01) return True + def cache_message(self, comm_id, msg): + """Message from a comm that might be opened later.""" + if comm_id not in self._cached_messages: + self._cached_messages[comm_id] = [] + self._cached_messages[comm_id].append(msg) + # --- Private -------- - def _wait_reply(self, call_id, call_name, timeout, retry=True): + def _check_comm_reply(self): + """ + Send comm message to frontend to check if the iopub channel is ready + """ + # Make sure the length doesn't change during iteration + pending_comms = list(self._pending_comms.values()) + if len(pending_comms) == 0: + return + for comm in pending_comms: + self._notify_comm_ready(comm) + self.kernel.io_loop.call_later(1, self._check_comm_reply) + + def _notify_comm_ready(self, comm): + """Send messages about comm readiness to frontend.""" + self.remote_call( + comm_id=comm.comm_id, + callback=self._comm_ready_callback + )._comm_ready() + + def _comm_ready_callback(self, ret): + """A comm has replied, so process all cached messages related to it.""" + comm = self._pending_comms.pop(self.calling_comm_id, None) + if not comm: + return + # Cached messages for that comm + if comm.comm_id in self._cached_messages: + for msg in self._cached_messages[comm.comm_id]: + comm.handle_msg(msg) + self._cached_messages.pop(comm.comm_id) + + + def _wait_reply(self, comm_id, call_id, call_name, timeout, retry=True): """Wait until the frontend replies to a request.""" def reply_received(): """The reply is there!""" return call_id in self._reply_inbox if not self.wait_until(reply_received): if retry: - self._wait_reply(call_id, call_name, timeout, False) + self._wait_reply(comm_id, call_id, call_name, timeout, False) return raise TimeoutError( "Timeout while waiting for '{}' reply.".format( @@ -224,18 +176,14 @@ def _comm_open(self, comm, msg): """ self.calling_comm_id = comm.comm_id self._register_comm(comm) - self._set_pickle_protocol(msg['content']['data']['pickle_protocol']) - self._send_comm_config() - - def on_outgoing_call(self, call_dict): - """A message is about to be sent""" - call_dict["comm_port"] = self.comm_port - return super(FrontendComm, self).on_outgoing_call(call_dict) + self._set_pickle_protocol( + msg['content']['data']['pickle_highest_protocol']) - def _send_comm_config(self): - """Send the comm config to the frontend.""" - self.remote_call()._set_comm_port(self.comm_port) - self.remote_call()._set_pickle_protocol(pickle.HIGHEST_PROTOCOL) + # IOPub might not be connected yet, keep sending messages until a + # reply is received. + self._pending_comms[comm.comm_id] = comm + self._notify_comm_ready(comm) + self.kernel.io_loop.call_later(.3, self._check_comm_reply) def _comm_close(self, msg): """Close comm.""" @@ -263,60 +211,7 @@ def handle_msg(msg): def _remote_callback(self, call_name, call_args, call_kwargs): """Call the callback function for the remote call.""" - with self.comm_lock: - current_stdout = sys.stdout - current_stderr = sys.stderr - saved_stdout_write = current_stdout.write - saved_stderr_write = current_stderr.write - thread_id = thread.get_ident() if PY2 else threading.get_ident() - current_stdout.write = WriteWrapper( - saved_stdout_write, call_name, thread_id) - current_stderr.write = WriteWrapper( - saved_stderr_write, call_name, thread_id) - try: - return super(FrontendComm, self)._remote_callback( - call_name, call_args, call_kwargs) - finally: - current_stdout.write = saved_stdout_write - current_stderr.write = saved_stderr_write - - -class WriteWrapper(object): - """Wrapper to warn user when text is printed.""" - - def __init__(self, write, name, thread_id): - self._write = write - self._name = name - self._thread_id = thread_id - self._warning_shown = False - - def is_benign_message(self, message): - """Determine if a message is benign in order to filter it.""" - benign_messages = [ - # Fixes spyder-ide/spyder#14928 - # Fixes spyder-ide/spyder-kernels#343 - 'DeprecationWarning', - # Fixes spyder-ide/spyder-kernels#365 - 'IOStream.flush timed out' - ] - - return any([msg in message for msg in benign_messages]) - - def __call__(self, string): - """Print warning once.""" - thread_id = thread.get_ident() if PY2 else threading.get_ident() - if self._thread_id != thread_id: - return self._write(string) - - if not self.is_benign_message(string): - if not self._warning_shown: - self._warning_shown = True - - # Don't print handler name for `show_mpl_backend_errors` - # because we have a specific message for it. - if repr(self._name) != "'show_mpl_backend_errors'": - self._write( - "\nOutput from spyder call " + repr(self._name) + ":\n" - ) + with WriteContext(call_name): + return super(FrontendComm, self)._remote_callback( + call_name, call_args, call_kwargs) - return self._write(string) diff --git a/external-deps/spyder-kernels/spyder_kernels/comms/utils.py b/external-deps/spyder-kernels/spyder_kernels/comms/utils.py new file mode 100644 index 00000000000..003323499af --- /dev/null +++ b/external-deps/spyder-kernels/spyder_kernels/comms/utils.py @@ -0,0 +1,82 @@ +# -*- coding: utf-8 -*- +# ----------------------------------------------------------------------------- +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- +""" +Comms Utilities +""" + +import sys +import threading + + +class WriteContext(object): + class_lock = threading.RLock() + def __init__(self, prefix): + self.prefix = prefix + + def __enter__(self): + self.class_lock.acquire() + + self.files = [sys.stdout, sys.stderr] + self.saved_writes = [f.write for f in self.files] + + thread_id = threading.get_ident() + + for f in self.files: + f.write = WriteWrapper(f.write, self.prefix, thread_id) + + def __exit__(self, exc_type, exc_value, traceback): + try: + for f, old_write in zip(self.files, self.saved_writes): + f.write = old_write + finally: + self.class_lock.release() + + +class WriteWrapper(object): + """Wrapper to warn user when text is printed.""" + + def __init__(self, write, name, thread_id): + self._write = write + self._name = name + self._thread_id = thread_id + self._warning_shown = False + + def is_benign_message(self, message): + """Determine if a message is benign in order to filter it.""" + benign_messages = [ + # Fixes spyder-ide/spyder#14928 + # Fixes spyder-ide/spyder-kernels#343 + 'DeprecationWarning', + # Fixes spyder-ide/spyder-kernels#365 + 'IOStream.flush timed out', + # Avoid unnecessary messages from set_configuration when changing + # Matplotlib options. + "Warning: Cannot change to a different GUI toolkit", + "%pylab is deprecated", + "Populating the interactive namespace", + "\n" + ] + + return any([msg in message for msg in benign_messages]) + + def __call__(self, string): + """Print warning once.""" + if self._thread_id != threading.get_ident(): + return self._write(string) + + if not self.is_benign_message(string): + if not self._warning_shown: + self._warning_shown = True + + # request_pdb_stop is expected to print messages. + if self._name not in ['request_pdb_stop']: + self._write( + "\nOutput from spyder call " + repr(self._name) + ":\n" + ) + + return self._write(string) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py index 306d2afab2d..15a407aa843 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/kernel.py @@ -11,29 +11,34 @@ """ # Standard library imports +import faulthandler +import json import logging import os +import re import sys +import traceback +import tempfile import threading # Third-party imports from ipykernel.ipkernel import IPythonKernel -from ipykernel import eventloops +from ipykernel import get_connection_info from traitlets.config.loader import LazyConfigValue +import zmq +from zmq.utils.garbage import gc # Local imports -from spyder_kernels.py3compat import ( - TEXT_TYPES, to_text_string, PY3) +import spyder_kernels from spyder_kernels.comms.frontendcomm import FrontendComm +from spyder_kernels.comms.decorators import ( + register_comm_handlers, comm_handler) from spyder_kernels.utils.iofuncs import iofunctions -from spyder_kernels.utils.mpl import ( - MPL_BACKENDS_FROM_SPYDER, MPL_BACKENDS_TO_SPYDER, INLINE_FIGURE_FORMATS) +from spyder_kernels.utils.mpl import automatic_backend, MPL_BACKENDS_TO_SPYDER from spyder_kernels.utils.nsview import ( get_remote_data, make_remote_view, get_size) from spyder_kernels.console.shell import SpyderShell - -if PY3: - import faulthandler +from spyder_kernels.comms.utils import WriteContext logger = logging.getLogger(__name__) @@ -56,55 +61,39 @@ def __init__(self, *args, **kwargs): self.frontend_comm = FrontendComm(self) # All functions that can be called through the comm - handlers = { - 'set_breakpoints': self.set_spyder_breakpoints, - 'set_pdb_ignore_lib': self.set_pdb_ignore_lib, - 'set_pdb_execute_events': self.set_pdb_execute_events, - 'set_pdb_use_exclamation_mark': self.set_pdb_use_exclamation_mark, - 'get_value': self.get_value, - 'load_data': self.load_data, - 'save_namespace': self.save_namespace, - 'is_defined': self.is_defined, - 'get_doc': self.get_doc, - 'get_source': self.get_source, - 'set_value': self.set_value, - 'remove_value': self.remove_value, - 'copy_value': self.copy_value, - 'set_cwd': self.set_cwd, - 'get_cwd': self.get_cwd, - 'get_syspath': self.get_syspath, - 'get_env': self.get_env, - 'close_all_mpl_figures': self.close_all_mpl_figures, - 'show_mpl_backend_errors': self.show_mpl_backend_errors, - 'get_namespace_view': self.get_namespace_view, - 'set_namespace_view_settings': self.set_namespace_view_settings, - 'get_var_properties': self.get_var_properties, - 'set_sympy_forecolor': self.set_sympy_forecolor, - 'update_syspath': self.update_syspath, - 'is_special_kernel_valid': self.is_special_kernel_valid, - 'get_matplotlib_backend': self.get_matplotlib_backend, - 'get_mpl_interactive_backend': self.get_mpl_interactive_backend, - 'pdb_input_reply': self.pdb_input_reply, - '_interrupt_eventloop': self._interrupt_eventloop, - 'enable_faulthandler': self.enable_faulthandler, - } - for call_id in handlers: - self.frontend_comm.register_call_handler( - call_id, handlers[call_id]) + register_comm_handlers(self, self.frontend_comm) + register_comm_handlers(self.shell, self.frontend_comm) self.namespace_view_settings = {} - self._mpl_backend_error = None - self._running_namespace = None self.faulthandler_handle = None + self._cwd_initialised = False + + # Add handlers to control to process messages while debugging + self.control_handlers['comm_msg'] = self.control_comm_msg + self.control_handlers['complete_request'] = self.shell_handlers[ + 'complete_request'] + + # Socket to signal shell_stream locally + self.loopback_socket = None + + # To track the interactive backend + self.interactive_backend = None + + @property + def kernel_info(self): + # Used for checking correct version by spyder + infos = super().kernel_info + infos.update({ + "spyder_kernels_info": ( + spyder_kernels.__version__, + sys.executable + ) + }) + return infos # -- Public API ----------------------------------------------------------- - def do_shutdown(self, restart): - """Disable faulthandler if enabled before proceeding.""" - self.disable_faulthandler() - super(SpyderKernel, self).do_shutdown(restart) - def frontend_call(self, blocking=False, broadcast=True, - timeout=None, callback=None): + timeout=None, callback=None, display_error=False): """Call the frontend.""" # If not broadcast, send only to the calling comm if broadcast: @@ -116,45 +105,181 @@ def frontend_call(self, blocking=False, broadcast=True, blocking=blocking, comm_id=comm_id, callback=callback, - timeout=timeout) + timeout=timeout, + display_error=display_error) + + def get_state(self): + """"get current state to send to the frontend""" + state = {} + with WriteContext("get_state"): + if self._cwd_initialised: + state["cwd"] = self.get_cwd() + state["namespace_view"] = self.get_namespace_view() + state["var_properties"] = self.get_var_properties() + return state + + def publish_state(self): + """Publish the current kernel state""" + if not self.frontend_comm.is_open(): + # No one to send to + return + try: + self.frontend_call(blocking=False).update_state(self.get_state()) + except Exception: + pass - def enable_faulthandler(self, fn): + def enable_faulthandler(self): """ Open a file to save the faulthandling and identifiers for internal threads. """ - if not PY3: - # Not implemented + fault_dir = None + if sys.platform.startswith('linux'): + # Do not use /tmp for temporary files + try: + from xdg.BaseDirectory import xdg_data_home + fault_dir = os.path.join(xdg_data_home, "spyder") + os.makedirs(fault_dir, exist_ok=True) + except Exception: + fault_dir = None + + self.faulthandler_handle = tempfile.NamedTemporaryFile( + 'wt', suffix='.fault', dir=fault_dir + ) + + main_id = threading.main_thread().ident + system_ids = [ + thread.ident for thread in threading.enumerate() + if thread is not threading.main_thread() + ] + faulthandler.enable(self.faulthandler_handle) + return self.faulthandler_handle.name, main_id, system_ids + + @comm_handler + def safe_exec(self, filename): + """Safely execute a file using IPKernelApp._exec_file.""" + self.parent._exec_file(filename) + + @comm_handler + def get_fault_text(self, fault_filename, main_id, ignore_ids): + """Get fault text from old run.""" + # Read file + try: + with open(fault_filename, 'r') as f: + fault = f.read() + except FileNotFoundError: return - self.disable_faulthandler() - f = open(fn, 'w') - self.faulthandler_handle = f - f.write("Main thread id:\n") - f.write(hex(threading.main_thread().ident)) - f.write('\nSystem threads ids:\n') - f.write(" ".join([hex(thread.ident) for thread in threading.enumerate() - if thread is not threading.main_thread()])) - f.write('\n') - faulthandler.enable(f) - - def disable_faulthandler(self): - """ - Cancel the faulthandling, close the file handle and remove the file. - """ - if not PY3: - # Not implemented + except UnicodeDecodeError as e: + return ( + "Can not read fault file!\n" + + "UnicodeDecodeError: " + str(e)) + + # Remove file + try: + os.remove(fault_filename) + except Exception: + pass + + # Process file + if not fault: return - if self.faulthandler_handle: - faulthandler.disable() - self.faulthandler_handle.close() - self.faulthandler_handle = None - # --- For the Variable Explorer - def set_namespace_view_settings(self, settings): - """Set namespace_view_settings.""" - self.namespace_view_settings = settings + thread_regex = ( + r"(Current thread|Thread) " + r"(0x[\da-f]+) \(most recent call first\):" + r"(?:.|\r\n|\r|\n)+?(?=Current thread|Thread|\Z)") + # Keep line for future improvements + # files_regex = r"File \"([^\"]+)\", line (\d+) in (\S+)" + + text = "" + start_idx = 0 + for idx, match in enumerate(re.finditer(thread_regex, fault)): + # Add anything non-matched + text += fault[start_idx:match.span()[0]] + start_idx = match.span()[1] + thread_id = int(match.group(2), base=16) + if thread_id != main_id: + if thread_id in ignore_ids: + continue + if "wurlitzer.py" in match.group(0): + # Wurlitzer threads are launched later + continue + text += "\n" + match.group(0) + "\n" + else: + try: + pattern = (r".*(?:/IPython/core/interactiveshell\.py|" + r"\\IPython\\core\\interactiveshell\.py).*") + match_internal = next(re.finditer(pattern, match.group(0))) + end_idx = match_internal.span()[0] + except StopIteration: + end_idx = None + text += "\nMain thread:\n" + match.group(0)[:end_idx] + "\n" + + # Add anything after match + text += fault[start_idx:] + return text + + def get_system_threads_id(self): + """Return the list of system threads id.""" + ignore_threads = [ + self.parent.poller, # Parent poller + self.shell.history_manager.save_thread, # history + self.parent.heartbeat, # heartbeat + self.parent.iopub_thread.thread, # iopub + gc.thread, # ZMQ garbage collector thread + self.parent.control_thread, # control + ] + return [ + thread.ident for thread in ignore_threads if thread is not None] + + def filter_stack(self, stack, is_main): + """Return the part of the stack the user needs to see.""" + # Remove wurlitzer frames + for frame_summary in stack: + if "wurlitzer.py" in frame_summary.filename: + return + # Cleanup main thread + if is_main: + start_idx = -1 + for idx in range(len(stack)): + if stack[idx].filename.endswith( + ("IPython/core/interactiveshell.py", + "IPython\\core\\interactiveshell.py")): + start_idx = idx + 1 + if start_idx != -1: + stack = stack[start_idx:] + else: + stack = [] + return stack + + @comm_handler + def get_current_frames(self, ignore_internal_threads=True): + """Get the current frames.""" + ignore_list = self.get_system_threads_id() + main_id = threading.main_thread().ident + frames = {} + thread_names = {thread.ident: thread.name + for thread in threading.enumerate()} + + for thread_id, frame in sys._current_frames().items(): + stack = traceback.StackSummary.extract( + traceback.walk_stack(frame)) + stack.reverse() + if ignore_internal_threads: + if thread_id in ignore_list: + continue + stack = self.filter_stack(stack, main_id == thread_id) + if stack is not None: + if thread_id in thread_names: + thread_name = thread_names[thread_id] + else: + thread_name = str(thread_id) + frames[thread_name] = stack + return frames - def get_namespace_view(self): + # --- For the Variable Explorer + @comm_handler + def get_namespace_view(self, frame=None): """ Return the namespace view @@ -183,12 +308,13 @@ def get_namespace_view(self): settings = self.namespace_view_settings if settings: - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace(frame=frame) view = make_remote_view(ns, settings, EXCLUDED_NAMES) return view else: return None + @comm_handler def get_var_properties(self): """ Get some properties of the variables in the current @@ -196,7 +322,7 @@ def get_var_properties(self): """ settings = self.namespace_view_settings if settings: - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace() data = get_remote_data(ns, settings, mode='editable', more_excluded_names=EXCLUDED_NAMES) @@ -219,27 +345,32 @@ def get_var_properties(self): else: return None + @comm_handler def get_value(self, name): """Get the value of a variable""" - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace() return ns[name] + @comm_handler def set_value(self, name, value): """Set the value of a variable""" - ns = self._get_reference_namespace(name) + ns = self.shell._get_reference_namespace(name) ns[name] = value self.log.debug(ns) + @comm_handler def remove_value(self, name): """Remove a variable""" - ns = self._get_reference_namespace(name) + ns = self.shell._get_reference_namespace(name) ns.pop(name) + @comm_handler def copy_value(self, orig_name, new_name): """Copy a variable""" - ns = self._get_reference_namespace(orig_name) + ns = self.shell._get_reference_namespace(orig_name) ns[new_name] = ns[orig_name] + @comm_handler def load_data(self, filename, ext, overwrite=False): """ Load data from filename. @@ -276,9 +407,10 @@ def load_data(self, filename, ext, overwrite=False): return None + @comm_handler def save_namespace(self, filename): """Save namespace into filename""" - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace() settings = self.namespace_view_settings data = get_remote_data(ns, settings, mode='picklable', more_excluded_names=EXCLUDED_NAMES).copy() @@ -299,63 +431,45 @@ def do_complete(self, code, cursor_pos): return self.shell.pdb_session.do_complete(code, cursor_pos) return self._do_complete(code, cursor_pos) - def set_spyder_breakpoints(self, breakpoints): + def interrupt_eventloop(self): """ - Handle a message from the frontend - """ - if self.shell.pdb_session: - self.shell.pdb_session.set_spyder_breakpoints(breakpoints) - - def set_pdb_ignore_lib(self, state): - """ - Change the "Ignore libraries while stepping" debugger setting. - """ - if self.shell.pdb_session: - self.shell.pdb_session.pdb_ignore_lib = state + Interrupts the eventloop. - def set_pdb_execute_events(self, state): - """ - Handle a message from the frontend - """ - if self.shell.pdb_session: - self.shell.pdb_session.pdb_execute_events = state + To be used when the main thread is blocked by a call to self.eventloop. + This can be called from another thread, e.g. the control thread. - def set_pdb_use_exclamation_mark(self, state): + note: + Interrupting the eventloop is only implemented when a message is + received on the shell channel, but this message is queued and + won't be processed because an `execute` message is being + processed. """ - Set an option on the current debugging session to decide wether - the Pdb commands needs to be prefixed by '!' - """ - if self.shell.pdb_session: - self.shell.pdb_session.pdb_use_exclamation_mark = state + if not self.eventloop: + return - def pdb_input_reply(self, line, echo_stack_entry=True): - """Get a pdb command from the frontend.""" - debugger = self.shell.pdb_session - if debugger: - debugger._disable_next_stack_entry = not echo_stack_entry - debugger._cmd_input_line = line - if self.eventloop: - # Interrupting the eventloop is only implemented when a message is - # received on the shell channel, but this message is queued and - # won't be processed because an `execute` message is being - # processed. Therefore we process the message here (control chan.) - # and request a dummy message to be sent on the shell channel to - # stop the eventloop. This will call back `_interrupt_eventloop`. - self.frontend_call().request_interrupt_eventloop() + if self.loopback_socket is None: + # Add socket to signal shell_stream locally + self.loopback_socket = self.shell_stream.socket.context.socket( + zmq.DEALER) + port = json.loads(get_connection_info())['shell_port'] + self.loopback_socket.connect("tcp://127.0.0.1:%i" % port) + # Add dummy handler + self.shell_handlers["interrupt_eventloop"] = ( + lambda stream, ident, parent: None) - def _interrupt_eventloop(self): - """Interrupts the eventloop.""" - # Receiving the request is enough to stop the eventloop. - pass + self.session.send( + self.loopback_socket, self.session.msg("interrupt_eventloop")) # --- For the Help plugin + @comm_handler def is_defined(self, obj, force_import=False): """Return True if object is defined in current namespace""" from spyder_kernels.utils.dochelpers import isdefined - ns = self._get_current_namespace(with_magics=True) + ns = self.shell._get_current_namespace(with_magics=True) return isdefined(obj, force_import=force_import, namespace=ns) + @comm_handler def get_doc(self, objtxt): """Get object documentation dictionary""" try: @@ -369,6 +483,7 @@ def get_doc(self, objtxt): if valid: return getdoc(obj) + @comm_handler def get_source(self, objtxt): """Get object source""" from spyder_kernels.utils.dochelpers import getsource @@ -378,82 +493,88 @@ def get_source(self, objtxt): return getsource(obj) # -- For Matplolib + @comm_handler def get_matplotlib_backend(self): """Get current matplotlib backend.""" try: import matplotlib - return MPL_BACKENDS_TO_SPYDER[matplotlib.get_backend()] + return MPL_BACKENDS_TO_SPYDER[matplotlib.get_backend().lower()] except Exception: return None + @comm_handler def get_mpl_interactive_backend(self): """ Get current Matplotlib interactive backend. This is different from the current backend because, for instance, the - user can set first the Qt5 backend, then the Inline one. In that case, - the current backend is Inline, but the current interactive one is Qt5, + user can set first the Qt backend, then the Inline one. In that case, + the current backend is Inline, but the current interactive one is Qt, and this backend can't be changed without a kernel restart. """ - # Mapping from frameworks to backend names. - mapping = { - 'qt': 'QtAgg', - 'tk': 'TkAgg', - 'macosx': 'MacOSX' - } - - # --- Get interactive framework - framework = None - - # Detect if there is a graphical framework running by checking the - # eventloop function attached to the kernel.eventloop attribute (see - # `ipykernel.eventloops.enable_gui` for context). - from IPython.core.getipython import get_ipython - loop_func = get_ipython().kernel.eventloop - - if loop_func is not None: - if loop_func == eventloops.loop_tk: - framework = 'tk' - elif loop_func == eventloops.loop_qt5: - framework = 'qt' - elif loop_func == eventloops.loop_cocoa: - framework = 'macosx' - else: - # Spyder doesn't handle other backends - framework = 'other' + # Backends that Spyder can handle + recognized_backends = {'qt', 'tk', 'macosx'} # --- Return backend according to framework - if framework is None: - # Since no interactive backend has been set yet, this is - # equivalent to having the inline one. - return 0 - elif framework in mapping: - return MPL_BACKENDS_TO_SPYDER[mapping[framework]] + if self.interactive_backend is None: + # Since no interactive backend has been set yet, this is equivalent + # to having the inline one. + return 'inline' + elif self.interactive_backend in recognized_backends: + return self.interactive_backend else: # This covers the case of other backends (e.g. Wx or Gtk) # which users can set interactively with the %matplotlib # magic but not through our Preferences. return -1 - def set_matplotlib_backend(self, backend, pylab=False): - """Set matplotlib backend given a Spyder backend option.""" - mpl_backend = MPL_BACKENDS_FROM_SPYDER[to_text_string(backend)] - self._set_mpl_backend(mpl_backend, pylab=pylab) - - def set_mpl_inline_figure_format(self, figure_format): - """Set the inline figure format to use with matplotlib.""" - mpl_figure_format = INLINE_FIGURE_FORMATS[figure_format] - self._set_config_option( - 'InlineBackend.figure_format', mpl_figure_format) - - def set_mpl_inline_resolution(self, resolution): - """Set inline figure resolution.""" - self._set_mpl_inline_rc_config('figure.dpi', resolution) + @comm_handler + def set_matplotlib_conf(self, conf): + """Set matplotlib configuration""" + pylab_autoload_n = 'pylab/autoload' + pylab_backend_n = 'pylab/backend' + figure_format_n = 'pylab/inline/figure_format' + resolution_n = 'pylab/inline/resolution' + width_n = 'pylab/inline/width' + height_n = 'pylab/inline/height' + fontsize_n = 'pylab/inline/fontsize' + bottom_n = 'pylab/inline/bottom' + bbox_inches_n = 'pylab/inline/bbox_inches' + inline_backend = 'inline' + + if pylab_autoload_n in conf or pylab_backend_n in conf: + self._set_mpl_backend( + conf.get(pylab_backend_n, inline_backend), + pylab=conf.get(pylab_autoload_n, False) + ) + + if figure_format_n in conf: + self._set_config_option( + 'InlineBackend.figure_format', + conf[figure_format_n] + ) + + if resolution_n in conf: + self._set_mpl_inline_rc_config('figure.dpi', conf[resolution_n]) + + if width_n in conf and height_n in conf: + self._set_mpl_inline_rc_config( + 'figure.figsize', + (conf[width_n], conf[height_n]) + ) + + if fontsize_n in conf: + self._set_mpl_inline_rc_config('font.size', conf[fontsize_n]) + + if bottom_n in conf: + self._set_mpl_inline_rc_config( + 'figure.subplot.bottom', + conf[bottom_n] + ) + + if bbox_inches_n in conf: + self.set_mpl_inline_bbox_inches(conf[bbox_inches_n]) - def set_mpl_inline_figure_size(self, width, height): - """Set inline figure size.""" - value = (width, height) - self._set_mpl_inline_rc_config('figure.figsize', value) def set_mpl_inline_bbox_inches(self, bbox_inches): """ @@ -461,8 +582,7 @@ def set_mpl_inline_bbox_inches(self, bbox_inches): The change is done by updating the 'print_figure_kwargs' config dict. """ - from IPython.core.getipython import get_ipython - config = get_ipython().kernel.config + config = self.config inline_config = ( config['InlineBackend'] if 'InlineBackend' in config else {}) print_figure_kwargs = ( @@ -496,9 +616,55 @@ def set_autocall(self, autocall): self._set_config_option('ZMQInteractiveShell.autocall', autocall) # --- Additional methods - def set_cwd(self, dirname): - """Set current working directory.""" - os.chdir(dirname) + @comm_handler + def set_configuration(self, conf): + """Set kernel configuration""" + ret = {} + for key, value in conf.items(): + if key == "cwd": + self._cwd_initialised = True + os.chdir(value) + self.publish_state() + elif key == "namespace_view_settings": + self.namespace_view_settings = value + self.publish_state() + elif key == "pdb": + self.shell.set_pdb_configuration(value) + elif key == "faulthandler": + if value: + ret[key] = self.enable_faulthandler() + elif key == "special_kernel": + try: + self.set_special_kernel(value) + except Exception: + ret["special_kernel_error"] = value + elif key == "color scheme": + self.set_color_scheme(value) + elif key == "jedi_completer": + self.set_jedi_completer(value) + elif key == "greedy_completer": + self.set_greedy_completer(value) + elif key == "autocall": + self.set_autocall(value) + elif key == "matplotlib": + self.set_matplotlib_conf(value) + elif key == "update_gui": + self.shell.update_gui_frontend = value + elif key == "wurlitzer": + if value: + self._load_wurlitzer() + elif key == "autoreload_magic": + self._autoreload_magic(value) + return ret + + def set_color_scheme(self, color_scheme): + if color_scheme == "dark": + # Needed to change the colors of tracebacks + self.shell.run_line_magic("colors", "linux") + self.set_sympy_forecolor(background_color='dark') + elif color_scheme == "light": + self.shell.run_line_magic("colors", "lightbg") + self.set_sympy_forecolor(background_color='light') def get_cwd(self): """Get current working directory.""" @@ -507,14 +673,17 @@ def get_cwd(self): except (IOError, OSError): pass + @comm_handler def get_syspath(self): """Return sys.path contents.""" return sys.path[:] + @comm_handler def get_env(self): """Get environment variables.""" return os.environ.copy() + @comm_handler def close_all_mpl_figures(self): """Close all Matplotlib figures.""" try: @@ -523,28 +692,59 @@ def close_all_mpl_figures(self): except: pass - def is_special_kernel_valid(self): + def set_special_kernel(self, special): """ Check if optional dependencies are available for special consoles. """ - try: - if os.environ.get('SPY_AUTOLOAD_PYLAB_O') == 'True': - import matplotlib - elif os.environ.get('SPY_SYMPY_O') == 'True': - import sympy - elif os.environ.get('SPY_RUN_CYTHON') == 'True': - import cython - except Exception: - # Use Exception instead of ImportError here because modules can - # fail to be imported due to a lot of issues. - if os.environ.get('SPY_AUTOLOAD_PYLAB_O') == 'True': - return u'matplotlib' - elif os.environ.get('SPY_SYMPY_O') == 'True': - return u'sympy' - elif os.environ.get('SPY_RUN_CYTHON') == 'True': - return u'cython' - return None + self.shell.special = None + if special is None: + return + + if special == "pylab": + import matplotlib # noqa + exec("from pylab import *", self.shell.user_ns) + self.shell.special = special + return + + if special == "sympy": + import sympy # noqa + sympy_init = "\n".join([ + "from sympy import *", + "x, y, z, t = symbols('x y z t')", + "k, m, n = symbols('k m n', integer=True)", + "f, g, h = symbols('f g h', cls=Function)", + "init_printing()", + ]) + exec(sympy_init, self.shell.user_ns) + self.shell.special = special + return + if special == "cython": + import cython # noqa + + # Import pyximport to enable Cython files support for + # import statement + import pyximport + pyx_setup_args = {} + + # Add Numpy include dir to pyximport/distutils + try: + import numpy + pyx_setup_args['include_dirs'] = numpy.get_include() + except Exception: + pass + + # Setup pyximport and enable Cython files reload + pyximport.install(setup_args=pyx_setup_args, + reload_support=True) + + self.shell.run_line_magic("reload_ext", "Cython") + self.shell.special = special + return + + raise NotImplementedError(f"{special}") + + @comm_handler def update_syspath(self, path_dict, new_path_dict): """ Update the PYTHONPATH of the kernel. @@ -570,49 +770,6 @@ def update_syspath(self, path_dict, new_path_dict): # -- Private API --------------------------------------------------- # --- For the Variable Explorer - def _get_current_namespace(self, with_magics=False): - """ - Return current namespace - - This is globals() if not debugging, or a dictionary containing - both locals() and globals() for current frame when debugging - """ - ns = {} - if self.shell.is_debugging() and self.shell.pdb_session.curframe: - # Stopped at a pdb prompt - ns.update(self.shell.user_ns) - ns.update(self.shell._pdb_locals) - else: - # Give access to the running namespace if there is one - if self._running_namespace is None: - ns.update(self.shell.user_ns) - else: - # This is true when a file is executing. - running_globals, running_locals = self._running_namespace - ns.update(running_globals) - if running_locals is not None: - ns.update(running_locals) - - # Add magics to ns so we can show help about them on the Help - # plugin - if with_magics: - line_magics = self.shell.magics_manager.magics['line'] - cell_magics = self.shell.magics_manager.magics['cell'] - ns.update(line_magics) - ns.update(cell_magics) - return ns - - def _get_reference_namespace(self, name): - """ - Return namespace where reference name is defined - - It returns the globals() if reference has not yet been defined - """ - lcls = self.shell._pdb_locals - if name in lcls: - return lcls - return self.shell.user_ns - def _get_len(self, var): """Return sequence length""" try: @@ -703,10 +860,9 @@ def _eval(self, text): where *obj* is the object represented by *text* and *valid* is True if object evaluation did not raise any exception """ - from spyder_kernels.py3compat import is_text_string - assert is_text_string(text) - ns = self._get_current_namespace(with_magics=True) + assert isinstance(text, str) + ns = self.shell._get_current_namespace(with_magics=True) try: return eval(text, ns), True except: @@ -723,7 +879,6 @@ def _set_mpl_backend(self, backend, pylab=False): namespace from numpy and matplotlib """ import traceback - from IPython.core.getipython import get_ipython # Don't proceed further if there's any error while importing Matplotlib try: @@ -740,6 +895,9 @@ def _set_mpl_backend(self, backend, pylab=False): magic = 'pylab' if pylab else 'matplotlib' + if backend == "auto": + backend = automatic_backend() + error = None try: # This prevents Matplotlib to automatically set the backend, which @@ -747,7 +905,7 @@ def _set_mpl_backend(self, backend, pylab=False): matplotlib.rcParams['backend'] = 'Agg' # Set the backend - get_ipython().run_line_magic(magic, backend) + self.shell.run_line_magic(magic, backend) except RuntimeError as err: # This catches errors generated by ipykernel when # trying to set a backend. See issue 5541 @@ -778,8 +936,8 @@ def _set_mpl_backend(self, backend, pylab=False): error = generic_error.format(err) + '\n\n' + additional_info except Exception: error = generic_error.format(traceback.format_exc()) - - self._mpl_backend_error = error + if error: + print(error) def _set_config_option(self, option, value): """ @@ -789,13 +947,12 @@ def _set_config_option(self, option, value): option: config option, for example 'InlineBackend.figure_format'. value: value of the option, for example 'SVG', 'Retina', etc. """ - from IPython.core.getipython import get_ipython try: base_config = "{option} = " value_line = ( - "'{value}'" if isinstance(value, TEXT_TYPES) else "{value}") + "'{value}'" if isinstance(value, str) else "{value}") config_line = base_config + value_line - get_ipython().run_line_magic( + self.shell.run_line_magic( 'config', config_line.format(option=option, value=value)) except Exception: @@ -812,31 +969,30 @@ def _set_mpl_inline_rc_config(self, option, value): # Needed in case matplolib isn't installed pass - def show_mpl_backend_errors(self): - """Show Matplotlib backend errors after the prompt is ready.""" - if self._mpl_backend_error is not None: - print(self._mpl_backend_error) # spyder: test-skip - def set_sympy_forecolor(self, background_color='dark'): """Set SymPy forecolor depending on console background.""" - if os.environ.get('SPY_SYMPY_O') == 'True': - try: - from sympy import init_printing - from IPython.core.getipython import get_ipython - if background_color == 'dark': - init_printing(forecolor='White', ip=get_ipython()) - elif background_color == 'light': - init_printing(forecolor='Black', ip=get_ipython()) - except Exception: - pass + if self.shell.special != "sympy": + return + + try: + from sympy import init_printing + if background_color == 'dark': + init_printing(forecolor='White', ip=self.shell) + elif background_color == 'light': + init_printing(forecolor='Black', ip=self.shell) + except Exception: + pass # --- Others - def _load_autoreload_magic(self): + def _autoreload_magic(self, enable): """Load %autoreload magic.""" - from IPython.core.getipython import get_ipython try: - get_ipython().run_line_magic('reload_ext', 'autoreload') - get_ipython().run_line_magic('autoreload', '2') + if enable: + self.shell.run_line_magic('reload_ext', 'autoreload') + self.shell.run_line_magic('autoreload', "2") + else: + self.shell.run_line_magic('autoreload', "off") + except Exception: pass @@ -844,12 +1000,11 @@ def _load_wurlitzer(self): """Load wurlitzer extension.""" # Wurlitzer has no effect on Windows if not os.name == 'nt': - from IPython.core.getipython import get_ipython # Enclose this in a try/except because if it fails the # console will be totally unusable. # Fixes spyder-ide/spyder#8668 try: - get_ipython().run_line_magic('reload_ext', 'wurlitzer') + self.shell.run_line_magic('reload_ext', 'wurlitzer') except Exception: pass @@ -865,3 +1020,32 @@ def _get_comm(self, comm_id): return self.comm_manager.comms[comm_id] except KeyError: pass + + def control_comm_msg(self, stream, ident, msg): + """ + Handler for comm_msg messages from control channel. + + If comm is not open yet, cache message. + """ + content = msg['content'] + comm_id = content['comm_id'] + comm = self.comm_manager.get_comm(comm_id) + if comm is None: + self.frontend_comm.cache_message(comm_id, msg) + return + try: + comm.handle_msg(msg) + except Exception: + self.comm_manager.log.error( + 'Exception in comm_msg for %s', comm_id, exc_info=True) + + def pre_handler_hook(self): + """Hook to execute before calling message handler""" + pass + + def post_handler_hook(self): + """Hook to execute after calling message handler""" + # keep ipykernel behavior of resetting sigint every call + self.shell.register_debugger_sigint() + # Reset tracing function so that pdb.set_trace works + sys.settrace(None) diff --git a/external-deps/spyder-kernels/spyder_kernels/console/shell.py b/external-deps/spyder-kernels/spyder_kernels/console/shell.py index 9d164701534..48cecaf3da0 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/shell.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/shell.py @@ -12,28 +12,64 @@ # Standard library imports import bdb +import logging +import os +import signal import sys +import traceback +from _thread import interrupt_main # Third-party imports from ipykernel.zmqshell import ZMQInteractiveShell +# Local imports +from spyder_kernels.customize.namespace_manager import NamespaceManager +from spyder_kernels.customize.spyderpdb import SpyderPdb +from spyder_kernels.customize.code_runner import SpyderCodeRunner +from spyder_kernels.comms.decorators import comm_handler +from spyder_kernels.utils.mpl import automatic_backend + + +logger = logging.getLogger(__name__) + class SpyderShell(ZMQInteractiveShell): """Spyder shell.""" + PDB_CONF_KEYS = [ + 'pdb_ignore_lib', + 'pdb_execute_events', + 'pdb_use_exclamation_mark', + 'pdb_stop_first_line', + 'breakpoints', + 'pdb_publish_stack' + ] + def __init__(self, *args, **kwargs): - # Create _pdb_obj before __init__ - self._pdb_obj = None + # Create _namespace_stack before __init__ + self._namespace_stack = [] + self._request_pdb_stop = False + self.special = None + self._pdb_conf = {} super(SpyderShell, self).__init__(*args, **kwargs) + self._allow_kbdint = False + self.register_debugger_sigint() + self.update_gui_frontend = False # register post_execute self.events.register('post_execute', self.do_post_execute) - # ---- Methods overriden by us. + def init_magics(self): + """Init magics""" + super().init_magics() + self.register_magics(SpyderCodeRunner) + def ask_exit(self): """Engage the exit actions.""" - self.kernel.frontend_comm.close_thread() - return super(SpyderShell, self).ask_exit() + if self.active_eventloop not in [None, "inline"]: + # Some eventloops prevent the kernel from shutting down + self.enable_gui('inline') + super().ask_exit() def _showtraceback(self, etype, evalue, stb): """ @@ -47,36 +83,126 @@ def _showtraceback(self, etype, evalue, stb): stb = [''] super(SpyderShell, self)._showtraceback(etype, evalue, stb) - # ---- For Pdb namespace integration - def get_local_scope(self, stack_depth): - """Get local scope at given frame depth.""" - frame = sys._getframe(stack_depth + 1) - if self._pdb_frame is frame: - # Avoid calling f_locals on _pdb_frame - return self._pdb_obj.curframe_locals - else: - return frame.f_locals + def enable_matplotlib(self, gui=None): + """Enable matplotlib.""" + if gui is None or gui.lower() == "auto": + gui = automatic_backend() - def get_global_scope(self, stack_depth): - """Get global scope at given frame depth.""" - frame = sys._getframe(stack_depth + 1) - return frame.f_globals + enabled_gui, backend = super().enable_matplotlib(gui) + + # This is necessary for IPython 8.24+, which returns None after + # enabling the Inline backend. + if enabled_gui is None and gui == "inline": + enabled_gui = "inline" + gui = enabled_gui + + # To easily track the current interactive backend + if self.kernel.interactive_backend is None: + self.kernel.interactive_backend = gui if gui != "inline" else None + + if self.update_gui_frontend: + try: + self.kernel.frontend_call( + blocking=False + ).update_matplotlib_gui(gui) + except Exception: + pass + + return gui, backend + + # --- For Pdb namespace integration + def set_pdb_configuration(self, pdb_conf): + """ + Set Pdb configuration. + + Parameters + ---------- + pdb_conf: dict + Dictionary containing the configuration. Its keys are part of the + `PDB_CONF_KEYS` class constant. + """ + for key in self.PDB_CONF_KEYS: + if key in pdb_conf: + self._pdb_conf[key] = pdb_conf[key] + if self.pdb_session: + setattr(self.pdb_session, key, pdb_conf[key]) def is_debugging(self): """ Check if we are currently debugging. """ - return bool(self._pdb_frame) + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb) and session.curframe is not None: + return True + return False @property def pdb_session(self): """Get current pdb session.""" - return self._pdb_obj + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb): + return session + return None + + def add_pdb_session(self, pdb_obj): + """Add a pdb object to the stack.""" + if self.pdb_session == pdb_obj: + # Already added + return + self._namespace_stack.append(pdb_obj) + + # Set config to pdb obj + self.set_pdb_configuration(self._pdb_conf) + + def remove_pdb_session(self, pdb_obj): + """Remove a pdb object from the stack.""" + if self.pdb_session != pdb_obj: + # Already removed + return + self._namespace_stack.pop() + + if self.pdb_session: + # Set config to newly active pdb obj + self.set_pdb_configuration(self._pdb_conf) + + def add_namespace_manager(self, ns_manager): + """Add namespace manager to stack.""" + self._namespace_stack.append(ns_manager) + + def remove_namespace_manager(self, ns_manager): + """Remove namespace manager.""" + if self._namespace_stack[-1] != ns_manager: + logger.debug("The namespace stack is inconsistent.") + return + self._namespace_stack.pop() - @pdb_session.setter - def pdb_session(self, pdb_obj): - """Register Pdb session to use it later""" - self._pdb_obj = pdb_obj + def get_local_scope(self, stack_depth): + """ + Get local scope at a given frame depth. + + Needed for magics that use "needs_local_scope" such as timeit + """ + frame = sys._getframe(stack_depth + 1) + return self.context_locals(frame) + + def context_locals(self, frame=None): + """ + Get context locals. + + If frame is not None, make sure frame.f_locals is not registered in a + debugger and return frame.f_locals + """ + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb) and session.curframe is not None: + if frame is None or frame == session.curframe: + return session.curframe_locals + elif frame is None and isinstance(session, NamespaceManager): + return session.ns_locals + + if frame is not None: + return frame.f_locals + + return None @property def _pdb_frame(self): @@ -84,32 +210,170 @@ def _pdb_frame(self): if self.pdb_session is not None: return self.pdb_session.curframe - @property - def _pdb_locals(self): - """ - Return current Pdb frame locals if available. Otherwise - return an empty dictionary - """ - if self._pdb_frame is not None: - return self._pdb_obj.curframe_locals - else: - return {} - @property def user_ns(self): """Get the current namespace.""" - if self._pdb_frame is not None: - return self._pdb_frame.f_globals - else: - return self.__user_ns + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb) and session.curframe is not None: + # Return first debugging namespace + return session.curframe.f_globals + elif isinstance(session, NamespaceManager): + return session.ns_globals + + return self.__user_ns @user_ns.setter def user_ns(self, namespace): """Set user_ns.""" self.__user_ns = namespace + def _get_current_namespace(self, with_magics=False, frame=None): + """Return a copy of the current namespace.""" + if frame is not None: + ns = frame.f_globals.copy() + ns.update(self.context_locals(frame)) + return ns + + ns = {} + ns.update(self.user_ns) + context_locals = self.context_locals() + if context_locals: + ns.update(context_locals) + + # Add magics to ns so we can show help about them on the Help + # plugin + if with_magics: + line_magics = self.magics_manager.magics['line'] + cell_magics = self.magics_manager.magics['cell'] + ns.update(line_magics) + ns.update(cell_magics) + + return ns + + def _get_reference_namespace(self, name): + """ + Return namespace where reference name is defined + + It returns the user namespace if name has not yet been defined. + """ + lcls = self.context_locals() + if lcls and name in lcls: + + return lcls + return self.user_ns + + def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None, + exception_only=False, running_compiled_code=False): + """Display the exception that just occurred.""" + super(SpyderShell, self).showtraceback( + exc_tuple, filename, tb_offset, + exception_only, running_compiled_code) + if not exception_only: + try: + etype, value, tb = self._get_exc_info(exc_tuple) + stack = traceback.extract_tb(tb.tb_next) + self.kernel.frontend_call(blocking=False).show_traceback( + etype, value, stack) + except Exception: + return + + def register_debugger_sigint(self): + """Register sigint handler.""" + signal.signal(signal.SIGINT, self.spyderkernel_sigint_handler) + + @comm_handler + def raise_interrupt_signal(self): + """Raise interrupt signal.""" + if os.name == "nt": + # Check if signal handler is callable to avoid + # 'int not callable' error (Python issue #23395) + if callable(signal.getsignal(signal.SIGINT)): + interrupt_main() + else: + self.kernel.log.error( + "Interrupt message not supported on Windows") + else: + # This is necessary to make the call below work for IPykernel + # versions equal or less than 6.21.2 and greater than it. + # See ipython/ipykernel#1101 + if hasattr(self.kernel, '_send_interupt_children'): + self.kernel._send_interupt_children() + else: + self.kernel._send_interrupt_children() + + @comm_handler + def request_pdb_stop(self): + """Request pdb to stop at the next possible position.""" + pdb_session = self.pdb_session + if pdb_session: + if pdb_session.interrupting: + # interrupt already requested, wait + return + # trace_dispatch is active, stop at the next possible position + pdb_session.interrupt() + elif (self.spyderkernel_sigint_handler + == signal.getsignal(signal.SIGINT)): + # Use spyderkernel_sigint_handler + self._request_pdb_stop = True + self.raise_interrupt_signal() + else: + logger.debug( + "Can not signal main thread to stop as SIGINT " + "handler was replaced and the debugger is not active. " + "The current handler is: " + + repr(signal.getsignal(signal.SIGINT)) + ) + + def spyderkernel_sigint_handler(self, signum, frame): + """SIGINT handler.""" + if self._request_pdb_stop: + # SIGINT called from request_pdb_stop + self._request_pdb_stop = False + debugger = SpyderPdb() + debugger.interrupt() + debugger.set_trace(frame) + return + + pdb_session = self.pdb_session + if pdb_session: + # SIGINT called while debugging + if pdb_session.allow_kbdint: + raise KeyboardInterrupt + if pdb_session.interrupting: + # second call to interrupt, raise + raise KeyboardInterrupt + pdb_session.interrupt() + return + + if self._allow_kbdint: + # Do not raise KeyboardInterrupt in the middle of ipython code + raise KeyboardInterrupt + + async def run_code(self, *args, **kwargs): + """Execute a code object.""" + try: + try: + self._allow_kbdint = True + return await super().run_code(*args, **kwargs) + finally: + self._allow_kbdint = False + except KeyboardInterrupt: + self.showtraceback() + + @comm_handler + def pdb_input_reply(self, line, echo_stack_entry=True): + """Get a pdb command from the frontend.""" + debugger = self.pdb_session + if not debugger: + return + debugger._disable_next_stack_entry = not echo_stack_entry + debugger._cmd_input_line = line + # Interrupts eventloop if needed + self.kernel.interrupt_eventloop() + def do_post_execute(self): """Flush __std*__ after execution.""" # Flush C standard streams. sys.__stderr__.flush() sys.__stdout__.flush() + self.kernel.publish_state() diff --git a/external-deps/spyder-kernels/spyder_kernels/console/start.py b/external-deps/spyder-kernels/spyder_kernels/console/start.py index f80111d76c9..eb910305957 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/start.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/start.py @@ -16,17 +16,11 @@ import sys import site +# Third-party imports from traitlets import DottedObjectName -import ipykernel # Local imports from spyder_kernels.utils.misc import is_module_installed -from spyder_kernels.utils.mpl import ( - MPL_BACKENDS_FROM_SPYDER, INLINE_FIGURE_FORMATS) - - -PY2 = sys.version[0] == '2' -IPYKERNEL_6 = ipykernel.__version__[0] >= '6' def import_spydercustomize(): @@ -52,24 +46,6 @@ def import_spydercustomize(): except ValueError: pass - -def sympy_config(mpl_backend): - """Sympy configuration""" - if mpl_backend is not None: - lines = """ -from sympy.interactive import init_session -init_session() -%matplotlib {0} -""".format(mpl_backend) - else: - lines = """ -from sympy.interactive import init_session -init_session() -""" - - return lines - - def kernel_config(): """Create a config object with IPython kernel options.""" from IPython.core.application import get_ipython_dir @@ -96,10 +72,9 @@ def kernel_config(): # Until we implement Issue 1052 spy_cfg.InteractiveShell.xmode = 'Plain' - # Jedi completer. It's only available in Python 3 + # Jedi completer. jedi_o = os.environ.get('SPY_JEDI_O') == 'True' - if not PY2: - spy_cfg.IPCompleter.use_jedi = jedi_o + spy_cfg.IPCompleter.use_jedi = jedi_o # Clear terminal arguments input. # This needs to be done before adding the exec_lines that come from @@ -109,15 +84,6 @@ def kernel_config(): clear_argv = "import sys; sys.argv = ['']; del sys" spy_cfg.IPKernelApp.exec_lines = [clear_argv] - # Set our runfile in builtins here to prevent other packages shadowing it. - # This started to be a problem since IPykernel 6.3.0. - if not PY2: - spy_cfg.IPKernelApp.exec_lines.append( - "import builtins; " - "builtins.runfile = builtins.spyder_runfile; " - "del builtins.spyder_runfile; del builtins" - ) - # Prevent other libraries to change the breakpoint builtin. # This started to be a problem since IPykernel 6.3.0. if sys.version_info[0:2] >= (3, 7): @@ -127,27 +93,15 @@ def kernel_config(): "del sys; del pdb" ) - # Run lines of code at startup - run_lines_o = os.environ.get('SPY_RUN_LINES_O') - if run_lines_o is not None: - spy_cfg.IPKernelApp.exec_lines += ( - [x.strip() for x in run_lines_o.split(';')] - ) - - # Load %autoreload magic - spy_cfg.IPKernelApp.exec_lines.append( - "get_ipython().kernel._load_autoreload_magic()") - - # Load wurlitzer extension - spy_cfg.IPKernelApp.exec_lines.append( - "get_ipython().kernel._load_wurlitzer()") - - # Default inline backend configuration + # Default inline backend configuration. # This is useful to have when people doesn't # use our config system to configure the # inline backend but want to use # '%matplotlib inline' at runtime spy_cfg.InlineBackend.rc = { + # The typical default figure size is too large for inline use, + # so we shrink the figure size to 6x4, and tweak fonts to + # make that fit. 'figure.figsize': (6.0, 4.0), # 72 dpi matches SVG/qtconsole. # This only affects PNG export, as SVG has no dpi setting. @@ -161,86 +115,24 @@ def kernel_config(): 'figure.edgecolor': 'white' } - # Pylab configuration - mpl_backend = None if is_module_installed('matplotlib'): - # Set Matplotlib backend with Spyder options - pylab_o = os.environ.get('SPY_PYLAB_O') - backend_o = os.environ.get('SPY_BACKEND_O') - if pylab_o == 'True' and backend_o is not None: - mpl_backend = MPL_BACKENDS_FROM_SPYDER[backend_o] - # Inline backend configuration - if mpl_backend == 'inline': - # Figure format - format_o = os.environ.get('SPY_FORMAT_O') - formats = INLINE_FIGURE_FORMATS - if format_o is not None: - spy_cfg.InlineBackend.figure_format = formats[format_o] - - # Resolution - resolution_o = os.environ.get('SPY_RESOLUTION_O') - if resolution_o is not None: - spy_cfg.InlineBackend.rc['figure.dpi'] = float( - resolution_o) - - # Figure size - width_o = float(os.environ.get('SPY_WIDTH_O')) - height_o = float(os.environ.get('SPY_HEIGHT_O')) - if width_o is not None and height_o is not None: - spy_cfg.InlineBackend.rc['figure.figsize'] = (width_o, - height_o) - - # Print figure kwargs - bbox_inches_o = os.environ.get('SPY_BBOX_INCHES_O') - bbox_inches = 'tight' if bbox_inches_o == 'True' else None - spy_cfg.InlineBackend.print_figure_kwargs.update( - {'bbox_inches': bbox_inches}) - else: - # Set Matplotlib backend to inline for external kernels. - # Fixes issue 108 - mpl_backend = 'inline' - - # Automatically load Pylab and Numpy, or only set Matplotlib - # backend - autoload_pylab_o = os.environ.get('SPY_AUTOLOAD_PYLAB_O') == 'True' - command = "get_ipython().kernel._set_mpl_backend('{0}', {1})" - spy_cfg.IPKernelApp.exec_lines.append( - command.format(mpl_backend, autoload_pylab_o)) - - # Enable Cython magic - run_cython = os.environ.get('SPY_RUN_CYTHON') == 'True' - if run_cython and is_module_installed('Cython'): - spy_cfg.IPKernelApp.exec_lines.append('%reload_ext Cython') - - # Run a file at startup - use_file_o = os.environ.get('SPY_USE_FILE_O') - run_file_o = os.environ.get('SPY_RUN_FILE_O') - if use_file_o == 'True' and run_file_o is not None: - if osp.exists(run_file_o): - spy_cfg.IPKernelApp.file_to_run = run_file_o + spy_cfg.IPKernelApp.matplotlib = "inline" # Autocall autocall_o = os.environ.get('SPY_AUTOCALL_O') if autocall_o is not None: spy_cfg.ZMQInteractiveShell.autocall = int(autocall_o) - # To handle the banner by ourselves in IPython 3+ + # To handle the banner by ourselves spy_cfg.ZMQInteractiveShell.banner1 = '' # Greedy completer greedy_o = os.environ.get('SPY_GREEDY_O') == 'True' spy_cfg.IPCompleter.greedy = greedy_o - # Sympy loading - sympy_o = os.environ.get('SPY_SYMPY_O') == 'True' - if sympy_o and is_module_installed('sympy'): - lines = sympy_config(mpl_backend) - spy_cfg.IPKernelApp.exec_lines.append(lines) - # Disable the new mechanism to capture and forward low-level output # in IPykernel 6. For that we have Wurlitzer. - if not PY2: - spy_cfg.IPKernelApp.capture_fd_output = False + spy_cfg.IPKernelApp.capture_fd_output = False # Merge IPython and Spyder configs. Spyder prefs will have prevalence # over IPython ones @@ -262,7 +154,7 @@ def varexp(line): except: import matplotlib.pyplot as pyplot pyplot.figure(); - getattr(pyplot, funcname[2:])(ip.kernel._get_current_namespace()[name]) + getattr(pyplot, funcname[2:])(ip._get_current_namespace()[name]) pyplot.show() @@ -296,9 +188,8 @@ def main(): class SpyderKernelApp(IPKernelApp): - if IPYKERNEL_6: - outstream_class = DottedObjectName( - 'spyder_kernels.console.outstream.TTYOutStream') + outstream_class = DottedObjectName( + 'spyder_kernels.console.outstream.TTYOutStream') def init_pdb(self): """ @@ -309,6 +200,13 @@ def init_pdb(self): """ pass + def close(self): + """Close the loopback socket.""" + socket = self.kernel.loopback_socket + if socket and not socket.closed: + socket.close() + return super().close() + # Fire up the kernel instance. kernel = SpyderKernelApp.instance() kernel.kernel_class = SpyderKernel diff --git a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py index 3e6a06509d7..6cb817415ac 100644 --- a/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py +++ b/external-deps/spyder-kernels/spyder_kernels/console/tests/test_console_kernel.py @@ -11,6 +11,7 @@ # Standard library imports import ast +import asyncio import os import os.path as osp from textwrap import dedent @@ -19,10 +20,10 @@ from subprocess import Popen, PIPE import sys import inspect +import uuid from collections import namedtuple # Test imports -import ipykernel import pytest from flaky import flaky from jupyter_core import paths @@ -30,23 +31,15 @@ import numpy as np # Local imports -from spyder_kernels.py3compat import PY2, PY3, to_text_string from spyder_kernels.utils.iofuncs import iofunctions -from spyder_kernels.utils.mpl import MPL_BACKENDS_FROM_SPYDER from spyder_kernels.utils.test_utils import get_kernel, get_log_text from spyder_kernels.customize.spyderpdb import SpyderPdb - -# For ipykernel 6 -try: - import asyncio -except ImportError: - pass +from spyder_kernels.comms.commbase import CommBase # ============================================================================= # Constants and utility functions # ============================================================================= FILES_PATH = os.path.dirname(os.path.realpath(__file__)) -IPYKERNEL_6 = ipykernel.__version__[0] >= '6' TIMEOUT = 15 SETUP_TIMEOUT = 60 @@ -68,7 +61,7 @@ def setup_kernel(cmd): """start an embedded kernel in a subprocess, and wait for it to be ready This function was taken from the ipykernel project. - We plan to remove it when dropping support for python 2. + We plan to remove it. Yields ------- @@ -89,8 +82,6 @@ def setup_kernel(cmd): if kernel.poll() is not None: o,e = kernel.communicate() - if not PY3 and isinstance(e, bytes): - e = e.decode() raise IOError("Kernel failed to start:\n%s" % e) if not os.path.exists(connection_file): @@ -99,7 +90,16 @@ def setup_kernel(cmd): raise IOError("Connection file %r never arrived" % connection_file) client = BlockingKernelClient(connection_file=connection_file) - client.load_connection_file() + tic = time.time() + while True: + try: + client.load_connection_file() + break + except ValueError: + # The file is not written yet + if time.time() > tic + SETUP_TIMEOUT: + # Give up after 5s + raise IOError("Kernel failed to write connection file") client.start_channels() client.wait_for_ready() try: @@ -107,8 +107,87 @@ def setup_kernel(cmd): finally: client.stop_channels() finally: - if not PY2: - kernel.terminate() + kernel.terminate() + + +class Comm(): + """ + Comm base class, copied from qtconsole without the qt stuff + """ + + def __init__(self, target_name, kernel_client, + msg_callback=None, close_callback=None): + """ + Create a new comm. Must call open to use. + """ + self.target_name = target_name + self.kernel_client = kernel_client + self.comm_id = uuid.uuid1().hex + self._msg_callback = msg_callback + self._close_callback = close_callback + self._send_channel = self.kernel_client.shell_channel + + def _send_msg(self, msg_type, content, data, metadata, buffers): + """ + Send a message on the shell channel. + """ + if data is None: + data = {} + if content is None: + content = {} + content['comm_id'] = self.comm_id + content['data'] = data + msg = self.kernel_client.session.msg( + msg_type, content, metadata=metadata) + if buffers: + msg['buffers'] = buffers + return self._send_channel.send(msg) + + # methods for sending messages + def open(self, data=None, metadata=None, buffers=None): + """Open the kernel-side version of this comm""" + return self._send_msg( + 'comm_open', {'target_name': self.target_name}, + data, metadata, buffers) + + def send(self, data=None, metadata=None, buffers=None): + """Send a message to the kernel-side version of this comm""" + return self._send_msg( + 'comm_msg', {}, data, metadata, buffers) + + def close(self, data=None, metadata=None, buffers=None): + """Close the kernel-side version of this comm""" + return self._send_msg( + 'comm_close', {}, data, metadata, buffers) + + def on_msg(self, callback): + """Register a callback for comm_msg + + Will be called with the `data` of any comm_msg messages. + + Call `on_msg(None)` to disable an existing callback. + """ + self._msg_callback = callback + + def on_close(self, callback): + """Register a callback for comm_close + + Will be called with the `data` of the close message. + + Call `on_close(None)` to disable an existing callback. + """ + self._close_callback = callback + + # methods for handling incoming messages + def handle_msg(self, msg): + """Handle a comm_msg message""" + if self._msg_callback: + return self._msg_callback(msg) + + def handle_close(self, msg): + """Handle a comm_close message""" + if self._close_callback: + return self._close_callback(msg) # ============================================================================= @@ -147,15 +226,13 @@ def kernel(request): 'False_', 'True_' ], - 'minmax': False + 'minmax': False, + 'filter_on':True } # Teardown def reset_kernel(): - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('reset -f', True)) - else: - kernel.do_execute('reset -f', True) + asyncio.run(kernel.do_execute('reset -f', True)) request.addfinalizer(reset_kernel) return kernel @@ -200,10 +277,7 @@ def test_get_namespace_view(kernel): """ Test the namespace view of the kernel. """ - if IPYKERNEL_6: - execute = asyncio.run(kernel.do_execute('a = 1', True)) - else: - execute = kernel.do_execute('a = 1', True) + execute = asyncio.run(kernel.do_execute('a = 1', True)) nsview = repr(kernel.get_namespace_view()) assert "'a':" in nsview @@ -211,21 +285,39 @@ def test_get_namespace_view(kernel): assert "'size': 1" in nsview assert "'view': '1'" in nsview assert "'numpy_type': 'Unknown'" in nsview + assert "'python_type': 'int'" in nsview + - if PY3: - assert "'python_type': 'int'" in nsview +@pytest.mark.parametrize("filter_on", [True, False]) +def test_get_namespace_view_filter_on(kernel, filter_on): + """ + Test the namespace view of the kernel with filters on and off. + """ + execute = asyncio.run(kernel.do_execute('a = 1', True)) + asyncio.run(kernel.do_execute('TestFilterOff = 1', True)) + + settings = kernel.namespace_view_settings + settings['filter_on'] = filter_on + settings['exclude_capitalized'] = True + nsview = kernel.get_namespace_view() + + if not filter_on: + assert 'a' in nsview + assert 'TestFilterOff' in nsview else: - assert "'python_type': u'int'" in nsview + assert 'TestFilterOff' not in nsview + assert 'a' in nsview + + # Restore settings for other tests + settings['filter_on'] = True + settings['exclude_capitalized'] = False def test_get_var_properties(kernel): """ Test the properties fo the variables in the namespace. """ - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 1', True)) - else: - kernel.do_execute('a = 1', True) + asyncio.run(kernel.do_execute('a = 1', True)) var_properties = repr(kernel.get_var_properties()) assert "'a'" in var_properties @@ -243,10 +335,7 @@ def test_get_var_properties(kernel): def test_get_value(kernel): """Test getting the value of a variable.""" name = 'a' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute("a = 124", True)) - else: - kernel.do_execute("a = 124", True) + asyncio.run(kernel.do_execute("a = 124", True)) # Check data type send assert kernel.get_value(name) == 124 @@ -255,10 +344,7 @@ def test_get_value(kernel): def test_set_value(kernel): """Test setting the value of a variable.""" name = 'a' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 0', True)) - else: - kernel.do_execute('a = 0', True) + asyncio.run(kernel.do_execute('a = 0', True)) value = 10 kernel.set_value(name, value) log_text = get_log_text(kernel) @@ -272,10 +358,7 @@ def test_set_value(kernel): def test_remove_value(kernel): """Test the removal of a variable.""" name = 'a' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 1', True)) - else: - kernel.do_execute('a = 1', True) + asyncio.run(kernel.do_execute('a = 1', True)) var_properties = repr(kernel.get_var_properties()) assert "'a'" in var_properties @@ -297,10 +380,7 @@ def test_copy_value(kernel): """Test the copy of a variable.""" orig_name = 'a' new_name = 'b' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 1', True)) - else: - kernel.do_execute('a = 1', True) + asyncio.run(kernel.do_execute('a = 1', True)) var_properties = repr(kernel.get_var_properties()) assert "'a'" in var_properties @@ -336,11 +416,7 @@ def test_load_npz_data(kernel, load): namespace_file = osp.join(FILES_PATH, 'load_data.npz') extention = '.npz' overwrite, execute, variables = load - - if IPYKERNEL_6: - asyncio.run(kernel.do_execute(execute, True)) - else: - kernel.do_execute(execute, True) + asyncio.run(kernel.do_execute(execute, True)) kernel.load_data(namespace_file, extention, overwrite=overwrite) for var, value in variables.items(): @@ -368,11 +444,7 @@ def test_load_data(kernel): def test_save_namespace(kernel): """Test saving the namespace into filename.""" namespace_file = osp.join(FILES_PATH, 'save_data.spydata') - - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('b = 1', True)) - else: - kernel.do_execute('b = 1', True) + asyncio.run(kernel.do_execute('b = 1', True)) kernel.save_namespace(namespace_file) assert osp.isfile(namespace_file) @@ -400,7 +472,7 @@ def test_get_doc(kernel): def test_get_source(kernel): """Test to get object source.""" objtxt = 'help' - assert 'class _Helper(object):' in kernel.get_source(objtxt) + assert 'class _Helper' in kernel.get_source(objtxt) # --- Other stuff @@ -416,11 +488,7 @@ def test_output_from_c_libraries(kernel, capsys): # With Wurlitzer we have the expected output kernel._load_wurlitzer() - - if IPYKERNEL_6: - asyncio.run(kernel.do_execute(code, True)) - else: - kernel.do_execute(code, True) + asyncio.run(kernel.do_execute(code, True)) captured = capsys.readouterr() assert captured.out == "Hello from C\n" @@ -435,11 +503,9 @@ def test_cwd_in_sys_path(): cmd = "from spyder_kernels.console import start; start.main()" with setup_kernel(cmd) as client: - msg_id = client.execute("import sys; sys_path = sys.path", - user_expressions={'output':'sys_path'}) - reply = client.get_shell_msg(timeout=TIMEOUT) - while 'user_expressions' not in reply['content']: - reply = client.get_shell_msg(timeout=TIMEOUT) + reply = client.execute_interactive( + "import sys; sys_path = sys.path", + user_expressions={'output':'sys_path'}, timeout=TIMEOUT) # Transform value obtained through user_expressions user_expressions = reply['content']['user_expressions'] @@ -451,19 +517,16 @@ def test_cwd_in_sys_path(): @flaky(max_runs=3) -@pytest.mark.skipif(not PY3, - reason="Only meant for Python 3") def test_multiprocessing(tmpdir): """ - Test that multiprocessing works on Python 3. + Test that multiprocessing works. """ # Command to start the kernel cmd = "from spyder_kernels.console import start; start.main()" with setup_kernel(cmd) as client: # Remove all variables - client.execute("%reset -f") - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive("%reset -f", timeout=TIMEOUT) # Write multiprocessing code to a file code = """ @@ -480,8 +543,8 @@ def f(x): p.write(code) # Run code - client.execute("runfile(r'{}')".format(to_text_string(p))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `result` variable is defined client.inspect('result') @@ -493,19 +556,16 @@ def f(x): @flaky(max_runs=3) -@pytest.mark.skipif(not PY3, - reason="Only meant for Python 3") def test_multiprocessing_2(tmpdir): """ - Test that multiprocessing works on Python 3. + Test that multiprocessing works. """ # Command to start the kernel cmd = "from spyder_kernels.console import start; start.main()" with setup_kernel(cmd) as client: # Remove all variables - client.execute("%reset -f") - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive("%reset -f", timeout=TIMEOUT) # Write multiprocessing code to a file code = """ @@ -527,8 +587,8 @@ def myFunc(i): p.write(code) # Run code - client.execute("runfile(r'{}')".format(to_text_string(p))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `result` variable is defined client.inspect('result') @@ -541,7 +601,6 @@ def myFunc(i): @flaky(max_runs=3) -@pytest.mark.skipif(not PY3, reason="Only meant for Python 3") @pytest.mark.skipif( sys.platform == 'darwin' and sys.version_info[:2] == (3, 8), reason="Fails on Mac with Python 3.8") @@ -550,15 +609,14 @@ def myFunc(i): reason="Doesn't work with pip packages") def test_dask_multiprocessing(tmpdir): """ - Test that dask multiprocessing works on Python 3. + Test that dask multiprocessing works. """ # Command to start the kernel cmd = "from spyder_kernels.console import start; start.main()" with setup_kernel(cmd) as client: # Remove all variables - client.execute("%reset -f") - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive("%reset -f") # Write multiprocessing code to a file # Runs two times to verify that in the second case it doesn't break @@ -574,11 +632,11 @@ def test_dask_multiprocessing(tmpdir): p.write(code) # Run code two times - client.execute("runfile(r'{}')".format(to_text_string(p))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) - client.execute("runfile(r'{}')".format(to_text_string(p))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `x` variable is defined client.inspect('x') @@ -599,16 +657,15 @@ def test_runfile(tmpdir): with setup_kernel(cmd) as client: # Remove all variables - client.execute("%reset -f") - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive("%reset -f", timeout=TIMEOUT) # Write defined variable code to a file - code = u"result = 'hello world'; error # make an error" + code = "result = 'hello world'; error # make an error" d = tmpdir.join("defined-test.py") d.write(code) # Write undefined variable code to a file - code = dedent(u""" + code = dedent(""" try: result3 = result except NameError: @@ -618,9 +675,8 @@ def test_runfile(tmpdir): u.write(code) # Run code file `d` to define `result` even after an error - client.execute("runfile(r'{}', current_namespace=False)" - .format(to_text_string(d))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(d))), timeout=TIMEOUT) # Verify that `result` is defined in the current namespace client.inspect('result') @@ -631,9 +687,8 @@ def test_runfile(tmpdir): assert content['found'] # Run code file `u` without current namespace - client.execute("runfile(r'{}', current_namespace=False)" - .format(to_text_string(u))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(u))), timeout=TIMEOUT) # Verify that the variable `result2` is defined client.inspect('result2') @@ -644,9 +699,8 @@ def test_runfile(tmpdir): assert content['found'] # Run code file `u` with current namespace - client.execute("runfile(r'{}', current_namespace=True)" - .format(to_text_string(u))) - msg = client.get_shell_msg(timeout=TIMEOUT) + msg = client.execute_interactive("%runfile {} --current-namespace" + .format(repr(str(u))), timeout=TIMEOUT) content = msg['content'] # Verify that the variable `result3` is defined @@ -678,30 +732,27 @@ def test_np_threshold(kernel): with setup_kernel(cmd) as client: # Set Numpy threshold, suppress and formatter - client.execute(""" + client.execute_interactive(""" import numpy as np; np.set_printoptions( threshold=np.inf, suppress=True, formatter={'float_kind':'{:0.2f}'.format}) - """) - client.get_shell_msg(timeout=TIMEOUT) + """, timeout=TIMEOUT) # Create a big Numpy array and an array to check decimal format - client.execute(""" + client.execute_interactive(""" x = np.random.rand(75000,5); a = np.array([123412341234.123412341234]) -""") - client.get_shell_msg(timeout=TIMEOUT) +""", timeout=TIMEOUT) # Assert that NumPy threshold, suppress and formatter # are the same as the ones set by the user - client.execute(""" + client.execute_interactive(""" t = np.get_printoptions()['threshold']; s = np.get_printoptions()['suppress']; f = np.get_printoptions()['formatter'] -""") - client.get_shell_msg(timeout=TIMEOUT) +""", timeout=TIMEOUT) # Check correct decimal format client.inspect('a') @@ -747,8 +798,7 @@ def test_turtle_launch(tmpdir): with setup_kernel(cmd) as client: # Remove all variables - client.execute("%reset -f") - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive("%reset -f", timeout=TIMEOUT) # Write turtle code to a file code = """ @@ -772,8 +822,8 @@ def test_turtle_launch(tmpdir): p.write(code) # Run code - client.execute("runfile(r'{}')".format(to_text_string(p))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `tess` variable is defined client.inspect('tess') @@ -790,8 +840,8 @@ def test_turtle_launch(tmpdir): p.write(code) # Run code again - client.execute("runfile(r'{}')".format(to_text_string(p))) - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `a` variable is defined client.inspect('a') @@ -811,10 +861,8 @@ def test_matplotlib_inline(kernel): with setup_kernel(cmd) as client: # Get current backend code = "import matplotlib; backend = matplotlib.get_backend()" - client.execute(code, user_expressions={'output': 'backend'}) - reply = client.get_shell_msg(timeout=TIMEOUT) - while 'user_expressions' not in reply['content']: - reply = client.get_shell_msg(timeout=TIMEOUT) + reply = client.execute_interactive( + code, user_expressions={'output': 'backend'}, timeout=TIMEOUT) # Transform value obtained through user_expressions user_expressions = reply['content']['user_expressions'] @@ -829,10 +877,7 @@ def test_do_complete(kernel): """ Check do complete works in normal and debugging mode. """ - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('abba = 1', True)) - else: - kernel.do_execute('abba = 1', True) + asyncio.run(kernel.do_execute('abba = 1', True)) assert kernel.get_value('abba') == 1 match = kernel.do_complete('ab', 2) assert 'abba' in match['matches'] @@ -841,7 +886,7 @@ def test_do_complete(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.completenames = lambda *ignore: ['baba'] - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] match = kernel.do_complete('ba', 2) assert 'baba' in match['matches'] pdb_obj.curframe = None @@ -855,17 +900,11 @@ def test_callables_and_modules(kernel, exclude_callables_and_modules, Tests that callables and modules are in the namespace view only when the right options are passed to the kernel. """ - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('import numpy', True)) - asyncio.run(kernel.do_execute('a = 10', True)) - asyncio.run(kernel.do_execute('def f(x): return x', True)) - else: - kernel.do_execute('import numpy', True) - kernel.do_execute('a = 10', True) - kernel.do_execute('def f(x): return x', True) + asyncio.run(kernel.do_execute('import numpy', True)) + asyncio.run(kernel.do_execute('a = 10', True)) + asyncio.run(kernel.do_execute('def f(x): return x', True)) settings = kernel.namespace_view_settings - settings['exclude_callables_and_modules'] = exclude_callables_and_modules settings['exclude_unsupported'] = exclude_unsupported nsview = kernel.get_namespace_view() @@ -899,7 +938,7 @@ def test_comprehensions_with_locals_in_pdb(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] # Create a local variable. kernel.shell.pdb_session.default('zz = 10') @@ -925,7 +964,7 @@ def test_comprehensions_with_locals_in_pdb_2(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] # Create a local variable. kernel.shell.pdb_session.default('aa = [1, 2]') @@ -951,18 +990,12 @@ def test_namespaces_in_pdb(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] # Check adding something to globals works pdb_obj.default("globals()['test2'] = 0") assert pdb_obj.curframe.f_globals["test2"] == 0 - if PY2: - # no error method in py2 - pdb_obj.curframe = None - pdb_obj.curframe_locals = None - return - # Create wrapper to check for errors old_error = pdb_obj.error pdb_obj._error_occured = False @@ -1000,7 +1033,7 @@ def test_functions_with_locals_in_pdb(kernel): Frame = namedtuple("Frame", ["f_globals"]) pdb_obj.curframe = Frame(f_globals=kernel.shell.user_ns) pdb_obj.curframe_locals = kernel.shell.user_ns - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] # Create a local function. kernel.shell.pdb_session.default( @@ -1032,7 +1065,7 @@ def test_functions_with_locals_in_pdb_2(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] # Create a local function. kernel.shell.pdb_session.default( @@ -1069,7 +1102,7 @@ def test_locals_globals_in_pdb(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell.pdb_session = pdb_obj + kernel.shell._namespace_stack = [pdb_obj] assert kernel.get_value('a') == 1 @@ -1104,52 +1137,95 @@ def test_locals_globals_in_pdb(kernel): @flaky(max_runs=3) -@pytest.mark.parametrize("backend", [None, 'inline', 'tk', 'qt5']) -@pytest.mark.skipif( - not sys.platform.startswith('linux'), - reason="Doesn't work reliably on Windows and Mac") +@pytest.mark.parametrize("backend", [None, 'inline', 'tk', 'qt']) @pytest.mark.skipif( os.environ.get('USE_CONDA') != 'true', reason="Doesn't work with pip packages") @pytest.mark.skipif( - sys.version_info[:2] < (3, 8), - reason="Too flaky in Python 3.7 and doesn't work in older versions") + sys.version_info[:2] < (3, 9), + reason="Too flaky in Python 3.8 and doesn't work in older versions") def test_get_interactive_backend(backend): """ Test that we correctly get the interactive backend set in the kernel. """ - cmd = "from spyder_kernels.console import start; start.main()" + # This test passes locally but fails on CIs. Don't know why. + if sys.platform == "darwin" and backend == "qt" and os.environ.get('CI'): + return + cmd = "from spyder_kernels.console import start; start.main()" with setup_kernel(cmd) as client: # Set backend if backend is not None: - client.execute("%matplotlib {}".format(backend)) - client.get_shell_msg(timeout=TIMEOUT) - client.execute("import time; time.sleep(.1)") - client.get_shell_msg(timeout=TIMEOUT) + client.execute_interactive( + "%matplotlib {}".format(backend), timeout=TIMEOUT) + client.execute_interactive( + "import time; time.sleep(.1)", timeout=TIMEOUT) # Get backend code = "backend = get_ipython().kernel.get_mpl_interactive_backend()" - client.execute(code, user_expressions={'output': 'backend'}) - reply = client.get_shell_msg(timeout=TIMEOUT) - while 'user_expressions' not in reply['content']: - reply = client.get_shell_msg(timeout=TIMEOUT) + reply = client.execute_interactive( + code, user_expressions={'output': 'backend'}, timeout=TIMEOUT) # Get value obtained through user_expressions user_expressions = reply['content']['user_expressions'] value = user_expressions['output']['data']['text/plain'] + # remove quotes + value = value[1:-1] + # Assert we got the right interactive backend if backend is not None: - assert MPL_BACKENDS_FROM_SPYDER[value] == backend + assert value == backend else: - assert value == '0' + assert value == 'inline' + + +def test_global_message(tmpdir): + """ + Test that using `global` triggers a warning. + """ + # Command to start the kernel + cmd = "from spyder_kernels.console import start; start.main()" + + with setup_kernel(cmd) as client: + # Remove all variables + client.execute_interactive("%reset -f", timeout=TIMEOUT) + + # Write code with a global to a file + code = ( + "def foo1():\n" + " global x\n" + " x = 2\n" + "x = 1\n" + "print(x)\n" + ) + + p = tmpdir.join("test.py") + p.write(code) + global found + found = False + + def check_found(msg): + if "text" in msg["content"]: + if ("WARNING: This file contains a global statement" in + msg["content"]["text"]): + global found + found = True + + # Run code in current namespace + client.execute_interactive("%runfile {} --current-namespace".format( + repr(str(p))), timeout=TIMEOUT, output_hook=check_found) + assert not found + + # Run code in empty namespace + client.execute_interactive( + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT, + output_hook=check_found) + + assert found @flaky(max_runs=3) -@pytest.mark.skipif( - sys.version_info[0] < 3, - reason="Fails with python 2") def test_debug_namespace(tmpdir): """ Test that the kernel uses the proper namespace while debugging. @@ -1163,7 +1239,7 @@ def test_debug_namespace(tmpdir): d.write('def func():\n bb = "hello"\n breakpoint()\nfunc()') # Run code file `d` - msg_id = client.execute("runfile(r'{}')".format(to_text_string(d))) + msg_id = client.execute("%runfile {}".format(repr(str(d)))) # make sure that 'bb' returns 'hello' client.get_stdin_msg(timeout=TIMEOUT) @@ -1190,6 +1266,105 @@ def test_debug_namespace(tmpdir): break +def test_interrupt(): + """ + Test that the kernel can be interrupted by calling a comm handler. + """ + # Command to start the kernel + cmd = "from spyder_kernels.console import start; start.main()" + import pickle + with setup_kernel(cmd) as client: + kernel_comm = CommBase() + + # Create new comm and send the highest protocol + comm = Comm(kernel_comm._comm_name, client) + comm.open(data={'pickle_highest_protocol': pickle.HIGHEST_PROTOCOL}) + comm._send_channel = client.control_channel + kernel_comm._register_comm(comm) + + client.execute_interactive("import time", timeout=TIMEOUT) + + # Try interrupting loop + t0 = time.time() + msg_id = client.execute("for i in range(100): time.sleep(.1)") + time.sleep(.2) + # Raise interrupt on control_channel + kernel_comm.remote_call().raise_interrupt_signal() + # Wait for shell message + while True: + assert time.time() - t0 < 5 + msg = client.get_shell_msg(timeout=TIMEOUT) + if msg["parent_header"].get("msg_id") != msg_id: + # not from my request + continue + break + assert time.time() - t0 < 5 + + if os.name == 'nt': + # Windows doesn't do "interrupting sleep" + return + + # Try interrupting sleep + t0 = time.time() + msg_id = client.execute("time.sleep(10)") + time.sleep(.2) + # Raise interrupt on control_channel + kernel_comm.remote_call().raise_interrupt_signal() + # Wait for shell message + while True: + assert time.time() - t0 < 5 + msg = client.get_shell_msg(timeout=TIMEOUT) + if msg["parent_header"].get("msg_id") != msg_id: + # not from my request + continue + break + assert time.time() - t0 < 5 + + +def test_enter_debug_after_interruption(): + """ + Test that we can enter the debugger after interrupting the current + execution. + """ + # Command to start the kernel + cmd = "from spyder_kernels.console import start; start.main()" + import pickle + with setup_kernel(cmd) as client: + kernel_comm = CommBase() + + # Create new comm and send the highest protocol + comm = Comm(kernel_comm._comm_name, client) + comm.open(data={'pickle_highest_protocol': pickle.HIGHEST_PROTOCOL}) + comm._send_channel = client.control_channel + kernel_comm._register_comm(comm) + + client.execute_interactive("import time", timeout=TIMEOUT) + + # Try interrupting loop + t0 = time.time() + msg_id = client.execute("for i in range(100): time.sleep(.1)") + time.sleep(.2) + # Request to enter the debugger + kernel_comm.remote_call().request_pdb_stop() + # Wait for debug message + while True: + assert time.time() - t0 < 5 + msg = client.get_iopub_msg(timeout=TIMEOUT) + if msg.get('msg_type') == 'stream': + print(msg["content"].get("text")) + if msg["parent_header"].get("msg_id") != msg_id: + # not from my request + continue + if msg.get('msg_type') == 'comm_msg': + if msg["content"].get("data", {}).get("content", {}).get( + 'call_name') == 'pdb_input': + # pdb entered + break + comm.handle_msg(msg) + + assert time.time() - t0 < 5 + + def test_non_strings_in_locals(kernel): """ Test that we can hande non-string entries in `locals` when bulding the @@ -1197,18 +1372,12 @@ def test_non_strings_in_locals(kernel): This is a regression test for issue spyder-ide/spyder#19145 """ - if IPYKERNEL_6: - execute = asyncio.run(kernel.do_execute( - 'locals().update({1:2})', True)) - else: - execute = kernel.do_execute('locals().update({1:2})', True) + execute = asyncio.run(kernel.do_execute('locals().update({1:2})', True)) nsview = repr(kernel.get_namespace_view()) assert "1:" in nsview -@pytest.mark.skipif( - sys.version_info[0] < 3, reason="Doesn't work with Python 2") def test_django_settings(kernel): """ Test that we don't generate errors when importing `django.conf.settings`. diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py b/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py new file mode 100644 index 00000000000..d3cc7a1f485 --- /dev/null +++ b/external-deps/spyder-kernels/spyder_kernels/customize/code_runner.py @@ -0,0 +1,614 @@ +# +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- + +""" +Spyder magics related to code execution, debugging, profiling, etc. +""" + +# Standard library imports +import ast +import bdb +import builtins +from contextlib import contextmanager +import io +import logging +import os +import pdb +import shlex +import sys +import time + +# Third-party imports +from IPython.core.inputtransformer2 import ( + TransformerManager, + leading_indent, + leading_empty_lines, +) +from IPython.core.magic import ( + needs_local_scope, + magics_class, + Magics, + line_magic, +) +from IPython.core import magic_arguments + +# Local imports +from spyder_kernels.comms.frontendcomm import frontend_request +from spyder_kernels.customize.namespace_manager import NamespaceManager +from spyder_kernels.customize.spyderpdb import SpyderPdb +from spyder_kernels.customize.umr import UserModuleReloader +from spyder_kernels.customize.utils import ( + capture_last_Expr, canonic, exec_encapsulate_locals +) + + +# For logging +logger = logging.getLogger(__name__) + + +def runfile_arguments(func): + """Decorator to add runfile magic arguments to magic.""" + decorators = [ + magic_arguments.magic_arguments(), + magic_arguments.argument( + "filename", + help=""" + Filename to run + """, + ), + magic_arguments.argument( + "--args", + help=""" + Command line arguments (string) + """, + ), + magic_arguments.argument( + "--wdir", + const=True, + nargs="?", + help=""" + Working directory + """, + ), + magic_arguments.argument( + "--post-mortem", + action="store_true", + help=""" + Enter post-mortem mode on errors + """, + ), + magic_arguments.argument( + "--current-namespace", + action="store_true", + help=""" + Use current namespace + """, + ), + magic_arguments.argument( + "--namespace", + help=""" + Namespace to run the file in + """, + ) + ] + for dec in reversed(decorators): + func = dec(func) + return func + + +def runcell_arguments(func): + """Decorator to add runcell magic arguments to magic.""" + decorators = [ + magic_arguments.magic_arguments(), + magic_arguments.argument( + "--name", "-n", + help=""" + Cell name. + """, + ), + magic_arguments.argument( + "--index", "-i", + help=""" + Cell index. + """, + ), + magic_arguments.argument( + "filename", + nargs="?", + help=""" + Filename + """, + ), + magic_arguments.argument( + "--post-mortem", + action="store_true", + default=False, + help=""" + Enter post-mortem mode on errors + """, + ) + ] + for dec in reversed(decorators): + func = dec(func) + return func + + +@magics_class +class SpyderCodeRunner(Magics): + """ + Functions and magics related to code execution, debugging, profiling, etc. + """ + def __init__(self, *args, **kwargs): + self.show_global_msg = True + self.show_invalid_syntax_msg = True + self.umr = UserModuleReloader( + namelist=os.environ.get("SPY_UMR_NAMELIST", None) + ) + super().__init__(*args, **kwargs) + + @runfile_arguments + @needs_local_scope + @line_magic + def runfile(self, line, local_ns=None): + """ + Run a file. + """ + args, local_ns = self._parse_runfile_argstring( + self.runfile, line, local_ns) + + return self._exec_file( + filename=args.filename, + canonic_filename=args.canonic_filename, + args=args.args, + wdir=args.wdir, + post_mortem=args.post_mortem, + current_namespace=args.current_namespace, + context_globals=args.namespace, + context_locals=local_ns, + ) + + @runfile_arguments + @needs_local_scope + @line_magic + def debugfile(self, line, local_ns=None): + """ + Debug a file. + """ + args, local_ns = self._parse_runfile_argstring( + self.debugfile, line, local_ns) + + with self._debugger_exec(args.canonic_filename, True) as debug_exec: + self._exec_file( + filename=args.filename, + canonic_filename=args.canonic_filename, + args=args.args, + wdir=args.wdir, + current_namespace=args.current_namespace, + exec_fun=debug_exec, + post_mortem=args.post_mortem, + context_globals=args.namespace, + context_locals=local_ns, + ) + + @runcell_arguments + @needs_local_scope + @line_magic + def runcell(self, line, local_ns=None): + """ + Run a code cell from an editor. + """ + args = self._parse_runcell_argstring(self.runcell, line) + + return self._exec_cell( + cell_id=args.cell_id, + filename=args.filename, + canonic_filename=args.canonic_filename, + post_mortem=args.post_mortem, + context_globals=self.shell.user_ns, + context_locals=local_ns, + ) + + @runcell_arguments + @needs_local_scope + @line_magic + def debugcell(self, line, local_ns=None): + """ + Debug a code cell from an editor. + """ + args = self._parse_runcell_argstring(self.debugcell, line) + + with self._debugger_exec(args.canonic_filename, False) as debug_exec: + return self._exec_cell( + cell_id=args.cell_id, + filename=args.filename, + canonic_filename=args.canonic_filename, + exec_fun=debug_exec, + post_mortem=args.post_mortem, + context_globals=self.shell.user_ns, + context_locals=local_ns, + ) + + @contextmanager + def _debugger_exec(self, filename, continue_if_has_breakpoints): + """Get an exec function to use for debugging.""" + if not self.shell.is_debugging(): + debugger = SpyderPdb() + debugger.set_remote_filename(filename) + debugger.continue_if_has_breakpoints = continue_if_has_breakpoints + yield debugger.run + return + + session = self.shell.pdb_session + with session.recursive_debugger() as debugger: + debugger.set_remote_filename(filename) + debugger.continue_if_has_breakpoints = continue_if_has_breakpoints + + def debug_exec(code, glob, loc): + return sys.call_tracing(debugger.run, (code, glob, loc)) + + # Enter recursive debugger + yield debug_exec + + def _exec_file( + self, + filename=None, + args=None, + wdir=None, + post_mortem=False, + current_namespace=False, + exec_fun=None, + canonic_filename=None, + context_locals=None, + context_globals=None, + ): + """ + Execute a file. + """ + if self.umr.enabled and self.shell.special != "cython": + self.umr.run() + if args is not None and not isinstance(args, str): + raise TypeError("expected a character buffer object") + + try: + file_code = self._get_file_code(filename, raise_exception=True) + except Exception: + print( + "This command failed to be executed because an error occurred " + "while trying to get the file code from Spyder's editor. " + "The error was:\n\n" + ) + self.shell.showtraceback(exception_only=True) + return + + # Here the remote filename has been used. It must now be valid locally. + filename = canonic_filename + + with NamespaceManager( + self.shell, + filename, + current_namespace=current_namespace, + file_code=file_code, + context_locals=context_locals, + context_globals=context_globals, + ) as (ns_globals, ns_locals): + sys.argv = [filename] + if args is not None: + # args are a sting in a string + for arg in shlex.split(args): + sys.argv.append(arg) + + if "multiprocessing" in sys.modules: + # See https://github.com/spyder-ide/spyder/issues/16696 + try: + sys.modules["__mp_main__"] = sys.modules["__main__"] + except Exception: + pass + + if wdir is not None: + if wdir is True: + # True means use file dir + wdir = os.path.dirname(filename) + if os.path.isdir(wdir): + os.chdir(wdir) + + # See https://github.com/spyder-ide/spyder/issues/13632 + if "multiprocessing.process" in sys.modules: + try: + import multiprocessing.process + multiprocessing.process.ORIGINAL_DIR = os.path.abspath(wdir) + except Exception: + pass + else: + print("Working directory {} doesn't exist.\n".format(wdir)) + + try: + if self.shell.special == "cython": + # Cython files + with io.open(filename, encoding="utf-8") as f: + self.shell.run_cell_magic("cython", "", f.read()) + else: + self._exec_code( + file_code, + filename, + ns_globals, + ns_locals, + post_mortem=post_mortem, + exec_fun=exec_fun, + capture_last_expression=False, + global_warning=not current_namespace, + ) + finally: + sys.argv = [""] + + def _exec_cell( + self, + cell_id, + filename=None, + post_mortem=False, + exec_fun=None, + canonic_filename=None, + context_locals=None, + context_globals=None, + ): + """ + Execute a code cell. + """ + try: + # Get code from spyder + cell_code = frontend_request(blocking=True).run_cell(cell_id, filename) + except Exception: + print( + "This command failed to be executed because an error occurred " + "while trying to get the cell code from Spyder's editor." + "The error was:\n\n" + ) + self.shell.showtraceback(exception_only=True) + return + + if not cell_code or cell_code.strip() == "": + print("Nothing to execute, this cell is empty.\n") + return + + # Trigger `post_execute` to exit the additional pre-execution. + # See Spyder PR #7310. + self.shell.events.trigger("post_execute") + file_code = self._get_file_code(filename, save_all=False) + + # Here the remote filename has been used. It must now be valid locally. + filename = canonic_filename + + with NamespaceManager( + self.shell, + filename, + current_namespace=True, + file_code=file_code, + context_locals=context_locals, + context_globals=context_globals + ) as (ns_globals, ns_locals): + return self._exec_code( + cell_code, + filename, + ns_globals, + ns_locals, + post_mortem=post_mortem, + exec_fun=exec_fun, + capture_last_expression=True, + ) + + def _get_current_file_name(self): + """Get the current editor file name.""" + try: + return frontend_request(blocking=True).current_filename() + except Exception: + print( + "This command failed to be executed because an error occurred " + "while trying to get the current file name from Spyder's editor." + "The error was:\n\n" + ) + self.shell.showtraceback(exception_only=True) + return None + + def _get_file_code(self, filename, save_all=True, raise_exception=False): + """Retrieve the content of a file.""" + # Get code from spyder + try: + return frontend_request(blocking=True).get_file_code( + filename, save_all=save_all + ) + except Exception: + # Maybe this is a local file + try: + with open(filename, "r") as f: + return f.read() + except FileNotFoundError: + pass + + if raise_exception: + raise + + # Finally return None + return None + + def _exec_code( + self, + code, + filename, + ns_globals, + ns_locals=None, + post_mortem=False, + exec_fun=None, + capture_last_expression=False, + global_warning=False, + ): + """Execute code and display any exception.""" + if exec_fun is None: + exec_fun = exec + + is_ipython = os.path.splitext(filename)[1] == ".ipy" + try: + if not is_ipython: + # TODO: Remove the try-except and let the SyntaxError raise + # because there should't be IPython code in a Python file. + try: + ast_code = ast.parse( + self._transform_cell(code, indent_only=True) + ) + except SyntaxError as e: + try: + ast_code = ast.parse(self._transform_cell(code)) + except SyntaxError: + raise e from None + else: + if self.show_invalid_syntax_msg: + print( + "\nWARNING: This is not valid Python code. " + "If you want to use IPython magics, " + "flexible indentation, and prompt removal, " + "we recommend that you save this file with the " + ".ipy extension.\n" + ) + self.show_invalid_syntax_msg = False + else: + ast_code = ast.parse(self._transform_cell(code)) + + # Print warning for global + if global_warning and self.show_global_msg: + has_global = any( + isinstance(node, ast.Global) for node in ast.walk(ast_code) + ) + if has_global: + print( + "\nWARNING: This file contains a global statement, " + "but it is run in an empty namespace. " + "Consider using the " + "'Run in console's namespace instead of an empty one' " + "option, that you can find in the menu 'Run > " + "Configuration per file', if you want to capture the " + "namespace.\n" + ) + self.show_global_msg = False + + if code.rstrip()[-1:] == ";": + # Supress output with ; + capture_last_expression = False + + if capture_last_expression: + ast_code, capture_last_expression = capture_last_Expr( + ast_code, "_spyder_out", ns_globals + ) + + exec_encapsulate_locals( + ast_code, ns_globals, ns_locals, exec_fun, filename + ) + + if capture_last_expression: + out = ns_globals.pop("_spyder_out", None) + if out is not None: + return out + except SystemExit as status: + # ignore exit(0) + if status.code: + self.shell.showtraceback(exception_only=True) + except BaseException as error: + if isinstance(error, bdb.BdbQuit) and self.shell.pdb_session: + # Ignore BdbQuit if we are debugging, as it is expected. + pass + elif post_mortem and isinstance(error, Exception): + error_type, error, tb = sys.exc_info() + self._post_mortem_excepthook(error_type, error, tb) + else: + # We ignore the call to exec + self.shell.showtraceback(tb_offset=1) + finally: + __tracebackhide__ = "__pdb_exit__" + + def _count_leading_empty_lines(self, cell): + """Count the number of leading empty cells.""" + lines = cell.splitlines(keepends=True) + if not lines: + return 0 + for i, line in enumerate(lines): + if line and not line.isspace(): + return i + return len(lines) + + def _transform_cell(self, code, indent_only=False): + """Transform IPython code to Python code.""" + number_empty_lines = self._count_leading_empty_lines(code) + if indent_only: + if not code.endswith("\n"): + code += "\n" # Ensure the cell has a trailing newline + lines = code.splitlines(keepends=True) + lines = leading_indent(leading_empty_lines(lines)) + code = "".join(lines) + else: + tm = TransformerManager() + code = tm.transform_cell(code) + return "\n" * number_empty_lines + code + + def _post_mortem_excepthook(self, type, value, tb): + """ + For post mortem exception handling, print a banner and enable post + mortem debugging. + """ + self.shell.showtraceback((type, value, tb)) + p = pdb.Pdb(self.shell.colors) + + if not type == SyntaxError: + # wait for stderr to print (stderr.flush does not work in this case) + time.sleep(0.1) + print("*" * 40) + print("Entering post mortem debugging...") + print("*" * 40) + # add ability to move between frames + p.reset() + frame = tb.tb_next.tb_frame + # wait for stdout to print + time.sleep(0.1) + p.interaction(frame, tb) + + def _parse_argstring(self, magic_func, argstring): + """ + Parse a string of arguments for a magic function. + + This is needed because magic_arguments.parse_argstring does + platform-dependent things with quotes and backslashes. For + example, on Windows, strings are removed and backslashes are + escaped. + """ + argv = shlex.split(argstring) + args = magic_func.parser.parse_args(argv) + if args.filename is None: + args.filename = self._get_current_file_name() + args.canonic_filename = canonic(args.filename) + return args + + def _parse_runfile_argstring(self, magic_func, argstring, local_ns): + """Parse an args string for runfile and debugfile.""" + args = self._parse_argstring(magic_func, argstring) + if args.namespace is None: + args.namespace = self.shell.user_ns + else: + if local_ns is not None and args.namespace in local_ns: + args.namespace = local_ns[args.namespace] + elif args.namespace in self.shell.user_ns: + args.namespace = self.shell.user_ns[args.namespace] + else: + raise NameError( + f"name '{args.namespace}' is not defined" + ) + local_ns = None + args.current_namespace = True + return args, local_ns + + def _parse_runcell_argstring(self, magic_func, argstring): + """Parse an args string for runcell and debugcell.""" + args = self._parse_argstring(magic_func, argstring) + args.cell_id = args.name + if args.cell_id is None: + args.cell_id = int(args.index) + return args diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/namespace_manager.py b/external-deps/spyder-kernels/spyder_kernels/customize/namespace_manager.py index e92214e8ca9..f758cf27711 100755 --- a/external-deps/spyder-kernels/spyder_kernels/customize/namespace_manager.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/namespace_manager.py @@ -9,10 +9,6 @@ import types import sys -from IPython.core.getipython import get_ipython - -from spyder_kernels.py3compat import PY2 - def new_main_mod(filename, modname): """ @@ -28,12 +24,12 @@ def new_main_mod(filename, modname): main_mod.__file__ = filename # It seems pydoc (and perhaps others) needs any module instance to # implement a __nonzero__ method - main_mod.__nonzero__ = lambda : True + main_mod.__nonzero__ = lambda: True return main_mod -class NamespaceManager(object): +class NamespaceManager: """ Get a namespace and set __file__ to filename for this namespace. @@ -41,52 +37,55 @@ class NamespaceManager(object): current_namespace is True, or a new namespace. """ - def __init__(self, filename, namespace=None, current_namespace=False, - file_code=None, stack_depth=1): + def __init__( + self, + shell, + filename, + current_namespace=False, + file_code=None, + context_locals=None, + context_globals=None, + ): + self.shell = shell self.filename = filename - self.ns_globals = namespace + self.ns_globals = None self.ns_locals = None self.current_namespace = current_namespace self._previous_filename = None self._previous_main = None - self._previous_running_namespace = None self._reset_main = False self._file_code = file_code - ipython_shell = get_ipython() - self.context_globals = ipython_shell.get_global_scope(stack_depth + 1) - self.context_locals = ipython_shell.get_local_scope(stack_depth + 1) + if context_globals is None: + context_globals = shell.user_ns + self.context_globals = context_globals + self.context_locals = context_locals def __enter__(self): """ Prepare the namespace. """ # Save previous __file__ - ipython_shell = get_ipython() - if self.ns_globals is None: - if self.current_namespace: - self.ns_globals = self.context_globals - self.ns_locals = self.context_locals - if '__file__' in self.ns_globals: - self._previous_filename = self.ns_globals['__file__'] - self.ns_globals['__file__'] = self.filename - else: - main_mod = new_main_mod(self.filename, '__main__') - self.ns_globals = main_mod.__dict__ - self.ns_locals = None - # Needed to allow pickle to reference main - if '__main__' in sys.modules: - self._previous_main = sys.modules['__main__'] - sys.modules['__main__'] = main_mod - self._reset_main = True + if self.current_namespace: + self.ns_globals = self.context_globals + self.ns_locals = self.context_locals + if '__file__' in self.ns_globals: + self._previous_filename = self.ns_globals['__file__'] + self.ns_globals['__file__'] = self.filename + else: + main_mod = new_main_mod(self.filename, '__main__') + self.ns_globals = main_mod.__dict__ + self.ns_locals = None + + # Needed to allow pickle to reference main + if '__main__' in sys.modules: + self._previous_main = sys.modules['__main__'] + sys.modules['__main__'] = main_mod + self._reset_main = True # Save current namespace for access by variable explorer - self._previous_running_namespace = ( - ipython_shell.kernel._running_namespace) - ipython_shell.kernel._running_namespace = ( - self.ns_globals, self.ns_locals) + self.shell.add_namespace_manager(self) if (self._file_code is not None - and not PY2 and isinstance(self._file_code, bytes)): try: self._file_code = self._file_code.decode() @@ -106,18 +105,17 @@ def __exit__(self, exc_type, exc_val, exc_tb): """ Reset the namespace. """ - ipython_shell = get_ipython() - ipython_shell.kernel._running_namespace = ( - self._previous_running_namespace) + self.shell.remove_namespace_manager(self) if self._previous_filename: self.ns_globals['__file__'] = self._previous_filename elif '__file__' in self.ns_globals: self.ns_globals.pop('__file__') if not self.current_namespace: - self.context_globals.update(self.ns_globals) - if self.context_locals and self.ns_locals: - self.context_locals.update(self.ns_locals) + if self.context_locals is not None: + self.context_locals.update(self.ns_globals) + else: + self.context_globals.update(self.ns_globals) if self._previous_main: sys.modules['__main__'] = self._previous_main diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py b/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py index b649b2b7453..99c2089890f 100644 --- a/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/spydercustomize.py @@ -11,35 +11,13 @@ # Spyder consoles sitecustomize # -import ast -import bdb -import io import logging import os import pdb -import shlex import sys -import time import warnings -from IPython.core.getipython import get_ipython - -from spyder_kernels.comms.frontendcomm import frontend_request -from spyder_kernels.customize.namespace_manager import NamespaceManager -from spyder_kernels.customize.spyderpdb import SpyderPdb, get_new_debugger -from spyder_kernels.customize.umr import UserModuleReloader -from spyder_kernels.py3compat import ( - PY2, _print, encode, compat_exec, FileNotFoundError) -from spyder_kernels.customize.utils import capture_last_Expr, canonic - -if not PY2: - from IPython.core.inputtransformer2 import ( - TransformerManager, leading_indent, leading_empty_lines) -else: - from IPython.core.inputsplitter import IPythonInputSplitter - - -logger = logging.getLogger(__name__) +from spyder_kernels.customize.spyderpdb import SpyderPdb # ============================================================================= @@ -50,50 +28,11 @@ if not hasattr(sys, 'argv'): sys.argv = [''] - # ============================================================================= # Main constants # ============================================================================= IS_EXT_INTERPRETER = os.environ.get('SPY_EXTERNAL_INTERPRETER') == "True" HIDE_CMD_WINDOWS = os.environ.get('SPY_HIDE_CMD') == "True" -SHOW_INVALID_SYNTAX_MSG = True - - -# ============================================================================= -# Execfile functions -# -# The definitions for Python 2 on Windows were taken from the IPython project -# Copyright (C) The IPython Development Team -# Distributed under the terms of the modified BSD license -# ============================================================================= -try: - # Python 2 - import __builtin__ as builtins - -except ImportError: - # Python 3 - import builtins - basestring = (str,) - - -# ============================================================================= -# Setting console encoding (otherwise Python does not recognize encoding) -# for Windows platforms -# ============================================================================= -if os.name == 'nt' and PY2: - try: - import locale, ctypes - _t, _cp = locale.getdefaultlocale('LANG') - try: - _cp = int(_cp[2:]) - ctypes.windll.kernel32.SetConsoleCP(_cp) - ctypes.windll.kernel32.SetConsoleOutputCP(_cp) - except (ValueError, TypeError): - # Code page number in locale is not valid - pass - except Exception: - pass - # ============================================================================= # Prevent subprocess.Popen calls to create visible console windows on Windows. @@ -110,7 +49,6 @@ def __init__(self, *args, **kwargs): subprocess.Popen = SubprocessPopen - # ============================================================================= # Importing user's sitecustomize # ============================================================================= @@ -119,18 +57,6 @@ def __init__(self, *args, **kwargs): except Exception: pass - -# ============================================================================= -# Add default filesystem encoding on Linux to avoid an error with -# Matplotlib 1.5 in Python 2 (Fixes Issue 2793) -# ============================================================================= -if PY2 and sys.platform.startswith('linux'): - def _getfilesystemencoding_wrapper(): - return 'utf-8' - - sys.getfilesystemencoding = _getfilesystemencoding_wrapper - - # ============================================================================= # Set PyQt API to #2 # ============================================================================= @@ -279,39 +205,37 @@ def spyder_bye(): # ============================================================================= # Multiprocessing adjustments # ============================================================================= -# This patch is only needed on Python 3 -if not PY2: - # This could fail with changes in Python itself, so we protect it - # with a try/except - try: - import multiprocessing.spawn - _old_preparation_data = multiprocessing.spawn.get_preparation_data - - def _patched_preparation_data(name): - """ - Patched get_preparation_data to work when all variables are - removed before execution. - """ - try: - d = _old_preparation_data(name) - except AttributeError: - main_module = sys.modules['__main__'] - # Any string for __spec__ does the job - main_module.__spec__ = '' - d = _old_preparation_data(name) - # On windows, there is no fork, so we need to save the main file - # and import it - if (os.name == 'nt' and 'init_main_from_path' in d - and not os.path.exists(d['init_main_from_path'])): - _print( - "Warning: multiprocessing may need the main file to exist. " - "Please save {}".format(d['init_main_from_path'])) - # Remove path as the subprocess can't do anything with it - del d['init_main_from_path'] - return d - multiprocessing.spawn.get_preparation_data = _patched_preparation_data - except Exception: - pass +# This could fail with changes in Python itself, so we protect it +# with a try/except +try: + import multiprocessing.spawn + _old_preparation_data = multiprocessing.spawn.get_preparation_data + + def _patched_preparation_data(name): + """ + Patched get_preparation_data to work when all variables are + removed before execution. + """ + try: + d = _old_preparation_data(name) + except AttributeError: + main_module = sys.modules['__main__'] + # Any string for __spec__ does the job + main_module.__spec__ = '' + d = _old_preparation_data(name) + # On windows, there is no fork, so we need to save the main file + # and import it + if (os.name == 'nt' and 'init_main_from_path' in d + and not os.path.exists(d['init_main_from_path'])): + print( + "Warning: multiprocessing may need the main file to exist. " + "Please save {}".format(d['init_main_from_path'])) + # Remove path as the subprocess can't do anything with it + del d['init_main_from_path'] + return d + multiprocessing.spawn.get_preparation_data = _patched_preparation_data +except Exception: + pass # ============================================================================= @@ -329,480 +253,6 @@ def _patched_get_terminal_size(fd=None): # ============================================================================= pdb.Pdb = SpyderPdb -# ============================================================================= -# User module reloader -# ============================================================================= -__umr__ = UserModuleReloader(namelist=os.environ.get("SPY_UMR_NAMELIST", None)) - - -# ============================================================================= -# Handle Post Mortem Debugging and Traceback Linkage to Spyder -# ============================================================================= -def post_mortem_excepthook(type, value, tb): - """ - For post mortem exception handling, print a banner and enable post - mortem debugging. - """ - ipython_shell = get_ipython() - ipython_shell.showtraceback((type, value, tb)) - p = pdb.Pdb(ipython_shell.colors) - - if not type == SyntaxError: - # wait for stderr to print (stderr.flush does not work in this case) - time.sleep(0.1) - _print('*' * 40) - _print('Entering post mortem debugging...') - _print('*' * 40) - - # Inform Spyder about position of exception: pdb.Pdb.interaction() calls - # cmd.Cmd.cmdloop(), which calls SpyderPdb.preloop() where - # send_initial_notification is handled. - p.send_initial_notification = True - - p.reset() - frame = tb.tb_next.tb_frame - # wait for stdout to print - time.sleep(0.1) - p.interaction(frame, tb) - - -# ============================================================================== -# runfile and debugfile commands -# ============================================================================== -def get_current_file_name(): - """Get the current file name.""" - try: - return frontend_request(blocking=True).current_filename() - except Exception: - _print("This command failed to be executed because an error occurred" - " while trying to get the current file name from Spyder's" - " editor. The error was:\n\n") - get_ipython().showtraceback(exception_only=True) - return None - - -def count_leading_empty_lines(cell): - """Count the number of leading empty cells.""" - if PY2: - lines = cell.splitlines(True) - else: - lines = cell.splitlines(keepends=True) - if not lines: - return 0 - for i, line in enumerate(lines): - if line and not line.isspace(): - return i - return len(lines) - - -def transform_cell(code, indent_only=False): - """Transform IPython code to Python code.""" - number_empty_lines = count_leading_empty_lines(code) - if indent_only: - # Not implemented for PY2 - if PY2: - return code - if not code.endswith('\n'): - code += '\n' # Ensure the cell has a trailing newline - lines = code.splitlines(keepends=True) - lines = leading_indent(leading_empty_lines(lines)) - code = ''.join(lines) - else: - if PY2: - tm = IPythonInputSplitter() - return tm.transform_cell(code) - else: - tm = TransformerManager() - code = tm.transform_cell(code) - return '\n' * number_empty_lines + code - - -def exec_code(code, filename, ns_globals, ns_locals=None, post_mortem=False, - exec_fun=None, capture_last_expression=False): - """Execute code and display any exception.""" - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - global SHOW_INVALID_SYNTAX_MSG - - if PY2: - filename = encode(filename) - code = encode(code) - - if exec_fun is None: - # Replace by exec when dropping Python 2 - exec_fun = compat_exec - - ipython_shell = get_ipython() - is_ipython = os.path.splitext(filename)[1] == '.ipy' - try: - if not is_ipython: - # TODO: remove the try-except and let the SyntaxError raise - # Because there should not be ipython code in a python file - try: - ast_code = ast.parse(transform_cell(code, indent_only=True)) - except SyntaxError as e: - try: - ast_code = ast.parse(transform_cell(code)) - except SyntaxError: - if PY2: - raise e - else: - # Need to call exec to avoid Syntax Error in Python 2. - # TODO: remove exec when dropping Python 2 support. - exec("raise e from None") - else: - if SHOW_INVALID_SYNTAX_MSG: - _print( - "\nWARNING: This is not valid Python code. " - "If you want to use IPython magics, " - "flexible indentation, and prompt removal, " - "we recommend that you save this file with the " - ".ipy extension.\n") - SHOW_INVALID_SYNTAX_MSG = False - else: - ast_code = ast.parse(transform_cell(code)) - - if code.rstrip()[-1:] == ";": - # Supress output with ; - capture_last_expression = False - - if capture_last_expression: - ast_code, capture_last_expression = capture_last_Expr( - ast_code, "_spyder_out") - - exec_fun(compile(ast_code, filename, 'exec'), ns_globals, ns_locals) - - if capture_last_expression: - out = ns_globals.pop("_spyder_out", None) - if out is not None: - return out - - except SystemExit as status: - # ignore exit(0) - if status.code: - ipython_shell.showtraceback(exception_only=True) - except BaseException as error: - if (isinstance(error, bdb.BdbQuit) - and ipython_shell.pdb_session): - # Ignore BdbQuit if we are debugging, as it is expected. - ipython_shell.pdb_session = None - elif post_mortem and isinstance(error, Exception): - error_type, error, tb = sys.exc_info() - post_mortem_excepthook(error_type, error, tb) - else: - # We ignore the call to exec - ipython_shell.showtraceback(tb_offset=1) - finally: - __tracebackhide__ = "__pdb_exit__" - - -def get_file_code(filename, save_all=True, raise_exception=False): - """Retrieve the content of a file.""" - # Get code from spyder - try: - return frontend_request(blocking=True).get_file_code( - filename, save_all=save_all) - except Exception: - # Maybe this is a local file - try: - with open(filename, 'r') as f: - return f.read() - except FileNotFoundError: - pass - if raise_exception: - raise - # Else return None - return None - - -def runfile(filename=None, args=None, wdir=None, namespace=None, - post_mortem=False, current_namespace=False): - """ - Run filename - args: command line arguments (string) - wdir: working directory - namespace: namespace for execution - post_mortem: boolean, whether to enter post-mortem mode on error - current_namespace: if true, run the file in the current namespace - """ - return _exec_file( - filename, args, wdir, namespace, - post_mortem, current_namespace, stack_depth=1) - - -def _exec_file(filename=None, args=None, wdir=None, namespace=None, - post_mortem=False, current_namespace=False, stack_depth=0, - exec_fun=None, canonic_filename=None): - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - ipython_shell = get_ipython() - if filename is None: - filename = get_current_file_name() - if filename is None: - return - - try: - filename = filename.decode('utf-8') - except (UnicodeError, TypeError, AttributeError): - # UnicodeError, TypeError --> eventually raised in Python 2 - # AttributeError --> systematically raised in Python 3 - pass - if PY2: - filename = encode(filename) - if __umr__.enabled: - __umr__.run() - if args is not None and not isinstance(args, basestring): - raise TypeError("expected a character buffer object") - - try: - file_code = get_file_code(filename, raise_exception=True) - except Exception: - # Show an error and return None - _print( - "This command failed to be executed because an error occurred" - " while trying to get the file code from Spyder's" - " editor. The error was:\n\n") - get_ipython().showtraceback(exception_only=True) - return - - # Here the remote filename has been used. It must now be valid locally. - if canonic_filename is not None: - filename = canonic_filename - else: - filename = canonic(filename) - - with NamespaceManager(filename, namespace, current_namespace, - file_code=file_code, stack_depth=stack_depth + 1 - ) as (ns_globals, ns_locals): - sys.argv = [filename] - if args is not None: - for arg in shlex.split(args): - sys.argv.append(arg) - - if "multiprocessing" in sys.modules: - # See https://github.com/spyder-ide/spyder/issues/16696 - try: - sys.modules['__mp_main__'] = sys.modules['__main__'] - except Exception: - pass - - if wdir is not None: - if PY2: - try: - wdir = wdir.decode('utf-8') - except (UnicodeError, TypeError): - # UnicodeError, TypeError --> eventually raised in Python 2 - pass - if os.path.isdir(wdir): - os.chdir(wdir) - # See https://github.com/spyder-ide/spyder/issues/13632 - if "multiprocessing.process" in sys.modules: - try: - import multiprocessing.process - multiprocessing.process.ORIGINAL_DIR = os.path.abspath( - wdir) - except Exception: - pass - else: - _print("Working directory {} doesn't exist.\n".format(wdir)) - - try: - if __umr__.has_cython: - # Cython files - with io.open(filename, encoding='utf-8') as f: - ipython_shell.run_cell_magic('cython', '', f.read()) - else: - exec_code(file_code, filename, ns_globals, ns_locals, - post_mortem=post_mortem, exec_fun=exec_fun, - capture_last_expression=False) - finally: - sys.argv = [''] - - -# IPykernel 6.3.0+ shadows our runfile because it depends on the Pydev -# debugger, which adds its own runfile to builtins. So we replace it with -# our own using exec_lines in start.py -if PY2: - builtins.runfile = runfile -else: - builtins.spyder_runfile = runfile - - -def debugfile(filename=None, args=None, wdir=None, post_mortem=False, - current_namespace=False): - """ - Debug filename - args: command line arguments (string) - wdir: working directory - post_mortem: boolean, included for compatiblity with runfile - """ - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - if filename is None: - filename = get_current_file_name() - if filename is None: - return - - shell = get_ipython() - if shell.is_debugging(): - # Recursive - code = ( - "runfile({}".format(repr(filename)) + - ", args=%r, wdir=%r, current_namespace=%r)" % ( - args, wdir, current_namespace) - ) - - shell.pdb_session.enter_recursive_debugger( - code, filename, True, - ) - else: - debugger = get_new_debugger(filename, True) - _exec_file( - filename=filename, - canonic_filename=debugger.canonic(filename), - args=args, - wdir=wdir, - current_namespace=current_namespace, - exec_fun=debugger.run, - stack_depth=1, - ) - - -builtins.debugfile = debugfile - - -def runcell(cellname, filename=None, post_mortem=False): - """ - Run a code cell from an editor as a file. - - Parameters - ---------- - cellname : str or int - Cell name or index. - filename : str - Needed to allow for proper traceback links. - post_mortem: bool - Automatically enter post mortem on exception. - """ - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - return _exec_cell(cellname, filename, post_mortem, stack_depth=1) - - -def _exec_cell(cellname, filename=None, post_mortem=False, stack_depth=0, - exec_fun=None, canonic_filename=None): - """ - Execute a code cell with a given exec function. - """ - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - if filename is None: - filename = get_current_file_name() - if filename is None: - return - try: - filename = filename.decode('utf-8') - except (UnicodeError, TypeError, AttributeError): - # UnicodeError, TypeError --> eventually raised in Python 2 - # AttributeError --> systematically raised in Python 3 - pass - ipython_shell = get_ipython() - try: - # Get code from spyder - cell_code = frontend_request( - blocking=True).run_cell(cellname, filename) - except Exception: - _print("This command failed to be executed because an error occurred" - " while trying to get the cell code from Spyder's" - " editor. The error was:\n\n") - get_ipython().showtraceback(exception_only=True) - return - - if not cell_code or cell_code.strip() == '': - _print("Nothing to execute, this cell is empty.\n") - return - - # Trigger `post_execute` to exit the additional pre-execution. - # See Spyder PR #7310. - ipython_shell.events.trigger('post_execute') - file_code = get_file_code(filename, save_all=False) - - # Here the remote filename has been used. It must now be valid locally. - if canonic_filename is not None: - filename = canonic_filename - else: - # Normalise the filename - filename = canonic(filename) - - with NamespaceManager(filename, current_namespace=True, - file_code=file_code, stack_depth=stack_depth + 1 - ) as (ns_globals, ns_locals): - return exec_code(cell_code, filename, ns_globals, ns_locals, - post_mortem=post_mortem, exec_fun=exec_fun, - capture_last_expression=True) - - -builtins.runcell = runcell - - -def debugcell(cellname, filename=None, post_mortem=False): - """Debug a cell.""" - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - if filename is None: - filename = get_current_file_name() - if filename is None: - return - - shell = get_ipython() - if shell.is_debugging(): - # Recursive - code = ( - "runcell({}, ".format(repr(cellname)) + - "{})".format(repr(filename)) - ) - shell.pdb_session.enter_recursive_debugger( - code, filename, False, - ) - else: - debugger = get_new_debugger(filename, False) - _exec_cell( - cellname=cellname, - filename=filename, - canonic_filename=debugger.canonic(filename), - exec_fun=debugger.run, - stack_depth=1 - ) - - -builtins.debugcell = debugcell - - -def cell_count(filename=None): - """ - Get the number of cells in a file. - - Parameters - ---------- - filename : str - The file to get the cells from. If None, the currently opened file. - """ - if filename is None: - filename = get_current_file_name() - if filename is None: - raise RuntimeError('Could not get cell count from frontend.') - try: - # Get code from spyder - cell_count = frontend_request(blocking=True).cell_count(filename) - return cell_count - except Exception: - etype, error, tb = sys.exc_info() - raise etype(error) - - -builtins.cell_count = cell_count - - # ============================================================================= # PYTHONPATH and sys.path Adjustments # ============================================================================= diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py b/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py index 7f687e5c2bb..944bd1e5950 100755 --- a/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/spyderpdb.py @@ -9,63 +9,58 @@ import ast import bdb +import builtins +from contextlib import contextmanager import logging import os import sys import traceback import threading from collections import namedtuple +from functools import lru_cache from IPython.core.autocall import ZMQExitAutocall from IPython.core.debugger import Pdb as ipyPdb -from IPython.core.getipython import get_ipython +from IPython.core.inputtransformer2 import TransformerManager +import spyder_kernels from spyder_kernels.comms.frontendcomm import CommError, frontend_request -from spyder_kernels.customize.utils import path_is_library, capture_last_Expr -from spyder_kernels.py3compat import ( - TimeoutError, PY2, _print, isidentifier, PY3, input) - -if not PY2: - from IPython.core.inputtransformer2 import TransformerManager - import builtins - basestring = (str,) -else: - import __builtin__ as builtins - from IPython.core.inputsplitter import IPythonInputSplitter as TransformerManager +from spyder_kernels.customize.utils import ( + path_is_library, capture_last_Expr, exec_encapsulate_locals +) logger = logging.getLogger(__name__) -class DebugWrapper(object): +class DebugWrapper: """ Notifies the frontend when debugging starts/stops """ def __init__(self, pdb_obj): self.pdb_obj = pdb_obj + self._cleanup = True def __enter__(self): """ Debugging starts. """ - self.pdb_obj._frontend_notified = True - try: - frontend_request(blocking=True).set_debug_state(True) - except (CommError, TimeoutError): - logger.debug("Could not send debugging state to the frontend.") + shell = self.pdb_obj.shell + if shell.pdb_session == self.pdb_obj: + self._cleanup = False + else: + shell.add_pdb_session(self.pdb_obj) + self._cleanup = True def __exit__(self, exc_type, exc_val, exc_tb): """ Debugging ends. """ - self.pdb_obj._frontend_notified = False - try: - frontend_request(blocking=True).set_debug_state(False) - except (CommError, TimeoutError): - logger.debug("Could not send debugging state to the frontend.") + if self._cleanup: + self.pdb_obj.shell.remove_pdb_session(self.pdb_obj) -class SpyderPdb(ipyPdb, object): # Inherits `object` to call super() in PY2 +class SpyderPdb(ipyPdb): """ Extends Pdb to add features: @@ -76,9 +71,6 @@ class SpyderPdb(ipyPdb, object): # Inherits `object` to call super() in PY2 - Add completion to non-command code. """ - send_initial_notification = True - starting = True - def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False): """Init Pdb.""" @@ -88,12 +80,11 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, self.pdb_ignore_lib = False self.pdb_execute_events = False self.pdb_use_exclamation_mark = False + self.pdb_publish_stack = False self._exclamation_warning_printed = False self.pdb_stop_first_line = True self._disable_next_stack_entry = False super(SpyderPdb, self).__init__() - self._pdb_breaking = False - self._frontend_notified = False # content of tuple: (filename, line number) self._previous_step = None @@ -102,23 +93,30 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, # has no effect in previous versions. self.report_skipped = False - # Keep track of remote filename self.remote_filename = None + # Needed to know which namespace to show (user or current frame) # Line received from the frontend self._cmd_input_line = None - # This is not available in IPython 5 - if hasattr(self, '_predicates'): - # Turn off IPython's debugger skip funcionality by default because - # it makes our debugger quite slow. It's also important to remark - # that this functionality doesn't do anything on its own. Users - # need to mark what frames they want to skip for it to be useful. - # So, we hope that knowledgeable users will find that they need to - # enable it in Spyder. - # Fixes spyder-ide/spyder#20639. - self._predicates["debuggerskip"] = False + # Disable sigint so we can do it ourselves + self.nosigint = True + + # Keep track of interrupting state to avoid several interruptions + self.interrupting = False + + # Should the frontend force go to the current line? + self._request_where = False + + # Turn off IPython's debugger skip funcionality by default because + # it makes our debugger quite slow. It's also important to remark + # that this functionality doesn't do anything on its own. Users + # need to mark what frames they want to skip for it to be useful. + # So, we hope that knowledgeable users will find that they need to + # enable it in Spyder. + # Fixes spyder-ide/spyder#20639. + self._predicates["debuggerskip"] = False # --- Methods overriden for code execution def print_exclamation_warning(self): @@ -139,28 +137,29 @@ def default(self, line): self.print_exclamation_warning() self.error("Unknown command '" + line.split()[0] + "'") return - # Disallow the use of %debug magic in the debugger - if line.startswith("%debug"): - self.error("Please don't use '%debug' in the debugger.\n" - "For a recursive debugger, use the pdb 'debug'" - " command instead") - return - locals = self.curframe_locals - globals = self.curframe.f_globals + + # Replace %debug magic in the debugger + if line.startswith("%debug") or line.startswith("%%debug"): + cmd, arg, _ = self.parseline(line.lstrip("%")) + if cmd == "debug": + return self.do_debug(arg) + + local_ns = self.curframe_locals + global_ns = self.curframe.f_globals if self.pdb_use_exclamation_mark: # Find pdb commands executed without ! cmd, arg, line = self.parseline(line) if cmd: cmd_in_namespace = ( - cmd in globals - or cmd in locals + cmd in global_ns + or cmd in local_ns or cmd in builtins.__dict__ ) # Special case for quit and exit if cmd in ("quit", "exit"): - if cmd in globals and isinstance( - globals[cmd], ZMQExitAutocall): + if cmd in global_ns and isinstance( + global_ns[cmd], ZMQExitAutocall): # Use the pdb call cmd_in_namespace = False cmd_func = getattr(self, 'do_' + cmd, None) @@ -184,6 +183,7 @@ def default(self, line): # The pdb command is masked by something self.print_exclamation_warning() try: + is_magic = line.startswith("%") line = TransformerManager().transform_cell(line) save_stdout = sys.stdout save_stdin = sys.stdin @@ -193,7 +193,7 @@ def default(self, line): sys.stdout = self.stdout sys.displayhook = self.displayhook if execute_events: - get_ipython().events.trigger('pre_execute') + self.shell.events.trigger('pre_execute') code_ast = ast.parse(line) @@ -202,73 +202,19 @@ def default(self, line): capture_last_expression = False else: code_ast, capture_last_expression = capture_last_Expr( - code_ast, "_spyderpdb_out") - - if locals is not globals: - # Mitigates a behaviour of CPython that makes it difficult - # to work with exec and the local namespace - # See: - # - https://bugs.python.org/issue41918 - # - https://bugs.python.org/issue46153 - # - https://bugs.python.org/issue21161 - # - spyder-ide/spyder#13909 - # - spyder-ide/spyder-kernels#345 - # - # The idea here is that the best way to emulate being in a - # function is to actually execute the code in a function. - # A function called `_spyderpdb_code` is created and - # called. It will first load the locals, execute the code, - # and then update the locals. - # - # One limitation of this approach is that locals() is only - # a copy of the curframe locals. This means that closures - # for example are early binding instead of late binding. - - # Create a function - indent = " " - code = ["def _spyderpdb_code():"] - - # Load the locals - globals["_spyderpdb_builtins_locals"] = builtins.locals - - # Save builtins locals in case it is shadowed - globals["_spyderpdb_locals"] = locals - - # Load locals if they have a valid name - # In comprehensions, locals could contain ".0" for example - code += [indent + "{k} = _spyderpdb_locals['{k}']".format( - k=k) for k in locals if isidentifier(k)] - - - # Update the locals - code += [indent + "_spyderpdb_locals.update(" - "_spyderpdb_builtins_locals())"] - - # Run the function - code += ["_spyderpdb_code()"] - - # Cleanup - code += [ - "del _spyderpdb_code", - "del _spyderpdb_locals", - "del _spyderpdb_builtins_locals" - ] - - # Parse the function - fun_ast = ast.parse('\n'.join(code) + '\n') - - # Inject code_ast in the function before the locals update - fun_ast.body[0].body = ( - fun_ast.body[0].body[:-1] # The locals - + code_ast.body # Code to run - + fun_ast.body[0].body[-1:] # Locals update - ) - code_ast = fun_ast - - exec(compile(code_ast, "", "exec"), globals) + code_ast, "_spyderpdb_out", global_ns) + + if is_magic: + # Magics like runcell use and modify local_ns. + # But the locals() dict can not be directly modified when + # encapsulated. Therefore they must encapsulate the locals + # themselves (see code_runner.py). + exec(compile(code_ast, "", "exec"), global_ns, local_ns) + else: + exec_encapsulate_locals(code_ast, global_ns, local_ns) if capture_last_expression: - out = globals.pop("_spyderpdb_out", None) + out = global_ns.pop("_spyderpdb_out", None) if out is not None: sys.stdout.flush() sys.stderr.flush() @@ -281,101 +227,109 @@ def default(self, line): finally: if execute_events: - get_ipython().events.trigger('post_execute') + self.shell.events.trigger('post_execute') sys.stdout = save_stdout sys.stdin = save_stdin sys.displayhook = save_displayhook except BaseException: - if PY2: - t, v = sys.exc_info()[:2] - if type(t) == type(''): - exc_type_name = t - else: exc_type_name = t.__name__ - print >>self.stdout, '***', exc_type_name + ':', v - else: - exc_info = sys.exc_info()[:2] - self.error( - traceback.format_exception_only(*exc_info)[-1].strip()) + exc_info = sys.exc_info()[:2] + self.error( + traceback.format_exception_only(*exc_info)[-1].strip()) # --- Methods overriden for signal handling - def sigint_handler(self, signum, frame): - """ - Handle a sigint signal. Break on the frame above this one. - - This method is not present in python2 so this won't be called there. - """ - if self.allow_kbdint: - raise KeyboardInterrupt + def interrupt(self): + """Stop debugger on next instruction.""" + self.interrupting = True self.message("\nProgram interrupted. (Use 'cont' to resume).") - # avoid stopping in set_trace - sys.settrace(None) - self._pdb_breaking = True self.set_step() - self.set_trace(sys._getframe()) + + def set_trace(self, frame=None): + """Register that debugger is tracing.""" + self.shell.add_pdb_session(self) + super(SpyderPdb, self).set_trace(frame) + + def set_quit(self): + """Register that debugger is not tracing.""" + self.shell.remove_pdb_session(self) + super(SpyderPdb, self).set_quit() def interaction(self, frame, traceback): """ Called when a user interaction is required. - - If this is from sigint, break on the upper frame. - If the frame is in spydercustomize.py, quit. - Notifies spyder and print current code. """ - if self._pdb_breaking: - self._pdb_breaking = False - if frame and frame.f_back: - return self.interaction(frame.f_back, traceback) - - # This is necessary to handle chained exceptions in Pdb, support for - # which was added in IPython 8.15 and will be the default in Python - # 3.13 (see ipython/ipython#14146). - if isinstance(traceback, BaseException): - _chained_exceptions, tb = self._get_tb_and_exceptions(traceback) - - with self._hold_exceptions(_chained_exceptions): - self.interaction(frame, tb) - - return - - self.setup(frame, traceback) - self.print_stack_entry(self.stack[self.curindex]) - - if self._frontend_notified: - self._cmdloop() - else: - with DebugWrapper(self): - self._cmdloop() - - self.forget() + with DebugWrapper(self): + # Wrapp in case the frontend was not notified, e.g. postmortem + return super(SpyderPdb, self).interaction( + frame, traceback) - def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', - context=None): + def print_stack_entry(self, *args, **kwargs): """Disable printing stack entry if requested.""" if self._disable_next_stack_entry: self._disable_next_stack_entry = False return - return super(SpyderPdb, self).print_stack_entry( - frame_lineno, prompt_prefix, context) + return super().print_stack_entry(*args, **kwargs) # --- Methods overriden for skipping libraries def stop_here(self, frame): """Check if pdb should stop here.""" - if (frame is not None - and "__tracebackhide__" in frame.f_locals - and frame.f_locals["__tracebackhide__"] == "__pdb_exit__"): + # Never stop if we are continuing unless there is a breakpoint + if self.stopframe == self.botframe and self.stoplineno == -1: + return False + if self.continue_if_has_breakpoints and self.should_continue(frame): + self.set_continue() + return False + if ( + frame is not None + and "__tracebackhide__" in frame.f_locals + and frame.f_locals["__tracebackhide__"] == "__pdb_exit__" + ): self.onecmd('exit') return False - if not super(SpyderPdb, self).stop_here(frame): + if not super().stop_here(frame): return False + if frame is self.stopframe: + return True filename = frame.f_code.co_filename if filename.startswith('<'): # This is not a file return True if self.pdb_ignore_lib and path_is_library(filename): return False + if self.skip_hidden and os.path.dirname(spyder_kernels.__file__) in filename: + # This is spyder-kernels internals + return False return True + def should_continue(self, frame): + """ + Jump to first breakpoint if needed. + + Fixes spyder-ide/spyder#2034 + """ + + if not self.continue_if_has_breakpoints: + # This was disabled + return False + self.continue_if_has_breakpoints = False + + # Get all breakpoints for the file we're going to debug + if not frame: + # We are not debugging, return. Solves spyder-ide/spyder#10290 + return False + + lineno = frame.f_lineno + breaks = self.get_file_breaks(frame.f_code.co_filename) + + # Do 'continue' if the first breakpoint is *not* placed + # where the debugger is going to land. + # Fixes spyder-ide/spyder#4681 + if self.pdb_stop_first_line: + return breaks and lineno < breaks[0] + + # The breakpoint could be in another file. + return not (breaks and lineno >= breaks[0]) + def do_where(self, arg): """w(here) Print a stack trace, with the most recent frame at the bottom. @@ -384,11 +338,8 @@ def do_where(self, arg): Take a number as argument as an (optional) number of context line to print""" - super(SpyderPdb, self).do_where(arg) - try: - frontend_request(blocking=False).do_where() - except (CommError, TimeoutError): - logger.debug("Could not send where request to the frontend.") + self._request_where = True + return super(SpyderPdb, self).do_where(arg) do_w = do_where @@ -444,7 +395,7 @@ def is_name_or_composed(text): if not text or text[0] == '.': return False # We want to keep value.subvalue - return isidentifier(text.replace('.', '')) + return text.replace('.', '').isidentifier() while text and not is_name_or_composed(text): text = text[1:] @@ -457,7 +408,6 @@ def is_name_or_composed(text): cursor_start = cursor_pos - len(text) if ipython_do_complete: - kernel = get_ipython().kernel # Make complete call with current frame if self.curframe: if self.curframe_locals: @@ -466,10 +416,10 @@ def is_name_or_composed(text): self.curframe.f_globals) else: frame = self.curframe - kernel.shell.set_completer_frame(frame) - result = kernel._do_complete(code, cursor_pos) + self.shell.set_completer_frame(frame) + result = self.shell.kernel._do_complete(code, cursor_pos) # Reset frame - kernel.shell.set_completer_frame() + self.shell.set_completer_frame() # If there is no Pdb results to merge, return the result if not compfunc: return result @@ -547,7 +497,7 @@ def is_name_or_composed(text): if not text or text[0] == '.': return False # We want to keep value.subvalue - return isidentifier(text.replace('.', '')) + return text.replace('.', '').isidentifier() while text and not is_name_or_composed(text): text = text[1:] @@ -565,7 +515,6 @@ def is_name_or_composed(text): 'status': 'ok' } - kernel = get_ipython().kernel # Make complete call with current frame if self.curframe: if self.curframe_locals: @@ -574,36 +523,19 @@ def is_name_or_composed(text): self.curframe.f_globals) else: frame = self.curframe - kernel.shell.set_completer_frame(frame) - result = kernel._do_complete(code, cursor_pos) + self.shell.set_completer_frame(frame) + result = self.shell.kernel._do_complete(code, cursor_pos) # Reset frame - kernel.shell.set_completer_frame() + self.shell.set_completer_frame() return result # --- Methods overriden by us for Spyder integration def postloop(self): # postloop() is called when the debugger’s input prompt exists. Reset - # _previous_step so that publish_pdb_state() actually notifies Spyder + # _previous_step so that get_pdb_state() actually notifies Spyder # about a changed frame the next the input prompt is entered again. self._previous_step = None - def preloop(self): - """Ask Spyder for breakpoints before the first prompt is created.""" - try: - pdb_settings = frontend_request(blocking=True).get_pdb_settings() - self.pdb_ignore_lib = pdb_settings['pdb_ignore_lib'] - self.pdb_execute_events = pdb_settings['pdb_execute_events'] - self.pdb_use_exclamation_mark = pdb_settings[ - 'pdb_use_exclamation_mark'] - self.pdb_stop_first_line = pdb_settings['pdb_stop_first_line'] - if self.starting: - self.set_spyder_breakpoints(pdb_settings['breakpoints']) - if self.send_initial_notification: - self.publish_pdb_state() - except (CommError, TimeoutError): - logger.debug("Could not get breakpoints from the frontend.") - super(SpyderPdb, self).preloop() - def set_continue(self): """ Stop only at breakpoints or when finished. @@ -614,13 +546,6 @@ def set_continue(self): # Don't stop except at breakpoints or when finished self._set_stopinfo(self.botframe, None, -1) - def reset(self): - """ - Register Pdb session after reset. - """ - super(SpyderPdb, self).reset() - get_ipython().pdb_session = self - def do_debug(self, arg): """ Debug code @@ -629,20 +554,43 @@ def do_debug(self, arg): argument (which is an arbitrary expression or statement to be executed in the current environment). """ - try: - super(SpyderPdb, self).do_debug(arg) - except Exception: - if PY2: - t, v = sys.exc_info()[:2] - if type(t) == type(''): - exc_type_name = t - else: exc_type_name = t.__name__ - print >>self.stdout, '***', exc_type_name + ':', v - else: + with self.recursive_debugger() as debugger: + self.message("Entering recursive debugger") + try: + global_ns = self.curframe.f_globals + local_ns = self.curframe_locals + return sys.call_tracing(debugger.run, (arg, global_ns, local_ns)) + except Exception: exc_info = sys.exc_info()[:2] self.error( traceback.format_exception_only(*exc_info)[-1].strip()) - get_ipython().pdb_session = self + finally: + self.message("Leaving recursive debugger") + + @contextmanager + def recursive_debugger(self): + """Get a recursive debugger.""" + # Save and restore tracing function + trace_function = sys.gettrace() + sys.settrace(None) + + # Create child debugger + debugger = self.__class__( + completekey=self.completekey, + stdin=self.stdin, stdout=self.stdout) + debugger.prompt = "(%s) " % self.prompt.strip() + try: + yield debugger + finally: + # Reset parent debugger + sys.settrace(trace_function) + self.lastcmd = debugger.lastcmd + + # Reset _previous_step so that get_pdb_state() notifies Spyder about + # a changed debugger position. The reset is required because the + # recursive debugger might change the position, but the parent + # debugger (self) is not aware of this. + self._previous_step = None def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" @@ -657,6 +605,7 @@ def user_return(self, frame, return_value): def _cmdloop(self): """Modifies the error text.""" + self.interrupting = False while True: try: # keyboard interrupts allow for an easy way to cancel @@ -666,10 +615,24 @@ def _cmdloop(self): self.allow_kbdint = False break except KeyboardInterrupt: - _print("--KeyboardInterrupt--\n" - "For copying text while debugging, use Ctrl+Shift+C", - file=self.stdout) + print("--KeyboardInterrupt--\n" + "For copying text while debugging, use Ctrl+Shift+C", + file=self.stdout) + @lru_cache + def canonic(self, filename): + """Return canonical form of filename.""" + return super().canonic(filename) + + def do_exitdb(self, arg): + """Exit the debugger""" + self._set_stopinfo(self.botframe, None, -1) + sys.settrace(None) + frame = sys._getframe().f_back + while frame and frame is not self.botframe: + del frame.f_trace + frame = frame.f_back + return 1 def cmdloop(self, intro=None): """ @@ -700,7 +663,7 @@ def cmd_input(self, prompt=''): """ Get input from frontend. Blocks until return """ - kernel = get_ipython().kernel + kernel = self.shell.kernel # Only works if the comm is open if not kernel.frontend_comm.is_open(): return input(prompt) @@ -713,15 +676,12 @@ def cmd_input(self, prompt=''): # Send the input request. self._cmd_input_line = None - kernel.frontend_call().pdb_input(prompt) + kernel.frontend_call(display_error=True).pdb_input( + prompt, state=self.get_pdb_state()) # Allow GUI event loop to update - if PY3: - is_main_thread = ( - threading.current_thread() is threading.main_thread()) - else: - is_main_thread = isinstance( - threading.current_thread(), threading._MainThread) + is_main_thread = ( + threading.current_thread() is threading.main_thread()) # Get input by running eventloop if is_main_thread and kernel.eventloop: @@ -767,41 +727,7 @@ def postcmd(self, stop, line): # Flush in case the command produced output on underlying outputs sys.__stderr__.flush() sys.__stdout__.flush() - self.publish_pdb_state() - return super(SpyderPdb, self).postcmd(stop, line) - - if PY2: - def break_here(self, frame): - """ - Breakpoints don't work for files with non-ascii chars in Python 2 - - Fixes Issue 1484 - """ - from bdb import effective - filename = self.canonic(frame.f_code.co_filename) - try: - filename = unicode(filename, "utf-8") - except TypeError: - pass - if filename not in self.breaks: - return False - lineno = frame.f_lineno - if lineno not in self.breaks[filename]: - # The line itself has no breakpoint, but maybe the line is the - # first line of a function with breakpoint set by function name - lineno = frame.f_code.co_firstlineno - if lineno not in self.breaks[filename]: - return False - - # flag says ok to delete temp. bp - (bp, flag) = effective(filename, lineno, frame) - if bp: - self.currentbp = bp.number - if (flag and bp.temporary): - self.do_clear(str(bp.number)) - return True - else: - return False + return stop # --- Methods defined by us for Spyder integration def set_spyder_breakpoints(self, breakpoints): @@ -825,89 +751,67 @@ def set_spyder_breakpoints(self, breakpoints): # The file is not readable pass - # Jump to first breakpoint. - # Fixes issue 2034 - if self.starting: - # Only run this after a Pdb session is created - self.starting = False - - # Get all breakpoints for the file we're going to debug - frame = self.curframe - if not frame: - # We are not debugging, return. Solves #10290 - return - lineno = frame.f_lineno - breaks = self.get_file_breaks(frame.f_code.co_filename) - - # Do 'continue' if the first breakpoint is *not* placed - # where the debugger is going to land. - # Fixes issue 4681 - if self.pdb_stop_first_line: - do_continue = ( - self.continue_if_has_breakpoints - and breaks - and lineno < breaks[0]) - else: - # The breakpoint could be in another file. - do_continue = ( - self.continue_if_has_breakpoints - and not (breaks and lineno >= breaks[0])) + breakpoints = property(fset=set_spyder_breakpoints) - if do_continue: - try: - if self.pdb_use_exclamation_mark: - cont_cmd = '!continue' - else: - cont_cmd = 'continue' - frontend_request(blocking=False).pdb_execute(cont_cmd) - except (CommError, TimeoutError): - logger.debug( - "Could not send a Pdb continue call to the frontend.") - - def publish_pdb_state(self): + def get_pdb_state(self): """ Send debugger state (frame position) to the frontend. The state is only sent if it has changed since the last update. """ + state = self.shell.kernel.get_state() frame = self.curframe if frame is None: self._previous_step = None - return + return state + + if self._request_where: + self._request_where = False + state["do_where"] = True # Get filename and line number of the current frame fname = self.canonic(frame.f_code.co_filename) - if PY2: - try: - fname = unicode(fname, "utf-8") - except TypeError: - pass if fname == self.mainpyfile and self.remote_filename is not None: fname = self.remote_filename lineno = frame.f_lineno if self._previous_step == (fname, lineno): - return + # Do not update state if not needed + return state # Set step of the current frame (if any) step = {} self._previous_step = None - if isinstance(fname, basestring) and isinstance(lineno, int): + if isinstance(fname, str) and isinstance(lineno, int): step = dict(fname=fname, lineno=lineno) self._previous_step = (fname, lineno) - try: - frontend_request(blocking=False).pdb_state(dict(step=step)) - except (CommError, TimeoutError): - logger.debug("Could not send Pdb state to the frontend.") + state['step'] = step + + if self.pdb_publish_stack: + # Publish Pdb stack so we can update the Debugger plugin on Spyder + pdb_stack = traceback.StackSummary.extract(self.stack) + pdb_index = self.curindex + + skip_hidden = getattr(self, 'skip_hidden', False) + + if skip_hidden: + # Filter out the hidden frames + hidden = self.hidden_frames(self.stack) + pdb_stack = [f for f, h in zip(pdb_stack, hidden) if not h] + # Adjust the index + pdb_index -= sum([bool(i) for i in hidden[:pdb_index]]) + + state['stack'] = (pdb_stack, pdb_index) + + return state def run(self, cmd, globals=None, locals=None): """Debug a statement executed via the exec() function. globals defaults to __main__.dict; locals defaults to globals. """ - self.starting = True with DebugWrapper(self): super(SpyderPdb, self).run(cmd, globals, locals) @@ -916,7 +820,6 @@ def runeval(self, expr, globals=None, locals=None): globals defaults to __main__.dict; locals defaults to globals. """ - self.starting = True with DebugWrapper(self): super(SpyderPdb, self).runeval(expr, globals, locals) @@ -925,51 +828,11 @@ def runcall(self, *args, **kwds): Return the result of the function call. """ - self.starting = True with DebugWrapper(self): super(SpyderPdb, self).runcall(*args, **kwds) - def enter_recursive_debugger(self, code, filename, - continue_if_has_breakpoints): - """ - Enter debugger recursively. - """ - sys.settrace(None) - globals = self.curframe.f_globals - locals = self.curframe_locals - # Create child debugger - debugger = SpyderPdb( - completekey=self.completekey, - stdin=self.stdin, stdout=self.stdout) - debugger.use_rawinput = self.use_rawinput - debugger.prompt = "(%s) " % self.prompt.strip() - - debugger.set_remote_filename(filename) - debugger.continue_if_has_breakpoints = continue_if_has_breakpoints - - # Enter recursive debugger - sys.call_tracing(debugger.run, (code, globals, locals)) - # Reset parent debugger - sys.settrace(self.trace_dispatch) - self.lastcmd = debugger.lastcmd - get_ipython().pdb_session = self - - # Reset _previous_step so that publish_pdb_state() called from within - # postcmd() notifies Spyder about a changed debugger position. The reset - # is required because the recursive debugger might change the position, - # but the parent debugger (self) is not aware of this. - self._previous_step = None - def set_remote_filename(self, filename): """Set remote filename to signal Spyder on mainpyfile.""" self.remote_filename = filename self.mainpyfile = self.canonic(filename) self._wait_for_mainpyfile = True - - -def get_new_debugger(filename, continue_if_has_breakpoints): - """Get a new debugger.""" - debugger = SpyderPdb() - debugger.set_remote_filename(filename) - debugger.continue_if_has_breakpoints = continue_if_has_breakpoints - return debugger diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py b/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py index fc3329ac5f8..fe84791b304 100644 --- a/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/tests/test_umr.py @@ -16,15 +16,14 @@ import pytest # Local imports -from spyder_kernels.py3compat import to_text_string from spyder_kernels.customize.umr import UserModuleReloader @pytest.fixture def user_module(tmpdir): """Create a simple module in tmpdir as an example of a user module.""" - if to_text_string(tmpdir) not in sys.path: - sys.path.append(to_text_string(tmpdir)) + if str(tmpdir) not in sys.path: + sys.path.append(str(tmpdir)) def create_module(modname): modfile = tmpdir.mkdir(modname).join('bar.py') @@ -40,27 +39,6 @@ def square(x): return create_module -def test_umr_skip_cython(user_module): - """ - Test that the UMR doesn't try to reload modules when Cython - support is active. - """ - # Create user module - user_module('foo') - - # Activate Cython support - os.environ['SPY_RUN_CYTHON'] = 'True' - - # Create UMR - umr = UserModuleReloader() - - import foo - assert umr.is_module_reloadable(foo, 'foo') == False - - # Deactivate Cython support - os.environ['SPY_RUN_CYTHON'] = 'False' - - def test_umr_run(user_module): """Test that UMR's run method is working correctly.""" # Create user module @@ -73,12 +51,11 @@ def test_umr_run(user_module): umr = UserModuleReloader() from foo1.bar import square - umr.run() - umr.modnames_to_reload == ['foo', 'foo.bar'] + assert umr.run() == ['foo1', 'foo1.bar'] def test_umr_previous_modules(user_module): - """Test that UMR's previos_modules is working as expected.""" + """Test that UMR's previous_modules is working as expected.""" # Create user module user_module('foo2') diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/umr.py b/external-deps/spyder-kernels/spyder_kernels/customize/umr.py index 6b71abf068f..e779ec336bd 100644 --- a/external-deps/spyder-kernels/spyder_kernels/customize/umr.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/umr.py @@ -9,10 +9,9 @@ import sys from spyder_kernels.customize.utils import path_is_library -from spyder_kernels.py3compat import PY2, _print -class UserModuleReloader(object): +class UserModuleReloader: """ User Module Reloader (UMR) aims at deleting user modules to force Python to deeply reload them during import @@ -43,9 +42,6 @@ def __init__(self, namelist=None, pathlist=None): # pythoncom: See spyder-ide/spyder#7190 # tensorflow: See spyder-ide/spyder#8697 other_modules = ['pytorch', 'pythoncom', 'tensorflow'] - if PY2: - py2_modules = ['astropy', 'fastmat'] - other_modules = other_modules + py2_modules self.namelist = namelist + spy_modules + mpl_modules + other_modules self.pathlist = pathlist @@ -53,13 +49,6 @@ def __init__(self, namelist=None, pathlist=None): # List of previously loaded modules self.previous_modules = list(sys.modules.keys()) - # List of module names to reload - self.modnames_to_reload = [] - - # Activate Cython support - self.has_cython = False - self.activate_cython() - # Check if the UMR is enabled or not enabled = os.environ.get("SPY_UMR_ENABLED", "") self.enabled = enabled.lower() == "true" @@ -70,54 +59,18 @@ def __init__(self, namelist=None, pathlist=None): def is_module_reloadable(self, module, modname): """Decide if a module is reloadable or not.""" - if self.has_cython: - # Don't return cached inline compiled .PYX files + if ( + path_is_library(getattr(module, '__file__', None), self.pathlist) + or self.is_module_in_namelist(modname) + ): return False else: - if (path_is_library(getattr(module, '__file__', None), - self.pathlist) or - self.is_module_in_namelist(modname)): - return False - else: - return True + return True def is_module_in_namelist(self, modname): """Decide if a module can be reloaded or not according to its name.""" return set(modname.split('.')) & set(self.namelist) - def activate_cython(self): - """ - Activate Cython support. - - We need to run this here because if the support is - active, we don't to run the UMR at all. - """ - run_cython = os.environ.get("SPY_RUN_CYTHON") == "True" - - if run_cython: - try: - __import__('Cython') - self.has_cython = True - except Exception: - pass - - if self.has_cython: - # Import pyximport to enable Cython files support for - # import statement - import pyximport - pyx_setup_args = {} - - # Add Numpy include dir to pyximport/distutils - try: - import numpy - pyx_setup_args['include_dirs'] = numpy.get_include() - except Exception: - pass - - # Setup pyximport and enable Cython files reload - pyximport.install(setup_args=pyx_setup_args, - reload_support=True) - def run(self): """ Delete user modules to force Python to deeply reload them @@ -126,18 +79,20 @@ def run(self): modules installed in subdirectories of Python interpreter's binary Do not del C modules """ - self.modnames_to_reload = [] + modnames_to_reload = [] for modname, module in list(sys.modules.items()): if modname not in self.previous_modules: # Decide if a module can be reloaded or not if self.is_module_reloadable(module, modname): - self.modnames_to_reload.append(modname) + modnames_to_reload.append(modname) del sys.modules[modname] else: continue # Report reloaded modules - if self.verbose and self.modnames_to_reload: - modnames = self.modnames_to_reload - _print("\x1b[4;33m%s\x1b[24m%s\x1b[0m" - % ("Reloaded modules", ": "+", ".join(modnames))) \ No newline at end of file + if self.verbose and modnames_to_reload: + modnames = modnames_to_reload + print("\x1b[4;33m%s\x1b[24m%s\x1b[0m" + % ("Reloaded modules", ": "+", ".join(modnames))) + + return modnames_to_reload diff --git a/external-deps/spyder-kernels/spyder_kernels/customize/utils.py b/external-deps/spyder-kernels/spyder_kernels/customize/utils.py index f34a0e5a770..fff18581b2c 100644 --- a/external-deps/spyder-kernels/spyder_kernels/customize/utils.py +++ b/external-deps/spyder-kernels/spyder_kernels/customize/utils.py @@ -6,6 +6,7 @@ """Utility functions.""" import ast +import builtins import os import re import sys @@ -96,7 +97,7 @@ def path_is_library(path, initial_pathlist=None): return False -def capture_last_Expr(code_ast, out_varname): +def capture_last_Expr(code_ast, out_varname, global_ns): """Parse line and modify code to capture in globals the last expression.""" # Modify ast code to capture the last expression capture_last_expression = False @@ -104,11 +105,13 @@ def capture_last_Expr(code_ast, out_varname): len(code_ast.body) and isinstance(code_ast.body[-1], ast.Expr) ): + global_ns["__spyder_builtins__"] = builtins capture_last_expression = True expr_node = code_ast.body[-1] # Create new assign node assign_node = ast.parse( - 'globals()[{}] = None'.format(repr(out_varname))).body[0] + '__spyder_builtins__.globals()[{}] = None'.format( + repr(out_varname))).body[0] # Replace None by the value assign_node.value = expr_node.value # Fix line number and column offset @@ -118,7 +121,7 @@ def capture_last_Expr(code_ast, out_varname): # Exists from 3.8, necessary from 3.11 assign_node.end_lineno = expr_node.end_lineno if assign_node.lineno == assign_node.end_lineno: - # Add 'globals()[{}] = ' and remove 'None' + # Add '__spyder_builtins__.globals()[{}] = ' and remove 'None' assign_node.end_col_offset += expr_node.end_col_offset - 4 else: assign_node.end_col_offset = expr_node.end_col_offset @@ -126,6 +129,94 @@ def capture_last_Expr(code_ast, out_varname): return code_ast, capture_last_expression +def exec_encapsulate_locals( + code_ast, globals, locals, exec_fun=None, filename=None +): + """ + Execute by encapsulating locals if needed. + + Notes + ----- + * In general, the dict returned by locals() might or might not be modified. + In this case, the encapsulated dict can not. + """ + use_locals_hack = locals is not None and locals is not globals + if use_locals_hack: + globals["__spyder_builtins__"] = builtins + + # Mitigates a behaviour of CPython that makes it difficult + # to work with exec and the local namespace + # See: + # - https://bugs.python.org/issue41918 + # - https://bugs.python.org/issue46153 + # - https://bugs.python.org/issue21161 + # - spyder-ide/spyder#13909 + # - spyder-ide/spyder-kernels#345 + # + # The idea here is that the best way to emulate being in a + # function is to actually execute the code in a function. + # A function called `_spyderpdb_code` is created and + # called. It will first load the locals, execute the code, + # and then update the locals. + # + # One limitation of this approach is that locals() is only + # a copy of the curframe locals. This means that closures + # for example are early binding instead of late binding. + + # Create a function + indent = " " + code = ["def _spyderpdb_code():"] + + # Add locals in globals + # If the debugger is recursive, the globals could already + # have a _spyderpdb_locals as it might be shared between + # levels + if "_spyderpdb_locals" in globals: + globals["_spyderpdb_locals"].append(locals) + else: + globals["_spyderpdb_locals"] = [locals] + + # Load locals if they have a valid name + # In comprehensions, locals could contain ".0" for example + code += [indent + "{k} = _spyderpdb_locals[-1]['{k}']".format( + k=k) for k in locals if k.isidentifier()] + + # The code comes here + + # Update the locals + code += [indent + "_spyderpdb_locals[-1].update(" + "__spyder_builtins__.locals())"] + + # Run the function + code += ["_spyderpdb_code()"] + + # Parse the function + fun_ast = ast.parse('\n'.join(code) + '\n') + + # Inject code_ast in the function before the locals update + fun_ast.body[0].body = ( + fun_ast.body[0].body[:-1] # The locals + + code_ast.body # Code to run + + fun_ast.body[0].body[-1:] # Locals update + ) + code_ast = fun_ast + + try: + if exec_fun is None: + exec_fun = exec + if filename is None: + filename = "" + exec_fun(compile(code_ast, filename, "exec"), globals) + finally: + if use_locals_hack: + # Cleanup code + globals.pop("_spyderpdb_code", None) + if len(globals["_spyderpdb_locals"]) > 1: + del globals["_spyderpdb_locals"][-1] + else: + del globals["_spyderpdb_locals"] + + def canonic(filename): """ Return canonical form of filename. diff --git a/external-deps/spyder-kernels/spyder_kernels/py3compat.py b/external-deps/spyder-kernels/spyder_kernels/py3compat.py deleted file mode 100644 index dfb0e322bbf..00000000000 --- a/external-deps/spyder-kernels/spyder_kernels/py3compat.py +++ /dev/null @@ -1,360 +0,0 @@ -# -*- coding: utf-8 -*- -# ----------------------------------------------------------------------------- -# Copyright (c) 2009- Spyder Kernels Contributors -# -# Licensed under the terms of the MIT License -# (see spyder_kernels/__init__.py for details) -# ----------------------------------------------------------------------------- - -""" -spyder.py3compat ----------------- - -Transitional module providing compatibility functions intended to help -migrating from Python 2 to Python 3. - -This module should be fully compatible with: - * Python >=v2.6 - * Python 3 -""" - -from __future__ import print_function - -import operator -import os -import sys - -PY2 = sys.version[0] == '2' -PY3 = sys.version[0] == '3' - -if PY3: - # keep reference to builtin_mod because the kernel overrides that value - # to forward requests to a frontend. - def input(prompt=''): - return builtin_mod.input(prompt) - builtin_mod_name = "builtins" - import builtins as builtin_mod -else: - # keep reference to builtin_mod because the kernel overrides that value - # to forward requests to a frontend. - def input(prompt=''): - return builtin_mod.raw_input(prompt) - builtin_mod_name = "__builtin__" - import __builtin__ as builtin_mod - - -#============================================================================== -# Data types -#============================================================================== -if PY2: - # Python 2 - TEXT_TYPES = (str, unicode) - INT_TYPES = (int, long) -else: - # Python 3 - TEXT_TYPES = (str,) - INT_TYPES = (int,) -NUMERIC_TYPES = tuple(list(INT_TYPES) + [float, complex]) - - -#============================================================================== -# Renamed/Reorganized modules -#============================================================================== -if PY2: - # Python 2 - import __builtin__ as builtins - import ConfigParser as configparser - try: - import _winreg as winreg - except ImportError: - pass - from sys import maxint as maxsize - try: - import CStringIO as io - except ImportError: - import StringIO as io - try: - import cPickle as pickle - except ImportError: - import pickle - from UserDict import DictMixin as MutableMapping - import thread as _thread - import repr as reprlib - import Queue -else: - # Python 3 - import builtins - import configparser - try: - import winreg - except ImportError: - pass - from sys import maxsize - import io - import pickle - from collections.abc import MutableMapping - import _thread - import reprlib - import queue as Queue - - -#============================================================================== -# Strings -#============================================================================== -def is_type_text_string(obj): - """Return True if `obj` is type text string, False if it is anything else, - like an instance of a class that extends the basestring class.""" - if PY2: - # Python 2 - return type(obj) in [str, unicode] - else: - # Python 3 - return type(obj) in [str, bytes] - -def is_text_string(obj): - """Return True if `obj` is a text string, False if it is anything else, - like binary data (Python 3) or QString (Python 2, PyQt API #1)""" - if PY2: - # Python 2 - return isinstance(obj, basestring) - else: - # Python 3 - return isinstance(obj, str) - -def is_binary_string(obj): - """Return True if `obj` is a binary string, False if it is anything else""" - if PY2: - # Python 2 - return isinstance(obj, str) - else: - # Python 3 - return isinstance(obj, bytes) - -def is_string(obj): - """Return True if `obj` is a text or binary Python string object, - False if it is anything else, like a QString (Python 2, PyQt API #1)""" - return is_text_string(obj) or is_binary_string(obj) - -def is_unicode(obj): - """Return True if `obj` is unicode""" - if PY2: - # Python 2 - return isinstance(obj, unicode) - else: - # Python 3 - return isinstance(obj, str) - -def to_text_string(obj, encoding=None): - """Convert `obj` to (unicode) text string""" - if PY2: - # Python 2 - if encoding is None: - return unicode(obj) - else: - return unicode(obj, encoding) - else: - # Python 3 - if encoding is None: - return str(obj) - elif isinstance(obj, str): - # In case this function is not used properly, this could happen - return obj - else: - return str(obj, encoding) - -def to_binary_string(obj, encoding=None): - """Convert `obj` to binary string (bytes in Python 3, str in Python 2)""" - if PY2: - # Python 2 - if encoding is None: - return str(obj) - else: - return obj.encode(encoding) - else: - # Python 3 - return bytes(obj, 'utf-8' if encoding is None else encoding) - - -#============================================================================== -# Function attributes -#============================================================================== -def get_func_code(func): - """Return function code object""" - if PY2: - # Python 2 - return func.func_code - else: - # Python 3 - return func.__code__ - -def get_func_name(func): - """Return function name""" - if PY2: - # Python 2 - return func.func_name - else: - # Python 3 - return func.__name__ - -def get_func_defaults(func): - """Return function default argument values""" - if PY2: - # Python 2 - return func.func_defaults - else: - # Python 3 - return func.__defaults__ - - -#============================================================================== -# Special method attributes -#============================================================================== -def get_meth_func(obj): - """Return method function object""" - if PY2: - # Python 2 - return obj.im_func - else: - # Python 3 - return obj.__func__ - -def get_meth_class_inst(obj): - """Return method class instance""" - if PY2: - # Python 2 - return obj.im_self - else: - # Python 3 - return obj.__self__ - -def get_meth_class(obj): - """Return method class""" - if PY2: - # Python 2 - return obj.im_class - else: - # Python 3 - return obj.__self__.__class__ - - -#============================================================================== -# Misc. -#============================================================================== -if PY2: - def _print(*objects, **options): - end = options.get('end', '\n') - file = options.get('file', sys.stdout) - sep = options.get('sep', ' ') - string = sep.join([str(obj) for obj in objects]) - print(string, file=file, end=end, sep=sep) -else: - _print = print - - -if PY2: - # Python 2 - getcwd = os.getcwdu - cmp = cmp - import string - str_lower = string.lower - from itertools import izip_longest as zip_longest - from backports.functools_lru_cache import lru_cache -else: - # Python 3 - getcwd = os.getcwd - def cmp(a, b): - return (a > b) - (a < b) - str_lower = str.lower - from itertools import zip_longest - from functools import lru_cache - -def qbytearray_to_str(qba): - """Convert QByteArray object to str in a way compatible with Python 2/3""" - return str(bytes(qba.toHex().data()).decode()) - -# ============================================================================= -# Dict funcs -# ============================================================================= -if PY3: - def iterkeys(d, **kw): - return iter(d.keys(**kw)) - - def itervalues(d, **kw): - return iter(d.values(**kw)) - - def iteritems(d, **kw): - return iter(d.items(**kw)) - - def iterlists(d, **kw): - return iter(d.lists(**kw)) - - viewkeys = operator.methodcaller("keys") - - viewvalues = operator.methodcaller("values") - - viewitems = operator.methodcaller("items") -else: - def iterkeys(d, **kw): - return d.iterkeys(**kw) - - def itervalues(d, **kw): - return d.itervalues(**kw) - - def iteritems(d, **kw): - return d.iteritems(**kw) - - def iterlists(d, **kw): - return d.iterlists(**kw) - - viewkeys = operator.methodcaller("viewkeys") - - viewvalues = operator.methodcaller("viewvalues") - - viewitems = operator.methodcaller("viewitems") - -# ============================================================================= -# Exceptions -# ============================================================================= -if PY2: - TimeoutError = RuntimeError - FileNotFoundError = IOError -else: - TimeoutError = TimeoutError - FileNotFoundError = FileNotFoundError - -if PY2: - import re - import tokenize - def isidentifier(string): - """Check if string can be a variable name.""" - return re.match(tokenize.Name + r'\Z', string) is not None - - if os.name == 'nt': - def encode(u): - """Try encoding with utf8.""" - if isinstance(u, unicode): - return u.encode('utf8', 'replace') - return u - else: - def encode(u): - """Try encoding with file system encoding.""" - if isinstance(u, unicode): - return u.encode(sys.getfilesystemencoding()) - return u -else: - def isidentifier(string): - """Check if string can be a variable name.""" - return string.isidentifier() - - def encode(u): - """Encoding is not a problem in python 3.""" - return u - - -def compat_exec(code, globals, locals): - # Wrap exec in a function - exec(code, globals, locals) - - -if __name__ == '__main__': - pass diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/dochelpers.py b/external-deps/spyder-kernels/spyder_kernels/utils/dochelpers.py index 83db1a43e00..2aace3afa97 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/dochelpers.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/dochelpers.py @@ -7,17 +7,10 @@ # ----------------------------------------------------------------------------- """Utilities and wrappers around inspect module""" - -from __future__ import print_function - +import builtins import inspect import re -# Local imports: -from spyder_kernels.py3compat import (is_text_string, builtins, get_meth_func, - get_meth_class_inst, get_meth_class, - get_func_defaults, to_text_string, PY2) - SYMBOLS = r"[^\'\"a-zA-Z0-9_.]" @@ -57,7 +50,7 @@ def getobjdir(obj): In special cases (e.g. WrapITK package), will return only string elements of result returned by dir(obj) """ - return [item for item in dir(obj) if is_text_string(item)] + return [item for item in dir(obj) if isinstance(item, str)] def getdoc(obj): @@ -84,7 +77,7 @@ def getdoc(obj): # yield anything, either. So assume the most commonly used # multi-byte file encoding (which also covers ascii). try: - docstring = to_text_string(docstring) + docstring = str(docstring) except: pass @@ -101,35 +94,29 @@ def getdoc(obj): doc['docstring'] = docstring return doc if inspect.ismethod(obj): - imclass = get_meth_class(obj) - if get_meth_class_inst(obj) is not None: + imclass = obj.__self__.__class__ + if obj.__self__ is not None: doc['note'] = 'Method of %s instance' \ - % get_meth_class_inst(obj).__class__.__name__ + % obj.__self__.__class__.__name__ else: doc['note'] = 'Unbound %s method' % imclass.__name__ - obj = get_meth_func(obj) + obj = obj.__func__ elif hasattr(obj, '__module__'): doc['note'] = 'Function of %s module' % obj.__module__ else: doc['note'] = 'Function' doc['name'] = obj.__name__ if inspect.isfunction(obj): - if PY2: - args, varargs, varkw, defaults = inspect.getargspec(obj) - doc['argspec'] = inspect.formatargspec( - args, varargs, varkw, defaults, - formatvalue=lambda o:'='+repr(o)) - else: - # This is necessary to catch errors for objects without a - # signature, like numpy.where. - # Fixes spyder-ide/spyder#21148 - try: - sig = inspect.signature(obj) - except ValueError: - sig = getargspecfromtext(doc['docstring']) - if not sig: - sig = '(...)' - doc['argspec'] = str(sig) + # This is necessary to catch errors for objects without a + # signature, like numpy.where. + # Fixes spyder-ide/spyder#21148 + try: + sig = inspect.signature(obj) + except ValueError: + sig = getargspecfromtext(doc['docstring']) + if not sig: + sig = '(...)' + doc['argspec'] = str(sig) if name == '': doc['name'] = name + ' lambda ' doc['argspec'] = doc['argspec'][1:-1] # remove parentheses @@ -166,10 +153,10 @@ def getsource(obj): """Wrapper around inspect.getsource""" try: try: - src = to_text_string(inspect.getsource(obj)) + src = str(inspect.getsource(obj)) except TypeError: if hasattr(obj, '__class__'): - src = to_text_string(inspect.getsource(obj.__class__)) + src = str(inspect.getsource(obj.__class__)) else: # Bindings like VTK or ITK require this case src = getdoc(obj) @@ -201,44 +188,35 @@ def getsignaturefromtext(text, objname): # others in doctests or other places, but those are not so important. sig = '' if sigs: - if PY2: - # We don't have an easy way to check if the identifier detected by - # signature_re is a valid one in Python 2. So, we simply select the - # first match. - sig = sigs[0] if objname else sigs[0][1] + # Default signatures returned by IPython. + # Notes: + # * These are not real signatures but only used to provide a + # placeholder. + # * We skip them if we can find other signatures in `text`. + # * This is necessary because we also use this function in Spyder + # to parse the content of inspect replies that come from the + # kernel, which can include these signatures. + default_ipy_sigs = ['(*args, **kwargs)', '(self, /, *args, **kwargs)'] + + if objname: + real_sigs = [s for s in sigs if s not in default_ipy_sigs] + + if real_sigs: + sig = real_sigs[0] + else: + sig = sigs[0] else: - # Default signatures returned by IPython. - # Notes: - # * These are not real signatures but only used to provide a - # placeholder. - # * We skip them if we can find other signatures in `text`. - # * This is necessary because we also use this function in Spyder - # to parse the content of inspect replies that come from the - # kernel, which can include these signatures. - default_ipy_sigs = [ - '(*args, **kwargs)', - '(self, /, *args, **kwargs)' - ] - - if objname: - real_sigs = [s for s in sigs if s not in default_ipy_sigs] + valid_sigs = [s for s in sigs if s[0].isidentifier()] + + if valid_sigs: + real_sigs = [ + s for s in valid_sigs if s[1] not in default_ipy_sigs + ] if real_sigs: - sig = real_sigs[0] + sig = real_sigs[0][1] else: - sig = sigs[0] - else: - valid_sigs = [s for s in sigs if s[0].isidentifier()] - - if valid_sigs: - real_sigs = [ - s for s in valid_sigs if s[1] not in default_ipy_sigs - ] - - if real_sigs: - sig = real_sigs[0][1] - else: - sig = valid_sigs[0][1] + sig = valid_sigs[0][1] return sig @@ -274,7 +252,7 @@ def getargs(obj): if inspect.isfunction(obj) or inspect.isbuiltin(obj): func_obj = obj elif inspect.ismethod(obj): - func_obj = get_meth_func(obj) + func_obj = obj.__func__ elif inspect.isclass(obj) and hasattr(obj, '__init__'): func_obj = getattr(obj, '__init__') else: @@ -298,7 +276,7 @@ def getargs(obj): if isinstance(arg, list): args[i_arg] = "(%s)" % ", ".join(arg) - defaults = get_func_defaults(func_obj) + defaults = func_obj.__defaults__ if defaults is not None: for index, default in enumerate(defaults): args[index + len(args) - len(defaults)] += '=' + repr(default) diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/iofuncs.py b/external-deps/spyder-kernels/spyder_kernels/utils/iofuncs.py index 05b988831c6..7da2828ff3a 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/iofuncs.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/iofuncs.py @@ -12,9 +12,6 @@ Note: 'load' functions has to return a dictionary from which a globals() namespace may be updated """ - -from __future__ import print_function - # Standard library imports import sys import os @@ -28,13 +25,15 @@ import dis import copy import glob +import pickle # Local imports -from spyder_kernels.py3compat import getcwd, pickle, PY2, to_text_string from spyder_kernels.utils.lazymodules import ( FakeObject, numpy as np, pandas as pd, PIL, scipy as sp) +# ---- For Matlab files +# ----------------------------------------------------------------------------- class MatlabStruct(dict): """ Matlab style struct, enhanced. @@ -44,7 +43,7 @@ class MatlabStruct(dict): Examples ======== - >>> from spyder.utils.iofuncs import MatlabStruct + >>> from spyder_kernels.utils.iofuncs import MatlabStruct >>> a = MatlabStruct() >>> a.b = 'spam' # a["b"] == 'spam' >>> a.c["d"] = 'eggs' # a.c.d == 'eggs' @@ -82,7 +81,6 @@ def _is_allowed(self, frame): dis.opmap.get('STOP_CODE', 0)] bytecode = frame.f_code.co_code instruction = bytecode[frame.f_lasti + 3] - instruction = ord(instruction) if PY2 else instruction return instruction in allowed __setattr__ = dict.__setitem__ @@ -169,6 +167,8 @@ def save_matlab(data, filename): return str(error) +# ---- For arrays +# ----------------------------------------------------------------------------- def load_array(filename): if np.load is FakeObject: return None, '' @@ -193,6 +193,8 @@ def __save_array(data, basename, index): return fname +# ---- For PIL images +# ----------------------------------------------------------------------------- if sys.byteorder == 'little': _ENDIAN = '<' else: @@ -237,6 +239,8 @@ def load_image(filename): return None, str(error) +# ---- For misc formats +# ----------------------------------------------------------------------------- def load_pickle(filename): """Load a pickle file as a dictionary""" try: @@ -253,21 +257,19 @@ def load_pickle(filename): def load_json(filename): """Load a json file as a dictionary""" try: - if PY2: - args = 'rb' - else: - args = 'r' - with open(filename, args) as fid: + with open(filename, 'r') as fid: data = json.load(fid) return data, None except Exception as err: return None, str(err) +# ---- For Spydata files +# ----------------------------------------------------------------------------- def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) - old_cwd = getcwd() + old_cwd = os.getcwd() os.chdir(osp.dirname(filename)) error_message = None skipped_keys = [] @@ -358,7 +360,7 @@ def save_dictionary(data, filename): tar.add(osp.basename(fname)) os.remove(fname) except (RuntimeError, pickle.PicklingError, TypeError) as error: - error_message = to_text_string(error) + error_message = str(error) else: if skipped_keys: skipped_keys.sort() @@ -377,15 +379,13 @@ def is_within_directory(directory, target): return prefix == abs_directory -def safe_extract(tar, path=".", members=None, numeric_owner=False): +def safe_extract(tar, path=".", members=None, *, numeric_owner=False): """Safely extract a tar file.""" for member in tar.getmembers(): member_path = os.path.join(path, member.name) if not is_within_directory(path, member_path): raise Exception( - "Attempted path traversal in tar file {}".format( - repr(tar.name) - ) + f"Attempted path traversal in tar file {tar.name!r}" ) tar.extractall(path, members, numeric_owner=numeric_owner) @@ -393,20 +393,17 @@ def safe_extract(tar, path=".", members=None, numeric_owner=False): def load_dictionary(filename): """Load dictionary from .spydata file""" filename = osp.abspath(filename) - old_cwd = getcwd() + old_cwd = os.getcwd() tmp_folder = tempfile.mkdtemp() os.chdir(tmp_folder) data = None error_message = None try: with tarfile.open(filename, "r") as tar: - if PY2: - tar.extractall() - else: - safe_extract(tar) + safe_extract(tar) pickle_filename = glob.glob('*.pickle')[0] - # 'New' format (Spyder >=2.2 for Python 2 and Python 3) + # 'New' format (Spyder >=2.2) with open(pickle_filename, 'rb') as fdesc: data = pickle.loads(fdesc.read()) saved_arrays = {} @@ -426,18 +423,101 @@ def load_dictionary(filename): pass # Except AttributeError from e.g. trying to load function no longer present except (AttributeError, EOFError, ValueError) as error: - error_message = to_text_string(error) + error_message = str(error) # To ensure working dir gets changed back and temp dir wiped no matter what finally: os.chdir(old_cwd) try: shutil.rmtree(tmp_folder) except OSError as error: - error_message = to_text_string(error) + error_message = str(error) return data, error_message -class IOFunctions(object): +# ---- For HDF5 files +# ----------------------------------------------------------------------------- +def load_hdf5(filename): + """ + Load an hdf5 file. + + Notes + ----- + - This is a fairly dumb implementation which reads the whole HDF5 file into + Spyder's variable explorer. Since HDF5 files are designed for storing + very large data-sets, it may be much better to work directly with the + HDF5 objects, thus keeping the data on disk. Nonetheless, this gives + quick and dirty but convenient access to them. + - There is no support for creating files with compression, chunking etc, + although these can be read without problem. + - When reading an HDF5 file with sub-groups, groups in the file will + correspond to dictionaries with the same layout. + """ + def get_group(group): + contents = {} + for name, obj in list(group.items()): + if isinstance(obj, h5py.Dataset): + contents[name] = np.array(obj) + elif isinstance(obj, h5py.Group): + # it is a group, so call self recursively + contents[name] = get_group(obj) + # other objects such as links are ignored + return contents + + try: + import h5py + + f = h5py.File(filename, 'r') + contents = get_group(f) + f.close() + return contents, None + except Exception as error: + return None, str(error) + + +def save_hdf5(data, filename): + """ + Save an hdf5 file. + + Notes + ----- + - All datatypes to be saved must be convertible to a numpy array, otherwise + an exception will be raised. + - Data attributes are currently ignored. + - When saving data after reading it with load_hdf5, dictionaries are not + turned into HDF5 groups. + """ + try: + import h5py + + f = h5py.File(filename, 'w') + for key, value in list(data.items()): + f[key] = np.array(value) + f.close() + except Exception as error: + return str(error) + + +# ---- For DICOM files +# ----------------------------------------------------------------------------- +def load_dicom(filename): + """Load a DICOM files.""" + try: + from pydicom import dicomio + + name = osp.splitext(osp.basename(filename))[0] + try: + data = dicomio.read_file(filename, force=True) + except TypeError: + data = dicomio.read_file(filename) + arr = data.pixel_array + return {name: arr}, None + except Exception as error: + return None, str(error) + + +# ---- Class to group all IO functionality +# ----------------------------------------------------------------------------- +class IOFunctions: def __init__(self): self.load_extensions = None self.save_extensions = None @@ -447,7 +527,7 @@ def __init__(self): self.save_funcs = None def setup(self): - iofuncs = self.get_internal_funcs()+self.get_3rd_party_funcs() + iofuncs = self.get_internal_funcs() load_extensions = {} save_extensions = {} load_funcs = {} @@ -455,8 +535,9 @@ def setup(self): load_filters = [] save_filters = [] load_ext = [] + for ext, name, loadfunc, savefunc in iofuncs: - filter_str = to_text_string(name + " (*%s)" % ext) + filter_str = str(name + " (*%s)" % ext) if loadfunc is not None: load_filters.append(filter_str) load_extensions[filter_str] = ext @@ -466,9 +547,12 @@ def setup(self): save_extensions[filter_str] = ext save_filters.append(filter_str) save_funcs[ext] = savefunc - load_filters.insert(0, to_text_string("Supported files"+" (*"+\ - " *".join(load_ext)+")")) - load_filters.append(to_text_string("All files (*.*)")) + + load_filters.insert( + 0, str("Supported files" + " (*" + " *".join(load_ext) + ")") + ) + load_filters.append(str("All files (*.*)")) + self.load_filters = "\n".join(load_filters) self.save_filters = "\n".join(save_filters) self.load_funcs = load_funcs @@ -478,35 +562,22 @@ def setup(self): def get_internal_funcs(self): return [ - ('.spydata', "Spyder data files", - load_dictionary, save_dictionary), - ('.npy', "NumPy arrays", load_array, None), - ('.npz', "NumPy zip arrays", load_array, None), - ('.mat', "Matlab files", load_matlab, save_matlab), - ('.csv', "CSV text files", 'import_wizard', None), - ('.txt', "Text files", 'import_wizard', None), - ('.jpg', "JPEG images", load_image, None), - ('.png', "PNG images", load_image, None), - ('.gif', "GIF images", load_image, None), - ('.tif', "TIFF images", load_image, None), - ('.pkl', "Pickle files", load_pickle, None), - ('.pickle', "Pickle files", load_pickle, None), - ('.json', "JSON files", load_json, None), - ] - - def get_3rd_party_funcs(self): - other_funcs = [] - try: - from spyder.otherplugins import get_spyderplugins_mods - for mod in get_spyderplugins_mods(io=True): - try: - other_funcs.append((mod.FORMAT_EXT, mod.FORMAT_NAME, - mod.FORMAT_LOAD, mod.FORMAT_SAVE)) - except AttributeError as error: - print("%s: %s" % (mod, str(error)), file=sys.stderr) - except ImportError: - pass - return other_funcs + ('.spydata', "Spyder data files", load_dictionary, save_dictionary), + ('.npy', "NumPy arrays", load_array, None), + ('.npz', "NumPy zip arrays", load_array, None), + ('.mat', "Matlab files", load_matlab, save_matlab), + ('.csv', "CSV text files", 'import_wizard', None), + ('.txt', "Text files", 'import_wizard', None), + ('.jpg', "JPEG images", load_image, None), + ('.png', "PNG images", load_image, None), + ('.gif', "GIF images", load_image, None), + ('.tif', "TIFF images", load_image, None), + ('.pkl', "Pickle files", load_pickle, None), + ('.pickle', "Pickle files", load_pickle, None), + ('.json', "JSON files", load_json, None), + ('.h5', "HDF5 files", load_hdf5, save_hdf5), + ('.dcm', "DICOM images", load_dicom, None), + ] def save(self, data, filename): ext = osp.splitext(filename)[1].lower() @@ -526,11 +597,8 @@ def load(self, filename): iofunctions.setup() -def save_auto(data, filename): - """Save data into filename, depending on file extension""" - pass - - +# ---- Test +# ----------------------------------------------------------------------------- if __name__ == "__main__": import datetime testdict = {'d': 1, 'a': np.random.rand(10, 10), 'b': [1, 2]} @@ -549,9 +617,9 @@ def save_auto(data, filename): import time t0 = time.time() save_dictionary(example, "test.spydata") - print(" Data saved in %.3f seconds" % (time.time()-t0)) # spyder: test-skip + print(" Data saved in %.3f seconds" % (time.time()-t0)) t0 = time.time() example2, ok = load_dictionary("test.spydata") os.remove("test.spydata") - print("Data loaded in %.3f seconds" % (time.time()-t0)) # spyder: test-skip + print("Data loaded in %.3f seconds" % (time.time()-t0)) diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/lazymodules.py b/external-deps/spyder-kernels/spyder_kernels/utils/lazymodules.py index d65847b44b3..75b5b3a94e1 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/lazymodules.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/lazymodules.py @@ -18,12 +18,12 @@ # ============================================================================= # Auxiliary classes # ============================================================================= -class FakeObject(object): +class FakeObject: """Fake class used in replacement of missing objects""" pass -class LazyModule(object): +class LazyModule: """Lazy module loader class.""" def __init__(self, modname, second_level_attrs=None): diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/misc.py b/external-deps/spyder-kernels/spyder_kernels/utils/misc.py index 56ec5f215ec..bbb9b050cac 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/misc.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/misc.py @@ -10,7 +10,7 @@ import re -from spyder_kernels.py3compat import lru_cache +from functools import lru_cache @lru_cache(maxsize=100) diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py index 7927e49da62..8ac557f7886 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/mpl.py @@ -11,13 +11,6 @@ from spyder_kernels.utils.misc import is_module_installed -# Mapping of inline figure formats -INLINE_FIGURE_FORMATS = { - '0': 'png', - '1': 'svg' -} - - # Inline backend if is_module_installed('matplotlib_inline'): inline_backend = 'module://matplotlib_inline.backend_inline' @@ -27,30 +20,21 @@ # Mapping of matlotlib backends options to Spyder MPL_BACKENDS_TO_SPYDER = { - inline_backend: 0, - 'Qt5Agg': 2, - 'QtAgg': 2, # For Matplotlib 3.5+ - 'TkAgg': 3, - 'MacOSX': 4, + 'inline': 'inline', # For Matplotlib >=3.9 + inline_backend: "inline", # For Matplotlib <3.9 + 'qt5agg': 'qt', + 'qtagg': 'qt', # For Matplotlib 3.5+ + 'tkagg': 'tk', + 'macosx': 'osx', } def automatic_backend(): """Get Matplolib automatic backend option.""" if is_module_installed('PyQt5'): - auto_backend = 'qt5' + auto_backend = 'qt' elif is_module_installed('_tkinter'): auto_backend = 'tk' else: auto_backend = 'inline' return auto_backend - - -# Mapping of Spyder options to backends -MPL_BACKENDS_FROM_SPYDER = { - '0': 'inline', - '1': automatic_backend(), - '2': 'qt5', - '3': 'tk', - '4': 'osx' -} diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/nsview.py b/external-deps/spyder-kernels/spyder_kernels/utils/nsview.py index 103490aa35d..d7e0ce5a727 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/nsview.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/nsview.py @@ -9,19 +9,10 @@ """ Utilities to build a namespace view. """ - -from __future__ import print_function - from itertools import islice import inspect import re -# Local imports -from spyder_kernels.py3compat import (NUMERIC_TYPES, INT_TYPES, TEXT_TYPES, - to_text_string, is_text_string, - is_type_text_string, - is_binary_string, PY2, - to_binary_string, iteritems) from spyder_kernels.utils.lazymodules import ( bs4, FakeObject, numpy as np, pandas as pd, PIL) @@ -274,7 +265,7 @@ def default_display(value, with_module=True): return name + ' object of ' + module + ' module' return name except Exception: - type_str = to_text_string(object_type) + type_str = str(object_type) return type_str[1:-1] @@ -285,7 +276,7 @@ def collections_display(value, level): # Get elements if is_dict: - elements = iteritems(value) + elements = iter(value.items()) else: elements = value @@ -373,27 +364,15 @@ def value_to_display(value, minmax=False, level=0): elif isinstance(value, pd.DataFrame): if level == 0: cols = value.columns - if PY2 and len(cols) > 0: - # Get rid of possible BOM utf-8 data present at the - # beginning of a file, which gets attached to the first - # column header when headers are present in the first - # row. - # Fixes Issue 2514 - try: - ini_col = to_text_string(cols[0], encoding='utf-8-sig') - except: - ini_col = to_text_string(cols[0]) - cols = [ini_col] + [to_text_string(c) for c in cols[1:]] - else: - cols = [to_text_string(c) for c in cols] + cols = [str(c) for c in cols] display = 'Column names: ' + ', '.join(list(cols)) else: display = 'Dataframe' elif isinstance(value, bs4.element.NavigableString): # Fixes Issue 2448 - display = to_text_string(value) + display = str(value) if level > 0: - display = u"'" + display + u"'" + display = "'" + display + "'" elif isinstance(value, pd.Index): if level == 0: try: @@ -402,33 +381,34 @@ def value_to_display(value, minmax=False, level=0): display = value.summary() else: display = 'Index' - elif is_binary_string(value): + elif isinstance(value, bytes): # We don't apply this to classes that extend string types # See issue 5636 - if is_type_text_string(value): + if type(value) in [str, bytes]: try: - display = to_text_string(value, 'utf8') + display = str(value, 'utf8') if level > 0: - display = u"'" + display + u"'" + display = "'" + display + "'" except: display = value if level > 0: display = b"'" + display + b"'" else: display = default_display(value) - elif is_text_string(value): + elif isinstance(value, str): # We don't apply this to classes that extend string types # See issue 5636 - if is_type_text_string(value): + if type(value) in [str, bytes]: display = value if level > 0: - display = u"'" + display + u"'" + display = "'" + display + "'" else: display = default_display(value) + elif (isinstance(value, datetime.date) or isinstance(value, datetime.timedelta)): display = str(value) - elif (isinstance(value, NUMERIC_TYPES) or + elif (isinstance(value, (int, float, complex)) or isinstance(value, bool) or isinstance(value, numeric_numpy_types)): display = repr(value) @@ -443,10 +423,10 @@ def value_to_display(value, minmax=False, level=0): # Truncate display at 70 chars to avoid freezing Spyder # because of large displays if len(display) > 70: - if is_binary_string(display): + if isinstance(display, bytes): ellipses = b' ...' else: - ellipses = u' ...' + ellipses = ' ...' display = display[:70].rstrip() + ellipses # Restore Numpy printoptions @@ -459,7 +439,7 @@ def value_to_display(value, minmax=False, level=0): def display_to_value(value, default_value, ignore_errors=True): """Convert back to value""" from qtpy.compat import from_qvariant - value = from_qvariant(value, to_text_string) + value = from_qvariant(value, str) try: np_dtype = get_numpy_dtype(default_value) if isinstance(default_value, bool): @@ -474,10 +454,10 @@ def display_to_value(value, default_value, ignore_errors=True): value = np_dtype(complex(value)) else: value = np_dtype(value) - elif is_binary_string(default_value): - value = to_binary_string(value, 'utf8') - elif is_text_string(default_value): - value = to_text_string(value) + elif isinstance(default_value, bytes): + value = bytes(value, 'utf-8') + elif isinstance(default_value, str): + value = str(value) elif isinstance(default_value, complex): value = complex(value) elif isinstance(default_value, float): @@ -531,7 +511,7 @@ def get_type_string(item): pass found = re.findall(r"<(?:type|class) '(\S*)'>", - to_text_string(type(item))) + str(type(item))) if found: if found[0] == 'type': return 'class' @@ -609,10 +589,12 @@ def is_callable_or_module(value): def globalsfilter(input_dict, check_all=False, filters=None, exclude_private=None, exclude_capitalized=None, exclude_uppercase=None, exclude_unsupported=None, - excluded_names=None, exclude_callables_and_modules=None): + excluded_names=None, exclude_callables_and_modules=None, + filter_on=True): """Keep objects in namespace view according to different criteria.""" output_dict = {} - _is_string = is_type_text_string + def _is_string(obj): + return type(obj) in [str, bytes] for key, value in list(input_dict.items()): excluded = ( @@ -624,7 +606,7 @@ def globalsfilter(input_dict, check_all=False, filters=None, (exclude_callables_and_modules and is_callable_or_module(value)) or (exclude_unsupported and not is_supported(value, check_all=check_all, filters=filters)) - ) + ) and filter_on if not excluded: output_dict[key] = value return output_dict @@ -636,7 +618,8 @@ def globalsfilter(input_dict, check_all=False, filters=None, REMOTE_SETTINGS = ('check_all', 'exclude_private', 'exclude_uppercase', 'exclude_capitalized', 'exclude_unsupported', 'excluded_names', 'minmax', 'show_callable_attributes', - 'show_special_attributes', 'exclude_callables_and_modules') + 'show_special_attributes', 'exclude_callables_and_modules', + 'filter_on') def get_supported_types(): @@ -650,7 +633,7 @@ def get_supported_types(): """ from datetime import date, timedelta editable_types = [int, float, complex, list, set, dict, tuple, date, - timedelta] + list(TEXT_TYPES) + list(INT_TYPES) + timedelta, str] try: from numpy import ndarray, matrix, generic editable_types += [ndarray, matrix, generic] @@ -692,7 +675,7 @@ def get_remote_data(data, settings, mode, more_excluded_names=None): exclude_capitalized=settings['exclude_capitalized'], exclude_unsupported=settings['exclude_unsupported'], exclude_callables_and_modules=settings['exclude_callables_and_modules'], - excluded_names=excluded_names) + excluded_names=excluded_names, filter_on=settings['filter_on']) def make_remote_view(data, settings, more_excluded_names=None): diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/tests/data.dcm b/external-deps/spyder-kernels/spyder_kernels/utils/tests/data.dcm new file mode 100644 index 00000000000..9d525ec3b3f Binary files /dev/null and b/external-deps/spyder-kernels/spyder_kernels/utils/tests/data.dcm differ diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_dochelpers.py b/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_dochelpers.py index f67d730fbe6..05197a0ab01 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_dochelpers.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_dochelpers.py @@ -21,7 +21,6 @@ from spyder_kernels.utils.dochelpers import ( getargtxt, getargspecfromtext, getdoc, getobj, getsignaturefromtext, isdefined) -from spyder_kernels.py3compat import PY2 class Test(object): @@ -29,8 +28,7 @@ def method(self, x, y=2): pass -@pytest.mark.skipif( - PY2 or os.name == 'nt', reason="Only works on Linux and Mac") +@pytest.mark.skipif(os.name == 'nt', reason="Only works on Linux and Mac") @pytest.mark.skipif( sys.platform == 'darwin' and sys.version_info[:2] == (3, 8), reason="Fails on Mac with Python 3.8") @@ -61,7 +59,6 @@ def test_dochelpers(): assert getobj('4.') == '4' -@pytest.mark.skipif(PY2, reason="Fails in Python 2") def test_no_signature(): """ Test that we can get documentation for objects for which Python can't get a @@ -76,18 +73,6 @@ def test_no_signature(): assert doc['docstring'] -@pytest.mark.parametrize( - 'text, name, expected', - [ - ('foo(x, y)', 'foo', '(x, y)'), - ('foo(x, y)', '', '(x, y)'), - ] -) -def test_getsignaturefromtext_py2(text, name, expected): - assert getsignaturefromtext(text, name) == expected - - -@pytest.mark.skipif(PY2, reason="Don't work in Python 2") @pytest.mark.parametrize( 'text, name, expected', [ diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_iofuncs.py b/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_iofuncs.py index b4a28c8b5fe..d88b2e103d5 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_iofuncs.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_iofuncs.py @@ -16,17 +16,18 @@ import copy # Third party imports +from PIL import ImageFile import pytest import numpy as np # Local imports import spyder_kernels.utils.iofuncs as iofuncs -from spyder_kernels.py3compat import is_text_string, PY2 # Full path to this file's parent directory for loading data -LOCATION = os.path.realpath(os.path.join(os.getcwd(), - os.path.dirname(__file__))) +LOCATION = os.path.realpath( + os.path.join(os.getcwd(), os.path.dirname(__file__)) +) # ============================================================================= @@ -48,7 +49,7 @@ def are_namespaces_equal(actual, expected): return are_equal -class CustomObj(object): +class CustomObj: """A custom class of objects for testing.""" def __init__(self, data): self.data = None @@ -227,7 +228,6 @@ def test_matlab_import(real_values): assert valid -@pytest.mark.skipif(PY2, reason="Fails on Python 2") @pytest.mark.parametrize('spydata_file_name', ['export_data.spydata', 'export_data_renamed.spydata']) def test_spydata_import(spydata_file_name, spydata_values): @@ -255,7 +255,7 @@ def test_spydata_import_witherror(): original_cwd = os.getcwd() path = os.path.join(LOCATION, 'export_data_withfunction.spydata') data, error = iofuncs.load_dictionary(path) - assert error and is_text_string(error) + assert error and isinstance(error, str) assert data is None assert os.getcwd() == original_cwd @@ -340,5 +340,26 @@ def test_spydata_export(input_namespace, expected_namespace, pass +def test_save_load_hdf5_files(tmp_path): + """Simple test to check that we can save and load HDF5 files.""" + h5_file = tmp_path / "test.h5" + data = {'a' : [1, 2, 3, 4], 'b' : 4.5} + iofuncs.save_hdf5(data, h5_file) + + expected = ({'a': np.array([1, 2, 3, 4]), 'b': np.array(4.5)}, None) + assert repr(iofuncs.load_hdf5(h5_file)) == repr(expected) + + +def test_load_dicom_files(): + """Check that we can load DICOM files.""" + # This test pass locally but we need to set the variable below for it to + # pass on CIs. + # See https://stackoverflow.com/a/47958486/438386 for context. + ImageFile.LOAD_TRUNCATED_IMAGES = True + + data = iofuncs.load_dicom(os.path.join(LOCATION, 'data.dcm')) + assert data[0]['data'].shape == (512, 512) + + if __name__ == "__main__": pytest.main() diff --git a/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_nsview.py b/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_nsview.py index 022f215810e..2d79c9f825c 100644 --- a/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_nsview.py +++ b/external-deps/spyder-kernels/spyder_kernels/utils/tests/test_nsview.py @@ -23,7 +23,6 @@ import PIL.Image # Local imports -from spyder_kernels.py3compat import PY2 from spyder_kernels.utils.nsview import ( sort_against, is_supported, value_to_display, get_size, get_supported_types, get_type_string, get_numpy_type_string, @@ -110,7 +109,7 @@ def test_none_values_are_supported(): def test_str_subclass_display(): - """Test for value_to_display of subclasses of str/basestring.""" + """Test for value_to_display of subclasses of str.""" class Test(str): def __repr__(self): return 'test' @@ -284,12 +283,6 @@ def test_str_in_container_display(): # Assert that both bytes and unicode return the right display assert value_to_display([b'a', u'b']) == "['a', 'b']" - # Encoded unicode gives bytes and it can't be transformed to - # unicode again. So this test the except part of - # is_binary_string(value) in value_to_display - if PY2: - assert value_to_display([u'Э'.encode('cp1251')]) == "['\xdd']" - def test_ellipses(tmpdir): """ @@ -314,11 +307,9 @@ def test_get_type_string(): # Bools assert get_type_string(True) == 'bool' - # Numeric types (PY2 has long, which disappeared in PY3) - if not PY2: - expected = ['int', 'float', 'complex'] - numeric_types = [1, 1.5, 1 + 2j] - assert [get_type_string(t) for t in numeric_types] == expected + expected = ['int', 'float', 'complex'] + numeric_types = [1, 1.5, 1 + 2j] + assert [get_type_string(t) for t in numeric_types] == expected # Lists assert get_type_string([1, 2, 3]) == 'list' @@ -333,8 +324,7 @@ def test_get_type_string(): assert get_type_string((1, 2, 3)) == 'tuple' # Strings - if not PY2: - assert get_type_string('foo') == 'str' + assert get_type_string('foo') == 'str' # Numpy objects assert get_type_string(np.array([1, 2, 3])) == 'NDArray'