diff --git a/.github/workflows/linux-pip-tests.yml b/.github/workflows/linux-pip-tests.yml index 917b7a67..87e5db58 100644 --- a/.github/workflows/linux-pip-tests.yml +++ b/.github/workflows/linux-pip-tests.yml @@ -22,10 +22,11 @@ jobs: CI: True PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} RUNNER_OS: 'ubuntu' + USE_CONDA: 'false' strategy: fail-fast: false matrix: - PYTHON_VERSION: ['3.7', '3.8', '3.9'] + PYTHON_VERSION: ['3.8', '3.9', '3.10'] timeout-minutes: 20 steps: - name: Checkout branch @@ -55,10 +56,11 @@ jobs: - name: Run tests shell: bash -l {0} run: | - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv - name: Upload coverage to Codecov - if: matrix.PYTHON_VERSION == '3.8' - shell: bash -l {0} - run: codecov -t 02fa9892-fa7f-4cf8-ac3c-d54143ddc933 + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: false + verbose: true diff --git a/.github/workflows/linux-tests.yml b/.github/workflows/linux-tests.yml index a175ac24..f0900308 100644 --- a/.github/workflows/linux-tests.yml +++ b/.github/workflows/linux-tests.yml @@ -62,10 +62,11 @@ jobs: - name: Run tests shell: bash -l {0} run: | - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + xvfb-run --auto-servernum pytest spyder_kernels --color=yes --cov=spyder_kernels -vv - name: Upload coverage to Codecov - if: matrix.PYTHON_VERSION == '3.8' - shell: bash -l {0} - run: codecov -t 02fa9892-fa7f-4cf8-ac3c-d54143ddc933 + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: false + verbose: true diff --git a/.github/workflows/macos-tests.yml b/.github/workflows/macos-tests.yml index b2ce6ef7..819f9a1e 100644 --- a/.github/workflows/macos-tests.yml +++ b/.github/workflows/macos-tests.yml @@ -22,6 +22,7 @@ jobs: CI: True PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} RUNNER_OS: 'macos' + USE_CONDA: 'true' strategy: fail-fast: false matrix: @@ -57,6 +58,11 @@ jobs: - name: Run tests shell: bash -l {0} run: | - pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv + pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + pytest spyder_kernels --color=yes --cov=spyder_kernels -vv + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: false + verbose: true diff --git a/.github/workflows/windows-tests.yml b/.github/workflows/windows-tests.yml index 8b08ef55..b402cb71 100644 --- a/.github/workflows/windows-tests.yml +++ b/.github/workflows/windows-tests.yml @@ -22,6 +22,7 @@ jobs: CI: True PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }} RUNNER_OS: 'windows' + USE_CONDA: 'true' strategy: fail-fast: false matrix: @@ -57,6 +58,11 @@ jobs: - name: Run tests shell: bash -l {0} run: | - pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - pytest spyder_kernels --color=yes --cov=spyder_kernels -x -vv || \ - pytest spyder_kernels --cov=spyder_kernels -x -vv + pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + pytest spyder_kernels --color=yes --cov=spyder_kernels -vv || \ + pytest spyder_kernels --cov=spyder_kernels -vv + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + fail_ci_if_error: false + verbose: true diff --git a/CHANGELOG.md b/CHANGELOG.md index a0b46eab..7484e8f6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,72 @@ # History of changes +## Version 3.0.0b1 (2023-06-14) + +### Issues Closed + +* [Issue 425](https://github.com/spyder-ide/spyder-kernels/issues/425) - Possible minor issues related to post mortem debugging ([PR 444](https://github.com/spyder-ide/spyder-kernels/pull/444) by [@impact27](https://github.com/impact27)) +* [Issue 340](https://github.com/spyder-ide/spyder-kernels/issues/340) - Drop support for Python 2 ([PR 341](https://github.com/spyder-ide/spyder-kernels/pull/341) by [@impact27](https://github.com/impact27)) + +In this release 2 issues were closed. + +### Pull Requests Merged + +* [PR 456](https://github.com/spyder-ide/spyder-kernels/pull/456) - PR: Remove unnecessary code for old IPykernel versions in our tests, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 453](https://github.com/spyder-ide/spyder-kernels/pull/453) - PR: Move code that loads and saves HDF5 and DICOM files from Spyder, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 447](https://github.com/spyder-ide/spyder-kernels/pull/447) - PR: Create magics for run|debug file|cell, by [@impact27](https://github.com/impact27) +* [PR 446](https://github.com/spyder-ide/spyder-kernels/pull/446) - PR: Make call to interrupt children processes work for IPykernel greater than 6.21.2, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 444](https://github.com/spyder-ide/spyder-kernels/pull/444) - PR: Fix post mortem for debugging, by [@impact27](https://github.com/impact27) ([425](https://github.com/spyder-ide/spyder-kernels/issues/425)) +* [PR 443](https://github.com/spyder-ide/spyder-kernels/pull/443) - PR: Fix small typo in Readme, by [@davidbrochart](https://github.com/davidbrochart) +* [PR 437](https://github.com/spyder-ide/spyder-kernels/pull/437) - PR: Remove imports from __future__, by [@oscargus](https://github.com/oscargus) +* [PR 421](https://github.com/spyder-ide/spyder-kernels/pull/421) - PR: Update variable explorer from the kernel, by [@impact27](https://github.com/impact27) +* [PR 417](https://github.com/spyder-ide/spyder-kernels/pull/417) - PR: Fix error in `globalsfilter`, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 412](https://github.com/spyder-ide/spyder-kernels/pull/412) - PR: Use temporary file for faulthandler and make sure iopub is open for comms, by [@impact27](https://github.com/impact27) +* [PR 409](https://github.com/spyder-ide/spyder-kernels/pull/409) - PR: Add Python executable as part of kernel info, by [@impact27](https://github.com/impact27) +* [PR 408](https://github.com/spyder-ide/spyder-kernels/pull/408) - PR: Expose package version in SpyderShell, by [@impact27](https://github.com/impact27) +* [PR 403](https://github.com/spyder-ide/spyder-kernels/pull/403) - PR: Make debugger faster by avoiding unnecessary comm messages, by [@impact27](https://github.com/impact27) +* [PR 401](https://github.com/spyder-ide/spyder-kernels/pull/401) - PR: Wait for connection file to be written, by [@impact27](https://github.com/impact27) +* [PR 400](https://github.com/spyder-ide/spyder-kernels/pull/400) - PR: Use `execute_interactive` to print errors during tests, by [@impact27](https://github.com/impact27) +* [PR 397](https://github.com/spyder-ide/spyder-kernels/pull/397) - PR: Remove Python 2 code introduced when merging PR #395, by [@impact27](https://github.com/impact27) +* [PR 396](https://github.com/spyder-ide/spyder-kernels/pull/396) - PR: Add handlers to interrupt executions and enter the debugger after that, by [@impact27](https://github.com/impact27) +* [PR 390](https://github.com/spyder-ide/spyder-kernels/pull/390) - PR: Filter comm socket thread, by [@impact27](https://github.com/impact27) +* [PR 387](https://github.com/spyder-ide/spyder-kernels/pull/387) - PR: Minor changes to finish the migration to Python 3, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 366](https://github.com/spyder-ide/spyder-kernels/pull/366) - PR: Print warning when using `global` in an empty namespace, by [@impact27](https://github.com/impact27) +* [PR 341](https://github.com/spyder-ide/spyder-kernels/pull/341) - PR: Drop support for Python 2, by [@impact27](https://github.com/impact27) ([340](https://github.com/spyder-ide/spyder-kernels/issues/340)) +* [PR 339](https://github.com/spyder-ide/spyder-kernels/pull/339) - PR: Use control channel for comms, by [@impact27](https://github.com/impact27) +* [PR 286](https://github.com/spyder-ide/spyder-kernels/pull/286) - PR: Improve and refactor the way we run and debug code, by [@impact27](https://github.com/impact27) +* [PR 257](https://github.com/spyder-ide/spyder-kernels/pull/257) - PR: Notify frontend of Matplotlib backend change, by [@impact27](https://github.com/impact27) +* [PR 171](https://github.com/spyder-ide/spyder-kernels/pull/171) - PR: Publish Pdb stack frames to Spyder, by [@impact27](https://github.com/impact27) + +In this release 25 pull requests were closed. + + +---- + + +## Version 2.4.3 (2023-04-02) + +### Issues Closed + +* [Issue 440](https://github.com/spyder-ide/spyder-kernels/issues/440) - distutils and LooseVersion deprecation ([PR 450](https://github.com/spyder-ide/spyder-kernels/pull/450) by [@ccordoba12](https://github.com/ccordoba12)) + +In this release 1 issue was closed. + +### Pull Requests Merged + +* [PR 452](https://github.com/spyder-ide/spyder-kernels/pull/452) - PR: Fix error when executing empty Python script, by [@rear1019](https://github.com/rear1019) +* [PR 450](https://github.com/spyder-ide/spyder-kernels/pull/450) - PR: Remove usage of `distutils.LooseVersion`, by [@ccordoba12](https://github.com/ccordoba12) ([440](https://github.com/spyder-ide/spyder-kernels/issues/440)) +* [PR 449](https://github.com/spyder-ide/spyder-kernels/pull/449) - PR: Add support for Jupyter-client 8, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 448](https://github.com/spyder-ide/spyder-kernels/pull/448) - PR: Skip IPython versions that give buggy code completions, by [@ccordoba12](https://github.com/ccordoba12) +* [PR 442](https://github.com/spyder-ide/spyder-kernels/pull/442) - PR: Add FreeBSD to `test_user_sitepackages_in_pathlist`, by [@rhurlin](https://github.com/rhurlin) +* [PR 434](https://github.com/spyder-ide/spyder-kernels/pull/434) - PR: Use `allow_pickle=True` when loading Numpy arrays, by [@nkleinbaer](https://github.com/nkleinbaer) +* [PR 430](https://github.com/spyder-ide/spyder-kernels/pull/430) - PR: Inform GUI about position of exception in post mortem debugging, by [@rear1019](https://github.com/rear1019) + +In this release 7 pull requests were closed. + + +---- + + ## Version 2.4.2 (2023-01-17) ### Issues Closed diff --git a/README.md b/README.md index 410e2fd5..b6307c10 100644 --- a/README.md +++ b/README.md @@ -8,7 +8,7 @@ Package that provides Jupyter kernels for use with the consoles of Spyder, the Scientific Python Development Environment. -These kernels can launched either through Spyder itself or in an independent +These kernels can be launched either through Spyder itself or in an independent Python session, and allow for interactive or file-based execution of Python code inside Spyder. diff --git a/requirements/posix.txt b/requirements/posix.txt index baf1d24e..2503a2c2 100644 --- a/requirements/posix.txt +++ b/requirements/posix.txt @@ -1,7 +1,7 @@ cloudpickle ipykernel>=6.16.1,<7 ipython>=7.31.1,<9 -jupyter_client>=7.4.9,<8 +jupyter_client>=7.4.9,<9 pyzmq>=22.1.0 wurlitzer>=1.0.3 pyxdg>=0.26 diff --git a/requirements/tests.txt b/requirements/tests.txt index 47d0fb60..42ef0167 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,4 +1,3 @@ -codecov cython dask flaky @@ -12,3 +11,5 @@ scipy xarray pillow django +h5py +pydicom diff --git a/requirements/windows.txt b/requirements/windows.txt index 28268954..da1002d0 100644 --- a/requirements/windows.txt +++ b/requirements/windows.txt @@ -1,5 +1,5 @@ cloudpickle ipykernel>=6.16.1,<7 ipython>=7.31.1,<9 -jupyter_client>=7.4.9,<8 +jupyter_client>=7.4.9,<9 pyzmq>=22.1.0 diff --git a/setup.py b/setup.py index aeda98fb..a0b7be2a 100644 --- a/setup.py +++ b/setup.py @@ -38,16 +38,14 @@ def get_version(module='spyder_kernels'): REQUIREMENTS = [ 'cloudpickle', 'ipykernel>=6.16.1,<7', - 'ipython>=7.31.1,<9', - 'jupyter-client>=7.4.9,<8', - 'packaging', + 'ipython>=7.31.1,<9,!=8.8.0,!=8.9.0,!=8.10.0,!=8.11.0,!=8.12.0,!=8.12.1', + 'jupyter-client>=7.4.9,<9', 'pyzmq>=22.1.0', 'wurlitzer>=1.0.3;platform_system!="Windows"', 'pyxdg>=0.26;platform_system=="Linux"', ] TEST_REQUIREMENTS = [ - 'codecov', 'cython', 'dask[distributed]', 'flaky', @@ -61,6 +59,8 @@ def get_version(module='spyder_kernels'): 'xarray', 'pillow', 'django', + 'h5py', + 'pydicom' ] setup( diff --git a/spyder_kernels/console/kernel.py b/spyder_kernels/console/kernel.py index fc620b62..d1a585a3 100644 --- a/spyder_kernels/console/kernel.py +++ b/spyder_kernels/console/kernel.py @@ -97,7 +97,6 @@ def __init__(self, *args, **kwargs): self.namespace_view_settings = {} self._mpl_backend_error = None - self._running_namespace = None self.faulthandler_handle = None self._cwd_initialised = False @@ -326,7 +325,7 @@ def get_namespace_view(self, frame=None): settings = self.namespace_view_settings if settings: - ns = self._get_current_namespace(frame=frame) + ns = self.shell._get_current_namespace(frame=frame) view = make_remote_view(ns, settings, EXCLUDED_NAMES) return view else: @@ -339,7 +338,7 @@ def get_var_properties(self): """ settings = self.namespace_view_settings if settings: - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace() data = get_remote_data(ns, settings, mode='editable', more_excluded_names=EXCLUDED_NAMES) @@ -364,23 +363,23 @@ def get_var_properties(self): def get_value(self, name): """Get the value of a variable""" - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace() return ns[name] def set_value(self, name, value): """Set the value of a variable""" - ns = self._get_reference_namespace(name) + ns = self.shell._get_reference_namespace(name) ns[name] = value self.log.debug(ns) def remove_value(self, name): """Remove a variable""" - ns = self._get_reference_namespace(name) + ns = self.shell._get_reference_namespace(name) ns.pop(name) def copy_value(self, orig_name, new_name): """Copy a variable""" - ns = self._get_reference_namespace(orig_name) + ns = self.shell._get_reference_namespace(orig_name) ns[new_name] = ns[orig_name] def load_data(self, filename, ext, overwrite=False): @@ -421,7 +420,7 @@ def load_data(self, filename, ext, overwrite=False): def save_namespace(self, filename): """Save namespace into filename""" - ns = self._get_current_namespace() + ns = self.shell._get_current_namespace() settings = self.namespace_view_settings data = get_remote_data(ns, settings, mode='picklable', more_excluded_names=EXCLUDED_NAMES).copy() @@ -476,7 +475,7 @@ def is_defined(self, obj, force_import=False): """Return True if object is defined in current namespace""" from spyder_kernels.utils.dochelpers import isdefined - ns = self._get_current_namespace(with_magics=True) + ns = self.shell._get_current_namespace(with_magics=True) return isdefined(obj, force_import=force_import, namespace=ns) def get_doc(self, objtxt): @@ -531,8 +530,7 @@ def get_mpl_interactive_backend(self): # Detect if there is a graphical framework running by checking the # eventloop function attached to the kernel.eventloop attribute (see # `ipykernel.eventloops.enable_gui` for context). - from IPython.core.getipython import get_ipython - loop_func = get_ipython().kernel.eventloop + loop_func = self.eventloop if loop_func is not None: if loop_func == eventloops.loop_tk: @@ -584,8 +582,7 @@ def set_mpl_inline_bbox_inches(self, bbox_inches): The change is done by updating the 'print_figure_kwargs' config dict. """ - from IPython.core.getipython import get_ipython - config = get_ipython().kernel.config + config = self.config inline_config = ( config['InlineBackend'] if 'InlineBackend' in config else {}) print_figure_kwargs = ( @@ -695,57 +692,6 @@ def update_syspath(self, path_dict, new_path_dict): # -- Private API --------------------------------------------------- # --- For the Variable Explorer - def _get_current_namespace(self, with_magics=False, frame=None): - """ - Return current namespace - - This is globals() if not debugging, or a dictionary containing - both locals() and globals() for current frame when debugging - """ - if frame is not None: - ns = frame.f_globals.copy() - if self.shell._pdb_frame is frame: - ns.update(self.shell._pdb_locals) - else: - ns.update(frame.f_locals) - return ns - - ns = {} - if self.shell.is_debugging() and self.shell.pdb_session.curframe: - # Stopped at a pdb prompt - ns.update(self.shell.user_ns) - ns.update(self.shell._pdb_locals) - else: - # Give access to the running namespace if there is one - if self._running_namespace is None: - ns.update(self.shell.user_ns) - else: - # This is true when a file is executing. - running_globals, running_locals = self._running_namespace - ns.update(running_globals) - if running_locals is not None: - ns.update(running_locals) - - # Add magics to ns so we can show help about them on the Help - # plugin - if with_magics: - line_magics = self.shell.magics_manager.magics['line'] - cell_magics = self.shell.magics_manager.magics['cell'] - ns.update(line_magics) - ns.update(cell_magics) - return ns - - def _get_reference_namespace(self, name): - """ - Return namespace where reference name is defined - - It returns the globals() if reference has not yet been defined - """ - lcls = self.shell._pdb_locals - if name in lcls: - return lcls - return self.shell.user_ns - def _get_len(self, var): """Return sequence length""" try: @@ -838,7 +784,7 @@ def _eval(self, text): """ assert isinstance(text, str) - ns = self._get_current_namespace(with_magics=True) + ns = self.shell._get_current_namespace(with_magics=True) try: return eval(text, ns), True except: @@ -855,7 +801,6 @@ def _set_mpl_backend(self, backend, pylab=False): namespace from numpy and matplotlib """ import traceback - from IPython.core.getipython import get_ipython # Don't proceed further if there's any error while importing Matplotlib try: @@ -879,7 +824,7 @@ def _set_mpl_backend(self, backend, pylab=False): matplotlib.rcParams['backend'] = 'Agg' # Set the backend - get_ipython().run_line_magic(magic, backend) + self.shell.run_line_magic(magic, backend) except RuntimeError as err: # This catches errors generated by ipykernel when # trying to set a backend. See issue 5541 @@ -921,13 +866,12 @@ def _set_config_option(self, option, value): option: config option, for example 'InlineBackend.figure_format'. value: value of the option, for example 'SVG', 'Retina', etc. """ - from IPython.core.getipython import get_ipython try: base_config = "{option} = " value_line = ( "'{value}'" if isinstance(value, str) else "{value}") config_line = base_config + value_line - get_ipython().run_line_magic( + self.shell.run_line_magic( 'config', config_line.format(option=option, value=value)) except Exception: @@ -954,21 +898,19 @@ def set_sympy_forecolor(self, background_color='dark'): if os.environ.get('SPY_SYMPY_O') == 'True': try: from sympy import init_printing - from IPython.core.getipython import get_ipython if background_color == 'dark': - init_printing(forecolor='White', ip=get_ipython()) + init_printing(forecolor='White', ip=self.shell) elif background_color == 'light': - init_printing(forecolor='Black', ip=get_ipython()) + init_printing(forecolor='Black', ip=self.shell) except Exception: pass # --- Others def _load_autoreload_magic(self): """Load %autoreload magic.""" - from IPython.core.getipython import get_ipython try: - get_ipython().run_line_magic('reload_ext', 'autoreload') - get_ipython().run_line_magic('autoreload', '2') + self.shell.run_line_magic('reload_ext', 'autoreload') + self.shell.run_line_magic('autoreload', '2') except Exception: pass @@ -976,12 +918,11 @@ def _load_wurlitzer(self): """Load wurlitzer extension.""" # Wurlitzer has no effect on Windows if not os.name == 'nt': - from IPython.core.getipython import get_ipython # Enclose this in a try/except because if it fails the # console will be totally unusable. # Fixes spyder-ide/spyder#8668 try: - get_ipython().run_line_magic('reload_ext', 'wurlitzer') + self.shell.run_line_magic('reload_ext', 'wurlitzer') except Exception: pass diff --git a/spyder_kernels/console/shell.py b/spyder_kernels/console/shell.py index cc55c78d..6dbbd759 100644 --- a/spyder_kernels/console/shell.py +++ b/spyder_kernels/console/shell.py @@ -24,7 +24,9 @@ # Local imports import spyder_kernels +from spyder_kernels.customize.namespace_manager import NamespaceManager from spyder_kernels.customize.spyderpdb import SpyderPdb +from spyder_kernels.customize.code_runner import SpyderCodeRunner from spyder_kernels.comms.frontendcomm import CommError from spyder_kernels.utils.mpl import automatic_backend @@ -45,8 +47,8 @@ class SpyderShell(ZMQInteractiveShell): ] def __init__(self, *args, **kwargs): - # Create _pdb_obj_stack before __init__ - self._pdb_obj_stack = [] + # Create _namespace_stack before __init__ + self._namespace_stack = [] self._request_pdb_stop = False self._pdb_conf = {} super(SpyderShell, self).__init__(*args, **kwargs) @@ -62,6 +64,11 @@ def __init__(self, *args, **kwargs): # register post_execute self.events.register('post_execute', self.do_post_execute) + def init_magics(self): + """Init magics""" + super().init_magics() + self.register_magics(SpyderCodeRunner) + def ask_exit(self): """Engage the exit actions.""" if self.active_eventloop not in [None, "inline"]: @@ -109,31 +116,21 @@ def set_pdb_configuration(self, pdb_conf): if self.pdb_session: setattr(self.pdb_session, key, pdb_conf[key]) - def get_local_scope(self, stack_depth): - """Get local scope at given frame depth.""" - frame = sys._getframe(stack_depth + 1) - if self._pdb_frame is frame: - # Avoid calling f_locals on _pdb_frame - return self.pdb_session.curframe_locals - else: - return frame.f_locals - - def get_global_scope(self, stack_depth): - """Get global scope at given frame depth.""" - frame = sys._getframe(stack_depth + 1) - return frame.f_globals - def is_debugging(self): """ Check if we are currently debugging. """ - return bool(self._pdb_frame) + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb) and session.curframe is not None: + return True + return False @property def pdb_session(self): """Get current pdb session.""" - if len(self._pdb_obj_stack) > 0: - return self._pdb_obj_stack[-1] + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb): + return session return None def add_pdb_session(self, pdb_obj): @@ -141,7 +138,7 @@ def add_pdb_session(self, pdb_obj): if self.pdb_session == pdb_obj: # Already added return - self._pdb_obj_stack.append(pdb_obj) + self._namespace_stack.append(pdb_obj) # Set config to pdb obj self.set_pdb_configuration(self._pdb_conf) @@ -151,42 +148,109 @@ def remove_pdb_session(self, pdb_obj): if self.pdb_session != pdb_obj: # Already removed return - self._pdb_obj_stack.pop() + self._namespace_stack.pop() if self.pdb_session: # Set config to newly active pdb obj self.set_pdb_configuration(self._pdb_conf) + def add_namespace_manager(self, ns_manager): + """Add namespace manager to stack.""" + self._namespace_stack.append(ns_manager) + + def remove_namespace_manager(self, ns_manager): + """Remove namespace manager.""" + if self._namespace_stack[-1] != ns_manager: + logger.debug("The namespace stack is inconsistent.") + return + self._namespace_stack.pop() + + def get_local_scope(self, stack_depth): + """ + Get local scope at a given frame depth. + + Needed for magics that use "needs_local_scope" such as timeit + """ + frame = sys._getframe(stack_depth + 1) + return self.context_locals(frame) + + def context_locals(self, frame=None): + """ + Get context locals. + + If frame is not None, make sure frame.f_locals is not registered in a + debugger and return frame.f_locals + """ + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb) and session.curframe is not None: + if frame is None or frame == session.curframe: + return session.curframe_locals + elif frame is None and isinstance(session, NamespaceManager): + return session.ns_locals + + if frame is not None: + return frame.f_locals + + return None + @property def _pdb_frame(self): """Return current Pdb frame if there is any""" if self.pdb_session is not None: return self.pdb_session.curframe - @property - def _pdb_locals(self): - """ - Return current Pdb frame locals if available. Otherwise - return an empty dictionary - """ - if self._pdb_frame is not None: - return self.pdb_session.curframe_locals - else: - return {} - @property def user_ns(self): """Get the current namespace.""" - if self._pdb_frame is not None: - return self._pdb_frame.f_globals - else: - return self.__user_ns + for session in self._namespace_stack[::-1]: + if isinstance(session, SpyderPdb) and session.curframe is not None: + # Return first debugging namespace + return session.curframe.f_globals + elif isinstance(session, NamespaceManager): + return session.ns_globals + + return self.__user_ns @user_ns.setter def user_ns(self, namespace): """Set user_ns.""" self.__user_ns = namespace + def _get_current_namespace(self, with_magics=False, frame=None): + """Return a copy of the current namespace.""" + if frame is not None: + ns = frame.f_globals.copy() + ns.update(self.context_locals(frame)) + return ns + + ns = {} + ns.update(self.user_ns) + context_locals = self.context_locals() + if context_locals: + ns.update(context_locals) + + # Add magics to ns so we can show help about them on the Help + # plugin + if with_magics: + line_magics = self.magics_manager.magics['line'] + cell_magics = self.magics_manager.magics['cell'] + ns.update(line_magics) + ns.update(cell_magics) + + return ns + + def _get_reference_namespace(self, name): + """ + Return namespace where reference name is defined + + It returns the user namespace if name has not yet been defined. + """ + lcls = self.context_locals() + if lcls and name in lcls: + + return lcls + return self.user_ns + def showtraceback(self, exc_tuple=None, filename=None, tb_offset=None, exception_only=False, running_compiled_code=False): """Display the exception that just occurred.""" @@ -221,7 +285,13 @@ def raise_interrupt_signal(self): self.kernel.log.error( "Interrupt message not supported on Windows") else: - self.kernel._send_interupt_children() + # This is necessary to make the call below work for IPykernel + # versions equal or less than 6.21.2 and greater than it. + # See ipython/ipykernel#1101 + if hasattr(self.kernel, '_send_interupt_children'): + self.kernel._send_interupt_children() + else: + self.kernel._send_interrupt_children() def request_pdb_stop(self): """Request pdb to stop at the next possible position.""" diff --git a/spyder_kernels/console/start.py b/spyder_kernels/console/start.py index f5e4b122..708319c0 100644 --- a/spyder_kernels/console/start.py +++ b/spyder_kernels/console/start.py @@ -17,7 +17,6 @@ import site # Third-party imports -from packaging.version import parse from traitlets import DottedObjectName # Local imports @@ -105,14 +104,6 @@ def kernel_config(): clear_argv = "import sys; sys.argv = ['']; del sys" spy_cfg.IPKernelApp.exec_lines = [clear_argv] - # Set our runfile in builtins here to prevent other packages shadowing it. - # This started to be a problem since IPykernel 6.3.0. - spy_cfg.IPKernelApp.exec_lines.append( - "import builtins; " - "builtins.runfile = builtins.spyder_runfile; " - "del builtins.spyder_runfile; del builtins" - ) - # Prevent other libraries to change the breakpoint builtin. # This started to be a problem since IPykernel 6.3.0. if sys.version_info[0:2] >= (3, 7): @@ -259,7 +250,7 @@ def varexp(line): except: import matplotlib.pyplot as pyplot pyplot.figure(); - getattr(pyplot, funcname[2:])(ip.kernel._get_current_namespace()[name]) + getattr(pyplot, funcname[2:])(ip._get_current_namespace()[name]) pyplot.show() diff --git a/spyder_kernels/console/tests/test_console_kernel.py b/spyder_kernels/console/tests/test_console_kernel.py index 7469610c..d7efae21 100644 --- a/spyder_kernels/console/tests/test_console_kernel.py +++ b/spyder_kernels/console/tests/test_console_kernel.py @@ -11,6 +11,7 @@ # Standard library imports import ast +import asyncio import os import os.path as osp from textwrap import dedent @@ -23,7 +24,6 @@ from collections import namedtuple # Test imports -import ipykernel import pytest from flaky import flaky from jupyter_core import paths @@ -37,17 +37,10 @@ from spyder_kernels.customize.spyderpdb import SpyderPdb from spyder_kernels.comms.commbase import CommBase -# For ipykernel 6 -try: - import asyncio -except ImportError: - pass - # ============================================================================= # Constants and utility functions # ============================================================================= FILES_PATH = os.path.dirname(os.path.realpath(__file__)) -IPYKERNEL_6 = ipykernel.__version__[0] >= '6' TIMEOUT = 15 SETUP_TIMEOUT = 60 @@ -239,10 +232,7 @@ def kernel(request): # Teardown def reset_kernel(): - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('reset -f', True)) - else: - kernel.do_execute('reset -f', True) + asyncio.run(kernel.do_execute('reset -f', True)) request.addfinalizer(reset_kernel) return kernel @@ -287,10 +277,7 @@ def test_get_namespace_view(kernel): """ Test the namespace view of the kernel. """ - if IPYKERNEL_6: - execute = asyncio.run(kernel.do_execute('a = 1', True)) - else: - execute = kernel.do_execute('a = 1', True) + execute = asyncio.run(kernel.do_execute('a = 1', True)) nsview = repr(kernel.get_namespace_view()) assert "'a':" in nsview @@ -305,10 +292,7 @@ def test_get_var_properties(kernel): """ Test the properties fo the variables in the namespace. """ - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 1', True)) - else: - kernel.do_execute('a = 1', True) + asyncio.run(kernel.do_execute('a = 1', True)) var_properties = repr(kernel.get_var_properties()) assert "'a'" in var_properties @@ -326,10 +310,7 @@ def test_get_var_properties(kernel): def test_get_value(kernel): """Test getting the value of a variable.""" name = 'a' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute("a = 124", True)) - else: - kernel.do_execute("a = 124", True) + asyncio.run(kernel.do_execute("a = 124", True)) # Check data type send assert kernel.get_value(name) == 124 @@ -338,10 +319,7 @@ def test_get_value(kernel): def test_set_value(kernel): """Test setting the value of a variable.""" name = 'a' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 0', True)) - else: - kernel.do_execute('a = 0', True) + asyncio.run(kernel.do_execute('a = 0', True)) value = 10 kernel.set_value(name, value) log_text = get_log_text(kernel) @@ -355,10 +333,7 @@ def test_set_value(kernel): def test_remove_value(kernel): """Test the removal of a variable.""" name = 'a' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 1', True)) - else: - kernel.do_execute('a = 1', True) + asyncio.run(kernel.do_execute('a = 1', True)) var_properties = repr(kernel.get_var_properties()) assert "'a'" in var_properties @@ -380,10 +355,7 @@ def test_copy_value(kernel): """Test the copy of a variable.""" orig_name = 'a' new_name = 'b' - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('a = 1', True)) - else: - kernel.do_execute('a = 1', True) + asyncio.run(kernel.do_execute('a = 1', True)) var_properties = repr(kernel.get_var_properties()) assert "'a'" in var_properties @@ -419,11 +391,7 @@ def test_load_npz_data(kernel, load): namespace_file = osp.join(FILES_PATH, 'load_data.npz') extention = '.npz' overwrite, execute, variables = load - - if IPYKERNEL_6: - asyncio.run(kernel.do_execute(execute, True)) - else: - kernel.do_execute(execute, True) + asyncio.run(kernel.do_execute(execute, True)) kernel.load_data(namespace_file, extention, overwrite=overwrite) for var, value in variables.items(): @@ -451,11 +419,7 @@ def test_load_data(kernel): def test_save_namespace(kernel): """Test saving the namespace into filename.""" namespace_file = osp.join(FILES_PATH, 'save_data.spydata') - - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('b = 1', True)) - else: - kernel.do_execute('b = 1', True) + asyncio.run(kernel.do_execute('b = 1', True)) kernel.save_namespace(namespace_file) assert osp.isfile(namespace_file) @@ -499,11 +463,7 @@ def test_output_from_c_libraries(kernel, capsys): # With Wurlitzer we have the expected output kernel._load_wurlitzer() - - if IPYKERNEL_6: - asyncio.run(kernel.do_execute(code, True)) - else: - kernel.do_execute(code, True) + asyncio.run(kernel.do_execute(code, True)) captured = capsys.readouterr() assert captured.out == "Hello from C\n" @@ -559,7 +519,7 @@ def f(x): # Run code client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT) + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `result` variable is defined client.inspect('result') @@ -603,7 +563,7 @@ def myFunc(i): # Run code client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT) + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `result` variable is defined client.inspect('result') @@ -619,6 +579,9 @@ def myFunc(i): @pytest.mark.skipif( sys.platform == 'darwin' and sys.version_info[:2] == (3, 8), reason="Fails on Mac with Python 3.8") +@pytest.mark.skipif( + os.environ.get('USE_CONDA') != 'true', + reason="Doesn't work with pip packages") def test_dask_multiprocessing(tmpdir): """ Test that dask multiprocessing works. @@ -645,10 +608,10 @@ def test_dask_multiprocessing(tmpdir): # Run code two times client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT) + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT) + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `x` variable is defined client.inspect('x') @@ -687,8 +650,8 @@ def test_runfile(tmpdir): u.write(code) # Run code file `d` to define `result` even after an error - client.execute_interactive("runfile(r'{}', current_namespace=False)" - .format(str(d)), timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(d))), timeout=TIMEOUT) # Verify that `result` is defined in the current namespace client.inspect('result') @@ -699,8 +662,8 @@ def test_runfile(tmpdir): assert content['found'] # Run code file `u` without current namespace - client.execute_interactive("runfile(r'{}', current_namespace=False)" - .format(str(u)), timeout=TIMEOUT) + client.execute_interactive( + "%runfile {}".format(repr(str(u))), timeout=TIMEOUT) # Verify that the variable `result2` is defined client.inspect('result2') @@ -711,8 +674,8 @@ def test_runfile(tmpdir): assert content['found'] # Run code file `u` with current namespace - msg = client.execute_interactive("runfile(r'{}', current_namespace=True)" - .format(str(u)), timeout=TIMEOUT) + msg = client.execute_interactive("%runfile {} --current-namespace" + .format(repr(str(u))), timeout=TIMEOUT) content = msg['content'] # Verify that the variable `result3` is defined @@ -835,7 +798,7 @@ def test_turtle_launch(tmpdir): # Run code client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT) + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `tess` variable is defined client.inspect('tess') @@ -853,7 +816,7 @@ def test_turtle_launch(tmpdir): # Run code again client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT) + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT) # Verify that the `a` variable is defined client.inspect('a') @@ -889,10 +852,7 @@ def test_do_complete(kernel): """ Check do complete works in normal and debugging mode. """ - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('abba = 1', True)) - else: - kernel.do_execute('abba = 1', True) + asyncio.run(kernel.do_execute('abba = 1', True)) assert kernel.get_value('abba') == 1 match = kernel.do_complete('ab', 2) assert 'abba' in match['matches'] @@ -901,7 +861,7 @@ def test_do_complete(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.completenames = lambda *ignore: ['baba'] - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] match = kernel.do_complete('ba', 2) assert 'baba' in match['matches'] pdb_obj.curframe = None @@ -915,17 +875,11 @@ def test_callables_and_modules(kernel, exclude_callables_and_modules, Tests that callables and modules are in the namespace view only when the right options are passed to the kernel. """ - if IPYKERNEL_6: - asyncio.run(kernel.do_execute('import numpy', True)) - asyncio.run(kernel.do_execute('a = 10', True)) - asyncio.run(kernel.do_execute('def f(x): return x', True)) - else: - kernel.do_execute('import numpy', True) - kernel.do_execute('a = 10', True) - kernel.do_execute('def f(x): return x', True) + asyncio.run(kernel.do_execute('import numpy', True)) + asyncio.run(kernel.do_execute('a = 10', True)) + asyncio.run(kernel.do_execute('def f(x): return x', True)) settings = kernel.namespace_view_settings - settings['exclude_callables_and_modules'] = exclude_callables_and_modules settings['exclude_unsupported'] = exclude_unsupported nsview = kernel.get_namespace_view() @@ -959,7 +913,7 @@ def test_comprehensions_with_locals_in_pdb(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] # Create a local variable. kernel.shell.pdb_session.default('zz = 10') @@ -985,7 +939,7 @@ def test_comprehensions_with_locals_in_pdb_2(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] # Create a local variable. kernel.shell.pdb_session.default('aa = [1, 2]') @@ -1011,7 +965,7 @@ def test_namespaces_in_pdb(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] # Check adding something to globals works pdb_obj.default("globals()['test2'] = 0") @@ -1054,7 +1008,7 @@ def test_functions_with_locals_in_pdb(kernel): Frame = namedtuple("Frame", ["f_globals"]) pdb_obj.curframe = Frame(f_globals=kernel.shell.user_ns) pdb_obj.curframe_locals = kernel.shell.user_ns - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] # Create a local function. kernel.shell.pdb_session.default( @@ -1086,7 +1040,7 @@ def test_functions_with_locals_in_pdb_2(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] # Create a local function. kernel.shell.pdb_session.default( @@ -1123,7 +1077,7 @@ def test_locals_globals_in_pdb(kernel): pdb_obj = SpyderPdb() pdb_obj.curframe = inspect.currentframe() pdb_obj.curframe_locals = pdb_obj.curframe.f_locals - kernel.shell._pdb_obj_stack = [pdb_obj] + kernel.shell._namespace_stack = [pdb_obj] assert kernel.get_value('a') == 1 @@ -1160,11 +1114,12 @@ def test_locals_globals_in_pdb(kernel): @flaky(max_runs=3) @pytest.mark.parametrize("backend", [None, 'inline', 'tk', 'qt5']) @pytest.mark.skipif( - not bool(os.environ.get('USE_CONDA')), + os.environ.get('USE_CONDA') != 'true', reason="Doesn't work with pip packages") @pytest.mark.skipif( sys.version_info[:2] < (3, 9), reason="Too flaky in Python 3.7/8 and doesn't work in older versions") +@pytest.mark.skipif(sys.platform == 'darwin', reason="Fails on Mac") def test_get_interactive_backend(backend): """ Test that we correctly get the interactive backend set in the kernel. @@ -1228,13 +1183,13 @@ def check_found(msg): found = True # Run code in current namespace - client.execute_interactive("runfile(r'{}', current_namespace=True)".format( - str(p)), timeout=TIMEOUT, output_hook=check_found) + client.execute_interactive("%runfile {} --current-namespace".format( + repr(str(p))), timeout=TIMEOUT, output_hook=check_found) assert not found # Run code in empty namespace client.execute_interactive( - "runfile(r'{}')".format(str(p)), timeout=TIMEOUT, + "%runfile {}".format(repr(str(p))), timeout=TIMEOUT, output_hook=check_found) assert found @@ -1254,7 +1209,7 @@ def test_debug_namespace(tmpdir): d.write('def func():\n bb = "hello"\n breakpoint()\nfunc()') # Run code file `d` - msg_id = client.execute("runfile(r'{}')".format(str(d))) + msg_id = client.execute("%runfile {}".format(repr(str(d)))) # make sure that 'bb' returns 'hello' client.get_stdin_msg(timeout=TIMEOUT) diff --git a/spyder_kernels/customize/code_runner.py b/spyder_kernels/customize/code_runner.py new file mode 100644 index 00000000..f4321eaa --- /dev/null +++ b/spyder_kernels/customize/code_runner.py @@ -0,0 +1,611 @@ +# +# Copyright (c) 2009- Spyder Kernels Contributors +# +# Licensed under the terms of the MIT License +# (see spyder_kernels/__init__.py for details) +# ----------------------------------------------------------------------------- + +""" +Spyder magics related to code execution, debugging, profiling, etc. +""" + +# Standard library imports +import ast +import bdb +import builtins +from contextlib import contextmanager +import io +import logging +import os +import pdb +import shlex +import sys +import time + +# Third-party imports +from IPython.core.inputtransformer2 import ( + TransformerManager, + leading_indent, + leading_empty_lines, +) +from IPython.core.magic import ( + needs_local_scope, + magics_class, + Magics, + line_magic, +) +from IPython.core import magic_arguments + +# Local imports +from spyder_kernels.comms.frontendcomm import frontend_request +from spyder_kernels.customize.namespace_manager import NamespaceManager +from spyder_kernels.customize.spyderpdb import SpyderPdb +from spyder_kernels.customize.umr import UserModuleReloader +from spyder_kernels.customize.utils import capture_last_Expr, canonic + + +# For logging +logger = logging.getLogger(__name__) + + +def runfile_arguments(func): + """Decorator to add runfile magic arguments to magic.""" + decorators = [ + magic_arguments.magic_arguments(), + magic_arguments.argument( + "filename", + help=""" + Filename to run + """, + ), + magic_arguments.argument( + "--args", + help=""" + Command line arguments (string) + """, + ), + magic_arguments.argument( + "--wdir", + const=True, + nargs="?", + help=""" + Working directory + """, + ), + magic_arguments.argument( + "--post-mortem", + action="store_true", + help=""" + Enter post-mortem mode on errors + """, + ), + magic_arguments.argument( + "--current-namespace", + action="store_true", + help=""" + Use current namespace + """, + ), + magic_arguments.argument( + "--namespace", + help=""" + Namespace to run the file in + """, + ) + ] + for dec in reversed(decorators): + func = dec(func) + return func + + +def runcell_arguments(func): + """Decorator to add runcell magic arguments to magic.""" + decorators = [ + magic_arguments.magic_arguments(), + magic_arguments.argument( + "--name", "-n", + help=""" + Cell name. + """, + ), + magic_arguments.argument( + "--index", "-i", + help=""" + Cell index. + """, + ), + magic_arguments.argument( + "filename", + nargs="?", + help=""" + Filename + """, + ), + magic_arguments.argument( + "--post-mortem", + action="store_true", + default=False, + help=""" + Enter post-mortem mode on errors + """, + ) + ] + for dec in reversed(decorators): + func = dec(func) + return func + + +@magics_class +class SpyderCodeRunner(Magics): + """ + Functions and magics related to code execution, debugging, profiling, etc. + """ + def __init__(self, *args, **kwargs): + self.show_global_msg = True + self.show_invalid_syntax_msg = True + self.umr = UserModuleReloader( + namelist=os.environ.get("SPY_UMR_NAMELIST", None) + ) + super().__init__(*args, **kwargs) + + @runfile_arguments + @needs_local_scope + @line_magic + def runfile(self, line, local_ns=None): + """ + Run a file. + """ + args, local_ns = self._parse_runfile_argstring( + self.runfile, line, local_ns) + + return self._exec_file( + filename=args.filename, + canonic_filename=args.canonic_filename, + args=args.args, + wdir=args.wdir, + post_mortem=args.post_mortem, + current_namespace=args.current_namespace, + context_globals=args.namespace, + context_locals=local_ns, + ) + + @runfile_arguments + @needs_local_scope + @line_magic + def debugfile(self, line, local_ns=None): + """ + Debug a file. + """ + args, local_ns = self._parse_runfile_argstring( + self.debugfile, line, local_ns) + + with self._debugger_exec(args.canonic_filename, True) as debug_exec: + self._exec_file( + filename=args.filename, + canonic_filename=args.canonic_filename, + args=args.args, + wdir=args.wdir, + current_namespace=args.current_namespace, + exec_fun=debug_exec, + post_mortem=args.post_mortem, + context_globals=args.namespace, + context_locals=local_ns, + ) + + @runcell_arguments + @needs_local_scope + @line_magic + def runcell(self, line, local_ns=None): + """ + Run a code cell from an editor. + """ + args = self._parse_runcell_argstring(self.runcell, line) + + return self._exec_cell( + cell_id=args.cell_id, + filename=args.filename, + canonic_filename=args.canonic_filename, + post_mortem=args.post_mortem, + context_globals=self.shell.user_ns, + context_locals=local_ns, + ) + + @runcell_arguments + @needs_local_scope + @line_magic + def debugcell(self, line, local_ns=None): + """ + Debug a code cell from an editor. + """ + args = self._parse_runcell_argstring(self.debugcell, line) + + with self._debugger_exec(args.canonic_filename, False) as debug_exec: + return self._exec_cell( + cell_id=args.cell_id, + filename=args.filename, + canonic_filename=args.canonic_filename, + exec_fun=debug_exec, + post_mortem=args.post_mortem, + context_globals=self.shell.user_ns, + context_locals=local_ns, + ) + + @contextmanager + def _debugger_exec(self, filename, continue_if_has_breakpoints): + """Get an exec function to use for debugging.""" + if not self.shell.is_debugging(): + debugger = SpyderPdb() + debugger.set_remote_filename(filename) + debugger.continue_if_has_breakpoints = continue_if_has_breakpoints + yield debugger.run + return + + session = self.shell.pdb_session + with session.recursive_debugger() as debugger: + debugger.set_remote_filename(filename) + debugger.continue_if_has_breakpoints = continue_if_has_breakpoints + + def debug_exec(code, glob, loc): + return sys.call_tracing(debugger.run, (code, glob, loc)) + + # Enter recursive debugger + yield debug_exec + + def _exec_file( + self, + filename=None, + args=None, + wdir=None, + post_mortem=False, + current_namespace=False, + exec_fun=None, + canonic_filename=None, + context_locals=None, + context_globals=None, + ): + """ + Execute a file. + """ + if self.umr.enabled: + self.umr.run() + if args is not None and not isinstance(args, str): + raise TypeError("expected a character buffer object") + + try: + file_code = self._get_file_code(filename, raise_exception=True) + except Exception: + print( + "This command failed to be executed because an error occurred " + "while trying to get the file code from Spyder's editor. " + "The error was:\n\n" + ) + self.shell.showtraceback(exception_only=True) + return + + # Here the remote filename has been used. It must now be valid locally. + filename = canonic_filename + + with NamespaceManager( + self.shell, + filename, + current_namespace=current_namespace, + file_code=file_code, + context_locals=context_locals, + context_globals=context_globals, + ) as (ns_globals, ns_locals): + sys.argv = [filename] + if args is not None: + # args are a sting in a string + for arg in shlex.split(args): + sys.argv.append(arg) + + if "multiprocessing" in sys.modules: + # See https://github.com/spyder-ide/spyder/issues/16696 + try: + sys.modules["__mp_main__"] = sys.modules["__main__"] + except Exception: + pass + + if wdir is not None: + if wdir is True: + # True means use file dir + wdir = os.path.dirname(filename) + if os.path.isdir(wdir): + os.chdir(wdir) + + # See https://github.com/spyder-ide/spyder/issues/13632 + if "multiprocessing.process" in sys.modules: + try: + import multiprocessing.process + multiprocessing.process.ORIGINAL_DIR = os.path.abspath(wdir) + except Exception: + pass + else: + print("Working directory {} doesn't exist.\n".format(wdir)) + + try: + if self.umr.has_cython: + # Cython files + with io.open(filename, encoding="utf-8") as f: + self.shell.run_cell_magic("cython", "", f.read()) + else: + self._exec_code( + file_code, + filename, + ns_globals, + ns_locals, + post_mortem=post_mortem, + exec_fun=exec_fun, + capture_last_expression=False, + global_warning=not current_namespace, + ) + finally: + sys.argv = [""] + + def _exec_cell( + self, + cell_id, + filename=None, + post_mortem=False, + exec_fun=None, + canonic_filename=None, + context_locals=None, + context_globals=None, + ): + """ + Execute a code cell. + """ + try: + # Get code from spyder + cell_code = frontend_request(blocking=True).run_cell(cell_id, filename) + except Exception: + print( + "This command failed to be executed because an error occurred " + "while trying to get the cell code from Spyder's editor." + "The error was:\n\n" + ) + self.shell.showtraceback(exception_only=True) + return + + if not cell_code or cell_code.strip() == "": + print("Nothing to execute, this cell is empty.\n") + return + + # Trigger `post_execute` to exit the additional pre-execution. + # See Spyder PR #7310. + self.shell.events.trigger("post_execute") + file_code = self._get_file_code(filename, save_all=False) + + # Here the remote filename has been used. It must now be valid locally. + filename = canonic_filename + + with NamespaceManager( + self.shell, + filename, + current_namespace=True, + file_code=file_code, + context_locals=context_locals, + context_globals=context_globals + ) as (ns_globals, ns_locals): + return self._exec_code( + cell_code, + filename, + ns_globals, + ns_locals, + post_mortem=post_mortem, + exec_fun=exec_fun, + capture_last_expression=True, + ) + + def _get_current_file_name(self): + """Get the current editor file name.""" + try: + return frontend_request(blocking=True).current_filename() + except Exception: + print( + "This command failed to be executed because an error occurred " + "while trying to get the current file name from Spyder's editor." + "The error was:\n\n" + ) + self.shell.showtraceback(exception_only=True) + return None + + def _get_file_code(self, filename, save_all=True, raise_exception=False): + """Retrieve the content of a file.""" + # Get code from spyder + try: + return frontend_request(blocking=True).get_file_code( + filename, save_all=save_all + ) + except Exception: + # Maybe this is a local file + try: + with open(filename, "r") as f: + return f.read() + except FileNotFoundError: + pass + + if raise_exception: + raise + + # Finally return None + return None + + def _exec_code( + self, + code, + filename, + ns_globals, + ns_locals=None, + post_mortem=False, + exec_fun=None, + capture_last_expression=False, + global_warning=False, + ): + """Execute code and display any exception.""" + if exec_fun is None: + exec_fun = exec + + is_ipython = os.path.splitext(filename)[1] == ".ipy" + try: + if not is_ipython: + # TODO: Remove the try-except and let the SyntaxError raise + # because there should't be IPython code in a Python file. + try: + ast_code = ast.parse( + self._transform_cell(code, indent_only=True) + ) + except SyntaxError as e: + try: + ast_code = ast.parse(self._transform_cell(code)) + except SyntaxError: + raise e from None + else: + if self.show_invalid_syntax_msg: + print( + "\nWARNING: This is not valid Python code. " + "If you want to use IPython magics, " + "flexible indentation, and prompt removal, " + "we recommend that you save this file with the " + ".ipy extension.\n" + ) + self.show_invalid_syntax_msg = False + else: + ast_code = ast.parse(self._transform_cell(code)) + + # Print warning for global + if global_warning and self.show_global_msg: + has_global = any( + isinstance(node, ast.Global) for node in ast.walk(ast_code) + ) + if has_global: + print( + "\nWARNING: This file contains a global statement, " + "but it is run in an empty namespace. " + "Consider using the " + "'Run in console's namespace instead of an empty one' " + "option, that you can find in the menu 'Run > " + "Configuration per file', if you want to capture the " + "namespace.\n" + ) + self.show_global_msg = False + + if code.rstrip()[-1:] == ";": + # Supress output with ; + capture_last_expression = False + + if capture_last_expression: + ast_code, capture_last_expression = capture_last_Expr( + ast_code, "_spyder_out" + ) + ns_globals["__spyder_builtins__"] = builtins + + exec_fun(compile(ast_code, filename, "exec"), ns_globals, ns_locals) + + if capture_last_expression: + out = ns_globals.pop("_spyder_out", None) + if out is not None: + return out + except SystemExit as status: + # ignore exit(0) + if status.code: + self.shell.showtraceback(exception_only=True) + except BaseException as error: + if isinstance(error, bdb.BdbQuit) and self.shell.pdb_session: + # Ignore BdbQuit if we are debugging, as it is expected. + pass + elif post_mortem and isinstance(error, Exception): + error_type, error, tb = sys.exc_info() + self._post_mortem_excepthook(error_type, error, tb) + else: + # We ignore the call to exec + self.shell.showtraceback(tb_offset=1) + finally: + __tracebackhide__ = "__pdb_exit__" + + def _count_leading_empty_lines(self, cell): + """Count the number of leading empty cells.""" + lines = cell.splitlines(keepends=True) + if not lines: + return 0 + for i, line in enumerate(lines): + if line and not line.isspace(): + return i + return len(lines) + + def _transform_cell(self, code, indent_only=False): + """Transform IPython code to Python code.""" + number_empty_lines = self._count_leading_empty_lines(code) + if indent_only: + if not code.endswith("\n"): + code += "\n" # Ensure the cell has a trailing newline + lines = code.splitlines(keepends=True) + lines = leading_indent(leading_empty_lines(lines)) + code = "".join(lines) + else: + tm = TransformerManager() + code = tm.transform_cell(code) + return "\n" * number_empty_lines + code + + def _post_mortem_excepthook(self, type, value, tb): + """ + For post mortem exception handling, print a banner and enable post + mortem debugging. + """ + self.shell.showtraceback((type, value, tb)) + p = pdb.Pdb(self.shell.colors) + + if not type == SyntaxError: + # wait for stderr to print (stderr.flush does not work in this case) + time.sleep(0.1) + print("*" * 40) + print("Entering post mortem debugging...") + print("*" * 40) + # add ability to move between frames + p.reset() + frame = tb.tb_next.tb_frame + # wait for stdout to print + time.sleep(0.1) + p.interaction(frame, tb) + + def _parse_argstring(self, magic_func, argstring): + """ + Parse a string of arguments for a magic function. + + This is needed because magic_arguments.parse_argstring does + platform-dependent things with quotes and backslashes. For + example, on Windows, strings are removed and backslashes are + escaped. + """ + argv = shlex.split(argstring) + args = magic_func.parser.parse_args(argv) + if args.filename is None: + args.filename = self._get_current_file_name() + args.canonic_filename = canonic(args.filename) + return args + + def _parse_runfile_argstring(self, magic_func, argstring, local_ns): + """Parse an args string for runfile and debugfile.""" + args = self._parse_argstring(magic_func, argstring) + if args.namespace is None: + args.namespace = self.shell.user_ns + else: + if local_ns is not None and args.namespace in local_ns: + args.namespace = local_ns[args.namespace] + elif args.namespace in self.shell.user_ns: + args.namespace = self.shell.user_ns[args.namespace] + else: + raise NameError( + f"name '{args.namespace}' is not defined" + ) + local_ns = None + args.current_namespace = True + return args, local_ns + + def _parse_runcell_argstring(self, magic_func, argstring): + """Parse an args string for runcell and debugcell.""" + args = self._parse_argstring(magic_func, argstring) + args.cell_id = args.name + if args.cell_id is None: + args.cell_id = int(args.index) + return args diff --git a/spyder_kernels/customize/namespace_manager.py b/spyder_kernels/customize/namespace_manager.py index 36cd2796..f758cf27 100755 --- a/spyder_kernels/customize/namespace_manager.py +++ b/spyder_kernels/customize/namespace_manager.py @@ -9,8 +9,6 @@ import types import sys -from IPython.core.getipython import get_ipython - def new_main_mod(filename, modname): """ @@ -39,49 +37,53 @@ class NamespaceManager: current_namespace is True, or a new namespace. """ - def __init__(self, filename, namespace=None, current_namespace=False, - file_code=None, stack_depth=1): + def __init__( + self, + shell, + filename, + current_namespace=False, + file_code=None, + context_locals=None, + context_globals=None, + ): + self.shell = shell self.filename = filename - self.ns_globals = namespace + self.ns_globals = None self.ns_locals = None self.current_namespace = current_namespace self._previous_filename = None self._previous_main = None - self._previous_running_namespace = None self._reset_main = False self._file_code = file_code - ipython_shell = get_ipython() - self.context_globals = ipython_shell.get_global_scope(stack_depth + 1) - self.context_locals = ipython_shell.get_local_scope(stack_depth + 1) + if context_globals is None: + context_globals = shell.user_ns + self.context_globals = context_globals + self.context_locals = context_locals def __enter__(self): """ Prepare the namespace. """ # Save previous __file__ - ipython_shell = get_ipython() - if self.ns_globals is None: - if self.current_namespace: - self.ns_globals = self.context_globals - self.ns_locals = self.context_locals - if '__file__' in self.ns_globals: - self._previous_filename = self.ns_globals['__file__'] - self.ns_globals['__file__'] = self.filename - else: - main_mod = new_main_mod(self.filename, '__main__') - self.ns_globals = main_mod.__dict__ - self.ns_locals = None - # Needed to allow pickle to reference main - if '__main__' in sys.modules: - self._previous_main = sys.modules['__main__'] - sys.modules['__main__'] = main_mod - self._reset_main = True + if self.current_namespace: + self.ns_globals = self.context_globals + self.ns_locals = self.context_locals + if '__file__' in self.ns_globals: + self._previous_filename = self.ns_globals['__file__'] + self.ns_globals['__file__'] = self.filename + else: + main_mod = new_main_mod(self.filename, '__main__') + self.ns_globals = main_mod.__dict__ + self.ns_locals = None + + # Needed to allow pickle to reference main + if '__main__' in sys.modules: + self._previous_main = sys.modules['__main__'] + sys.modules['__main__'] = main_mod + self._reset_main = True # Save current namespace for access by variable explorer - self._previous_running_namespace = ( - ipython_shell.kernel._running_namespace) - ipython_shell.kernel._running_namespace = ( - self.ns_globals, self.ns_locals) + self.shell.add_namespace_manager(self) if (self._file_code is not None and isinstance(self._file_code, bytes)): @@ -103,18 +105,17 @@ def __exit__(self, exc_type, exc_val, exc_tb): """ Reset the namespace. """ - ipython_shell = get_ipython() - ipython_shell.kernel._running_namespace = ( - self._previous_running_namespace) + self.shell.remove_namespace_manager(self) if self._previous_filename: self.ns_globals['__file__'] = self._previous_filename elif '__file__' in self.ns_globals: self.ns_globals.pop('__file__') if not self.current_namespace: - self.context_globals.update(self.ns_globals) - if self.context_locals and self.ns_locals: - self.context_locals.update(self.ns_locals) + if self.context_locals is not None: + self.context_locals.update(self.ns_globals) + else: + self.context_globals.update(self.ns_globals) if self._previous_main: sys.modules['__main__'] = self._previous_main diff --git a/spyder_kernels/customize/spydercustomize.py b/spyder_kernels/customize/spydercustomize.py index e6a81f67..99c20898 100644 --- a/spyder_kernels/customize/spydercustomize.py +++ b/spyder_kernels/customize/spydercustomize.py @@ -11,30 +11,13 @@ # Spyder consoles sitecustomize # -import ast -import bdb -import builtins -import io import logging import os import pdb -import shlex import sys -import time import warnings -from IPython.core.getipython import get_ipython -from IPython.core.inputtransformer2 import ( - TransformerManager, leading_indent, leading_empty_lines) - -from spyder_kernels.comms.frontendcomm import frontend_request -from spyder_kernels.customize.namespace_manager import NamespaceManager -from spyder_kernels.customize.spyderpdb import SpyderPdb, get_new_debugger -from spyder_kernels.customize.umr import UserModuleReloader -from spyder_kernels.customize.utils import capture_last_Expr, canonic - - -logger = logging.getLogger(__name__) +from spyder_kernels.customize.spyderpdb import SpyderPdb # ============================================================================= @@ -45,15 +28,11 @@ if not hasattr(sys, 'argv'): sys.argv = [''] - # ============================================================================= # Main constants # ============================================================================= IS_EXT_INTERPRETER = os.environ.get('SPY_EXTERNAL_INTERPRETER') == "True" HIDE_CMD_WINDOWS = os.environ.get('SPY_HIDE_CMD') == "True" -SHOW_INVALID_SYNTAX_MSG = True -SHOW_GLOBAL_MSG = True - # ============================================================================= # Prevent subprocess.Popen calls to create visible console windows on Windows. @@ -70,7 +49,6 @@ def __init__(self, *args, **kwargs): subprocess.Popen = SubprocessPopen - # ============================================================================= # Importing user's sitecustomize # ============================================================================= @@ -79,7 +57,6 @@ def __init__(self, *args, **kwargs): except Exception: pass - # ============================================================================= # Set PyQt API to #2 # ============================================================================= @@ -276,450 +253,6 @@ def _patched_get_terminal_size(fd=None): # ============================================================================= pdb.Pdb = SpyderPdb -# ============================================================================= -# User module reloader -# ============================================================================= -__umr__ = UserModuleReloader(namelist=os.environ.get("SPY_UMR_NAMELIST", None)) - - -# ============================================================================= -# Handle Post Mortem Debugging and Traceback Linkage to Spyder -# ============================================================================= -def post_mortem_excepthook(type, value, tb): - """ - For post mortem exception handling, print a banner and enable post - mortem debugging. - """ - ipython_shell = get_ipython() - ipython_shell.showtraceback((type, value, tb)) - p = pdb.Pdb(ipython_shell.colors) - - if not type == SyntaxError: - # wait for stderr to print (stderr.flush does not work in this case) - time.sleep(0.1) - print('*' * 40) - print('Entering post mortem debugging...') - print('*' * 40) - # add ability to move between frames - p.reset() - frame = tb.tb_next.tb_frame - # wait for stdout to print - time.sleep(0.1) - p.interaction(frame, tb) - - -# ============================================================================== -# runfile and debugfile commands -# ============================================================================== -def get_current_file_name(): - """Get the current file name.""" - try: - return frontend_request(blocking=True).current_filename() - except Exception: - print("This command failed to be executed because an error occurred" - " while trying to get the current file name from Spyder's" - " editor. The error was:\n\n") - get_ipython().showtraceback(exception_only=True) - return None - - -def count_leading_empty_lines(cell): - """Count the number of leading empty cells.""" - lines = cell.splitlines(keepends=True) - if not lines: - return 0 - for i, line in enumerate(lines): - if line and not line.isspace(): - return i - return len(lines) - - -def transform_cell(code, indent_only=False): - """Transform IPython code to Python code.""" - number_empty_lines = count_leading_empty_lines(code) - if indent_only: - if not code.endswith('\n'): - code += '\n' # Ensure the cell has a trailing newline - lines = code.splitlines(keepends=True) - lines = leading_indent(leading_empty_lines(lines)) - code = ''.join(lines) - else: - tm = TransformerManager() - code = tm.transform_cell(code) - return '\n' * number_empty_lines + code - - -def exec_code(code, filename, ns_globals, ns_locals=None, post_mortem=False, - exec_fun=None, capture_last_expression=False, - global_warning=False): - """Execute code and display any exception.""" - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - global SHOW_INVALID_SYNTAX_MSG - global SHOW_GLOBAL_MSG - - if exec_fun is None: - exec_fun = exec - - ipython_shell = get_ipython() - is_ipython = os.path.splitext(filename)[1] == '.ipy' - try: - if not is_ipython: - # TODO: remove the try-except and let the SyntaxError raise - # Because there should not be ipython code in a python file - try: - ast_code = ast.parse(transform_cell(code, indent_only=True)) - except SyntaxError as e: - try: - ast_code = ast.parse(transform_cell(code)) - except SyntaxError: - raise e from None - else: - if SHOW_INVALID_SYNTAX_MSG: - print( - "\nWARNING: This is not valid Python code. " - "If you want to use IPython magics, " - "flexible indentation, and prompt removal, " - "we recommend that you save this file with the " - ".ipy extension.\n") - SHOW_INVALID_SYNTAX_MSG = False - else: - ast_code = ast.parse(transform_cell(code)) - - # Print warning for global - if global_warning and SHOW_GLOBAL_MSG: - has_global = any( - isinstance(node, ast.Global) for node in ast.walk(ast_code)) - if has_global: - print( - "\nWARNING: This file contains a global statement, " - "but it is run in an empty namespace. " - "Consider using the " - "'Run in console's namespace instead of an empty one' " - "option, that you can find in the menu 'Run > " - "Configuration per file', if you want to capture the " - "namespace.\n" - ) - SHOW_GLOBAL_MSG = False - - if code.rstrip()[-1] == ";": - # Supress output with ; - capture_last_expression = False - - if capture_last_expression: - ast_code, capture_last_expression = capture_last_Expr( - ast_code, "_spyder_out") - - exec_fun(compile(ast_code, filename, 'exec'), ns_globals, ns_locals) - - if capture_last_expression: - out = ns_globals.pop("_spyder_out", None) - if out is not None: - return out - - except SystemExit as status: - # ignore exit(0) - if status.code: - ipython_shell.showtraceback(exception_only=True) - except BaseException as error: - if (isinstance(error, bdb.BdbQuit) - and ipython_shell.pdb_session): - # Ignore BdbQuit if we are debugging, as it is expected. - pass - elif post_mortem and isinstance(error, Exception): - error_type, error, tb = sys.exc_info() - post_mortem_excepthook(error_type, error, tb) - else: - # We ignore the call to exec - ipython_shell.showtraceback(tb_offset=1) - finally: - __tracebackhide__ = "__pdb_exit__" - - -def get_file_code(filename, save_all=True, raise_exception=False): - """Retrieve the content of a file.""" - # Get code from spyder - try: - return frontend_request(blocking=True).get_file_code( - filename, save_all=save_all) - except Exception: - # Maybe this is a local file - try: - with open(filename, 'r') as f: - return f.read() - except FileNotFoundError: - pass - if raise_exception: - raise - # Else return None - return None - - -def runfile(filename=None, args=None, wdir=None, namespace=None, - post_mortem=False, current_namespace=False): - """ - Run filename - args: command line arguments (string) - wdir: working directory - namespace: namespace for execution - post_mortem: boolean, whether to enter post-mortem mode on error - current_namespace: if true, run the file in the current namespace - """ - return _exec_file( - filename, args, wdir, namespace, - post_mortem, current_namespace, stack_depth=1) - - -def _exec_file(filename=None, args=None, wdir=None, namespace=None, - post_mortem=False, current_namespace=False, stack_depth=0, - exec_fun=None, canonic_filename=None): - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - ipython_shell = get_ipython() - if filename is None: - filename = get_current_file_name() - if filename is None: - return - - if __umr__.enabled: - __umr__.run() - if args is not None and not isinstance(args, str): - raise TypeError("expected a character buffer object") - - try: - file_code = get_file_code(filename, raise_exception=True) - except Exception: - print( - "This command failed to be executed because an error occurred" - " while trying to get the file code from Spyder's" - " editor. The error was:\n\n") - get_ipython().showtraceback(exception_only=True) - return - - # Here the remote filename has been used. It must now be valid locally. - if canonic_filename is not None: - filename = canonic_filename - else: - filename = canonic(filename) - - with NamespaceManager(filename, namespace, current_namespace, - file_code=file_code, stack_depth=stack_depth + 1 - ) as (ns_globals, ns_locals): - sys.argv = [filename] - if args is not None: - for arg in shlex.split(args): - sys.argv.append(arg) - - if "multiprocessing" in sys.modules: - # See https://github.com/spyder-ide/spyder/issues/16696 - try: - sys.modules['__mp_main__'] = sys.modules['__main__'] - except Exception: - pass - - if wdir is not None: - if os.path.isdir(wdir): - os.chdir(wdir) - # See https://github.com/spyder-ide/spyder/issues/13632 - if "multiprocessing.process" in sys.modules: - try: - import multiprocessing.process - multiprocessing.process.ORIGINAL_DIR = os.path.abspath( - wdir) - except Exception: - pass - else: - print("Working directory {} doesn't exist.\n".format(wdir)) - - try: - if __umr__.has_cython: - # Cython files - with io.open(filename, encoding='utf-8') as f: - ipython_shell.run_cell_magic('cython', '', f.read()) - else: - exec_code(file_code, filename, ns_globals, ns_locals, - post_mortem=post_mortem, exec_fun=exec_fun, - capture_last_expression=False, - global_warning=not current_namespace) - finally: - sys.argv = [''] - - -# IPykernel 6.3.0+ shadows our runfile because it depends on the Pydev -# debugger, which adds its own runfile to builtins. So we replace it with -# our own using exec_lines in start.py -builtins.spyder_runfile = runfile - - -def debugfile(filename=None, args=None, wdir=None, post_mortem=False, - current_namespace=False): - """ - Debug filename - args: command line arguments (string) - wdir: working directory - post_mortem: boolean, included for compatiblity with runfile - """ - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - if filename is None: - filename = get_current_file_name() - if filename is None: - return - - shell = get_ipython() - if shell.is_debugging(): - # Recursive - code = ( - "runfile({}".format(repr(filename)) + - ", args=%r, wdir=%r, current_namespace=%r)" % ( - args, wdir, current_namespace) - ) - - shell.pdb_session.enter_recursive_debugger( - code, filename, True, - ) - else: - debugger = get_new_debugger(filename, True) - _exec_file( - filename=filename, - canonic_filename=debugger.canonic(filename), - args=args, - wdir=wdir, - current_namespace=current_namespace, - exec_fun=debugger.run, - stack_depth=1, - ) - - -builtins.debugfile = debugfile - - -def runcell(cellname, filename=None, post_mortem=False): - """ - Run a code cell from an editor as a file. - - Parameters - ---------- - cellname : str or int - Cell name or index. - filename : str - Needed to allow for proper traceback links. - post_mortem: bool - Automatically enter post mortem on exception. - """ - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - return _exec_cell(cellname, filename, post_mortem, stack_depth=1) - - -def _exec_cell(cellname, filename=None, post_mortem=False, stack_depth=0, - exec_fun=None, canonic_filename=None): - """ - Execute a code cell with a given exec function. - """ - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - if filename is None: - filename = get_current_file_name() - if filename is None: - return - ipython_shell = get_ipython() - try: - # Get code from spyder - cell_code = frontend_request( - blocking=True).run_cell(cellname, filename) - except Exception: - print("This command failed to be executed because an error occurred" - " while trying to get the cell code from Spyder's" - " editor. The error was:\n\n") - get_ipython().showtraceback(exception_only=True) - return - - if not cell_code or cell_code.strip() == '': - print("Nothing to execute, this cell is empty.\n") - return - - # Trigger `post_execute` to exit the additional pre-execution. - # See Spyder PR #7310. - ipython_shell.events.trigger('post_execute') - file_code = get_file_code(filename, save_all=False) - - # Here the remote filename has been used. It must now be valid locally. - if canonic_filename is not None: - filename = canonic_filename - else: - # Normalise the filename - filename = canonic(filename) - - with NamespaceManager(filename, current_namespace=True, - file_code=file_code, stack_depth=stack_depth + 1 - ) as (ns_globals, ns_locals): - return exec_code(cell_code, filename, ns_globals, ns_locals, - post_mortem=post_mortem, exec_fun=exec_fun, - capture_last_expression=True) - - -builtins.runcell = runcell - - -def debugcell(cellname, filename=None, post_mortem=False): - """Debug a cell.""" - # Tell IPython to hide this frame (>7.16) - __tracebackhide__ = True - if filename is None: - filename = get_current_file_name() - if filename is None: - return - - shell = get_ipython() - if shell.is_debugging(): - # Recursive - code = ( - "runcell({}, ".format(repr(cellname)) + - "{})".format(repr(filename)) - ) - shell.pdb_session.enter_recursive_debugger( - code, filename, False, - ) - else: - debugger = get_new_debugger(filename, False) - _exec_cell( - cellname=cellname, - filename=filename, - canonic_filename=debugger.canonic(filename), - exec_fun=debugger.run, - stack_depth=1 - ) - - -builtins.debugcell = debugcell - - -def cell_count(filename=None): - """ - Get the number of cells in a file. - - Parameters - ---------- - filename : str - The file to get the cells from. If None, the currently opened file. - """ - if filename is None: - filename = get_current_file_name() - if filename is None: - raise RuntimeError('Could not get cell count from frontend.') - try: - # Get code from spyder - cell_count = frontend_request(blocking=True).cell_count(filename) - return cell_count - except Exception: - etype, error, tb = sys.exc_info() - raise etype(error) - - -builtins.cell_count = cell_count - - # ============================================================================= # PYTHONPATH and sys.path Adjustments # ============================================================================= diff --git a/spyder_kernels/customize/spyderpdb.py b/spyder_kernels/customize/spyderpdb.py index 0264e765..ec505368 100755 --- a/spyder_kernels/customize/spyderpdb.py +++ b/spyder_kernels/customize/spyderpdb.py @@ -10,6 +10,7 @@ import ast import bdb import builtins +from contextlib import contextmanager import logging import os import sys @@ -19,7 +20,6 @@ from IPython.core.autocall import ZMQExitAutocall from IPython.core.debugger import Pdb as ipyPdb -from IPython.core.getipython import get_ipython from IPython.core.inputtransformer2 import TransformerManager import spyder_kernels @@ -42,7 +42,7 @@ def __enter__(self): """ Debugging starts. """ - shell = get_ipython() + shell = self.pdb_obj.shell if shell.pdb_session == self.pdb_obj: self._cleanup = False else: @@ -54,7 +54,8 @@ def __exit__(self, exc_type, exc_val, exc_tb): Debugging ends. """ if self._cleanup: - get_ipython().remove_pdb_session(self.pdb_obj) + self.pdb_obj.shell.remove_pdb_session(self.pdb_obj) + class SpyderPdb(ipyPdb): """ @@ -67,8 +68,6 @@ class SpyderPdb(ipyPdb): - Add completion to non-command code. """ - starting = True - def __init__(self, completekey='tab', stdin=None, stdout=None, skip=None, nosigint=False): """Init Pdb.""" @@ -107,6 +106,15 @@ def __init__(self, completekey='tab', stdin=None, stdout=None, # Should the frontend force go to the current line? self._request_where = False + # Turn off IPython's debugger skip funcionality by default because + # it makes our debugger quite slow. It's also important to remark + # that this functionality doesn't do anything on its own. Users + # need to mark what frames they want to skip for it to be useful. + # So, we hope that knowledgeable users will find that they need to + # enable it in Spyder. + # Fixes spyder-ide/spyder#20639. + self._predicates["debuggerskip"] = False + # --- Methods overriden for code execution def print_exclamation_warning(self): """Print pdb warning for exclamation mark.""" @@ -126,12 +134,13 @@ def default(self, line): self.print_exclamation_warning() self.error("Unknown command '" + line.split()[0] + "'") return - # Disallow the use of %debug magic in the debugger - if line.startswith("%debug"): - self.error("Please don't use '%debug' in the debugger.\n" - "For a recursive debugger, use the pdb 'debug'" - " command instead") - return + + # Replace %debug magic in the debugger + if line.startswith("%debug") or line.startswith("%%debug"): + cmd, arg, _ = self.parseline(line.lstrip("%")) + if cmd == "debug": + return self.do_debug(arg) + locals = self.curframe_locals globals = self.curframe.f_globals @@ -180,17 +189,19 @@ def default(self, line): sys.stdout = self.stdout sys.displayhook = self.displayhook if execute_events: - get_ipython().events.trigger('pre_execute') + self.shell.events.trigger('pre_execute') code_ast = ast.parse(line) - if line.rstrip()[-1] == ";": + if line.rstrip()[-1:] == ";": # Supress output with ; capture_last_expression = False else: code_ast, capture_last_expression = capture_last_Expr( code_ast, "_spyderpdb_out") + globals["__spyder_builtins__"] = builtins + if locals is not globals: # Mitigates a behaviour of CPython that makes it difficult # to work with exec and the local namespace @@ -215,32 +226,29 @@ def default(self, line): indent = " " code = ["def _spyderpdb_code():"] - # Load the locals - globals["_spyderpdb_builtins_locals"] = builtins.locals - - # Save builtins locals in case it is shadowed - globals["_spyderpdb_locals"] = locals + # Add locals in globals + # If the debugger is recursive, the globals could already + # have a _spyderpdb_locals as it might be shared between + # levels + if "_spyderpdb_locals" in globals: + globals["_spyderpdb_locals"].append(locals) + else: + globals["_spyderpdb_locals"] = [locals] # Load locals if they have a valid name # In comprehensions, locals could contain ".0" for example - code += [indent + "{k} = _spyderpdb_locals['{k}']".format( + code += [indent + "{k} = _spyderpdb_locals[-1]['{k}']".format( k=k) for k in locals if k.isidentifier()] + # The code comes here # Update the locals - code += [indent + "_spyderpdb_locals.update(" - "_spyderpdb_builtins_locals())"] + code += [indent + "_spyderpdb_locals[-1].update(" + "__spyder_builtins__.locals())"] # Run the function code += ["_spyderpdb_code()"] - # Cleanup - code += [ - "del _spyderpdb_code", - "del _spyderpdb_locals", - "del _spyderpdb_builtins_locals" - ] - # Parse the function fun_ast = ast.parse('\n'.join(code) + '\n') @@ -252,7 +260,17 @@ def default(self, line): ) code_ast = fun_ast - exec(compile(code_ast, "", "exec"), globals) + try: + exec(compile(code_ast, "", "exec"), globals) + finally: + if locals is not globals: + # CLeanup code + globals.pop("_spyderpdb_code", None) + if len(globals["_spyderpdb_locals"]) > 1: + del globals["_spyderpdb_locals"][-1] + else: + del globals["_spyderpdb_locals"] + if capture_last_expression: out = globals.pop("_spyderpdb_out", None) @@ -268,7 +286,7 @@ def default(self, line): finally: if execute_events: - get_ipython().events.trigger('post_execute') + self.shell.events.trigger('post_execute') sys.stdout = save_stdout sys.stdin = save_stdin sys.displayhook = save_displayhook @@ -286,12 +304,12 @@ def interrupt(self): def set_trace(self, frame=None): """Register that debugger is tracing.""" - get_ipython().add_pdb_session(self) + self.shell.add_pdb_session(self) super(SpyderPdb, self).set_trace(frame) def set_quit(self): """Register that debugger is not tracing.""" - get_ipython().remove_pdb_session(self) + self.shell.remove_pdb_session(self) super(SpyderPdb, self).set_quit() def interaction(self, frame, traceback): @@ -303,36 +321,73 @@ def interaction(self, frame, traceback): return super(SpyderPdb, self).interaction( frame, traceback) - def print_stack_entry(self, frame_lineno, prompt_prefix='\n-> ', - context=None): + def print_stack_entry(self, *args, **kwargs): """Disable printing stack entry if requested.""" if self._disable_next_stack_entry: self._disable_next_stack_entry = False return - return super(SpyderPdb, self).print_stack_entry( - frame_lineno, prompt_prefix, context) + return super().print_stack_entry(*args, **kwargs) # --- Methods overriden for skipping libraries def stop_here(self, frame): """Check if pdb should stop here.""" - if (frame is not None - and "__tracebackhide__" in frame.f_locals - and frame.f_locals["__tracebackhide__"] == "__pdb_exit__"): + # Never stop if we are continuing unless there is a breakpoint + if self.stopframe == self.botframe and self.stoplineno == -1: + return False + if self.continue_if_has_breakpoints and self.should_continue(frame): + self.set_continue() + return False + if ( + frame is not None + and "__tracebackhide__" in frame.f_locals + and frame.f_locals["__tracebackhide__"] == "__pdb_exit__" + ): self.onecmd('exit') return False - if not super(SpyderPdb, self).stop_here(frame): + if not super().stop_here(frame): return False + if frame is self.stopframe: + return True filename = frame.f_code.co_filename if filename.startswith('<'): # This is not a file return True if self.pdb_ignore_lib and path_is_library(filename): return False - if os.path.dirname(spyder_kernels.__file__) in filename: + if self.skip_hidden and os.path.dirname(spyder_kernels.__file__) in filename: # This is spyder-kernels internals return False return True + + def should_continue(self, frame): + """ + Jump to first breakpoint if needed. + + Fixes spyder-ide/spyder#2034 + """ + + if not self.continue_if_has_breakpoints: + # This was disabled + return False + self.continue_if_has_breakpoints = False + + # Get all breakpoints for the file we're going to debug + if not frame: + # We are not debugging, return. Solves spyder-ide/spyder#10290 + return False + + lineno = frame.f_lineno + breaks = self.get_file_breaks(frame.f_code.co_filename) + + # Do 'continue' if the first breakpoint is *not* placed + # where the debugger is going to land. + # Fixes spyder-ide/spyder#4681 + if self.pdb_stop_first_line: + return breaks and lineno < breaks[0] + + # The breakpoint could be in another file. + return not (breaks and lineno >= breaks[0]) def do_where(self, arg): """w(here) @@ -412,7 +467,6 @@ def is_name_or_composed(text): cursor_start = cursor_pos - len(text) if ipython_do_complete: - kernel = get_ipython().kernel # Make complete call with current frame if self.curframe: if self.curframe_locals: @@ -421,10 +475,10 @@ def is_name_or_composed(text): self.curframe.f_globals) else: frame = self.curframe - kernel.shell.set_completer_frame(frame) - result = kernel._do_complete(code, cursor_pos) + self.shell.set_completer_frame(frame) + result = self.shell.kernel._do_complete(code, cursor_pos) # Reset frame - kernel.shell.set_completer_frame() + self.shell.set_completer_frame() # If there is no Pdb results to merge, return the result if not compfunc: return result @@ -520,7 +574,6 @@ def is_name_or_composed(text): 'status': 'ok' } - kernel = get_ipython().kernel # Make complete call with current frame if self.curframe: if self.curframe_locals: @@ -529,10 +582,10 @@ def is_name_or_composed(text): self.curframe.f_globals) else: frame = self.curframe - kernel.shell.set_completer_frame(frame) - result = kernel._do_complete(code, cursor_pos) + self.shell.set_completer_frame(frame) + result = self.shell.kernel._do_complete(code, cursor_pos) # Reset frame - kernel.shell.set_completer_frame() + self.shell.set_completer_frame() return result # --- Methods overriden by us for Spyder integration @@ -560,12 +613,43 @@ def do_debug(self, arg): argument (which is an arbitrary expression or statement to be executed in the current environment). """ + with self.recursive_debugger() as debugger: + self.message("Entering recursive debugger") + try: + globals = self.curframe.f_globals + locals = self.curframe_locals + return sys.call_tracing(debugger.run, (arg, globals, locals)) + except Exception: + exc_info = sys.exc_info()[:2] + self.error( + traceback.format_exception_only(*exc_info)[-1].strip()) + finally: + self.message("Leaving recursive debugger") + + @contextmanager + def recursive_debugger(self): + """Get a recursive debugger.""" + # Save and restore tracing function + trace_function = sys.gettrace() + sys.settrace(None) + + # Create child debugger + debugger = self.__class__( + completekey=self.completekey, + stdin=self.stdin, stdout=self.stdout) + debugger.prompt = "(%s) " % self.prompt.strip() try: - super(SpyderPdb, self).do_debug(arg) - except Exception: - exc_info = sys.exc_info()[:2] - self.error( - traceback.format_exception_only(*exc_info)[-1].strip()) + yield debugger + finally: + # Reset parent debugger + sys.settrace(trace_function) + self.lastcmd = debugger.lastcmd + + # Reset _previous_step so that get_pdb_state() notifies Spyder about + # a changed debugger position. The reset is required because the + # recursive debugger might change the position, but the parent + # debugger (self) is not aware of this. + self._previous_step = None def user_return(self, frame, return_value): """This function is called when a return trap is set here.""" @@ -623,7 +707,7 @@ def cmd_input(self, prompt=''): """ Get input from frontend. Blocks until return """ - kernel = get_ipython().kernel + kernel = self.shell.kernel # Only works if the comm is open if not kernel.frontend_comm.is_open(): return input(prompt) @@ -713,52 +797,13 @@ def set_spyder_breakpoints(self, breakpoints): breakpoints = property(fset=set_spyder_breakpoints) - def should_do_continue(self): - """ - Jump to first breakpoint if needed - - Fixes spyder-ide/spyder#2034 - """ - if not self.starting: - # Only run this after a Pdb session is created - return False - self.starting = False - - if not self.continue_if_has_breakpoints: - # This was disabled - return False - - # Get all breakpoints for the file we're going to debug - frame = self.curframe - if not frame: - # We are not debugging, return. Solves spyder-ide/spyder#10290 - return False - - lineno = frame.f_lineno - breaks = self.get_file_breaks(frame.f_code.co_filename) - - # Do 'continue' if the first breakpoint is *not* placed - # where the debugger is going to land. - # Fixes spyder-ide/spyder#4681 - if self.pdb_stop_first_line: - return breaks and lineno < breaks[0] - - # The breakpoint could be in another file. - return not (breaks and lineno >= breaks[0]) - def get_pdb_state(self): """ Send debugger state (frame position) to the frontend. The state is only sent if it has changed since the last update. """ - state = get_ipython().kernel.get_state() - if self.starting and self.should_do_continue(): - if self.pdb_use_exclamation_mark: - cont_cmd = '!continue' - else: - cont_cmd = 'continue' - state['request_pdb_input'] = cont_cmd + state = self.shell.kernel.get_state() frame = self.curframe if frame is None: @@ -811,7 +856,6 @@ def run(self, cmd, globals=None, locals=None): globals defaults to __main__.dict; locals defaults to globals. """ - self.starting = True with DebugWrapper(self): super(SpyderPdb, self).run(cmd, globals, locals) @@ -820,7 +864,6 @@ def runeval(self, expr, globals=None, locals=None): globals defaults to __main__.dict; locals defaults to globals. """ - self.starting = True with DebugWrapper(self): super(SpyderPdb, self).runeval(expr, globals, locals) @@ -829,50 +872,11 @@ def runcall(self, *args, **kwds): Return the result of the function call. """ - self.starting = True with DebugWrapper(self): super(SpyderPdb, self).runcall(*args, **kwds) - def enter_recursive_debugger(self, code, filename, - continue_if_has_breakpoints): - """ - Enter debugger recursively. - """ - sys.settrace(None) - globals = self.curframe.f_globals - locals = self.curframe_locals - # Create child debugger - debugger = SpyderPdb( - completekey=self.completekey, - stdin=self.stdin, stdout=self.stdout) - debugger.use_rawinput = self.use_rawinput - debugger.prompt = "(%s) " % self.prompt.strip() - - debugger.set_remote_filename(filename) - debugger.continue_if_has_breakpoints = continue_if_has_breakpoints - - # Enter recursive debugger - sys.call_tracing(debugger.run, (code, globals, locals)) - # Reset parent debugger - sys.settrace(self.trace_dispatch) - self.lastcmd = debugger.lastcmd - - # Reset _previous_step so that get_pdb_state() notifies Spyder about - # a changed debugger position. The reset is required because the - # recursive debugger might change the position, but the parent debugger - # (self) is not aware of this. - self._previous_step = None - def set_remote_filename(self, filename): """Set remote filename to signal Spyder on mainpyfile.""" self.remote_filename = filename self.mainpyfile = self.canonic(filename) self._wait_for_mainpyfile = True - - -def get_new_debugger(filename, continue_if_has_breakpoints): - """Get a new debugger.""" - debugger = SpyderPdb() - debugger.set_remote_filename(filename) - debugger.continue_if_has_breakpoints = continue_if_has_breakpoints - return debugger diff --git a/spyder_kernels/customize/utils.py b/spyder_kernels/customize/utils.py index f34a0e5a..d74b6d7c 100644 --- a/spyder_kernels/customize/utils.py +++ b/spyder_kernels/customize/utils.py @@ -97,7 +97,11 @@ def path_is_library(path, initial_pathlist=None): def capture_last_Expr(code_ast, out_varname): - """Parse line and modify code to capture in globals the last expression.""" + """ + Parse line and modify code to capture in globals the last expression. + + The namespace must contain __spyder_builtins__, which is the builtins module. + """ # Modify ast code to capture the last expression capture_last_expression = False if ( @@ -108,7 +112,8 @@ def capture_last_Expr(code_ast, out_varname): expr_node = code_ast.body[-1] # Create new assign node assign_node = ast.parse( - 'globals()[{}] = None'.format(repr(out_varname))).body[0] + '__spyder_builtins__.globals()[{}] = None'.format( + repr(out_varname))).body[0] # Replace None by the value assign_node.value = expr_node.value # Fix line number and column offset @@ -118,7 +123,7 @@ def capture_last_Expr(code_ast, out_varname): # Exists from 3.8, necessary from 3.11 assign_node.end_lineno = expr_node.end_lineno if assign_node.lineno == assign_node.end_lineno: - # Add 'globals()[{}] = ' and remove 'None' + # Add '__spyder_builtins__.globals()[{}] = ' and remove 'None' assign_node.end_col_offset += expr_node.end_col_offset - 4 else: assign_node.end_col_offset = expr_node.end_col_offset diff --git a/spyder_kernels/utils/iofuncs.py b/spyder_kernels/utils/iofuncs.py index 2764f559..7da2828f 100644 --- a/spyder_kernels/utils/iofuncs.py +++ b/spyder_kernels/utils/iofuncs.py @@ -32,6 +32,8 @@ FakeObject, numpy as np, pandas as pd, PIL, scipy as sp) +# ---- For Matlab files +# ----------------------------------------------------------------------------- class MatlabStruct(dict): """ Matlab style struct, enhanced. @@ -41,7 +43,7 @@ class MatlabStruct(dict): Examples ======== - >>> from spyder.utils.iofuncs import MatlabStruct + >>> from spyder_kernels.utils.iofuncs import MatlabStruct >>> a = MatlabStruct() >>> a.b = 'spam' # a["b"] == 'spam' >>> a.c["d"] = 'eggs' # a.c.d == 'eggs' @@ -165,6 +167,8 @@ def save_matlab(data, filename): return str(error) +# ---- For arrays +# ----------------------------------------------------------------------------- def load_array(filename): if np.load is FakeObject: return None, '' @@ -189,6 +193,8 @@ def __save_array(data, basename, index): return fname +# ---- For PIL images +# ----------------------------------------------------------------------------- if sys.byteorder == 'little': _ENDIAN = '<' else: @@ -233,6 +239,8 @@ def load_image(filename): return None, str(error) +# ---- For misc formats +# ----------------------------------------------------------------------------- def load_pickle(filename): """Load a pickle file as a dictionary""" try: @@ -256,6 +264,8 @@ def load_json(filename): return None, str(err) +# ---- For Spydata files +# ----------------------------------------------------------------------------- def save_dictionary(data, filename): """Save dictionary in a single file .spydata file""" filename = osp.abspath(filename) @@ -402,7 +412,7 @@ def load_dictionary(filename): try: saved_arrays = data.pop('__saved_arrays__') for (name, index), fname in list(saved_arrays.items()): - arr = np.load(osp.join(tmp_folder, fname)) + arr = np.load(osp.join(tmp_folder, fname), allow_pickle=True) if index is None: data[name] = arr elif isinstance(data[name], dict): @@ -424,6 +434,89 @@ def load_dictionary(filename): return data, error_message +# ---- For HDF5 files +# ----------------------------------------------------------------------------- +def load_hdf5(filename): + """ + Load an hdf5 file. + + Notes + ----- + - This is a fairly dumb implementation which reads the whole HDF5 file into + Spyder's variable explorer. Since HDF5 files are designed for storing + very large data-sets, it may be much better to work directly with the + HDF5 objects, thus keeping the data on disk. Nonetheless, this gives + quick and dirty but convenient access to them. + - There is no support for creating files with compression, chunking etc, + although these can be read without problem. + - When reading an HDF5 file with sub-groups, groups in the file will + correspond to dictionaries with the same layout. + """ + def get_group(group): + contents = {} + for name, obj in list(group.items()): + if isinstance(obj, h5py.Dataset): + contents[name] = np.array(obj) + elif isinstance(obj, h5py.Group): + # it is a group, so call self recursively + contents[name] = get_group(obj) + # other objects such as links are ignored + return contents + + try: + import h5py + + f = h5py.File(filename, 'r') + contents = get_group(f) + f.close() + return contents, None + except Exception as error: + return None, str(error) + + +def save_hdf5(data, filename): + """ + Save an hdf5 file. + + Notes + ----- + - All datatypes to be saved must be convertible to a numpy array, otherwise + an exception will be raised. + - Data attributes are currently ignored. + - When saving data after reading it with load_hdf5, dictionaries are not + turned into HDF5 groups. + """ + try: + import h5py + + f = h5py.File(filename, 'w') + for key, value in list(data.items()): + f[key] = np.array(value) + f.close() + except Exception as error: + return str(error) + + +# ---- For DICOM files +# ----------------------------------------------------------------------------- +def load_dicom(filename): + """Load a DICOM files.""" + try: + from pydicom import dicomio + + name = osp.splitext(osp.basename(filename))[0] + try: + data = dicomio.read_file(filename, force=True) + except TypeError: + data = dicomio.read_file(filename) + arr = data.pixel_array + return {name: arr}, None + except Exception as error: + return None, str(error) + + +# ---- Class to group all IO functionality +# ----------------------------------------------------------------------------- class IOFunctions: def __init__(self): self.load_extensions = None @@ -434,7 +527,7 @@ def __init__(self): self.save_funcs = None def setup(self): - iofuncs = self.get_internal_funcs()+self.get_3rd_party_funcs() + iofuncs = self.get_internal_funcs() load_extensions = {} save_extensions = {} load_funcs = {} @@ -442,6 +535,7 @@ def setup(self): load_filters = [] save_filters = [] load_ext = [] + for ext, name, loadfunc, savefunc in iofuncs: filter_str = str(name + " (*%s)" % ext) if loadfunc is not None: @@ -453,9 +547,12 @@ def setup(self): save_extensions[filter_str] = ext save_filters.append(filter_str) save_funcs[ext] = savefunc + load_filters.insert( - 0, str("Supported files" + " (*" + " *".join(load_ext) + ")")) + 0, str("Supported files" + " (*" + " *".join(load_ext) + ")") + ) load_filters.append(str("All files (*.*)")) + self.load_filters = "\n".join(load_filters) self.save_filters = "\n".join(save_filters) self.load_funcs = load_funcs @@ -465,35 +562,22 @@ def setup(self): def get_internal_funcs(self): return [ - ('.spydata', "Spyder data files", - load_dictionary, save_dictionary), - ('.npy', "NumPy arrays", load_array, None), - ('.npz', "NumPy zip arrays", load_array, None), - ('.mat', "Matlab files", load_matlab, save_matlab), - ('.csv', "CSV text files", 'import_wizard', None), - ('.txt', "Text files", 'import_wizard', None), - ('.jpg', "JPEG images", load_image, None), - ('.png', "PNG images", load_image, None), - ('.gif', "GIF images", load_image, None), - ('.tif', "TIFF images", load_image, None), - ('.pkl', "Pickle files", load_pickle, None), - ('.pickle', "Pickle files", load_pickle, None), - ('.json', "JSON files", load_json, None), - ] - - def get_3rd_party_funcs(self): - other_funcs = [] - try: - from spyder.otherplugins import get_spyderplugins_mods - for mod in get_spyderplugins_mods(io=True): - try: - other_funcs.append((mod.FORMAT_EXT, mod.FORMAT_NAME, - mod.FORMAT_LOAD, mod.FORMAT_SAVE)) - except AttributeError as error: - print("%s: %s" % (mod, str(error)), file=sys.stderr) - except ImportError: - pass - return other_funcs + ('.spydata', "Spyder data files", load_dictionary, save_dictionary), + ('.npy', "NumPy arrays", load_array, None), + ('.npz', "NumPy zip arrays", load_array, None), + ('.mat', "Matlab files", load_matlab, save_matlab), + ('.csv', "CSV text files", 'import_wizard', None), + ('.txt', "Text files", 'import_wizard', None), + ('.jpg', "JPEG images", load_image, None), + ('.png', "PNG images", load_image, None), + ('.gif', "GIF images", load_image, None), + ('.tif', "TIFF images", load_image, None), + ('.pkl', "Pickle files", load_pickle, None), + ('.pickle', "Pickle files", load_pickle, None), + ('.json', "JSON files", load_json, None), + ('.h5', "HDF5 files", load_hdf5, save_hdf5), + ('.dcm', "DICOM images", load_dicom, None), + ] def save(self, data, filename): ext = osp.splitext(filename)[1].lower() @@ -513,11 +597,8 @@ def load(self, filename): iofunctions.setup() -def save_auto(data, filename): - """Save data into filename, depending on file extension""" - pass - - +# ---- Test +# ----------------------------------------------------------------------------- if __name__ == "__main__": import datetime testdict = {'d': 1, 'a': np.random.rand(10, 10), 'b': [1, 2]} @@ -536,9 +617,9 @@ def save_auto(data, filename): import time t0 = time.time() save_dictionary(example, "test.spydata") - print(" Data saved in %.3f seconds" % (time.time()-t0)) # spyder: test-skip + print(" Data saved in %.3f seconds" % (time.time()-t0)) t0 = time.time() example2, ok = load_dictionary("test.spydata") os.remove("test.spydata") - print("Data loaded in %.3f seconds" % (time.time()-t0)) # spyder: test-skip + print("Data loaded in %.3f seconds" % (time.time()-t0)) diff --git a/spyder_kernels/utils/tests/data.dcm b/spyder_kernels/utils/tests/data.dcm new file mode 100644 index 00000000..9d525ec3 Binary files /dev/null and b/spyder_kernels/utils/tests/data.dcm differ diff --git a/spyder_kernels/utils/tests/export_data.spydata b/spyder_kernels/utils/tests/export_data.spydata index 38ba220e..60efd7a0 100644 Binary files a/spyder_kernels/utils/tests/export_data.spydata and b/spyder_kernels/utils/tests/export_data.spydata differ diff --git a/spyder_kernels/utils/tests/export_data_renamed.spydata b/spyder_kernels/utils/tests/export_data_renamed.spydata index 38ba220e..60efd7a0 100644 Binary files a/spyder_kernels/utils/tests/export_data_renamed.spydata and b/spyder_kernels/utils/tests/export_data_renamed.spydata differ diff --git a/spyder_kernels/utils/tests/test_iofuncs.py b/spyder_kernels/utils/tests/test_iofuncs.py index eb29241f..ebb07c85 100644 --- a/spyder_kernels/utils/tests/test_iofuncs.py +++ b/spyder_kernels/utils/tests/test_iofuncs.py @@ -16,6 +16,7 @@ import copy # Third party imports +from PIL import ImageFile import pytest import numpy as np @@ -24,8 +25,9 @@ # Full path to this file's parent directory for loading data -LOCATION = os.path.realpath(os.path.join(os.getcwd(), - os.path.dirname(__file__))) +LOCATION = os.path.realpath( + os.path.join(os.getcwd(), os.path.dirname(__file__)) +) # ============================================================================= @@ -87,7 +89,7 @@ def spydata_values(): B = 'ham' C = np.eye(3) D = {'a': True, 'b': np.eye(4, dtype=np.complex128)} - E = [np.eye(2, dtype=np.int64), 42.0, np.eye(3, dtype=np.bool_)] + E = [np.eye(2, dtype=np.int64), 42.0, np.eye(3, dtype=np.bool_), np.eye(4, dtype=object)] return {'A': A, 'B': B, 'C': C, 'D': D, 'E': E} @@ -338,5 +340,25 @@ def test_spydata_export(input_namespace, expected_namespace, pass +def test_save_load_hdf5_files(): + """Simple test to check that we can save and load HDF5 files.""" + data = {'a' : [1, 2, 3, 4], 'b' : 4.5} + iofuncs.save_hdf5(data, "test.h5") + + expected = ({'a': np.array([1, 2, 3, 4]), 'b': np.array(4.5)}, None) + assert repr(iofuncs.load_hdf5("test.h5")) == repr(expected) + + +def test_load_dicom_files(): + """Check that we can load DICOM files.""" + # This test pass locally but we need to set the variable below for it to + # pass on CIs. + # See https://stackoverflow.com/a/47958486/438386 for context. + ImageFile.LOAD_TRUNCATED_IMAGES = True + + data = iofuncs.load_dicom(os.path.join(LOCATION, 'data.dcm')) + assert data[0]['data'].shape == (512, 512) + + if __name__ == "__main__": pytest.main() diff --git a/spyder_kernels/utils/tests/test_nsview.py b/spyder_kernels/utils/tests/test_nsview.py index e7539ee0..2d79c9f8 100644 --- a/spyder_kernels/utils/tests/test_nsview.py +++ b/spyder_kernels/utils/tests/test_nsview.py @@ -343,7 +343,7 @@ def test_get_type_string(): assert get_type_string(series) == 'Series' index = pd.Index([1, 2, 3]) - assert get_type_string(index) == 'Int64Index' + assert get_type_string(index) in ['Int64Index', 'Index'] # PIL images img = PIL.Image.new('RGB', (256,256))