Script editor debugger - experimental

Makefile

@@ -84,6 +84,7 @@ uninstall-src: # Uninstalls source extensions if they're still installed
 	- jupyter labextension uninstall --no-build @elyra/r-editor-extension
 	- jupyter labextension uninstall --no-build @elyra/scala-editor-extension
 	- jupyter labextension uninstall --no-build @elyra/code-viewer-extension
+	- jupyter labextension uninstall --no-build @elyra/script-debugger-extension
 	- jupyter labextension unlink --no-build @elyra/pipeline-services
 	- jupyter labextension unlink --no-build @elyra/pipeline-editor
 

create-release.py

@@ -597,14 +597,14 @@ def prepare_extensions_release() -> None:
             f"See https://elyra.readthedocs.io/en/v{config.new_version}/user_guide/pipelines.html",
         ),
         "elyra-python-editor-extension": SimpleNamespace(
-            packages=["python-editor-extension", "metadata-extension", "theme-extension"],
+            packages=["python-editor-extension", "metadata-extension", "theme-extension", "script-debugger-extension"],
             description=f"The Python Script editor extension contains support for Python files, "
             f"which can take advantage of the Hybrid Runtime Support enabling users to "
-            f"locally edit .py scripts and execute them against local or cloud-based resources."
+            f"locally edit, execute and debug .py scripts against local or cloud-based resources."
             f"See https://elyra.readthedocs.io/en/v{config.new_version}/user_guide/enhanced-script-support.html",
         ),
         "elyra-r-editor-extension": SimpleNamespace(
-            packages=["r-editor-extension", "metadata-extension", "theme-extension"],
+            packages=["r-editor-extension", "metadata-extension", "theme-extension", "script-debugger-extension"],
             description=f"The R Script editor extension contains support for R files, which can take "
             f"advantage of the Hybrid Runtime Support enabling users to locally edit .R scripts "
             f"and execute them against local or cloud-based resources."

docs/source/getting_started/overview.md

@@ -89,10 +89,11 @@ Refer to the
 Elyra contributes a Script editor with support for Python and R files, which can take advantage of the
 **Hybrid Runtime Support** enabling users to locally edit scripts and execute
 them against local or cloud-based resources seamlessly.
+Elyra Script editors are now integrated with JupyterLab's debugger feature, allowing scripts to be easily debugged within the editors' UI. For the debugger to be enabled and visible in the editor's toolbar, a kernel with support for debugging is required.
 
 ![Enhanced Python Support](../images/python-editor.png)
 
-For information on how to use the Script editor refer to the [_Enhanced Script Support_ topic](../user_guide/enhanced-script-support) in the User Guide.
+For information on how to use the Script editor and the debugger requirements refer to the [_Enhanced Script Support_ topic](../user_guide/enhanced-script-support) in the User Guide.
 
 The Script editor feature can optionally be [installed as a stand-alone extension](installation).
 

docs/source/user_guide/enhanced-script-support.md

@@ -17,13 +17,14 @@ limitations under the License.
 -->
 # Enhanced Script Support
 
-Elyra provides **Enhanced Script Support** where Python and R scripts can be developed and
-executed. It also leverages the **Hybrid Runtime Support** to enable running
+Elyra provides **Enhanced Script Support** where Python, R and Scala scripts can be edited and executed. It also leverages the **Hybrid Runtime Support** to enable running
 these scripts in remote environments.
 
 ![Enhanced Python Support](../images/python-editor.gif)
 
-The execution of these scripts leverages the available Python and R based Kernels. This enables users to run their scripts in different configurations and environments.
+The execution of these scripts leverages the available Python, R and Scala based kernels. This enables users to run their scripts in different configurations and environments.
+
+Debugging Python scripts is now available as an experimental feature, where users no longer need the extra steps of enabling it by opening a console for the editor, or switching to a general-purpose IDE.
 
 Elyra also allows submitting a Python and R scripts as a single node pipeline for execution in a Kubeflow Pipelines or Apache Airflow environment in the cloud. This feature is accessible when the Elyra [AI Pipelines](../user_guide/pipelines.md) extension is also enabled.
 
@@ -38,6 +39,39 @@ allowing users to run their scripts with remote kernels with more specialized re
 
 To run your script locally, select the `Python 3` option in the dropdown menu, and click the `Run` icon.
 
+## Python script debugging support (experimental)
+
+Elyra users can now expand their development experience and start debugging scripts from the Python Editor.
+In this experimental stage we provide an integration between [Elyra's Script Editor](https://github.com/elyra-ai/elyra/tree/main/packages/script-editor) and the existing [JupyterLab's debugger](https://jupyterlab.readthedocs.io/en/stable/user/debugger.html), allowing basic debugging tasks such as setting breakpoints to Python scripts, inspecting variables and navigating the call stack.
+
+Elyra extends JupyterLab's visual debugger which can be visible and enabled from the editor's toolbar if a kernel with debugger support is installed and selected.
+
+Currently only the Jupyter kernels below are known to support the Jupyter Debug Protocol, the first two for Python programming language:
+- [ipykernel](https://github.com/ipython/ipykernel) (6.0+)
+- [xeus-python](https://github.com/jupyter-xeus/xeus-python)
+- [xeus-robot](https://github.com/jupyter-xeus/xeus-robot)
+
+To list installed kernels run in a terminal window
+```bash
+jupyter kernelspec list
+```
+
+Once a kernel with supporting debugger is selected, the debugger can be enabled by clicking the bug button in the editor's toolbar. A sidebar will display a variable explorer, the list of breakpoints, a source preview and buttons to navigate the call stack.
+
+The user sets breakpoints from the editor's UI, the `Run` button executes the script, and visual markers will indicate where the current execution has hit a breakpoint.
+
+![Debugger usage](../images/debugger.gif)
+
+Since Elyra's Python debugging support is experimental, [here](https://github.com/elyra-ai/elyra/pull/2087) you can find a list of known issues.
+For feedback, bug reports and enhancement suggestions, before opening an issue in Elyra's repository, please also check [existing debugger issues open in JupyterLab](https://github.com/jupyterlab/jupyterlab/issues?q=is%3Aopen+is%3Aissue+label%3Apkg%3Adebugger).
+
+### Troubleshooting
+- Interrupting the kernel while debugger is running does not trigger breakpoints on subsequent debug runs (same behavior in notebooks).
+Solution:
+Open `Running termninal and kernels` left side tab, find and select the relevant file path under `Kernels`, click the `x` button to shut down and reload the page.
+
+![Manually restart the debugger service](../images/kernel-shutdown.png)
+
 ## R script execution support
 
 In the JupyterLab Launcher, click the `R Editor` icon to create a new R script and open the R Editor.

packages/python-editor/package.json

@@ -47,7 +47,6 @@
     "@elyra/ui-components": "3.11.0-dev",
     "@jupyterlab/application": "^3.4.0",
     "@jupyterlab/apputils": "^3.4.0",
-    "@jupyterlab/builder": "^3.4.0",
     "@jupyterlab/codeeditor": "^3.4.0",
     "@jupyterlab/docregistry": "^3.4.0",
     "@jupyterlab/filebrowser": "^3.4.0",
@@ -59,6 +58,7 @@
     "@lumino/coreutils": "^1.5.6"
   },
   "devDependencies": {
+    "@jupyterlab/builder": "^3.4.0",
     "rimraf": "^3.0.2",
     "typescript": "~4.1.3"
   },

packages/script-debugger/package.json

@@ -0,0 +1,65 @@
+{
+  "name": "@elyra/script-debugger-extension",
+  "version": "3.10.0-dev",
+  "description": "JupyterLab extension - visual debugging support for script editors",
+  "keywords": [
+    "jupyter",
+    "jupyterlab",
+    "jupyterlab-extension"
+  ],
+  "homepage": "https://github.com/elyra-ai/elyra",
+  "bugs": {
+    "url": "https://github.com/elyra-ai/elyra/issues"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/elyra-ai/elyra/"
+  },
+  "license": "Apache-2.0",
+  "main": "lib/index.js",
+  "types": "lib/index.d.ts",
+  "style": "style/index.css",
+  "files": [
+    "lib/**/*.{d.ts,eot,gif,html,jpg,js,js.map,json,png,svg,woff2,ttf}",
+    "src/**/*.{ts,tsx}",
+    "style/**/*.{css,eot,gif,html,jpg,json,png,svg,woff2,ttf}"
+  ],
+  "scripts": {
+    "build": "jlpm run build:lib && jlpm run build:labextension:dev",
+    "build:prod": "jlpm run build:lib && jlpm run build:labextension",
+    "build:lib": "tsc",
+    "build:labextension": "jupyter labextension build .",
+    "build:labextension:dev": "jupyter labextension build --development True .",
+    "clean": "rimraf lib tsconfig.tsbuildinfo ../../dist/labextensions/@elyra/script-debugger-extension",
+    "lab:dev": "jupyter labextension develop --overwrite ../../dist/labextensions/@elyra/script-debugger-extension",
+    "dist": "npm pack .",
+    "prepare": "npm run build",
+    "watch": "run-p watch:src watch:labextension",
+    "watch:src": "tsc -w",
+    "watch:labextension": "jupyter labextension watch .",
+    "lab:install": "jupyter labextension install --no-build",
+    "lab:uninstall": "jupyter labextension uninstall --no-build",
+    "link:dev": "yarn link @jupyterlab/builder",
+    "unlink:dev": "yarn unlink @jupyterlab/builder"
+  },
+  "dependencies": {
+    "@elyra/script-editor": "3.11.0-dev",
+    "@jupyterlab/application": "^3.4.0",
+    "@jupyterlab/debugger": "^3.4.0",
+    "@jupyterlab/fileeditor": "^3.4.0",
+    "@jupyterlab/services": "^6.4.0",
+    "@lumino/widgets": "^1.19.0"
+  },
+  "devDependencies": {
+    "@jupyterlab/builder": "^3.4.0",
+    "rimraf": "^3.0.2",
+    "typescript": "~4.1.3"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "jupyterlab": {
+    "extension": true,
+    "outputDir": "../../dist/labextensions/@elyra/script-debugger-extension"
+  }
+}

packages/script-debugger/src/index.ts

@@ -0,0 +1,194 @@
+/*
+ * Copyright 2018-2022 Elyra Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import { ScriptEditor } from '@elyra/script-editor';
+import {
+  ILabShell,
+  JupyterFrontEnd,
+  JupyterFrontEndPlugin
+} from '@jupyterlab/application';
+import { Debugger, IDebugger } from '@jupyterlab/debugger';
+import { IEditorTracker } from '@jupyterlab/fileeditor';
+import { KernelManager, Session, SessionManager } from '@jupyterlab/services';
+import { Widget } from '@lumino/widgets';
+
+/**
+ * Debugger plugin.
+ * Adapted from JupyterLab debugger extension.
+ * A plugin that provides visual debugging support for script editors.
+ */
+const scriptEditorDebuggerExtension: JupyterFrontEndPlugin<void> = {
+  id: 'elyra-script-debugger',
+  autoStart: true,
+  requires: [IDebugger, IEditorTracker],
+  optional: [ILabShell],
+  activate: (
+    app: JupyterFrontEnd,
+    debug: IDebugger,
+    editorTracker: IEditorTracker,
+    labShell: ILabShell | null
+  ) => {
+    console.log('Elyra - script-debugger extension is activated!');
+
+    const handler = new Debugger.Handler({
+      type: 'file',
+      shell: app.shell,
+      service: debug
+    });
+
+    const activeSessions: { [id: string]: Session.ISessionConnection } = {};
+    const kernelManager = new KernelManager();
+    const sessionManager = new SessionManager({
+      kernelManager: kernelManager
+    });
+
+    const updateDebugger = async (widget: ScriptEditor): Promise<void> => {
+      const widgetInFocus = app.shell.currentWidget;
+      if (widget !== widgetInFocus) {
+        return;
+      }
+
+      const kernelSelection = (widget as ScriptEditor).kernelSelection;
+      const debuggerAvailable = await widget.debuggerAvailable(kernelSelection);
+
+      if (!debuggerAvailable) {
+        return;
+      }
+
+      const sessions = app.serviceManager.sessions;
+      try {
+        const path = widget.context.path;
+        let sessionModel = await sessions.findByPath(path);
+        if (!sessionModel) {
+          // Start a kernel session for the selected kernel supporting debug
+          const sessionConnection = await startSession(kernelSelection, path);
+          sessionModel = await sessions.findByPath(path);
+          if (sessionConnection && sessionModel) {
+            activeSessions[sessionModel.id] = sessionConnection;
+          }
+        }
+
+        if (sessionModel) {
+          let sessionConnection: Session.ISessionConnection | null =
+            activeSessions[sessionModel.id];
+          if (!sessionConnection) {
+            // Use `connectTo` only if the session does not exist.
+            // `connectTo` sends a kernel_info_request on the shell
+            // channel, which blocks the debug session restore when waiting
+            // for the kernel to be ready
+            sessionConnection = sessions.connectTo({ model: sessionModel });
+            activeSessions[sessionModel.id] = sessionConnection;
+          }
+          if (sessionModel.kernel?.name !== kernelSelection) {
+            // New kernel selection detected, restart session connection for new kernel
+            await shutdownSession(sessionConnection);
+
+            sessionConnection = await startSession(kernelSelection, path);
+            sessionModel = await sessions.findByPath(path);
+            if (sessionConnection && sessionModel) {
+              activeSessions[sessionModel.id] = sessionConnection;
+            }
+          }
+
+          // Temporary solution to give enough time for the handler to update the UI on page reload.
+          setTimeout(async () => {
+            await handler.update(widget, sessionConnection);
+            app.commands.notifyCommandChanged();
+          }, 1000);
+        }
+      } catch (error) {
+        console.warn(
+          'Exception: session connection = ' + JSON.stringify(error)
+        );
+      }
+    };
+
+    // Use a weakmap to track the callback function used by signal listeners
+    // The object is cleared by garbabe collector when no longer in use avoiding memory leaks
+    // Key: ScriptEditor widget
+    // Value: instance of updateDebugger function
+    const callbackControl = new WeakMap<ScriptEditor, () => Promise<void>>();
+
+    const update = async (widget: Widget | null): Promise<void> => {
+      if (widget instanceof ScriptEditor) {
+        let callbackFn = callbackControl.get(widget);
+        if (!callbackFn) {
+          callbackFn = (): Promise<void> => updateDebugger(widget);
+          callbackControl.set(widget, callbackFn);
+        }
+        updateDebugger(widget);
+
+        // Listen to possible kernel selection changes
+        widget.kernelSelectionChanged.disconnect(callbackFn);
+        widget.kernelSelectionChanged.connect(callbackFn);
+      }
+    };
+
+    if (labShell) {
+      // Listen to main area's current focus changes.
+      labShell.currentChanged.connect((_, widget) => {
+        return update(widget.newValue);
+      });
+    }
+
+    if (editorTracker) {
+      // Listen to script editor's current instance changes.
+      editorTracker.currentChanged.connect((_, widget) => {
+        return update(widget);
+      });
+    }
+
+    const startSession = async (
+      kernelSelection: string,
+      path: string
+    ): Promise<Session.ISessionConnection | null> => {
+      const options: Session.ISessionOptions = {
+        kernel: {
+          name: kernelSelection
+        },
+        path: path,
+        type: 'file',
+        name: path
+      };
+      let sessionConnection;
+      try {
+        sessionConnection = await sessionManager.startNew(options);
+        sessionConnection.setPath(path);
+        console.log(
+          `Kernel session started for ${path} with selected ${kernelSelection} kernel.`
+        );
+      } catch (error) {
+        console.warn('Exception: start session = ' + JSON.stringify(error));
+        sessionConnection = null;
+      }
+      return sessionConnection;
+    };
+
+    const shutdownSession = async (
+      sessionConnection: Session.ISessionConnection
+    ): Promise<void> => {
+      try {
+        const kernelName = sessionConnection.kernel?.name;
+        await sessionConnection.shutdown();
+        console.log(`${kernelName} kernel shut down.`);
+      } catch (error) {
+        console.warn('Exception: shutdown = ' + JSON.stringify(error));
+      }
+    };
+  }
+};
+
+export default scriptEditorDebuggerExtension;

packages/script-debugger/style/index.css

@@ -0,0 +1,15 @@
+/*
+ * Copyright 2018-2022 Elyra Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */

packages/script-debugger/tsconfig.json

@@ -0,0 +1,7 @@
+{
+  "extends": "../../tsconfig.base.json",
+  "compilerOptions": {
+    "outDir": "lib"
+  },
+  "include": ["src"]
+}