diff --git a/DEEPNOTE_KERNEL_IMPLEMENTATION.md b/DEEPNOTE_KERNEL_IMPLEMENTATION.md new file mode 100644 index 0000000000..91caf6208a --- /dev/null +++ b/DEEPNOTE_KERNEL_IMPLEMENTATION.md @@ -0,0 +1,391 @@ +# Deepnote Kernel Auto-Start Implementation + +## Overview + +This implementation adds automatic kernel selection and startup for `.deepnote` notebook files. When a user opens and runs cells in a `.deepnote` file, the extension will: + +1. Automatically detect the file type +2. Install the `deepnote-toolkit` Python package (if not already installed) +3. Start a Jupyter server using `deepnote-toolkit` +4. Create and auto-select a Deepnote kernel controller +5. Execute cells on the Deepnote kernel + +## Architecture + +### Components Created + +#### 1. **Deepnote Kernel Types** (`src/kernels/deepnote/types.ts`) +- `DeepnoteKernelConnectionMetadata`: Connection metadata for Deepnote kernels (similar to RemoteKernelSpecConnectionMetadata) +- `IDeepnoteToolkitInstaller`: Interface for toolkit installation service +- `IDeepnoteServerStarter`: Interface for server management +- `IDeepnoteKernelAutoSelector`: Interface for automatic kernel selection +- `DeepnoteServerInfo`: Server connection information (URL, port, token) +- Constants for wheel URL, default port, and notebook type + +#### 2. **Deepnote Toolkit Installer** (`src/kernels/deepnote/deepnoteToolkitInstaller.node.ts`) +- Creates a dedicated virtual environment per `.deepnote` file +- Checks if `deepnote-toolkit` is installed in the venv +- Installs the toolkit and `ipykernel` from the hardcoded S3 wheel URL +- **Registers a kernel spec** using `ipykernel install --user --name deepnote-venv-` that points to the venv's Python interpreter +- This ensures packages installed via `pip` are available to the kernel +- Outputs installation progress to the output channel +- Verifies successful installation +- Reuses existing venvs for the same `.deepnote` file + +**Key Methods:** +- `getVenvInterpreter(deepnoteFileUri)`: Gets the venv Python interpreter for a specific file +- `ensureInstalled(interpreter, deepnoteFileUri)`: Creates venv, installs toolkit and ipykernel, and registers kernel spec +- `getVenvHash(deepnoteFileUri)`: Creates a unique hash for both kernel naming and venv directory paths +- `getDisplayName(deepnoteFileUri)`: Gets a friendly display name for the kernel + +#### 3. **Deepnote Server Starter** (`src/kernels/deepnote/deepnoteServerStarter.node.ts`) +- Manages the lifecycle of deepnote-toolkit Jupyter servers (one per `.deepnote` file) +- Finds an available port (starting from 8888) +- Starts the server with `python -m deepnote_toolkit server --jupyter-port ` +- **Sets environment variables** so shell commands use the venv's Python: + - Prepends venv's bin directory to `PATH` + - Sets `VIRTUAL_ENV` to the venv path + - Removes `PYTHONHOME` to avoid conflicts +- Monitors server output and logs it +- Waits for server to be ready before returning connection info +- Reuses existing server for the same `.deepnote` file if already running +- Manages multiple servers for different `.deepnote` files simultaneously + +**Key Methods:** +- `getOrStartServer(interpreter, deepnoteFileUri)`: Returns server info for a file, starting if needed +- `stopServer(deepnoteFileUri)`: Stops the running server for a specific file +- `isServerRunning(serverInfo)`: Checks if server is responsive + +#### 4. **Deepnote Server Provider** (`src/kernels/deepnote/deepnoteServerProvider.node.ts`) +- Jupyter server provider that registers and resolves Deepnote toolkit servers +- Implements `JupyterServerProvider` interface from VSCode Jupyter API +- Maintains a map of server handles to server connection information +- Allows the kernel infrastructure to resolve server connections + +**Key Methods:** +- `activate()`: Registers the server provider with the Jupyter server provider registry +- `registerServer(handle, serverInfo)`: Registers a Deepnote server for a specific handle +- `provideJupyterServers(token)`: Lists all registered Deepnote servers +- `resolveJupyterServer(server, token)`: Resolves server connection info by handle + +#### 5. **Deepnote Kernel Auto-Selector** (`src/notebooks/deepnote/deepnoteKernelAutoSelector.node.ts`) +- Activation service that listens for notebook open events and controller selection changes +- Automatically selects Deepnote kernel for `.deepnote` files +- Queries the Deepnote server for available kernel specs +- **Prefers the venv kernel spec** (`deepnote-venv-`) that was registered by the installer and uses the venv's Python interpreter +- This ensures the kernel uses the same environment where packages are installed +- Falls back to other Python kernel specs if venv kernel not found +- Registers the server with the server provider +- Creates kernel connection metadata +- Registers the controller with VSCode +- Auto-selects the kernel for the notebook +- **Reuses existing controllers and servers** for persistent kernel sessions +- **Automatically reselects kernel** if it becomes unselected after errors +- Tracks controllers per notebook file for efficient reuse + +**Key Methods:** +- `activate()`: Registers event listeners for notebook open/close and controller selection changes +- `ensureKernelSelected(notebook)`: Main logic for auto-selection, kernel spec selection, and kernel reuse +- `onDidOpenNotebook(notebook)`: Event handler for notebook opens +- `onControllerSelectionChanged(event)`: Event handler for controller selection changes (auto-reselects if needed) +- `onDidCloseNotebook(notebook)`: Event handler for notebook closes (preserves controllers for reuse) + +#### 6. **Service Registry Updates** (`src/notebooks/serviceRegistry.node.ts`) +- Registers all new Deepnote kernel services +- Binds `DeepnoteServerProvider` as an activation service +- Binds `IDeepnoteKernelAutoSelector` as an activation service + +#### 7. **Kernel Types Updates** (`src/kernels/types.ts`) +- Adds `DeepnoteKernelConnectionMetadata` to `RemoteKernelConnectionMetadata` union type +- Adds deserialization support for `'startUsingDeepnoteKernel'` kind + +## Flow Diagram + +``` +User opens .deepnote file + ↓ +DeepnoteKernelAutoSelector.onDidOpenNotebook() + ↓ +Check if kernel already selected → Yes → Exit + ↓ No +Get active Python interpreter + ↓ +DeepnoteToolkitInstaller.ensureInstalled() + ↓ +Extract base file URI (remove query params) + ↓ +Check if venv exists for this file → Yes → Skip to server + ↓ No +Create venv for this .deepnote file + ↓ +pip install deepnote-toolkit[server] and ipykernel in venv + ↓ +Register kernel spec pointing to venv's Python + ↓ +Verify installation + ↓ +DeepnoteServerStarter.getOrStartServer(venv, fileUri) + ↓ +Check if server running for this file → Yes → Return info + ↓ No +Find available port + ↓ +Start: python -m deepnote_toolkit server --jupyter-port + ↓ +Wait for server to be ready (poll /api endpoint) + ↓ +Register server with DeepnoteServerProvider + ↓ +Query server for available kernel specs + ↓ +Select venv kernel spec (deepnote-venv-) or fall back to other Python kernel + ↓ +Create DeepnoteKernelConnectionMetadata with server kernel spec + ↓ +Register controller with IControllerRegistration + ↓ +Set controller affinity to Preferred (auto-selects kernel) + ↓ +User runs cell → Executes on Deepnote kernel +``` + +## Configuration + +### Hardcoded Values (as requested) +- **Wheel URL**: `https://deepnote-staging-runtime-artifactory.s3.amazonaws.com/deepnote-toolkit-packages/0.2.30.post20/deepnote_toolkit-0.2.30.post20-py3-none-any.whl` +- **Default Port**: `8888` (will find next available if occupied) +- **Notebook Type**: `deepnote` +- **Venv Location**: `~/.vscode/extensions/storage/deepnote-venvs//` (e.g., `venv_a1b2c3d4`) +- **Server Provider ID**: `deepnote-server` +- **Kernel Spec Name**: `deepnote-venv-` (registered via ipykernel to point to venv Python) +- **Kernel Display Name**: `Deepnote ()` + +## Usage + +1. **Open a .deepnote file** in VSCode + - A temporary "Loading Deepnote Kernel..." controller is automatically selected + - A progress notification appears in the bottom-right +2. **You can immediately click "Run All" or run individual cells** + - Cells will wait for the kernel to be ready before executing + - No kernel selection dialog will appear +3. **Once the progress notification shows "Kernel ready!"**: + - The loading controller is automatically replaced with the real Deepnote kernel + - Your cells start executing +4. The extension automatically: + - Installs deepnote-toolkit in a dedicated virtual environment (first time only) + - Starts a Deepnote server on an available port (if not already running) + - Selects the appropriate Deepnote kernel + - Executes your cells + +**First-time setup** takes 15-30 seconds. **Subsequent opens** of the same file reuse the existing environment and server, taking less than 1 second. + +## Benefits + +- **Zero configuration**: No manual kernel selection needed +- **Automatic setup**: Toolkit installation and server startup handled automatically +- **Isolated environments**: Each `.deepnote` file gets its own virtual environment +- **Multi-file support**: Can run multiple `.deepnote` files with separate servers +- **Resource efficiency**: Reuses venv and server for notebooks within the same `.deepnote` file +- **Clean integration**: Uses existing VSCode notebook controller infrastructure +- **Proper server resolution**: Implements Jupyter server provider for proper kernel connection handling +- **Compatible kernel specs**: Uses kernel specs that exist on the Deepnote server +- **Persistent kernel sessions**: Controllers and servers remain available even after errors +- **Automatic recovery**: If kernel becomes unselected, it's automatically reselected +- **Seamless reusability**: Run cells as many times as you want without manual kernel selection + +## UI Customization + +### Hidden UI Elements for Deepnote Notebooks + +To provide a cleaner experience for Deepnote notebooks, the following UI elements are hidden when working with `.deepnote` files: + +1. **Notebook Toolbar Buttons:** + - Restart Kernel + - Variable View + - Outline View + - Export + - Codespace Integration + +2. **Cell Title Menu Items:** + - Run by Line + - Run by Line Next/Stop + - Select Precedent/Dependent Cells + +3. **Cell Execute Menu Items:** + - Run and Debug Cell + - Run Precedent/Dependent Cells + +These items are hidden by adding `notebookType != 'deepnote'` conditions to the `when` clauses in `package.json`. The standard cell run buttons (play icons) remain visible as they are the primary way to execute cells. + +### Progress Indicators + +The extension shows a visual progress notification while the Deepnote kernel is being set up: + +- **Location**: Notification area (bottom-right) +- **Title**: "Loading Deepnote Kernel" +- **Cancellable**: Yes +- **Progress Steps**: + 1. "Setting up Deepnote kernel..." + 2. "Finding Python interpreter..." + 3. "Installing Deepnote toolkit..." (shown if installation is needed) + 4. "Starting Deepnote server..." (shown if server needs to be started) + 5. "Connecting to kernel..." + 6. "Finalizing kernel setup..." + 7. "Kernel ready!" + +For notebooks that already have a running kernel, the notification shows "Reusing existing kernel..." and completes quickly. + +**Important**: When you first open a `.deepnote` file, a temporary "Loading Deepnote Kernel..." controller is automatically selected. This prevents the kernel selection dialog from appearing. The kernel setup happens automatically in the background. During this loading period (typically 5-30 seconds for first-time setup, < 1 second for subsequent opens), if you try to run cells, they will wait until the real kernel is ready. Once ready, the loading controller is automatically replaced with the real Deepnote kernel and your cells will execute. + +## Future Enhancements + +1. **PyPI Package**: Replace S3 URL with PyPI package name once published +2. **Configuration**: Add settings for custom ports, wheel URLs, etc. +3. **Server Management UI**: Add commands to start/stop/restart servers for specific files +4. **Venv Cleanup**: Add command to clean up unused venvs +5. **Error Recovery**: Better handling of server crashes and auto-restart + +## Testing + +To test the implementation: + +1. Create a `.deepnote` file +2. Add Python code cells +3. Run a cell +4. Verify: + - Toolkit gets installed (check output channel) + - Server starts (check output channel) + - Kernel is auto-selected (check kernel picker) + - Code executes successfully + +## Files Modified/Created + +### Created: +- `src/kernels/deepnote/types.ts` - Type definitions and interfaces +- `src/kernels/deepnote/deepnoteToolkitInstaller.node.ts` - Toolkit installation service +- `src/kernels/deepnote/deepnoteServerStarter.node.ts` - Server lifecycle management +- `src/kernels/deepnote/deepnoteServerProvider.node.ts` - Jupyter server provider implementation +- `src/notebooks/deepnote/deepnoteKernelAutoSelector.node.ts` - Automatic kernel selection + +### Modified: +- `src/kernels/types.ts` - Added DeepnoteKernelConnectionMetadata to union types +- `src/notebooks/serviceRegistry.node.ts` - Registered new services + +## Dependencies + +- `get-port`: For finding available ports +- Existing VSCode notebook infrastructure +- Existing kernel controller system +- Python interpreter service +- Jupyter server provider registry +- JupyterLab session management + +## Technical Details + +### Server Provider Architecture + +The implementation uses VSCode's Jupyter server provider API to properly integrate Deepnote servers: + +1. **DeepnoteServerProvider** implements the `JupyterServerProvider` interface +2. It registers with the `IJupyterServerProviderRegistry` during activation +3. When a Deepnote server is started, it's registered with the provider using a unique handle +4. The kernel infrastructure can then resolve the server connection through this provider +5. This allows the kernel session factory to properly connect to the Deepnote server + +### Kernel Spec Resolution + +The implementation uses a hybrid approach: + +1. **Registers per-venv kernel specs**: The installer registers kernel specs using `ipykernel install --user --name deepnote-venv-` that point to each venv's Python interpreter +2. **Queries server for available specs**: Connects to the running Deepnote server using `JupyterLabHelper` and queries available kernel specs via `getKernelSpecs()` +3. **Prefers venv kernel specs**: Looks for the registered venv kernel spec (`deepnote-venv-`) first +4. **Falls back gracefully**: Falls back to other Python kernel specs (like `python3-venv`) if the venv kernel spec is not found +5. **Uses server-compatible specs**: This ensures compatibility with the Deepnote server's kernel configuration while maintaining venv isolation + +### Virtual Environment Path Handling + +The implementation uses a robust hashing approach for virtual environment directory names: + +1. **Path Hashing**: Uses `getVenvHash()` to create short, unique identifiers from file paths +2. **Hash Algorithm**: Implements a djb2-style hash function for better distribution +3. **Format**: Generates identifiers like `venv_a1b2c3d4` (max 16 characters) +4. **Benefits**: + - Avoids Windows MAX_PATH (260 character) limitations + - Prevents directory structure leakage into extension storage + - Provides consistent naming for both venv directories and kernel specs + - Reduces collision risk with better hash distribution + +## Troubleshooting & Key Fixes + +### Issue 1: "Unable to get resolved server information" + +**Problem**: The kernel infrastructure couldn't resolve the server connection because the `serverProviderHandle` pointed to a non-existent server provider. + +**Solution**: Created `DeepnoteServerProvider` that implements the `JupyterServerProvider` interface and registered it with the `IJupyterServerProviderRegistry`. This allows the kernel session factory to properly resolve server connections. + +### Issue 2: "No such kernel named python31211jvsc74a57bd0..." + +**Problem**: The extension was creating a custom kernel spec name based on the interpreter hash, but this kernel spec didn't exist on the Deepnote server. + +**Solution**: Instead of creating a custom kernel spec, the implementation now: +- Queries the Deepnote server for available kernel specs +- Selects an existing Python kernel (typically `python3-venv`) +- Uses this server-native kernel spec for the connection + +### Issue 3: "Kernel becomes unregistered after errors" + +**Problem**: When a cell execution resulted in an error, the kernel controller would sometimes become unselected or disposed. Subsequent attempts to run cells would fail because no kernel was selected, requiring manual intervention. + +**Solution**: Implemented persistent kernel tracking and automatic reselection: +- Controllers and connection metadata are stored per notebook file and reused across sessions +- Listens to `onControllerSelectionChanged` events to detect when a Deepnote kernel becomes unselected +- Automatically reselects the same kernel controller when it becomes deselected +- Reuses existing servers and controllers instead of creating new ones +- Ensures the same kernel remains available for the entire session, even after errors + +### Issue 4: "Controllers getting disposed causing repeated recreation" + +**Problem**: Controllers were being automatically disposed by VSCode's `ControllerRegistration` system when: +1. The kernel finder refreshed its list of available kernels +2. The Deepnote kernel wasn't in that list (because it's created dynamically) +3. The `loadControllers()` method would dispose controllers that weren't in the kernel finder's list +4. This led to a cycle of recreation, disposal, and race conditions + +**Root Cause**: The `ControllerRegistration.loadControllers()` method periodically checks if controllers are still valid by comparing them against the kernel finder's list. Controllers that aren't found and aren't "protected" get disposed. Deepnote controllers weren't protected, so they were being disposed and recreated repeatedly. + +**Solution**: Mark Deepnote controllers as protected using `trackActiveInterpreterControllers()`: +- Call `controllerRegistration.trackActiveInterpreterControllers(controllers)` when creating Deepnote controllers +- This adds them to the `_activeInterpreterControllerIds` set, which prevents disposal in `canControllerBeDisposed()` +- Controllers are now created **once** and persist for the entire session +- **No more recreation, no more debouncing, no more race conditions** +- The same controller instance handles all cell executions, even after errors + +These changes ensure that Deepnote notebooks can execute cells reliably by: +1. Providing a valid server provider that can be resolved +2. Using kernel specs that actually exist on the Deepnote server +3. Maintaining persistent kernel sessions that survive errors and can be reused indefinitely +4. **Preventing controller disposal entirely** - controllers are created once and reused forever + +### Issue 5: "Packages installed via pip not available in kernel" + +**Problem**: When users ran `!pip install matplotlib`, the package was installed successfully, but when they tried to import it, they got `ModuleNotFoundError`. This happened because: +1. The Jupyter server was running in the venv +2. But the kernel was using a different Python interpreter (system Python or different environment) +3. So `pip install` went to one environment, but imports came from another + +**Root Cause**: The kernel was using the venv's Python (correct), but shell commands (`!pip install`) were using the system Python or pyenv (wrong) because they inherit the shell's PATH environment variable. + +**Solution**: Two-part fix: +1. **Kernel spec registration** (ensures kernel uses venv Python): + - Install `ipykernel` in the venv along with deepnote-toolkit + - Use `python -m ipykernel install --user --name deepnote-venv-` to register a kernel spec that points to the venv's Python interpreter + - In the kernel selection logic, prefer the venv kernel spec (`deepnote-venv-`) when querying the server for available specs + +2. **Environment variable configuration** (ensures shell commands use venv Python): + - When starting the Jupyter server, set environment variables: + - Prepend venv's `bin/` directory to `PATH` + - Set `VIRTUAL_ENV` to point to the venv + - Remove `PYTHONHOME` (can interfere with venv) + - This ensures `!pip install` and other shell commands use the venv's Python + +**Result**: Both the kernel and shell commands now use the same Python environment (the venv), so packages installed via `!pip install` or `%pip install` are immediately available for import. diff --git a/package.json b/package.json index 4c4996533d..10c5773425 100644 --- a/package.json +++ b/package.json @@ -710,80 +710,80 @@ { "command": "jupyter.restartkernel", "group": "navigation/execute@5", - "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && notebookType == 'jupyter-notebook' && isWorkspaceTrusted && jupyter.kernel.isjupyter" + "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && isWorkspaceTrusted && jupyter.kernel.isjupyter" }, { "command": "jupyter.openVariableView", "group": "navigation@1", - "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && notebookType == 'jupyter-notebook' && isWorkspaceTrusted && jupyter.ispythonnotebook && jupyter.kernel.isjupyter" + "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && isWorkspaceTrusted && jupyter.ispythonnotebook && jupyter.kernel.isjupyter" }, { "command": "jupyter.openOutlineView", "group": "navigation@2", - "when": "notebookType == 'jupyter-notebook' && config.jupyter.showOutlineButtonInNotebookToolbar" + "when": "notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && config.jupyter.showOutlineButtonInNotebookToolbar" }, { "command": "jupyter.continueEditSessionInCodespace", "group": "navigation@3", - "when": "notebookType == 'jupyter-notebook' && jupyter.kernelSource == 'github-codespaces'" + "when": "notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && jupyter.kernelSource == 'github-codespaces'" }, { "command": "jupyter.notebookeditor.export", "group": "Jupyter", - "when": "notebookType == 'jupyter-notebook' && isWorkspaceTrusted" + "when": "notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && isWorkspaceTrusted" }, { "command": "jupyter.replayPylanceLogStep", "group": "navigation@1", - "when": "notebookType == 'jupyter-notebook' && isWorkspaceTrusted && jupyter.replayLogLoaded" + "when": "notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && isWorkspaceTrusted && jupyter.replayLogLoaded" } ], "notebook/cell/title": [ { "command": "jupyter.runByLine", - "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && notebookType == jupyter-notebook && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted && resource not in jupyter.notebookeditor.runByLineDocuments || !notebookKernel && notebookType == jupyter-notebook && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted", + "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && notebookType == jupyter-notebook && notebookType != deepnote && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted && resource not in jupyter.notebookeditor.runByLineDocuments || !notebookKernel && notebookType == jupyter-notebook && notebookType != deepnote && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted", "group": "inline/cell@0" }, { "command": "jupyter.runByLineNext", - "when": "notebookCellResource in jupyter.notebookeditor.runByLineCells", + "when": "notebookCellResource in jupyter.notebookeditor.runByLineCells && notebookType != deepnote", "group": "inline/cell@0" }, { "command": "jupyter.runByLineStop", - "when": "notebookCellResource in jupyter.notebookeditor.runByLineCells && notebookCellToolbarLocation == left", + "when": "notebookCellResource in jupyter.notebookeditor.runByLineCells && notebookType != deepnote && notebookCellToolbarLocation == left", "group": "inline/cell@1" }, { "command": "jupyter.runByLineStop", - "when": "notebookCellResource in jupyter.notebookeditor.runByLineCells && notebookCellToolbarLocation == right", + "when": "notebookCellResource in jupyter.notebookeditor.runByLineCells && notebookType != deepnote && notebookCellToolbarLocation == right", "group": "inline/cell@0" }, { "command": "jupyter.selectPrecedentCells", - "when": "notebookType == 'jupyter-notebook' && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", + "when": "notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", "group": "executionAnalysis@0" }, { "command": "jupyter.selectDependentCells", - "when": "notebookType == 'jupyter-notebook' && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", + "when": "notebookType == 'jupyter-notebook' && notebookType != 'deepnote' && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", "group": "executionAnalysis@1" } ], "notebook/cell/execute": [ { "command": "jupyter.runAndDebugCell", - "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted && resource not in jupyter.notebookeditor.debugDocuments || !notebookKernel && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted", + "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && jupyter.ispythonnotebook && notebookType != deepnote && notebookCellType == code && isWorkspaceTrusted && resource not in jupyter.notebookeditor.debugDocuments || !notebookKernel && jupyter.ispythonnotebook && notebookType != deepnote && notebookCellType == code && isWorkspaceTrusted", "group": "jupyterCellExecute@0" }, { "command": "jupyter.runPrecedentCells", - "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", + "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && jupyter.ispythonnotebook && notebookType != deepnote && notebookCellType == code && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", "group": "jupyterCellExecute@1" }, { "command": "jupyter.runDependentCells", - "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && jupyter.ispythonnotebook && notebookCellType == code && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", + "when": "notebookKernel =~ /^ms-toolsai.jupyter\\// && jupyter.ispythonnotebook && notebookType != deepnote && notebookCellType == code && isWorkspaceTrusted && config.jupyter.executionAnalysis.enabled", "group": "jupyterCellExecute@2" } ], diff --git a/src/kernels/deepnote/deepnoteServerProvider.node.ts b/src/kernels/deepnote/deepnoteServerProvider.node.ts new file mode 100644 index 0000000000..22450f3d69 --- /dev/null +++ b/src/kernels/deepnote/deepnoteServerProvider.node.ts @@ -0,0 +1,119 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { inject, injectable } from 'inversify'; +import { CancellationToken, Uri, Event, EventEmitter } from 'vscode'; +import { JupyterServer, JupyterServerProvider } from '../../api'; +import { IExtensionSyncActivationService } from '../../platform/activation/types'; +import { IDisposableRegistry } from '../../platform/common/types'; +import { IJupyterServerProviderRegistry } from '../jupyter/types'; +import { JVSC_EXTENSION_ID } from '../../platform/common/constants'; +import { logger } from '../../platform/logging'; +import { DeepnoteServerNotFoundError } from '../../platform/errors/deepnoteServerNotFoundError'; +import { DeepnoteServerInfo, IDeepnoteServerProvider } from './types'; + +/** + * Jupyter Server Provider for Deepnote kernels. + * This provider resolves server connections for Deepnote kernels. + */ +@injectable() +export class DeepnoteServerProvider + implements IDeepnoteServerProvider, IExtensionSyncActivationService, JupyterServerProvider +{ + public readonly id = 'deepnote-server'; + private readonly _onDidChangeServers = new EventEmitter(); + public readonly onDidChangeServers: Event = this._onDidChangeServers.event; + + // Map of server handles to server info + private servers = new Map(); + + constructor( + @inject(IJupyterServerProviderRegistry) + private readonly jupyterServerProviderRegistry: IJupyterServerProviderRegistry, + @inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry + ) {} + + public activate() { + // Register this server provider + const collection = this.jupyterServerProviderRegistry.createJupyterServerCollection( + JVSC_EXTENSION_ID, + this.id, + 'Deepnote Toolkit Server', + this + ); + this.disposables.push(collection); + logger.info('Deepnote server provider registered'); + } + + /** + * Register a server for a specific handle. + * Called by DeepnoteKernelAutoSelector when a server is started. + */ + public registerServer(handle: string, serverInfo: DeepnoteServerInfo): void { + logger.info(`Registering Deepnote server: ${handle} -> ${serverInfo.url}`); + this.servers.set(handle, serverInfo); + this._onDidChangeServers.fire(); + } + + /** + * Unregister a server for a specific handle. + * Called when the server is no longer needed or notebook is closed. + * No-op if the handle doesn't exist. + */ + public unregisterServer(handle: string): void { + if (this.servers.has(handle)) { + logger.info(`Unregistering Deepnote server: ${handle}`); + this.servers.delete(handle); + this._onDidChangeServers.fire(); + } + } + + /** + * Dispose of all servers and resources. + */ + public dispose(): void { + logger.info('Disposing Deepnote server provider, clearing all registered servers'); + this.servers.clear(); + this._onDidChangeServers.dispose(); + } + + /** + * Provides the list of available Deepnote servers. + */ + public async provideJupyterServers(_token: CancellationToken): Promise { + const servers: JupyterServer[] = []; + for (const [handle, info] of this.servers.entries()) { + servers.push({ + id: handle, + label: `Deepnote Toolkit (${info.port})`, + connectionInformation: { + baseUrl: Uri.parse(info.url), + token: info.token || '' + } + }); + } + return servers; + } + + /** + * Resolves a Jupyter server by its handle. + * This is called by the kernel infrastructure when starting a kernel. + */ + public async resolveJupyterServer(server: JupyterServer, _token: CancellationToken): Promise { + logger.info(`Resolving Deepnote server: ${server.id}`); + const serverInfo = this.servers.get(server.id); + + if (!serverInfo) { + throw new DeepnoteServerNotFoundError(server.id); + } + + return { + id: server.id, + label: server.label, + connectionInformation: { + baseUrl: Uri.parse(serverInfo.url), + token: serverInfo.token || '' + } + }; + } +} diff --git a/src/kernels/deepnote/deepnoteServerStarter.node.ts b/src/kernels/deepnote/deepnoteServerStarter.node.ts new file mode 100644 index 0000000000..cf4ee4e34f --- /dev/null +++ b/src/kernels/deepnote/deepnoteServerStarter.node.ts @@ -0,0 +1,280 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { inject, injectable, named } from 'inversify'; +import { CancellationToken, Uri } from 'vscode'; +import { PythonEnvironment } from '../../platform/pythonEnvironments/info'; +import { IDeepnoteServerStarter, IDeepnoteToolkitInstaller, DeepnoteServerInfo, DEEPNOTE_DEFAULT_PORT } from './types'; +import { IProcessServiceFactory, ObservableExecutionResult } from '../../platform/common/process/types.node'; +import { logger } from '../../platform/logging'; +import { IOutputChannel, IDisposable, IHttpClient } from '../../platform/common/types'; +import { STANDARD_OUTPUT_CHANNEL } from '../../platform/common/constants'; +import { sleep } from '../../platform/common/utils/async'; +import { Cancellation, raceCancellationError } from '../../platform/common/cancellation'; +import getPort from 'get-port'; + +/** + * Starts and manages the deepnote-toolkit Jupyter server. + */ +@injectable() +export class DeepnoteServerStarter implements IDeepnoteServerStarter { + private readonly serverProcesses: Map> = new Map(); + private readonly serverInfos: Map = new Map(); + private readonly disposablesByFile: Map = new Map(); + // Track in-flight operations per file to prevent concurrent start/stop + private readonly pendingOperations: Map> = new Map(); + + constructor( + @inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory, + @inject(IDeepnoteToolkitInstaller) private readonly toolkitInstaller: IDeepnoteToolkitInstaller, + @inject(IOutputChannel) @named(STANDARD_OUTPUT_CHANNEL) private readonly outputChannel: IOutputChannel, + @inject(IHttpClient) private readonly httpClient: IHttpClient + ) {} + + public async getOrStartServer( + interpreter: PythonEnvironment, + deepnoteFileUri: Uri, + token?: CancellationToken + ): Promise { + const fileKey = deepnoteFileUri.fsPath; + + // Wait for any pending operations on this file to complete + const pendingOp = this.pendingOperations.get(fileKey); + if (pendingOp) { + logger.info(`Waiting for pending operation on ${fileKey} to complete...`); + try { + await pendingOp; + } catch { + // Ignore errors from previous operations + } + } + + // If server is already running for this file, return existing info + const existingServerInfo = this.serverInfos.get(fileKey); + if (existingServerInfo && (await this.isServerRunning(existingServerInfo))) { + logger.info(`Deepnote server already running at ${existingServerInfo.url} for ${fileKey}`); + return existingServerInfo; + } + + // Start the operation and track it + const operation = this.startServerImpl(interpreter, deepnoteFileUri, token); + this.pendingOperations.set(fileKey, operation); + + try { + const result = await operation; + return result; + } finally { + // Remove from pending operations when done + if (this.pendingOperations.get(fileKey) === operation) { + this.pendingOperations.delete(fileKey); + } + } + } + + private async startServerImpl( + interpreter: PythonEnvironment, + deepnoteFileUri: Uri, + token?: CancellationToken + ): Promise { + const fileKey = deepnoteFileUri.fsPath; + + Cancellation.throwIfCanceled(token); + + // Ensure toolkit is installed + logger.info(`Ensuring deepnote-toolkit is installed for ${fileKey}...`); + const installed = await this.toolkitInstaller.ensureInstalled(interpreter, deepnoteFileUri, token); + if (!installed) { + throw new Error('Failed to install deepnote-toolkit. Please check the output for details.'); + } + + Cancellation.throwIfCanceled(token); + + // Find available port + const port = await getPort({ host: 'localhost', port: DEEPNOTE_DEFAULT_PORT }); + logger.info(`Starting deepnote-toolkit server on port ${port} for ${fileKey}`); + this.outputChannel.appendLine(`Starting Deepnote server on port ${port} for ${deepnoteFileUri.fsPath}...`); + + // Start the server with venv's Python in PATH + // This ensures shell commands (!) in notebooks use the venv's Python + // Use undefined as resource to get full system environment (including git in PATH) + const processService = await this.processServiceFactory.create(undefined); + + // Set up environment to ensure the venv's Python is used for shell commands + const venvBinDir = interpreter.uri.fsPath.replace(/\/python$/, '').replace(/\\python\.exe$/, ''); + const env = { ...process.env }; + + // Prepend venv bin directory to PATH so shell commands use venv's Python + env.PATH = `${venvBinDir}${process.platform === 'win32' ? ';' : ':'}${env.PATH || ''}`; + + // Also set VIRTUAL_ENV to indicate we're in a venv + const venvPath = venvBinDir.replace(/\/bin$/, '').replace(/\\Scripts$/, ''); + env.VIRTUAL_ENV = venvPath; + + // Remove PYTHONHOME if it exists (can interfere with venv) + delete env.PYTHONHOME; + + // Get the directory containing the notebook file to set as working directory + // This ensures relative file paths in the notebook work correctly + const notebookDir = Uri.joinPath(deepnoteFileUri, '..').fsPath; + + const serverProcess = processService.execObservable( + interpreter.uri.fsPath, + ['-m', 'deepnote_toolkit', 'server', '--jupyter-port', port.toString()], + { + env, + cwd: notebookDir + } + ); + + this.serverProcesses.set(fileKey, serverProcess); + + // Track disposables for this file + const disposables: IDisposable[] = []; + this.disposablesByFile.set(fileKey, disposables); + + // Monitor server output + serverProcess.out.onDidChange( + (output) => { + if (output.source === 'stdout') { + logger.trace(`Deepnote server (${fileKey}): ${output.out}`); + this.outputChannel.appendLine(output.out); + } else if (output.source === 'stderr') { + logger.warn(`Deepnote server stderr (${fileKey}): ${output.out}`); + this.outputChannel.appendLine(output.out); + } + }, + this, + disposables + ); + + // Wait for server to be ready + const url = `http://localhost:${port}`; + const serverInfo = { url, port }; + this.serverInfos.set(fileKey, serverInfo); + + try { + const serverReady = await this.waitForServer(serverInfo, 120000, token); + if (!serverReady) { + await this.stopServerImpl(deepnoteFileUri); + throw new Error('Deepnote server failed to start within timeout period'); + } + } catch (error) { + // Clean up leaked server before rethrowing + await this.stopServerImpl(deepnoteFileUri); + throw error; + } + + logger.info(`Deepnote server started successfully at ${url} for ${fileKey}`); + this.outputChannel.appendLine(`✓ Deepnote server running at ${url}`); + + return serverInfo; + } + + public async stopServer(deepnoteFileUri: Uri): Promise { + const fileKey = deepnoteFileUri.fsPath; + + // Wait for any pending operations on this file to complete + const pendingOp = this.pendingOperations.get(fileKey); + if (pendingOp) { + logger.info(`Waiting for pending operation on ${fileKey} before stopping...`); + try { + await pendingOp; + } catch { + // Ignore errors from previous operations + } + } + + // Start the stop operation and track it + const operation = this.stopServerImpl(deepnoteFileUri); + this.pendingOperations.set(fileKey, operation); + + try { + await operation; + } finally { + // Remove from pending operations when done + if (this.pendingOperations.get(fileKey) === operation) { + this.pendingOperations.delete(fileKey); + } + } + } + + private async stopServerImpl(deepnoteFileUri: Uri): Promise { + const fileKey = deepnoteFileUri.fsPath; + const serverProcess = this.serverProcesses.get(fileKey); + + if (serverProcess) { + try { + logger.info(`Stopping Deepnote server for ${fileKey}...`); + serverProcess.proc?.kill(); + this.serverProcesses.delete(fileKey); + this.serverInfos.delete(fileKey); + this.outputChannel.appendLine(`Deepnote server stopped for ${fileKey}`); + } catch (ex) { + logger.error(`Error stopping Deepnote server: ${ex}`); + } + } + + const disposables = this.disposablesByFile.get(fileKey); + if (disposables) { + disposables.forEach((d) => d.dispose()); + this.disposablesByFile.delete(fileKey); + } + } + + private async waitForServer( + serverInfo: DeepnoteServerInfo, + timeout: number, + token?: CancellationToken + ): Promise { + const startTime = Date.now(); + while (Date.now() - startTime < timeout) { + Cancellation.throwIfCanceled(token); + if (await this.isServerRunning(serverInfo)) { + return true; + } + await raceCancellationError(token, sleep(500)); + } + return false; + } + + private async isServerRunning(serverInfo: DeepnoteServerInfo): Promise { + try { + // Try to connect to the Jupyter API endpoint + const exists = await this.httpClient.exists(`${serverInfo.url}/api`).catch(() => false); + return exists; + } catch { + return false; + } + } + + public dispose(): void { + logger.info('Disposing DeepnoteServerStarter - stopping all servers...'); + + // Stop all server processes + for (const [fileKey, serverProcess] of this.serverProcesses.entries()) { + try { + logger.info(`Stopping Deepnote server for ${fileKey}...`); + serverProcess.proc?.kill(); + } catch (ex) { + logger.error(`Error stopping Deepnote server for ${fileKey}: ${ex}`); + } + } + + // Dispose all tracked disposables + for (const [fileKey, disposables] of this.disposablesByFile.entries()) { + try { + disposables.forEach((d) => d.dispose()); + } catch (ex) { + logger.error(`Error disposing resources for ${fileKey}: ${ex}`); + } + } + + // Clear all maps + this.serverProcesses.clear(); + this.serverInfos.clear(); + this.disposablesByFile.clear(); + this.pendingOperations.clear(); + + logger.info('DeepnoteServerStarter disposed successfully'); + } +} diff --git a/src/kernels/deepnote/deepnoteSharedToolkitInstaller.node.ts b/src/kernels/deepnote/deepnoteSharedToolkitInstaller.node.ts new file mode 100644 index 0000000000..5ae50c931c --- /dev/null +++ b/src/kernels/deepnote/deepnoteSharedToolkitInstaller.node.ts @@ -0,0 +1,256 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { inject, injectable, named } from 'inversify'; +import { CancellationToken, Uri } from 'vscode'; +import { PythonEnvironment } from '../../platform/pythonEnvironments/info'; +import { IProcessServiceFactory } from '../../platform/common/process/types.node'; +import { logger } from '../../platform/logging'; +import { IOutputChannel, IExtensionContext } from '../../platform/common/types'; +import { STANDARD_OUTPUT_CHANNEL } from '../../platform/common/constants'; +import { IFileSystem } from '../../platform/common/platform/types'; +import { Cancellation } from '../../platform/common/cancellation'; +import { DEEPNOTE_TOOLKIT_WHEEL_URL, DEEPNOTE_TOOLKIT_VERSION } from './types'; + +/** + * Manages a shared installation of deepnote-toolkit in a versioned extension directory. + * This avoids installing the heavy wheel package in every virtual environment. + */ +@injectable() +export class DeepnoteSharedToolkitInstaller { + private readonly sharedInstallationPath: Uri; + private readonly versionFilePath: Uri; + private readonly toolkitVersion: string; + private installationPromise: Promise | undefined; + + constructor( + @inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory, + @inject(IOutputChannel) @named(STANDARD_OUTPUT_CHANNEL) private readonly outputChannel: IOutputChannel, + @inject(IExtensionContext) private readonly context: IExtensionContext, + @inject(IFileSystem) private readonly fs: IFileSystem + ) { + // Create versioned directory for shared toolkit installation + this.toolkitVersion = DEEPNOTE_TOOLKIT_VERSION; + this.sharedInstallationPath = Uri.joinPath( + this.context.globalStorageUri, + 'deepnote-shared-toolkit', + this.toolkitVersion + ); + this.versionFilePath = Uri.joinPath(this.sharedInstallationPath, 'version.txt'); + } + + /** + * Ensures the shared deepnote-toolkit installation is available. + * @param baseInterpreter The base Python interpreter to use for installation + * @param token Cancellation token + * @returns True if installation is ready, false if failed + */ + public async ensureSharedInstallation( + baseInterpreter: PythonEnvironment, + token?: CancellationToken + ): Promise { + // Check if already installed and up to date + if (await this.isInstalled()) { + logger.info(`Shared deepnote-toolkit v${this.toolkitVersion} is already installed`); + return true; + } + + // Prevent concurrent installations + if (this.installationPromise) { + logger.info('Waiting for existing shared toolkit installation to complete...'); + return await this.installationPromise; + } + + this.installationPromise = this.installSharedToolkit(baseInterpreter, token); + try { + const result = await this.installationPromise; + return result; + } finally { + this.installationPromise = undefined; + } + } + + /** + * Gets the path to the shared toolkit installation. + */ + public getSharedInstallationPath(): Uri { + return this.sharedInstallationPath; + } + + /** + * Tests if the shared installation can be imported by a Python interpreter. + * Useful for debugging import issues. + */ + public async testSharedInstallation(interpreter: PythonEnvironment): Promise { + try { + const processService = await this.processServiceFactory.create(interpreter.uri); + + // Test import with explicit path + const result = await processService.exec( + interpreter.uri.fsPath, + [ + '-c', + `import sys; sys.path.insert(0, '${this.sharedInstallationPath.fsPath}'); import deepnote_toolkit; print('shared import successful')` + ], + { throwOnStdErr: false } + ); + + const success = result.stdout.toLowerCase().includes('shared import successful'); + logger.info(`Shared installation test result: ${success ? 'SUCCESS' : 'FAILED'}`); + if (!success) { + logger.warn(`Shared installation test failed: stdout=${result.stdout}, stderr=${result.stderr}`); + } + return success; + } catch (ex) { + logger.error(`Shared installation test error: ${ex}`); + return false; + } + } + + /** + * Creates a .pth file in the given venv that points to the shared toolkit installation. + * @param venvInterpreter The venv Python interpreter + * @param token Cancellation token + */ + public async createPthFile(venvInterpreter: PythonEnvironment, token?: CancellationToken): Promise { + Cancellation.throwIfCanceled(token); + + // Ensure shared installation is available first + const isInstalled = await this.ensureSharedInstallation(venvInterpreter, token); + if (!isInstalled) { + throw new Error('Failed to ensure shared deepnote-toolkit installation'); + } + + // Find the correct site-packages directory by querying Python + const processService = await this.processServiceFactory.create(venvInterpreter.uri); + const sitePackagesResult = await processService.exec( + venvInterpreter.uri.fsPath, + ['-c', 'import site; print(site.getsitepackages()[0])'], + { throwOnStdErr: false } + ); + + if (!sitePackagesResult.stdout) { + throw new Error('Failed to determine site-packages directory'); + } + + const sitePackagesPath = Uri.file(sitePackagesResult.stdout.trim()); + + // Create site-packages directory if it doesn't exist + if (!(await this.fs.exists(sitePackagesPath))) { + await this.fs.createDirectory(sitePackagesPath); + } + + // Create .pth file pointing to shared installation + const pthFilePath = Uri.joinPath(sitePackagesPath, 'deepnote-toolkit.pth'); + const pthContent = `${this.sharedInstallationPath.fsPath}\n`; + + await this.fs.writeFile(pthFilePath, Buffer.from(pthContent, 'utf8')); + logger.info( + `Created .pth file at ${pthFilePath.fsPath} pointing to shared installation ${this.sharedInstallationPath.fsPath}` + ); + + // Verify the .pth file is working by testing import + const testResult = await processService.exec( + venvInterpreter.uri.fsPath, + ['-c', 'import sys; print("\\n".join(sys.path))'], + { throwOnStdErr: false } + ); + logger.info(`Python sys.path after .pth file creation: ${testResult.stdout}`); + } + + /** + * Checks if the shared installation exists and is up to date. + */ + private async isInstalled(): Promise { + try { + // Check if version file exists and matches current version + if (!(await this.fs.exists(this.versionFilePath))) { + return false; + } + + const versionContent = await this.fs.readFile(this.versionFilePath); + const installedVersion = versionContent.toString().trim(); + + if (installedVersion !== this.toolkitVersion) { + logger.info(`Version mismatch: installed=${installedVersion}, expected=${this.toolkitVersion}`); + return false; + } + + // Check if the actual package is installed + const packagePath = Uri.joinPath(this.sharedInstallationPath, 'deepnote_toolkit'); + return await this.fs.exists(packagePath); + } catch (ex) { + logger.debug(`Error checking shared installation: ${ex}`); + return false; + } + } + + /** + * Installs the shared toolkit in the versioned directory. + */ + private async installSharedToolkit( + baseInterpreter: PythonEnvironment, + token?: CancellationToken + ): Promise { + try { + Cancellation.throwIfCanceled(token); + + logger.info( + `Installing shared deepnote-toolkit v${this.toolkitVersion} to ${this.sharedInstallationPath.fsPath}` + ); + this.outputChannel.appendLine(`Installing shared deepnote-toolkit v${this.toolkitVersion}...`); + + // Create shared installation directory + await this.fs.createDirectory(this.sharedInstallationPath); + + // Remove existing installation if it exists + const existingPackage = Uri.joinPath(this.sharedInstallationPath, 'deepnote_toolkit'); + if (await this.fs.exists(existingPackage)) { + await this.fs.delete(existingPackage); + } + + // Install deepnote-toolkit to the shared directory + const processService = await this.processServiceFactory.create(baseInterpreter.uri); + const installResult = await processService.exec( + baseInterpreter.uri.fsPath, + [ + '-m', + 'pip', + 'install', + '--target', + this.sharedInstallationPath.fsPath, + '--upgrade', + `deepnote-toolkit[server] @ ${DEEPNOTE_TOOLKIT_WHEEL_URL}` + ], + { throwOnStdErr: false } + ); + + Cancellation.throwIfCanceled(token); + + if (installResult.stdout) { + this.outputChannel.appendLine(installResult.stdout); + } + if (installResult.stderr) { + this.outputChannel.appendLine(installResult.stderr); + } + + // Verify installation + if (await this.fs.exists(existingPackage)) { + // Write version file + await this.fs.writeFile(this.versionFilePath, Buffer.from(this.toolkitVersion, 'utf8')); + + logger.info(`Shared deepnote-toolkit v${this.toolkitVersion} installed successfully`); + this.outputChannel.appendLine(`✓ Shared deepnote-toolkit v${this.toolkitVersion} ready`); + return true; + } else { + logger.error('Shared deepnote-toolkit installation failed - package not found'); + this.outputChannel.appendLine('✗ Shared deepnote-toolkit installation failed'); + return false; + } + } catch (ex) { + logger.error(`Failed to install shared deepnote-toolkit: ${ex}`); + this.outputChannel.appendLine(`Error installing shared deepnote-toolkit: ${ex}`); + return false; + } + } +} diff --git a/src/kernels/deepnote/deepnoteToolkitInstaller.node.ts b/src/kernels/deepnote/deepnoteToolkitInstaller.node.ts new file mode 100644 index 0000000000..f893947a22 --- /dev/null +++ b/src/kernels/deepnote/deepnoteToolkitInstaller.node.ts @@ -0,0 +1,269 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { inject, injectable, named } from 'inversify'; +import { CancellationToken, Uri, workspace } from 'vscode'; +import { PythonEnvironment } from '../../platform/pythonEnvironments/info'; +import { IDeepnoteToolkitInstaller, DEEPNOTE_TOOLKIT_WHEEL_URL } from './types'; +import { IProcessServiceFactory } from '../../platform/common/process/types.node'; +import { logger } from '../../platform/logging'; +import { IOutputChannel, IExtensionContext } from '../../platform/common/types'; +import { STANDARD_OUTPUT_CHANNEL } from '../../platform/common/constants'; +import { IFileSystem } from '../../platform/common/platform/types'; +import { Cancellation } from '../../platform/common/cancellation'; + +/** + * Handles installation of the deepnote-toolkit Python package. + */ +@injectable() +export class DeepnoteToolkitInstaller implements IDeepnoteToolkitInstaller { + private readonly venvPythonPaths: Map = new Map(); + // Track in-flight installations per venv path to prevent concurrent installs + private readonly pendingInstallations: Map> = new Map(); + + constructor( + @inject(IProcessServiceFactory) private readonly processServiceFactory: IProcessServiceFactory, + @inject(IOutputChannel) @named(STANDARD_OUTPUT_CHANNEL) private readonly outputChannel: IOutputChannel, + @inject(IExtensionContext) private readonly context: IExtensionContext, + @inject(IFileSystem) private readonly fs: IFileSystem + ) {} + + private getVenvPath(deepnoteFileUri: Uri): Uri { + // Create a unique venv name based on the file path using a hash + // This avoids Windows MAX_PATH issues and prevents directory structure leakage + const hash = this.getVenvHash(deepnoteFileUri); + return Uri.joinPath(this.context.globalStorageUri, 'deepnote-venvs', hash); + } + + public async getVenvInterpreter(deepnoteFileUri: Uri): Promise { + const venvPath = this.getVenvPath(deepnoteFileUri); + const cacheKey = venvPath.fsPath; + + if (this.venvPythonPaths.has(cacheKey)) { + return { uri: this.venvPythonPaths.get(cacheKey)!, id: this.venvPythonPaths.get(cacheKey)!.fsPath }; + } + + // Check if venv exists + const pythonInVenv = + process.platform === 'win32' + ? Uri.joinPath(venvPath, 'Scripts', 'python.exe') + : Uri.joinPath(venvPath, 'bin', 'python'); + + if (await this.fs.exists(pythonInVenv)) { + this.venvPythonPaths.set(cacheKey, pythonInVenv); + return { uri: pythonInVenv, id: pythonInVenv.fsPath }; + } + + return undefined; + } + + public async ensureInstalled( + baseInterpreter: PythonEnvironment, + deepnoteFileUri: Uri, + token?: CancellationToken + ): Promise { + const venvPath = this.getVenvPath(deepnoteFileUri); + const venvKey = venvPath.fsPath; + + // Wait for any pending installation for this venv to complete + const pendingInstall = this.pendingInstallations.get(venvKey); + if (pendingInstall) { + logger.info(`Waiting for pending installation for ${venvKey} to complete...`); + try { + return await pendingInstall; + } catch { + // If the previous installation failed, continue to retry + logger.info(`Previous installation for ${venvKey} failed, retrying...`); + } + } + + // Check if venv already exists with toolkit installed + const existingVenv = await this.getVenvInterpreter(deepnoteFileUri); + if (existingVenv && (await this.isToolkitInstalled(existingVenv))) { + logger.info(`deepnote-toolkit venv already exists and is ready for ${deepnoteFileUri.fsPath}`); + return existingVenv; + } + + // Double-check for race condition: another caller might have started installation + // while we were checking the venv + const pendingAfterCheck = this.pendingInstallations.get(venvKey); + if (pendingAfterCheck) { + logger.info(`Another installation started for ${venvKey} while checking, waiting for it...`); + try { + return await pendingAfterCheck; + } catch { + logger.info(`Concurrent installation for ${venvKey} failed, retrying...`); + } + } + + // Start the installation and track it + const installation = this.installImpl(baseInterpreter, deepnoteFileUri, venvPath, token); + this.pendingInstallations.set(venvKey, installation); + + try { + const result = await installation; + return result; + } finally { + // Remove from pending installations when done + if (this.pendingInstallations.get(venvKey) === installation) { + this.pendingInstallations.delete(venvKey); + } + } + } + + private async installImpl( + baseInterpreter: PythonEnvironment, + deepnoteFileUri: Uri, + venvPath: Uri, + token?: CancellationToken + ): Promise { + try { + Cancellation.throwIfCanceled(token); + + logger.info(`Creating virtual environment at ${venvPath.fsPath} for ${deepnoteFileUri.fsPath}`); + this.outputChannel.appendLine(`Setting up Deepnote toolkit environment for ${deepnoteFileUri.fsPath}...`); + + // Create venv parent directory if it doesn't exist + const venvParentDir = Uri.joinPath(this.context.globalStorageUri, 'deepnote-venvs'); + await this.fs.createDirectory(venvParentDir); + + // Remove old venv if it exists but is broken + if (await this.fs.exists(venvPath)) { + logger.info('Removing existing broken venv'); + await workspace.fs.delete(venvPath, { recursive: true }); + } + + // Create new venv + // Use undefined as resource to get full system environment + const processService = await this.processServiceFactory.create(undefined); + const venvResult = await processService.exec(baseInterpreter.uri.fsPath, ['-m', 'venv', venvPath.fsPath], { + throwOnStdErr: false + }); + + // Log any stderr output (warnings, etc.) but don't fail on it + if (venvResult.stderr) { + logger.info(`venv creation stderr: ${venvResult.stderr}`); + } + + Cancellation.throwIfCanceled(token); + + // Verify venv was created successfully by checking for the Python interpreter + const venvInterpreter = await this.getVenvInterpreter(deepnoteFileUri); + if (!venvInterpreter) { + logger.error('Failed to create venv: Python interpreter not found after venv creation'); + if (venvResult.stderr) { + logger.error(`venv stderr: ${venvResult.stderr}`); + } + this.outputChannel.appendLine('Error: Failed to create virtual environment'); + return undefined; + } + + // Install deepnote-toolkit and ipykernel in venv + logger.info(`Installing deepnote-toolkit and ipykernel in venv from ${DEEPNOTE_TOOLKIT_WHEEL_URL}`); + this.outputChannel.appendLine('Installing deepnote-toolkit and ipykernel...'); + + // Use undefined as resource to get full system environment (including git in PATH) + const venvProcessService = await this.processServiceFactory.create(undefined); + const installResult = await venvProcessService.exec( + venvInterpreter.uri.fsPath, + [ + '-m', + 'pip', + 'install', + '--upgrade', + `deepnote-toolkit[server] @ ${DEEPNOTE_TOOLKIT_WHEEL_URL}`, + 'ipykernel' + ], + { throwOnStdErr: true } + ); + + Cancellation.throwIfCanceled(token); + + if (installResult.stdout) { + this.outputChannel.appendLine(installResult.stdout); + } + if (installResult.stderr) { + this.outputChannel.appendLine(installResult.stderr); + } + + // Verify installation + if (await this.isToolkitInstalled(venvInterpreter)) { + logger.info('deepnote-toolkit installed successfully in venv'); + + // Install kernel spec so the kernel uses this venv's Python + logger.info('Installing kernel spec for venv...'); + try { + // Reuse the process service with system environment + await venvProcessService.exec( + venvInterpreter.uri.fsPath, + [ + '-m', + 'ipykernel', + 'install', + '--user', + '--name', + `deepnote-venv-${this.getVenvHash(deepnoteFileUri)}`, + '--display-name', + `Deepnote (${this.getDisplayName(deepnoteFileUri)})` + ], + { throwOnStdErr: false } + ); + logger.info('Kernel spec installed successfully'); + } catch (ex) { + logger.warn(`Failed to install kernel spec: ${ex}`); + // Don't fail the entire installation if kernel spec creation fails + } + + this.outputChannel.appendLine('✓ Deepnote toolkit ready'); + return venvInterpreter; + } else { + logger.error('deepnote-toolkit installation failed'); + this.outputChannel.appendLine('✗ deepnote-toolkit installation failed'); + return undefined; + } + } catch (ex) { + logger.error(`Failed to set up deepnote-toolkit: ${ex}`); + this.outputChannel.appendLine(`Error setting up deepnote-toolkit: ${ex}`); + return undefined; + } + } + + private async isToolkitInstalled(interpreter: PythonEnvironment): Promise { + try { + // Use undefined as resource to get full system environment + const processService = await this.processServiceFactory.create(undefined); + const result = await processService.exec(interpreter.uri.fsPath, [ + '-c', + "import deepnote_toolkit; print('installed')" + ]); + return result.stdout.toLowerCase().includes('installed'); + } catch (ex) { + logger.debug(`deepnote-toolkit not found: ${ex}`); + return false; + } + } + + private getVenvHash(deepnoteFileUri: Uri): string { + // Create a short hash from the file path for kernel naming and venv directory + // This provides better uniqueness and prevents directory structure leakage + const path = deepnoteFileUri.fsPath; + + // Use a simple hash function for better distribution + let hash = 0; + for (let i = 0; i < path.length; i++) { + const char = path.charCodeAt(i); + hash = (hash << 5) - hash + char; + hash = hash & hash; // Convert to 32-bit integer + } + + // Convert to positive hex string and limit length + const hashStr = Math.abs(hash).toString(16); + return `venv_${hashStr}`.substring(0, 16); + } + + private getDisplayName(deepnoteFileUri: Uri): string { + // Get a friendly display name from the file path + const parts = deepnoteFileUri.fsPath.split('/'); + return parts[parts.length - 1] || 'notebook'; + } +} diff --git a/src/kernels/deepnote/types.ts b/src/kernels/deepnote/types.ts new file mode 100644 index 0000000000..2d74fdad57 --- /dev/null +++ b/src/kernels/deepnote/types.ts @@ -0,0 +1,148 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import * as vscode from 'vscode'; +import { IJupyterKernelSpec } from '../types'; +import { PythonEnvironment } from '../../platform/pythonEnvironments/info'; +import { JupyterServerProviderHandle } from '../jupyter/types'; +import { serializePythonEnvironment } from '../../platform/api/pythonApi'; +import { getTelemetrySafeHashedString } from '../../platform/telemetry/helpers'; + +/** + * Connection metadata for Deepnote Toolkit Kernels. + * This kernel connects to a Jupyter server started by deepnote-toolkit. + */ +export class DeepnoteKernelConnectionMetadata { + public readonly kernelModel?: undefined; + public readonly kind = 'startUsingDeepnoteKernel' as const; + public readonly id: string; + public readonly kernelSpec: IJupyterKernelSpec; + public readonly baseUrl: string; + public readonly interpreter?: PythonEnvironment; + public readonly serverProviderHandle: JupyterServerProviderHandle; + public readonly serverInfo?: DeepnoteServerInfo; // Store server info for connection + + private constructor(options: { + interpreter?: PythonEnvironment; + kernelSpec: IJupyterKernelSpec; + baseUrl: string; + id: string; + serverProviderHandle: JupyterServerProviderHandle; + serverInfo?: DeepnoteServerInfo; + }) { + this.interpreter = options.interpreter; + this.kernelSpec = options.kernelSpec; + this.baseUrl = options.baseUrl; + this.id = options.id; + this.serverProviderHandle = options.serverProviderHandle; + this.serverInfo = options.serverInfo; + } + + public static create(options: { + interpreter?: PythonEnvironment; + kernelSpec: IJupyterKernelSpec; + baseUrl: string; + id: string; + serverProviderHandle: JupyterServerProviderHandle; + serverInfo?: DeepnoteServerInfo; + }) { + return new DeepnoteKernelConnectionMetadata(options); + } + + public getHashId() { + return getTelemetrySafeHashedString(this.id); + } + + public toJSON() { + return { + id: this.id, + kernelSpec: this.kernelSpec, + interpreter: serializePythonEnvironment(this.interpreter), + baseUrl: this.baseUrl, + kind: this.kind, + serverProviderHandle: this.serverProviderHandle + }; + } +} + +export const IDeepnoteToolkitInstaller = Symbol('IDeepnoteToolkitInstaller'); +export interface IDeepnoteToolkitInstaller { + /** + * Ensures deepnote-toolkit is installed in a dedicated virtual environment. + * @param baseInterpreter The base Python interpreter to use for creating the venv + * @param deepnoteFileUri The URI of the .deepnote file (used to create a unique venv per file) + * @param token Cancellation token to cancel the operation + * @returns The Python interpreter from the venv if installed successfully, undefined otherwise + */ + ensureInstalled( + baseInterpreter: PythonEnvironment, + deepnoteFileUri: vscode.Uri, + token?: vscode.CancellationToken + ): Promise; + + /** + * Gets the venv Python interpreter if toolkit is installed, undefined otherwise. + * @param deepnoteFileUri The URI of the .deepnote file + */ + getVenvInterpreter(deepnoteFileUri: vscode.Uri): Promise; +} + +export const IDeepnoteServerStarter = Symbol('IDeepnoteServerStarter'); +export interface IDeepnoteServerStarter extends vscode.Disposable { + /** + * Starts or gets an existing deepnote-toolkit Jupyter server. + * @param interpreter The Python interpreter to use + * @param deepnoteFileUri The URI of the .deepnote file (for server management per file) + * @param token Cancellation token to cancel the operation + * @returns Connection information (URL, port, etc.) + */ + getOrStartServer( + interpreter: PythonEnvironment, + deepnoteFileUri: vscode.Uri, + token?: vscode.CancellationToken + ): Promise; + + /** + * Stops the deepnote-toolkit server if running. + * @param deepnoteFileUri The URI of the .deepnote file + */ + stopServer(deepnoteFileUri: vscode.Uri): Promise; +} + +export interface DeepnoteServerInfo { + url: string; + port: number; + token?: string; +} + +export const IDeepnoteServerProvider = Symbol('IDeepnoteServerProvider'); +export interface IDeepnoteServerProvider { + /** + * Register a server for a specific handle. + * Called by DeepnoteKernelAutoSelector when a server is started. + */ + registerServer(handle: string, serverInfo: DeepnoteServerInfo): void; + + /** + * Unregister a server for a specific handle. + * Called when the server is no longer needed or notebook is closed. + * No-op if the handle doesn't exist. + */ + unregisterServer(handle: string): void; +} + +export const IDeepnoteKernelAutoSelector = Symbol('IDeepnoteKernelAutoSelector'); +export interface IDeepnoteKernelAutoSelector { + /** + * Automatically selects and starts a Deepnote kernel for the given notebook. + * @param notebook The notebook document + * @param token Cancellation token to cancel the operation + */ + ensureKernelSelected(notebook: vscode.NotebookDocument, token?: vscode.CancellationToken): Promise; +} + +export const DEEPNOTE_TOOLKIT_WHEEL_URL = + 'https://deepnote-staging-runtime-artifactory.s3.amazonaws.com/deepnote-toolkit-packages/0.2.30.post20/deepnote_toolkit-0.2.30.post20-py3-none-any.whl'; +export const DEEPNOTE_TOOLKIT_VERSION = '0.2.30.post20'; +export const DEEPNOTE_DEFAULT_PORT = 8888; +export const DEEPNOTE_NOTEBOOK_TYPE = 'deepnote'; diff --git a/src/kernels/jupyter/finder/remoteKernelFinder.ts b/src/kernels/jupyter/finder/remoteKernelFinder.ts index dad319ac46..9a2154506c 100644 --- a/src/kernels/jupyter/finder/remoteKernelFinder.ts +++ b/src/kernels/jupyter/finder/remoteKernelFinder.ts @@ -457,6 +457,9 @@ export class RemoteKernelFinder extends ObservableDisposable implements IRemoteK return false; case 'connectToLiveRemoteKernel': return this.cachedRemoteKernelValidator.isValid(kernel); + case 'startUsingDeepnoteKernel': + // Deepnote kernels are dynamically created, no need to cache/validate + return false; } } } diff --git a/src/kernels/types.ts b/src/kernels/types.ts index c0e10139f3..fbc44b2ebc 100644 --- a/src/kernels/types.ts +++ b/src/kernels/types.ts @@ -79,6 +79,10 @@ export class BaseKernelConnectionMetadata { case 'startUsingPythonInterpreter': // eslint-disable-next-line @typescript-eslint/no-use-before-define return PythonKernelConnectionMetadata.create(clone as PythonKernelConnectionMetadata); + case 'startUsingDeepnoteKernel': + // eslint-disable-next-line @typescript-eslint/no-use-before-define + const { DeepnoteKernelConnectionMetadata } = require('./deepnote/types'); + return DeepnoteKernelConnectionMetadata.create(clone); default: throw new Error(`Invalid object to be deserialized into a connection, kind = ${clone.kind}`); } @@ -311,7 +315,8 @@ export type LocalKernelConnectionMetadata = */ export type RemoteKernelConnectionMetadata = | Readonly - | Readonly; + | Readonly + | Readonly; export function isLocalConnection( kernelConnection: KernelConnectionMetadata diff --git a/src/notebooks/controllers/controllerRegistration.ts b/src/notebooks/controllers/controllerRegistration.ts index a16fa77534..977292a845 100644 --- a/src/notebooks/controllers/controllerRegistration.ts +++ b/src/notebooks/controllers/controllerRegistration.ts @@ -30,6 +30,7 @@ import { } from './types'; import { VSCodeNotebookController } from './vscodeNotebookController'; import { IJupyterVariablesProvider } from '../../kernels/variables/types'; +import { DEEPNOTE_NOTEBOOK_TYPE } from '../../kernels/deepnote/types'; /** * Keeps track of registered controllers and available KernelConnectionMetadatas. @@ -236,14 +237,14 @@ export class ControllerRegistration implements IControllerRegistration, IExtensi } addOrUpdate( metadata: KernelConnectionMetadata, - types: ('jupyter-notebook' | 'interactive')[] + types: (typeof JupyterNotebookView | typeof InteractiveWindowView | typeof DEEPNOTE_NOTEBOOK_TYPE)[] ): IVSCodeNotebookController[] { const { added, existing } = this.addImpl(metadata, types, true); return added.concat(existing); } addImpl( metadata: KernelConnectionMetadata, - types: ('jupyter-notebook' | 'interactive')[], + types: (typeof JupyterNotebookView | typeof InteractiveWindowView | typeof DEEPNOTE_NOTEBOOK_TYPE)[], triggerChangeEvent: boolean ): { added: IVSCodeNotebookController[]; existing: IVSCodeNotebookController[] } { const added: IVSCodeNotebookController[] = []; @@ -350,7 +351,7 @@ export class ControllerRegistration implements IControllerRegistration, IExtensi } get( metadata: KernelConnectionMetadata, - notebookType: 'jupyter-notebook' | 'interactive' + notebookType: 'jupyter-notebook' | 'interactive' | 'deepnote' ): IVSCodeNotebookController | undefined { const id = this.getControllerId(metadata, notebookType); return this.registeredControllers.get(id); @@ -358,8 +359,11 @@ export class ControllerRegistration implements IControllerRegistration, IExtensi private getControllerId( metadata: KernelConnectionMetadata, - viewType: typeof JupyterNotebookView | typeof InteractiveWindowView + viewType: typeof JupyterNotebookView | typeof InteractiveWindowView | 'deepnote' ) { + if (viewType === 'deepnote') { + return metadata.id; + } return viewType === JupyterNotebookView ? metadata.id : `${metadata.id}${InteractiveControllerIdSuffix}`; } diff --git a/src/notebooks/controllers/types.ts b/src/notebooks/controllers/types.ts index 5871ad1952..80ea92157b 100644 --- a/src/notebooks/controllers/types.ts +++ b/src/notebooks/controllers/types.ts @@ -16,6 +16,7 @@ import { IDisposable } from '../../platform/common/types'; import { JupyterServerCollection } from '../../api'; import { EnvironmentPath } from '@vscode/python-extension'; import type { VSCodeNotebookController } from './vscodeNotebookController'; +import { DEEPNOTE_NOTEBOOK_TYPE } from '../../kernels/deepnote/types'; export const InteractiveControllerIdSuffix = ' (Interactive)'; @@ -83,7 +84,7 @@ export interface IControllerRegistration { */ addOrUpdate( metadata: KernelConnectionMetadata, - types: (typeof JupyterNotebookView | typeof InteractiveWindowView)[] + types: (typeof JupyterNotebookView | typeof InteractiveWindowView | typeof DEEPNOTE_NOTEBOOK_TYPE)[] ): IVSCodeNotebookController[]; /** * Gets the controller for a particular connection @@ -92,7 +93,7 @@ export interface IControllerRegistration { */ get( connection: KernelConnectionMetadata, - notebookType: typeof JupyterNotebookView | typeof InteractiveWindowView + notebookType: typeof JupyterNotebookView | typeof InteractiveWindowView | typeof DEEPNOTE_NOTEBOOK_TYPE ): IVSCodeNotebookController | undefined; } diff --git a/src/notebooks/deepnote/deepnoteKernelAutoSelector.node.ts b/src/notebooks/deepnote/deepnoteKernelAutoSelector.node.ts new file mode 100644 index 0000000000..ac9521d7c9 --- /dev/null +++ b/src/notebooks/deepnote/deepnoteKernelAutoSelector.node.ts @@ -0,0 +1,430 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { inject, injectable, optional } from 'inversify'; +import { + CancellationToken, + NotebookDocument, + workspace, + NotebookControllerAffinity, + window, + ProgressLocation, + notebooks, + NotebookController +} from 'vscode'; +import { IExtensionSyncActivationService } from '../../platform/activation/types'; +import { IDisposableRegistry } from '../../platform/common/types'; +import { logger } from '../../platform/logging'; +import { IInterpreterService } from '../../platform/interpreter/contracts'; +import { + IDeepnoteKernelAutoSelector, + IDeepnoteServerStarter, + IDeepnoteToolkitInstaller, + IDeepnoteServerProvider, + DEEPNOTE_NOTEBOOK_TYPE, + DeepnoteKernelConnectionMetadata +} from '../../kernels/deepnote/types'; +import { IControllerRegistration, IVSCodeNotebookController } from '../controllers/types'; +import { JVSC_EXTENSION_ID } from '../../platform/common/constants'; +import { getDisplayPath } from '../../platform/common/platform/fs-paths'; +import { JupyterServerProviderHandle } from '../../kernels/jupyter/types'; +import { IPythonExtensionChecker } from '../../platform/api/types'; +import { JupyterLabHelper } from '../../kernels/jupyter/session/jupyterLabHelper'; +import { createJupyterConnectionInfo } from '../../kernels/jupyter/jupyterUtils'; +import { IJupyterRequestCreator, IJupyterRequestAgentCreator } from '../../kernels/jupyter/types'; +import { IConfigurationService } from '../../platform/common/types'; +import { disposeAsync } from '../../platform/common/utils'; + +/** + * Automatically selects and starts Deepnote kernel for .deepnote notebooks + */ +@injectable() +export class DeepnoteKernelAutoSelector implements IDeepnoteKernelAutoSelector, IExtensionSyncActivationService { + // Track server handles per notebook URI for cleanup + private readonly notebookServerHandles = new Map(); + // Track registered controllers per notebook file (base URI) for reuse + private readonly notebookControllers = new Map(); + // Track connection metadata per notebook file for reuse + private readonly notebookConnectionMetadata = new Map(); + // Track temporary loading controllers that get disposed when real controller is ready + private readonly loadingControllers = new Map(); + + constructor( + @inject(IDisposableRegistry) private readonly disposables: IDisposableRegistry, + @inject(IControllerRegistration) private readonly controllerRegistration: IControllerRegistration, + @inject(IInterpreterService) private readonly interpreterService: IInterpreterService, + @inject(IDeepnoteToolkitInstaller) private readonly toolkitInstaller: IDeepnoteToolkitInstaller, + @inject(IDeepnoteServerStarter) private readonly serverStarter: IDeepnoteServerStarter, + @inject(IPythonExtensionChecker) private readonly pythonExtensionChecker: IPythonExtensionChecker, + @inject(IDeepnoteServerProvider) private readonly serverProvider: IDeepnoteServerProvider, + @inject(IJupyterRequestCreator) private readonly requestCreator: IJupyterRequestCreator, + @inject(IJupyterRequestAgentCreator) + @optional() + private readonly requestAgentCreator: IJupyterRequestAgentCreator | undefined, + @inject(IConfigurationService) private readonly configService: IConfigurationService + ) {} + + public activate() { + // Listen to notebook open events + workspace.onDidOpenNotebookDocument(this.onDidOpenNotebook, this, this.disposables); + + // Listen to notebook close events for cleanup + workspace.onDidCloseNotebookDocument(this.onDidCloseNotebook, this, this.disposables); + + // Listen to controller selection changes to detect when kernel becomes unselected + // (This is now mostly a safety net since controllers are protected from disposal) + this.controllerRegistration.onControllerSelectionChanged( + this.onControllerSelectionChanged, + this, + this.disposables + ); + + // Handle currently open notebooks - await all async operations + Promise.all(workspace.notebookDocuments.map((d) => this.onDidOpenNotebook(d))).catch((error) => { + logger.error(`Error handling open notebooks during activation: ${error}`); + }); + } + + private async onDidOpenNotebook(notebook: NotebookDocument) { + // Only handle deepnote notebooks + if (notebook.notebookType !== DEEPNOTE_NOTEBOOK_TYPE) { + return; + } + + logger.info(`Deepnote notebook opened: ${getDisplayPath(notebook.uri)}`); + + // Check if we already have a controller ready for this notebook + const baseFileUri = notebook.uri.with({ query: '', fragment: '' }); + const notebookKey = baseFileUri.fsPath; + const hasExistingController = this.notebookControllers.has(notebookKey); + + // If no existing controller, create a temporary "Loading" controller immediately + // This prevents the kernel selector from appearing when user clicks Run All + if (!hasExistingController) { + this.createLoadingController(notebook, notebookKey); + } + + // Always try to ensure kernel is selected (this will reuse existing controllers) + // Don't await - let it happen in background so notebook opens quickly + void this.ensureKernelSelected(notebook).catch((error) => { + logger.error(`Failed to auto-select Deepnote kernel for ${getDisplayPath(notebook.uri)}: ${error}`); + void window.showErrorMessage(`Failed to load Deepnote kernel: ${error}`); + }); + } + + private onControllerSelectionChanged(event: { + notebook: NotebookDocument; + controller: IVSCodeNotebookController; + selected: boolean; + }) { + // Only handle deepnote notebooks + if (event.notebook.notebookType !== DEEPNOTE_NOTEBOOK_TYPE) { + return; + } + + const baseFileUri = event.notebook.uri.with({ query: '', fragment: '' }); + const notebookKey = baseFileUri.fsPath; + + // If the Deepnote controller for this notebook was deselected, try to reselect it + // Since controllers are now protected from disposal, this should rarely happen + if (!event.selected) { + const ourController = this.notebookControllers.get(notebookKey); + if (ourController && ourController.id === event.controller.id) { + logger.warn( + `Deepnote controller was unexpectedly deselected for ${getDisplayPath( + event.notebook.uri + )}. Reselecting...` + ); + // Reselect the controller + ourController.controller.updateNotebookAffinity(event.notebook, NotebookControllerAffinity.Preferred); + } + } + } + + private onDidCloseNotebook(notebook: NotebookDocument) { + // Only handle deepnote notebooks + if (notebook.notebookType !== DEEPNOTE_NOTEBOOK_TYPE) { + return; + } + + logger.info(`Deepnote notebook closed: ${getDisplayPath(notebook.uri)}`); + + // Extract the base file URI to match what we used when registering + const baseFileUri = notebook.uri.with({ query: '', fragment: '' }); + const notebookKey = baseFileUri.fsPath; + + // Note: We intentionally don't clean up controllers, connection metadata, or servers here. + // This allows the kernel to be reused if the user reopens the same .deepnote file. + // The server will continue running and can be reused for better performance. + // Cleanup will happen when the extension is disposed or when explicitly requested. + + // However, we do unregister the server from the provider to keep it clean + const serverHandle = this.notebookServerHandles.get(notebookKey); + if (serverHandle) { + logger.info(`Unregistering server for closed notebook: ${serverHandle}`); + this.serverProvider.unregisterServer(serverHandle); + this.notebookServerHandles.delete(notebookKey); + } + } + + public async ensureKernelSelected(notebook: NotebookDocument, _token?: CancellationToken): Promise { + return window.withProgress( + { + location: ProgressLocation.Notification, + title: 'Loading Deepnote Kernel', + cancellable: true + }, + async (progress, progressToken) => { + try { + logger.info(`Ensuring Deepnote kernel is selected for ${getDisplayPath(notebook.uri)}`); + + // Extract the base file URI (without query parameters) + // Notebooks from the same .deepnote file have different URIs with ?notebook=id query params + const baseFileUri = notebook.uri.with({ query: '', fragment: '' }); + const notebookKey = baseFileUri.fsPath; + logger.info(`Base Deepnote file: ${getDisplayPath(baseFileUri)}`); + + // Check if we already have a controller for this notebook file + let existingController = this.notebookControllers.get(notebookKey); + const connectionMetadata = this.notebookConnectionMetadata.get(notebookKey); + + // If we have an existing controller, reuse it (controllers are now protected from disposal) + if (existingController && connectionMetadata) { + logger.info( + `Reusing existing Deepnote controller ${existingController.id} for ${getDisplayPath( + notebook.uri + )}` + ); + progress.report({ message: 'Reusing existing kernel...' }); + + // Ensure server is registered with the provider (it might have been unregistered on close) + if (connectionMetadata.serverInfo) { + const serverProviderHandle = connectionMetadata.serverProviderHandle; + this.serverProvider.registerServer( + serverProviderHandle.handle, + connectionMetadata.serverInfo + ); + this.notebookServerHandles.set(notebookKey, serverProviderHandle.handle); + logger.info(`Re-registered server for reuse: ${serverProviderHandle.handle}`); + } + + // Check if this controller is already selected for this notebook + const selectedController = this.controllerRegistration.getSelected(notebook); + if (selectedController && selectedController.id === existingController.id) { + logger.info(`Controller already selected for ${getDisplayPath(notebook.uri)}`); + return; + } + + // Auto-select the existing controller for this notebook + existingController.controller.updateNotebookAffinity( + notebook, + NotebookControllerAffinity.Preferred + ); + logger.info(`Reselected existing Deepnote kernel for ${getDisplayPath(notebook.uri)}`); + + // Dispose the loading controller if it exists + const loadingController = this.loadingControllers.get(notebookKey); + if (loadingController) { + loadingController.dispose(); + this.loadingControllers.delete(notebookKey); + logger.info(`Disposed loading controller for ${notebookKey}`); + } + + return; + } + + // No existing controller, so create a new one + logger.info(`Creating new Deepnote kernel for ${getDisplayPath(notebook.uri)}`); + progress.report({ message: 'Setting up Deepnote kernel...' }); + + // Check if Python extension is installed + if (!this.pythonExtensionChecker.isPythonExtensionInstalled) { + logger.warn('Python extension is not installed. Prompting user to install it.'); + await this.pythonExtensionChecker.showPythonExtensionInstallRequiredPrompt(); + return; // Exit - user needs to install Python extension first + } + + // Get active Python interpreter + progress.report({ message: 'Finding Python interpreter...' }); + const interpreter = await this.interpreterService.getActiveInterpreter(notebook.uri); + if (!interpreter) { + logger.warn( + 'No Python interpreter found for Deepnote notebook. Kernel selection will be manual.' + ); + return; // Exit gracefully - user can select kernel manually + } + + logger.info(`Using base interpreter: ${getDisplayPath(interpreter.uri)}`); + + // Ensure deepnote-toolkit is installed in a venv and get the venv interpreter + progress.report({ message: 'Installing Deepnote toolkit...' }); + const venvInterpreter = await this.toolkitInstaller.ensureInstalled( + interpreter, + baseFileUri, + progressToken + ); + if (!venvInterpreter) { + logger.error('Failed to set up Deepnote toolkit environment'); + return; // Exit gracefully + } + + logger.info(`Deepnote toolkit venv ready at: ${getDisplayPath(venvInterpreter.uri)}`); + + // Start the Deepnote server using the venv interpreter + progress.report({ message: 'Starting Deepnote server...' }); + const serverInfo = await this.serverStarter.getOrStartServer( + venvInterpreter, + baseFileUri, + progressToken + ); + logger.info(`Deepnote server running at ${serverInfo.url}`); + + // Create server provider handle + const serverProviderHandle: JupyterServerProviderHandle = { + extensionId: JVSC_EXTENSION_ID, + id: 'deepnote-server', + handle: `deepnote-toolkit-server-${baseFileUri.fsPath}` + }; + + // Register the server with the provider so it can be resolved + this.serverProvider.registerServer(serverProviderHandle.handle, serverInfo); + + // Track the server handle for cleanup when notebook is closed + this.notebookServerHandles.set(notebookKey, serverProviderHandle.handle); + + // Connect to the server and get available kernel specs + progress.report({ message: 'Connecting to kernel...' }); + const connectionInfo = createJupyterConnectionInfo( + serverProviderHandle, + { + baseUrl: serverInfo.url, + token: serverInfo.token || '', + displayName: 'Deepnote Server', + authorizationHeader: {} + }, + this.requestCreator, + this.requestAgentCreator, + this.configService, + baseFileUri + ); + + const sessionManager = JupyterLabHelper.create(connectionInfo.settings); + let kernelSpec; + try { + const kernelSpecs = await sessionManager.getKernelSpecs(); + logger.info( + `Available kernel specs on Deepnote server: ${kernelSpecs.map((s) => s.name).join(', ')}` + ); + + // Create expected kernel name based on file path (same logic as in installer) + const safePath = baseFileUri.fsPath.replace(/[^a-zA-Z0-9]/g, '_'); + const venvHash = safePath.substring(0, 16); + const expectedKernelName = `deepnote-venv-${venvHash}`; + logger.info(`Looking for venv kernel spec: ${expectedKernelName}`); + + // Prefer the venv kernel spec that uses the venv's Python interpreter + // This ensures packages installed via pip are available to the kernel + kernelSpec = + kernelSpecs.find((s) => s.name === expectedKernelName) || + kernelSpecs.find((s) => s.language === 'python') || + kernelSpecs.find((s) => s.name === 'python3-venv') || + kernelSpecs[0]; + + if (!kernelSpec) { + throw new Error('No kernel specs available on Deepnote server'); + } + + logger.info(`Using kernel spec: ${kernelSpec.name} (${kernelSpec.display_name})`); + } finally { + await disposeAsync(sessionManager); + } + + progress.report({ message: 'Finalizing kernel setup...' }); + const newConnectionMetadata = DeepnoteKernelConnectionMetadata.create({ + interpreter, + kernelSpec, + baseUrl: serverInfo.url, + id: `deepnote-kernel-${interpreter.id}`, + serverProviderHandle, + serverInfo // Pass the server info so we can use it later + }); + + // Store connection metadata for reuse + this.notebookConnectionMetadata.set(notebookKey, newConnectionMetadata); + + // Register controller for deepnote notebook type + const controllers = this.controllerRegistration.addOrUpdate(newConnectionMetadata, [ + DEEPNOTE_NOTEBOOK_TYPE + ]); + + if (controllers.length === 0) { + logger.error('Failed to create Deepnote kernel controller'); + throw new Error('Failed to create Deepnote kernel controller'); + } + + const controller = controllers[0]; + logger.info(`Created Deepnote kernel controller: ${controller.id}`); + + // Store the controller for reuse + this.notebookControllers.set(notebookKey, controller); + + // Mark this controller as protected so it won't be automatically disposed + // This is similar to how active interpreter controllers are protected + this.controllerRegistration.trackActiveInterpreterControllers(controllers); + logger.info(`Marked Deepnote controller as protected from automatic disposal`); + + // Listen to controller disposal so we can clean up our tracking + controller.onDidDispose(() => { + logger.info(`Deepnote controller ${controller!.id} disposed, removing from tracking`); + this.notebookControllers.delete(notebookKey); + // Keep connection metadata for quick recreation + // The metadata is still valid and can be used to recreate the controller + }); + + // Auto-select the controller for this notebook using affinity + // Setting NotebookControllerAffinity.Preferred will make VSCode automatically select this controller + controller.controller.updateNotebookAffinity(notebook, NotebookControllerAffinity.Preferred); + + logger.info(`Successfully auto-selected Deepnote kernel for ${getDisplayPath(notebook.uri)}`); + progress.report({ message: 'Kernel ready!' }); + + // Dispose the loading controller once the real one is ready + const loadingController = this.loadingControllers.get(notebookKey); + if (loadingController) { + loadingController.dispose(); + this.loadingControllers.delete(notebookKey); + logger.info(`Disposed loading controller for ${notebookKey}`); + } + } catch (ex) { + logger.error(`Failed to auto-select Deepnote kernel: ${ex}`); + throw ex; + } + } + ); + } + + private createLoadingController(notebook: NotebookDocument, notebookKey: string): void { + // Create a temporary controller that shows "Loading..." and prevents kernel selection prompt + const loadingController = notebooks.createNotebookController( + `deepnote-loading-${notebookKey}`, + DEEPNOTE_NOTEBOOK_TYPE, + 'Loading Deepnote Kernel...' + ); + + // Set it as the preferred controller immediately + loadingController.supportsExecutionOrder = false; + loadingController.supportedLanguages = ['python']; + + // Execution handler that does nothing - cells will just sit there until real kernel is ready + loadingController.executeHandler = () => { + // No-op: execution is blocked until the real controller takes over + }; + + // Select this controller for the notebook + loadingController.updateNotebookAffinity(notebook, NotebookControllerAffinity.Preferred); + + // Store it so we can dispose it later + this.loadingControllers.set(notebookKey, loadingController); + logger.info(`Created loading controller for ${notebookKey}`); + } +} diff --git a/src/notebooks/serviceRegistry.node.ts b/src/notebooks/serviceRegistry.node.ts index fc8ac895f1..d0e20c2491 100644 --- a/src/notebooks/serviceRegistry.node.ts +++ b/src/notebooks/serviceRegistry.node.ts @@ -43,6 +43,16 @@ import { INotebookEditorProvider, INotebookPythonEnvironmentService } from './ty import { DeepnoteActivationService } from './deepnote/deepnoteActivationService'; import { DeepnoteNotebookManager } from './deepnote/deepnoteNotebookManager'; import { IDeepnoteNotebookManager } from './types'; +import { + IDeepnoteToolkitInstaller, + IDeepnoteServerStarter, + IDeepnoteKernelAutoSelector, + IDeepnoteServerProvider +} from '../kernels/deepnote/types'; +import { DeepnoteToolkitInstaller } from '../kernels/deepnote/deepnoteToolkitInstaller.node'; +import { DeepnoteServerStarter } from '../kernels/deepnote/deepnoteServerStarter.node'; +import { DeepnoteKernelAutoSelector } from './deepnote/deepnoteKernelAutoSelector.node'; +import { DeepnoteServerProvider } from '../kernels/deepnote/deepnoteServerProvider.node'; export function registerTypes(serviceManager: IServiceManager, isDevMode: boolean) { registerControllerTypes(serviceManager, isDevMode); @@ -117,6 +127,14 @@ export function registerTypes(serviceManager: IServiceManager, isDevMode: boolea ); serviceManager.addSingleton(IDeepnoteNotebookManager, DeepnoteNotebookManager); + // Deepnote kernel services + serviceManager.addSingleton(IDeepnoteToolkitInstaller, DeepnoteToolkitInstaller); + serviceManager.addSingleton(IDeepnoteServerStarter, DeepnoteServerStarter); + serviceManager.addSingleton(IDeepnoteServerProvider, DeepnoteServerProvider); + serviceManager.addBinding(IDeepnoteServerProvider, IExtensionSyncActivationService); + serviceManager.addSingleton(IDeepnoteKernelAutoSelector, DeepnoteKernelAutoSelector); + serviceManager.addBinding(IDeepnoteKernelAutoSelector, IExtensionSyncActivationService); + // File export/import serviceManager.addSingleton(IFileConverter, FileConverter); serviceManager.addSingleton(ExportInterpreterFinder, ExportInterpreterFinder); diff --git a/src/platform/common/net/httpClient.ts b/src/platform/common/net/httpClient.ts index 721300663b..5643669c8d 100644 --- a/src/platform/common/net/httpClient.ts +++ b/src/platform/common/net/httpClient.ts @@ -1,6 +1,7 @@ // Copyright (c) Microsoft Corporation. // Licensed under the MIT License. +import { injectable } from 'inversify'; import { IHttpClient } from '../types'; import { logger } from '../../logging'; import * as fetch from 'cross-fetch'; @@ -9,6 +10,7 @@ import { workspace } from 'vscode'; /** * Class used to verify http connections and make GET requests */ +@injectable() export class HttpClient implements IHttpClient { private readonly requestOptions: RequestInit = {}; constructor(private readonly fetchImplementation: typeof fetch.fetch = fetch.fetch) { diff --git a/src/platform/common/serviceRegistry.node.ts b/src/platform/common/serviceRegistry.node.ts index cfd460aad2..0ff56bbd31 100644 --- a/src/platform/common/serviceRegistry.node.ts +++ b/src/platform/common/serviceRegistry.node.ts @@ -20,6 +20,7 @@ import { ICryptoUtils, IExtensions, IFeaturesManager, + IHttpClient, IPersistentStateFactory, IsWindows } from './types'; @@ -31,6 +32,7 @@ import { registerTypes as variableRegisterTypes } from './variables/serviceRegis import { RunInDedicatedExtensionHostCommandHandler } from './application/commands/runInDedicatedExtensionHost.node'; import { OldCacheCleaner } from './cache'; import { ApplicationEnvironment } from './application/applicationEnvironment'; +import { HttpClient } from './net/httpClient'; // eslint-disable-next-line export function registerTypes(serviceManager: IServiceManager) { @@ -45,6 +47,7 @@ export function registerTypes(serviceManager: IServiceManager) { serviceManager.addSingleton(IExperimentService, ExperimentService); serviceManager.addSingleton(IFeaturesManager, FeatureManager); + serviceManager.addSingleton(IHttpClient, HttpClient); serviceManager.addSingleton(IAsyncDisposableRegistry, AsyncDisposableRegistry); serviceManager.addSingleton(IMultiStepInputFactory, MultiStepInputFactory); diff --git a/src/platform/common/types.ts b/src/platform/common/types.ts index 33b019c1e6..f82997cc6c 100644 --- a/src/platform/common/types.ts +++ b/src/platform/common/types.ts @@ -161,6 +161,7 @@ export type DownloadOptions = { extension: 'tmp' | string; }; +export const IHttpClient = Symbol('IHttpClient'); export interface IHttpClient { downloadFile(uri: string): Promise; /** diff --git a/src/platform/errors/deepnoteServerNotFoundError.ts b/src/platform/errors/deepnoteServerNotFoundError.ts new file mode 100644 index 0000000000..c0c9d2ea05 --- /dev/null +++ b/src/platform/errors/deepnoteServerNotFoundError.ts @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. +// Licensed under the MIT License. + +import { BaseError } from './types'; + +/** + * Error thrown when a Deepnote server handle cannot be resolved. + * + * Cause: + * The requested Deepnote server handle was not found in the registered servers. + * This typically happens when trying to resolve a server that hasn't been registered yet + * or has been removed from the registry. + * + * Handled by: + * The error should be logged and the user should be notified that the server connection + * could not be established. + */ +export class DeepnoteServerNotFoundError extends BaseError { + constructor(serverId: string) { + super('deepnoteserver', `Deepnote server not found: ${serverId}`); + } +} diff --git a/src/platform/errors/types.ts b/src/platform/errors/types.ts index d2c12cf4ee..9c79583bfd 100644 --- a/src/platform/errors/types.ts +++ b/src/platform/errors/types.ts @@ -104,7 +104,8 @@ export type ErrorCategory = | 'kernelProcessFailedToLaunch' | 'unknownProduct' | 'invalidInterpreter' - | 'pythonAPINotInitialized'; + | 'pythonAPINotInitialized' + | 'deepnoteserver'; // If there are errors, then the are added to the telementry properties. export type TelemetryErrorProperties = { diff --git a/src/telemetry.ts b/src/telemetry.ts index 14cb893d31..c02aa6c1af 100644 --- a/src/telemetry.ts +++ b/src/telemetry.ts @@ -890,7 +890,8 @@ export class IEventNamePropertyMapping { | 'startUsingPythonInterpreter' | 'startUsingLocalKernelSpec' | 'startUsingRemoteKernelSpec' - | 'connectToLiveRemoteKernel'; + | 'connectToLiveRemoteKernel' + | 'startUsingDeepnoteKernel'; /** * Language of the kernel spec. */ @@ -3533,7 +3534,8 @@ export class IEventNamePropertyMapping { | 'startUsingPythonInterpreter' | 'startUsingLocalKernelSpec' | 'startUsingRemoteKernelSpec' - | 'connectToLiveRemoteKernel'; + | 'connectToLiveRemoteKernel' + | 'startUsingDeepnoteKernel'; /** * Language of the kernel spec. */ @@ -3653,7 +3655,8 @@ export class IEventNamePropertyMapping { | 'startUsingPythonInterpreter' | 'startUsingLocalKernelSpec' | 'startUsingRemoteKernelSpec' - | 'connectToLiveRemoteKernel'; + | 'connectToLiveRemoteKernel' + | 'startUsingDeepnoteKernel'; /** * Language of the kernel spec. */