Add basic integration tests for .deepnote file handling (GRN-4766)

.github/workflows/ci.yml

@@ -143,6 +143,44 @@ jobs:
       - name: Check licenses
         run: npm run check-licenses
 
+  integration-tests:
+    name: Integration Tests
+    runs-on: ubicloud
+    timeout-minutes: 30
+    permissions:
+      id-token: write
+      contents: read
+      packages: read
+    steps:
+      - name: Checkout
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5
+
+      - name: Setup Node.js
+        uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 # v5
+        with:
+          cache: 'npm'
+          node-version-file: '.nvmrc'
+          registry-url: 'https://npm.pkg.github.com'
+          scope: '@deepnote'
+
+      - name: Setup Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
+        with:
+          python-version: '3.12'
+
+      - name: Install dependencies
+        run: npm ci --prefer-offline --no-audit
+        env:
+          NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Compile TypeScript
+        run: npm run compile
+
+      - name: Run integration tests
+        run: xvfb-run -a -s "-screen 0 1024x768x24" npm run test:integration
+        env:
+          VSC_JUPYTER_CI_TEST_GREP: 'Deepnote Integration Tests'
+
   check_licenses:
     name: Check Licenses
     runs-on: ubuntu-latest

src/kernels/execution/codeExecution.unit.test.ts

@@ -21,7 +21,7 @@ import {
 } from '@jupyterlab/services/lib/kernel/messages';
 import { Deferred, createDeferred } from '../../platform/common/utils/async';
 import { NotebookCellOutput, NotebookCellOutputItem } from 'vscode';
-import { JVSC_EXTENSION_ID_FOR_TESTS } from '../../test/constants';
+import { JVSC_EXTENSION_ID } from '../../platform/common/constants';
 
 suite('Code Execution', () => {
     let disposables: IDisposable[] = [];
@@ -318,7 +318,7 @@ suite('Code Execution', () => {
     });
     test('Cancelling pending Internal Jupyter execution code should not interrupt the kernel', async () => {
         const code = `print('Hello World')`;
-        const execution = createExecution(code, JVSC_EXTENSION_ID_FOR_TESTS);
+        const execution = createExecution(code, JVSC_EXTENSION_ID);
 
         const outputs: NotebookCellOutput[] = [];
         disposables.push(execution.onDidEmitOutput((output) => outputs.push(output)));

src/kernels/jupyter/finder/remoteKernelFinderController.unit.test.ts

@@ -27,7 +27,7 @@ import { IFileSystem } from '../../../platform/common/platform/types';
 import { RemoteKernelFinderController } from './remoteKernelFinderController';
 import { JupyterServerCollection, JupyterServerProvider } from '../../../api';
 import { UserJupyterServerPickerProviderId } from '../../../platform/constants';
-import { JVSC_EXTENSION_ID_FOR_TESTS } from '../../../test/constants';
+import { JVSC_EXTENSION_ID } from '../../../platform/common/constants';
 
 suite(`Remote Kernel Finder Controller`, () => {
     let disposables: Disposable[] = [];
@@ -50,7 +50,7 @@ suite(`Remote Kernel Finder Controller`, () => {
         provider: {
             id: UserJupyterServerPickerProviderId,
             handle: '2',
-            extensionId: JVSC_EXTENSION_ID_FOR_TESTS
+            extensionId: JVSC_EXTENSION_ID
         }
     };
     let serverUriStorage: IJupyterServerUriStorage;
@@ -127,7 +127,7 @@ suite(`Remote Kernel Finder Controller`, () => {
         const collectionForRemote = mock<JupyterServerCollection>();
         when(collectionForRemote.id).thenReturn(UserJupyterServerPickerProviderId);
         when(collectionForRemote.label).thenReturn('Quick Label');
-        when(collectionForRemote.extensionId).thenReturn(JVSC_EXTENSION_ID_FOR_TESTS);
+        when(collectionForRemote.extensionId).thenReturn(JVSC_EXTENSION_ID);
         const serverProvider = mock<JupyterServerProvider>();
         when(serverProvider.provideJupyterServers(anything())).thenResolve();
         when(collectionForRemote.serverProvider).thenReturn(instance(serverProvider));

src/notebooks/deepnote/deepnoteExplorerView.ts

@@ -1,7 +1,6 @@
 import { injectable, inject } from 'inversify';
 import { commands, window, workspace, type TreeView, Uri, l10n } from 'vscode';
 import * as yaml from 'js-yaml';
-import { convertIpynbFilesToDeepnoteFile } from '@deepnote/convert';
 
 import { IExtensionContext } from '../../platform/common/types';
 import { IDeepnoteNotebookManager } from '../types';
@@ -357,6 +356,7 @@ export class DeepnoteExplorerView {
                 const outputFileName = `${projectName}.deepnote`;
                 const outputPath = Uri.joinPath(workspaceFolder.uri, outputFileName).path;
 
+                const convertIpynbFilesToDeepnoteFile = await this.getConverter();
                 await convertIpynbFilesToDeepnoteFile(inputFilePaths, {
                     outputPath: outputPath,
                     projectName: projectName
@@ -379,6 +379,11 @@ export class DeepnoteExplorerView {
         }
     }
 
+    private async getConverter() {
+        const { convertIpynbFilesToDeepnoteFile } = await import('@deepnote/convert');
+        return convertIpynbFilesToDeepnoteFile;
+    }
+
     private async importJupyterNotebook(): Promise<void> {
         if (!workspace.workspaceFolders || workspace.workspaceFolders.length === 0) {
             const selection = await window.showInformationMessage(
@@ -429,6 +434,7 @@ export class DeepnoteExplorerView {
                 // File doesn't exist, continue
             }
 
+            const convertIpynbFilesToDeepnoteFile = await this.getConverter();
             await convertIpynbFilesToDeepnoteFile(inputFilePaths, {
                 outputPath: outputUri.path,
                 projectName: projectName

src/standalone/api/kernels/kernel.unit.test.ts

@@ -31,7 +31,7 @@ import { createMockedNotebookDocument } from '../../../test/datascience/editor-i
 import { IControllerRegistration, IVSCodeNotebookController } from '../../../notebooks/controllers/types';
 import { createKernelApiForExtension } from './kernel';
 import { noop } from '../../../test/core';
-import { JVSC_EXTENSION_ID_FOR_TESTS } from '../../../test/constants';
+import { JVSC_EXTENSION_ID } from '../../../platform/common/constants';
 import { IKernelConnection } from '@jupyterlab/services/lib/kernel/kernel';
 import { NotebookCellOutput } from 'vscode';
 
@@ -127,7 +127,7 @@ suite('Kernel Api', () => {
         when(kernel.shutdown()).thenResolve();
         when(kernel.dispose()).thenCall(() => when(kernel.status).thenReturn('dead'));
 
-        const { api } = createKernelApiForExtension(JVSC_EXTENSION_ID_FOR_TESTS, instance(kernel));
+        const { api } = createKernelApiForExtension(JVSC_EXTENSION_ID, instance(kernel));
         // eslint-disable-next-line @typescript-eslint/no-unused-vars
         for await (const _ of api.executeCode('bogus', token)) {
             //

src/test/constants.node.ts

@@ -16,7 +16,7 @@ export const EXTENSION_TEST_DIR_FOR_FILES = path.join(
     'datascience',
     'temp'
 );
-export const JVSC_EXTENSION_ID_FOR_TESTS = 'ms-toolsai.jupyter';
+export const JVSC_EXTENSION_ID_FOR_TESTS = 'Deepnote.vscode-deepnote';
 
 export const SMOKE_TEST_EXTENSIONS_DIR = path.join(
     EXTENSION_ROOT_DIR_FOR_TESTS,

src/test/constants.ts

@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation.
 // Licensed under the MIT License.
 
-export const JVSC_EXTENSION_ID_FOR_TESTS = 'ms-toolsai.jupyter';
+export const JVSC_EXTENSION_ID_FOR_TESTS = 'Deepnote.vscode-deepnote';
 export const PerformanceExtensionId = 'ms-toolsai.vscode-notebook-perf';
 
 export type TestSettingsType = {

src/test/datascience/notebook/deepnote.vscode.test.ts

@@ -0,0 +1,109 @@
+/* eslint-disable @typescript-eslint/no-require-imports, @typescript-eslint/no-var-requires */
+import { assert } from 'chai';
+import { Uri, workspace } from 'vscode';
+import { IDisposable } from '../../../platform/common/types';
+import { captureScreenShot, IExtensionTestApi } from '../../common.node';
+import { EXTENSION_ROOT_DIR_FOR_TESTS, initialize } from '../../initialize.node';
+import { closeNotebooksAndCleanUpAfterTests } from './helper.node';
+import { logger } from '../../../platform/logging';
+import { IDeepnoteNotebookManager } from '../../../notebooks/types';
+
+/* eslint-disable @typescript-eslint/no-explicit-any, no-invalid-this */
+suite('Deepnote Integration Tests @kernelCore', function () {
+    let api: IExtensionTestApi;
+    const disposables: IDisposable[] = [];
+    const deepnoteFilePath = Uri.joinPath(
+        Uri.file(EXTENSION_ROOT_DIR_FOR_TESTS),
+        'src',
+        'test',
+        'datascience',
+        'notebook',
+        'test.deepnote'
+    );
+    this.timeout(240_000);
+
+    suiteSetup(async function () {
+        logger.info('Suite Setup VS Code Notebook - Deepnote Integration');
+        this.timeout(240_000);
+        try {
+            api = await initialize();
+            logger.info('Suite Setup (completed)');
+        } catch (e) {
+            logger.error('Suite Setup (failed) - Deepnote Integration', e);
+            await captureScreenShot('deepnote-suite');
+            throw e;
+        }
+    });
+
+    setup(function () {
+        logger.info(`Start Test (completed) ${this.currentTest?.title}`);
+    });
+
+    teardown(async function () {
+        if (this.currentTest?.isFailed()) {
+            await captureScreenShot(this);
+        }
+        logger.info(`Ended Test (completed) ${this.currentTest?.title}`);
+    });
+
+    suiteTeardown(() => closeNotebooksAndCleanUpAfterTests(disposables));
+
+    test('Load .deepnote file and verify structure', async function () {
+        logger.debug('Test: Load .deepnote file - starting');
+
+        const notebookManager = api.serviceContainer.get<IDeepnoteNotebookManager>(IDeepnoteNotebookManager);
+        assert.isOk(notebookManager, 'Notebook manager should be available');
+
+        notebookManager.selectNotebookForProject('test-project-id', 'main-notebook-id');
+
+        const nbDocument = await workspace.openNotebookDocument(deepnoteFilePath);
+
+        logger.debug(`Opened notebook with type: ${nbDocument.notebookType}, cells: ${nbDocument.cellCount}`);
+
+        assert.equal(nbDocument.notebookType, 'deepnote', 'Notebook type should be deepnote');
+        assert.equal(nbDocument.cellCount, 3, 'Notebook should have 3 cells');
+
+        assert.equal(nbDocument.metadata?.deepnoteProjectId, 'test-project-id', 'Project ID should match');
+        assert.equal(nbDocument.metadata?.deepnoteNotebookId, 'main-notebook-id', 'Notebook ID should match');
+
+        logger.debug('Test: Load .deepnote file - completed');
+    });
+
+    test('Verify notebook cells are correctly deserialized', async function () {
+        logger.debug('Test: Verify cells - starting');
+
+        const notebookManager = api.serviceContainer.get<IDeepnoteNotebookManager>(IDeepnoteNotebookManager);
+        notebookManager.selectNotebookForProject('test-project-id', 'main-notebook-id');
+
+        const nbDocument = await workspace.openNotebookDocument(deepnoteFilePath);
+
+        assert.isAtLeast(nbDocument.cellCount, 3, 'Notebook should have at least 3 cells');
+
+        let hasCodeCell = false;
+        let hasMarkdownCell = false;
+
+        for (let i = 0; i < nbDocument.cellCount; i++) {
+            const cell = nbDocument.cellAt(i);
+            if (cell.kind === 1) {
+                hasCodeCell = true;
+            }
+            if (cell.kind === 2) {
+                hasMarkdownCell = true;
+            }
+        }
+
+        assert.isTrue(hasCodeCell, 'Notebook should have at least one code cell');
+        assert.isTrue(hasMarkdownCell, 'Notebook should have at least one markdown cell');
+
+        logger.debug('Test: Verify cells - completed');
+    });
+
+    test('Extension services are available', async function () {
+        logger.debug('Test: Extension services are available - starting');
+
+        const notebookManager = api.serviceContainer.get<IDeepnoteNotebookManager>(IDeepnoteNotebookManager);
+        assert.isOk(notebookManager, 'Notebook manager should be available');
+
+        logger.debug('Test: Extension services are available - completed');
+    });
+});

src/test/datascience/notebook/test.deepnote

@@ -0,0 +1,39 @@
+metadata:
+  createdAt: '2025-01-01T00:00:00.000Z'
+  modifiedAt: '2025-01-01T00:00:00.000Z'
+project:
+  id: test-project-id
+  name: Test Project
+  initNotebookId: init-notebook-id
+  notebooks:
+    - id: init-notebook-id
+      name: Init Notebook
+      blocks:
+        - id: init-block-1
+          type: code
+          content: |
+            # This is the init notebook
+            import sys
+            print("Init notebook executed")
+          sortingKey: '0001'
+    - id: main-notebook-id
+      name: Main Notebook
+      blocks:
+        - id: block-1
+          type: code
+          content: |
+            print("Hello World")
+          sortingKey: '0001'
+        - id: block-2
+          type: code
+          content: |
+            x = 42
+            print(f"The answer is {x}")
+          sortingKey: '0002'
+        - id: block-3
+          type: markdown
+          content: |
+            # Test Markdown
+            This is a test markdown block.
+          sortingKey: '0003'
+version: '1.0'