From b9aa168d9a192c1bd02e402976efe7c129320074 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Fri, 12 Sep 2025 17:48:01 +0200 Subject: [PATCH 01/37] Switch notebooks Signed-off-by: Andy Jakubowski --- src/index.tsx | 36 ++++++++++++++++++++++++++++++++++-- 1 file changed, 34 insertions(+), 2 deletions(-) diff --git a/src/index.tsx b/src/index.tsx index 6382c0f..9cd4658 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -68,7 +68,7 @@ class NotebookPicker extends ReactWidget { ? metadataNames : []; - this.selected = names.length > 0 ? names[0] : null; + this.selected = names.length === 0 ? null : (names[0] ?? null); this.update(); }); } @@ -104,7 +104,40 @@ class NotebookPicker extends ReactWidget { this.update(); }; + private handleChange = (event: React.ChangeEvent) => { + const model = this.panel.model; + if (!model) { + return; + } + + const selected = event.target.value; + const deepnoteMetadata = this.panel.context.model.getMetadata('deepnote'); + const notebooks = deepnoteMetadata?.notebooks; + + if (notebooks && selected in notebooks) { + // clone the notebook JSON + const newModelData = { ...notebooks[selected] }; + + // preserve deepnote metadata *without* re-inserting all notebooks + newModelData.metadata = { + ...(newModelData.metadata ?? {}), + deepnote: { + notebook_names: deepnoteMetadata?.notebook_names ?? [], + notebooks: deepnoteMetadata?.notebooks ?? {} + } + }; + + model.fromJSON(newModelData); + model.dirty = false; + } + + this.selected = selected; + this.update(); + }; + render(): JSX.Element { + const deepnoteMetadata = this.panel.context.model.getMetadata('deepnote'); + const metadataNames = deepnoteMetadata?.notebook_names; const deepnoteMetadata = this.panel.context.model.getMetadata('deepnote'); const metadataNames = deepnoteMetadata?.notebook_names; const names = @@ -115,7 +148,6 @@ class NotebookPicker extends ReactWidget { return ( {}} From 2102e3278e0abf06eff3fc5d7a1f24fa050fce2f Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Fri, 26 Sep 2025 12:13:07 +0200 Subject: [PATCH 02/37] Add example request logs Signed-off-by: Andy Jakubowski --- deepnote-file-request-examples.txt | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 deepnote-file-request-examples.txt diff --git a/deepnote-file-request-examples.txt b/deepnote-file-request-examples.txt new file mode 100644 index 0000000..ad33e03 --- /dev/null +++ b/deepnote-file-request-examples.txt @@ -0,0 +1,7 @@ +[W 2025-09-18 11:22:56.083 ServerApp] 404 GET /api/contents/all-block-types-no-outputs.deepnote?content=0&hash=0&1758187376077 (::1): file or directory does not exist: 'all-block-types-no-outputs.deepnote' +[D 2025-09-18 11:22:56.083 ServerApp] Accepting token-authenticated request from ::1 +[D 2025-09-18 11:22:56.083 ServerApp] 200 GET /api/contents/example.ipynb?content=0&hash=0&1758187376077 (b4931ce4427142b3a2011a47bec521d2@::1) 1.80ms +[D 2025-09-18 11:22:56.083 ServerApp] Accepting token-authenticated request from ::1 +[D 2025-09-18 11:22:56.083 ServerApp] 200 GET /api/contents/example.deepnote?content=0&hash=0&1758187376077 (b4931ce4427142b3a2011a47bec521d2@::1) 2.10ms +[D 2025-09-18 11:22:56.288 ServerApp] 200 GET /api/contents/example.deepnote?type=notebook&content=1&hash=1&contentProviderId=undefined&1758187376168 (b4931ce4427142b3a2011a47bec521d2@::1) 57.88ms +[D 2025-09-18 11:22:56.296 ServerApp] 200 GET /api/contents/example.deepnote/checkpoints?1758187376293 (b4931ce4427142b3a2011a47bec521d2@::1) 1.35ms From 28cae642d8405c9679c8f9a3d691b748ada000a3 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Fri, 26 Sep 2025 12:13:36 +0200 Subject: [PATCH 03/37] Add custom content provider on frontend Signed-off-by: Andy Jakubowski --- src/deepnote-content-provider.ts | 22 ++++++++++++++++++++++ src/index.tsx | 22 ++++++++++++++++++++++ 2 files changed, 44 insertions(+) create mode 100644 src/deepnote-content-provider.ts diff --git a/src/deepnote-content-provider.ts b/src/deepnote-content-provider.ts new file mode 100644 index 0000000..f6dd172 --- /dev/null +++ b/src/deepnote-content-provider.ts @@ -0,0 +1,22 @@ +import { Contents, RestContentProvider } from '@jupyterlab/services'; + +export const deepnoteContentProviderName = 'deepnote-content-provider'; + +export class DeepnoteContentProvider extends RestContentProvider { + async get( + localPath: string, + options?: Contents.IFetchOptions + ): Promise { + const model = await super.get(localPath, options); + const isDeepnoteFile = + localPath.endsWith('.deepnote') && model.type === 'notebook'; + + if (!isDeepnoteFile) { + // Not a .deepnote file, return as-is + return model; + } + + model.content.cells = []; + return model; + } +} diff --git a/src/index.tsx b/src/index.tsx index 9cd4658..8a59471 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -11,6 +11,11 @@ import { } from '@jupyterlab/notebook'; import { Widget } from '@lumino/widgets'; import { HTMLSelect } from '@jupyterlab/ui-components'; +import { ContentsManager } from '@jupyterlab/services'; +import { + DeepnoteContentProvider, + deepnoteContentProviderName +} from './deepnote-content-provider'; const plugin: JupyterFrontEndPlugin = { id: 'jupyterlab-deepnote:plugin', @@ -22,6 +27,23 @@ const plugin: JupyterFrontEndPlugin = { notebookWidgetFactory: NotebookWidgetFactory, toolbarRegistry: IToolbarWidgetRegistry ) => { + const drive = (app.serviceManager.contents as ContentsManager).defaultDrive; + const registry = drive?.contentProviderRegistry; + if (!registry) { + // If content provider is a non-essential feature and support for JupyterLab <4.4 is desired: + console.error( + 'Cannot initialize content provider: no content provider registry.' + ); + return; + } + const deepnoteContentProvider = new DeepnoteContentProvider({ + // These options are only required if extending the `RestContentProvider`. + apiEndpoint: '/api/contents', + serverSettings: app.serviceManager.serverSettings + }); + registry.register(deepnoteContentProviderName, deepnoteContentProvider); + notebookWidgetFactory.contentProviderId = deepnoteContentProviderName; + app.docRegistry.addFileType( { name: 'deepnote', From e4887595498e16543c907383e7213da1dad7b5af Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Fri, 26 Sep 2025 16:42:27 +0200 Subject: [PATCH 04/37] Remove Python YAML conversion logic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We’ll do conversion in TypeScript on the frontend Signed-off-by: Andy Jakubowski --- jupyterlab_deepnote/contents.py | 66 ++------------------------------- 1 file changed, 4 insertions(+), 62 deletions(-) diff --git a/jupyterlab_deepnote/contents.py b/jupyterlab_deepnote/contents.py index 679d01f..fc9aec0 100644 --- a/jupyterlab_deepnote/contents.py +++ b/jupyterlab_deepnote/contents.py @@ -2,63 +2,7 @@ from jupyter_server.services.contents.filemanager import FileContentsManager from typing import cast -import yaml -from nbformat.v4 import new_notebook, new_code_cell, new_markdown_cell - - -def yaml_to_ipynb(yaml_text: str): - """Convert Deepnote YAML into a minimal Jupyter nbformat v4 notebook.""" - try: - data = yaml.safe_load(yaml_text) - except Exception: - return new_notebook(cells=[]) - - notebooks = ( - data.get("project", {}).get("notebooks", []) if isinstance(data, dict) else [] - ) - - if not notebooks: - return new_notebook(cells=[]) - - # Collect notebook names - notebook_names = [nb.get("name", "") for nb in notebooks] - - # Build all_notebooks dict: name -> full nbformat notebook JSON - all_notebooks = {} - for nb in notebooks: - nb_blocks = nb.get("blocks", []) - nb_cells = [] - for block in sorted(nb_blocks, key=lambda b: b.get("sortingKey", "")): - btype = block.get("type", "code") - content = block.get("content", "") - if btype == "code": - nb_cells.append(new_code_cell(content)) - else: - nb_cells.append(new_markdown_cell(content)) - # Use the notebook name as key - nb_name = nb.get("name", "") - all_notebooks[nb_name] = new_notebook(cells=nb_cells) - - # Use first notebook's cells to render initially - nb0 = notebooks[0] - blocks = nb0.get("blocks", []) - cells = [] - for block in sorted(blocks, key=lambda b: b.get("sortingKey", "")): - btype = block.get("type", "code") - content = block.get("content", "") - if btype == "code": - cells.append(new_code_cell(content)) - else: - cells.append(new_markdown_cell(content)) - - metadata = { - "deepnote": {"notebook_names": notebook_names, "notebooks": all_notebooks} - } - return new_notebook(cells=cells, metadata=metadata) - - -def yaml_to_ipynb_dummy(yaml_text: str) -> dict: - return {"nbformat": 4, "nbformat_minor": 5, "metadata": {}, "cells": []} +from nbformat.v4 import new_notebook class DeepnoteContentsManager(FileContentsManager): @@ -74,15 +18,13 @@ def get(self, path, content=True, type=None, format=None, require_hash=False): else: yaml_text = cast(str, _content) - nb_node = yaml_to_ipynb(yaml_text) - model = self._base_model(path) model["type"] = "notebook" model["format"] = "json" - model["content"] = nb_node + model["content"] = new_notebook( + cells=[], metadata={"deepnote": {"rawYamlString": yaml_text}} + ) model["writable"] = False - self.mark_trusted_cells(nb_node, path) - self.validate_notebook_model(model, validation_error={}) if require_hash: # Accept 2- or 3-tuple; we only need the bytes From a15597701c6e02476a4ec3692b66ff49ea642183 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Mon, 29 Sep 2025 15:21:16 +0200 Subject: [PATCH 05/37] Edit tsconfig.json to include all files in `src` Signed-off-by: Andy Jakubowski --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index 25af040..8dc0b3e 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -21,5 +21,5 @@ "target": "ES2018", "types": ["jest"] }, - "include": ["src/*"] + "include": ["src"] } From 50e853693f0835406df3a8d6a4b53e5391802e15 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Mon, 29 Sep 2025 15:21:29 +0200 Subject: [PATCH 06/37] Extract `NotebookPicker` to own file Signed-off-by: Andy Jakubowski --- .../NotebookPicker.tsx} | 80 +------------------ src/index.ts | 77 ++++++++++++++++++ 2 files changed, 80 insertions(+), 77 deletions(-) rename src/{index.tsx => components/NotebookPicker.tsx} (59%) create mode 100644 src/index.ts diff --git a/src/index.tsx b/src/components/NotebookPicker.tsx similarity index 59% rename from src/index.tsx rename to src/components/NotebookPicker.tsx index 8a59471..cb1f2d4 100644 --- a/src/index.tsx +++ b/src/components/NotebookPicker.tsx @@ -1,81 +1,9 @@ -import { - JupyterFrontEnd, - JupyterFrontEndPlugin -} from '@jupyterlab/application'; import React from 'react'; -import { IToolbarWidgetRegistry, ReactWidget } from '@jupyterlab/apputils'; -import { - INotebookWidgetFactory, - NotebookPanel, - NotebookWidgetFactory -} from '@jupyterlab/notebook'; -import { Widget } from '@lumino/widgets'; +import { ReactWidget } from '@jupyterlab/apputils'; +import { NotebookPanel } from '@jupyterlab/notebook'; import { HTMLSelect } from '@jupyterlab/ui-components'; -import { ContentsManager } from '@jupyterlab/services'; -import { - DeepnoteContentProvider, - deepnoteContentProviderName -} from './deepnote-content-provider'; - -const plugin: JupyterFrontEndPlugin = { - id: 'jupyterlab-deepnote:plugin', - description: 'Open .deepnote files as notebooks.', - autoStart: true, - requires: [INotebookWidgetFactory, IToolbarWidgetRegistry], - activate: ( - app: JupyterFrontEnd, - notebookWidgetFactory: NotebookWidgetFactory, - toolbarRegistry: IToolbarWidgetRegistry - ) => { - const drive = (app.serviceManager.contents as ContentsManager).defaultDrive; - const registry = drive?.contentProviderRegistry; - if (!registry) { - // If content provider is a non-essential feature and support for JupyterLab <4.4 is desired: - console.error( - 'Cannot initialize content provider: no content provider registry.' - ); - return; - } - const deepnoteContentProvider = new DeepnoteContentProvider({ - // These options are only required if extending the `RestContentProvider`. - apiEndpoint: '/api/contents', - serverSettings: app.serviceManager.serverSettings - }); - registry.register(deepnoteContentProviderName, deepnoteContentProvider); - notebookWidgetFactory.contentProviderId = deepnoteContentProviderName; - - app.docRegistry.addFileType( - { - name: 'deepnote', - displayName: 'Deepnote Notebook', - extensions: ['.deepnote'], - mimeTypes: ['text/yaml', 'application/x-yaml'], - fileFormat: 'text', - contentType: 'file' - }, - [notebookWidgetFactory.name] - ); - - app.docRegistry.setDefaultWidgetFactory( - 'deepnote', - notebookWidgetFactory.name - ); - toolbarRegistry.addFactory( - notebookWidgetFactory.name, - 'deepnote:switch-notebook', - panel => { - if (!panel.context.path.endsWith('.deepnote')) { - return new Widget(); // don’t render for .ipynb or others - } - - return new NotebookPicker(panel); - } - ); - } -}; - -class NotebookPicker extends ReactWidget { +export class NotebookPicker extends ReactWidget { private selected: string | null = null; constructor(private panel: NotebookPanel) { @@ -195,5 +123,3 @@ class NotebookPicker extends ReactWidget { ); } } - -export default plugin; diff --git a/src/index.ts b/src/index.ts new file mode 100644 index 0000000..f77735c --- /dev/null +++ b/src/index.ts @@ -0,0 +1,77 @@ +import { + JupyterFrontEnd, + JupyterFrontEndPlugin +} from '@jupyterlab/application'; +import { IToolbarWidgetRegistry } from '@jupyterlab/apputils'; +import { + INotebookWidgetFactory, + NotebookPanel, + NotebookWidgetFactory +} from '@jupyterlab/notebook'; +import { Widget } from '@lumino/widgets'; +import { ContentsManager } from '@jupyterlab/services'; +import { + DeepnoteContentProvider, + deepnoteContentProviderName +} from './deepnote-content-provider'; +import { NotebookPicker } from './components/NotebookPicker'; + +const plugin: JupyterFrontEndPlugin = { + id: 'jupyterlab-deepnote:plugin', + description: 'Open .deepnote files as notebooks.', + autoStart: true, + requires: [INotebookWidgetFactory, IToolbarWidgetRegistry], + activate: ( + app: JupyterFrontEnd, + notebookWidgetFactory: NotebookWidgetFactory, + toolbarRegistry: IToolbarWidgetRegistry + ) => { + const drive = (app.serviceManager.contents as ContentsManager).defaultDrive; + const registry = drive?.contentProviderRegistry; + if (!registry) { + // If content provider is a non-essential feature and support for JupyterLab <4.4 is desired: + console.error( + 'Cannot initialize content provider: no content provider registry.' + ); + return; + } + const deepnoteContentProvider = new DeepnoteContentProvider({ + // These options are only required if extending the `RestContentProvider`. + apiEndpoint: '/api/contents', + serverSettings: app.serviceManager.serverSettings + }); + registry.register(deepnoteContentProviderName, deepnoteContentProvider); + notebookWidgetFactory.contentProviderId = deepnoteContentProviderName; + + app.docRegistry.addFileType( + { + name: 'deepnote', + displayName: 'Deepnote Notebook', + extensions: ['.deepnote'], + mimeTypes: ['text/yaml', 'application/x-yaml'], + fileFormat: 'text', + contentType: 'file' + }, + [notebookWidgetFactory.name] + ); + + app.docRegistry.setDefaultWidgetFactory( + 'deepnote', + notebookWidgetFactory.name + ); + + toolbarRegistry.addFactory( + notebookWidgetFactory.name, + 'deepnote:switch-notebook', + panel => { + if (!panel.context.path.endsWith('.deepnote')) { + return new Widget(); // don’t render for .ipynb or others + } + + return new NotebookPicker(panel); + } + ); + } +}; + +export default plugin; From 91bc7e5dc586d0706182ba4c27d0d7edcb20e460 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Mon, 29 Sep 2025 15:24:47 +0200 Subject: [PATCH 07/37] Add explanatory comments to `index.tsx` Signed-off-by: Andy Jakubowski --- src/index.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/index.ts b/src/index.ts index f77735c..c54e61a 100644 --- a/src/index.ts +++ b/src/index.ts @@ -26,6 +26,7 @@ const plugin: JupyterFrontEndPlugin = { notebookWidgetFactory: NotebookWidgetFactory, toolbarRegistry: IToolbarWidgetRegistry ) => { + // Register a custom contents provider for the default notebook widget factory. const drive = (app.serviceManager.contents as ContentsManager).defaultDrive; const registry = drive?.contentProviderRegistry; if (!registry) { @@ -43,6 +44,7 @@ const plugin: JupyterFrontEndPlugin = { registry.register(deepnoteContentProviderName, deepnoteContentProvider); notebookWidgetFactory.contentProviderId = deepnoteContentProviderName; + // Register .deepnote file type and set the notebook widget factory as the default. app.docRegistry.addFileType( { name: 'deepnote', @@ -54,12 +56,12 @@ const plugin: JupyterFrontEndPlugin = { }, [notebookWidgetFactory.name] ); - app.docRegistry.setDefaultWidgetFactory( 'deepnote', notebookWidgetFactory.name ); + // Add a toolbar item to switch between notebooks in a .deepnote file. toolbarRegistry.addFactory( notebookWidgetFactory.name, 'deepnote:switch-notebook', From 7e98a3c9be781f78461f215b170d6ae63257a2cc Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Mon, 29 Sep 2025 15:42:11 +0200 Subject: [PATCH 08/37] Validate model content with Zod schema Signed-off-by: Andy Jakubowski --- package.json | 3 ++- src/deepnote-content-provider.ts | 27 ++++++++++++++++++++++++++- yarn.lock | 8 ++++++++ 3 files changed, 36 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 964568c..8ecb797 100644 --- a/package.json +++ b/package.json @@ -63,7 +63,8 @@ "@jupyterlab/notebook": "^4.4.7", "@jupyterlab/services": "^7.0.0", "@jupyterlab/settingregistry": "^4.0.0", - "@lumino/widgets": "^2.7.1" + "@lumino/widgets": "^2.7.1", + "zod": "^4.1.11" }, "devDependencies": { "@jupyterlab/builder": "^4.0.0", diff --git a/src/deepnote-content-provider.ts b/src/deepnote-content-provider.ts index f6dd172..fcaa035 100644 --- a/src/deepnote-content-provider.ts +++ b/src/deepnote-content-provider.ts @@ -1,7 +1,19 @@ import { Contents, RestContentProvider } from '@jupyterlab/services'; +import { z } from 'zod'; export const deepnoteContentProviderName = 'deepnote-content-provider'; +const deepnoteNotebookSchema = z.object({ + cells: z.array(z.any()), // or refine further with nbformat + metadata: z.object({ + deepnote: z.object({ + rawYamlString: z.string() + }) + }), + nbformat: z.number(), + nbformat_minor: z.number() +}); + export class DeepnoteContentProvider extends RestContentProvider { async get( localPath: string, @@ -16,7 +28,20 @@ export class DeepnoteContentProvider extends RestContentProvider { return model; } - model.content.cells = []; + const validatedModelContent = deepnoteNotebookSchema.safeParse( + model.content + ); + + if (!validatedModelContent.success) { + console.error( + 'Invalid .deepnote file content:', + validatedModelContent.error + ); + // Return an empty notebook instead of throwing an error + model.content.cells = []; + return model; + } + return model; } } diff --git a/yarn.lock b/yarn.lock index dd8960e..7b00140 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6826,6 +6826,7 @@ __metadata: stylelint-prettier: ^4.0.0 typescript: ~5.8.0 yjs: ^13.5.0 + zod: ^4.1.11 languageName: unknown linkType: soft @@ -9805,3 +9806,10 @@ __metadata: checksum: f77b3d8d00310def622123df93d4ee654fc6a0096182af8bd60679ddcdfb3474c56c6c7190817c84a2785648cdee9d721c0154eb45698c62176c322fb46fc700 languageName: node linkType: hard + +"zod@npm:^4.1.11": + version: 4.1.11 + resolution: "zod@npm:4.1.11" + checksum: 022d59f85ebe054835fbcdc96a93c01479a64321104f846cb5644812c91e00d17ae3479f823956ec9b04e4351dd32841e1f12c567e81bc43f6e21ef5cc02ce3c + languageName: node + linkType: hard From fab6c3969f78c529b9fb415f8cf86915ef72cd7f Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Mon, 29 Sep 2025 15:53:28 +0200 Subject: [PATCH 09/37] Stub out transformDeepnoteYamlToNotebookContent Signed-off-by: Andy Jakubowski --- src/deepnote-content-provider.ts | 12 ++++++- ...sform-deepnote-yaml-to-notebook-content.ts | 36 +++++++++++++++++++ 2 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 src/transform-deepnote-yaml-to-notebook-content.ts diff --git a/src/deepnote-content-provider.ts b/src/deepnote-content-provider.ts index fcaa035..da011b3 100644 --- a/src/deepnote-content-provider.ts +++ b/src/deepnote-content-provider.ts @@ -1,5 +1,6 @@ import { Contents, RestContentProvider } from '@jupyterlab/services'; import { z } from 'zod'; +import { transformDeepnoteYamlToNotebookContent } from './transform-deepnote-yaml-to-notebook-content'; export const deepnoteContentProviderName = 'deepnote-content-provider'; @@ -42,6 +43,15 @@ export class DeepnoteContentProvider extends RestContentProvider { return model; } - return model; + const transformedModelContent = transformDeepnoteYamlToNotebookContent( + validatedModelContent.data.metadata.deepnote.rawYamlString + ); + + const transformedModel = { + ...model, + content: transformedModelContent + }; + + return transformedModel; } } diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts new file mode 100644 index 0000000..3344f80 --- /dev/null +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -0,0 +1,36 @@ +import { INotebookContent, INotebookMetadata } from '@jupyterlab/nbformat'; + +export interface IDeepnoteNotebookMetadata extends INotebookMetadata { + deepnote: { + rawYamlString: string; + }; +} + +export interface IDeepnoteNotebookContent + extends Omit { + metadata: IDeepnoteNotebookMetadata; +} + +export function transformDeepnoteYamlToNotebookContent( + yamlString: string +): IDeepnoteNotebookContent { + // Placeholder implementation + return { + cells: [ + { + cell_type: 'code', + source: '# Transformed from Deepnote YAML\n', + metadata: {}, + outputs: [], + execution_count: null + } + ], + metadata: { + deepnote: { + rawYamlString: yamlString + } + }, + nbformat: 4, + nbformat_minor: 0 + }; +} From 9863caed20d289d2540059ec8a6528692f599940 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Mon, 29 Sep 2025 17:01:18 +0200 Subject: [PATCH 10/37] Deserialize Deepnote file YAML into object Signed-off-by: Andy Jakubowski --- package.json | 1 + src/deepnote-content-provider.ts | 7 +- src/deepnote-convert/deepnote-file-schema.ts | 72 +++++++++++++++++++ .../deserialize-deepnote-file.ts | 38 ++++++++++ src/deepnote-convert/parse-yaml.ts | 22 ++++++ src/fallback-data.ts | 30 ++++++++ ...sform-deepnote-yaml-to-notebook-content.ts | 58 +++++++-------- src/types.ts | 14 ++++ yarn.lock | 10 +++ 9 files changed, 217 insertions(+), 35 deletions(-) create mode 100644 src/deepnote-convert/deepnote-file-schema.ts create mode 100644 src/deepnote-convert/deserialize-deepnote-file.ts create mode 100644 src/deepnote-convert/parse-yaml.ts create mode 100644 src/fallback-data.ts create mode 100644 src/types.ts diff --git a/package.json b/package.json index 8ecb797..93edfc2 100644 --- a/package.json +++ b/package.json @@ -64,6 +64,7 @@ "@jupyterlab/services": "^7.0.0", "@jupyterlab/settingregistry": "^4.0.0", "@lumino/widgets": "^2.7.1", + "yaml": "^2.8.1", "zod": "^4.1.11" }, "devDependencies": { diff --git a/src/deepnote-content-provider.ts b/src/deepnote-content-provider.ts index da011b3..86ec0e0 100644 --- a/src/deepnote-content-provider.ts +++ b/src/deepnote-content-provider.ts @@ -43,9 +43,10 @@ export class DeepnoteContentProvider extends RestContentProvider { return model; } - const transformedModelContent = transformDeepnoteYamlToNotebookContent( - validatedModelContent.data.metadata.deepnote.rawYamlString - ); + const transformedModelContent = + await transformDeepnoteYamlToNotebookContent( + validatedModelContent.data.metadata.deepnote.rawYamlString + ); const transformedModel = { ...model, diff --git a/src/deepnote-convert/deepnote-file-schema.ts b/src/deepnote-convert/deepnote-file-schema.ts new file mode 100644 index 0000000..6df5332 --- /dev/null +++ b/src/deepnote-convert/deepnote-file-schema.ts @@ -0,0 +1,72 @@ +import { z } from 'zod'; + +// Source: +// deepnote-internal +// +// Path: +// apps/webapp/server/modules/export-and-import-project/deepnote-file.ts + +// Commit SHA: +// 3ec11e794c6aca998ef88d894f18e4611586cc30 + +export const deepnoteFileSchema = z.object({ + metadata: z.object({ + checksum: z.string().optional(), + createdAt: z.string(), + exportedAt: z.string().optional(), + modifiedAt: z.string().optional() + }), + + project: z.object({ + id: z.string(), + + initNotebookId: z.string().optional(), + integrations: z + .array( + z.object({ + id: z.string(), + name: z.string(), + type: z.string() + }) + ) + .optional(), + name: z.string(), + notebooks: z.array( + z.object({ + blocks: z.array( + z.object({ + blockGroup: z.string().optional(), + content: z.string().optional(), + executionCount: z.number().optional(), + id: z.string(), + metadata: z.record(z.string(), z.any()).optional(), + outputs: z.array(z.any()).optional(), + sortingKey: z.string(), + type: z.string(), + version: z.number().optional() + }) + ), + executionMode: z.enum(['block', 'downstream']).optional(), + id: z.string(), + isModule: z.boolean().optional(), + name: z.string(), + workingDirectory: z.string().optional() + }) + ), + settings: z + .object({ + environment: z + .object({ + customImage: z.string().optional(), + pythonVersion: z.string().optional() + }) + .optional(), + requirements: z.array(z.string()).optional(), + sqlCacheMaxAge: z.number().optional() + }) + .optional() + }), + version: z.string() +}); + +export type DeepnoteFile = z.infer; diff --git a/src/deepnote-convert/deserialize-deepnote-file.ts b/src/deepnote-convert/deserialize-deepnote-file.ts new file mode 100644 index 0000000..b2f3807 --- /dev/null +++ b/src/deepnote-convert/deserialize-deepnote-file.ts @@ -0,0 +1,38 @@ +import { DeepnoteFile, deepnoteFileSchema } from './deepnote-file-schema'; +import { parseYaml } from './parse-yaml'; + +// Source: +// deepnote-internal +// +// Path: +// apps/webapp/server/modules/export-and-import-project/index.ts + +// Commit SHA: +// 3ec11e794c6aca998ef88d894f18e4611586cc30 + +/** + * Deserialize a YAML string into a DeepnoteFile object. + */ +export function deserializeDeepnoteFile(yamlContent: string): DeepnoteFile { + const parsed = parseYaml(yamlContent); + const result = deepnoteFileSchema.safeParse(parsed); + + if (!result.success) { + const issue = result.error.issues[0]; + + if (!issue) { + console.error('Invalid Deepnote file with no issues.'); + + throw new Error('Invalid Deepnote file.'); + } + + const path = issue.path.join('.'); + const message = path ? `${path}: ${issue.message}` : issue.message; + + console.error(`Failed to parse the Deepnote file: ${message}.`); + + throw new Error(`Failed to parse the Deepnote file: ${message}.`); + } + + return result.data; +} diff --git a/src/deepnote-convert/parse-yaml.ts b/src/deepnote-convert/parse-yaml.ts new file mode 100644 index 0000000..a183fc8 --- /dev/null +++ b/src/deepnote-convert/parse-yaml.ts @@ -0,0 +1,22 @@ +import { parse } from 'yaml'; + +// Source: +// deepnote-internal +// +// Path: +// apps/webapp/server/modules/export-and-import-project/index.ts + +// Commit SHA: +// 3ec11e794c6aca998ef88d894f18e4611586cc30 + +export function parseYaml(yamlContent: string): unknown { + try { + const parsed = parse(yamlContent); + + return parsed; + } catch (e) { + console.error('Failed to parse Deepnote file as YAML.', e); + + throw new Error('Failed to parse Deepnote file.'); + } +} diff --git a/src/fallback-data.ts b/src/fallback-data.ts new file mode 100644 index 0000000..7d3f45c --- /dev/null +++ b/src/fallback-data.ts @@ -0,0 +1,30 @@ +import { ICodeCell } from '@jupyterlab/nbformat'; +import { IDeepnoteNotebookContent } from './types'; + +export const blankCodeCell: ICodeCell = { + cell_type: 'code', + source: '', + metadata: {}, + outputs: [], + execution_count: null +}; + +export const blankDeepnoteNotebookContent: IDeepnoteNotebookContent = { + cells: [ + { + cell_type: 'code', + source: '# Transformed from Deepnote YAML\n', + metadata: {}, + outputs: [], + execution_count: null + } + ], + metadata: { + deepnote: { + rawYamlString: null, + deepnoteFile: null + } + }, + nbformat: 4, + nbformat_minor: 0 +}; diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 3344f80..74dbffe 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -1,36 +1,30 @@ -import { INotebookContent, INotebookMetadata } from '@jupyterlab/nbformat'; +import { deserializeDeepnoteFile } from './deepnote-convert/deserialize-deepnote-file'; +import { IDeepnoteNotebookContent } from './types'; +import { blankCodeCell, blankDeepnoteNotebookContent } from './fallback-data'; -export interface IDeepnoteNotebookMetadata extends INotebookMetadata { - deepnote: { - rawYamlString: string; - }; -} +export async function transformDeepnoteYamlToNotebookContent( + yamlString: string +): Promise { + try { + const deepnoteFile = await deserializeDeepnoteFile(yamlString); -export interface IDeepnoteNotebookContent - extends Omit { - metadata: IDeepnoteNotebookMetadata; -} + const selectedNotebook = deepnoteFile.project.notebooks[0]; -export function transformDeepnoteYamlToNotebookContent( - yamlString: string -): IDeepnoteNotebookContent { - // Placeholder implementation - return { - cells: [ - { - cell_type: 'code', - source: '# Transformed from Deepnote YAML\n', - metadata: {}, - outputs: [], - execution_count: null - } - ], - metadata: { - deepnote: { - rawYamlString: yamlString - } - }, - nbformat: 4, - nbformat_minor: 0 - }; + if (!selectedNotebook) { + return { + ...blankDeepnoteNotebookContent, + cells: [ + { + ...blankCodeCell, + source: '# No notebooks found in Deepnote file.\n' + } + ] + }; + } + + return blankDeepnoteNotebookContent; + } catch (error) { + console.error('Failed to deserialize Deepnote file:', error); + throw new Error('Failed to transform Deepnote YAML to notebook content.'); + } } diff --git a/src/types.ts b/src/types.ts new file mode 100644 index 0000000..477566c --- /dev/null +++ b/src/types.ts @@ -0,0 +1,14 @@ +import { INotebookContent, INotebookMetadata } from '@jupyterlab/nbformat'; +import { DeepnoteFile } from './deepnote-convert/deepnote-file-schema'; + +export interface IDeepnoteNotebookMetadata extends INotebookMetadata { + deepnote: { + rawYamlString: string | null; + deepnoteFile: DeepnoteFile | null; + }; +} + +export interface IDeepnoteNotebookContent + extends Omit { + metadata: IDeepnoteNotebookMetadata; +} diff --git a/yarn.lock b/yarn.lock index 7b00140..13e9de5 100644 --- a/yarn.lock +++ b/yarn.lock @@ -6825,6 +6825,7 @@ __metadata: stylelint-csstree-validator: ^3.0.0 stylelint-prettier: ^4.0.0 typescript: ~5.8.0 + yaml: ^2.8.1 yjs: ^13.5.0 zod: ^4.1.11 languageName: unknown @@ -9762,6 +9763,15 @@ __metadata: languageName: node linkType: hard +"yaml@npm:^2.8.1": + version: 2.8.1 + resolution: "yaml@npm:2.8.1" + bin: + yaml: bin.mjs + checksum: 35b46150d48bc1da2fd5b1521a48a4fa36d68deaabe496f3c3fa9646d5796b6b974f3930a02c4b5aee6c85c860d7d7f79009416724465e835f40b87898c36de4 + languageName: node + linkType: hard + "yargs-parser@npm:^20.2.9": version: 20.2.9 resolution: "yargs-parser@npm:20.2.9" From aec4f6be40cde5f76850437f5536c1796141fb18 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 30 Sep 2025 11:34:15 +0200 Subject: [PATCH 11/37] Enable noUncheckedIndexedAccess in TS config Signed-off-by: Andy Jakubowski --- tsconfig.json | 1 + 1 file changed, 1 insertion(+) diff --git a/tsconfig.json b/tsconfig.json index 8dc0b3e..d05083b 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -18,6 +18,7 @@ "rootDir": "src", "strict": true, "strictNullChecks": true, + "noUncheckedIndexedAccess": true, "target": "ES2018", "types": ["jest"] }, From 3ed238f5c83e139e67bd1801e962bf63751c48a1 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 30 Sep 2025 11:40:56 +0200 Subject: [PATCH 12/37] Stub out block conversion Signed-off-by: Andy Jakubowski --- src/deepnote-convert/deepnote-file-schema.ts | 30 ++++++++------- ...sform-deepnote-yaml-to-notebook-content.ts | 38 ++++++++++++++++++- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/src/deepnote-convert/deepnote-file-schema.ts b/src/deepnote-convert/deepnote-file-schema.ts index 6df5332..46ed95a 100644 --- a/src/deepnote-convert/deepnote-file-schema.ts +++ b/src/deepnote-convert/deepnote-file-schema.ts @@ -1,5 +1,7 @@ import { z } from 'zod'; +// Below schema has been modified from the original deepnote-internal schema + // Source: // deepnote-internal // @@ -9,6 +11,20 @@ import { z } from 'zod'; // Commit SHA: // 3ec11e794c6aca998ef88d894f18e4611586cc30 +export const deepnoteFileBlockSchema = z.object({ + blockGroup: z.string().optional(), + content: z.string().optional(), + executionCount: z.number().optional(), + id: z.string(), + metadata: z.record(z.string(), z.any()).optional(), + outputs: z.array(z.any()).optional(), + sortingKey: z.string(), + type: z.string(), + version: z.number().optional() +}); + +export type DeepnoteFileBlock = z.infer; + export const deepnoteFileSchema = z.object({ metadata: z.object({ checksum: z.string().optional(), @@ -33,19 +49,7 @@ export const deepnoteFileSchema = z.object({ name: z.string(), notebooks: z.array( z.object({ - blocks: z.array( - z.object({ - blockGroup: z.string().optional(), - content: z.string().optional(), - executionCount: z.number().optional(), - id: z.string(), - metadata: z.record(z.string(), z.any()).optional(), - outputs: z.array(z.any()).optional(), - sortingKey: z.string(), - type: z.string(), - version: z.number().optional() - }) - ), + blocks: z.array(deepnoteFileBlockSchema), executionMode: z.enum(['block', 'downstream']).optional(), id: z.string(), isModule: z.boolean().optional(), diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 74dbffe..2240fe6 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -1,6 +1,35 @@ import { deserializeDeepnoteFile } from './deepnote-convert/deserialize-deepnote-file'; import { IDeepnoteNotebookContent } from './types'; import { blankCodeCell, blankDeepnoteNotebookContent } from './fallback-data'; +import { DeepnoteFileBlock } from './deepnote-convert/deepnote-file-schema'; +import { ICodeCell, IMarkdownCell } from '@jupyterlab/nbformat'; + +function convertDeepnoteBlockToJupyterCell( + block: DeepnoteFileBlock +): ICodeCell | IMarkdownCell { + if (block.type === 'code') { + return { + cell_type: 'code', + source: block.content || '', + metadata: {}, + outputs: block.outputs || [], + execution_count: block.executionCount || null + }; + } else if (block.type === 'markdown') { + return { + cell_type: 'markdown', + source: block.content || '', + metadata: {} + }; + } else { + // For unsupported block types, return a markdown cell indicating it's unsupported + return { + cell_type: 'markdown', + source: `# Unsupported block type: ${block.type}\n`, + metadata: {} + }; + } +} export async function transformDeepnoteYamlToNotebookContent( yamlString: string @@ -22,7 +51,14 @@ export async function transformDeepnoteYamlToNotebookContent( }; } - return blankDeepnoteNotebookContent; + const cells = selectedNotebook.blocks.map( + convertDeepnoteBlockToJupyterCell + ); + + return { + ...blankDeepnoteNotebookContent, + cells + }; } catch (error) { console.error('Failed to deserialize Deepnote file:', error); throw new Error('Failed to transform Deepnote YAML to notebook content.'); From c765dad0587a16f43005712f0bb334057d47549e Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 11:12:23 +0200 Subject: [PATCH 13/37] Update TS config Signed-off-by: Andy Jakubowski --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index d05083b..ae922ef 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,7 +6,7 @@ "esModuleInterop": true, "incremental": true, "jsx": "react", - "lib": ["DOM", "ES2018", "ES2020.Intl"], + "lib": ["ES2022"], "module": "esnext", "moduleResolution": "node", "noEmitOnError": true, From 5799272053545f0e56bf3b94d681ac65b31a705d Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 11:13:15 +0200 Subject: [PATCH 14/37] Add temporary conversion code Signed-off-by: Andy Jakubowski --- package.json | 13 - src/convert-deepnote-block-to-jupyter-cell.ts | 31 + src/deepnote-convert/assert-unreachable.ts | 16 + .../blocks/block-metadata-utils.ts | 86 +++ src/deepnote-convert/cells/cell-utils.ts | 457 +++++++++++++ .../convert-cell-type-to-jupyter.ts | 56 ++ .../convert-deepnote-cell-to-jupyter-cell.ts | 627 ++++++++++++++++++ src/deepnote-convert/deepnote-file-schema.ts | 3 +- .../shared-table-state-schemas.ts | 197 ++++++ src/deepnote-convert/types.ts | 59 ++ src/deepnote-convert/utils/object-utils.ts | 50 ++ src/deepnote-convert/vega-lite.types.ts | 414 ++++++++++++ ...sform-deepnote-yaml-to-notebook-content.ts | 34 +- 13 files changed, 1997 insertions(+), 46 deletions(-) create mode 100644 src/convert-deepnote-block-to-jupyter-cell.ts create mode 100644 src/deepnote-convert/assert-unreachable.ts create mode 100644 src/deepnote-convert/blocks/block-metadata-utils.ts create mode 100644 src/deepnote-convert/cells/cell-utils.ts create mode 100644 src/deepnote-convert/convert-cell-type-to-jupyter.ts create mode 100644 src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts create mode 100644 src/deepnote-convert/shared-table-state-schemas.ts create mode 100644 src/deepnote-convert/types.ts create mode 100644 src/deepnote-convert/utils/object-utils.ts create mode 100644 src/deepnote-convert/vega-lite.types.ts diff --git a/package.json b/package.json index 93edfc2..1db3271 100644 --- a/package.json +++ b/package.json @@ -143,19 +143,6 @@ "@typescript-eslint" ], "rules": { - "@typescript-eslint/naming-convention": [ - "error", - { - "selector": "interface", - "format": [ - "PascalCase" - ], - "custom": { - "regex": "^I[A-Z]", - "match": true - } - } - ], "@typescript-eslint/no-unused-vars": [ "warn", { diff --git a/src/convert-deepnote-block-to-jupyter-cell.ts b/src/convert-deepnote-block-to-jupyter-cell.ts new file mode 100644 index 0000000..13ea18d --- /dev/null +++ b/src/convert-deepnote-block-to-jupyter-cell.ts @@ -0,0 +1,31 @@ +import { convertCellTypeToJupyter } from './deepnote-convert/convert-cell-type-to-jupyter'; +import { DeepnoteFileBlock } from './deepnote-convert/deepnote-file-schema'; +import { ICodeCell, IMarkdownCell } from '@jupyterlab/nbformat'; + +export function convertDeepnoteBlockToJupyterCell( + block: DeepnoteFileBlock +): ICodeCell | IMarkdownCell { + const jupyterCellType = convertCellTypeToJupyter(block.type); + if (jupyterCellType === 'code') { + return { + cell_type: 'code', + source: block.content || '', + metadata: {}, + outputs: block.outputs || [], + execution_count: block.executionCount || null + }; + } else if (jupyterCellType === 'markdown') { + return { + cell_type: 'markdown', + source: block.content || '', + metadata: {} + }; + } else { + // For unsupported block types, return a markdown cell indicating it's unsupported + return { + cell_type: 'markdown', + source: `# Unsupported block type: ${block.type}\n`, + metadata: {} + }; + } +} diff --git a/src/deepnote-convert/assert-unreachable.ts b/src/deepnote-convert/assert-unreachable.ts new file mode 100644 index 0000000..cbb30df --- /dev/null +++ b/src/deepnote-convert/assert-unreachable.ts @@ -0,0 +1,16 @@ +// Source: +// deepnote-internal +// +// Path: +// libs/shared/src/utils/assert-unreachable.ts + +// Commit SHA: +// ebdadfe8f16b7fb2279c24b2362734909cab5d4d + +/** + * Utility function that uses the typechecker to check that some conditions never happen. + * Very useful e.g. for checking that we have covered all cases in a switch statement of in a long list of if's. + * This is purely for typechecking purposes and has no runtime implications (it's just an identity function at runtime). + * @example See Icon.tsx + */ +export const assertUnreachable = (x: never) => x; diff --git a/src/deepnote-convert/blocks/block-metadata-utils.ts b/src/deepnote-convert/blocks/block-metadata-utils.ts new file mode 100644 index 0000000..03296b1 --- /dev/null +++ b/src/deepnote-convert/blocks/block-metadata-utils.ts @@ -0,0 +1,86 @@ +import { pickBy } from '../utils' + +import type { SharedTableState } from '../data-table/shared-table-state-schemas' +import type { ExecutableCellMetadata } from '../types' + +export type BlockExecutionStateData = Pick< + ExecutableCellMetadata, + | 'output_cleared' + | 'execution_start' + | 'execution_millis' + | 'source_hash' + | 'deepnote_to_be_reexecuted' + | 'last_executed_function_notebook_id' + | 'last_function_run_started_at' + | 'function_notebook_export_states' +> & { + 'execution_count': number | null + 'outputs_reference': string | null + 'execution_context_id'?: string + 'deepnote_table_state'?: SharedTableState + 'deepnote_table_loading'?: boolean + 'deepnote_table_invalid'?: boolean +} + +export function createMetadataAndStateFromBlock>(block: { + block_type: T + metadata: M + execution_count: number | null + outputs_reference: string | null +}) { + const { metadata, state: stateFromMetadata } = splitBlockMetadataAndState(block.metadata) + + const state: BlockExecutionStateData = { + ...stateFromMetadata, + execution_count: block.execution_count, + outputs_reference: block.outputs_reference, + } + + return { + blockType: block.block_type, + metadata, + state, + } +} + +export function splitBlockMetadataAndState>(fullMetadata: M) { + const { + output_cleared, + execution_start, + execution_millis, + source_hash, + execution_context_id, + deepnote_to_be_reexecuted, + deepnote_table_state: tableStateMetadataKey, + deepnote_table_loading: tableLoadingMetadataKey, + deepnote_table_invalid: tableInvalidMetadataKey, + last_executed_function_notebook_id: lastExecutedFunctionNotebookId, + last_function_run_started_at: lastExecutedFunctionTimestamp, + function_notebook_export_states: functionNotebookExportStates, + cell_id: droppedCellId, + ...metadata + } = fullMetadata + + const state: Omit = pickBy( + { + output_cleared, + execution_start, + execution_millis, + source_hash, + execution_context_id, + deepnote_to_be_reexecuted, + deepnote_table_state: tableStateMetadataKey, + deepnote_table_loading: tableLoadingMetadataKey, + deepnote_table_invalid: tableInvalidMetadataKey, + last_executed_function_notebook_id: lastExecutedFunctionNotebookId, + last_function_run_started_at: lastExecutedFunctionTimestamp, + function_notebook_export_states: functionNotebookExportStates, + }, + value => typeof value !== 'undefined' + ) + + return { + metadata, + state, + } +} diff --git a/src/deepnote-convert/cells/cell-utils.ts b/src/deepnote-convert/cells/cell-utils.ts new file mode 100644 index 0000000..f9b9d8d --- /dev/null +++ b/src/deepnote-convert/cells/cell-utils.ts @@ -0,0 +1,457 @@ +import _countBy from 'lodash/countBy'; + +import { createMetadataAndStateFromBlock } from '../blocks/block-metadata-utils'; +import { INPUT_CELL_TYPES, isBigNumberCell, TEXT_CELL_TYPES } from '../types'; +import { bigNumberCellUtils } from './big-number-cell-utils'; +import { buttonCellUtils } from './button-cell-utils'; +import { codeCellUtils } from './code-cell-utils'; +import { imageCellUtils } from './image-cell-utils'; +import { inputCheckboxCellUtils } from './input-checkbox-cell-utils'; +import { inputDateCellUtils } from './input-date-cell-utils'; +import { inputDateRangeCellUtils } from './input-date-range-cell-utils'; +import { inputFileCellUtils } from './input-file-cell-utils'; +import { + inputSelectCellUtils, + validateSelectInputVariable +} from './input-select-cell-utils'; +import { inputSliderCellUtils } from './input-slider-cell-utils'; +import { inputTextCellUtils } from './input-text-cell-utils'; +import { inputTextareaCellUtils } from './input-textarea-cell-utils'; +import { markdownCellUtils } from './markdown-cell-utils'; +import { notebookFunctionCellUtils } from './notebook-function-cell-utils'; +import { separatorCellUtils } from './separator-cell-utils'; +import { sqlCellUtils } from './sql-cell-utils'; +import { + textCellBulletUtils, + textCellCalloutUtils, + textCellH1Utils, + textCellH2Utils, + textCellH3Utils, + textCellParagraphUtils, + textCellTodoUtils +} from './text-cell-utils'; +import { visualizationCellUtils } from './visualization-cell-utils'; + +import type { JupyterCell } from '@deepnote/jupyter'; +import type { IOutput } from '@jupyterlab/nbformat'; +import type { + ButtonCell, + Cell, + CellByType, + CellType, + CodeCell, + DataTableCell, + DetachedCell, + DetachedExecutableCell, + DetachedExecutableCellWithContentDeps, + EmbeddableCell, + ExecutableCell, + ExecutableCellWithContentDeps, + ImageCell, + InputCell, + InputCellType, + InputCheckboxCell, + InputDateCell, + InputDateRangeCell, + InputFileCell, + InputSelectCell, + InputSliderCell, + InputTextareaCell, + InputTextCell, + MarkdownCell, + NotebookFunctionCell, + SeparatorCell, + SqlCell, + TextCell, + TextCellType, + VisualizationCell +} from '../types'; + +export { validateSelectInputVariable }; + +// see https://github.com/deepnote/compute-helpers/blob/af569f06396a36bbc7e221445dfb23093ab8db39/code/sql_utils.py#L15 +export type SqlCacheMode = 'cache_disabled' | 'always_write' | 'read_or_write'; + +export type SubmittedBy = 'user' | 'scheduling' | 'api' | 'ai'; + +// More specific type for workload context to track why workloads are running +export type WorkloadType = 'notebook' | 'app' | 'api' | 'scheduled-run'; + +export type VariableContext = string[]; + +export interface ExecutionContext { + user?: { id: string; email: string | null }; + submittedBy?: SubmittedBy; + // TODO: It would probably be nicer to use `SqlAlchemyInput` type here. + // To achieve that, we would need to move `SqlAlchemyInput` to `libs/shared` package. + federatedIntegrationConnectionString?: string | null; + parentNotebookFunctionRunId?: string | null; + notebookFunctionApiToken?: string | null; + /** + * If set, button blocks with this variable name that are being executed + * will resolve their source code to set the variable to True, False otherwise. + */ + variableContext?: VariableContext; + workspaceId: string; + projectId: string; + notebookId: string; + sqlCacheMode: SqlCacheMode; + tenantDomain?: string; +} + +export interface DeepnoteCellUtils { + parseJupyterSource: ( + cellId: string, + jupyterCell: JupyterCell + ) => Extract; + createJupyterSource: ( + data: { + type: TCell['cell_type']; + source: TCell['source']; + metadata: TCell['metadata']; + }, + executionContext?: ExecutionContext + ) => string; + // NOTE: This wrapper is not included in source hash calculations. + wrapJupyterSource?: ( + code: string, + data: { + type: TCell['cell_type']; + source: TCell['source']; + metadata: TCell['metadata']; + }, + executionContext?: ExecutionContext + ) => string; + createInterruptJupyterSource?: ( + data: { + type: TCell['cell_type']; + source: TCell['source']; + metadata: TCell['metadata']; + }, + executionContext?: ExecutionContext + ) => string | null; + cleanJupyterOutputs?: (outputs: IOutput[]) => IOutput[]; + createNewCell(params: { + id: string; + existingCellVariableNames: Set; + source?: string; + metadata?: TCell['metadata']; + }): TCell & { cellId: string }; +} + +export const cellUtils: { + [cellType in CellType]: DeepnoteCellUtils; +} = { + 'big-number': bigNumberCellUtils, + sql: sqlCellUtils, + code: codeCellUtils, + markdown: markdownCellUtils, + 'notebook-function': notebookFunctionCellUtils, + 'input-text': inputTextCellUtils, + 'input-textarea': inputTextareaCellUtils, + 'input-file': inputFileCellUtils, + 'input-select': inputSelectCellUtils, + 'input-date': inputDateCellUtils, + 'input-date-range': inputDateRangeCellUtils, + 'input-slider': inputSliderCellUtils, + 'input-checkbox': inputCheckboxCellUtils, + 'text-cell-h1': textCellH1Utils, + 'text-cell-h2': textCellH2Utils, + 'text-cell-h3': textCellH3Utils, + 'text-cell-p': textCellParagraphUtils, + 'text-cell-bullet': textCellBulletUtils, + 'text-cell-todo': textCellTodoUtils, + 'text-cell-callout': textCellCalloutUtils, + visualization: visualizationCellUtils, + image: imageCellUtils, + button: buttonCellUtils, + separator: separatorCellUtils +} as const; + +export function createJupyterSource( + data: { + type: CellByType[T]['cell_type']; + source: CellByType[T]['source']; + metadata: CellByType[T]['metadata']; + }, + executionContext?: ExecutionContext +): string { + const cellUtil = cellUtils[data.type]; + return cellUtil.createJupyterSource(data, executionContext); +} + +export function wrapJupyterSource( + code: string, + data: { + type: CellByType[T]['cell_type']; + source: CellByType[T]['source']; + metadata: CellByType[T]['metadata']; + }, + executionContext?: ExecutionContext +): string { + const cellUtil = cellUtils[data.type]; + return cellUtil.wrapJupyterSource + ? cellUtil.wrapJupyterSource(code, data, executionContext) + : code; +} + +export function createInterruptJupyterSource( + data: { + type: CellByType[T]['cell_type']; + source: CellByType[T]['source']; + metadata: CellByType[T]['metadata']; + }, + executionContext?: ExecutionContext +): string | null { + const cellUtil = cellUtils[data.type]; + return ( + cellUtil.createInterruptJupyterSource?.(data, executionContext) ?? null + ); +} + +export function cleanJupyterOutputs( + outputs: IOutput[], + cellType: T +): IOutput[] { + const cellUtil = cellUtils[cellType]; + return cellUtil.cleanJupyterOutputs + ? cellUtil.cleanJupyterOutputs(outputs) + : outputs; +} + +export function parseJupyterSource( + cellType: CellByType[T]['cell_type'], + cellId: string, + jupyterCell: JupyterCell +): Extract { + const cellUtil = cellUtils[cellType]; + return cellUtil.parseJupyterSource(cellId, jupyterCell); +} + +export function createNewCell( + cellType: T, + params: { + id: string; + existingCellVariableNames: Set; + source?: CellByType[T]['source']; + metadata?: CellByType[T]['metadata']; + } +): CellByType[T] & { cellId: string } { + const cellUtil = cellUtils[cellType]; + return cellUtil.createNewCell(params); +} + +export function getUserEditableContent( + cell: TCell +) { + return { + type: cell.cell_type, + source: cell.source, + metadata: isExecutableCell(cell) + ? createMetadataAndStateFromBlock({ + block_type: cell.cell_type, + metadata: cell.metadata, + execution_count: cell.execution_count, + outputs_reference: cell.outputs_reference + }).metadata + : cell.metadata, + block_group: cell.block_group + }; +} + +// these 2 variable assignments function as compile-time tests that the keys of the object above has 1-to-1 mapping with CellType +// exporting `test1` and `test2` only to prevent unused variables TS error +export const test1: CellType = null as unknown as keyof typeof cellUtils; +export const test2: keyof typeof cellUtils = null as unknown as CellType; + +export function getCellCountsByType(cells: { [id: string]: Cell }): { + [cellType in CellType]?: number; +} { + return _countBy(Object.values(cells), cell => cell.cell_type); +} + +export function canCellHaveDataTableOutput( + cell: Cell | undefined | null +): cell is DataTableCell { + return isCodeCell(cell) || isSqlCell(cell); +} + +export function canCellHaveHiddenDefinition(cell: Cell | undefined | null) { + return isCodeCell(cell) || isSqlCell(cell) || isNotebookFunctionCell(cell); +} + +export function canCellHaveFunctionExport(cell: Cell | undefined | null) { + return isCodeCell(cell) || isSqlCell(cell); +} + +export function isCellType(maybeCellType: string): maybeCellType is CellType { + return maybeCellType in cellUtils; +} + +export function isTextCellType(cellType: CellType): cellType is TextCellType { + return (TEXT_CELL_TYPES as ReadonlyArray).includes(cellType); +} + +export function isExecutableCellType( + cellType: CellType +): cellType is ExecutableCell['cell_type'] { + return isExecutableCell({ cell_type: cellType } as unknown as Cell); +} + +export function isExecutableCellWithContentDepsType( + cellType: CellType +): cellType is ExecutableCellWithContentDeps['cell_type'] { + return isExecutableCellWithContentDeps({ + cell_type: cellType + } as unknown as Cell); +} + +export function isExecutableCell( + cell: T | undefined | null + // @ts-expect-error Conditional types in type predicates fail but it works from the outside. +): cell is T extends DetachedCell ? DetachedExecutableCell : ExecutableCell { + return ( + isCodeCell(cell) || + isSqlCell(cell) || + isNotebookFunctionCell(cell) || + isInputCell(cell) || + isVisualizationCell(cell) || + isBigNumberCell(cell) || + isButtonCell(cell) + ); +} + +export function isExecutableCellWithContentDeps( + cell: T | undefined | null + // @ts-expect-error Conditional types in type predicates fail but it works from the outside. +): cell is T extends DetachedCell + ? DetachedExecutableCellWithContentDeps + : ExecutableCellWithContentDeps { + return isCodeCell(cell) || isSqlCell(cell); +} + +export function isFunctionExportableCellType(cellType: CellType) { + return cellType === 'code' || cellType === 'sql'; +} + +export function isEmbeddableCell( + cell: Cell | undefined | null +): cell is EmbeddableCell { + return ( + isCodeCell(cell) || + isSqlCell(cell) || + isVisualizationCell(cell) || + isNotebookFunctionCell(cell) + ); +} + +export function isCodeCell(cell: Cell | undefined | null): cell is CodeCell { + return cell?.cell_type === 'code'; +} + +export function isSqlCell(cell: Cell | undefined | null): cell is SqlCell { + return cell?.cell_type === 'sql'; +} + +export function isMarkdownCell( + cell: Cell | undefined | null +): cell is MarkdownCell { + return cell?.cell_type === 'markdown'; +} + +export function isNotebookFunctionCell( + cell: Cell | undefined | null +): cell is NotebookFunctionCell { + return cell?.cell_type === 'notebook-function'; +} + +export function isInputCellType(cellType: string): cellType is InputCellType { + return (INPUT_CELL_TYPES as ReadonlyArray).includes(cellType); +} + +export function isInputCell(cell: Cell | undefined | null): cell is InputCell { + return ( + isInputCheckboxCell(cell) || + isInputTextCell(cell) || + isInputTextareaCell(cell) || + isInputSelectCell(cell) || + isInputDateCell(cell) || + isInputSliderCell(cell) || + isInputFileCell(cell) || + isInputDateRangeCell(cell) + ); +} + +export function isButtonCell( + cell: Cell | undefined | null +): cell is ButtonCell { + return cell?.cell_type === 'button'; +} + +export function isSeparatorCell( + cell: Cell | undefined | null +): cell is SeparatorCell { + return cell?.cell_type === 'separator'; +} + +export function isTextCell(cell: Cell | undefined | null): cell is TextCell { + return !!cell?.cell_type && isTextCellType(cell.cell_type); +} + +export function isInputTextCell( + cell: Cell | undefined | null +): cell is InputTextCell { + return cell?.cell_type === 'input-text'; +} + +export function isInputCheckboxCell( + cell: Cell | undefined | null +): cell is InputCheckboxCell { + return cell?.cell_type === 'input-checkbox'; +} + +export function isInputTextareaCell( + cell: Cell | undefined | null +): cell is InputTextareaCell { + return cell?.cell_type === 'input-textarea'; +} + +export function isInputFileCell( + cell: Cell | undefined | null +): cell is InputFileCell { + return cell?.cell_type === 'input-file'; +} + +export function isInputSelectCell( + cell: Cell | undefined | null +): cell is InputSelectCell { + return cell?.cell_type === 'input-select'; +} + +export function isInputSliderCell( + cell: Cell | undefined | null +): cell is InputSliderCell { + return cell?.cell_type === 'input-slider'; +} + +export function isInputDateCell( + cell: Cell | undefined | null +): cell is InputDateCell { + return cell?.cell_type === 'input-date'; +} + +export function isInputDateRangeCell( + cell: Cell | undefined | null +): cell is InputDateRangeCell { + return cell?.cell_type === 'input-date-range'; +} + +export function isVisualizationCell( + cell: Cell | undefined | null +): cell is VisualizationCell { + return cell?.cell_type === 'visualization'; +} + +export function isImageCell(cell: Cell | undefined | null): cell is ImageCell { + return cell?.cell_type === 'image'; +} diff --git a/src/deepnote-convert/convert-cell-type-to-jupyter.ts b/src/deepnote-convert/convert-cell-type-to-jupyter.ts new file mode 100644 index 0000000..a022691 --- /dev/null +++ b/src/deepnote-convert/convert-cell-type-to-jupyter.ts @@ -0,0 +1,56 @@ +import { assertUnreachable } from './assert-unreachable'; +import { CellType, JupyterCellType } from './types'; + +// Source: +// deepnote-internal +// +// Path: +// libs/shared/src/utils/utils.ts +// +// Commit SHA: +// 97e072bee9089c3122bb3ea82ff478e890280014 + +// eslint-disable-next-line complexity +export function convertCellTypeToJupyter(cellType: CellType): JupyterCellType { + switch (cellType) { + case 'big-number': + return 'code'; + case 'code': + return 'code'; + case 'sql': + return 'code'; + case 'notebook-function': + return 'code'; + + case 'markdown': + return 'markdown'; + + case 'text-cell-h1': + case 'text-cell-h3': + case 'text-cell-h2': + case 'text-cell-p': + case 'text-cell-bullet': + case 'text-cell-todo': + case 'text-cell-callout': + case 'image': + case 'button': + case 'separator': + return 'markdown'; + case 'input-text': + case 'input-checkbox': + case 'input-textarea': + case 'input-file': + case 'input-select': + case 'input-date-range': + case 'input-date': + case 'input-slider': + return 'code'; + + case 'visualization': + return 'code'; + + default: + assertUnreachable(cellType); + throw new Error(`Invalid cell type ${cellType}`); + } +} diff --git a/src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts b/src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts new file mode 100644 index 0000000..044028b --- /dev/null +++ b/src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts @@ -0,0 +1,627 @@ +import { + convertCellTypeToJupyter, + createJupyterSource, + isExecutableCell, + isSqlCell +} from '@deepnote/shared'; +import _cloneDeep from 'lodash/cloneDeep'; + +import type { JupyterCell } from '@deepnote/jupyter'; +import type { IOutput } from '@jupyterlab/nbformat'; +import { SharedTableState } from './shared-table-state-schemas'; +import type { ValueOf } from 'ts-essentials'; +import { type DataframeFilter } from './shared-table-state-schemas'; +import { VegaLiteSpec } from './vega-lite.types'; + +// NOTE: All cells extend this base type. The type should not be used directly. +interface CellBase { + // NOTE: This grouping property is used to place multiple blocks (with the same group value) as columns + // in the same row. Ordering within the row is still determined by the sorting_key and there can + // be multiple non-consecutive groups of blocks with the same group value (although we try to avoid that). + block_group: string; +} + +interface ExecutableCellBase extends CellBase { + execution_count: number | null; + outputs: IOutput[]; + outputs_reference: string | null; +} + +interface ExecutableCodeCellBase extends ExecutableCellBase { + content_dependencies: BlockContentDependencies | null; +} + +export interface CellMetadata { + /** + * @deprecated Do not use, use cell.cellId instead. + */ + cell_id?: unknown; + /** Whether the code block is hidden in the app */ + deepnote_app_is_code_hidden?: boolean; + /** Whether the output of the code block is hidden in the app */ + deepnote_app_is_output_hidden?: boolean; + /** Whether the block is visible in the app */ + deepnote_app_block_visible?: boolean; + /** The width of the block in the app as a percentage value */ + deepnote_app_block_width?: number; + /** The group id of the block in the app. Items with the same group id are + * rendered in the same row. */ + deepnote_app_block_group_id?: string | null; + /** The subgroup id of the block in the app. Items with the same subgroup id + * are rendered in the same column within a row. */ + deepnote_app_block_subgroup_id?: string; + /** The order of the block in the app (which can differ from the order in the + * notebook) */ + deepnote_app_block_order?: number; + /** + * This metadata is used to display "Run the app" banner in the published app. + * Without it we don't know if the outputs are not there because the user cleared them + * or they were not there in the first place. + */ + deepnote_app_outputs_were_cleared?: boolean; +} + +export interface MarkdownCellMetadata extends CellMetadata { + deepnote_cell_height?: number; +} + +export interface SeparatorCellMetadata extends CellMetadata {} + +export type CellEmbedMode = 'code_output' | 'code' | 'output' | false; + +/* Dependencies (AST) fetched from Lambda for given block content */ +export interface BlockContentDependencies { + error?: { + // This is currently used for SyntaxError (which we want to show to the user instead of block dependencies) + message: string; + type: string; + }; + definedVariables: string[]; + usedVariables: string[]; + importedModules?: string[]; +} + +export interface NotebookExportState { + table_loading?: boolean; + table_state?: SharedTableState; + table_invalid?: boolean; +} + +export type NotebookExportStates = { + [exportName in string]: NotebookExportState; +}; + +/** + * Height of output, when it's null it means output is native (not rendered in iframe) + */ +export type CellOutputsHeights = (number | null)[]; + +export interface ExecutableCellMetadata extends CellMetadata { + allow_embed?: boolean | CellEmbedMode; + is_code_hidden?: boolean; + is_output_hidden?: boolean; + /** @deprecated The outputs are actually being cleared. This remains here only to be able to migrate historical versions of blocks. */ + output_cleared?: boolean; + execution_start?: number; // UTC timestamp in millis + execution_millis?: number; + source_hash?: string; + execution_context_id?: string; + deepnote_to_be_reexecuted?: boolean; // Whether the cell was marked for reexecution (being in executionQueue is not enough, it's possible that the cell was marked for reexecution but some other cell earlier in the execution queue errorred out and cancelled the queue) + deepnote_cell_height?: number; + deepnote_output_heights?: CellOutputsHeights; + deepnote_table_state?: SharedTableState; + deepnote_table_loading?: boolean; + deepnote_table_invalid?: boolean; + /** + * If enabled, the output will have no max-height and will grow to fit the content. + */ + deepnote_output_height_limit_disabled?: boolean; + last_executed_function_notebook_id?: string; + last_function_run_started_at?: number; + function_notebook_export_states?: NotebookExportStates; +} + +export interface CodeCellMetadata extends ExecutableCellMetadata { + function_export_name?: string; +} + +export type SqlCellVariableType = 'dataframe' | 'query_preview'; + +export interface SqlCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name?: string; + deepnote_return_variable_type?: SqlCellVariableType; + sql_integration_id?: string; + is_compiled_sql_query_visible?: boolean; + function_export_name?: string; +} + +export interface ButtonCellMetadata extends ExecutableCellMetadata { + deepnote_button_title?: string; + deepnote_button_color_scheme?: + | 'blue' + | 'red' + | 'neutral' + | 'green' + | 'yellow'; + deepnote_button_behavior?: 'run' | 'set_variable'; + // deepnote_variable_name is applicable only when deepnote_button_behavior is 'set_variable' + deepnote_variable_name?: string; +} + +// NOTE: We must allow all types of input values for all inputs since input block definitions can change over time +// and what used to be a multi-option select box can become a text input for instance, or numbers can become dates. +export type NotebookFunctionInputValue = + InputCell['metadata']['deepnote_variable_value']; +export interface NotebookFunctionInput { + custom_value?: NotebookFunctionInputValue | null; + variable_name?: string | null; +} +export type NotebookFunctionInputs = { + [inputName in string]: NotebookFunctionInput; +}; + +export interface NotebookExportMapping { + enabled: boolean; + variable_name: string | null; +} +export type NotebookExportMappings = { + [exportName in string]: NotebookExportMapping; +}; + +export interface NotebookFunctionCellMetadata extends ExecutableCellMetadata { + function_notebook_id: string | null; + function_notebook_inputs?: NotebookFunctionInputs; + function_notebook_export_mappings?: NotebookExportMappings; +} + +export interface InputCheckboxCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: boolean; + deepnote_variable_default_value?: boolean; + deepnote_input_checkbox_label?: string; + deepnote_input_label?: string; +} + +export interface InputTextCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: string; + deepnote_variable_default_value?: string; + deepnote_input_label?: string; +} + +export interface InputTextareaCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: string; + deepnote_variable_default_value?: string; + deepnote_input_label?: string; +} + +export interface InputFileCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: string; + deepnote_input_label?: string; + deepnote_allowed_file_extensions?: string; +} + +export const InputCellSelectTypes = { + FROM_OPTIONS: 'from-options', + FROM_VARIABLE: 'from-variable' +} as const; + +export type InputCellSelectType = ValueOf; + +export interface InputSelectCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: string | string[]; + deepnote_variable_default_value?: string | string[]; + deepnote_variable_options: string[]; + deepnote_variable_custom_options: string[]; + deepnote_variable_selected_variable: string; + deepnote_variable_select_type: InputCellSelectType; + deepnote_allow_multiple_values?: boolean; + deepnote_allow_empty_values?: boolean; + deepnote_input_label?: string; +} + +export interface InputSliderCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: string; + deepnote_variable_default_value?: string; + deepnote_slider_min_value: number; + deepnote_slider_max_value: number; + deepnote_slider_step: number; + deepnote_input_label?: string; +} + +export interface InputDateCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: string; + deepnote_variable_default_value?: string; + deepnote_allow_empty_values?: boolean; + /** + * Version 2 returns a Date object + * Version 1 or no version returns a DateTime object + * This exists to keep backward compatibility for notebooks created with version 1 Date picker. + */ + deepnote_input_date_version?: number; + deepnote_input_label?: string; +} + +export interface InputBlockValueOverrides { + [inputName: string]: InputCell['metadata']['deepnote_variable_value']; +} + +export type ImageBlockAlignmentType = 'left' | 'center' | 'right'; +export type ImageBlockWidthType = 'actual' | '50%' | '75%' | '100%'; + +export interface ImageCellMetadata extends CellMetadata { + deepnote_img_src?: string; + deepnote_img_width?: ImageBlockWidthType; + deepnote_img_alignment?: ImageBlockAlignmentType; +} + +export interface ValueSelector { + selectionType: 'values'; + field: string; + values: RangeUnits; +} + +/** + * String representing the ISO format of the time + */ +export type RangeUnits = number[] | string[]; +export type RangeUnit = RangeUnits[number]; + +export interface RangeSelector { + selectionType: 'range'; + field: string; + start: RangeUnit; + end: RangeUnit; +} + +export type ChartDataValueSelector = ValueSelector; + +export interface ChartDataIntervalSelector { + selectionType: 'data'; + axes: (RangeSelector | ValueSelector)[]; +} + +export type ChartDataSelector = + | ChartDataValueSelector + | ChartDataIntervalSelector; + +export type FilterClause = ChartDataSelector & { + filterType: 'include' | 'exclude'; + source: 'color_legend' | 'size_legend' | 'data'; +}; + +export interface FilterMetadata { + /** @deprecated Use advancedFilters instead */ + filter?: FilterClause[]; + advancedFilters?: DataframeFilter[]; +} + +export interface VisualizationCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name?: string; + deepnote_visualization_spec?: VegaLiteSpec; + deepnote_config_collapsed?: boolean; + deepnote_chart_height?: number; + deepnote_chart_filter?: FilterMetadata; +} + +export interface CodeCell extends ExecutableCodeCellBase { + cell_type: 'code'; + metadata: CodeCellMetadata; + source: string; +} + +export interface SqlCell extends ExecutableCodeCellBase { + cell_type: 'sql'; + metadata: SqlCellMetadata; + source: string; +} + +export interface NotebookFunctionCell extends ExecutableCodeCellBase { + cell_type: 'notebook-function'; + metadata: NotebookFunctionCellMetadata; + source: ''; +} + +export interface InputTextCell extends ExecutableCellBase { + cell_type: 'input-text'; + metadata: InputTextCellMetadata; + source: string; +} + +export interface InputCheckboxCell extends ExecutableCellBase { + cell_type: 'input-checkbox'; + metadata: InputCheckboxCellMetadata; + source: string; +} + +export interface InputTextareaCell extends ExecutableCellBase { + cell_type: 'input-textarea'; + metadata: InputTextareaCellMetadata; + source: string; +} + +export interface InputFileCell extends ExecutableCellBase { + cell_type: 'input-file'; + metadata: InputFileCellMetadata; + source: string; +} + +export interface ButtonCell extends ExecutableCellBase { + cell_type: 'button'; + source: ''; + metadata: ButtonCellMetadata; +} + +export interface SeparatorCell extends CellBase { + cell_type: 'separator'; + source: ''; + metadata: SeparatorCellMetadata; +} + +export interface FormatMarks { + bold?: boolean; + italic?: boolean; + underline?: boolean; + strike?: boolean; + code?: boolean; + /* the color accepts any valid CSS color value ('#000000', 'red', 'rgb(2,2,2)') */ + color?: string; +} + +export interface FormattedRangeMarks extends FormatMarks {} + +export type FormattedRange = FormattedRangeText | FormattedRangeLink; + +export interface FormattedRangeLink { + type: 'link'; + url: string; + fromCodePoint: number; + toCodePoint: number; + // ranges are local to the link + ranges: FormattedRangeText[]; +} + +export interface FormattedRangeText { + type?: 'marks'; + fromCodePoint: number; + toCodePoint: number; + marks: FormattedRangeMarks; +} + +export interface TextCellMetadata extends CellMetadata { + is_collapsed?: boolean; + formattedRanges?: FormattedRange[]; +} + +export interface TodoTextCellMetadata extends TextCellMetadata { + checked?: boolean; +} + +export type CalloutTextCellColor = + | 'blue' + | 'green' + | 'yellow' + | 'red' + | 'purple'; + +export interface CalloutTextCellMetadata extends TextCellMetadata { + color?: CalloutTextCellColor; +} + +export interface ParagraphTextCell extends CellBase { + cell_type: 'text-cell-p'; + metadata: TextCellMetadata; + source: string; +} + +export type HeadingTextCellType = + | 'text-cell-h1' + | 'text-cell-h2' + | 'text-cell-h3' + | 'text-cell-h4' + | 'text-cell-h5' + | 'text-cell-h6'; + +export interface Heading1TextCell extends CellBase { + cell_type: 'text-cell-h1'; + metadata: TextCellMetadata; + source: string; +} + +export interface Heading2TextCell extends CellBase { + cell_type: 'text-cell-h2'; + metadata: TextCellMetadata; + source: string; +} + +export interface Heading3TextCell extends CellBase { + cell_type: 'text-cell-h3'; + metadata: TextCellMetadata; + source: string; +} + +export type HeadingTextCell = + | Heading1TextCell + | Heading2TextCell + | Heading3TextCell; + +export interface BulletTextCell extends CellBase { + cell_type: 'text-cell-bullet'; + metadata: TextCellMetadata; + source: string; +} + +export interface TodoTextCell extends CellBase { + cell_type: 'text-cell-todo'; + metadata: TodoTextCellMetadata; + source: string; +} + +export interface CalloutTextCell extends CellBase { + cell_type: 'text-cell-callout'; + metadata: CalloutTextCellMetadata; + source: string; +} + +export type TextCell = + | ParagraphTextCell + | HeadingTextCell + | BulletTextCell + | TodoTextCell + | CalloutTextCell; + +export interface InputSelectCell extends ExecutableCellBase { + cell_type: 'input-select'; + metadata: InputSelectCellMetadata; + source: string; +} + +export interface InputSliderCell extends ExecutableCellBase { + cell_type: 'input-slider'; + metadata: InputSliderCellMetadata; + source: string; +} + +export interface InputDateCell extends ExecutableCellBase { + cell_type: 'input-date'; + metadata: InputDateCellMetadata; + source: string; +} + +export interface MarkdownCell extends CellBase { + cell_type: 'markdown'; + metadata: MarkdownCellMetadata; + source: string; +} + +export interface VisualizationCell extends ExecutableCellBase { + cell_type: 'visualization'; + source: ''; + metadata: VisualizationCellMetadata; +} + +export interface ImageCell extends CellBase { + cell_type: 'image'; + metadata: ImageCellMetadata; + source: ''; +} + +export interface BigNumberCell extends ExecutableCellBase { + cell_type: 'big-number'; + metadata: BigNumberCellMetadata; + source: string; +} + +export interface BigNumberCellMetadata extends ExecutableCellMetadata { + deepnote_big_number_title: string; + deepnote_big_number_value: string; + deepnote_big_number_format: string; + deepnote_big_number_comparison_enabled?: boolean; + deepnote_big_number_comparison_title?: string; + deepnote_big_number_comparison_value?: string; + deepnote_big_number_comparison_type?: string; + deepnote_big_number_comparison_format?: string; +} + +export type Cell = + | MarkdownCell + | CodeCell + | SqlCell + | NotebookFunctionCell + | InputCell + | TextCell + | VisualizationCell + | ImageCell + | ButtonCell + | SeparatorCell + | BigNumberCell; +export type InputCell = + | InputCheckboxCell + | InputTextCell + | InputSelectCell + | InputDateCell + | InputDateRangeCell + | InputSliderCell + | InputTextareaCell + | InputFileCell; + +export interface InputDateRangeCellMetadata extends ExecutableCellMetadata { + deepnote_variable_name: string; + deepnote_variable_value: [string, string] | string; + deepnote_variable_default_value?: [string, string] | string; + deepnote_input_label?: string; +} + +export interface InputDateRangeCell extends ExecutableCellBase { + cell_type: 'input-date-range'; + metadata: InputDateRangeCellMetadata; + source: string; +} + +// Source: +// deepnote-internal +// +// Path: +// libs/notebook-conversions/src/convert-deepnote-cell-to-jupyter-cell.ts +// +// Commit SHA: +// d943d4a9e9fb1a80f608072875f07a25fad3ce9e + +export function convertDeepnoteCellToJupyterCell( + cellId: string, + unclonedCell: Cell, + executionContext?: ExecutionContext +): JupyterCell { + const cell = _cloneDeep(unclonedCell); + + const jupyterCellMetadata: JupyterCell['metadata'] = { + ...cell.metadata, + cell_id: cellId + }; + + jupyterCellMetadata.deepnote_cell_type = cell.cell_type; + + if (isSqlCell(cell)) { + // let's store the raw SQL query in the metadata, since cell.source will be replaced by runnable python code + jupyterCellMetadata.deepnote_sql_source = cell.source; + } + + if (isExecutableCell(cell)) { + // Jupyter spec requires an explicit null here + cell.execution_count = + cell.execution_count === undefined ? null : cell.execution_count; + cell.outputs = cell.outputs === undefined ? [] : cell.outputs; + + if (cell.outputs) { + cell.outputs.forEach(output => { + delete output.truncated; + }); + } else { + console.warn( + '[convertDeepnoteToJupyter] Cell outputs not present in a code cell' + ); + } + } + + const source = createJupyterSource( + { + type: cell.cell_type, + source: cell.source, + metadata: cell.metadata + }, + executionContext + ); + + const jupyterCell: JupyterCell = { + ...cell, + cell_type: convertCellTypeToJupyter(cell.cell_type), + metadata: jupyterCellMetadata, + source + }; + return jupyterCell; +} diff --git a/src/deepnote-convert/deepnote-file-schema.ts b/src/deepnote-convert/deepnote-file-schema.ts index 46ed95a..aa3ee9a 100644 --- a/src/deepnote-convert/deepnote-file-schema.ts +++ b/src/deepnote-convert/deepnote-file-schema.ts @@ -1,4 +1,5 @@ import { z } from 'zod'; +import { CELL_TYPES } from './types'; // Below schema has been modified from the original deepnote-internal schema @@ -19,7 +20,7 @@ export const deepnoteFileBlockSchema = z.object({ metadata: z.record(z.string(), z.any()).optional(), outputs: z.array(z.any()).optional(), sortingKey: z.string(), - type: z.string(), + type: z.enum(CELL_TYPES), version: z.number().optional() }); diff --git a/src/deepnote-convert/shared-table-state-schemas.ts b/src/deepnote-convert/shared-table-state-schemas.ts new file mode 100644 index 0000000..992e329 --- /dev/null +++ b/src/deepnote-convert/shared-table-state-schemas.ts @@ -0,0 +1,197 @@ +import { z } from 'zod'; + +const columnIdSchema = z.string(); +export type ColumnId = z.infer; + +export const dataTableCellFormattingStyleNames = [ + 'default', + 'defaultBold', + 'positive', + 'positiveBold', + 'positiveProminentBold', + 'attention', + 'attentionBold', + 'attentionProminentBold', + 'critical', + 'criticalBold', + 'criticalProminentBold' +] as const; +export type DataTableCellFormattingStyleName = + (typeof dataTableCellFormattingStyleNames)[number]; + +const dataTableColumnSelectionModeSchema = z.enum(['all', 'allExcept', 'only']); +export type DataTableColumnSelectionMode = z.infer< + typeof dataTableColumnSelectionModeSchema +>; + +export const dataframeFilterOperators = [ + 'is-equal', + 'is-not-equal', + 'is-one-of', + 'is-not-one-of', + 'is-not-null', + 'is-null', + 'text-contains', + 'text-does-not-contain', + 'greater-than', + 'greater-than-or-equal', + 'less-than', + 'less-than-or-equal', + 'between', + 'outside-of', + 'is-relative-today', + 'is-after', + 'is-before', + 'is-on' +] as const; +const dataframeFilterOperatorSchema = z.enum(dataframeFilterOperators); +export type DataframeFilterOperator = z.infer< + typeof dataframeFilterOperatorSchema +>; + +const dataTableCellFormattingRuleSingleColorSchema = z.object({ + type: z.literal('singleColor'), + columnSelectionMode: dataTableColumnSelectionModeSchema, + columnNames: z.array(columnIdSchema).nullable(), + operator: dataframeFilterOperatorSchema, + comparativeValues: z.array(z.string()), + styleName: z.enum(dataTableCellFormattingStyleNames) +}); +export type DataTableCellFormattingRuleSingleColor = z.infer< + typeof dataTableCellFormattingRuleSingleColorSchema +>; + +export const dataTableCellFormattingScaleNames = [ + 'defaultToPositive', + 'defaultToAttention', + 'defaultToCritical', + 'positiveToDefault', + 'attentionToDefault', + 'criticalToDefault', + 'criticalToDefaultToPositive', + 'criticalToAttentionToPositive', + 'positiveToDefaultToCritical', + 'positiveToAttentionToCritical' +] as const; +export type DataTableCellFormattingColorScaleName = + (typeof dataTableCellFormattingScaleNames)[number]; + +const dataTableCellFormattingRuleColorScalePointSchema = z.object({ + valueType: z.enum(['autoAllColumns', 'number', 'percentage']), + value: z.number().nullable(), + customColor: z.string().nullable() +}); +export type DataTableCellFormattingRuleColorScalePoint = z.infer< + typeof dataTableCellFormattingRuleColorScalePointSchema +>; + +const dataTableCellFormattingRuleColorScaleSchema = z.object({ + type: z.literal('colorScale'), + columnSelectionMode: dataTableColumnSelectionModeSchema, + columnNames: z.array(columnIdSchema).nullable(), + pointMin: dataTableCellFormattingRuleColorScalePointSchema, + pointMid: dataTableCellFormattingRuleColorScalePointSchema, + pointMax: dataTableCellFormattingRuleColorScalePointSchema, + scaleName: z.enum(dataTableCellFormattingScaleNames) +}); +export type DataTableCellFormattingRuleColorScale = z.infer< + typeof dataTableCellFormattingRuleColorScaleSchema +>; + +const dataTableCellFormattingRuleColorScaleResolvedValueSchema = z.object({ + quantitative: z + .object({ + min: z.number(), + mid: z.number(), + max: z.number() + }) + .nullable(), + temporal: z + .object({ + min: z.date(), + mid: z.date(), + max: z.date() + }) + .nullable() +}); +export type DataTableCellFormattingRuleColorScaleResolvedValue = z.infer< + typeof dataTableCellFormattingRuleColorScaleResolvedValueSchema +>; + +const dataTableCellFormattingRuleColorScaleResolvedSchema = + dataTableCellFormattingRuleColorScaleSchema.extend({ + resolvedValue: dataTableCellFormattingRuleColorScaleResolvedValueSchema + }); +export type DataTableCellFormattingRuleColorScaleResolved = z.infer< + typeof dataTableCellFormattingRuleColorScaleResolvedSchema +>; + +const dataTableCellFormattingRuleSchema = z.union([ + dataTableCellFormattingRuleSingleColorSchema, + dataTableCellFormattingRuleColorScaleSchema +]); +export type DataTableCellFormattingRule = z.infer< + typeof dataTableCellFormattingRuleSchema +>; + +const dataTableCellFormattingRuleResolvedSchema = z.union([ + dataTableCellFormattingRuleSingleColorSchema, + dataTableCellFormattingRuleColorScaleResolvedSchema +]); +export type DataTableCellFormattingRuleResolved = z.infer< + typeof dataTableCellFormattingRuleResolvedSchema +>; + +const dataTableCellFormattingColorScaleDefinitionSchema = z.object({ + min: z.string(), + mid: z.string().optional(), + max: z.string() +}); +export type DataTableCellFormattingColorScaleDefinition = z.infer< + typeof dataTableCellFormattingColorScaleDefinitionSchema +>; + +export const dataframeFilterSchema = z.object({ + column: columnIdSchema, + operator: dataframeFilterOperatorSchema, + comparativeValues: z.array(z.string()) +}); +export type DataframeFilter = z.infer; + +const sortBySchema = z.object({ + id: columnIdSchema, + type: z.enum(['asc', 'desc']) +}); +export type SortBy = z.infer; + +const columnContainsFilterSchema = z.object({ + id: columnIdSchema, + value: z.string(), + type: z.literal('contains') +}); +export type ColumnContainsFilter = z.infer; + +const columnFilterSchema = columnContainsFilterSchema; +export type ColumnFilter = z.infer; + +const columnDisplayNameRecordSchema = z.object({ + columnName: columnIdSchema, + displayName: z.string() +}); +export type ColumnDisplayNameRecord = z.infer< + typeof columnDisplayNameRecordSchema +>; + +export const sharedTableStateSchema = z.object({ + pageSize: z.number().optional(), + pageIndex: z.number().optional(), + filters: z.array(columnFilterSchema).optional(), + conditionalFilters: z.array(dataframeFilterSchema).optional(), + sortBy: z.array(sortBySchema).optional(), + cellFormattingRules: z.array(dataTableCellFormattingRuleSchema).optional(), + wrappedTextColumnIds: z.array(columnIdSchema).optional(), + hiddenColumnIds: z.array(columnIdSchema).optional(), + columnOrder: z.array(columnIdSchema).optional(), + columnDisplayNames: z.array(columnDisplayNameRecordSchema).optional() +}); +export type SharedTableState = z.infer; diff --git a/src/deepnote-convert/types.ts b/src/deepnote-convert/types.ts new file mode 100644 index 0000000..e265a87 --- /dev/null +++ b/src/deepnote-convert/types.ts @@ -0,0 +1,59 @@ +// Source: +// deepnote-internal +// +// Path: +// libs/shared/src/types.ts + +// Commit SHA: +// 3a0bab71e1ee86530c74eb5ada2e1873848c5fea + +export const TEXT_CELL_TYPES = [ + 'text-cell-h1', + 'text-cell-h2', + 'text-cell-h3', + 'text-cell-p', + 'text-cell-bullet', + 'text-cell-todo', + 'text-cell-callout' +] as const; +export type TextCellType = (typeof TEXT_CELL_TYPES)[number]; + +export const INPUT_CELL_TYPES = [ + 'input-text', + 'input-textarea', + 'input-select', + 'input-date', + 'input-date-range', + 'input-slider', + 'input-file', + 'input-checkbox' +] as const; +export type InputCellType = (typeof INPUT_CELL_TYPES)[number]; +export const CELL_TYPES = [ + 'code', + 'sql', + 'markdown', + 'notebook-function', + ...INPUT_CELL_TYPES, + ...TEXT_CELL_TYPES, + 'visualization', + 'image', + 'button', + 'separator', + 'big-number' +] as const; +export type CellType = (typeof CELL_TYPES)[number]; + +// Source: +// deepnote-internal +// +// Path: +// libs/jupyter/src/jupyter.ts + +// Commit SHA: +// 0ac69479e192dbbb04ae10cd7b027872d09fbb14 + +/** + * A type which describes the type of cell. + */ +export type JupyterCellType = 'code' | 'markdown' | 'raw'; diff --git a/src/deepnote-convert/utils/object-utils.ts b/src/deepnote-convert/utils/object-utils.ts new file mode 100644 index 0000000..5c1192c --- /dev/null +++ b/src/deepnote-convert/utils/object-utils.ts @@ -0,0 +1,50 @@ +import _isEqual from 'lodash/isEqual'; + +export function pickBy( + obj: T, + predicate: (value: T[keyof T]) => boolean +) { + return Object.fromEntries( + Object.entries(obj).filter(([_, v]) => predicate(v as T[keyof T])) + ) as Partial; +} + +export function pickNonEmpty(obj: T) { + return pickBy(obj, v => v != null) as Partial<{ + [K in keyof T]: NonNullable; + }>; +} + +export function getObjectDiff( + objectA: E, + objectB: E +): Partial | null { + const diff: Partial = {}; + + // NOTE: We need to cover the situation where some keys are missing in one of the objects. + const keys = new Set([ + ...Object.keys(objectA), + ...Object.keys(objectB)  + ] as Array); + for (const key of keys) { + if (!_isEqual(objectA[key], objectB[key])) { + diff[key] = objectB[key]; + } + } + + return Object.keys(diff).length > 0 ? diff : null; +} + +export function nullKeysToUndefined(obj: T) { + return Object.fromEntries( + Object.entries(obj).map(([k, v]) => [k, v ?? undefined]) + ) as { + [K in keyof T]: NonNullable | undefined; + }; +} + +export function getSortedObjectEntries(obj: T) { + return Object.entries(obj).sort(([a], [b]) => a.localeCompare(b)) as Array< + [keyof T, T[keyof T]] + >; +} diff --git a/src/deepnote-convert/vega-lite.types.ts b/src/deepnote-convert/vega-lite.types.ts new file mode 100644 index 0000000..4636312 --- /dev/null +++ b/src/deepnote-convert/vega-lite.types.ts @@ -0,0 +1,414 @@ +import { z } from 'zod'; + +import type { TopLevelParameter } from 'vega-lite/build/src/spec/toplevel'; +import type { Transform } from 'vega-lite/build/src/transform'; + +export type { Spec as VegaSpec } from 'vega'; + +// Source: +// deepnote-internal +// +// Path: +// libs/shared/src/cells/vega-lite.types.ts +// +// Commit SHA: +// 7a06bd352cf59577b7cd100cb14747908400440e + +/** In v1 chart blocks, ordinal type was supported. In v2, it's not - so we're only keeping it here for migration purposes. + * Existing ordinal types encoding are converted to quantitative type. + */ + +export const legacyEncodingTypeSchema = z.enum([ + 'quantitative', + 'ordinal', + 'nominal', + 'temporal' +]); +export type LegacyEncodingType = z.infer; + +const markTypeSchema = z.enum([ + 'area', + 'bar', + 'trail', + 'line', + 'point', + 'circle', + 'text', + 'arc' +]); +export type MarkType = z.infer; + +export const encodingChannelSchema = z.enum([ + 'x', + 'y', + 'color', + 'opacity', + 'order', + 'size', + 'xOffset', + 'yOffset', + 'text', + 'theta', + 'tooltip', + 'detail', + 'stroke' +]); +export type EncodingChannel = z.infer; + +export const sortOrderSchema = z.enum(['ascending', 'descending']); +export type SortOrder = z.infer; + +export const scaleTypeSchema = z.enum(['linear', 'log', 'sqrt', 'time']); +export type ScaleType = z.infer; + +export const numberTypeSchema = z.enum([ + 'default', + 'number', + 'scientific', + 'percent' +]); +export type NumberType = z.infer; + +export const numberFormatSchema = z.object({ + type: numberTypeSchema.default('default'), + decimals: z.number().nullable().default(null) +}); +export type NumberFormat = z.infer; + +export const aggregateTypeSchema = z.enum([ + 'argmax', + 'argmin', + 'count', + 'average', + 'distinct', + 'max', + 'median', + 'min', + 'sum', + 'variance', + 'none' +]); +export type AggregateType = z.infer; + +export const timeUnitSchema = z.enum([ + 'yearmonthdatehours', + 'yearmonthdate', + 'yearmonth', + 'yearweek', + 'year', + 'monthdatehours', + 'monthdate', + 'month', + 'week', + 'datehours', + 'date', + 'day', + 'hours' +]); +export type TimeUnit = z.infer; + +export type FieldType = string; + +export type GridType = 'x' | 'y' | 'full' | 'disabled'; + +export interface Encoding { + field?: FieldType; + type?: LegacyEncodingType; + title?: string; + sort?: + | { + order: SortOrder; + } + | { + encoding: EncodingChannel; + order: SortOrder; + } + | { + order: SortOrder; + field?: string | null; + op?: AggregateType; + } + | SortOrder + | null; + aggregate?: AggregateType; + bin?: boolean | { step?: number; maxbins?: number }; + timeUnit?: TimeUnit | null; + axis?: { + grid?: boolean; + title?: string | null; + ticks?: boolean; + labels?: boolean; + formatType?: string; + format?: string | NumberFormat; + }; + scale?: { + scheme?: string; + type?: ScaleType; + zero?: boolean; + domainMin?: number; + domainMax?: number; + // Used only for color encoding to set custom label + color for legend + domain?: [string]; + range?: [string]; + }; + formatType?: string; + format?: string | NumberFormat; + datum?: number | string; + stack?: 'normalize' | 'zero' | true; + condition?: Record; + value?: number | string; + legend?: Record; + bandPosition?: number; +} + +export type VegaLiteMark = + | { + type: MarkType; + outerRadius?: { + expr: string; + }; + innerRadius?: { + expr: string; + }; + tooltip?: boolean; + } + | { + type: MarkType; + tooltip: boolean | { content: string }; + color?: string; + clip?: boolean; + } + | MarkType; + +// NOTE: tooltip is used in encoding to display and format tooltips in the Pie / Arc chart +export type EncodingsMap = { + [channel in EncodingChannel]?: channel extends 'tooltip' | 'detail' + ? Encoding[] + : Encoding; +}; + +export interface VegaLiteDataLayer { + mark: VegaLiteMark; + encoding: EncodingsMap; + params?: Array; + transform?: Array; +} + +/** + * This mark config is only used for the purposes of "value labels". See NB-353. + * That's why it's not a full mark config, but only a subset of properties that we need. + */ +interface VegaLiteTextMarkConfig { + type: 'text'; + align?: 'left' | 'right' | 'center'; + baseline?: 'top' | 'middle' | 'bottom'; + fill: 'black'; + dx?: number; + dy?: number; + radius?: { + expr: string; + }; +} + +/** + * This text layer is only used for the purposes of adding "value labels" to another layer. See NB-353 + * This layer will always be nested with a sibling data layer. See `VegaLiteParentLayer` + */ +export interface VegaLiteTextLayer { + mark: VegaLiteTextMarkConfig; + params?: Array; + + encoding: EncodingsMap & { + text: Encoding; + }; + transform?: Array; +} + +/** + * This point layer is only used for the purpose of adding a better tooltip to another, line layer. + * This layer will always be nested with a sibling data layer. See `VegaLiteParentLayer` + * ! Be careful with type-guarding this layer. It looks very similar to a VegaLiteDataLayer with a point mark, + * ! but with only a subset of properties that we need. + */ +export interface VegaLiteLineTooltipLayer { + mark: + | { type: 'point'; tooltip: true; size: number; opacity: 0 } + | { + type: 'text'; + radius: { + expr: string; + }; + }; + encoding: EncodingsMap; + params?: Array; +} + +export type VegaLiteHelperLayer = VegaLiteTextLayer | VegaLiteLineTooltipLayer; + +// This is a leaf layer that doesn't have any children layer +export type VegaLiteLeafLayer = VegaLiteDataLayer | VegaLiteHelperLayer; + +export interface VegaLiteParentLayer { + title?: string; + // an array with at least 1 element - first element is always a data layer, and the rest are 0+ helper layers + layer: [VegaLiteDataLayer, ...VegaLiteHelperLayer[]]; + encoding?: { [channel in EncodingChannel]?: Encoding }; + mark?: VegaLiteMark; +} + +export type VegaLiteMultiLayerSpecV1TopLevelLayer = + | VegaLiteDataLayer + | VegaLiteParentLayer; + +interface VegaLiteSpecShared { + $schema: + | 'https://vega.github.io/schema/vega-lite/v4.json' + | 'https://vega.github.io/schema/vega-lite/v5.json'; + title?: string; + config?: { + legend?: { + disable?: boolean; + orient?: 'left' | 'right' | 'top' | 'bottom'; + labelFont?: string; + labelFontSize?: number; + labelFontWeight?: string; + labelOverlap?: boolean; + labelColor?: string; + padding?: number; + rowPadding?: number; + symbolSize?: number; + symbolType?: string; + titleColor?: string; + titlePadding?: number; + titleFont?: string; + titleFontSize?: number; + titleFontWeight?: string; + }; + title?: { + anchor?: string; + color?: string; + font?: string; + fontSize?: number; + fontWeight?: string | number; + dy?: number; + offset?: number; + orient?: 'left' | 'right' | 'top' | 'bottom'; + }; + axis?: { + labelFont?: string; + labelFontSize?: number; + labelFontWeight?: string | number; + titleFont?: string; + titleFontSize?: number; + titleFontWeight?: string | number; + labelOverlap?: string; + labelColor?: string; + titleColor?: string; + titlePadding?: number; + domainCap?: 'butt' | 'round' | 'square'; + gridCap?: 'butt' | 'round' | 'square'; + gridWidth?: number; + gridColor?: string; + }; + axisX?: { + labelPadding?: number; + }; + axisY?: { + labelPadding?: number; + }; + axisBand?: { + tickCap?: 'butt' | 'round' | 'square'; + tickExtra?: boolean; + }; + axisQuantitative?: { + tickCount?: number; + }; + view?: { + stroke?: string; + }; + line?: { + strokeCap?: 'butt' | 'round' | 'square'; + strokeWidth?: number; + strokeJoin?: 'bevel'; + }; + bar?: { + cornerRadiusTopRight?: number; + cornerRadiusBottomLeft?: number; + cornerRadiusBottomRight?: number; + cornerRadiusTopLeft?: number; + }; + area?: { fill?: 'category'; line?: boolean; opacity?: number }; + circle?: { size?: number }; + mark?: { color?: string }; + range?: { + category?: readonly string[]; + ramp?: string[]; + }; + + customFormatTypes?: boolean; + }; + encoding: EncodingsMap; + usermeta?: { + tooltipDefaultMode?: boolean; + aditionalTypeInfo?: { + histogramLayerIndexes: number[]; + }; + seriesOrder?: number[]; + seriesNames?: string[]; + specSchemaVersion?: number; + }; + resolve?: { scale?: { [channel in 'x' | 'y']?: 'independent' | 'shared' } }; + transform?: Array; + params?: Array; + + autosize?: { + type: 'fit'; + }; + background?: string; + width?: 'container' | number; + height?: 'container' | number; + padding?: + | number + | { left: number; right: number; top: number; bottom: number }; + data?: { + name?: string; + values?: Array>; + }; +} + +export interface VegaLiteMultiLayerSpecV1 extends VegaLiteSpecShared { + mark?: undefined; + // In spec v1 we had arbitrary number of layers (roughly corresponds to series in current chart config), + // each of which consisted of one data layer and 0 or more helper layers (e.g. for value labels) + layer: Array; +} + +export interface VegaLiteAxisGroup { + resolve: { + scale: { + color: 'independent'; + }; + }; + layer: VegaLiteParentLayer[]; +} + +export interface VegaLiteMultiLayerSpecV2 extends VegaLiteSpecShared { + mark?: undefined; + // Second version of spec adds one more nesting level to `layer`. Now, there are always 1-2 + // root layers (one for each measure axis), which then consist of arbitrary number of series, + // each of which has one data layer and 0 or more helper layers (e.g. for value labels) + layer: [VegaLiteAxisGroup] | [VegaLiteAxisGroup, VegaLiteAxisGroup]; +} + +export interface VegaLiteTopLayerSpec extends VegaLiteSpecShared { + mark: VegaLiteMark; + layer?: undefined; +} + +/** + * This is a subset of vega-lite spec that we support the editing of in the chart block UI. + */ +export type VegaLiteSpec = + | VegaLiteTopLayerSpec + | VegaLiteMultiLayerSpecV1 + | VegaLiteMultiLayerSpecV2; diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 2240fe6..94d1039 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -1,35 +1,7 @@ import { deserializeDeepnoteFile } from './deepnote-convert/deserialize-deepnote-file'; import { IDeepnoteNotebookContent } from './types'; import { blankCodeCell, blankDeepnoteNotebookContent } from './fallback-data'; -import { DeepnoteFileBlock } from './deepnote-convert/deepnote-file-schema'; -import { ICodeCell, IMarkdownCell } from '@jupyterlab/nbformat'; - -function convertDeepnoteBlockToJupyterCell( - block: DeepnoteFileBlock -): ICodeCell | IMarkdownCell { - if (block.type === 'code') { - return { - cell_type: 'code', - source: block.content || '', - metadata: {}, - outputs: block.outputs || [], - execution_count: block.executionCount || null - }; - } else if (block.type === 'markdown') { - return { - cell_type: 'markdown', - source: block.content || '', - metadata: {} - }; - } else { - // For unsupported block types, return a markdown cell indicating it's unsupported - return { - cell_type: 'markdown', - source: `# Unsupported block type: ${block.type}\n`, - metadata: {} - }; - } -} +import { convertDeepnoteCellToJupyterCell } from './deepnote-convert/convert-deepnote-cell-to-jupyter-cell'; export async function transformDeepnoteYamlToNotebookContent( yamlString: string @@ -51,9 +23,7 @@ export async function transformDeepnoteYamlToNotebookContent( }; } - const cells = selectedNotebook.blocks.map( - convertDeepnoteBlockToJupyterCell - ); + const cells = selectedNotebook.blocks.map(convertDeepnoteCellToJupyterCell); return { ...blankDeepnoteNotebookContent, From d8c1a9ad281c9c74a9568d964f4ba535b3bbe9c3 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 11:29:49 +0200 Subject: [PATCH 15/37] Add DOM to TS config lib Signed-off-by: Andy Jakubowski --- tsconfig.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tsconfig.json b/tsconfig.json index ae922ef..81a7a75 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -6,7 +6,7 @@ "esModuleInterop": true, "incremental": true, "jsx": "react", - "lib": ["ES2022"], + "lib": ["ES2022", "DOM"], "module": "esnext", "moduleResolution": "node", "noEmitOnError": true, From a4fa6c726dbdfa65b5ebf1a611550b28aeaca3fc Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 11:31:33 +0200 Subject: [PATCH 16/37] Remove unused code Signed-off-by: Andy Jakubowski --- src/transform-deepnote-yaml-to-notebook-content.ts | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 94d1039..74dbffe 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -1,7 +1,6 @@ import { deserializeDeepnoteFile } from './deepnote-convert/deserialize-deepnote-file'; import { IDeepnoteNotebookContent } from './types'; import { blankCodeCell, blankDeepnoteNotebookContent } from './fallback-data'; -import { convertDeepnoteCellToJupyterCell } from './deepnote-convert/convert-deepnote-cell-to-jupyter-cell'; export async function transformDeepnoteYamlToNotebookContent( yamlString: string @@ -23,12 +22,7 @@ export async function transformDeepnoteYamlToNotebookContent( }; } - const cells = selectedNotebook.blocks.map(convertDeepnoteCellToJupyterCell); - - return { - ...blankDeepnoteNotebookContent, - cells - }; + return blankDeepnoteNotebookContent; } catch (error) { console.error('Failed to deserialize Deepnote file:', error); throw new Error('Failed to transform Deepnote YAML to notebook content.'); From 5120d9eed8924240f2cadd19ca45ce510e903262 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 14:50:47 +0200 Subject: [PATCH 17/37] Test @deepnote/blocks Signed-off-by: Andy Jakubowski --- package.json | 1 + src/deepnote-convert/deepnote-file-schema.ts | 3 +-- ...sform-deepnote-yaml-to-notebook-content.ts | 2 +- yarn.lock | 26 +++++++++++++++++++ 4 files changed, 29 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 1db3271..bf9aeef 100644 --- a/package.json +++ b/package.json @@ -58,6 +58,7 @@ "watch:labextension": "jupyter labextension watch ." }, "dependencies": { + "@deepnote/blocks": "file:/Users/work/repos/deepnote/deepnote/packages/blocks", "@jupyterlab/application": "^4.0.0", "@jupyterlab/coreutils": "^6.0.0", "@jupyterlab/notebook": "^4.4.7", diff --git a/src/deepnote-convert/deepnote-file-schema.ts b/src/deepnote-convert/deepnote-file-schema.ts index aa3ee9a..46ed95a 100644 --- a/src/deepnote-convert/deepnote-file-schema.ts +++ b/src/deepnote-convert/deepnote-file-schema.ts @@ -1,5 +1,4 @@ import { z } from 'zod'; -import { CELL_TYPES } from './types'; // Below schema has been modified from the original deepnote-internal schema @@ -20,7 +19,7 @@ export const deepnoteFileBlockSchema = z.object({ metadata: z.record(z.string(), z.any()).optional(), outputs: z.array(z.any()).optional(), sortingKey: z.string(), - type: z.enum(CELL_TYPES), + type: z.string(), version: z.number().optional() }); diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 74dbffe..2b17087 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -1,6 +1,6 @@ -import { deserializeDeepnoteFile } from './deepnote-convert/deserialize-deepnote-file'; import { IDeepnoteNotebookContent } from './types'; import { blankCodeCell, blankDeepnoteNotebookContent } from './fallback-data'; +import { deserializeDeepnoteFile } from '@deepnote/blocks'; export async function transformDeepnoteYamlToNotebookContent( yamlString: string diff --git a/yarn.lock b/yarn.lock index 13e9de5..6bd97d4 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1577,6 +1577,17 @@ __metadata: languageName: node linkType: hard +"@deepnote/blocks@file:/Users/work/repos/deepnote/deepnote/packages/blocks::locator=jupyterlab-deepnote%40workspace%3A.": + version: 1.0.0 + resolution: "@deepnote/blocks@file:/Users/work/repos/deepnote/deepnote/packages/blocks#/Users/work/repos/deepnote/deepnote/packages/blocks::hash=193053&locator=jupyterlab-deepnote%40workspace%3A." + dependencies: + ts-dedent: ^2.2.0 + yaml: ^2.8.1 + zod: ^4.1.12 + checksum: d47ee3fe23825c2f3012c50cd87216a4099d0abb07a46de67c50650943448baf624c3ec93e9a4fc6626995c9fae33d3fc3500e37e3044d99c9660887ea88e2de + languageName: node + linkType: hard + "@discoveryjs/json-ext@npm:^0.5.0": version: 0.5.7 resolution: "@discoveryjs/json-ext@npm:0.5.7" @@ -6794,6 +6805,7 @@ __metadata: version: 0.0.0-use.local resolution: "jupyterlab-deepnote@workspace:." dependencies: + "@deepnote/blocks": "file:/Users/work/repos/deepnote/deepnote/packages/blocks" "@jupyterlab/application": ^4.0.0 "@jupyterlab/builder": ^4.0.0 "@jupyterlab/coreutils": ^6.0.0 @@ -9018,6 +9030,13 @@ __metadata: languageName: node linkType: hard +"ts-dedent@npm:^2.2.0": + version: 2.2.0 + resolution: "ts-dedent@npm:2.2.0" + checksum: 93ed8f7878b6d5ed3c08d99b740010eede6bccfe64bce61c5a4da06a2c17d6ddbb80a8c49c2d15251de7594a4f93ffa21dd10e7be75ef66a4dc9951b4a94e2af + languageName: node + linkType: hard + "ts-jest@npm:^29.1.0": version: 29.4.1 resolution: "ts-jest@npm:29.4.1" @@ -9823,3 +9842,10 @@ __metadata: checksum: 022d59f85ebe054835fbcdc96a93c01479a64321104f846cb5644812c91e00d17ae3479f823956ec9b04e4351dd32841e1f12c567e81bc43f6e21ef5cc02ce3c languageName: node linkType: hard + +"zod@npm:^4.1.12": + version: 4.1.12 + resolution: "zod@npm:4.1.12" + checksum: 91174acc7d2ca5572ad522643474ddd60640cf6877b5d76e5d583eb25e3c4072c6f5eb92ab94f231ec5ce61c6acdfc3e0166de45fb1005b1ea54986b026b765f + languageName: node + linkType: hard From af2227b12708941c42218c557f06a6c891033c50 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 14:51:36 +0200 Subject: [PATCH 18/37] Delete `deepnote-convert` code Signed-off-by: Andy Jakubowski --- src/deepnote-convert/assert-unreachable.ts | 16 - .../blocks/block-metadata-utils.ts | 86 --- src/deepnote-convert/cells/cell-utils.ts | 457 ------------- .../convert-cell-type-to-jupyter.ts | 56 -- .../convert-deepnote-cell-to-jupyter-cell.ts | 627 ------------------ src/deepnote-convert/deepnote-file-schema.ts | 76 --- .../deserialize-deepnote-file.ts | 38 -- src/deepnote-convert/parse-yaml.ts | 22 - .../shared-table-state-schemas.ts | 197 ------ src/deepnote-convert/types.ts | 59 -- src/deepnote-convert/utils/object-utils.ts | 50 -- src/deepnote-convert/vega-lite.types.ts | 414 ------------ 12 files changed, 2098 deletions(-) delete mode 100644 src/deepnote-convert/assert-unreachable.ts delete mode 100644 src/deepnote-convert/blocks/block-metadata-utils.ts delete mode 100644 src/deepnote-convert/cells/cell-utils.ts delete mode 100644 src/deepnote-convert/convert-cell-type-to-jupyter.ts delete mode 100644 src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts delete mode 100644 src/deepnote-convert/deepnote-file-schema.ts delete mode 100644 src/deepnote-convert/deserialize-deepnote-file.ts delete mode 100644 src/deepnote-convert/parse-yaml.ts delete mode 100644 src/deepnote-convert/shared-table-state-schemas.ts delete mode 100644 src/deepnote-convert/types.ts delete mode 100644 src/deepnote-convert/utils/object-utils.ts delete mode 100644 src/deepnote-convert/vega-lite.types.ts diff --git a/src/deepnote-convert/assert-unreachable.ts b/src/deepnote-convert/assert-unreachable.ts deleted file mode 100644 index cbb30df..0000000 --- a/src/deepnote-convert/assert-unreachable.ts +++ /dev/null @@ -1,16 +0,0 @@ -// Source: -// deepnote-internal -// -// Path: -// libs/shared/src/utils/assert-unreachable.ts - -// Commit SHA: -// ebdadfe8f16b7fb2279c24b2362734909cab5d4d - -/** - * Utility function that uses the typechecker to check that some conditions never happen. - * Very useful e.g. for checking that we have covered all cases in a switch statement of in a long list of if's. - * This is purely for typechecking purposes and has no runtime implications (it's just an identity function at runtime). - * @example See Icon.tsx - */ -export const assertUnreachable = (x: never) => x; diff --git a/src/deepnote-convert/blocks/block-metadata-utils.ts b/src/deepnote-convert/blocks/block-metadata-utils.ts deleted file mode 100644 index 03296b1..0000000 --- a/src/deepnote-convert/blocks/block-metadata-utils.ts +++ /dev/null @@ -1,86 +0,0 @@ -import { pickBy } from '../utils' - -import type { SharedTableState } from '../data-table/shared-table-state-schemas' -import type { ExecutableCellMetadata } from '../types' - -export type BlockExecutionStateData = Pick< - ExecutableCellMetadata, - | 'output_cleared' - | 'execution_start' - | 'execution_millis' - | 'source_hash' - | 'deepnote_to_be_reexecuted' - | 'last_executed_function_notebook_id' - | 'last_function_run_started_at' - | 'function_notebook_export_states' -> & { - 'execution_count': number | null - 'outputs_reference': string | null - 'execution_context_id'?: string - 'deepnote_table_state'?: SharedTableState - 'deepnote_table_loading'?: boolean - 'deepnote_table_invalid'?: boolean -} - -export function createMetadataAndStateFromBlock>(block: { - block_type: T - metadata: M - execution_count: number | null - outputs_reference: string | null -}) { - const { metadata, state: stateFromMetadata } = splitBlockMetadataAndState(block.metadata) - - const state: BlockExecutionStateData = { - ...stateFromMetadata, - execution_count: block.execution_count, - outputs_reference: block.outputs_reference, - } - - return { - blockType: block.block_type, - metadata, - state, - } -} - -export function splitBlockMetadataAndState>(fullMetadata: M) { - const { - output_cleared, - execution_start, - execution_millis, - source_hash, - execution_context_id, - deepnote_to_be_reexecuted, - deepnote_table_state: tableStateMetadataKey, - deepnote_table_loading: tableLoadingMetadataKey, - deepnote_table_invalid: tableInvalidMetadataKey, - last_executed_function_notebook_id: lastExecutedFunctionNotebookId, - last_function_run_started_at: lastExecutedFunctionTimestamp, - function_notebook_export_states: functionNotebookExportStates, - cell_id: droppedCellId, - ...metadata - } = fullMetadata - - const state: Omit = pickBy( - { - output_cleared, - execution_start, - execution_millis, - source_hash, - execution_context_id, - deepnote_to_be_reexecuted, - deepnote_table_state: tableStateMetadataKey, - deepnote_table_loading: tableLoadingMetadataKey, - deepnote_table_invalid: tableInvalidMetadataKey, - last_executed_function_notebook_id: lastExecutedFunctionNotebookId, - last_function_run_started_at: lastExecutedFunctionTimestamp, - function_notebook_export_states: functionNotebookExportStates, - }, - value => typeof value !== 'undefined' - ) - - return { - metadata, - state, - } -} diff --git a/src/deepnote-convert/cells/cell-utils.ts b/src/deepnote-convert/cells/cell-utils.ts deleted file mode 100644 index f9b9d8d..0000000 --- a/src/deepnote-convert/cells/cell-utils.ts +++ /dev/null @@ -1,457 +0,0 @@ -import _countBy from 'lodash/countBy'; - -import { createMetadataAndStateFromBlock } from '../blocks/block-metadata-utils'; -import { INPUT_CELL_TYPES, isBigNumberCell, TEXT_CELL_TYPES } from '../types'; -import { bigNumberCellUtils } from './big-number-cell-utils'; -import { buttonCellUtils } from './button-cell-utils'; -import { codeCellUtils } from './code-cell-utils'; -import { imageCellUtils } from './image-cell-utils'; -import { inputCheckboxCellUtils } from './input-checkbox-cell-utils'; -import { inputDateCellUtils } from './input-date-cell-utils'; -import { inputDateRangeCellUtils } from './input-date-range-cell-utils'; -import { inputFileCellUtils } from './input-file-cell-utils'; -import { - inputSelectCellUtils, - validateSelectInputVariable -} from './input-select-cell-utils'; -import { inputSliderCellUtils } from './input-slider-cell-utils'; -import { inputTextCellUtils } from './input-text-cell-utils'; -import { inputTextareaCellUtils } from './input-textarea-cell-utils'; -import { markdownCellUtils } from './markdown-cell-utils'; -import { notebookFunctionCellUtils } from './notebook-function-cell-utils'; -import { separatorCellUtils } from './separator-cell-utils'; -import { sqlCellUtils } from './sql-cell-utils'; -import { - textCellBulletUtils, - textCellCalloutUtils, - textCellH1Utils, - textCellH2Utils, - textCellH3Utils, - textCellParagraphUtils, - textCellTodoUtils -} from './text-cell-utils'; -import { visualizationCellUtils } from './visualization-cell-utils'; - -import type { JupyterCell } from '@deepnote/jupyter'; -import type { IOutput } from '@jupyterlab/nbformat'; -import type { - ButtonCell, - Cell, - CellByType, - CellType, - CodeCell, - DataTableCell, - DetachedCell, - DetachedExecutableCell, - DetachedExecutableCellWithContentDeps, - EmbeddableCell, - ExecutableCell, - ExecutableCellWithContentDeps, - ImageCell, - InputCell, - InputCellType, - InputCheckboxCell, - InputDateCell, - InputDateRangeCell, - InputFileCell, - InputSelectCell, - InputSliderCell, - InputTextareaCell, - InputTextCell, - MarkdownCell, - NotebookFunctionCell, - SeparatorCell, - SqlCell, - TextCell, - TextCellType, - VisualizationCell -} from '../types'; - -export { validateSelectInputVariable }; - -// see https://github.com/deepnote/compute-helpers/blob/af569f06396a36bbc7e221445dfb23093ab8db39/code/sql_utils.py#L15 -export type SqlCacheMode = 'cache_disabled' | 'always_write' | 'read_or_write'; - -export type SubmittedBy = 'user' | 'scheduling' | 'api' | 'ai'; - -// More specific type for workload context to track why workloads are running -export type WorkloadType = 'notebook' | 'app' | 'api' | 'scheduled-run'; - -export type VariableContext = string[]; - -export interface ExecutionContext { - user?: { id: string; email: string | null }; - submittedBy?: SubmittedBy; - // TODO: It would probably be nicer to use `SqlAlchemyInput` type here. - // To achieve that, we would need to move `SqlAlchemyInput` to `libs/shared` package. - federatedIntegrationConnectionString?: string | null; - parentNotebookFunctionRunId?: string | null; - notebookFunctionApiToken?: string | null; - /** - * If set, button blocks with this variable name that are being executed - * will resolve their source code to set the variable to True, False otherwise. - */ - variableContext?: VariableContext; - workspaceId: string; - projectId: string; - notebookId: string; - sqlCacheMode: SqlCacheMode; - tenantDomain?: string; -} - -export interface DeepnoteCellUtils { - parseJupyterSource: ( - cellId: string, - jupyterCell: JupyterCell - ) => Extract; - createJupyterSource: ( - data: { - type: TCell['cell_type']; - source: TCell['source']; - metadata: TCell['metadata']; - }, - executionContext?: ExecutionContext - ) => string; - // NOTE: This wrapper is not included in source hash calculations. - wrapJupyterSource?: ( - code: string, - data: { - type: TCell['cell_type']; - source: TCell['source']; - metadata: TCell['metadata']; - }, - executionContext?: ExecutionContext - ) => string; - createInterruptJupyterSource?: ( - data: { - type: TCell['cell_type']; - source: TCell['source']; - metadata: TCell['metadata']; - }, - executionContext?: ExecutionContext - ) => string | null; - cleanJupyterOutputs?: (outputs: IOutput[]) => IOutput[]; - createNewCell(params: { - id: string; - existingCellVariableNames: Set; - source?: string; - metadata?: TCell['metadata']; - }): TCell & { cellId: string }; -} - -export const cellUtils: { - [cellType in CellType]: DeepnoteCellUtils; -} = { - 'big-number': bigNumberCellUtils, - sql: sqlCellUtils, - code: codeCellUtils, - markdown: markdownCellUtils, - 'notebook-function': notebookFunctionCellUtils, - 'input-text': inputTextCellUtils, - 'input-textarea': inputTextareaCellUtils, - 'input-file': inputFileCellUtils, - 'input-select': inputSelectCellUtils, - 'input-date': inputDateCellUtils, - 'input-date-range': inputDateRangeCellUtils, - 'input-slider': inputSliderCellUtils, - 'input-checkbox': inputCheckboxCellUtils, - 'text-cell-h1': textCellH1Utils, - 'text-cell-h2': textCellH2Utils, - 'text-cell-h3': textCellH3Utils, - 'text-cell-p': textCellParagraphUtils, - 'text-cell-bullet': textCellBulletUtils, - 'text-cell-todo': textCellTodoUtils, - 'text-cell-callout': textCellCalloutUtils, - visualization: visualizationCellUtils, - image: imageCellUtils, - button: buttonCellUtils, - separator: separatorCellUtils -} as const; - -export function createJupyterSource( - data: { - type: CellByType[T]['cell_type']; - source: CellByType[T]['source']; - metadata: CellByType[T]['metadata']; - }, - executionContext?: ExecutionContext -): string { - const cellUtil = cellUtils[data.type]; - return cellUtil.createJupyterSource(data, executionContext); -} - -export function wrapJupyterSource( - code: string, - data: { - type: CellByType[T]['cell_type']; - source: CellByType[T]['source']; - metadata: CellByType[T]['metadata']; - }, - executionContext?: ExecutionContext -): string { - const cellUtil = cellUtils[data.type]; - return cellUtil.wrapJupyterSource - ? cellUtil.wrapJupyterSource(code, data, executionContext) - : code; -} - -export function createInterruptJupyterSource( - data: { - type: CellByType[T]['cell_type']; - source: CellByType[T]['source']; - metadata: CellByType[T]['metadata']; - }, - executionContext?: ExecutionContext -): string | null { - const cellUtil = cellUtils[data.type]; - return ( - cellUtil.createInterruptJupyterSource?.(data, executionContext) ?? null - ); -} - -export function cleanJupyterOutputs( - outputs: IOutput[], - cellType: T -): IOutput[] { - const cellUtil = cellUtils[cellType]; - return cellUtil.cleanJupyterOutputs - ? cellUtil.cleanJupyterOutputs(outputs) - : outputs; -} - -export function parseJupyterSource( - cellType: CellByType[T]['cell_type'], - cellId: string, - jupyterCell: JupyterCell -): Extract { - const cellUtil = cellUtils[cellType]; - return cellUtil.parseJupyterSource(cellId, jupyterCell); -} - -export function createNewCell( - cellType: T, - params: { - id: string; - existingCellVariableNames: Set; - source?: CellByType[T]['source']; - metadata?: CellByType[T]['metadata']; - } -): CellByType[T] & { cellId: string } { - const cellUtil = cellUtils[cellType]; - return cellUtil.createNewCell(params); -} - -export function getUserEditableContent( - cell: TCell -) { - return { - type: cell.cell_type, - source: cell.source, - metadata: isExecutableCell(cell) - ? createMetadataAndStateFromBlock({ - block_type: cell.cell_type, - metadata: cell.metadata, - execution_count: cell.execution_count, - outputs_reference: cell.outputs_reference - }).metadata - : cell.metadata, - block_group: cell.block_group - }; -} - -// these 2 variable assignments function as compile-time tests that the keys of the object above has 1-to-1 mapping with CellType -// exporting `test1` and `test2` only to prevent unused variables TS error -export const test1: CellType = null as unknown as keyof typeof cellUtils; -export const test2: keyof typeof cellUtils = null as unknown as CellType; - -export function getCellCountsByType(cells: { [id: string]: Cell }): { - [cellType in CellType]?: number; -} { - return _countBy(Object.values(cells), cell => cell.cell_type); -} - -export function canCellHaveDataTableOutput( - cell: Cell | undefined | null -): cell is DataTableCell { - return isCodeCell(cell) || isSqlCell(cell); -} - -export function canCellHaveHiddenDefinition(cell: Cell | undefined | null) { - return isCodeCell(cell) || isSqlCell(cell) || isNotebookFunctionCell(cell); -} - -export function canCellHaveFunctionExport(cell: Cell | undefined | null) { - return isCodeCell(cell) || isSqlCell(cell); -} - -export function isCellType(maybeCellType: string): maybeCellType is CellType { - return maybeCellType in cellUtils; -} - -export function isTextCellType(cellType: CellType): cellType is TextCellType { - return (TEXT_CELL_TYPES as ReadonlyArray).includes(cellType); -} - -export function isExecutableCellType( - cellType: CellType -): cellType is ExecutableCell['cell_type'] { - return isExecutableCell({ cell_type: cellType } as unknown as Cell); -} - -export function isExecutableCellWithContentDepsType( - cellType: CellType -): cellType is ExecutableCellWithContentDeps['cell_type'] { - return isExecutableCellWithContentDeps({ - cell_type: cellType - } as unknown as Cell); -} - -export function isExecutableCell( - cell: T | undefined | null - // @ts-expect-error Conditional types in type predicates fail but it works from the outside. -): cell is T extends DetachedCell ? DetachedExecutableCell : ExecutableCell { - return ( - isCodeCell(cell) || - isSqlCell(cell) || - isNotebookFunctionCell(cell) || - isInputCell(cell) || - isVisualizationCell(cell) || - isBigNumberCell(cell) || - isButtonCell(cell) - ); -} - -export function isExecutableCellWithContentDeps( - cell: T | undefined | null - // @ts-expect-error Conditional types in type predicates fail but it works from the outside. -): cell is T extends DetachedCell - ? DetachedExecutableCellWithContentDeps - : ExecutableCellWithContentDeps { - return isCodeCell(cell) || isSqlCell(cell); -} - -export function isFunctionExportableCellType(cellType: CellType) { - return cellType === 'code' || cellType === 'sql'; -} - -export function isEmbeddableCell( - cell: Cell | undefined | null -): cell is EmbeddableCell { - return ( - isCodeCell(cell) || - isSqlCell(cell) || - isVisualizationCell(cell) || - isNotebookFunctionCell(cell) - ); -} - -export function isCodeCell(cell: Cell | undefined | null): cell is CodeCell { - return cell?.cell_type === 'code'; -} - -export function isSqlCell(cell: Cell | undefined | null): cell is SqlCell { - return cell?.cell_type === 'sql'; -} - -export function isMarkdownCell( - cell: Cell | undefined | null -): cell is MarkdownCell { - return cell?.cell_type === 'markdown'; -} - -export function isNotebookFunctionCell( - cell: Cell | undefined | null -): cell is NotebookFunctionCell { - return cell?.cell_type === 'notebook-function'; -} - -export function isInputCellType(cellType: string): cellType is InputCellType { - return (INPUT_CELL_TYPES as ReadonlyArray).includes(cellType); -} - -export function isInputCell(cell: Cell | undefined | null): cell is InputCell { - return ( - isInputCheckboxCell(cell) || - isInputTextCell(cell) || - isInputTextareaCell(cell) || - isInputSelectCell(cell) || - isInputDateCell(cell) || - isInputSliderCell(cell) || - isInputFileCell(cell) || - isInputDateRangeCell(cell) - ); -} - -export function isButtonCell( - cell: Cell | undefined | null -): cell is ButtonCell { - return cell?.cell_type === 'button'; -} - -export function isSeparatorCell( - cell: Cell | undefined | null -): cell is SeparatorCell { - return cell?.cell_type === 'separator'; -} - -export function isTextCell(cell: Cell | undefined | null): cell is TextCell { - return !!cell?.cell_type && isTextCellType(cell.cell_type); -} - -export function isInputTextCell( - cell: Cell | undefined | null -): cell is InputTextCell { - return cell?.cell_type === 'input-text'; -} - -export function isInputCheckboxCell( - cell: Cell | undefined | null -): cell is InputCheckboxCell { - return cell?.cell_type === 'input-checkbox'; -} - -export function isInputTextareaCell( - cell: Cell | undefined | null -): cell is InputTextareaCell { - return cell?.cell_type === 'input-textarea'; -} - -export function isInputFileCell( - cell: Cell | undefined | null -): cell is InputFileCell { - return cell?.cell_type === 'input-file'; -} - -export function isInputSelectCell( - cell: Cell | undefined | null -): cell is InputSelectCell { - return cell?.cell_type === 'input-select'; -} - -export function isInputSliderCell( - cell: Cell | undefined | null -): cell is InputSliderCell { - return cell?.cell_type === 'input-slider'; -} - -export function isInputDateCell( - cell: Cell | undefined | null -): cell is InputDateCell { - return cell?.cell_type === 'input-date'; -} - -export function isInputDateRangeCell( - cell: Cell | undefined | null -): cell is InputDateRangeCell { - return cell?.cell_type === 'input-date-range'; -} - -export function isVisualizationCell( - cell: Cell | undefined | null -): cell is VisualizationCell { - return cell?.cell_type === 'visualization'; -} - -export function isImageCell(cell: Cell | undefined | null): cell is ImageCell { - return cell?.cell_type === 'image'; -} diff --git a/src/deepnote-convert/convert-cell-type-to-jupyter.ts b/src/deepnote-convert/convert-cell-type-to-jupyter.ts deleted file mode 100644 index a022691..0000000 --- a/src/deepnote-convert/convert-cell-type-to-jupyter.ts +++ /dev/null @@ -1,56 +0,0 @@ -import { assertUnreachable } from './assert-unreachable'; -import { CellType, JupyterCellType } from './types'; - -// Source: -// deepnote-internal -// -// Path: -// libs/shared/src/utils/utils.ts -// -// Commit SHA: -// 97e072bee9089c3122bb3ea82ff478e890280014 - -// eslint-disable-next-line complexity -export function convertCellTypeToJupyter(cellType: CellType): JupyterCellType { - switch (cellType) { - case 'big-number': - return 'code'; - case 'code': - return 'code'; - case 'sql': - return 'code'; - case 'notebook-function': - return 'code'; - - case 'markdown': - return 'markdown'; - - case 'text-cell-h1': - case 'text-cell-h3': - case 'text-cell-h2': - case 'text-cell-p': - case 'text-cell-bullet': - case 'text-cell-todo': - case 'text-cell-callout': - case 'image': - case 'button': - case 'separator': - return 'markdown'; - case 'input-text': - case 'input-checkbox': - case 'input-textarea': - case 'input-file': - case 'input-select': - case 'input-date-range': - case 'input-date': - case 'input-slider': - return 'code'; - - case 'visualization': - return 'code'; - - default: - assertUnreachable(cellType); - throw new Error(`Invalid cell type ${cellType}`); - } -} diff --git a/src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts b/src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts deleted file mode 100644 index 044028b..0000000 --- a/src/deepnote-convert/convert-deepnote-cell-to-jupyter-cell.ts +++ /dev/null @@ -1,627 +0,0 @@ -import { - convertCellTypeToJupyter, - createJupyterSource, - isExecutableCell, - isSqlCell -} from '@deepnote/shared'; -import _cloneDeep from 'lodash/cloneDeep'; - -import type { JupyterCell } from '@deepnote/jupyter'; -import type { IOutput } from '@jupyterlab/nbformat'; -import { SharedTableState } from './shared-table-state-schemas'; -import type { ValueOf } from 'ts-essentials'; -import { type DataframeFilter } from './shared-table-state-schemas'; -import { VegaLiteSpec } from './vega-lite.types'; - -// NOTE: All cells extend this base type. The type should not be used directly. -interface CellBase { - // NOTE: This grouping property is used to place multiple blocks (with the same group value) as columns - // in the same row. Ordering within the row is still determined by the sorting_key and there can - // be multiple non-consecutive groups of blocks with the same group value (although we try to avoid that). - block_group: string; -} - -interface ExecutableCellBase extends CellBase { - execution_count: number | null; - outputs: IOutput[]; - outputs_reference: string | null; -} - -interface ExecutableCodeCellBase extends ExecutableCellBase { - content_dependencies: BlockContentDependencies | null; -} - -export interface CellMetadata { - /** - * @deprecated Do not use, use cell.cellId instead. - */ - cell_id?: unknown; - /** Whether the code block is hidden in the app */ - deepnote_app_is_code_hidden?: boolean; - /** Whether the output of the code block is hidden in the app */ - deepnote_app_is_output_hidden?: boolean; - /** Whether the block is visible in the app */ - deepnote_app_block_visible?: boolean; - /** The width of the block in the app as a percentage value */ - deepnote_app_block_width?: number; - /** The group id of the block in the app. Items with the same group id are - * rendered in the same row. */ - deepnote_app_block_group_id?: string | null; - /** The subgroup id of the block in the app. Items with the same subgroup id - * are rendered in the same column within a row. */ - deepnote_app_block_subgroup_id?: string; - /** The order of the block in the app (which can differ from the order in the - * notebook) */ - deepnote_app_block_order?: number; - /** - * This metadata is used to display "Run the app" banner in the published app. - * Without it we don't know if the outputs are not there because the user cleared them - * or they were not there in the first place. - */ - deepnote_app_outputs_were_cleared?: boolean; -} - -export interface MarkdownCellMetadata extends CellMetadata { - deepnote_cell_height?: number; -} - -export interface SeparatorCellMetadata extends CellMetadata {} - -export type CellEmbedMode = 'code_output' | 'code' | 'output' | false; - -/* Dependencies (AST) fetched from Lambda for given block content */ -export interface BlockContentDependencies { - error?: { - // This is currently used for SyntaxError (which we want to show to the user instead of block dependencies) - message: string; - type: string; - }; - definedVariables: string[]; - usedVariables: string[]; - importedModules?: string[]; -} - -export interface NotebookExportState { - table_loading?: boolean; - table_state?: SharedTableState; - table_invalid?: boolean; -} - -export type NotebookExportStates = { - [exportName in string]: NotebookExportState; -}; - -/** - * Height of output, when it's null it means output is native (not rendered in iframe) - */ -export type CellOutputsHeights = (number | null)[]; - -export interface ExecutableCellMetadata extends CellMetadata { - allow_embed?: boolean | CellEmbedMode; - is_code_hidden?: boolean; - is_output_hidden?: boolean; - /** @deprecated The outputs are actually being cleared. This remains here only to be able to migrate historical versions of blocks. */ - output_cleared?: boolean; - execution_start?: number; // UTC timestamp in millis - execution_millis?: number; - source_hash?: string; - execution_context_id?: string; - deepnote_to_be_reexecuted?: boolean; // Whether the cell was marked for reexecution (being in executionQueue is not enough, it's possible that the cell was marked for reexecution but some other cell earlier in the execution queue errorred out and cancelled the queue) - deepnote_cell_height?: number; - deepnote_output_heights?: CellOutputsHeights; - deepnote_table_state?: SharedTableState; - deepnote_table_loading?: boolean; - deepnote_table_invalid?: boolean; - /** - * If enabled, the output will have no max-height and will grow to fit the content. - */ - deepnote_output_height_limit_disabled?: boolean; - last_executed_function_notebook_id?: string; - last_function_run_started_at?: number; - function_notebook_export_states?: NotebookExportStates; -} - -export interface CodeCellMetadata extends ExecutableCellMetadata { - function_export_name?: string; -} - -export type SqlCellVariableType = 'dataframe' | 'query_preview'; - -export interface SqlCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name?: string; - deepnote_return_variable_type?: SqlCellVariableType; - sql_integration_id?: string; - is_compiled_sql_query_visible?: boolean; - function_export_name?: string; -} - -export interface ButtonCellMetadata extends ExecutableCellMetadata { - deepnote_button_title?: string; - deepnote_button_color_scheme?: - | 'blue' - | 'red' - | 'neutral' - | 'green' - | 'yellow'; - deepnote_button_behavior?: 'run' | 'set_variable'; - // deepnote_variable_name is applicable only when deepnote_button_behavior is 'set_variable' - deepnote_variable_name?: string; -} - -// NOTE: We must allow all types of input values for all inputs since input block definitions can change over time -// and what used to be a multi-option select box can become a text input for instance, or numbers can become dates. -export type NotebookFunctionInputValue = - InputCell['metadata']['deepnote_variable_value']; -export interface NotebookFunctionInput { - custom_value?: NotebookFunctionInputValue | null; - variable_name?: string | null; -} -export type NotebookFunctionInputs = { - [inputName in string]: NotebookFunctionInput; -}; - -export interface NotebookExportMapping { - enabled: boolean; - variable_name: string | null; -} -export type NotebookExportMappings = { - [exportName in string]: NotebookExportMapping; -}; - -export interface NotebookFunctionCellMetadata extends ExecutableCellMetadata { - function_notebook_id: string | null; - function_notebook_inputs?: NotebookFunctionInputs; - function_notebook_export_mappings?: NotebookExportMappings; -} - -export interface InputCheckboxCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: boolean; - deepnote_variable_default_value?: boolean; - deepnote_input_checkbox_label?: string; - deepnote_input_label?: string; -} - -export interface InputTextCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: string; - deepnote_variable_default_value?: string; - deepnote_input_label?: string; -} - -export interface InputTextareaCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: string; - deepnote_variable_default_value?: string; - deepnote_input_label?: string; -} - -export interface InputFileCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: string; - deepnote_input_label?: string; - deepnote_allowed_file_extensions?: string; -} - -export const InputCellSelectTypes = { - FROM_OPTIONS: 'from-options', - FROM_VARIABLE: 'from-variable' -} as const; - -export type InputCellSelectType = ValueOf; - -export interface InputSelectCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: string | string[]; - deepnote_variable_default_value?: string | string[]; - deepnote_variable_options: string[]; - deepnote_variable_custom_options: string[]; - deepnote_variable_selected_variable: string; - deepnote_variable_select_type: InputCellSelectType; - deepnote_allow_multiple_values?: boolean; - deepnote_allow_empty_values?: boolean; - deepnote_input_label?: string; -} - -export interface InputSliderCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: string; - deepnote_variable_default_value?: string; - deepnote_slider_min_value: number; - deepnote_slider_max_value: number; - deepnote_slider_step: number; - deepnote_input_label?: string; -} - -export interface InputDateCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: string; - deepnote_variable_default_value?: string; - deepnote_allow_empty_values?: boolean; - /** - * Version 2 returns a Date object - * Version 1 or no version returns a DateTime object - * This exists to keep backward compatibility for notebooks created with version 1 Date picker. - */ - deepnote_input_date_version?: number; - deepnote_input_label?: string; -} - -export interface InputBlockValueOverrides { - [inputName: string]: InputCell['metadata']['deepnote_variable_value']; -} - -export type ImageBlockAlignmentType = 'left' | 'center' | 'right'; -export type ImageBlockWidthType = 'actual' | '50%' | '75%' | '100%'; - -export interface ImageCellMetadata extends CellMetadata { - deepnote_img_src?: string; - deepnote_img_width?: ImageBlockWidthType; - deepnote_img_alignment?: ImageBlockAlignmentType; -} - -export interface ValueSelector { - selectionType: 'values'; - field: string; - values: RangeUnits; -} - -/** - * String representing the ISO format of the time - */ -export type RangeUnits = number[] | string[]; -export type RangeUnit = RangeUnits[number]; - -export interface RangeSelector { - selectionType: 'range'; - field: string; - start: RangeUnit; - end: RangeUnit; -} - -export type ChartDataValueSelector = ValueSelector; - -export interface ChartDataIntervalSelector { - selectionType: 'data'; - axes: (RangeSelector | ValueSelector)[]; -} - -export type ChartDataSelector = - | ChartDataValueSelector - | ChartDataIntervalSelector; - -export type FilterClause = ChartDataSelector & { - filterType: 'include' | 'exclude'; - source: 'color_legend' | 'size_legend' | 'data'; -}; - -export interface FilterMetadata { - /** @deprecated Use advancedFilters instead */ - filter?: FilterClause[]; - advancedFilters?: DataframeFilter[]; -} - -export interface VisualizationCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name?: string; - deepnote_visualization_spec?: VegaLiteSpec; - deepnote_config_collapsed?: boolean; - deepnote_chart_height?: number; - deepnote_chart_filter?: FilterMetadata; -} - -export interface CodeCell extends ExecutableCodeCellBase { - cell_type: 'code'; - metadata: CodeCellMetadata; - source: string; -} - -export interface SqlCell extends ExecutableCodeCellBase { - cell_type: 'sql'; - metadata: SqlCellMetadata; - source: string; -} - -export interface NotebookFunctionCell extends ExecutableCodeCellBase { - cell_type: 'notebook-function'; - metadata: NotebookFunctionCellMetadata; - source: ''; -} - -export interface InputTextCell extends ExecutableCellBase { - cell_type: 'input-text'; - metadata: InputTextCellMetadata; - source: string; -} - -export interface InputCheckboxCell extends ExecutableCellBase { - cell_type: 'input-checkbox'; - metadata: InputCheckboxCellMetadata; - source: string; -} - -export interface InputTextareaCell extends ExecutableCellBase { - cell_type: 'input-textarea'; - metadata: InputTextareaCellMetadata; - source: string; -} - -export interface InputFileCell extends ExecutableCellBase { - cell_type: 'input-file'; - metadata: InputFileCellMetadata; - source: string; -} - -export interface ButtonCell extends ExecutableCellBase { - cell_type: 'button'; - source: ''; - metadata: ButtonCellMetadata; -} - -export interface SeparatorCell extends CellBase { - cell_type: 'separator'; - source: ''; - metadata: SeparatorCellMetadata; -} - -export interface FormatMarks { - bold?: boolean; - italic?: boolean; - underline?: boolean; - strike?: boolean; - code?: boolean; - /* the color accepts any valid CSS color value ('#000000', 'red', 'rgb(2,2,2)') */ - color?: string; -} - -export interface FormattedRangeMarks extends FormatMarks {} - -export type FormattedRange = FormattedRangeText | FormattedRangeLink; - -export interface FormattedRangeLink { - type: 'link'; - url: string; - fromCodePoint: number; - toCodePoint: number; - // ranges are local to the link - ranges: FormattedRangeText[]; -} - -export interface FormattedRangeText { - type?: 'marks'; - fromCodePoint: number; - toCodePoint: number; - marks: FormattedRangeMarks; -} - -export interface TextCellMetadata extends CellMetadata { - is_collapsed?: boolean; - formattedRanges?: FormattedRange[]; -} - -export interface TodoTextCellMetadata extends TextCellMetadata { - checked?: boolean; -} - -export type CalloutTextCellColor = - | 'blue' - | 'green' - | 'yellow' - | 'red' - | 'purple'; - -export interface CalloutTextCellMetadata extends TextCellMetadata { - color?: CalloutTextCellColor; -} - -export interface ParagraphTextCell extends CellBase { - cell_type: 'text-cell-p'; - metadata: TextCellMetadata; - source: string; -} - -export type HeadingTextCellType = - | 'text-cell-h1' - | 'text-cell-h2' - | 'text-cell-h3' - | 'text-cell-h4' - | 'text-cell-h5' - | 'text-cell-h6'; - -export interface Heading1TextCell extends CellBase { - cell_type: 'text-cell-h1'; - metadata: TextCellMetadata; - source: string; -} - -export interface Heading2TextCell extends CellBase { - cell_type: 'text-cell-h2'; - metadata: TextCellMetadata; - source: string; -} - -export interface Heading3TextCell extends CellBase { - cell_type: 'text-cell-h3'; - metadata: TextCellMetadata; - source: string; -} - -export type HeadingTextCell = - | Heading1TextCell - | Heading2TextCell - | Heading3TextCell; - -export interface BulletTextCell extends CellBase { - cell_type: 'text-cell-bullet'; - metadata: TextCellMetadata; - source: string; -} - -export interface TodoTextCell extends CellBase { - cell_type: 'text-cell-todo'; - metadata: TodoTextCellMetadata; - source: string; -} - -export interface CalloutTextCell extends CellBase { - cell_type: 'text-cell-callout'; - metadata: CalloutTextCellMetadata; - source: string; -} - -export type TextCell = - | ParagraphTextCell - | HeadingTextCell - | BulletTextCell - | TodoTextCell - | CalloutTextCell; - -export interface InputSelectCell extends ExecutableCellBase { - cell_type: 'input-select'; - metadata: InputSelectCellMetadata; - source: string; -} - -export interface InputSliderCell extends ExecutableCellBase { - cell_type: 'input-slider'; - metadata: InputSliderCellMetadata; - source: string; -} - -export interface InputDateCell extends ExecutableCellBase { - cell_type: 'input-date'; - metadata: InputDateCellMetadata; - source: string; -} - -export interface MarkdownCell extends CellBase { - cell_type: 'markdown'; - metadata: MarkdownCellMetadata; - source: string; -} - -export interface VisualizationCell extends ExecutableCellBase { - cell_type: 'visualization'; - source: ''; - metadata: VisualizationCellMetadata; -} - -export interface ImageCell extends CellBase { - cell_type: 'image'; - metadata: ImageCellMetadata; - source: ''; -} - -export interface BigNumberCell extends ExecutableCellBase { - cell_type: 'big-number'; - metadata: BigNumberCellMetadata; - source: string; -} - -export interface BigNumberCellMetadata extends ExecutableCellMetadata { - deepnote_big_number_title: string; - deepnote_big_number_value: string; - deepnote_big_number_format: string; - deepnote_big_number_comparison_enabled?: boolean; - deepnote_big_number_comparison_title?: string; - deepnote_big_number_comparison_value?: string; - deepnote_big_number_comparison_type?: string; - deepnote_big_number_comparison_format?: string; -} - -export type Cell = - | MarkdownCell - | CodeCell - | SqlCell - | NotebookFunctionCell - | InputCell - | TextCell - | VisualizationCell - | ImageCell - | ButtonCell - | SeparatorCell - | BigNumberCell; -export type InputCell = - | InputCheckboxCell - | InputTextCell - | InputSelectCell - | InputDateCell - | InputDateRangeCell - | InputSliderCell - | InputTextareaCell - | InputFileCell; - -export interface InputDateRangeCellMetadata extends ExecutableCellMetadata { - deepnote_variable_name: string; - deepnote_variable_value: [string, string] | string; - deepnote_variable_default_value?: [string, string] | string; - deepnote_input_label?: string; -} - -export interface InputDateRangeCell extends ExecutableCellBase { - cell_type: 'input-date-range'; - metadata: InputDateRangeCellMetadata; - source: string; -} - -// Source: -// deepnote-internal -// -// Path: -// libs/notebook-conversions/src/convert-deepnote-cell-to-jupyter-cell.ts -// -// Commit SHA: -// d943d4a9e9fb1a80f608072875f07a25fad3ce9e - -export function convertDeepnoteCellToJupyterCell( - cellId: string, - unclonedCell: Cell, - executionContext?: ExecutionContext -): JupyterCell { - const cell = _cloneDeep(unclonedCell); - - const jupyterCellMetadata: JupyterCell['metadata'] = { - ...cell.metadata, - cell_id: cellId - }; - - jupyterCellMetadata.deepnote_cell_type = cell.cell_type; - - if (isSqlCell(cell)) { - // let's store the raw SQL query in the metadata, since cell.source will be replaced by runnable python code - jupyterCellMetadata.deepnote_sql_source = cell.source; - } - - if (isExecutableCell(cell)) { - // Jupyter spec requires an explicit null here - cell.execution_count = - cell.execution_count === undefined ? null : cell.execution_count; - cell.outputs = cell.outputs === undefined ? [] : cell.outputs; - - if (cell.outputs) { - cell.outputs.forEach(output => { - delete output.truncated; - }); - } else { - console.warn( - '[convertDeepnoteToJupyter] Cell outputs not present in a code cell' - ); - } - } - - const source = createJupyterSource( - { - type: cell.cell_type, - source: cell.source, - metadata: cell.metadata - }, - executionContext - ); - - const jupyterCell: JupyterCell = { - ...cell, - cell_type: convertCellTypeToJupyter(cell.cell_type), - metadata: jupyterCellMetadata, - source - }; - return jupyterCell; -} diff --git a/src/deepnote-convert/deepnote-file-schema.ts b/src/deepnote-convert/deepnote-file-schema.ts deleted file mode 100644 index 46ed95a..0000000 --- a/src/deepnote-convert/deepnote-file-schema.ts +++ /dev/null @@ -1,76 +0,0 @@ -import { z } from 'zod'; - -// Below schema has been modified from the original deepnote-internal schema - -// Source: -// deepnote-internal -// -// Path: -// apps/webapp/server/modules/export-and-import-project/deepnote-file.ts - -// Commit SHA: -// 3ec11e794c6aca998ef88d894f18e4611586cc30 - -export const deepnoteFileBlockSchema = z.object({ - blockGroup: z.string().optional(), - content: z.string().optional(), - executionCount: z.number().optional(), - id: z.string(), - metadata: z.record(z.string(), z.any()).optional(), - outputs: z.array(z.any()).optional(), - sortingKey: z.string(), - type: z.string(), - version: z.number().optional() -}); - -export type DeepnoteFileBlock = z.infer; - -export const deepnoteFileSchema = z.object({ - metadata: z.object({ - checksum: z.string().optional(), - createdAt: z.string(), - exportedAt: z.string().optional(), - modifiedAt: z.string().optional() - }), - - project: z.object({ - id: z.string(), - - initNotebookId: z.string().optional(), - integrations: z - .array( - z.object({ - id: z.string(), - name: z.string(), - type: z.string() - }) - ) - .optional(), - name: z.string(), - notebooks: z.array( - z.object({ - blocks: z.array(deepnoteFileBlockSchema), - executionMode: z.enum(['block', 'downstream']).optional(), - id: z.string(), - isModule: z.boolean().optional(), - name: z.string(), - workingDirectory: z.string().optional() - }) - ), - settings: z - .object({ - environment: z - .object({ - customImage: z.string().optional(), - pythonVersion: z.string().optional() - }) - .optional(), - requirements: z.array(z.string()).optional(), - sqlCacheMaxAge: z.number().optional() - }) - .optional() - }), - version: z.string() -}); - -export type DeepnoteFile = z.infer; diff --git a/src/deepnote-convert/deserialize-deepnote-file.ts b/src/deepnote-convert/deserialize-deepnote-file.ts deleted file mode 100644 index b2f3807..0000000 --- a/src/deepnote-convert/deserialize-deepnote-file.ts +++ /dev/null @@ -1,38 +0,0 @@ -import { DeepnoteFile, deepnoteFileSchema } from './deepnote-file-schema'; -import { parseYaml } from './parse-yaml'; - -// Source: -// deepnote-internal -// -// Path: -// apps/webapp/server/modules/export-and-import-project/index.ts - -// Commit SHA: -// 3ec11e794c6aca998ef88d894f18e4611586cc30 - -/** - * Deserialize a YAML string into a DeepnoteFile object. - */ -export function deserializeDeepnoteFile(yamlContent: string): DeepnoteFile { - const parsed = parseYaml(yamlContent); - const result = deepnoteFileSchema.safeParse(parsed); - - if (!result.success) { - const issue = result.error.issues[0]; - - if (!issue) { - console.error('Invalid Deepnote file with no issues.'); - - throw new Error('Invalid Deepnote file.'); - } - - const path = issue.path.join('.'); - const message = path ? `${path}: ${issue.message}` : issue.message; - - console.error(`Failed to parse the Deepnote file: ${message}.`); - - throw new Error(`Failed to parse the Deepnote file: ${message}.`); - } - - return result.data; -} diff --git a/src/deepnote-convert/parse-yaml.ts b/src/deepnote-convert/parse-yaml.ts deleted file mode 100644 index a183fc8..0000000 --- a/src/deepnote-convert/parse-yaml.ts +++ /dev/null @@ -1,22 +0,0 @@ -import { parse } from 'yaml'; - -// Source: -// deepnote-internal -// -// Path: -// apps/webapp/server/modules/export-and-import-project/index.ts - -// Commit SHA: -// 3ec11e794c6aca998ef88d894f18e4611586cc30 - -export function parseYaml(yamlContent: string): unknown { - try { - const parsed = parse(yamlContent); - - return parsed; - } catch (e) { - console.error('Failed to parse Deepnote file as YAML.', e); - - throw new Error('Failed to parse Deepnote file.'); - } -} diff --git a/src/deepnote-convert/shared-table-state-schemas.ts b/src/deepnote-convert/shared-table-state-schemas.ts deleted file mode 100644 index 992e329..0000000 --- a/src/deepnote-convert/shared-table-state-schemas.ts +++ /dev/null @@ -1,197 +0,0 @@ -import { z } from 'zod'; - -const columnIdSchema = z.string(); -export type ColumnId = z.infer; - -export const dataTableCellFormattingStyleNames = [ - 'default', - 'defaultBold', - 'positive', - 'positiveBold', - 'positiveProminentBold', - 'attention', - 'attentionBold', - 'attentionProminentBold', - 'critical', - 'criticalBold', - 'criticalProminentBold' -] as const; -export type DataTableCellFormattingStyleName = - (typeof dataTableCellFormattingStyleNames)[number]; - -const dataTableColumnSelectionModeSchema = z.enum(['all', 'allExcept', 'only']); -export type DataTableColumnSelectionMode = z.infer< - typeof dataTableColumnSelectionModeSchema ->; - -export const dataframeFilterOperators = [ - 'is-equal', - 'is-not-equal', - 'is-one-of', - 'is-not-one-of', - 'is-not-null', - 'is-null', - 'text-contains', - 'text-does-not-contain', - 'greater-than', - 'greater-than-or-equal', - 'less-than', - 'less-than-or-equal', - 'between', - 'outside-of', - 'is-relative-today', - 'is-after', - 'is-before', - 'is-on' -] as const; -const dataframeFilterOperatorSchema = z.enum(dataframeFilterOperators); -export type DataframeFilterOperator = z.infer< - typeof dataframeFilterOperatorSchema ->; - -const dataTableCellFormattingRuleSingleColorSchema = z.object({ - type: z.literal('singleColor'), - columnSelectionMode: dataTableColumnSelectionModeSchema, - columnNames: z.array(columnIdSchema).nullable(), - operator: dataframeFilterOperatorSchema, - comparativeValues: z.array(z.string()), - styleName: z.enum(dataTableCellFormattingStyleNames) -}); -export type DataTableCellFormattingRuleSingleColor = z.infer< - typeof dataTableCellFormattingRuleSingleColorSchema ->; - -export const dataTableCellFormattingScaleNames = [ - 'defaultToPositive', - 'defaultToAttention', - 'defaultToCritical', - 'positiveToDefault', - 'attentionToDefault', - 'criticalToDefault', - 'criticalToDefaultToPositive', - 'criticalToAttentionToPositive', - 'positiveToDefaultToCritical', - 'positiveToAttentionToCritical' -] as const; -export type DataTableCellFormattingColorScaleName = - (typeof dataTableCellFormattingScaleNames)[number]; - -const dataTableCellFormattingRuleColorScalePointSchema = z.object({ - valueType: z.enum(['autoAllColumns', 'number', 'percentage']), - value: z.number().nullable(), - customColor: z.string().nullable() -}); -export type DataTableCellFormattingRuleColorScalePoint = z.infer< - typeof dataTableCellFormattingRuleColorScalePointSchema ->; - -const dataTableCellFormattingRuleColorScaleSchema = z.object({ - type: z.literal('colorScale'), - columnSelectionMode: dataTableColumnSelectionModeSchema, - columnNames: z.array(columnIdSchema).nullable(), - pointMin: dataTableCellFormattingRuleColorScalePointSchema, - pointMid: dataTableCellFormattingRuleColorScalePointSchema, - pointMax: dataTableCellFormattingRuleColorScalePointSchema, - scaleName: z.enum(dataTableCellFormattingScaleNames) -}); -export type DataTableCellFormattingRuleColorScale = z.infer< - typeof dataTableCellFormattingRuleColorScaleSchema ->; - -const dataTableCellFormattingRuleColorScaleResolvedValueSchema = z.object({ - quantitative: z - .object({ - min: z.number(), - mid: z.number(), - max: z.number() - }) - .nullable(), - temporal: z - .object({ - min: z.date(), - mid: z.date(), - max: z.date() - }) - .nullable() -}); -export type DataTableCellFormattingRuleColorScaleResolvedValue = z.infer< - typeof dataTableCellFormattingRuleColorScaleResolvedValueSchema ->; - -const dataTableCellFormattingRuleColorScaleResolvedSchema = - dataTableCellFormattingRuleColorScaleSchema.extend({ - resolvedValue: dataTableCellFormattingRuleColorScaleResolvedValueSchema - }); -export type DataTableCellFormattingRuleColorScaleResolved = z.infer< - typeof dataTableCellFormattingRuleColorScaleResolvedSchema ->; - -const dataTableCellFormattingRuleSchema = z.union([ - dataTableCellFormattingRuleSingleColorSchema, - dataTableCellFormattingRuleColorScaleSchema -]); -export type DataTableCellFormattingRule = z.infer< - typeof dataTableCellFormattingRuleSchema ->; - -const dataTableCellFormattingRuleResolvedSchema = z.union([ - dataTableCellFormattingRuleSingleColorSchema, - dataTableCellFormattingRuleColorScaleResolvedSchema -]); -export type DataTableCellFormattingRuleResolved = z.infer< - typeof dataTableCellFormattingRuleResolvedSchema ->; - -const dataTableCellFormattingColorScaleDefinitionSchema = z.object({ - min: z.string(), - mid: z.string().optional(), - max: z.string() -}); -export type DataTableCellFormattingColorScaleDefinition = z.infer< - typeof dataTableCellFormattingColorScaleDefinitionSchema ->; - -export const dataframeFilterSchema = z.object({ - column: columnIdSchema, - operator: dataframeFilterOperatorSchema, - comparativeValues: z.array(z.string()) -}); -export type DataframeFilter = z.infer; - -const sortBySchema = z.object({ - id: columnIdSchema, - type: z.enum(['asc', 'desc']) -}); -export type SortBy = z.infer; - -const columnContainsFilterSchema = z.object({ - id: columnIdSchema, - value: z.string(), - type: z.literal('contains') -}); -export type ColumnContainsFilter = z.infer; - -const columnFilterSchema = columnContainsFilterSchema; -export type ColumnFilter = z.infer; - -const columnDisplayNameRecordSchema = z.object({ - columnName: columnIdSchema, - displayName: z.string() -}); -export type ColumnDisplayNameRecord = z.infer< - typeof columnDisplayNameRecordSchema ->; - -export const sharedTableStateSchema = z.object({ - pageSize: z.number().optional(), - pageIndex: z.number().optional(), - filters: z.array(columnFilterSchema).optional(), - conditionalFilters: z.array(dataframeFilterSchema).optional(), - sortBy: z.array(sortBySchema).optional(), - cellFormattingRules: z.array(dataTableCellFormattingRuleSchema).optional(), - wrappedTextColumnIds: z.array(columnIdSchema).optional(), - hiddenColumnIds: z.array(columnIdSchema).optional(), - columnOrder: z.array(columnIdSchema).optional(), - columnDisplayNames: z.array(columnDisplayNameRecordSchema).optional() -}); -export type SharedTableState = z.infer; diff --git a/src/deepnote-convert/types.ts b/src/deepnote-convert/types.ts deleted file mode 100644 index e265a87..0000000 --- a/src/deepnote-convert/types.ts +++ /dev/null @@ -1,59 +0,0 @@ -// Source: -// deepnote-internal -// -// Path: -// libs/shared/src/types.ts - -// Commit SHA: -// 3a0bab71e1ee86530c74eb5ada2e1873848c5fea - -export const TEXT_CELL_TYPES = [ - 'text-cell-h1', - 'text-cell-h2', - 'text-cell-h3', - 'text-cell-p', - 'text-cell-bullet', - 'text-cell-todo', - 'text-cell-callout' -] as const; -export type TextCellType = (typeof TEXT_CELL_TYPES)[number]; - -export const INPUT_CELL_TYPES = [ - 'input-text', - 'input-textarea', - 'input-select', - 'input-date', - 'input-date-range', - 'input-slider', - 'input-file', - 'input-checkbox' -] as const; -export type InputCellType = (typeof INPUT_CELL_TYPES)[number]; -export const CELL_TYPES = [ - 'code', - 'sql', - 'markdown', - 'notebook-function', - ...INPUT_CELL_TYPES, - ...TEXT_CELL_TYPES, - 'visualization', - 'image', - 'button', - 'separator', - 'big-number' -] as const; -export type CellType = (typeof CELL_TYPES)[number]; - -// Source: -// deepnote-internal -// -// Path: -// libs/jupyter/src/jupyter.ts - -// Commit SHA: -// 0ac69479e192dbbb04ae10cd7b027872d09fbb14 - -/** - * A type which describes the type of cell. - */ -export type JupyterCellType = 'code' | 'markdown' | 'raw'; diff --git a/src/deepnote-convert/utils/object-utils.ts b/src/deepnote-convert/utils/object-utils.ts deleted file mode 100644 index 5c1192c..0000000 --- a/src/deepnote-convert/utils/object-utils.ts +++ /dev/null @@ -1,50 +0,0 @@ -import _isEqual from 'lodash/isEqual'; - -export function pickBy( - obj: T, - predicate: (value: T[keyof T]) => boolean -) { - return Object.fromEntries( - Object.entries(obj).filter(([_, v]) => predicate(v as T[keyof T])) - ) as Partial; -} - -export function pickNonEmpty(obj: T) { - return pickBy(obj, v => v != null) as Partial<{ - [K in keyof T]: NonNullable; - }>; -} - -export function getObjectDiff( - objectA: E, - objectB: E -): Partial | null { - const diff: Partial = {}; - - // NOTE: We need to cover the situation where some keys are missing in one of the objects. - const keys = new Set([ - ...Object.keys(objectA), - ...Object.keys(objectB)  - ] as Array); - for (const key of keys) { - if (!_isEqual(objectA[key], objectB[key])) { - diff[key] = objectB[key]; - } - } - - return Object.keys(diff).length > 0 ? diff : null; -} - -export function nullKeysToUndefined(obj: T) { - return Object.fromEntries( - Object.entries(obj).map(([k, v]) => [k, v ?? undefined]) - ) as { - [K in keyof T]: NonNullable | undefined; - }; -} - -export function getSortedObjectEntries(obj: T) { - return Object.entries(obj).sort(([a], [b]) => a.localeCompare(b)) as Array< - [keyof T, T[keyof T]] - >; -} diff --git a/src/deepnote-convert/vega-lite.types.ts b/src/deepnote-convert/vega-lite.types.ts deleted file mode 100644 index 4636312..0000000 --- a/src/deepnote-convert/vega-lite.types.ts +++ /dev/null @@ -1,414 +0,0 @@ -import { z } from 'zod'; - -import type { TopLevelParameter } from 'vega-lite/build/src/spec/toplevel'; -import type { Transform } from 'vega-lite/build/src/transform'; - -export type { Spec as VegaSpec } from 'vega'; - -// Source: -// deepnote-internal -// -// Path: -// libs/shared/src/cells/vega-lite.types.ts -// -// Commit SHA: -// 7a06bd352cf59577b7cd100cb14747908400440e - -/** In v1 chart blocks, ordinal type was supported. In v2, it's not - so we're only keeping it here for migration purposes. - * Existing ordinal types encoding are converted to quantitative type. - */ - -export const legacyEncodingTypeSchema = z.enum([ - 'quantitative', - 'ordinal', - 'nominal', - 'temporal' -]); -export type LegacyEncodingType = z.infer; - -const markTypeSchema = z.enum([ - 'area', - 'bar', - 'trail', - 'line', - 'point', - 'circle', - 'text', - 'arc' -]); -export type MarkType = z.infer; - -export const encodingChannelSchema = z.enum([ - 'x', - 'y', - 'color', - 'opacity', - 'order', - 'size', - 'xOffset', - 'yOffset', - 'text', - 'theta', - 'tooltip', - 'detail', - 'stroke' -]); -export type EncodingChannel = z.infer; - -export const sortOrderSchema = z.enum(['ascending', 'descending']); -export type SortOrder = z.infer; - -export const scaleTypeSchema = z.enum(['linear', 'log', 'sqrt', 'time']); -export type ScaleType = z.infer; - -export const numberTypeSchema = z.enum([ - 'default', - 'number', - 'scientific', - 'percent' -]); -export type NumberType = z.infer; - -export const numberFormatSchema = z.object({ - type: numberTypeSchema.default('default'), - decimals: z.number().nullable().default(null) -}); -export type NumberFormat = z.infer; - -export const aggregateTypeSchema = z.enum([ - 'argmax', - 'argmin', - 'count', - 'average', - 'distinct', - 'max', - 'median', - 'min', - 'sum', - 'variance', - 'none' -]); -export type AggregateType = z.infer; - -export const timeUnitSchema = z.enum([ - 'yearmonthdatehours', - 'yearmonthdate', - 'yearmonth', - 'yearweek', - 'year', - 'monthdatehours', - 'monthdate', - 'month', - 'week', - 'datehours', - 'date', - 'day', - 'hours' -]); -export type TimeUnit = z.infer; - -export type FieldType = string; - -export type GridType = 'x' | 'y' | 'full' | 'disabled'; - -export interface Encoding { - field?: FieldType; - type?: LegacyEncodingType; - title?: string; - sort?: - | { - order: SortOrder; - } - | { - encoding: EncodingChannel; - order: SortOrder; - } - | { - order: SortOrder; - field?: string | null; - op?: AggregateType; - } - | SortOrder - | null; - aggregate?: AggregateType; - bin?: boolean | { step?: number; maxbins?: number }; - timeUnit?: TimeUnit | null; - axis?: { - grid?: boolean; - title?: string | null; - ticks?: boolean; - labels?: boolean; - formatType?: string; - format?: string | NumberFormat; - }; - scale?: { - scheme?: string; - type?: ScaleType; - zero?: boolean; - domainMin?: number; - domainMax?: number; - // Used only for color encoding to set custom label + color for legend - domain?: [string]; - range?: [string]; - }; - formatType?: string; - format?: string | NumberFormat; - datum?: number | string; - stack?: 'normalize' | 'zero' | true; - condition?: Record; - value?: number | string; - legend?: Record; - bandPosition?: number; -} - -export type VegaLiteMark = - | { - type: MarkType; - outerRadius?: { - expr: string; - }; - innerRadius?: { - expr: string; - }; - tooltip?: boolean; - } - | { - type: MarkType; - tooltip: boolean | { content: string }; - color?: string; - clip?: boolean; - } - | MarkType; - -// NOTE: tooltip is used in encoding to display and format tooltips in the Pie / Arc chart -export type EncodingsMap = { - [channel in EncodingChannel]?: channel extends 'tooltip' | 'detail' - ? Encoding[] - : Encoding; -}; - -export interface VegaLiteDataLayer { - mark: VegaLiteMark; - encoding: EncodingsMap; - params?: Array; - transform?: Array; -} - -/** - * This mark config is only used for the purposes of "value labels". See NB-353. - * That's why it's not a full mark config, but only a subset of properties that we need. - */ -interface VegaLiteTextMarkConfig { - type: 'text'; - align?: 'left' | 'right' | 'center'; - baseline?: 'top' | 'middle' | 'bottom'; - fill: 'black'; - dx?: number; - dy?: number; - radius?: { - expr: string; - }; -} - -/** - * This text layer is only used for the purposes of adding "value labels" to another layer. See NB-353 - * This layer will always be nested with a sibling data layer. See `VegaLiteParentLayer` - */ -export interface VegaLiteTextLayer { - mark: VegaLiteTextMarkConfig; - params?: Array; - - encoding: EncodingsMap & { - text: Encoding; - }; - transform?: Array; -} - -/** - * This point layer is only used for the purpose of adding a better tooltip to another, line layer. - * This layer will always be nested with a sibling data layer. See `VegaLiteParentLayer` - * ! Be careful with type-guarding this layer. It looks very similar to a VegaLiteDataLayer with a point mark, - * ! but with only a subset of properties that we need. - */ -export interface VegaLiteLineTooltipLayer { - mark: - | { type: 'point'; tooltip: true; size: number; opacity: 0 } - | { - type: 'text'; - radius: { - expr: string; - }; - }; - encoding: EncodingsMap; - params?: Array; -} - -export type VegaLiteHelperLayer = VegaLiteTextLayer | VegaLiteLineTooltipLayer; - -// This is a leaf layer that doesn't have any children layer -export type VegaLiteLeafLayer = VegaLiteDataLayer | VegaLiteHelperLayer; - -export interface VegaLiteParentLayer { - title?: string; - // an array with at least 1 element - first element is always a data layer, and the rest are 0+ helper layers - layer: [VegaLiteDataLayer, ...VegaLiteHelperLayer[]]; - encoding?: { [channel in EncodingChannel]?: Encoding }; - mark?: VegaLiteMark; -} - -export type VegaLiteMultiLayerSpecV1TopLevelLayer = - | VegaLiteDataLayer - | VegaLiteParentLayer; - -interface VegaLiteSpecShared { - $schema: - | 'https://vega.github.io/schema/vega-lite/v4.json' - | 'https://vega.github.io/schema/vega-lite/v5.json'; - title?: string; - config?: { - legend?: { - disable?: boolean; - orient?: 'left' | 'right' | 'top' | 'bottom'; - labelFont?: string; - labelFontSize?: number; - labelFontWeight?: string; - labelOverlap?: boolean; - labelColor?: string; - padding?: number; - rowPadding?: number; - symbolSize?: number; - symbolType?: string; - titleColor?: string; - titlePadding?: number; - titleFont?: string; - titleFontSize?: number; - titleFontWeight?: string; - }; - title?: { - anchor?: string; - color?: string; - font?: string; - fontSize?: number; - fontWeight?: string | number; - dy?: number; - offset?: number; - orient?: 'left' | 'right' | 'top' | 'bottom'; - }; - axis?: { - labelFont?: string; - labelFontSize?: number; - labelFontWeight?: string | number; - titleFont?: string; - titleFontSize?: number; - titleFontWeight?: string | number; - labelOverlap?: string; - labelColor?: string; - titleColor?: string; - titlePadding?: number; - domainCap?: 'butt' | 'round' | 'square'; - gridCap?: 'butt' | 'round' | 'square'; - gridWidth?: number; - gridColor?: string; - }; - axisX?: { - labelPadding?: number; - }; - axisY?: { - labelPadding?: number; - }; - axisBand?: { - tickCap?: 'butt' | 'round' | 'square'; - tickExtra?: boolean; - }; - axisQuantitative?: { - tickCount?: number; - }; - view?: { - stroke?: string; - }; - line?: { - strokeCap?: 'butt' | 'round' | 'square'; - strokeWidth?: number; - strokeJoin?: 'bevel'; - }; - bar?: { - cornerRadiusTopRight?: number; - cornerRadiusBottomLeft?: number; - cornerRadiusBottomRight?: number; - cornerRadiusTopLeft?: number; - }; - area?: { fill?: 'category'; line?: boolean; opacity?: number }; - circle?: { size?: number }; - mark?: { color?: string }; - range?: { - category?: readonly string[]; - ramp?: string[]; - }; - - customFormatTypes?: boolean; - }; - encoding: EncodingsMap; - usermeta?: { - tooltipDefaultMode?: boolean; - aditionalTypeInfo?: { - histogramLayerIndexes: number[]; - }; - seriesOrder?: number[]; - seriesNames?: string[]; - specSchemaVersion?: number; - }; - resolve?: { scale?: { [channel in 'x' | 'y']?: 'independent' | 'shared' } }; - transform?: Array; - params?: Array; - - autosize?: { - type: 'fit'; - }; - background?: string; - width?: 'container' | number; - height?: 'container' | number; - padding?: - | number - | { left: number; right: number; top: number; bottom: number }; - data?: { - name?: string; - values?: Array>; - }; -} - -export interface VegaLiteMultiLayerSpecV1 extends VegaLiteSpecShared { - mark?: undefined; - // In spec v1 we had arbitrary number of layers (roughly corresponds to series in current chart config), - // each of which consisted of one data layer and 0 or more helper layers (e.g. for value labels) - layer: Array; -} - -export interface VegaLiteAxisGroup { - resolve: { - scale: { - color: 'independent'; - }; - }; - layer: VegaLiteParentLayer[]; -} - -export interface VegaLiteMultiLayerSpecV2 extends VegaLiteSpecShared { - mark?: undefined; - // Second version of spec adds one more nesting level to `layer`. Now, there are always 1-2 - // root layers (one for each measure axis), which then consist of arbitrary number of series, - // each of which has one data layer and 0 or more helper layers (e.g. for value labels) - layer: [VegaLiteAxisGroup] | [VegaLiteAxisGroup, VegaLiteAxisGroup]; -} - -export interface VegaLiteTopLayerSpec extends VegaLiteSpecShared { - mark: VegaLiteMark; - layer?: undefined; -} - -/** - * This is a subset of vega-lite spec that we support the editing of in the chart block UI. - */ -export type VegaLiteSpec = - | VegaLiteTopLayerSpec - | VegaLiteMultiLayerSpecV1 - | VegaLiteMultiLayerSpecV2; From 07bb5494a5b7ebf5925d8b591f25bee2be208912 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 16:31:08 +0200 Subject: [PATCH 19/37] Remove duplicated code Signed-off-by: Andy Jakubowski --- src/components/NotebookPicker.tsx | 33 ------------------------------- 1 file changed, 33 deletions(-) diff --git a/src/components/NotebookPicker.tsx b/src/components/NotebookPicker.tsx index cb1f2d4..a7f15f6 100644 --- a/src/components/NotebookPicker.tsx +++ b/src/components/NotebookPicker.tsx @@ -54,40 +54,7 @@ export class NotebookPicker extends ReactWidget { this.update(); }; - private handleChange = (event: React.ChangeEvent) => { - const model = this.panel.model; - if (!model) { - return; - } - - const selected = event.target.value; - const deepnoteMetadata = this.panel.context.model.getMetadata('deepnote'); - const notebooks = deepnoteMetadata?.notebooks; - - if (notebooks && selected in notebooks) { - // clone the notebook JSON - const newModelData = { ...notebooks[selected] }; - - // preserve deepnote metadata *without* re-inserting all notebooks - newModelData.metadata = { - ...(newModelData.metadata ?? {}), - deepnote: { - notebook_names: deepnoteMetadata?.notebook_names ?? [], - notebooks: deepnoteMetadata?.notebooks ?? {} - } - }; - - model.fromJSON(newModelData); - model.dirty = false; - } - - this.selected = selected; - this.update(); - }; - render(): JSX.Element { - const deepnoteMetadata = this.panel.context.model.getMetadata('deepnote'); - const metadataNames = deepnoteMetadata?.notebook_names; const deepnoteMetadata = this.panel.context.model.getMetadata('deepnote'); const metadataNames = deepnoteMetadata?.notebook_names; const names = From cb26de05b2a708c52e8bbb546e898f096932adbd Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Tue, 7 Oct 2025 16:58:25 +0200 Subject: [PATCH 20/37] Convert Deepnote file to Jupyter notebook Signed-off-by: Andy Jakubowski --- .yarnrc.yml | 5 ++ package.json | 4 +- src/convert-deepnote-block-to-jupyter-cell.ts | 58 ++++++++++++------- src/convert-deepnote-block-type-to-jupyter.ts | 32 ++++++++++ ...sform-deepnote-yaml-to-notebook-content.ts | 10 +++- src/types.ts | 2 +- yarn.lock | 26 ++++----- 7 files changed, 100 insertions(+), 37 deletions(-) create mode 100644 src/convert-deepnote-block-type-to-jupyter.ts diff --git a/.yarnrc.yml b/.yarnrc.yml index 3186f3f..37de770 100644 --- a/.yarnrc.yml +++ b/.yarnrc.yml @@ -1 +1,6 @@ nodeLinker: node-modules +npmScopes: + deepnote: + npmRegistryServer: 'https://npm.pkg.github.com' + npmAlwaysAuth: true + npmAuthToken: '${GITHUB_TOKEN}' diff --git a/package.json b/package.json index bf9aeef..818c54c 100644 --- a/package.json +++ b/package.json @@ -58,13 +58,15 @@ "watch:labextension": "jupyter labextension watch ." }, "dependencies": { - "@deepnote/blocks": "file:/Users/work/repos/deepnote/deepnote/packages/blocks", + "@deepnote/blocks": "^1.0.0", "@jupyterlab/application": "^4.0.0", "@jupyterlab/coreutils": "^6.0.0", "@jupyterlab/notebook": "^4.4.7", "@jupyterlab/services": "^7.0.0", "@jupyterlab/settingregistry": "^4.0.0", "@lumino/widgets": "^2.7.1", + "@types/lodash": "^4.17.20", + "lodash": "^4.17.21", "yaml": "^2.8.1", "zod": "^4.1.11" }, diff --git a/src/convert-deepnote-block-to-jupyter-cell.ts b/src/convert-deepnote-block-to-jupyter-cell.ts index 13ea18d..a0ef666 100644 --- a/src/convert-deepnote-block-to-jupyter-cell.ts +++ b/src/convert-deepnote-block-to-jupyter-cell.ts @@ -1,31 +1,47 @@ -import { convertCellTypeToJupyter } from './deepnote-convert/convert-cell-type-to-jupyter'; -import { DeepnoteFileBlock } from './deepnote-convert/deepnote-file-schema'; +import { + createMarkdown, + createPythonCode, + DeepnoteBlock +} from '@deepnote/blocks'; +import _cloneDeep from 'lodash/cloneDeep'; import { ICodeCell, IMarkdownCell } from '@jupyterlab/nbformat'; +import { convertDeepnoteBlockTypeToJupyter } from './convert-deepnote-block-type-to-jupyter'; + +export function convertDeepnoteBlockToJupyterCell(block: DeepnoteBlock) { + const blockCopy = _cloneDeep(block); + const jupyterCellMetadata = { ...blockCopy.metadata, cell_id: blockCopy.id }; + const jupyterCellType = convertDeepnoteBlockTypeToJupyter(blockCopy.type); -export function convertDeepnoteBlockToJupyterCell( - block: DeepnoteFileBlock -): ICodeCell | IMarkdownCell { - const jupyterCellType = convertCellTypeToJupyter(block.type); if (jupyterCellType === 'code') { - return { + const blockOutputs = blockCopy.outputs ?? []; + + if (Array.isArray(blockOutputs)) { + blockOutputs.forEach(output => { + delete output.truncated; + }); + } + + const source = createPythonCode(blockCopy); + + const jupyterCell: ICodeCell = { cell_type: 'code', - source: block.content || '', - metadata: {}, - outputs: block.outputs || [], - execution_count: block.executionCount || null - }; - } else if (jupyterCellType === 'markdown') { - return { - cell_type: 'markdown', - source: block.content || '', - metadata: {} + metadata: jupyterCellMetadata, + execution_count: + blockCopy.executionCount !== undefined + ? blockCopy.executionCount + : null, + outputs: blockOutputs, + source }; + return jupyterCell; } else { - // For unsupported block types, return a markdown cell indicating it's unsupported - return { + // Markdown cell + const source = createMarkdown(blockCopy); + const jupyterCell: IMarkdownCell = { cell_type: 'markdown', - source: `# Unsupported block type: ${block.type}\n`, - metadata: {} + metadata: {}, + source }; + return jupyterCell; } } diff --git a/src/convert-deepnote-block-type-to-jupyter.ts b/src/convert-deepnote-block-type-to-jupyter.ts new file mode 100644 index 0000000..6ed37d0 --- /dev/null +++ b/src/convert-deepnote-block-type-to-jupyter.ts @@ -0,0 +1,32 @@ +export function convertDeepnoteBlockTypeToJupyter(blockType: string) { + switch (blockType) { + case 'big-number': + case 'code': + case 'sql': + case 'notebook-function': + case 'input-text': + case 'input-checkbox': + case 'input-textarea': + case 'input-file': + case 'input-select': + case 'input-date-range': + case 'input-date': + case 'input-slider': + case 'visualization': + return 'code'; + + case 'markdown': + case 'text-cell-h1': + case 'text-cell-h3': + case 'text-cell-h2': + case 'text-cell-p': + case 'text-cell-bullet': + case 'text-cell-todo': + case 'text-cell-callout': + case 'image': + case 'button': + case 'separator': + default: + return 'markdown'; + } +} \ No newline at end of file diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 2b17087..17360f9 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -1,6 +1,7 @@ import { IDeepnoteNotebookContent } from './types'; import { blankCodeCell, blankDeepnoteNotebookContent } from './fallback-data'; import { deserializeDeepnoteFile } from '@deepnote/blocks'; +import { convertDeepnoteBlockToJupyterCell } from './convert-deepnote-block-to-jupyter-cell'; export async function transformDeepnoteYamlToNotebookContent( yamlString: string @@ -22,7 +23,14 @@ export async function transformDeepnoteYamlToNotebookContent( }; } - return blankDeepnoteNotebookContent; + const cells = selectedNotebook.blocks.map( + convertDeepnoteBlockToJupyterCell + ); + + return { + ...blankDeepnoteNotebookContent, + cells + }; } catch (error) { console.error('Failed to deserialize Deepnote file:', error); throw new Error('Failed to transform Deepnote YAML to notebook content.'); diff --git a/src/types.ts b/src/types.ts index 477566c..73f8435 100644 --- a/src/types.ts +++ b/src/types.ts @@ -1,5 +1,5 @@ import { INotebookContent, INotebookMetadata } from '@jupyterlab/nbformat'; -import { DeepnoteFile } from './deepnote-convert/deepnote-file-schema'; +import type { DeepnoteFile } from '@deepnote/blocks'; export interface IDeepnoteNotebookMetadata extends INotebookMetadata { deepnote: { diff --git a/yarn.lock b/yarn.lock index 6bd97d4..030588c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1577,14 +1577,12 @@ __metadata: languageName: node linkType: hard -"@deepnote/blocks@file:/Users/work/repos/deepnote/deepnote/packages/blocks::locator=jupyterlab-deepnote%40workspace%3A.": +"@deepnote/blocks@npm:^1.0.0": version: 1.0.0 - resolution: "@deepnote/blocks@file:/Users/work/repos/deepnote/deepnote/packages/blocks#/Users/work/repos/deepnote/deepnote/packages/blocks::hash=193053&locator=jupyterlab-deepnote%40workspace%3A." + resolution: "@deepnote/blocks@npm:1.0.0::__archiveUrl=https%3A%2F%2Fnpm.pkg.github.com%2Fdownload%2F%40deepnote%2Fblocks%2F1.0.0%2Fbbad3593046ed93df89035400c7e7d85a4742494" dependencies: ts-dedent: ^2.2.0 - yaml: ^2.8.1 - zod: ^4.1.12 - checksum: d47ee3fe23825c2f3012c50cd87216a4099d0abb07a46de67c50650943448baf624c3ec93e9a4fc6626995c9fae33d3fc3500e37e3044d99c9660887ea88e2de + checksum: 69339b4c5d5b1da33c0deb248714b1aa151f9e80f623ba3a5b16669273fd0516db6c811b59c8cec3add16506e5bcf4b87c944744aa7d7761682139d3c556a06f languageName: node linkType: hard @@ -3277,6 +3275,13 @@ __metadata: languageName: node linkType: hard +"@types/lodash@npm:^4.17.20": + version: 4.17.20 + resolution: "@types/lodash@npm:4.17.20" + checksum: dc7bb4653514dd91117a4c4cec2c37e2b5a163d7643445e4757d76a360fabe064422ec7a42dde7450c5e7e0e7e678d5e6eae6d2a919abcddf581d81e63e63839 + languageName: node + linkType: hard + "@types/minimist@npm:^1.2.2": version: 1.2.5 resolution: "@types/minimist@npm:1.2.5" @@ -6805,7 +6810,7 @@ __metadata: version: 0.0.0-use.local resolution: "jupyterlab-deepnote@workspace:." dependencies: - "@deepnote/blocks": "file:/Users/work/repos/deepnote/deepnote/packages/blocks" + "@deepnote/blocks": ^1.0.0 "@jupyterlab/application": ^4.0.0 "@jupyterlab/builder": ^4.0.0 "@jupyterlab/coreutils": ^6.0.0 @@ -6816,6 +6821,7 @@ __metadata: "@lumino/widgets": ^2.7.1 "@types/jest": ^29.2.0 "@types/json-schema": ^7.0.11 + "@types/lodash": ^4.17.20 "@types/react": ^18.0.26 "@types/react-addons-linked-state-mixin": ^0.14.22 "@typescript-eslint/eslint-plugin": ^6.1.0 @@ -6825,6 +6831,7 @@ __metadata: eslint-config-prettier: ^8.8.0 eslint-plugin-prettier: ^5.0.0 jest: ^29.2.0 + lodash: ^4.17.21 mkdirp: ^1.0.3 npm-run-all2: ^7.0.1 prettier: ^3.0.0 @@ -9842,10 +9849,3 @@ __metadata: checksum: 022d59f85ebe054835fbcdc96a93c01479a64321104f846cb5644812c91e00d17ae3479f823956ec9b04e4351dd32841e1f12c567e81bc43f6e21ef5cc02ce3c languageName: node linkType: hard - -"zod@npm:^4.1.12": - version: 4.1.12 - resolution: "zod@npm:4.1.12" - checksum: 91174acc7d2ca5572ad522643474ddd60640cf6877b5d76e5d583eb25e3c4072c6f5eb92ab94f231ec5ce61c6acdfc3e0166de45fb1005b1ea54986b026b765f - languageName: node - linkType: hard From 017e92fefb8053a5b0fde3effcad018fa53c13ab Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:03:12 +0200 Subject: [PATCH 21/37] Set GITHUB_TOKEN in CI build workflow Signed-off-by: Andy Jakubowski --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b0f6d58..75ef841 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -29,6 +29,8 @@ jobs: set -eux jlpm jlpm run lint:check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Test the extension run: | From 1d72fb118a3163274d2ae9bc31dc8bff813cf73e Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:10:21 +0200 Subject: [PATCH 22/37] Add setup instructions for GitHub package registry Signed-off-by: Andy Jakubowski --- README.md | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/README.md b/README.md index a799e60..38ede3d 100644 --- a/README.md +++ b/README.md @@ -72,6 +72,22 @@ Install `jupyterlab`. The extension package itself doesn’t depend on `jupyterl uv pip install jupyterlab ``` +**Configure Access to @deepnote/blocks Package** + +The `@deepnote/blocks` package is published on GitHub Packages. To install it, you'll need to authenticate with GitHub: + +1. Create a GitHub Personal Access Token (classic) with `read:packages` scope: + - Go to https://github.com/settings/tokens + - Click "Generate new token (classic)" + - Select the `read:packages` scope + - Generate and copy the token + +2. Set the `GITHUB_TOKEN` environment variable to ensure `jlpm` (which is a wrapper around Yarn) can download the `@deepnote/blocks` package from the GitHub package registry. You can set the variable in `.zshrc` or manually like: + ```shell + export GITHUB_TOKEN=your_token_here + ``` + Replace `YOUR_TOKEN_HERE` with your actual token. + Install the extension package in editable mode. It installs the package’s dependencies, too: ```shell From 08c2d1be931569fda37847ec0316034c2dbca5f8 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:11:46 +0200 Subject: [PATCH 23/37] Fix linting Signed-off-by: Andy Jakubowski --- src/convert-deepnote-block-type-to-jupyter.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/convert-deepnote-block-type-to-jupyter.ts b/src/convert-deepnote-block-type-to-jupyter.ts index 6ed37d0..8543e9d 100644 --- a/src/convert-deepnote-block-type-to-jupyter.ts +++ b/src/convert-deepnote-block-type-to-jupyter.ts @@ -29,4 +29,4 @@ export function convertDeepnoteBlockTypeToJupyter(blockType: string) { default: return 'markdown'; } -} \ No newline at end of file +} From 95cebf977b38c4b871143e71fcc176a299ac2067 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:13:00 +0200 Subject: [PATCH 24/37] Remove request examples file Signed-off-by: Andy Jakubowski --- deepnote-file-request-examples.txt | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 deepnote-file-request-examples.txt diff --git a/deepnote-file-request-examples.txt b/deepnote-file-request-examples.txt deleted file mode 100644 index ad33e03..0000000 --- a/deepnote-file-request-examples.txt +++ /dev/null @@ -1,7 +0,0 @@ -[W 2025-09-18 11:22:56.083 ServerApp] 404 GET /api/contents/all-block-types-no-outputs.deepnote?content=0&hash=0&1758187376077 (::1): file or directory does not exist: 'all-block-types-no-outputs.deepnote' -[D 2025-09-18 11:22:56.083 ServerApp] Accepting token-authenticated request from ::1 -[D 2025-09-18 11:22:56.083 ServerApp] 200 GET /api/contents/example.ipynb?content=0&hash=0&1758187376077 (b4931ce4427142b3a2011a47bec521d2@::1) 1.80ms -[D 2025-09-18 11:22:56.083 ServerApp] Accepting token-authenticated request from ::1 -[D 2025-09-18 11:22:56.083 ServerApp] 200 GET /api/contents/example.deepnote?content=0&hash=0&1758187376077 (b4931ce4427142b3a2011a47bec521d2@::1) 2.10ms -[D 2025-09-18 11:22:56.288 ServerApp] 200 GET /api/contents/example.deepnote?type=notebook&content=1&hash=1&contentProviderId=undefined&1758187376168 (b4931ce4427142b3a2011a47bec521d2@::1) 57.88ms -[D 2025-09-18 11:22:56.296 ServerApp] 200 GET /api/contents/example.deepnote/checkpoints?1758187376293 (b4931ce4427142b3a2011a47bec521d2@::1) 1.35ms From 5e6f2356cd1a04ab4f0cbac9e4727acc127176e6 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:14:43 +0200 Subject: [PATCH 25/37] Set env var in Test the extension job Signed-off-by: Andy Jakubowski --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 75ef841..469d88b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -36,6 +36,8 @@ jobs: run: | set -eux jlpm run test + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Build the extension run: | From 0ae0e567866084689834cd8c4dbc9fdcc6792f48 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:15:16 +0200 Subject: [PATCH 26/37] Move lodash to dev deps Signed-off-by: Andy Jakubowski --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index 818c54c..11e7592 100644 --- a/package.json +++ b/package.json @@ -65,7 +65,6 @@ "@jupyterlab/services": "^7.0.0", "@jupyterlab/settingregistry": "^4.0.0", "@lumino/widgets": "^2.7.1", - "@types/lodash": "^4.17.20", "lodash": "^4.17.21", "yaml": "^2.8.1", "zod": "^4.1.11" @@ -75,6 +74,7 @@ "@jupyterlab/testutils": "^4.0.0", "@types/jest": "^29.2.0", "@types/json-schema": "^7.0.11", + "@types/lodash": "^4.17.20", "@types/react": "^18.0.26", "@types/react-addons-linked-state-mixin": "^0.14.22", "@typescript-eslint/eslint-plugin": "^6.1.0", From d7d82569c534b5b201dfa8511b56facf14b892c1 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:16:58 +0200 Subject: [PATCH 27/37] Remove pointless ternary operator Signed-off-by: Andy Jakubowski --- src/convert-deepnote-block-to-jupyter-cell.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/convert-deepnote-block-to-jupyter-cell.ts b/src/convert-deepnote-block-to-jupyter-cell.ts index a0ef666..5799004 100644 --- a/src/convert-deepnote-block-to-jupyter-cell.ts +++ b/src/convert-deepnote-block-to-jupyter-cell.ts @@ -26,10 +26,7 @@ export function convertDeepnoteBlockToJupyterCell(block: DeepnoteBlock) { const jupyterCell: ICodeCell = { cell_type: 'code', metadata: jupyterCellMetadata, - execution_count: - blockCopy.executionCount !== undefined - ? blockCopy.executionCount - : null, + execution_count: blockCopy.executionCount ?? null, outputs: blockOutputs, source }; From 8426c75f536b5ecc8519d3f97dbd8553d1c7abfc Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:21:46 +0200 Subject: [PATCH 28/37] Bump `@deepnote/blocks` version Signed-off-by: Andy Jakubowski --- package.json | 2 +- yarn.lock | 19 ++++++++++++++----- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/package.json b/package.json index 11e7592..bb47bba 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "watch:labextension": "jupyter labextension watch ." }, "dependencies": { - "@deepnote/blocks": "^1.0.0", + "@deepnote/blocks": "^1.1.0", "@jupyterlab/application": "^4.0.0", "@jupyterlab/coreutils": "^6.0.0", "@jupyterlab/notebook": "^4.4.7", diff --git a/yarn.lock b/yarn.lock index 030588c..22a4354 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1577,12 +1577,14 @@ __metadata: languageName: node linkType: hard -"@deepnote/blocks@npm:^1.0.0": - version: 1.0.0 - resolution: "@deepnote/blocks@npm:1.0.0::__archiveUrl=https%3A%2F%2Fnpm.pkg.github.com%2Fdownload%2F%40deepnote%2Fblocks%2F1.0.0%2Fbbad3593046ed93df89035400c7e7d85a4742494" +"@deepnote/blocks@npm:^1.1.0": + version: 1.1.0 + resolution: "@deepnote/blocks@npm:1.1.0::__archiveUrl=https%3A%2F%2Fnpm.pkg.github.com%2Fdownload%2F%40deepnote%2Fblocks%2F1.1.0%2Feaed628f944e73fe64a6a9fd5f93b2bcfe130b44" dependencies: ts-dedent: ^2.2.0 - checksum: 69339b4c5d5b1da33c0deb248714b1aa151f9e80f623ba3a5b16669273fd0516db6c811b59c8cec3add16506e5bcf4b87c944744aa7d7761682139d3c556a06f + yaml: ^2.8.1 + zod: ^4.1.12 + checksum: 00ceb43c10c9c50e921bb83e2cd4b2cb8316f61edc1ebc89e3588d03ebe15bd5d2f9b2af4402c15e2b9e49a7a0b5603509103b86fc0814b19f6d83a40d82ce87 languageName: node linkType: hard @@ -6810,7 +6812,7 @@ __metadata: version: 0.0.0-use.local resolution: "jupyterlab-deepnote@workspace:." dependencies: - "@deepnote/blocks": ^1.0.0 + "@deepnote/blocks": ^1.1.0 "@jupyterlab/application": ^4.0.0 "@jupyterlab/builder": ^4.0.0 "@jupyterlab/coreutils": ^6.0.0 @@ -9849,3 +9851,10 @@ __metadata: checksum: 022d59f85ebe054835fbcdc96a93c01479a64321104f846cb5644812c91e00d17ae3479f823956ec9b04e4351dd32841e1f12c567e81bc43f6e21ef5cc02ce3c languageName: node linkType: hard + +"zod@npm:^4.1.12": + version: 4.1.12 + resolution: "zod@npm:4.1.12" + checksum: 91174acc7d2ca5572ad522643474ddd60640cf6877b5d76e5d583eb25e3c4072c6f5eb92ab94f231ec5ce61c6acdfc3e0166de45fb1005b1ea54986b026b765f + languageName: node + linkType: hard From 8dbf78a010ac8d08eb2222a1b8f1038d2b624324 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:21:57 +0200 Subject: [PATCH 29/37] Set env var in Build the extension job Signed-off-by: Andy Jakubowski --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 469d88b..7274354 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -51,6 +51,8 @@ jobs: jupyter labextension list jupyter labextension list 2>&1 | grep -ie "jupyterlab-deepnote.*OK" python -m jupyterlab.browser_check + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Package the extension run: | From 44818e90b15eaa185a7b255788897346a9261408 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:23:25 +0200 Subject: [PATCH 30/37] Set env var in Package the extension job Signed-off-by: Andy Jakubowski --- .github/workflows/build.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 7274354..146472b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -61,6 +61,8 @@ jobs: pip install build python -m build pip uninstall -y "jupyterlab_deepnote" jupyterlab + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload extension packages uses: actions/upload-artifact@v4 From 1b3b3698173c3c747c13f27deb6c10f633cc9b68 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:27:49 +0200 Subject: [PATCH 31/37] Fix release CI workflow Signed-off-by: Andy Jakubowski --- .github/workflows/check-release.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 8308798..e0405ac 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -1,9 +1,9 @@ name: Check Release on: push: - branches: ["main"] + branches: ['main'] pull_request: - branches: ["*"] + branches: ['*'] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -20,8 +20,9 @@ jobs: - name: Check Release uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 with: - token: ${{ secrets.GITHUB_TOKEN }} + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload Distributions uses: actions/upload-artifact@v4 From eac52df7af4e1e78e654fb430af27e95e21842dc Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Wed, 8 Oct 2025 12:37:55 +0200 Subject: [PATCH 32/37] Set up registry for packages Signed-off-by: Andy Jakubowski --- .github/workflows/check-release.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index e0405ac..0d5bd4e 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -15,6 +15,13 @@ jobs: steps: - name: Checkout uses: actions/checkout@v4 + + - name: Configure npm registry for GitHub Packages + run: | + echo "@deepnote:registry=https://npm.pkg.github.com" >> ~/.npmrc + echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" >> ~/.npmrc + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 - name: Check Release From 765d53168689449376603c4ca8bc545ee9b2f4c1 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Thu, 9 Oct 2025 11:03:20 +0200 Subject: [PATCH 33/37] Update CI job Signed-off-by: Andy Jakubowski --- .github/workflows/check-release.yml | 7 +------ .npmrc | 1 + 2 files changed, 2 insertions(+), 6 deletions(-) create mode 100644 .npmrc diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 0d5bd4e..1a8a7fc 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -16,14 +16,9 @@ jobs: - name: Checkout uses: actions/checkout@v4 - - name: Configure npm registry for GitHub Packages - run: | - echo "@deepnote:registry=https://npm.pkg.github.com" >> ~/.npmrc - echo "//npm.pkg.github.com/:_authToken=${GITHUB_TOKEN}" >> ~/.npmrc - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 + - name: Check Release uses: jupyter-server/jupyter_releaser/.github/actions/check-release@v2 with: diff --git a/.npmrc b/.npmrc new file mode 100644 index 0000000..9349458 --- /dev/null +++ b/.npmrc @@ -0,0 +1 @@ +@deepnote:registry=https://npm.pkg.github.com \ No newline at end of file From 0caae379a26822ac42eff1158221d5775151c86f Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Thu, 9 Oct 2025 11:10:05 +0200 Subject: [PATCH 34/37] Fix typo Co-authored-by: Christoffer Artmann Signed-off-by: Andy Jakubowski --- src/transform-deepnote-yaml-to-notebook-content.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/transform-deepnote-yaml-to-notebook-content.ts b/src/transform-deepnote-yaml-to-notebook-content.ts index 17360f9..83e7fdc 100644 --- a/src/transform-deepnote-yaml-to-notebook-content.ts +++ b/src/transform-deepnote-yaml-to-notebook-content.ts @@ -17,7 +17,7 @@ export async function transformDeepnoteYamlToNotebookContent( cells: [ { ...blankCodeCell, - source: '# No notebooks found in Deepnote file.\n' + source: '# No notebooks found in the Deepnote file.\n' } ] }; From 9538a18485b107caea0c25c7003a8c9970f56982 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Thu, 9 Oct 2025 11:54:13 +0200 Subject: [PATCH 35/37] Fix CI Signed-off-by: Andy Jakubowski --- .github/workflows/check-release.yml | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 1a8a7fc..14931ce 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -5,6 +5,9 @@ on: pull_request: branches: ['*'] +env: + NODE_VERSION: 22.x + concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true @@ -16,6 +19,14 @@ jobs: - name: Checkout uses: actions/checkout@v4 + - name: Setup Node.js + uses: actions/setup-node@v5 + with: + cache: 'npm' + node-version: ${{ env.NODE_VERSION }} + registry-url: 'https://npm.pkg.github.com' + scope: '@deepnote' + - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1 @@ -25,6 +36,7 @@ jobs: token: ${{ secrets.GITHUB_TOKEN }} env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + NODE_AUTH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Upload Distributions uses: actions/upload-artifact@v4 From fe7c9f0816c182836f4b7bacd8f45640b0651d25 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Thu, 9 Oct 2025 11:54:37 +0200 Subject: [PATCH 36/37] Delete .npmrc Signed-off-by: Andy Jakubowski --- .npmrc | 1 - 1 file changed, 1 deletion(-) delete mode 100644 .npmrc diff --git a/.npmrc b/.npmrc deleted file mode 100644 index 9349458..0000000 --- a/.npmrc +++ /dev/null @@ -1 +0,0 @@ -@deepnote:registry=https://npm.pkg.github.com \ No newline at end of file From 09a1e0a3bb895b37f185eae37e415883b903c7b7 Mon Sep 17 00:00:00 2001 From: Andy Jakubowski Date: Thu, 9 Oct 2025 12:01:36 +0200 Subject: [PATCH 37/37] Enable always-auth for GitHub packages Co-authored-by: coderabbitai[bot] <136622811+coderabbitai[bot]@users.noreply.github.com> --- .github/workflows/check-release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/check-release.yml b/.github/workflows/check-release.yml index 14931ce..6527f73 100644 --- a/.github/workflows/check-release.yml +++ b/.github/workflows/check-release.yml @@ -26,7 +26,7 @@ jobs: node-version: ${{ env.NODE_VERSION }} registry-url: 'https://npm.pkg.github.com' scope: '@deepnote' - + always-auth: true - name: Base Setup uses: jupyterlab/maintainer-tools/.github/actions/base-setup@v1