diff --git a/src/notebooks/deepnote/MimeTypeProcessor.ts b/src/notebooks/deepnote/MimeTypeProcessor.ts deleted file mode 100644 index dff2330e56..0000000000 --- a/src/notebooks/deepnote/MimeTypeProcessor.ts +++ /dev/null @@ -1,175 +0,0 @@ -import { NotebookCellOutputItem } from 'vscode'; -import { parseJsonSafely, convertBase64ToUint8Array } from './dataConversionUtils'; - -export interface MimeProcessor { - canHandle(mimeType: string): boolean; - processForDeepnote(content: unknown, mimeType: string): unknown; - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null; -} - -/** - * Handles text-based MIME types - */ -export class TextMimeProcessor implements MimeProcessor { - private readonly supportedTypes = ['text/plain', 'text/html']; - - canHandle(mimeType: string): boolean { - return this.supportedTypes.includes(mimeType); - } - - processForDeepnote(content: unknown): unknown { - return typeof content === 'string' ? content : String(content); - } - - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null { - if (mimeType === 'text/plain') { - return NotebookCellOutputItem.text(content as string, 'text/plain'); - } - if (mimeType === 'text/html') { - return NotebookCellOutputItem.text(content as string, 'text/html'); - } - return null; - } -} - -/** - * Handles image MIME types - */ -export class ImageMimeProcessor implements MimeProcessor { - canHandle(mimeType: string): boolean { - return mimeType.startsWith('image/'); - } - - processForDeepnote(content: unknown, _mimeType: string): unknown { - if (content instanceof Uint8Array) { - const base64String = btoa(String.fromCharCode(...content)); - return base64String; - } - // If it's already a string (base64 or data URL), return as-is - return content; - } - - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null { - try { - let uint8Array: Uint8Array; - - if (typeof content === 'string') { - uint8Array = convertBase64ToUint8Array(content); - // Store the original base64 string for round-trip preservation - const item = new NotebookCellOutputItem(uint8Array, mimeType); - // Use a property that won't interfere with VS Code but preserves the original data - (item as NotebookCellOutputItem & { _originalBase64?: string })._originalBase64 = content; - return item; - } else if (content instanceof ArrayBuffer) { - uint8Array = new Uint8Array(content); - } else if (content instanceof Uint8Array) { - uint8Array = content; - } else { - return null; - } - - return new NotebookCellOutputItem(uint8Array, mimeType); - } catch { - return NotebookCellOutputItem.text(String(content), mimeType); - } - } -} - -/** - * Handles JSON MIME types - */ -export class JsonMimeProcessor implements MimeProcessor { - canHandle(mimeType: string): boolean { - return mimeType === 'application/json'; - } - - processForDeepnote(content: unknown): unknown { - if (typeof content === 'string') { - return parseJsonSafely(content); - } - return content; - } - - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null { - try { - let jsonObject: unknown; - - if (typeof content === 'string') { - jsonObject = JSON.parse(content); - } else if (typeof content === 'object' && content !== null) { - jsonObject = content; - } else { - return NotebookCellOutputItem.text(String(content), mimeType); - } - - return NotebookCellOutputItem.text(JSON.stringify(jsonObject, null, 2), mimeType); - } catch { - return NotebookCellOutputItem.text(String(content), mimeType); - } - } -} - -/** - * Handles other application MIME types - */ -export class ApplicationMimeProcessor implements MimeProcessor { - canHandle(mimeType: string): boolean { - return mimeType.startsWith('application/') && mimeType !== 'application/json'; - } - - processForDeepnote(content: unknown): unknown { - if (typeof content === 'string') { - return parseJsonSafely(content); - } - return content; - } - - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null { - const textContent = typeof content === 'string' ? content : JSON.stringify(content, null, 2); - return NotebookCellOutputItem.text(textContent, mimeType); - } -} - -/** - * Generic fallback processor - */ -export class GenericMimeProcessor implements MimeProcessor { - canHandle(): boolean { - return true; // Always can handle as fallback - } - - processForDeepnote(content: unknown): unknown { - return content; - } - - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null { - return NotebookCellOutputItem.text(String(content), mimeType); - } -} - -/** - * Registry for MIME type processors - */ -export class MimeTypeProcessorRegistry { - private readonly processors: MimeProcessor[] = [ - new TextMimeProcessor(), - new ImageMimeProcessor(), - new JsonMimeProcessor(), - new ApplicationMimeProcessor(), - new GenericMimeProcessor() // Must be last as fallback - ]; - - getProcessor(mimeType: string): MimeProcessor { - return this.processors.find((processor) => processor.canHandle(mimeType)) || new GenericMimeProcessor(); - } - - processForDeepnote(content: unknown, mimeType: string): unknown { - const processor = this.getProcessor(mimeType); - return processor.processForDeepnote(content, mimeType); - } - - processForVSCode(content: unknown, mimeType: string): NotebookCellOutputItem | null { - const processor = this.getProcessor(mimeType); - return processor.processForVSCode(content, mimeType); - } -} diff --git a/src/notebooks/deepnote/OutputTypeDetector.ts b/src/notebooks/deepnote/OutputTypeDetector.ts deleted file mode 100644 index d69f8a9dc8..0000000000 --- a/src/notebooks/deepnote/OutputTypeDetector.ts +++ /dev/null @@ -1,60 +0,0 @@ -import { NotebookCellOutput } from 'vscode'; - -export type DetectedOutputType = 'error' | 'stream' | 'rich'; - -export interface OutputTypeResult { - type: DetectedOutputType; - streamMimes?: string[]; - errorItem?: { mime: string; data: Uint8Array }; -} - -/** - * Detects the appropriate output type from VS Code NotebookCellOutput items - */ -export class OutputTypeDetector { - private readonly streamMimes = [ - 'text/plain', - 'application/vnd.code.notebook.stdout', - 'application/vnd.code.notebook.stderr' - ]; - - detect(output: NotebookCellOutput): OutputTypeResult { - if (output.items.length === 0) { - return { type: 'rich' }; - } - - // Check for error output first - const errorItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.error'); - if (errorItem) { - return { - type: 'error', - errorItem: { mime: errorItem.mime, data: errorItem.data } - }; - } - - // Check for stream outputs - only if ALL items are stream mimes - // or if it contains stdout/stderr specific mimes - const hasStdoutStderr = output.items.some( - (item) => - item.mime === 'application/vnd.code.notebook.stdout' || - item.mime === 'application/vnd.code.notebook.stderr' - ); - - const allItemsAreStream = output.items.every((item) => this.streamMimes.includes(item.mime)); - - if (hasStdoutStderr || (allItemsAreStream && output.items.length === 1)) { - const streamItems = output.items.filter((item) => this.streamMimes.includes(item.mime)); - return { - type: 'stream', - streamMimes: streamItems.map((item) => item.mime) - }; - } - - // Default to rich output - return { type: 'rich' }; - } - - isStreamMime(mimeType: string): boolean { - return this.streamMimes.includes(mimeType); - } -} diff --git a/src/notebooks/deepnote/converters/blockConverter.ts b/src/notebooks/deepnote/converters/blockConverter.ts new file mode 100644 index 0000000000..58aa8d52ec --- /dev/null +++ b/src/notebooks/deepnote/converters/blockConverter.ts @@ -0,0 +1,10 @@ +import type { NotebookCellData } from 'vscode'; + +import type { DeepnoteBlock } from '../deepnoteTypes'; + +export interface BlockConverter { + applyChangesToBlock(block: DeepnoteBlock, cell: NotebookCellData): void; + canConvert(blockType: string): boolean; + convertToCell(block: DeepnoteBlock): NotebookCellData; + getSupportedTypes(): string[]; +} diff --git a/src/notebooks/deepnote/converters/codeBlockConverter.ts b/src/notebooks/deepnote/converters/codeBlockConverter.ts new file mode 100644 index 0000000000..ab412b5d94 --- /dev/null +++ b/src/notebooks/deepnote/converters/codeBlockConverter.ts @@ -0,0 +1,24 @@ +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import type { BlockConverter } from './blockConverter'; +import type { DeepnoteBlock } from '../deepnoteTypes'; + +export class CodeBlockConverter implements BlockConverter { + applyChangesToBlock(block: DeepnoteBlock, cell: NotebookCellData): void { + block.content = cell.value || ''; + } + + canConvert(blockType: string): boolean { + return blockType.toLowerCase() === 'code'; + } + + convertToCell(block: DeepnoteBlock): NotebookCellData { + const cell = new NotebookCellData(NotebookCellKind.Code, block.content || '', 'python'); + + return cell; + } + + getSupportedTypes(): string[] { + return ['code']; + } +} diff --git a/src/notebooks/deepnote/converters/codeBlockConverter.unit.test.ts b/src/notebooks/deepnote/converters/codeBlockConverter.unit.test.ts new file mode 100644 index 0000000000..55efcd3a23 --- /dev/null +++ b/src/notebooks/deepnote/converters/codeBlockConverter.unit.test.ts @@ -0,0 +1,146 @@ +import { assert } from 'chai'; +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import type { DeepnoteBlock } from '../deepnoteTypes'; +import { CodeBlockConverter } from './codeBlockConverter'; + +suite('CodeBlockConverter', () => { + let converter: CodeBlockConverter; + + setup(() => { + converter = new CodeBlockConverter(); + }); + + suite('canConvert', () => { + test('returns true for code type', () => { + assert.isTrue(converter.canConvert('code')); + }); + + test('canConvert ignores case', () => { + assert.isTrue(converter.canConvert('CODE')); + assert.isTrue(converter.canConvert('Code')); + assert.isTrue(converter.canConvert('CoDe')); + }); + + test('returns false for non-code types', () => { + assert.isFalse(converter.canConvert('markdown')); + assert.isFalse(converter.canConvert('text-cell-h1')); + assert.isFalse(converter.canConvert('unknown')); + }); + }); + + suite('getSupportedTypes', () => { + test('returns array with code type', () => { + const types = converter.getSupportedTypes(); + + assert.deepStrictEqual(types, ['code']); + }); + }); + + suite('convertToCell', () => { + test('converts code block to cell', () => { + const block: DeepnoteBlock = { + content: 'print("hello")', + id: 'block-123', + sortingKey: 'a0', + type: 'code' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.kind, NotebookCellKind.Code); + assert.strictEqual(cell.value, 'print("hello")'); + assert.strictEqual(cell.languageId, 'python'); + }); + + test('handles empty content', () => { + const block: DeepnoteBlock = { + content: '', + id: 'block-123', + sortingKey: 'a0', + type: 'code' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, ''); + }); + + test('handles missing content', () => { + const block = { + id: 'block-123', + sortingKey: 'a0', + type: 'code' + } as DeepnoteBlock; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, ''); + }); + + test('handles multi-line content', () => { + const block: DeepnoteBlock = { + content: 'import numpy as np\nimport pandas as pd\n\nprint("hello")', + id: 'block-123', + sortingKey: 'a0', + type: 'code' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, 'import numpy as np\nimport pandas as pd\n\nprint("hello")'); + }); + }); + + suite('applyChangesToBlock', () => { + test('applies cell content to block', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'code' + }; + const cell = new NotebookCellData(NotebookCellKind.Code, 'new content', 'python'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'new content'); + }); + + test('handles empty cell value', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'code' + }; + const cell = new NotebookCellData(NotebookCellKind.Code, '', 'python'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, ''); + }); + + test('does not modify other block properties', () => { + const block: DeepnoteBlock = { + content: 'old content', + executionCount: 5, + id: 'block-123', + metadata: { custom: 'value' }, + outputs: [], + sortingKey: 'a0', + type: 'code' + }; + const cell = new NotebookCellData(NotebookCellKind.Code, 'new content', 'python'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'new content'); + assert.strictEqual(block.id, 'block-123'); + assert.strictEqual(block.type, 'code'); + assert.strictEqual(block.sortingKey, 'a0'); + assert.strictEqual(block.executionCount, 5); + assert.deepStrictEqual(block.metadata, { custom: 'value' }); + }); + }); +}); diff --git a/src/notebooks/deepnote/converters/converterRegistry.ts b/src/notebooks/deepnote/converters/converterRegistry.ts new file mode 100644 index 0000000000..aa060cf714 --- /dev/null +++ b/src/notebooks/deepnote/converters/converterRegistry.ts @@ -0,0 +1,19 @@ +import type { BlockConverter } from './blockConverter'; + +export class ConverterRegistry { + private readonly typeToConverterMap: Map = new Map(); + + findConverter(blockType: string): BlockConverter | undefined { + return this.typeToConverterMap.get(blockType.toLowerCase()); + } + + listSupportedTypes(): string[] { + return Array.from(this.typeToConverterMap.keys()).sort(); + } + + register(converter: BlockConverter): void { + converter.getSupportedTypes().forEach((type) => { + this.typeToConverterMap.set(type, converter); + }); + } +} diff --git a/src/notebooks/deepnote/converters/markdownBlockConverter.ts b/src/notebooks/deepnote/converters/markdownBlockConverter.ts new file mode 100644 index 0000000000..fca485cb4a --- /dev/null +++ b/src/notebooks/deepnote/converters/markdownBlockConverter.ts @@ -0,0 +1,24 @@ +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import type { BlockConverter } from './blockConverter'; +import type { DeepnoteBlock } from '../deepnoteTypes'; + +export class MarkdownBlockConverter implements BlockConverter { + applyChangesToBlock(block: DeepnoteBlock, cell: NotebookCellData): void { + block.content = cell.value || ''; + } + + canConvert(blockType: string): boolean { + return blockType.toLowerCase() === 'markdown'; + } + + convertToCell(block: DeepnoteBlock): NotebookCellData { + const cell = new NotebookCellData(NotebookCellKind.Markup, block.content || '', 'markdown'); + + return cell; + } + + getSupportedTypes(): string[] { + return ['markdown']; + } +} diff --git a/src/notebooks/deepnote/converters/markdownBlockConverter.unit.test.ts b/src/notebooks/deepnote/converters/markdownBlockConverter.unit.test.ts new file mode 100644 index 0000000000..12228bca9c --- /dev/null +++ b/src/notebooks/deepnote/converters/markdownBlockConverter.unit.test.ts @@ -0,0 +1,161 @@ +import { assert } from 'chai'; +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import type { DeepnoteBlock } from '../deepnoteTypes'; +import { MarkdownBlockConverter } from './markdownBlockConverter'; + +suite('MarkdownBlockConverter', () => { + let converter: MarkdownBlockConverter; + + setup(() => { + converter = new MarkdownBlockConverter(); + }); + + suite('canConvert', () => { + test('returns true for markdown type', () => { + assert.isTrue(converter.canConvert('markdown')); + }); + + test('canConvert ignores case', () => { + assert.isTrue(converter.canConvert('MARKDOWN')); + assert.isTrue(converter.canConvert('Markdown')); + assert.isTrue(converter.canConvert('MaRkDoWn')); + }); + + test('returns false for non-markdown types', () => { + assert.isFalse(converter.canConvert('code')); + assert.isFalse(converter.canConvert('text-cell-h1')); + assert.isFalse(converter.canConvert('unknown')); + }); + }); + + suite('getSupportedTypes', () => { + test('returns array with markdown type', () => { + const types = converter.getSupportedTypes(); + + assert.deepStrictEqual(types, ['markdown']); + }); + }); + + suite('convertToCell', () => { + test('converts markdown block to cell', () => { + const block: DeepnoteBlock = { + content: '# Title\n\nParagraph text', + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.kind, NotebookCellKind.Markup); + assert.strictEqual(cell.value, '# Title\n\nParagraph text'); + assert.strictEqual(cell.languageId, 'markdown'); + }); + + test('handles empty content', () => { + const block: DeepnoteBlock = { + content: '', + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, ''); + }); + + test('handles missing content', () => { + const block = { + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + } as DeepnoteBlock; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, ''); + }); + + test('handles complex markdown', () => { + const block: DeepnoteBlock = { + content: '# Title\n\n## Subtitle\n\n- List item 1\n- List item 2\n\n```python\nprint("code")\n```', + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual( + cell.value, + '# Title\n\n## Subtitle\n\n- List item 1\n- List item 2\n\n```python\nprint("code")\n```' + ); + }); + }); + + suite('applyChangesToBlock', () => { + test('applies cell content to block', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '# New Content', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, '# New Content'); + }); + + test('handles empty cell value', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, ''); + }); + + test('does not modify other block properties', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + metadata: { custom: 'value' }, + sortingKey: 'a0', + type: 'markdown' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, 'new content', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'new content'); + assert.strictEqual(block.id, 'block-123'); + assert.strictEqual(block.type, 'markdown'); + assert.strictEqual(block.sortingKey, 'a0'); + assert.deepStrictEqual(block.metadata, { custom: 'value' }); + }); + + test('handles complex markdown content', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'markdown' + }; + const newContent = '# Title\n\n## Subtitle\n\n- Item 1\n- Item 2\n\n**Bold** and *italic*'; + const cell = new NotebookCellData(NotebookCellKind.Markup, newContent, 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, newContent); + }); + }); +}); diff --git a/src/notebooks/deepnote/converters/textBlockConverter.ts b/src/notebooks/deepnote/converters/textBlockConverter.ts new file mode 100644 index 0000000000..5c650e0396 --- /dev/null +++ b/src/notebooks/deepnote/converters/textBlockConverter.ts @@ -0,0 +1,48 @@ +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import type { BlockConverter } from './blockConverter'; +import type { DeepnoteBlock } from '../deepnoteTypes'; + +export class TextBlockConverter implements BlockConverter { + protected static readonly textBlockTypes = ['text-cell-h1', 'text-cell-h2', 'text-cell-h3', 'text-cell-p']; + + applyChangesToBlock(block: DeepnoteBlock, cell: NotebookCellData): void { + let value = cell.value || ''; + + if (block.type === 'text-cell-h1') { + value = value.replace(/^\s*#\s+/, ''); + } else if (block.type === 'text-cell-h2') { + value = value.replace(/^\s*##\s+/, ''); + } else if (block.type === 'text-cell-h3') { + value = value.replace(/^\s*###\s+/, ''); + } + + block.content = value; + } + + canConvert(blockType: string): boolean { + return TextBlockConverter.textBlockTypes.includes(blockType.toLowerCase()); + } + + convertToCell(block: DeepnoteBlock): NotebookCellData { + // TODO: Use the library to handle the markdown conversion here in the future. + + let value = block.content || ''; + + if (block.type === 'text-cell-h1') { + value = `# ${value}`; + } else if (block.type === 'text-cell-h2') { + value = `## ${value}`; + } else if (block.type === 'text-cell-h3') { + value = `### ${value}`; + } + + const cell = new NotebookCellData(NotebookCellKind.Markup, value, 'markdown'); + + return cell; + } + + getSupportedTypes(): string[] { + return [...TextBlockConverter.textBlockTypes]; + } +} diff --git a/src/notebooks/deepnote/converters/textBlockConverter.unit.test.ts b/src/notebooks/deepnote/converters/textBlockConverter.unit.test.ts new file mode 100644 index 0000000000..f0f7dfaf9e --- /dev/null +++ b/src/notebooks/deepnote/converters/textBlockConverter.unit.test.ts @@ -0,0 +1,269 @@ +import { assert } from 'chai'; +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import type { DeepnoteBlock } from '../deepnoteTypes'; +import { TextBlockConverter } from './textBlockConverter'; + +suite('TextBlockConverter', () => { + let converter: TextBlockConverter; + + setup(() => { + converter = new TextBlockConverter(); + }); + + suite('canConvert', () => { + test('returns true for text-cell types', () => { + assert.isTrue(converter.canConvert('text-cell-h1')); + assert.isTrue(converter.canConvert('text-cell-h2')); + assert.isTrue(converter.canConvert('text-cell-h3')); + assert.isTrue(converter.canConvert('text-cell-p')); + }); + + test('is case insensitive', () => { + assert.isTrue(converter.canConvert('TEXT-CELL-H1')); + assert.isTrue(converter.canConvert('Text-Cell-H2')); + }); + + test('returns false for non-text-cell types', () => { + assert.isFalse(converter.canConvert('code')); + assert.isFalse(converter.canConvert('markdown')); + assert.isFalse(converter.canConvert('unknown')); + }); + }); + + suite('getSupportedTypes', () => { + test('returns array of text-cell types', () => { + const types = converter.getSupportedTypes(); + + assert.deepStrictEqual(types, ['text-cell-h1', 'text-cell-h2', 'text-cell-h3', 'text-cell-p']); + }); + }); + + suite('convertToCell', () => { + test('converts text-cell-h1 to cell with # prefix', () => { + const block: DeepnoteBlock = { + content: 'Main Title', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h1' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.kind, NotebookCellKind.Markup); + assert.strictEqual(cell.value, '# Main Title'); + assert.strictEqual(cell.languageId, 'markdown'); + }); + + test('converts text-cell-h2 to cell with ## prefix', () => { + const block: DeepnoteBlock = { + content: 'Section Title', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h2' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, '## Section Title'); + }); + + test('converts text-cell-h3 to cell with ### prefix', () => { + const block: DeepnoteBlock = { + content: 'Subsection Title', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h3' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, '### Subsection Title'); + }); + + test('converts text-cell-p to cell without prefix', () => { + const block: DeepnoteBlock = { + content: 'Paragraph text', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-p' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, 'Paragraph text'); + }); + + test('handles empty content', () => { + const block: DeepnoteBlock = { + content: '', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h1' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, '# '); + }); + + test('handles missing content', () => { + const block = { + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h2' + } as DeepnoteBlock; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.value, '## '); + }); + }); + + suite('applyChangesToBlock', () => { + test('strips # prefix from h1 cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h1' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '# New Title', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Title'); + }); + + test('strips # prefix with leading whitespace from h1 cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h1' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, ' # New Title', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Title'); + }); + + test('strips ## prefix from h2 cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h2' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '## New Section', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Section'); + }); + + test('strips ## prefix with leading whitespace from h2 cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h2' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, ' ## New Section', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Section'); + }); + + test('strips ### prefix from h3 cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h3' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '### New Subsection', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Subsection'); + }); + + test('strips ### prefix with leading whitespace from h3 cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h3' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '\t### New Subsection', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Subsection'); + }); + + test('does not strip prefix from paragraph cell', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-p' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '# Not a heading', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, '# Not a heading'); + }); + + test('handles content without expected prefix', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h1' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, 'No prefix', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'No prefix'); + }); + + test('handles empty cell value', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + sortingKey: 'a0', + type: 'text-cell-h1' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, ''); + }); + + test('does not modify other block properties', () => { + const block: DeepnoteBlock = { + content: 'old content', + id: 'block-123', + metadata: { custom: 'value' }, + sortingKey: 'a0', + type: 'text-cell-h1' + }; + const cell = new NotebookCellData(NotebookCellKind.Markup, '# New Title', 'markdown'); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'New Title'); + assert.strictEqual(block.id, 'block-123'); + assert.strictEqual(block.type, 'text-cell-h1'); + assert.strictEqual(block.sortingKey, 'a0'); + assert.deepStrictEqual(block.metadata, { custom: 'value' }); + }); + }); +}); diff --git a/src/notebooks/deepnote/dataConversionUtils.ts b/src/notebooks/deepnote/dataConversionUtils.ts index cd21fa8770..a3a82a14e6 100644 --- a/src/notebooks/deepnote/dataConversionUtils.ts +++ b/src/notebooks/deepnote/dataConversionUtils.ts @@ -1,74 +1,9 @@ /** - * Utility functions for data transformation in Deepnote conversion + * Utility functions for Deepnote block ID and sorting key generation */ /** - * Safely decode content using TextDecoder - */ -export function decodeContent(data: Uint8Array): string { - return new TextDecoder().decode(data); -} - -/** - * Safely parse JSON with fallback to original content - */ -export function parseJsonSafely(content: string): unknown { - try { - return JSON.parse(content); - } catch { - return content; - } -} - -/** - * Convert base64 string to Uint8Array - */ -export function convertBase64ToUint8Array(base64Content: string): Uint8Array { - const base64Data = base64Content.includes(',') ? base64Content.split(',')[1] : base64Content; - const binaryString = atob(base64Data); - const uint8Array = new Uint8Array(binaryString.length); - for (let i = 0; i < binaryString.length; i++) { - uint8Array[i] = binaryString.charCodeAt(i); - } - return uint8Array; -} - -/** - * Convert Uint8Array to base64 data URL - */ -export function convertUint8ArrayToBase64DataUrl(data: Uint8Array, mimeType: string): string { - const base64String = btoa(String.fromCharCode(...data)); - return `data:${mimeType};base64,${base64String}`; -} - -/** - * Merge metadata objects, filtering out undefined values - */ -export function mergeMetadata(...metadataObjects: (Record | undefined)[]): Record { - const result: Record = {}; - - for (const metadata of metadataObjects) { - if (metadata) { - Object.entries(metadata).forEach(([key, value]) => { - if (value !== undefined) { - result[key] = value; - } - }); - } - } - - return result; -} - -/** - * Check if metadata object has any content - */ -export function hasMetadataContent(metadata: Record): boolean { - return Object.keys(metadata).length > 0; -} - -/** - * Generate a random hex ID for blocks + * Generate a random hex ID for blocks (32 character hex string) */ export function generateBlockId(): string { const chars = '0123456789abcdef'; @@ -80,7 +15,7 @@ export function generateBlockId(): string { } /** - * Generate sorting key based on index + * Generate sorting key based on index (format: a0, a1, ..., a99, b0, b1, ...) */ export function generateSortingKey(index: number): string { const alphabet = 'abcdefghijklmnopqrstuvwxyz'; diff --git a/src/notebooks/deepnote/deepnoteDataConverter.ts b/src/notebooks/deepnote/deepnoteDataConverter.ts index 506468ed11..658f564b36 100644 --- a/src/notebooks/deepnote/deepnoteDataConverter.ts +++ b/src/notebooks/deepnote/deepnoteDataConverter.ts @@ -1,21 +1,25 @@ import { NotebookCellData, NotebookCellKind, NotebookCellOutput, NotebookCellOutputItem } from 'vscode'; import type { DeepnoteBlock, DeepnoteOutput } from './deepnoteTypes'; -import { OutputTypeDetector } from './OutputTypeDetector'; -import { StreamOutputHandler } from './outputHandlers/StreamOutputHandler'; -import { ErrorOutputHandler } from './outputHandlers/ErrorOutputHandler'; -import { RichOutputHandler } from './outputHandlers/RichOutputHandler'; -import { mergeMetadata, hasMetadataContent, generateBlockId, generateSortingKey } from './dataConversionUtils'; +import { generateBlockId, generateSortingKey } from './dataConversionUtils'; +import { ConverterRegistry } from './converters/converterRegistry'; +import { CodeBlockConverter } from './converters/codeBlockConverter'; +import { addPocketToCellMetadata, createBlockFromPocket } from './pocket'; +import { TextBlockConverter } from './converters/textBlockConverter'; +import { MarkdownBlockConverter } from './converters/markdownBlockConverter'; /** * Utility class for converting between Deepnote block structures and VS Code notebook cells. * Handles bidirectional conversion while preserving metadata and execution state. */ export class DeepnoteDataConverter { - private readonly outputDetector = new OutputTypeDetector(); - private readonly streamHandler = new StreamOutputHandler(); - private readonly errorHandler = new ErrorOutputHandler(); - private readonly richHandler = new RichOutputHandler(); + private readonly registry = new ConverterRegistry(); + + constructor() { + this.registry.register(new CodeBlockConverter()); + this.registry.register(new TextBlockConverter()); + this.registry.register(new MarkdownBlockConverter()); + } /** * Converts Deepnote blocks to VS Code notebook cells. @@ -24,7 +28,36 @@ export class DeepnoteDataConverter { * @returns Array of VS Code notebook cell data */ convertBlocksToCells(blocks: DeepnoteBlock[]): NotebookCellData[] { - return blocks.map((block) => this.convertBlockToCell(block)); + return blocks.map((block) => { + const converter = this.registry.findConverter(block.type); + + if (!converter) { + // Fallback for unknown types - convert to markdown + console.warn(`Unknown block type: ${block.type}, converting to markdown`); + return this.createFallbackCell(block); + } + + const cell = converter.convertToCell(block); + + const blockWithOptionalFields = block as DeepnoteBlock & { blockGroup?: string }; + + cell.metadata = { + ...block.metadata, + id: block.id, + type: block.type, + sortingKey: block.sortingKey, + ...(blockWithOptionalFields.blockGroup && { blockGroup: blockWithOptionalFields.blockGroup }), + ...(block.executionCount !== undefined && { executionCount: block.executionCount }), + ...(block.outputs !== undefined && { outputs: block.outputs }) + }; + + // The pocket is a place to tuck away Deepnote-specific fields for later. + addPocketToCellMetadata(cell); + + cell.outputs = this.transformOutputsForVsCode(block.outputs || []); + + return cell; + }); } /** @@ -34,149 +67,255 @@ export class DeepnoteDataConverter { * @returns Array of Deepnote blocks */ convertCellsToBlocks(cells: NotebookCellData[]): DeepnoteBlock[] { - return cells.map((cell, index) => this.convertCellToBlock(cell, index)); + return cells.map((cell, index) => { + const block = createBlockFromPocket(cell, index); + + const converter = this.registry.findConverter(block.type); + + if (!converter) { + return this.createFallbackBlock(cell, index); + } + + converter.applyChangesToBlock(block, cell); + + // If pocket didn't have outputs, but cell does, convert VS Code outputs to Deepnote format + if (!block.outputs && cell.outputs && cell.outputs.length > 0) { + block.outputs = this.transformOutputsForDeepnote(cell.outputs); + } + + return block; + }); } - private convertBlockToCell(block: DeepnoteBlock): NotebookCellData { - const cellKind = block.type === 'code' ? NotebookCellKind.Code : NotebookCellKind.Markup; - const source = block.content || ''; + private base64ToUint8Array(base64: string): Uint8Array { + const binaryString = atob(base64); + const bytes = new Uint8Array(binaryString.length); - // Create the cell with proper language ID - let cell: NotebookCellData; - if (block.type === 'code') { - cell = new NotebookCellData(cellKind, source, 'python'); - } else { - cell = new NotebookCellData(cellKind, source, 'markdown'); + for (let i = 0; i < binaryString.length; i++) { + bytes[i] = binaryString.charCodeAt(i); } - // Set metadata after creation + return bytes; + } + + private createFallbackBlock(cell: NotebookCellData, index: number): DeepnoteBlock { + return { + id: generateBlockId(), + sortingKey: generateSortingKey(index), + type: cell.kind === NotebookCellKind.Code ? 'code' : 'markdown', + content: cell.value || '' + }; + } + + private createFallbackCell(block: DeepnoteBlock): NotebookCellData { + const cell = new NotebookCellData(NotebookCellKind.Markup, block.content || '', 'markdown'); + cell.metadata = { deepnoteBlockId: block.id, deepnoteBlockType: block.type, deepnoteSortingKey: block.sortingKey, - deepnoteMetadata: block.metadata, - ...(typeof block.executionCount === 'number' && { executionCount: block.executionCount }), - ...(block.outputReference && { deepnoteOutputReference: block.outputReference }) + deepnoteMetadata: block.metadata }; - // Only set outputs if they exist - if (block.outputs && block.outputs.length > 0) { - cell.outputs = this.convertDeepnoteOutputsToVSCodeOutputs(block.outputs); - } else { - cell.outputs = []; - } - return cell; } - private convertCellToBlock(cell: NotebookCellData, index: number): DeepnoteBlock { - const blockId = cell.metadata?.deepnoteBlockId || generateBlockId(); - const sortingKey = cell.metadata?.deepnoteSortingKey || generateSortingKey(index); - const originalMetadata = cell.metadata?.deepnoteMetadata; + private transformOutputsForDeepnote(outputs: NotebookCellOutput[]): DeepnoteOutput[] { + return outputs.map((output) => { + // Check if this is an error output + const errorItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.error'); - const block: DeepnoteBlock = { - id: blockId, - sortingKey: sortingKey, - type: cell.kind === NotebookCellKind.Code ? 'code' : 'markdown', - content: cell.value || '' - }; + if (errorItem) { + try { + const errorData = JSON.parse(new TextDecoder().decode(errorItem.data)); - // Only add metadata if it exists and is not empty - if (originalMetadata && Object.keys(originalMetadata).length > 0) { - block.metadata = originalMetadata; - } + return { + ename: errorData.name || 'Error', + evalue: errorData.message || '', + output_type: 'error', + traceback: errorData.stack ? errorData.stack.split('\n') : [] + } as DeepnoteOutput; + } catch { + return { + ename: 'Error', + evalue: '', + output_type: 'error', + traceback: [] + } as DeepnoteOutput; + } + } + + // Check if this is a stream output + const stdoutItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.stdout'); + const stderrItem = output.items.find((item) => item.mime === 'application/vnd.code.notebook.stderr'); - if (cell.kind === NotebookCellKind.Code) { - const executionCount = cell.metadata?.executionCount ?? cell.executionSummary?.executionOrder; + if (stdoutItem || stderrItem) { + const item = stdoutItem || stderrItem; + const text = new TextDecoder().decode(item!.data); - if (executionCount !== undefined) { - block.executionCount = executionCount; + return { + name: stderrItem ? 'stderr' : 'stdout', + output_type: 'stream', + text + } as DeepnoteOutput; } - } - if (cell.metadata?.deepnoteOutputReference) { - block.outputReference = cell.metadata.deepnoteOutputReference; - } + // Rich output (execute_result or display_data) + const data: Record = {}; - if (cell.outputs && cell.outputs.length > 0) { - block.outputs = this.convertVSCodeOutputsToDeepnoteOutputs(cell.outputs); - } + for (const item of output.items) { + if (item.mime === 'text/plain') { + data['text/plain'] = new TextDecoder().decode(item.data); + } else if (item.mime === 'text/html') { + data['text/html'] = new TextDecoder().decode(item.data); + } else if (item.mime === 'application/json') { + data['application/json'] = JSON.parse(new TextDecoder().decode(item.data)); + } else if (item.mime === 'image/png') { + data['image/png'] = btoa(String.fromCharCode(...new Uint8Array(item.data))); + } else if (item.mime === 'image/jpeg') { + data['image/jpeg'] = btoa(String.fromCharCode(...new Uint8Array(item.data))); + } else if (item.mime === 'application/vnd.deepnote.dataframe.v3+json') { + data['application/vnd.deepnote.dataframe.v3+json'] = JSON.parse( + new TextDecoder().decode(item.data) + ); + } + } - return block; - } + const deepnoteOutput: DeepnoteOutput = { + data, + execution_count: (output.metadata?.executionCount as number) || 0, + output_type: 'execute_result' + }; + + // Add metadata if present (excluding executionCount which we already handled) + if (output.metadata) { + // eslint-disable-next-line @typescript-eslint/no-unused-vars + const { executionCount, ...restMetadata } = output.metadata; - private convertDeepnoteOutputsToVSCodeOutputs(deepnoteOutputs: DeepnoteOutput[]): NotebookCellOutput[] { - return deepnoteOutputs.map((output) => this.convertSingleOutput(output)); + if (Object.keys(restMetadata).length > 0) { + (deepnoteOutput as DeepnoteOutput & { metadata?: Record }).metadata = restMetadata; + } + } + + return deepnoteOutput; + }); } - private convertSingleOutput(output: DeepnoteOutput): NotebookCellOutput { - const outputItems = this.createOutputItems(output); + private transformOutputsForVsCode(outputs: DeepnoteOutput[]): NotebookCellOutput[] { + return outputs.map((output) => { + if ('output_type' in output) { + if (output.output_type === 'error') { + const errorOutput = output as { ename?: string; evalue?: string; traceback?: string[] }; + const error = { + name: errorOutput.ename || 'Error', + message: errorOutput.evalue || '', + stack: errorOutput.traceback ? errorOutput.traceback.join('\n') : '' + }; - const metadata = mergeMetadata( - output.metadata, - output.execution_count !== undefined ? { executionCount: output.execution_count } : undefined - ); + return new NotebookCellOutput([NotebookCellOutputItem.error(error)]); + } - return hasMetadataContent(metadata) - ? new NotebookCellOutput(outputItems, metadata) - : new NotebookCellOutput(outputItems); - } + if (output.output_type === 'execute_result' || output.output_type === 'display_data') { + const items: NotebookCellOutputItem[] = []; - private convertVSCodeOutputsToDeepnoteOutputs(vscodeOutputs: NotebookCellOutput[]): DeepnoteOutput[] { - return vscodeOutputs.map((output) => this.convertVSCodeSingleOutput(output)); - } + // Handle text fallback if data is not present + if (!output.data && 'text' in output && output.text) { + items.push(NotebookCellOutputItem.text(String(output.text), 'text/plain')); + } else if (output.data && typeof output.data === 'object') { + const data = output.data as Record; - private convertVSCodeSingleOutput(output: NotebookCellOutput): DeepnoteOutput { - // Detect output type and delegate to appropriate handler - const detection = this.outputDetector.detect(output); - let deepnoteOutput: DeepnoteOutput; - - switch (detection.type) { - case 'error': - deepnoteOutput = this.errorHandler.convertToDeepnote(detection.errorItem!); - break; - case 'stream': - deepnoteOutput = this.streamHandler.convertToDeepnote(output); - break; - case 'rich': - default: - deepnoteOutput = this.richHandler.convertToDeepnote(output); - break; - } + // Order matters! Rich formats first, text/plain last + if (data['text/html']) { + items.push( + new NotebookCellOutputItem( + new TextEncoder().encode(data['text/html'] as string), + 'text/html' + ) + ); + } - // Preserve metadata from VS Code output - if (output.metadata) { - // Extract execution count from metadata before merging - const { executionCount, ...restMetadata } = output.metadata; + if (data['application/vnd.deepnote.dataframe.v3+json']) { + items.push( + NotebookCellOutputItem.json( + data['application/vnd.deepnote.dataframe.v3+json'], + 'application/vnd.deepnote.dataframe.v3+json' + ) + ); + } - if (executionCount !== undefined && deepnoteOutput.execution_count === undefined) { - deepnoteOutput.execution_count = executionCount as number; - } + if (data['application/json']) { + items.push(NotebookCellOutputItem.json(data['application/json'], 'application/json')); + } - // Only merge non-executionCount metadata - if (Object.keys(restMetadata).length > 0) { - deepnoteOutput.metadata = mergeMetadata(deepnoteOutput.metadata, restMetadata); - } - } + // Images (base64 encoded) + if (data['image/png']) { + items.push( + new NotebookCellOutputItem( + this.base64ToUint8Array(data['image/png'] as string), + 'image/png' + ) + ); + } - return deepnoteOutput; - } + if (data['image/jpeg']) { + items.push( + new NotebookCellOutputItem( + this.base64ToUint8Array(data['image/jpeg'] as string), + 'image/jpeg' + ) + ); + } + + // Plain text as fallback (always last) + if (data['text/plain']) { + items.push(NotebookCellOutputItem.text(data['text/plain'] as string)); + } + } + + // Preserve metadata and execution_count + const metadata: Record = {}; + + if (output.execution_count !== undefined) { + metadata.executionCount = output.execution_count; + } + + if ('metadata' in output && output.metadata) { + Object.assign(metadata, output.metadata); + } - private createOutputItems(output: DeepnoteOutput): NotebookCellOutputItem[] { - switch (output.output_type) { - case 'stream': - return this.streamHandler.convertToVSCode(output); - case 'error': - return this.errorHandler.convertToVSCode(output); - case 'execute_result': - case 'display_data': - return this.richHandler.convertToVSCode(output); - default: - // Fallback for unknown types with text - if (output.text) { - return [NotebookCellOutputItem.text(output.text)]; + return Object.keys(metadata).length > 0 + ? new NotebookCellOutput(items, metadata) + : new NotebookCellOutput(items); } - return []; - } + + if (output.output_type === 'stream') { + if (!output.text) { + return new NotebookCellOutput([]); + } + + const mimeType = + 'name' in output && output.name === 'stderr' + ? 'application/vnd.code.notebook.stderr' + : 'application/vnd.code.notebook.stdout'; + + return new NotebookCellOutput([NotebookCellOutputItem.text(String(output.text), mimeType)]); + } + + // Unknown output type - return as text if available + if ('text' in output && output.text) { + return new NotebookCellOutput([NotebookCellOutputItem.text(String(output.text), 'text/plain')]); + } + + // No text, return empty output + return new NotebookCellOutput([]); + } + + // Fallback for outputs without output_type but with text + if ('text' in output && output.text) { + return new NotebookCellOutput([NotebookCellOutputItem.text(String(output.text), 'text/plain')]); + } + + return new NotebookCellOutput([]); + }); } } diff --git a/src/notebooks/deepnote/deepnoteDataConverter.unit.test.ts b/src/notebooks/deepnote/deepnoteDataConverter.unit.test.ts index a586497db2..bd861344d3 100644 --- a/src/notebooks/deepnote/deepnoteDataConverter.unit.test.ts +++ b/src/notebooks/deepnote/deepnoteDataConverter.unit.test.ts @@ -29,10 +29,10 @@ suite('DeepnoteDataConverter', () => { assert.strictEqual(cells[0].kind, NotebookCellKind.Code); assert.strictEqual(cells[0].value, 'print("hello")'); assert.strictEqual(cells[0].languageId, 'python'); - assert.strictEqual(cells[0].metadata?.deepnoteBlockId, 'block1'); - assert.strictEqual(cells[0].metadata?.deepnoteBlockType, 'code'); - assert.strictEqual(cells[0].metadata?.deepnoteSortingKey, 'a0'); - assert.deepStrictEqual(cells[0].metadata?.deepnoteMetadata, { custom: 'data' }); + assert.strictEqual(cells[0].metadata?.__deepnotePocket?.id, 'block1'); + assert.strictEqual(cells[0].metadata?.__deepnotePocket?.type, 'code'); + assert.strictEqual(cells[0].metadata?.__deepnotePocket?.sortingKey, 'a0'); + assert.strictEqual(cells[0].metadata?.custom, 'data'); }); test('converts simple markdown block to cell', () => { @@ -51,26 +51,24 @@ suite('DeepnoteDataConverter', () => { assert.strictEqual(cells[0].kind, NotebookCellKind.Markup); assert.strictEqual(cells[0].value, '# Title'); assert.strictEqual(cells[0].languageId, 'markdown'); - assert.strictEqual(cells[0].metadata?.deepnoteBlockId, 'block2'); - assert.strictEqual(cells[0].metadata?.deepnoteBlockType, 'markdown'); + assert.strictEqual(cells[0].metadata?.__deepnotePocket?.id, 'block2'); + assert.strictEqual(cells[0].metadata?.__deepnotePocket?.type, 'markdown'); }); - test('handles execution count and output reference', () => { + test('handles execution count', () => { const blocks: DeepnoteBlock[] = [ { id: 'block1', type: 'code', content: 'x = 1', sortingKey: 'a0', - executionCount: 5, - outputReference: 'output-ref-123' + executionCount: 5 } ]; const cells = converter.convertBlocksToCells(blocks); - assert.strictEqual(cells[0].metadata?.executionCount, 5); - assert.strictEqual(cells[0].metadata?.deepnoteOutputReference, 'output-ref-123'); + assert.strictEqual(cells[0].metadata?.__deepnotePocket?.executionCount, 5); }); test('converts blocks with outputs', () => { @@ -105,9 +103,12 @@ suite('DeepnoteDataConverter', () => { value: 'print("test")', languageId: 'python', metadata: { - deepnoteBlockId: 'existing-id', - deepnoteSortingKey: 'a5', - deepnoteMetadata: { original: 'metadata' } + __deepnotePocket: { + id: 'existing-id', + type: 'code', + sortingKey: 'a5' + }, + original: 'metadata' } } ]; @@ -127,7 +128,12 @@ suite('DeepnoteDataConverter', () => { { kind: NotebookCellKind.Markup, value: '## Heading', - languageId: 'markdown' + languageId: 'markdown', + metadata: { + __deepnotePocket: { + type: 'markdown' + } + } } ]; @@ -161,19 +167,29 @@ suite('DeepnoteDataConverter', () => { assert.strictEqual(blocks[1].sortingKey, 'a1'); }); - test('handles execution count from metadata and executionSummary', () => { + test('handles execution count from pocket', () => { const cells: NotebookCellData[] = [ { kind: NotebookCellKind.Code, value: 'x = 1', languageId: 'python', - metadata: { executionCount: 10 } + metadata: { + __deepnotePocket: { + type: 'code', + executionCount: 10 + } + } }, { kind: NotebookCellKind.Code, value: 'y = 2', languageId: 'python', - executionSummary: { executionOrder: 20 } + metadata: { + __deepnotePocket: { + type: 'code', + executionCount: 20 + } + } } ]; @@ -182,21 +198,6 @@ suite('DeepnoteDataConverter', () => { assert.strictEqual(blocks[0].executionCount, 10); assert.strictEqual(blocks[1].executionCount, 20); }); - - test('includes output reference when present', () => { - const cells: NotebookCellData[] = [ - { - kind: NotebookCellKind.Code, - value: 'print("test")', - languageId: 'python', - metadata: { deepnoteOutputReference: 'ref-123' } - } - ]; - - const blocks = converter.convertCellsToBlocks(cells); - - assert.strictEqual(blocks[0].outputReference, 'ref-123'); - }); }); suite('output conversion', () => { @@ -402,9 +403,9 @@ suite('DeepnoteDataConverter', () => { sortingKey: 'a0', executionCount: 5, metadata: { custom: 'data' }, - outputReference: 'ref-123', outputs: [ { + name: 'stdout', output_type: 'stream', text: 'hello\n' } @@ -425,201 +426,80 @@ suite('DeepnoteDataConverter', () => { assert.deepStrictEqual(roundTripBlocks, originalBlocks); }); - test('blocks -> cells -> blocks preserves minimal multi-mime outputs', () => { - const originalBlocks: DeepnoteBlock[] = [ + test('real deepnote notebook round-trips without losing data', () => { + // Inline test data representing a real Deepnote notebook with various block types + // blockGroup is an optional field not in the DeepnoteBlock interface, so we cast as any + const originalBlocks = [ { - id: 'simple-multi', - type: 'code', - content: 'test', - sortingKey: 'z0', - outputs: [ - { - output_type: 'execute_result', - execution_count: 1, - data: { - 'text/plain': 'Result object', - 'text/html': '
Result
' - } - } - ] - } - ]; - - const cells = converter.convertBlocksToCells(originalBlocks); - const roundTripBlocks = converter.convertCellsToBlocks(cells); - - assert.deepStrictEqual(roundTripBlocks, originalBlocks); - }); - - test('blocks -> cells -> blocks preserves simpler multi-mime outputs', () => { - const originalBlocks: DeepnoteBlock[] = [ - { - id: 'multi-output-1', - type: 'code', - content: 'display(data)', - sortingKey: 'b0', - executionCount: 2, - outputs: [ - // Stream with name - { - output_type: 'stream', - name: 'stdout', - text: 'Starting...\n' - }, - // Execute result with multiple mimes - { - output_type: 'execute_result', - execution_count: 2, - data: { - 'text/plain': 'Result object', - 'text/html': '
Result
' - }, - metadata: { - custom: 'value' - } - }, - // Display data - { - output_type: 'display_data', - data: { - 'text/plain': 'Display text', - 'application/json': { result: true } - } - } - ] + blockGroup: '1a4224497bcd499ba180e5795990aaa8', + content: '# Data Exploration\n\nThis notebook demonstrates basic data exploration.', + id: 'b0524a309dff421e95f2efd64aaca02a', + metadata: {}, + sortingKey: 'a0', + type: 'markdown' }, { - id: 'error-block', - type: 'code', - content: 'error()', - sortingKey: 'b1', - executionCount: 3, - outputs: [ - { - output_type: 'error', - ename: 'RuntimeError', - evalue: 'Something went wrong', - traceback: ['Line 1: error'] - } - ] - } - ]; - - const cells = converter.convertBlocksToCells(originalBlocks); - const roundTripBlocks = converter.convertCellsToBlocks(cells); - - assert.deepStrictEqual(roundTripBlocks, originalBlocks); - }); - - test('blocks -> cells -> blocks preserves complex multi-mime rich outputs', () => { - const originalBlocks: DeepnoteBlock[] = [ + blockGroup: '9dd9578e604a4235a552d1f4a53336ee', + content: 'import pandas as pd\nimport numpy as np\n\nnp.random.seed(42)', + executionCount: 1, + id: 'b75d3ada977549b29f4c7f2183d52fcf', + metadata: { + execution_start: 1759390294701, + execution_millis: 1 + }, + outputs: [], + sortingKey: 'm', + type: 'code' + }, { - id: 'rich-block-1', - type: 'code', - content: 'import matplotlib.pyplot as plt\nplt.plot([1,2,3])', - sortingKey: 'aa0', + blockGroup: 'cf243a3bbe914b7598eb86935c9f5cf4', + content: 'print("Dataset shape:", df.shape)', executionCount: 3, - metadata: { slideshow: { slide_type: 'slide' } }, + id: '6e8982f5cae54715b7620c9dc58e6de5', + metadata: { + execution_start: 1759390294821, + execution_millis: 3 + }, outputs: [ - // Stream output { - output_type: 'stream', name: 'stdout', - text: 'Processing data...\n' - }, - // Execute result with multiple mime types - { - output_type: 'execute_result', - execution_count: 3, - data: { - 'text/plain': '', - 'text/html': '
Plot rendered
', - 'application/json': { type: 'plot', data: [1, 2, 3] } - }, - metadata: { - needs_background: 'light' - } - }, - // Display data with image - { - output_type: 'display_data', - data: { - 'image/png': - 'iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==', - 'text/plain': '
' - }, - metadata: { - image: { - width: 640, - height: 480 - } - } + output_type: 'stream', + text: 'Dataset shape: (100, 5)\n' } - ] + ], + sortingKey: 'y', + type: 'code' }, { - id: 'rich-block-2', - type: 'code', - content: 'raise ValueError("Test error")', - sortingKey: 'aa1', - executionCount: 4, + blockGroup: 'e3e8eea67aa64ed981c86edff029dc40', + content: 'print(r)', + executionCount: 7, + id: '8b99dc5b5ee94e0e9ec278466344ae2b', + metadata: { + execution_start: 1759390787589, + execution_millis: 320 + }, outputs: [ - // Error output with traceback { + ename: 'NameError', + evalue: "name 'r' is not defined", output_type: 'error', - ename: 'ValueError', - evalue: 'Test error', traceback: [ '\u001b[0;31m---------------------------------------------------------------------------\u001b[0m', - '\u001b[0;31mValueError\u001b[0m Traceback (most recent call last)', - '\u001b[0;32m\u001b[0m in \u001b[0;36m\u001b[0;34m\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0;32mraise\u001b[0m \u001b[0mValueError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m"Test error"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m', - '\u001b[0;31mValueError\u001b[0m: Test error' + '\u001b[0;31mNameError\u001b[0m Traceback (most recent call last)' ] } - ] - }, - { - id: 'rich-block-3', - type: 'code', - content: - 'from IPython.display import display, HTML, JSON\ndisplay(HTML("Bold text"), JSON({"key": "value"}))', - sortingKey: 'aa2', - executionCount: 5, - outputReference: 'output-ref-789', - outputs: [ - // Multiple display_data outputs - { - output_type: 'display_data', - data: { - 'text/html': 'Bold text', - 'text/plain': 'Bold text' - } - }, - { - output_type: 'display_data', - data: { - 'application/json': { key: 'value' }, - 'text/plain': "{'key': 'value'}" - }, - metadata: { - expanded: false, - root: 'object' - } - } - ] - }, - { - id: 'markdown-block', - type: 'markdown', - content: '## Results\n\nThe above cells demonstrate various output types.', - sortingKey: 'aa3', - metadata: { tags: ['documentation'] } + ], + sortingKey: 'yj', + type: 'code' } - ]; + ] as unknown as DeepnoteBlock[]; + // Convert blocks -> cells -> blocks const cells = converter.convertBlocksToCells(originalBlocks); const roundTripBlocks = converter.convertCellsToBlocks(cells); + // Should preserve all blocks without data loss assert.deepStrictEqual(roundTripBlocks, originalBlocks); }); }); diff --git a/src/notebooks/deepnote/deepnoteTypes.ts b/src/notebooks/deepnote/deepnoteTypes.ts index e8388986e4..e0d9e5e8bb 100644 --- a/src/notebooks/deepnote/deepnoteTypes.ts +++ b/src/notebooks/deepnote/deepnoteTypes.ts @@ -34,29 +34,45 @@ export interface DeepnoteNotebook { * Can be either a code block or a markdown block. */ export interface DeepnoteBlock { + blockGroup?: string; content: string; executionCount?: number; id: string; metadata?: Record; - outputReference?: string; outputs?: DeepnoteOutput[]; sortingKey: string; - type: 'code' | 'markdown'; + type: string; } /** * Represents output data generated by executing a code block. */ -export interface DeepnoteOutput { - data?: Record; - ename?: string; - error?: unknown; - evalue?: string; - execution_count?: number; - metadata?: Record; - name?: string; - output_type: string; - stack?: string; - text?: string; - traceback?: string[]; -} +export type DeepnoteOutput = + | { + data?: Record; + execution_count?: number; + metadata?: Record; + output_type: 'display_data' | 'execute_result'; + text?: string; + } + | { + name?: string; + text?: string; + } + | { + name?: string; + output_type: 'stream'; + text?: string; + } + | { + ename?: string; + evalue?: string; + output_type: 'error'; + text?: string; + traceback?: string[]; + } + | { + output_type: string; + text?: string; + [key: string]: unknown; + }; diff --git a/src/notebooks/deepnote/deepnoteVirtualDocumentProvider.ts b/src/notebooks/deepnote/deepnoteVirtualDocumentProvider.ts deleted file mode 100644 index f27f006518..0000000000 --- a/src/notebooks/deepnote/deepnoteVirtualDocumentProvider.ts +++ /dev/null @@ -1,84 +0,0 @@ -import { CancellationToken, Event, EventEmitter, TextDocumentContentProvider, Uri, workspace } from 'vscode'; -import * as yaml from 'js-yaml'; -import type { DeepnoteProject } from './deepnoteTypes'; -import { DeepnoteDataConverter } from './deepnoteDataConverter'; - -export class DeepnoteVirtualDocumentProvider implements TextDocumentContentProvider { - private onDidChangeEmitter = new EventEmitter(); - private converter = new DeepnoteDataConverter(); - - readonly onDidChange: Event = this.onDidChangeEmitter.event; - - async provideTextDocumentContent(uri: Uri, token: CancellationToken): Promise { - if (token.isCancellationRequested) { - throw new Error('Content provision cancelled'); - } - - const { filePath, notebookId } = this.parseVirtualUri(uri); - - try { - const fileUri = Uri.file(filePath); - const rawContent = await workspace.fs.readFile(fileUri); - const contentString = new TextDecoder('utf-8').decode(rawContent); - const deepnoteProject = yaml.load(contentString) as DeepnoteProject; - - if (!deepnoteProject.project?.notebooks) { - throw new Error('Invalid Deepnote file: no notebooks found'); - } - - const selectedNotebook = deepnoteProject.project.notebooks.find((nb) => nb.id === notebookId); - - if (!selectedNotebook) { - throw new Error(`Notebook with ID ${notebookId} not found`); - } - - const cells = this.converter.convertBlocksToCells(selectedNotebook.blocks); - - const notebookData = { - cells, - metadata: { - deepnoteProjectId: deepnoteProject.project.id, - deepnoteProjectName: deepnoteProject.project.name, - deepnoteNotebookId: selectedNotebook.id, - deepnoteNotebookName: selectedNotebook.name, - deepnoteVersion: deepnoteProject.version, - deepnoteFilePath: filePath - } - }; - - return JSON.stringify(notebookData, null, 2); - } catch (error) { - console.error('Error providing virtual document content:', error); - throw new Error(`Failed to provide content: ${error instanceof Error ? error.message : 'Unknown error'}`); - } - } - - private parseVirtualUri(uri: Uri): { filePath: string; notebookId: string } { - const query = new URLSearchParams(uri.query); - const filePath = query.get('filePath'); - const notebookId = query.get('notebookId'); - - if (!filePath || !notebookId) { - throw new Error('Invalid virtual URI: missing filePath or notebookId'); - } - - return { filePath, notebookId }; - } - - public static createVirtualUri(filePath: string, notebookId: string): Uri { - const query = new URLSearchParams({ - filePath, - notebookId - }); - - return Uri.parse(`deepnotenotebook://${notebookId}?${query.toString()}`); - } - - public fireDidChange(uri: Uri): void { - this.onDidChangeEmitter.fire(uri); - } - - dispose(): void { - this.onDidChangeEmitter.dispose(); - } -} diff --git a/src/notebooks/deepnote/outputHandlers/ErrorOutputHandler.ts b/src/notebooks/deepnote/outputHandlers/ErrorOutputHandler.ts deleted file mode 100644 index dcc582b873..0000000000 --- a/src/notebooks/deepnote/outputHandlers/ErrorOutputHandler.ts +++ /dev/null @@ -1,96 +0,0 @@ -import { NotebookCellOutputItem } from 'vscode'; -import { decodeContent, parseJsonSafely } from '../dataConversionUtils'; -import type { DeepnoteOutput } from '../deepnoteTypes'; - -/** - * Handles error outputs conversion between Deepnote and VS Code formats - */ -export class ErrorOutputHandler { - /** - * Convert VS Code error output to Deepnote format - */ - convertToDeepnote(errorItem: { mime: string; data: Uint8Array }): DeepnoteOutput { - const deepnoteOutput: DeepnoteOutput = { - output_type: 'error' - }; - - try { - const errorData = parseJsonSafely(decodeContent(errorItem.data)); - if (typeof errorData === 'object' && errorData !== null) { - const errorObj = errorData as Record; - // Prefer Jupyter-style fields if they exist (for round-trip preservation) - // Otherwise fallback to VS Code error structure - deepnoteOutput.ename = (errorObj.ename as string) || (errorObj.name as string) || 'Error'; - deepnoteOutput.evalue = (errorObj.evalue as string) || (errorObj.message as string) || ''; - - // Handle traceback - prefer original traceback array if it exists - if (errorObj.traceback && Array.isArray(errorObj.traceback)) { - deepnoteOutput.traceback = errorObj.traceback as string[]; - } else if (errorObj.stack && typeof errorObj.stack === 'string') { - // Try extracting traceback from stack trace if custom properties weren't preserved - if (errorObj.stack.includes('__TRACEBACK_START__')) { - // Parse our special format - const tracebackMatch = errorObj.stack.match(/__TRACEBACK_START__\n(.*?)\n__TRACEBACK_END__/s); - if (tracebackMatch) { - deepnoteOutput.traceback = tracebackMatch[1].split('\n__TRACEBACK_LINE__\n'); - } else { - deepnoteOutput.traceback = []; - } - } else { - const stackLines = errorObj.stack.split('\n'); - // Skip the first line which is the error name/message - deepnoteOutput.traceback = stackLines.slice(1); - } - } else { - deepnoteOutput.traceback = []; - } - } else { - // Fallback if error data is not valid JSON object - const errorText = String(errorData); - deepnoteOutput.ename = 'Error'; - deepnoteOutput.evalue = errorText; - deepnoteOutput.traceback = [errorText]; - } - } catch { - // Final fallback if parsing completely fails - const errorText = decodeContent(errorItem.data); - deepnoteOutput.ename = 'Error'; - deepnoteOutput.evalue = errorText; - deepnoteOutput.traceback = [errorText]; - } - - return deepnoteOutput; - } - - /** - * Convert Deepnote error output to VS Code format - */ - convertToVSCode(output: DeepnoteOutput): NotebookCellOutputItem[] { - // Create a simple error with just the evalue as message - const error = new Error(output.evalue || output.text || 'Error'); - - // Store the original Deepnote error data for round-trip preservation - if (output.ename) { - error.name = output.ename; - Object.assign(error, { ename: output.ename }); - } - if (output.evalue) { - Object.assign(error, { evalue: output.evalue }); - } - if (output.traceback) { - Object.assign(error, { traceback: output.traceback }); - // Also encode in the stack trace for better preservation - // Join traceback with a special separator that we can split on later - if (Array.isArray(output.traceback) && output.traceback.length > 0) { - error.stack = `${output.ename || 'Error'}: ${ - output.evalue || 'Unknown error' - }\n__TRACEBACK_START__\n${output.traceback.join('\n__TRACEBACK_LINE__\n')}\n__TRACEBACK_END__`; - } - } - if (output.error) { - Object.assign(error, { deepnoteError: output.error }); - } - - return [NotebookCellOutputItem.error(error)]; - } -} diff --git a/src/notebooks/deepnote/outputHandlers/RichOutputHandler.ts b/src/notebooks/deepnote/outputHandlers/RichOutputHandler.ts deleted file mode 100644 index f0a99e8e65..0000000000 --- a/src/notebooks/deepnote/outputHandlers/RichOutputHandler.ts +++ /dev/null @@ -1,85 +0,0 @@ -import { NotebookCellOutput, NotebookCellOutputItem } from 'vscode'; -import { decodeContent } from '../dataConversionUtils'; -import { MimeTypeProcessorRegistry } from '../MimeTypeProcessor'; -import type { DeepnoteOutput } from '../deepnoteTypes'; - -/** - * Handles rich/display data outputs conversion between Deepnote and VS Code formats - */ -export class RichOutputHandler { - private readonly mimeRegistry = new MimeTypeProcessorRegistry(); - - /** - * Convert VS Code rich output to Deepnote format - */ - convertToDeepnote(output: NotebookCellOutput): DeepnoteOutput { - const deepnoteOutput: DeepnoteOutput = { - output_type: 'display_data', - data: {} - }; - - // Check for execution count in metadata - if (output.metadata?.executionCount !== undefined) { - deepnoteOutput.execution_count = output.metadata.executionCount; - deepnoteOutput.output_type = 'execute_result'; - } - - for (const item of output.items) { - // Skip only specific VS Code notebook stream and error mimes, but allow text/plain in rich context - if ( - item.mime !== 'application/vnd.code.notebook.error' && - item.mime !== 'application/vnd.code.notebook.stdout' && - item.mime !== 'application/vnd.code.notebook.stderr' - ) { - try { - // Check if this item has preserved original base64 data - if ( - (item as NotebookCellOutputItem & { _originalBase64?: string })._originalBase64 && - item.mime.startsWith('image/') - ) { - deepnoteOutput.data![item.mime] = ( - item as NotebookCellOutputItem & { _originalBase64?: string } - )._originalBase64; - } else { - const decodedContent = decodeContent(item.data); - deepnoteOutput.data![item.mime] = this.mimeRegistry.processForDeepnote( - decodedContent, - item.mime - ); - } - } catch (error) { - // Fallback: treat as text if any processing fails - try { - const decodedContent = decodeContent(item.data); - deepnoteOutput.data![item.mime] = decodedContent; - } catch { - // Skip this item if even text decoding fails - console.warn(`Failed to process output item with mime type: ${item.mime}`, error); - } - } - } - } - - return deepnoteOutput; - } - - /** - * Convert Deepnote rich output to VS Code format - */ - convertToVSCode(output: DeepnoteOutput): NotebookCellOutputItem[] { - if (!output.data) { - return output.text ? [NotebookCellOutputItem.text(output.text)] : []; - } - - const items: NotebookCellOutputItem[] = []; - - for (const [mimeType, content] of Object.entries(output.data)) { - const item = this.mimeRegistry.processForVSCode(content, mimeType); - if (item) { - items.push(item); - } - } - - return items; - } -} diff --git a/src/notebooks/deepnote/outputHandlers/StreamOutputHandler.ts b/src/notebooks/deepnote/outputHandlers/StreamOutputHandler.ts deleted file mode 100644 index 3a63865284..0000000000 --- a/src/notebooks/deepnote/outputHandlers/StreamOutputHandler.ts +++ /dev/null @@ -1,69 +0,0 @@ -import { NotebookCellOutput, NotebookCellOutputItem } from 'vscode'; -import { decodeContent } from '../dataConversionUtils'; -import type { DeepnoteOutput } from '../deepnoteTypes'; - -/** - * Handles stream outputs (stdout/stderr) conversion between Deepnote and VS Code formats - */ -export class StreamOutputHandler { - private readonly streamMimes = [ - 'text/plain', - 'application/vnd.code.notebook.stdout', - 'application/vnd.code.notebook.stderr' - ]; - - /** - * Convert VS Code stream output to Deepnote format - */ - convertToDeepnote(output: NotebookCellOutput): DeepnoteOutput { - const streamItems = output.items.filter((item) => this.streamMimes.includes(item.mime)); - - // Combine all stream text - const streamTexts = streamItems.map((item) => decodeContent(item.data)); - const text = streamTexts.join(''); - - const deepnoteOutput: DeepnoteOutput = { - output_type: 'stream', - text - }; - - // Only set stream name if we can definitively determine it from mime type - const stderrItem = streamItems.find((item) => item.mime === 'application/vnd.code.notebook.stderr'); - const stdoutItem = streamItems.find((item) => item.mime === 'application/vnd.code.notebook.stdout'); - const unnamedStreamItem = streamItems.find( - (item) => (item as NotebookCellOutputItem & { _wasUnnamedStream?: boolean })._wasUnnamedStream - ); - - if (stderrItem) { - deepnoteOutput.name = 'stderr'; - } else if (stdoutItem && !unnamedStreamItem) { - // Only set stdout name if it wasn't originally unnamed - deepnoteOutput.name = 'stdout'; - } - // Don't set name for streams that were originally unnamed - - return deepnoteOutput; - } - - /** - * Convert Deepnote stream output to VS Code format - */ - convertToVSCode(output: DeepnoteOutput): NotebookCellOutputItem[] { - if (!output.text) { - return []; - } - - // Route to appropriate stream type based on Deepnote stream name - if (output.name === 'stderr') { - return [NotebookCellOutputItem.stderr(output.text)]; - } else if (output.name === 'stdout') { - return [NotebookCellOutputItem.stdout(output.text)]; - } else { - // For streams without explicit name, use stdout for proper VS Code display - // but mark it as originally unnamed for round-trip preservation - const item = NotebookCellOutputItem.stdout(output.text); - (item as NotebookCellOutputItem & { _wasUnnamedStream?: boolean })._wasUnnamedStream = true; - return [item]; - } - } -} diff --git a/src/notebooks/deepnote/pocket.ts b/src/notebooks/deepnote/pocket.ts new file mode 100644 index 0000000000..e486e696ad --- /dev/null +++ b/src/notebooks/deepnote/pocket.ts @@ -0,0 +1,82 @@ +import type { NotebookCellData } from 'vscode'; + +import type { DeepnoteBlock, DeepnoteOutput } from './deepnoteTypes'; +import { generateBlockId, generateSortingKey } from './dataConversionUtils'; + +const deepnoteBlockSpecificFields = ['blockGroup', 'executionCount', 'id', 'outputs', 'sortingKey', 'type'] as const; + +// Stores extra Deepnote-specific fields for each block that are not part of the standard VSCode NotebookCellData structure. +export interface Pocket { + blockGroup?: string; + executionCount?: number; + id?: string; + outputs?: DeepnoteOutput[]; + sortingKey?: string; + type?: string; +} + +export function addPocketToCellMetadata(cell: NotebookCellData): void { + const src: Record = cell.metadata ? { ...cell.metadata } : {}; + const pocket: Pocket = {}; + let found = false; + + for (const field of deepnoteBlockSpecificFields) { + if (Object.prototype.hasOwnProperty.call(src, field)) { + const value = src[field]; + (pocket as Record)[field] = value; + delete src[field]; + found = true; + } + } + + if (!found) { + return; + } + + cell.metadata = { + ...src, + __deepnotePocket: pocket + }; +} + +export function extractPocketFromCellMetadata(cell: NotebookCellData): Pocket | undefined { + return cell.metadata?.__deepnotePocket; +} + +export function createBlockFromPocket(cell: NotebookCellData, index: number): DeepnoteBlock { + const pocket = extractPocketFromCellMetadata(cell); + + const metadata = cell.metadata ? { ...cell.metadata } : undefined; + + if (metadata) { + // Remove pocket and all pocket fields from metadata + delete metadata.__deepnotePocket; + + for (const field of deepnoteBlockSpecificFields) { + delete metadata[field]; + } + } + + const block: DeepnoteBlock = { + content: cell.value, + id: pocket?.id || generateBlockId(), + metadata, + sortingKey: pocket?.sortingKey || generateSortingKey(index), + type: pocket?.type || 'code' + }; + + // Only add optional fields if they exist + if (pocket?.blockGroup) { + block.blockGroup = pocket.blockGroup; + } + + if (pocket?.executionCount !== undefined) { + block.executionCount = pocket.executionCount; + } + + if (pocket?.outputs !== undefined) { + block.outputs = pocket.outputs; + } + + return block; +} diff --git a/src/notebooks/deepnote/pocket.unit.test.ts b/src/notebooks/deepnote/pocket.unit.test.ts new file mode 100644 index 0000000000..49f9717b21 --- /dev/null +++ b/src/notebooks/deepnote/pocket.unit.test.ts @@ -0,0 +1,207 @@ +import { assert } from 'chai'; +import { NotebookCellData, NotebookCellKind } from 'vscode'; + +import { addPocketToCellMetadata, createBlockFromPocket, extractPocketFromCellMetadata } from './pocket'; + +suite('Pocket', () => { + suite('addPocketToCellMetadata', () => { + test('adds pocket with Deepnote-specific fields', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + id: 'block-123', + type: 'code', + sortingKey: 'a0', + executionCount: 5, + other: 'value' + }; + + addPocketToCellMetadata(cell); + + assert.deepStrictEqual(cell.metadata.__deepnotePocket, { + id: 'block-123', + type: 'code', + sortingKey: 'a0', + executionCount: 5 + }); + assert.strictEqual(cell.metadata.other, 'value'); + }); + + test('does not add pocket if no Deepnote-specific fields exist', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + other: 'value' + }; + + addPocketToCellMetadata(cell); + + assert.isUndefined(cell.metadata.__deepnotePocket); + }); + + test('handles cell with no metadata', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + addPocketToCellMetadata(cell); + + assert.isUndefined(cell.metadata); + }); + + test('handles partial Deepnote fields', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + id: 'block-123', + type: 'code' + }; + + addPocketToCellMetadata(cell); + + assert.deepStrictEqual(cell.metadata.__deepnotePocket, { + id: 'block-123', + type: 'code' + }); + }); + }); + + suite('extractPocketFromCellMetadata', () => { + test('extracts pocket from cell metadata', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + __deepnotePocket: { + id: 'block-123', + type: 'code', + sortingKey: 'a0', + executionCount: 5 + }, + other: 'value' + }; + + const pocket = extractPocketFromCellMetadata(cell); + + assert.deepStrictEqual(pocket, { + id: 'block-123', + type: 'code', + sortingKey: 'a0', + executionCount: 5 + }); + }); + + test('returns undefined if no pocket exists', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + other: 'value' + }; + + const pocket = extractPocketFromCellMetadata(cell); + + assert.isUndefined(pocket); + }); + + test('returns undefined if cell has no metadata', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + const pocket = extractPocketFromCellMetadata(cell); + + assert.isUndefined(pocket); + }); + }); + + suite('createBlockFromPocket', () => { + test('creates block from pocket metadata', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + __deepnotePocket: { + id: 'block-123', + type: 'code', + sortingKey: 'a0', + executionCount: 5 + }, + custom: 'value' + }; + + const block = createBlockFromPocket(cell, 0); + + assert.strictEqual(block.id, 'block-123'); + assert.strictEqual(block.type, 'code'); + assert.strictEqual(block.sortingKey, 'a0'); + assert.strictEqual(block.executionCount, 5); + assert.strictEqual(block.content, 'print("hello")'); + assert.strictEqual(block.outputs, undefined); + }); + + test('creates block with generated ID and sortingKey when no pocket exists', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + const block = createBlockFromPocket(cell, 5); + + assert.match(block.id, /^[0-9a-f]{32}$/); + assert.strictEqual(block.type, 'code'); + assert.strictEqual(block.sortingKey, 'a5'); + assert.isUndefined(block.executionCount); + }); + + test('removes __deepnotePocket from block metadata', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + __deepnotePocket: { + id: 'block-123', + type: 'code' + }, + custom: 'value' + }; + + const block = createBlockFromPocket(cell, 0); + + assert.isUndefined(block.metadata?.__deepnotePocket); + assert.strictEqual(block.metadata?.custom, 'value'); + }); + + test('preserves other metadata fields', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + __deepnotePocket: { + id: 'block-123', + type: 'code' + }, + custom: 'value', + slideshow: { slide_type: 'slide' } + }; + + const block = createBlockFromPocket(cell, 0); + + assert.strictEqual(block.metadata?.custom, 'value'); + assert.deepStrictEqual(block.metadata?.slideshow, { slide_type: 'slide' }); + }); + + test('uses default type when no pocket exists', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + const block = createBlockFromPocket(cell, 0); + + assert.strictEqual(block.type, 'code'); + }); + + test('handles partial pocket data', () => { + const cell = new NotebookCellData(NotebookCellKind.Code, 'print("hello")', 'python'); + + cell.metadata = { + __deepnotePocket: { + id: 'block-123' + } + }; + + const block = createBlockFromPocket(cell, 3); + + assert.strictEqual(block.id, 'block-123'); + assert.strictEqual(block.type, 'code'); + assert.strictEqual(block.sortingKey, 'a3'); + assert.isUndefined(block.executionCount); + }); + }); +}); diff --git a/src/notebooks/deepnote/serialization.md b/src/notebooks/deepnote/serialization.md new file mode 100644 index 0000000000..b4f25644b0 --- /dev/null +++ b/src/notebooks/deepnote/serialization.md @@ -0,0 +1,305 @@ +# Deepnote Serialization Architecture + +This document explains how Deepnote notebooks are serialized and deserialized between the Deepnote YAML format and VS Code's notebook format. + +## Overview + +The serialization system converts Deepnote blocks to VS Code cells and vice versa while preserving all Deepnote-specific metadata. This is accomplished through a converter pattern and a "pocket" system for storing extra data. + +## Key Concepts + +### The Pocket System + +The **pocket** is a special metadata field (`__deepnotePocket`) that stores Deepnote-specific block information that doesn't have a direct equivalent in VS Code's notebook format. This ensures perfect round-trip conversion without data loss. + +Pocket fields include: +- `id`: Deepnote block ID +- `type`: Deepnote block type (e.g., 'code', 'markdown', 'text-cell-h1') +- `sortingKey`: Deepnote sorting key for block ordering +- `executionCount`: Execution count for code blocks +- `outputs`: Original Deepnote outputs (for round-trip preservation) +- `blockGroup`: Deepnote block group identifier + +### The Converter Pattern + +Each Deepnote block type has a corresponding converter that knows how to: +1. Convert a Deepnote block to a VS Code cell (`convertToCell`) +2. Apply changes from a VS Code cell back to a Deepnote block (`applyChangesToBlock`) + +Converters are registered in a `ConverterRegistry` and retrieved based on block type. + +## Serialization Flow (Deepnote → VS Code) + +When opening a Deepnote notebook in VS Code: + +```text +Deepnote YAML File + ↓ +Parse YAML (js-yaml) + ↓ +Extract blocks from selected notebook + ↓ +For each block: + 1. Find converter for block.type + 2. converter.convertToCell(block) → creates VS Code cell + 3. Store Deepnote fields in cell.metadata + 4. addPocketToCellMetadata(cell) → moves fields to __deepnotePocket + 5. transformOutputsForVsCode(block.outputs) → convert outputs + ↓ +VS Code NotebookData with cells +``` + +**Key functions:** +- `deepnoteDataConverter.ts::convertBlocksToCells(blocks)` - Main entry point +- `pocket.ts::addPocketToCellMetadata(cell)` - Creates the pocket +- `deepnoteDataConverter.ts::transformOutputsForVsCode(outputs)` - Converts outputs + +## Deserialization Flow (VS Code → Deepnote) + +When saving a notebook in VS Code: + +```text +VS Code NotebookData with cells + ↓ +For each cell: + 1. createBlockFromPocket(cell, index) → creates base block + 2. Find converter for block.type + 3. converter.applyChangesToBlock(block, cell) → updates content + 4. If needed: transformOutputsForDeepnote(cell.outputs) → convert new outputs + ↓ +Array of Deepnote blocks + ↓ +Update project YAML structure + ↓ +Serialize to YAML (js-yaml) + ↓ +Deepnote YAML File +``` + +**Key functions:** +- `deepnoteDataConverter.ts::convertCellsToBlocks(cells)` - Main entry point +- `pocket.ts::createBlockFromPocket(cell, index)` - Extracts block from pocket +- `deepnoteDataConverter.ts::transformOutputsForDeepnote(outputs)` - Converts outputs + +## Adding Support for a New Block Type + +Follow these steps to add support for a new Deepnote block type: + +### 1. Create a Converter Class + +Create a new file in `src/notebooks/deepnote/converters/`: + +```typescript +// src/notebooks/deepnote/converters/myBlockConverter.ts +import { NotebookCellData, NotebookCellKind } from 'vscode'; +import type { BlockConverter } from './blockConverter'; +import type { DeepnoteBlock } from '../deepnoteTypes'; + +export class MyBlockConverter implements BlockConverter { + // Block types this converter handles + get blockTypes(): string[] { + return ['my-block-type']; + } + + // Convert Deepnote block to VS Code cell + convertToCell(block: DeepnoteBlock): NotebookCellData { + // Choose appropriate cell kind + const cell = new NotebookCellData( + NotebookCellKind.Markup, // or NotebookCellKind.Code + block.content || '', + 'markdown' // or 'python', etc. + ); + + return cell; + } + + // Apply VS Code cell changes back to Deepnote block + applyChangesToBlock(block: DeepnoteBlock, cell: NotebookCellData): void { + // Update the block content from the cell + block.content = cell.value || ''; + + // Apply any transformations needed + // (e.g., strip prefixes, convert formats, etc.) + } +} +``` + +### 2. Register the Converter + +Add your converter to the registry in `deepnoteDataConverter.ts`: + +```typescript +import { MyBlockConverter } from './converters/myBlockConverter'; + +export class DeepnoteDataConverter { + private readonly registry = new ConverterRegistry(); + + constructor() { + this.registry.register(new CodeBlockConverter()); + this.registry.register(new TextBlockConverter()); + this.registry.register(new MarkdownBlockConverter()); + this.registry.register(new MyBlockConverter()); // Add this line + } + // ... +} +``` + +### 3. Create Tests + +Create comprehensive tests in `src/notebooks/deepnote/converters/myBlockConverter.unit.test.ts`: + +```typescript +import { assert } from 'chai'; +import { NotebookCellKind } from 'vscode'; +import { MyBlockConverter } from './myBlockConverter'; +import type { DeepnoteBlock } from '../deepnoteTypes'; + +suite('MyBlockConverter', () => { + let converter: MyBlockConverter; + + setup(() => { + converter = new MyBlockConverter(); + }); + + suite('convertToCell', () => { + test('converts my-block-type to cell', () => { + const block: DeepnoteBlock = { + id: 'block1', + type: 'my-block-type', + content: 'Hello World', + sortingKey: 'a0' + }; + + const cell = converter.convertToCell(block); + + assert.strictEqual(cell.kind, NotebookCellKind.Markup); + assert.strictEqual(cell.value, 'Hello World'); + assert.strictEqual(cell.languageId, 'markdown'); + }); + }); + + suite('applyChangesToBlock', () => { + test('applies cell changes to block', () => { + const block: DeepnoteBlock = { + id: 'block1', + type: 'my-block-type', + content: '', + sortingKey: 'a0' + }; + + const cell = new NotebookCellData( + NotebookCellKind.Markup, + 'Updated content', + 'markdown' + ); + + converter.applyChangesToBlock(block, cell); + + assert.strictEqual(block.content, 'Updated content'); + }); + }); +}); +``` + +### 4. Add Round-Trip Tests + +Ensure your converter preserves data correctly in `deepnoteDataConverter.unit.test.ts`: + +```typescript +test('my-block-type round-trips correctly', () => { + const originalBlock: DeepnoteBlock = { + id: 'block1', + type: 'my-block-type', + content: 'Test content', + sortingKey: 'a0', + metadata: { custom: 'data' } + }; + + const cell = converter.convertBlocksToCells([originalBlock])[0]; + const roundTripBlock = converter.convertCellsToBlocks([cell])[0]; + + assert.deepStrictEqual(roundTripBlock, originalBlock); +}); +``` + +## Important Guidelines + +### DO: + +- ✅ Use the pocket system for Deepnote-specific fields +- ✅ Preserve all block metadata during conversion +- ✅ Test round-trip conversion (blocks → cells → blocks) +- ✅ Handle both empty and undefined fields correctly +- ✅ Use `assert.deepStrictEqual()` for object comparisons in tests + +### DON'T: + +- ❌ Store Deepnote-specific data directly in cell metadata (use the pocket) +- ❌ Modify the pocket in converters (it's managed automatically) +- ❌ Assume all optional fields exist (check for undefined) +- ❌ Convert undefined to empty arrays/objects (preserve exact structure) + +## Example: TextBlockConverter + +Here's a real example showing header transformations: + +```typescript +export class TextBlockConverter implements BlockConverter { + get blockTypes(): string[] { + return ['text-cell-h1', 'text-cell-h2', 'text-cell-h3', 'text-cell']; + } + + convertToCell(block: DeepnoteBlock): NotebookCellData { + let content = block.content || ''; + + // Add markdown prefix based on block type + if (block.type === 'text-cell-h1') { + content = `# ${content}`; + } else if (block.type === 'text-cell-h2') { + content = `## ${content}`; + } else if (block.type === 'text-cell-h3') { + content = `### ${content}`; + } + + return new NotebookCellData(NotebookCellKind.Markup, content, 'markdown'); + } + + applyChangesToBlock(block: DeepnoteBlock, cell: NotebookCellData): void { + let value = cell.value || ''; + + // Strip markdown prefix when converting back + if (block.type === 'text-cell-h1') { + value = value.replace(/^#\s+/, ''); + } else if (block.type === 'text-cell-h2') { + value = value.replace(/^##\s+/, ''); + } else if (block.type === 'text-cell-h3') { + value = value.replace(/^###\s+/, ''); + } + + block.content = value; + } +} +``` + +This example shows: +1. Supporting multiple block types in one converter +2. Transforming content during conversion (adding markdown prefixes) +3. Reversing the transformation when converting back (stripping prefixes) +4. Preserving the original block type through the pocket system + +## Testing Your Converter + +Run the tests to ensure everything works: + +```bash +# Run all tests +npm test + +# Run only your converter tests +npx mocha --config ./build/.mocha.unittests.js.json ./out/notebooks/deepnote/converters/myBlockConverter.unit.test.js +``` + +Make sure: +1. All tests pass +2. Round-trip conversion preserves all data +3. The real Deepnote notebook test still passes