SQL Support in Quadratic (Via DuckDB Wasm)

.vscode/settings.json

@@ -2,6 +2,7 @@
     "editor.formatOnSave": true,
     "cSpell.words": [
         "dgraph",
+        "duckdb",
         "Fuzzysort",
         "hljs",
         "openai",

package.json

@@ -10,6 +10,7 @@
   "private": true,
   "dependencies": {
     "@auth0/auth0-react": "^1.11.0",
+    "@duckdb/duckdb-wasm": "^1.24.0",
     "@emotion/react": "^11.7.0",
     "@emotion/styled": "^11.6.0",
     "@monaco-editor/react": "^4.3.1",
@@ -30,6 +31,7 @@
     "@types/react-dom": "^17.0.10",
     "@types/uuid": "^9.0.1",
     "ace-builds": "^1.4.13",
+    "apache-arrow": "^11.0.0",
     "axios": "^0.24.0",
     "color": "^4.2.3",
     "date-fns": "^2.28.0",

src/global.d.ts

@@ -2,3 +2,13 @@ declare module '*.py' {
   const content: string;
   export default content;
 }
+
+declare module '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm' {
+  const value: string;
+  export default value;
+}
+
+declare module '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm' {
+  const value: string;
+  export default value;
+}

src/grid/actions/updateCellAndDCells.ts

@@ -91,7 +91,7 @@ export const updateCellAndDCells = async (args: ArgsType) => {
       });
     } else {
       // We are evaluating a cell
-      if (cell.type === 'PYTHON' || cell.type === 'FORMULA' || cell.type === 'AI') {
+      if (cell.type === 'PYTHON' || cell.type === 'FORMULA' || cell.type === 'AI' || cell.type === 'SQL') {
         // run cell and format results
         // let result = await runPython(cell.python_code || '', pyodide);
         let result = await runCellComputation(cell, pyodide);

src/grid/computations/runCellComputation.ts

@@ -2,6 +2,7 @@ import { Cell } from '../sheet/gridTypes';
 import { runAI } from './ai/runAI';
 import { runFormula } from './formulas/runFormula';
 import { runPython } from './python/runPython';
+import { runSQL } from './sql/runSQL';
 import { CellEvaluationResult } from './types';
 
 export const runCellComputation = async (cell: Cell, pyodide?: any): Promise<CellEvaluationResult> => {
@@ -41,6 +42,18 @@ export const runCellComputation = async (cell: Cell, pyodide?: any): Promise<Cel
       formatted_code: '',
       error_span: null,
     };
+  } else if (cell.type === 'SQL') {
+    let result = await runSQL(cell.sql_statement || '');
+    return {
+      success: true,
+      std_out: undefined,
+      std_err: result.stderr,
+      output_value: result.output_value,
+      cells_accessed: [],
+      array_output: result.array_output || [],
+      formatted_code: '',
+      error_span: null,
+    };
   } else {
     throw new Error('Unsupported cell type');
   }

src/grid/computations/sql/runSQL.ts

@@ -0,0 +1,101 @@
+import * as duckdb from '@duckdb/duckdb-wasm';
+import duckdb_wasm from '@duckdb/duckdb-wasm/dist/duckdb-mvp.wasm';
+import duckdb_wasm_next from '@duckdb/duckdb-wasm/dist/duckdb-eh.wasm';
+import type { AsyncDuckDB } from '@duckdb/duckdb-wasm';
+
+const MANUAL_BUNDLES: duckdb.DuckDBBundles = {
+  mvp: {
+    mainModule: duckdb_wasm,
+    mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-mvp.worker.js', import.meta.url).toString(),
+  },
+  eh: {
+    mainModule: duckdb_wasm_next,
+    mainWorker: new URL('@duckdb/duckdb-wasm/dist/duckdb-browser-eh.worker.js', import.meta.url).toString(),
+  },
+};
+
+let db: AsyncDuckDB | null = null;
+
+export interface runSQLReturnType {
+  output_value: string | null;
+  stderr: string | undefined;
+  array_output?: (string | number | boolean)[][];
+}
+
+export const initDB = async () => {
+  if (db) {
+    return db; // Return existing database, if any
+  }
+
+  // Select a bundle based on browser checks
+  const bundle = await duckdb.selectBundle(MANUAL_BUNDLES);
+  // Instantiate the asynchronous version of DuckDB-wasm
+  const worker = new Worker(bundle.mainWorker!);
+  const logger = new duckdb.ConsoleLogger();
+  db = new duckdb.AsyncDuckDB(logger, worker);
+  await db.instantiate(bundle.mainModule, bundle.pthreadWorker);
+
+  return db;
+};
+
+export async function runSQL(sql_statement: string): Promise<runSQLReturnType> {
+  if (!db) {
+    await initDB();
+  }
+  if (!db) {
+    throw new Error('DuckDB database not initialized');
+  }
+
+  try {
+    const conn = await db.connect();
+
+    // Either materialize the query result
+    const result = await conn.query(sql_statement);
+
+    // Close the connection to release memory
+    await conn.close();
+
+    // Initialize an empty 2D array of strings
+    const resultArray: string[][] = [];
+
+    // Add column names as the first row
+    const columnNames = result.schema.fields.map((field) => field.name);
+    resultArray.push(columnNames);
+
+    // Iterate through the table's batches
+    result.batches.forEach((recordBatch) => {
+      // Iterate through the table's rows
+      const numCols = recordBatch.schema.fields.length;
+      const numRows = recordBatch.numRows;
+
+      // Iterate through the rows of the recordBatch
+      for (let row = 0; row < numRows; row++) {
+        // Initialize an empty rowData array
+        const rowData = [];
+
+        // Iterate through the columns of the recordBatch
+        for (let col = 0; col < numCols; col++) {
+          // Get the column data by column index
+          const column = recordBatch.getChildAt(col);
+          // Get the value at the specific row for the current column
+          const cellData = column?.get(row);
+          // Convert the cell data to a string and add it to the rowData array
+          rowData.push(cellData.toString());
+        }
+        // Add rowData to the resultArray
+        resultArray.push(rowData);
+      }
+    });
+
+    return {
+      output_value: null,
+      array_output: resultArray,
+    } as runSQLReturnType;
+  } catch (error: any) {
+    return {
+      output_value: null,
+      stderr: error.toString(),
+      array_output: [],
+    } as runSQLReturnType;
+  }
+}

src/grid/sheet/gridTypes.ts

@@ -17,6 +17,7 @@ export const CellSchema = z.object({
   last_modified: z.string().optional(),
   ai_prompt: z.string().optional(),
   python_code: z.string().optional(),
+  sql_statement: z.string().optional(),
 });
 export type Cell = z.infer<typeof CellSchema>;
 

src/gridGL/UI/Cursor.ts

@@ -122,6 +122,8 @@ export class Cursor extends Graphics {
             ? colors.cellColorUserFormula
             : editorInteractionState.mode === 'AI'
             ? colors.cellColorUserAI
+            : editorInteractionState.mode === 'SQL'
+            ? colors.cellColorUserSQL
             : colors.cursorCell;
       this.beginFill(color).drawShape(this.indicator).endFill();
     }
@@ -139,6 +141,8 @@ export class Cursor extends Graphics {
           ? colors.cellColorUserFormula
           : editorInteractionState.mode === 'AI'
           ? colors.cellColorUserAI
+          : editorInteractionState.mode === 'SQL'
+          ? colors.cellColorUserSQL
           : colors.independence;
       this.lineStyle({
         width: CURSOR_THICKNESS * 1.5,

src/gridGL/UI/cells/Cells.ts

@@ -160,6 +160,8 @@ export class Cells extends Container {
               this.cellsMarkers.add(x, y, 'FormulaIcon', error);
             } else if (entry.cell?.type === 'AI') {
               this.cellsMarkers.add(x, y, 'AIIcon', error);
+            } else if (entry.cell?.type === 'SQL') {
+              this.cellsMarkers.add(x, y, 'SQLIcon', error);
             }
 
           // show cell text

src/gridGL/UI/cells/CellsArray.ts

@@ -51,6 +51,8 @@ export class CellsArray extends Container {
           ? colors.cellColorUserFormula
           : type === 'AI'
           ? colors.cellColorUserAI
+          : type === 'SQL'
+          ? colors.cellColorUserSQL
           : colors.independence,
       alpha: 0.5,
       x,

src/gridGL/UI/cells/CellsBorder.ts

@@ -78,6 +78,8 @@ export class CellsBorder extends Container {
         drawInputBorder(input, colors.cellColorUserFormula, 0.75);
       } else if (input.cell.type === 'AI') {
         drawInputBorder(input, colors.cellColorUserAI, 0.75);
+      } else if (input.cell.type === 'SQL') {
+        drawInputBorder(input, colors.cellColorUserSQL, 0.75);
       } else if (input.cell.type === 'COMPUTED') {
         // drawInputBorder(input, colors.independence, 0.75);
       }

src/gridGL/UI/cells/CellsMarkers.ts

@@ -1,7 +1,7 @@
 import { Container, Sprite, Texture } from 'pixi.js';
 import { colors } from '../../../theme/colors';
 
-export type CellsMarkerTypes = 'CodeIcon' | 'FormulaIcon' | 'AIIcon' | 'ErrorIcon';
+export type CellsMarkerTypes = 'CodeIcon' | 'FormulaIcon' | 'AIIcon' | 'ErrorIcon' | 'SQLIcon';
 
 export class CellsMarkers extends Container {
   private visibleIndex = 0;
@@ -38,6 +38,11 @@ export class CellsMarkers extends Container {
       child.texture = Texture.from('images/ai-icon.png');
       child.tint = colors.cellColorUserAI;
       child.width = child.height = 4;
+    } else if (type === 'SQLIcon') {
+      child.position.set(x + 1.25, y + 1.25);
+      child.texture = Texture.from('images/sql-icon.png');
+      child.tint = colors.cellColorUserSQL;
+      child.width = child.height = 4;
     } else if (type === 'ErrorIcon') {
       child.position.set(x, y);
       child.texture = Texture.from('images/error-icon.png');

src/theme/colors.ts

@@ -5,6 +5,7 @@ export const colors = {
   cellColorUserPython: 0x3776ab,
   cellColorUserFormula: 0x8c1a6a,
   cellColorUserAI: 0x1a8c5d,
+  cellColorUserSQL: 0xe0c75a,
   cellColorError: 0xf25f5c,
   cursorCell: 0x6cd4ff,
   independence: 0x5d576b,
@@ -23,5 +24,6 @@ export const colors = {
   languagePython: '#3776ab',
   languageFormula: '#8c1a6a',
   languageAI: '#1a8c5d',
+  languageSQL: '#e0c75a',
   error: '#f25f5c',
 };

src/ui/menus/CellTypeMenu/CellTypeMenu.tsx

@@ -64,9 +64,8 @@ let CELL_TYPE_OPTIONS = [
   {
     name: 'SQL Query',
     mode: 'SQL',
-    icon: <Sql color="disabled" />,
-    description: 'Import your data with queries.',
-    disabled: true,
+    icon: <Sql sx={{ color: colors.languageSQL }} />,
+    description: 'Powered by DuckDB.',
   },
   {
     name: 'JavaScript',

src/ui/menus/CodeEditor/CodeEditor.tsx

@@ -23,7 +23,7 @@ import { updateCellAndDCells } from '../../../grid/actions/updateCellAndDCells';
 import { FormulaCompletionProvider, FormulaLanguageConfig } from './FormulaLanguageModel';
 import { CellEvaluationResult } from '../../../grid/computations/types';
 import { Close, FiberManualRecord, PlayArrow, Subject } from '@mui/icons-material';
-import { AI, Formula, Python } from '../../icons';
+import { AI, Formula, Python, Sql } from '../../icons';
 import { TooltipHint } from '../../components/TooltipHint';
 import { KeyboardSymbols } from '../../../helpers/keyboardSymbols';
 import { ResizeControl } from './ResizeControl';
@@ -82,6 +82,8 @@ export const CodeEditor = (props: CodeEditorProps) => {
       ? selectedCell?.formula_code !== editorContent
       : editorMode === 'AI'
       ? selectedCell?.ai_prompt !== editorContent
+      : editorMode === 'SQL'
+      ? selectedCell?.sql_statement !== editorContent
       : false);
 
   // When changing mode
@@ -161,6 +163,8 @@ export const CodeEditor = (props: CodeEditorProps) => {
         setEditorContent(cell?.formula_code);
       } else if (editorMode === 'AI') {
         setEditorContent(cell?.ai_prompt);
+      } else if (editorMode === 'SQL') {
+        setEditorContent(cell?.sql_statement);
       }
     } else {
       // create blank cell
@@ -192,6 +196,8 @@ export const CodeEditor = (props: CodeEditorProps) => {
       selectedCell.formula_code = editorContent;
     } else if (editorMode === 'AI') {
       selectedCell.ai_prompt = editorContent;
+    } else if (editorMode === 'SQL') {
+      selectedCell.sql_statement = editorContent;
     }
 
     await updateCellAndDCells({
@@ -309,6 +315,8 @@ export const CodeEditor = (props: CodeEditorProps) => {
             <Formula sx={{ color: colors.languageFormula }} fontSize="small" />
           ) : editorMode === 'AI' ? (
             <AI sx={{ color: colors.languageAI }} fontSize="small" />
+          ) : editorMode === 'SQL' ? (
+            <Sql sx={{ color: colors.languageSQL }} fontSize="small" />
           ) : (
             <Subject />
           )}
@@ -318,7 +326,7 @@ export const CodeEditor = (props: CodeEditorProps) => {
             }}
           >
             Cell ({selectedCell.x}, {selectedCell.y}) -{' '}
-            {selectedCell.type === 'AI' ? 'AI' : capitalize(selectedCell.type)}
+            {['AI', 'SQL'].includes(selectedCell.type) ? selectedCell.type : capitalize(selectedCell.type)}
             {hasUnsavedChanges && (
               <TooltipHint title="Your changes haven’t been saved or run">
                 <FiberManualRecord
@@ -369,7 +377,15 @@ export const CodeEditor = (props: CodeEditorProps) => {
         <Editor
           height="100%"
           width="100%"
-          language={editorMode === 'PYTHON' ? 'python' : editorMode === 'FORMULA' ? 'formula' : 'plaintext'}
+          language={
+            editorMode === 'PYTHON'
+              ? 'python'
+              : editorMode === 'FORMULA'
+              ? 'formula'
+              : editorMode === 'SQL'
+              ? 'sql'
+              : 'plaintext'
+          }
           value={editorContent}
           onChange={(value) => {
             setEditorContent(value);
@@ -405,6 +421,7 @@ export const CodeEditor = (props: CodeEditorProps) => {
       >
         {(editorInteractionState.mode === 'PYTHON' ||
           editorInteractionState.mode === 'FORMULA' ||
+          editorInteractionState.mode === 'SQL' ||
           editorInteractionState.mode === 'AI') && (
           <Console evalResult={evalResult} editorMode={editorMode} editorContent={editorContent} />
         )}

src/ui/menus/CodeEditor/Console.tsx

@@ -187,6 +187,12 @@ export function Console({ evalResult, editorMode, editorContent }: ConsoleProps)
                 in Quadratic, you can be confident that your data is always accurate and up-to-date.
               </p>
             </>
+          ) : editorMode === 'SQL' ? (
+            <>
+              <br></br>
+              <Chip label="Experimental" size="small" color="warning" variant="outlined" />
+              <h3>SQL Docs</h3>
+            </>
           ) : (
             <>
               <h3>Spreadsheet formulas</h3>