Require pagination on table panels, fall back to regular input when s…

…hapes aren't valid in FieldPicker, better Show/Hide Options UI, allow Google Sheets (#235) * Basic pagination for table * Try a new options styling * Increase graph max * Started supporting google sheets * Google sheets working * Cleanup * Fixes for fmt * Fix for gofmt * Fix for tests * Fixes for panel names * Unnecessary neo4j step * Move everything to bufio * Fixes for eslint * Tests passing locally * Start moving fetch results to runner * Sketch out fetch * Add tests for graphtable * Graph and table panels return values * Fixes for tests * No expand when no body * Cleanup page buttons * Fix google sheet test * Select so that we can break
multiprocessio · May 2, 2022 · 69a8137 · 69a8137
1 parent 5a50d7f
commit 69a8137
Show file tree

Hide file tree

Showing 40 changed files with 1,603 additions and 933 deletions.
diff --git a/.eslintignore b/.eslintignore
@@ -1,2 +1 @@
-ui/scripts/*
-ui/state.test.js
+ui/scripts/*
diff --git a/desktop/panel/columns.test.js b/desktop/panel/columns.test.js
@@ -32,6 +32,7 @@ for (const runner of RUNNERS) {
     const tp = new TablePanelInfo(null, {
       columns: [{ field: 'a' }],
       panelSource: lp.id,
+      name: 'Table',
     });
 
     let finished = false;
@@ -59,7 +60,7 @@ for (const runner of RUNNERS) {
         expect(result.preview).toStrictEqual(preview(testData));
         expect(result.contentType).toBe('application/json');
 
-        const p = await makeEvalHandler().handler(
+        const p = await makeEvalHandler(runner).handler(
           project.projectName,
           { panelId: tp.id },
           dispatch

diff --git a/desktop/panel/columns.ts b/desktop/panel/columns.ts
@@ -1,65 +1,11 @@
 import fs from 'fs';
 import { PanelBody } from '../../shared/rpc';
-import {
-  GraphPanelInfo,
-  PanelInfo,
-  ProjectState,
-  TablePanelInfo,
-} from '../../shared/state';
-import { columnsFromObject } from '../../shared/table';
 import { Dispatch, RPCHandler } from '../rpc';
 import { getProjectResultsFile } from '../store';
-import { getPanelResult, getProjectAndPanel } from './shared';
-import { EvalHandlerExtra, EvalHandlerResponse, guardPanel } from './types';
-
-export async function evalColumns(
-  project: ProjectState,
-  panel: PanelInfo,
-  { idMap }: EvalHandlerExtra,
-  dispatch: Dispatch
-): Promise<EvalHandlerResponse> {
-  let columns: Array<string>;
-  let panelSource: string;
-  if (panel.type === 'graph') {
-    const gp = panel as GraphPanelInfo;
-    columns = [
-      gp.graph.x,
-      gp.graph.uniqueBy,
-      ...gp.graph.ys.map((y) => y.field),
-    ].filter(Boolean);
-    panelSource = gp.graph.panelSource;
-  } else if (panel.type === 'table') {
-    const tp = panel as TablePanelInfo;
-    columns = tp.table.columns.map((c) => c.field);
-    panelSource = tp.table.panelSource;
-  } else {
-    // Let guardPanel throw a nice error.
-    guardPanel<GraphPanelInfo>(panel, 'graph');
-  }
-
-  if (!panelSource) {
-    throw new Error('Panel source not specified, cannot eval.');
-  }
-
-  const { value } = await getPanelResult(
-    dispatch,
-    project.projectName,
-    panelSource
-  );
-
-  const valueWithRequestedColumns = columnsFromObject(
-    value,
-    columns,
-    // Assumes that position always comes before panel name in the idmap
-    +Object.keys(idMap).find((key) => idMap[key] === panelSource)
-  );
-
-  return {
-    value: valueWithRequestedColumns,
-    returnValue: true,
-  };
-}
+import { getProjectAndPanel } from './shared';
+import { EvalHandlerResponse } from './types';
 
+// TODO: this needs to be ported to go
 export const fetchResultsHandler: RPCHandler<PanelBody, EvalHandlerResponse> = {
   resource: 'fetchResults',
   handler: async function (
@@ -75,6 +21,7 @@ export const fetchResultsHandler: RPCHandler<PanelBody, EvalHandlerResponse> = {
 
     // Maybe the only appropriate place to call this in this package?
     const projectResultsFile = getProjectResultsFile(projectId);
+    // TODO: this is a 4GB file limit!
     const f = fs.readFileSync(projectResultsFile + panel.id);
 
     // Everything gets stored as JSON on disk. Even literals and files get rewritten as JSON.

diff --git a/desktop/panel/database.test.js b/desktop/panel/database.test.js
@@ -470,9 +470,6 @@ for (const subprocess of RUNNERS) {
         return;
       }
 
-      // Mongo doesn't work yet.
-      return;
-
       const connectors = [
         new DatabaseConnectorInfo({
           type: 'google-sheets',
@@ -496,11 +493,11 @@ for (const subprocess of RUNNERS) {
 
           const v = JSON.parse(panelValueBuffer.toString());
           expect(v).toStrictEqual([
-            { age: 52, name: 'Emma' },
-            { age: 50, name: 'Karl' },
-            { age: 43, name: 'Garry' },
-            { age: 41, name: 'Nile' },
-            { age: 39, name: 'Mina' },
+            { age: '43', name: 'Garry' },
+            { age: '39', name: 'Mina' },
+            { age: '50', name: 'Karl' },
+            { age: '41', name: 'Nile' },
+            { age: '52', name: 'Emma' },
           ]);
 
           finished = true;

diff --git a/desktop/panel/eval.ts b/desktop/panel/eval.ts
@@ -1,28 +1,14 @@
 import { execFile } from 'child_process';
 import fs from 'fs';
-import jsesc from 'jsesc';
 import circularSafeStringify from 'json-stringify-safe';
 import { EOL } from 'os';
 import path from 'path';
-import { preview } from 'preview';
-import { shape, Shape } from 'shape';
 import { file as makeTmpFile } from 'tmp-promise';
-import {
-  Cancelled,
-  EVAL_ERRORS,
-  InvalidDependentPanelError,
-  NoResultError,
-} from '../../shared/errors';
+import { Cancelled, EVAL_ERRORS, NoResultError } from '../../shared/errors';
 import log from '../../shared/log';
 import { newId } from '../../shared/object';
 import { PanelBody } from '../../shared/rpc';
-import {
-  ConnectorInfo,
-  PanelInfo,
-  PanelInfoType,
-  PanelResult,
-  ProjectState,
-} from '../../shared/state';
+import { ConnectorInfo, PanelInfo, PanelResult } from '../../shared/state';
 import {
   CODE_ROOT,
   DISK_ROOT,
@@ -37,31 +23,6 @@ import { parsePartialJSONFile } from '../partial';
 import { Dispatch, RPCHandler } from '../rpc';
 import { getProjectResultsFile } from '../store';
 import { getProjectAndPanel } from './shared';
-import { EvalHandlerExtra, EvalHandlerResponse } from './types';
-
-type EvalHandler = (
-  project: ProjectState,
-  panel: PanelInfo,
-  extra: EvalHandlerExtra,
-  dispatch: Dispatch
-) => Promise<EvalHandlerResponse>;
-
-function unimplementedInJavaScript(): EvalHandler {
-  return function () {
-    throw new Error('There is a bug, this condition should not be possible.');
-  };
-}
-
-const EVAL_HANDLERS: { [k in PanelInfoType]: () => EvalHandler } = {
-  table: () => require('./columns').evalColumns,
-  graph: () => require('./columns').evalColumns,
-  literal: unimplementedInJavaScript,
-  database: unimplementedInJavaScript,
-  file: unimplementedInJavaScript,
-  http: unimplementedInJavaScript,
-  program: unimplementedInJavaScript,
-  filagg: unimplementedInJavaScript,
-};
 
 const runningProcesses: Record<string, Set<number>> = {};
 const cancelledPids = new Set<number>();
@@ -84,10 +45,6 @@ function killAllByPanelId(panelId: string) {
   }
 }
 
-function canUseGoRunner(panel: PanelInfo, connectors: ConnectorInfo[]) {
-  return !['table', 'graph'].includes(panel.type);
-}
-
 export async function evalInSubprocess(
   subprocess: {
     node: string;
@@ -122,7 +79,7 @@ export async function evalInSubprocess(
       args.push(SETTINGS_FILE_FLAG, subprocess.settingsFileOverride);
     }
 
-    if (subprocess.go && canUseGoRunner(panel, connectors)) {
+    if (subprocess.go) {
       base = subprocess.go;
       args.shift();
     }
@@ -213,6 +170,14 @@ export async function evalInSubprocess(
         throw e;
       }
     }
+
+    // Table and graph panels get their results passed back to me displayed in the UI
+    if (['table', 'graph'].includes(panel.type)) {
+      const projectResultsFile = getProjectResultsFile(projectName);
+      const bytes = fs.readFileSync(projectResultsFile + panel.id);
+      rm.value = JSON.parse(bytes.toString());
+    }
+
     return [{ ...rm, ...resultMeta }, stderr];
   } finally {
     try {
@@ -227,30 +192,6 @@ export async function evalInSubprocess(
   }
 }
 
-function assertValidDependentPanels(
-  projectId: string,
-  content: string,
-  idMap: Record<string | number, string>
-) {
-  const projectResultsFile = getProjectResultsFile(projectId);
-  const re =
-    /(DM_getPanel\((?<number>[0-9]+)\))|(DM_getPanel\((?<singlequote>'(?:[^'\\]|\\.)*\')\))|(DM_getPanel\((?<doublequote>"(?:[^"\\]|\\.)*\")\))/g;
-  let match = null;
-  while ((match = re.exec(content)) !== null) {
-    if (match && match.groups) {
-      const { number, singlequote, doublequote } = match.groups;
-      let m = doublequote || singlequote || number;
-      if (["'", '"'].includes(m.charAt(0))) {
-        m = m.slice(1, m.length - 1);
-      }
-
-      if (!fs.existsSync(projectResultsFile + idMap[m])) {
-        throw new InvalidDependentPanelError(m);
-      }
-    }
-  }
-}
-
 async function evalNoUpdate(
   projectId: string,
   body: PanelBody,
@@ -260,7 +201,7 @@ async function evalNoUpdate(
     go?: string;
   }
 ): Promise<[Partial<PanelResult>, string]> {
-  const { project, panel, panelPage } = await getProjectAndPanel(
+  const { project, panel } = await getProjectAndPanel(
     dispatch,
     projectId,
     body.panelId
@@ -273,66 +214,16 @@ async function evalNoUpdate(
     body: { data: new PanelResult(), panelId: panel.id },
   });
 
-  if (subprocessEval) {
-    return evalInSubprocess(
-      subprocessEval,
-      project.projectName,
-      panel,
-      project.connectors
-    );
+  if (!subprocessEval) {
+    throw new Error('Developer error: all eval must use subprocess');
   }
 
-  const idMap: Record<string | number, string> = {};
-  const idShapeMap: Record<string | number, Shape> = {};
-  project.pages[panelPage].panels.forEach((p, i) => {
-    idMap[i] = p.id;
-    idMap[p.name] = p.id;
-    idShapeMap[i] = p.resultMeta.shape;
-    idShapeMap[p.name] = p.resultMeta.shape;
-  });
-
-  assertValidDependentPanels(projectId, panel.content, idMap);
-
-  const evalHandler = EVAL_HANDLERS[panel.type]();
-  const res = await evalHandler(
-    project,
+  return evalInSubprocess(
+    subprocessEval,
+    project.projectName,
     panel,
-    {
-      idMap,
-      idShapeMap,
-    },
-    dispatch
+    project.connectors
   );
-
-  // TODO: is it a problem panels like Program skip this escaping?
-  // This library is important for escaping responses otherwise some
-  // characters can blow up various panel processes.
-  const json = jsesc(res.value, { quotes: 'double', json: true });
-
-  if (!res.skipWrite) {
-    const projectResultsFile = getProjectResultsFile(projectId);
-    fs.writeFileSync(projectResultsFile + panel.id, json);
-  }
-
-  const s = shape(res.value);
-
-  return [
-    {
-      stdout: res.stdout || '',
-      preview: preview(res.value),
-      shape: s,
-      value: res.returnValue ? res.value : null,
-      size: res.size === undefined ? json.length : res.size,
-      arrayCount:
-        res.arrayCount === undefined
-          ? s.kind === 'array'
-            ? (res.value || []).length
-            : null
-          : res.arrayCount,
-      contentType: res.contentType || 'application/json',
-    },
-    '',
-  ];
 }
 
 export const makeEvalHandler = (subprocessEval?: {