diff --git a/desktop/panel/eval.ts b/desktop/panel/eval.ts index 9d931fee6..ef74182e6 100644 --- a/desktop/panel/eval.ts +++ b/desktop/panel/eval.ts @@ -205,8 +205,13 @@ export const makeEvalHandler = ( preview: preview(res.value), shape: s, value: res.returnValue ? res.value : null, - size: json.length, - arrayCount: s.kind === 'array' ? (res.value || []).length : null, + size: res.size === undefined ? json.length : res.size, + arrayCount: + res.arrayCount === undefined + ? s.kind === 'array' + ? (res.value || []).length + : null + : res.arrayCount, contentType: res.contentType || 'application/json', }; }, diff --git a/desktop/panel/program.test.js b/desktop/panel/program.test.js index 1732e6ad9..9a288e9f2 100644 --- a/desktop/panel/program.test.js +++ b/desktop/panel/program.test.js @@ -1,4 +1,5 @@ const path = require('path'); +import { file as makeTmpFile } from 'tmp-promise'; const { LANGUAGES } = require('../../shared/languages'); const { getProjectResultsFile } = require('../store'); const fs = require('fs'); @@ -7,6 +8,7 @@ const { updateProjectHandler } = require('../store'); const { CODE_ROOT } = require('../constants'); const { makeEvalHandler } = require('./eval'); const { inPath, withSavedPanels } = require('./testutil'); +const { parsePartialJSONFile } = require('./program'); const TESTS = [ { @@ -172,3 +174,47 @@ for (const t of TESTS) { } }); } + +describe('parsePartialJSONFile', function parsePartialJSONFileTest() { + test('correctly fills out partial file', async function fillsOutPartial() { + const f = await makeTmpFile(); + try { + const whole = '[{"foo": "bar"}, {"big": "bad"}]'; + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ foo: 'bar' }]); + } finally { + f.cleanup(); + } + }); + + test('handles open-close in string', async function handlesOpenCloseInString() { + for (const c of ['{', '}', '[', ']']) { + const f = await makeTmpFile(); + try { + const whole = `[{"foo": "${c}bar"}, {"big": "bad"}]`; + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ foo: `${c}bar` }]); + } finally { + f.cleanup(); + } + } + }); + + test('handles escaped quotes in string', async function handlesEscapedQuotesInString() { + const f = await makeTmpFile(); + try { + const whole = `[{"foo":"bar\\" { "},{"big":"bad"}]`; + expect(JSON.stringify(JSON.parse(whole))).toEqual(whole); + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ foo: `bar" { ` }]); + } finally { + f.cleanup(); + } + }); +}); diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 057d3ffdc..0c482da29 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -10,6 +10,103 @@ import { SETTINGS } from '../settings'; import { getProjectResultsFile } from '../store'; import { EvalHandlerExtra, EvalHandlerResponse, guardPanel } from './types'; +export function parsePartialJSONFile(file: string, maxBytesToRead: number) { + let fd: number; + try { + fd = fs.openSync(file, 'r'); + } catch (e) { + throw new NoResultError(); + } + + const { size } = fs.statSync(file); + + if (size < maxBytesToRead) { + const f = fs.readFileSync(file).toString(); + const value = JSON.parse(f); + return { + size, + value, + arrayCount: f.charAt(0) === '[' ? value.length : null, + }; + } + + try { + let done = false; + let f = ''; + const incomplete = []; + let inString = false; + + while (!done) { + const bufferSize = 1024; + const b = Buffer.alloc(bufferSize); + const bytesRead = fs.readSync(fd, b); + + // To be able to iterate over code points + let bs = Array.from(b.toString()); + outer: for (let i = 0; i < bs.length; i++) { + const c = bs[i]; + if (c !== '"' && inString) { + continue; + } + + switch (c) { + case '"': + const previous = + i + bs.length === 0 + ? '' + : i > 0 + ? bs[i - 1] + : f.charAt(f.length - 1); + const isEscaped = previous === '\\'; + if (!isEscaped) { + inString = !inString; + } + break; + case '{': + case '[': + incomplete.push(c); + break; + case ']': + case '}': + if (f.length + bufferSize >= maxBytesToRead) { + bs = bs.slice(0, i); + // Need to not count additional openings after this + done = true; + break outer; + } + + // Otherwise, pop it + incomplete.pop(); + break; + } + } + + f += bs.join(''); + if (bytesRead < bufferSize) { + break; + } + } + + while (incomplete.length) { + if (incomplete.pop() === '{') { + f += '}'; + } else { + f += ']'; + } + } + + const value = JSON.parse(f); + + return { + size, + value, + arrayCount: f.charAt(0) === '[' ? 'More than ' + value.length : null, + }; + } finally { + fs.closeSync(fd); + } +} + export async function evalProgram( project: ProjectState, panel: PanelInfo, @@ -76,18 +173,17 @@ export async function evalProgram( throw new Error(stderr); } - let f: Buffer; - try { - f = fs.readFileSync(projectResultsFile + ppi.id); - } catch (e) { - throw new NoResultError(); - } - const value = JSON.parse(f.toString()); + const { size, value, arrayCount } = parsePartialJSONFile( + projectResultsFile + ppi.id, + 100_000 + ); return { skipWrite: true, value, stdout: out, + size, + arrayCount, }; } catch (e) { const resultsFileRE = new RegExp( diff --git a/desktop/panel/types.ts b/desktop/panel/types.ts index 38f77343a..2a0667426 100644 --- a/desktop/panel/types.ts +++ b/desktop/panel/types.ts @@ -7,6 +7,8 @@ export type EvalHandlerResponse = { stdout?: string; contentType?: string; value: any; + size?: any; + arrayCount?: any; }; export type EvalHandlerExtra = { diff --git a/shared/languages/javascript.ts b/shared/languages/javascript.ts index b0a030a5e..ff6d9adbc 100644 --- a/shared/languages/javascript.ts +++ b/shared/languages/javascript.ts @@ -38,7 +38,21 @@ function DM_getPanel(i) { } function DM_setPanel(v) { const fs = require('fs'); - fs.writeFileSync('${resultsFile + panelId}', JSON.stringify(v)); + const fd = fs.openSync('${resultsFile + panelId}', 'w'); + if (Array.isArray(v)) { + fs.writeSync(fd, '['); + for (let i = 0; i < v.length; i++) { + const row = v[i]; + let rowJSON = JSON.stringify(row); + if (i < v.length - 1) { + rowJSON += ','; + } + fs.writeSync(fd, rowJSON); + } + fs.writeSync(fd, ']'); + } else { + fs.writeSync(fd, JSON.stringify(v)); + } }`; } diff --git a/ui/app.test.jsx b/ui/app.test.jsx index d455e615f..074c14953 100644 --- a/ui/app.test.jsx +++ b/ui/app.test.jsx @@ -96,7 +96,6 @@ test( for (let i = 0; i < connectors.length; i++) { const c = connectors.at(i); act(() => { - console.log(c.debug()); c.find({ 'data-testid': 'show-hide-connector', icon: true, diff --git a/ui/style.css b/ui/style.css index fa2067b56..21769fe02 100644 --- a/ui/style.css +++ b/ui/style.css @@ -43,6 +43,7 @@ a:hover { .main-body { flex: 1; + overflow-y: auto; } header { @@ -342,7 +343,6 @@ label.select select { border-bottom: 1px solid #ccc; background: white; width: 350px; - overflow-y: auto; background: #fbfbfb; padding: 15px; } @@ -812,7 +812,7 @@ th { } tbody tr:nth-child(odd) { - background: #f9f0ff; + background: #f2f2f7; } .confirm-wrapper { @@ -887,6 +887,10 @@ body.dark .connector--expanded { body.dark .button:hover { background: #383852; } + +body.dark tbody tr:nth-child(odd) { + background: #31314e; +} /* End cssplus break */ body.dark {