From ddc402f1dc9a5f962d0b8578b759c86a70651ce1 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 21:47:09 +0000 Subject: [PATCH 01/18] Fix table style and add support for reading partial program results --- desktop/panel/eval.ts | 4 +-- desktop/panel/program.ts | 77 ++++++++++++++++++++++++++++++++++++---- desktop/panel/types.ts | 2 ++ ui/style.css | 6 +++- 4 files changed, 79 insertions(+), 10 deletions(-) diff --git a/desktop/panel/eval.ts b/desktop/panel/eval.ts index 9d931fee6..743cdbb20 100644 --- a/desktop/panel/eval.ts +++ b/desktop/panel/eval.ts @@ -205,8 +205,8 @@ export const makeEvalHandler = ( preview: preview(res.value), shape: s, value: res.returnValue ? res.value : null, - size: json.length, - arrayCount: s.kind === 'array' ? (res.value || []).length : null, + size: res.size === undefined ? json.length : res.size, + arrayCount: res.arrayCount === undefined ? ( s.kind === 'array' ? (res.value || []).length : null ) : res.arrayCount, contentType: res.contentType || 'application/json', }; }, diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 057d3ffdc..a4063dae7 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -10,6 +10,73 @@ import { SETTINGS } from '../settings'; import { getProjectResultsFile } from '../store'; import { EvalHandlerExtra, EvalHandlerResponse, guardPanel } from './types'; +export function parsePartialJSONFile(file, maxBytesToRead) { + let fd: number; + try { + fd = fs.openSync(file); + } catch (e) { + throw new NoResultError(); + } + + const { size } = fs.statSync(file); + + if (size < maxBytesToRead) { + const f = fs.readFileSync(file).toString(); + const value = JSON.parse(f); + return { size, value, arrayCount: f.charAt(0) === '[' ? value.length : null}; + } + + try { + let done = false; + let f = ''; + const incomplete = [] + + while (true) { + const b = new Buffer(); + const bufferSize = 1024; + fs.readSync(fd, b, 0, bufferSize); + const bs = b.toString(); + + let closingIndex = bufferSize; + outer: + for (let i = 0; i < bs.length; i++) { + const c = bs.charAt(i); + switch (c) { + case '{': + case '[': + incomplete.push(c); + break; + case ']': + case '}': + incomplete.pop(); + closingIndex = i; + if (f.length + bufferSize >= maxBytesToRead) { + // Need to not count additional openings after this + break outer; + } + break; + } + } + + f += bs.slice(0, closingIndex); + } + + while (incomplete.length) { + if (incomplete.pop() === '{') { + f += '}'; + } else { + f += ']'; + } + } + + const value = JSON.parse(f.toString()); + + return { size, value, arrayCount: value.charAt(0) === '[' ? ('More than ' + value.length) : null }; + } finally { + fs.closeSync(fd); + } +} + export async function evalProgram( project: ProjectState, panel: PanelInfo, @@ -76,18 +143,14 @@ export async function evalProgram( throw new Error(stderr); } - let f: Buffer; - try { - f = fs.readFileSync(projectResultsFile + ppi.id); - } catch (e) { - throw new NoResultError(); - } - const value = JSON.parse(f.toString()); + const { size, value, arrayCount } = parsePartialJSONFile(projectResultsFile + ppi.id, 100_000); return { skipWrite: true, value, stdout: out, + size, + arrayCount, }; } catch (e) { const resultsFileRE = new RegExp( diff --git a/desktop/panel/types.ts b/desktop/panel/types.ts index 38f77343a..2a0667426 100644 --- a/desktop/panel/types.ts +++ b/desktop/panel/types.ts @@ -7,6 +7,8 @@ export type EvalHandlerResponse = { stdout?: string; contentType?: string; value: any; + size?: any; + arrayCount?: any; }; export type EvalHandlerExtra = { diff --git a/ui/style.css b/ui/style.css index fa2067b56..cc5e9fcce 100644 --- a/ui/style.css +++ b/ui/style.css @@ -812,7 +812,7 @@ th { } tbody tr:nth-child(odd) { - background: #f9f0ff; + background: #f2f2f7; } .confirm-wrapper { @@ -887,6 +887,10 @@ body.dark .connector--expanded { body.dark .button:hover { background: #383852; } + +body.dark tbody tr:nth-child(odd) { + background: #31314e; +} /* End cssplus break */ body.dark { From af656c3956ea2f6d5a399344d1fc0b204b9e5013 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 21:50:46 +0000 Subject: [PATCH 02/18] Alloc buffer --- desktop/panel/program.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index a4063dae7..f6b144fb0 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -32,8 +32,8 @@ export function parsePartialJSONFile(file, maxBytesToRead) { const incomplete = [] while (true) { - const b = new Buffer(); const bufferSize = 1024; + const b = Buffer.alloc(bufferSize); fs.readSync(fd, b, 0, bufferSize); const bs = b.toString(); From 613da5dc5c6811e55ce40f45d503030641fafb3e Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 21:53:04 +0000 Subject: [PATCH 03/18] Fix sidebar --- ui/style.css | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ui/style.css b/ui/style.css index cc5e9fcce..7d84f3734 100644 --- a/ui/style.css +++ b/ui/style.css @@ -42,7 +42,8 @@ a:hover { } .main-body { - flex: 1; + flex: 1; + overflow-y: auto; } header { @@ -342,7 +343,6 @@ label.select select { border-bottom: 1px solid #ccc; background: white; width: 350px; - overflow-y: auto; background: #fbfbfb; padding: 15px; } From 6b6ab286bf9b77cfe1aac9b75ab9568d328890dc Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 21:55:27 +0000 Subject: [PATCH 04/18] Add console log --- desktop/panel/program.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index f6b144fb0..f18a5f922 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -58,6 +58,7 @@ export function parsePartialJSONFile(file, maxBytesToRead) { } } + console.log(closingIndex); f += bs.slice(0, closingIndex); } From 34eef8481aa5a265193c670bc893fa39b11c0cba Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 21:58:31 +0000 Subject: [PATCH 05/18] Try something else --- desktop/panel/program.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index f18a5f922..92b8474c2 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -49,8 +49,8 @@ export function parsePartialJSONFile(file, maxBytesToRead) { case ']': case '}': incomplete.pop(); - closingIndex = i; if (f.length + bufferSize >= maxBytesToRead) { + closingIndex = i; // Need to not count additional openings after this break outer; } @@ -58,7 +58,6 @@ export function parsePartialJSONFile(file, maxBytesToRead) { } } - console.log(closingIndex); f += bs.slice(0, closingIndex); } From 0dc65465153d35f6b4f8f3d6d88b34cb2d8f6e0c Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 22:03:39 +0000 Subject: [PATCH 06/18] Iterate over code points --- desktop/panel/program.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 92b8474c2..e3cae95a1 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -35,12 +35,12 @@ export function parsePartialJSONFile(file, maxBytesToRead) { const bufferSize = 1024; const b = Buffer.alloc(bufferSize); fs.readSync(fd, b, 0, bufferSize); - const bs = b.toString(); - let closingIndex = bufferSize; + // To be able to iterate over code points + let bs = Array.from(b.toString()); outer: for (let i = 0; i < bs.length; i++) { - const c = bs.charAt(i); + const c = bs[i]; switch (c) { case '{': case '[': @@ -50,7 +50,7 @@ export function parsePartialJSONFile(file, maxBytesToRead) { case '}': incomplete.pop(); if (f.length + bufferSize >= maxBytesToRead) { - closingIndex = i; + bs = bs.slice(0, i); // Need to not count additional openings after this break outer; } @@ -58,7 +58,7 @@ export function parsePartialJSONFile(file, maxBytesToRead) { } } - f += bs.slice(0, closingIndex); + f += bs.join(''); } while (incomplete.length) { From a455eea8e67deccfb3466b9f705a6f8ce770c96d Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 22:07:44 +0000 Subject: [PATCH 07/18] Actually exit loop --- desktop/panel/program.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index e3cae95a1..4a086823c 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -31,7 +31,7 @@ export function parsePartialJSONFile(file, maxBytesToRead) { let f = ''; const incomplete = [] - while (true) { + while (!done) { const bufferSize = 1024; const b = Buffer.alloc(bufferSize); fs.readSync(fd, b, 0, bufferSize); @@ -52,6 +52,7 @@ export function parsePartialJSONFile(file, maxBytesToRead) { if (f.length + bufferSize >= maxBytesToRead) { bs = bs.slice(0, i); // Need to not count additional openings after this + done = true; break outer; } break; From 1b1dfc22c495c844255a498c369e2f470848e353 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 22:23:26 +0000 Subject: [PATCH 08/18] Fix and test --- desktop/panel/program.test.js | 319 ++++++++++++++++++---------------- desktop/panel/program.ts | 13 +- 2 files changed, 176 insertions(+), 156 deletions(-) diff --git a/desktop/panel/program.test.js b/desktop/panel/program.test.js index 1732e6ad9..f95abbfaa 100644 --- a/desktop/panel/program.test.js +++ b/desktop/panel/program.test.js @@ -1,4 +1,5 @@ const path = require('path'); +import { file as makeTmpFile } from 'tmp-promise'; const { LANGUAGES } = require('../../shared/languages'); const { getProjectResultsFile } = require('../store'); const fs = require('fs'); @@ -7,168 +8,184 @@ const { updateProjectHandler } = require('../store'); const { CODE_ROOT } = require('../constants'); const { makeEvalHandler } = require('./eval'); const { inPath, withSavedPanels } = require('./testutil'); +const { parsePartialJSONFile } = require('./program'); -const TESTS = [ - { - type: 'javascript', - content: - 'const prev = DM_getPanel(0); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);', - condition: true, - }, - { - type: 'javascript', - content: `const prev = DM_getPanel('Raw Data'); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);`, - condition: true, - }, - { - type: 'sql', - content: 'SELECT name, age::INT + 10 AS age FROM DM_getPanel(0)', - condition: true, - }, - { - type: 'sql', - content: `SELECT name, age::INT + 10 AS age FROM DM_getPanel('Raw Data')`, - condition: true, - }, - // Rest are only mandatory-tested on Linux to make CI easier for now - { - type: 'python', - content: - 'prev = DM_getPanel(0)\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)', - condition: process.platform === 'linux' || inPath('python3'), - }, - { - type: 'python', - content: `prev = DM_getPanel('Raw Data')\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)`, - condition: process.platform === 'linux' || inPath('python3'), - }, - { - type: 'ruby', - content: - 'prev = DM_getPanel(0)\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)', - condition: process.platform === 'linux' || inPath('ruby'), - }, - { - type: 'ruby', - content: `prev = DM_getPanel('Raw Data')\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)`, - condition: process.platform === 'linux' || inPath('ruby'), - }, - { - type: 'julia', - content: - 'prev = DM_getPanel(0)\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)', - condition: process.platform === 'linux' || inPath('julia'), - }, - { - type: 'julia', - content: `prev = DM_getPanel("Raw Data")\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)`, - condition: process.platform === 'linux' || inPath('julia'), - }, - { - type: 'r', - content: - 'prev = DM_getPanel(0)\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)', - condition: process.platform === 'linux' || inPath('Rscript'), - }, - { - type: 'r', - content: `prev = DM_getPanel('Raw Data')\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)`, - condition: process.platform === 'linux' || inPath('Rscript'), - }, -]; +// const TESTS = [ +// { +// type: 'javascript', +// content: +// 'const prev = DM_getPanel(0); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);', +// condition: true, +// }, +// { +// type: 'javascript', +// content: `const prev = DM_getPanel('Raw Data'); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);`, +// condition: true, +// }, +// { +// type: 'sql', +// content: 'SELECT name, age::INT + 10 AS age FROM DM_getPanel(0)', +// condition: true, +// }, +// { +// type: 'sql', +// content: `SELECT name, age::INT + 10 AS age FROM DM_getPanel('Raw Data')`, +// condition: true, +// }, +// // Rest are only mandatory-tested on Linux to make CI easier for now +// { +// type: 'python', +// content: +// 'prev = DM_getPanel(0)\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)', +// condition: process.platform === 'linux' || inPath('python3'), +// }, +// { +// type: 'python', +// content: `prev = DM_getPanel('Raw Data')\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)`, +// condition: process.platform === 'linux' || inPath('python3'), +// }, +// { +// type: 'ruby', +// content: +// 'prev = DM_getPanel(0)\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)', +// condition: process.platform === 'linux' || inPath('ruby'), +// }, +// { +// type: 'ruby', +// content: `prev = DM_getPanel('Raw Data')\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)`, +// condition: process.platform === 'linux' || inPath('ruby'), +// }, +// { +// type: 'julia', +// content: +// 'prev = DM_getPanel(0)\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)', +// condition: process.platform === 'linux' || inPath('julia'), +// }, +// { +// type: 'julia', +// content: `prev = DM_getPanel("Raw Data")\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)`, +// condition: process.platform === 'linux' || inPath('julia'), +// }, +// { +// type: 'r', +// content: +// 'prev = DM_getPanel(0)\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)', +// condition: process.platform === 'linux' || inPath('Rscript'), +// }, +// { +// type: 'r', +// content: `prev = DM_getPanel('Raw Data')\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)`, +// condition: process.platform === 'linux' || inPath('Rscript'), +// }, +// ]; -for (const t of TESTS) { - if (!t.condition) { - continue; - } +// for (const t of TESTS) { +// if (!t.condition) { +// continue; +// } - describe(t.type, () => { - // First pass runs in process, second pass runs in subprocess - for (const subprocessName of [ - undefined, - path.join(CODE_ROOT, 'build', 'desktop_runner.js'), - ]) { - test(`runs ${t.type} programs to perform addition via ${ - subprocessName ? subprocessName : 'same-process' - }`, async () => { - const lp = new LiteralPanelInfo({ - contentTypeInfo: { type: 'text/csv' }, - content: 'age,name\n12,Kev\n18,Nyra', - name: 'Raw Data', - }); +// describe(t.type, () => { +// // First pass runs in process, second pass runs in subprocess +// for (const subprocessName of [ +// undefined, +// path.join(CODE_ROOT, 'build', 'desktop_runner.js'), +// ]) { +// test(`runs ${t.type} programs to perform addition via ${ +// subprocessName ? subprocessName : 'same-process' +// }`, async () => { +// const lp = new LiteralPanelInfo({ +// contentTypeInfo: { type: 'text/csv' }, +// content: 'age,name\n12,Kev\n18,Nyra', +// name: 'Raw Data', +// }); - const pp = new ProgramPanelInfo({ - type: t.type, - content: t.content, - }); +// const pp = new ProgramPanelInfo({ +// type: t.type, +// content: t.content, +// }); - let finished = false; - const panels = [lp, pp]; - await withSavedPanels( - panels, - async (project) => { - const panelValueBuffer = fs.readFileSync( - getProjectResultsFile(project.projectName) + pp.id - ); - expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual([ - { name: 'Kev', age: 22 }, - { name: 'Nyra', age: 28 }, - ]); +// let finished = false; +// const panels = [lp, pp]; +// await withSavedPanels( +// panels, +// async (project) => { +// const panelValueBuffer = fs.readFileSync( +// getProjectResultsFile(project.projectName) + pp.id +// ); +// expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual([ +// { name: 'Kev', age: 22 }, +// { name: 'Nyra', age: 28 }, +// ]); - finished = true; - }, - { evalPanels: true, subprocessName } - ); +// finished = true; +// }, +// { evalPanels: true, subprocessName } +// ); - if (!finished) { - throw new Error('Callback did not finish'); - } - }, 300_000); - } +// if (!finished) { +// throw new Error('Callback did not finish'); +// } +// }, 300_000); +// } + +// for (const n of [0, 1]) { +// test(`${t.type} default content ${ +// n === 0 ? 'first' : 'second' +// } panel`, async () => { +// const lp = new LiteralPanelInfo(); +// lp.literal.contentTypeInfo = { type: 'text/csv' }; +// lp.content = 'age,name\n12,Kev\n18,Nyra'; - for (const n of [0, 1]) { - test(`${t.type} default content ${ - n === 0 ? 'first' : 'second' - } panel`, async () => { - const lp = new LiteralPanelInfo(); - lp.literal.contentTypeInfo = { type: 'text/csv' }; - lp.content = 'age,name\n12,Kev\n18,Nyra'; +// const pp = new ProgramPanelInfo(); +// pp.program.type = t.type; +// pp.content = LANGUAGES[t.type].defaultContent(n); - const pp = new ProgramPanelInfo(); - pp.program.type = t.type; - pp.content = LANGUAGES[t.type].defaultContent(n); +// let finished = false; +// const panels = [lp, pp]; +// await withSavedPanels( +// panels, +// async (project) => { +// const panelValueBuffer = fs.readFileSync( +// getProjectResultsFile(project.projectName) + pp.id +// ); +// // defaultContent(0) returns [] and defaultContent(!0) returns the previous panel +// expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual( +// n === 0 +// ? t.type === 'r' +// ? null +// : t.type === 'sql' +// ? [{ NULL: null }] +// : [] +// : [ +// { name: 'Kev', age: '12' }, +// { name: 'Nyra', age: '18' }, +// ] +// ); - let finished = false; - const panels = [lp, pp]; - await withSavedPanels( - panels, - async (project) => { - const panelValueBuffer = fs.readFileSync( - getProjectResultsFile(project.projectName) + pp.id - ); - // defaultContent(0) returns [] and defaultContent(!0) returns the previous panel - expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual( - n === 0 - ? t.type === 'r' - ? null - : t.type === 'sql' - ? [{ NULL: null }] - : [] - : [ - { name: 'Kev', age: '12' }, - { name: 'Nyra', age: '18' }, - ] - ); +// finished = true; +// }, +// { evalPanels: true } +// ); - finished = true; - }, - { evalPanels: true } - ); +// if (!finished) { +// throw new Error('Callback did not finish'); +// } +// }, 300_000); +// } +// }); +// } - if (!finished) { - throw new Error('Callback did not finish'); - } - }, 300_000); +describe('parsePartialJSONFile', function parsePartialJSONFileTest() { + test('correctly fills out partial file', async function fillsOutPartial() { + const f = await makeTmpFile(); + try { + const whole = '[{"foo": "bar"}, {"big": "bad"}]'; + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ "foo": "bar" }]); + } finally { + f.cleanup(); } }); -} +}); diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 4a086823c..6ea1783ec 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -10,10 +10,10 @@ import { SETTINGS } from '../settings'; import { getProjectResultsFile } from '../store'; import { EvalHandlerExtra, EvalHandlerResponse, guardPanel } from './types'; -export function parsePartialJSONFile(file, maxBytesToRead) { +export function parsePartialJSONFile(file: string, maxBytesToRead: number) { let fd: number; try { - fd = fs.openSync(file); + fd = fs.openSync(file, 'r'); } catch (e) { throw new NoResultError(); } @@ -34,7 +34,7 @@ export function parsePartialJSONFile(file, maxBytesToRead) { while (!done) { const bufferSize = 1024; const b = Buffer.alloc(bufferSize); - fs.readSync(fd, b, 0, bufferSize); + fs.readSync(fd, b); // To be able to iterate over code points let bs = Array.from(b.toString()); @@ -48,13 +48,15 @@ export function parsePartialJSONFile(file, maxBytesToRead) { break; case ']': case '}': - incomplete.pop(); if (f.length + bufferSize >= maxBytesToRead) { bs = bs.slice(0, i); // Need to not count additional openings after this done = true; break outer; } + + // Otherwise, pop it + incomplete.pop(); break; } } @@ -70,9 +72,10 @@ export function parsePartialJSONFile(file, maxBytesToRead) { } } + console.log({f}); const value = JSON.parse(f.toString()); - return { size, value, arrayCount: value.charAt(0) === '[' ? ('More than ' + value.length) : null }; + return { size, value, arrayCount: f.charAt(0) === '[' ? ('More than ' + value.length) : null }; } finally { fs.closeSync(fd); } From 71431e605ac90fae1ccdaef7355d11a4328c60ba Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 22:23:40 +0000 Subject: [PATCH 09/18] Drop console log --- desktop/panel/eval.ts | 7 ++++++- desktop/panel/program.test.js | 2 +- desktop/panel/program.ts | 1 - 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/desktop/panel/eval.ts b/desktop/panel/eval.ts index 743cdbb20..ef74182e6 100644 --- a/desktop/panel/eval.ts +++ b/desktop/panel/eval.ts @@ -206,7 +206,12 @@ export const makeEvalHandler = ( shape: s, value: res.returnValue ? res.value : null, size: res.size === undefined ? json.length : res.size, - arrayCount: res.arrayCount === undefined ? ( s.kind === 'array' ? (res.value || []).length : null ) : res.arrayCount, + arrayCount: + res.arrayCount === undefined + ? s.kind === 'array' + ? (res.value || []).length + : null + : res.arrayCount, contentType: res.contentType || 'application/json', }; }, diff --git a/desktop/panel/program.test.js b/desktop/panel/program.test.js index f95abbfaa..99909d6e4 100644 --- a/desktop/panel/program.test.js +++ b/desktop/panel/program.test.js @@ -183,7 +183,7 @@ describe('parsePartialJSONFile', function parsePartialJSONFileTest() { fs.writeFileSync(f.path, whole); const { value, size } = parsePartialJSONFile(f.path, 3); expect(size).toBe(whole.length); - expect(value).toStrictEqual([{ "foo": "bar" }]); + expect(value).toStrictEqual([{ foo: 'bar' }]); } finally { f.cleanup(); } diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 6ea1783ec..7448d51f1 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -72,7 +72,6 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { } } - console.log({f}); const value = JSON.parse(f.toString()); return { size, value, arrayCount: f.charAt(0) === '[' ? ('More than ' + value.length) : null }; From d085fad3c6158d0b96ed271dadb80ca344e14cc1 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 22:23:46 +0000 Subject: [PATCH 10/18] Fix for format --- desktop/panel/program.ts | 68 +++++++++++++++++++++++----------------- ui/style.css | 4 +-- 2 files changed, 41 insertions(+), 31 deletions(-) diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 7448d51f1..2d3ec6bff 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -23,13 +23,17 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { if (size < maxBytesToRead) { const f = fs.readFileSync(file).toString(); const value = JSON.parse(f); - return { size, value, arrayCount: f.charAt(0) === '[' ? value.length : null}; + return { + size, + value, + arrayCount: f.charAt(0) === '[' ? value.length : null, + }; } try { let done = false; let f = ''; - const incomplete = [] + const incomplete = []; while (!done) { const bufferSize = 1024; @@ -38,27 +42,26 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { // To be able to iterate over code points let bs = Array.from(b.toString()); - outer: - for (let i = 0; i < bs.length; i++) { - const c = bs[i]; - switch (c) { - case '{': - case '[': - incomplete.push(c); - break; - case ']': - case '}': - if (f.length + bufferSize >= maxBytesToRead) { - bs = bs.slice(0, i); - // Need to not count additional openings after this - done = true; - break outer; - } - - // Otherwise, pop it - incomplete.pop(); - break; - } + outer: for (let i = 0; i < bs.length; i++) { + const c = bs[i]; + switch (c) { + case '{': + case '[': + incomplete.push(c); + break; + case ']': + case '}': + if (f.length + bufferSize >= maxBytesToRead) { + bs = bs.slice(0, i); + // Need to not count additional openings after this + done = true; + break outer; + } + + // Otherwise, pop it + incomplete.pop(); + break; + } } f += bs.join(''); @@ -66,15 +69,19 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { while (incomplete.length) { if (incomplete.pop() === '{') { - f += '}'; + f += '}'; } else { - f += ']'; + f += ']'; } } const value = JSON.parse(f.toString()); - return { size, value, arrayCount: f.charAt(0) === '[' ? ('More than ' + value.length) : null }; + return { + size, + value, + arrayCount: f.charAt(0) === '[' ? 'More than ' + value.length : null, + }; } finally { fs.closeSync(fd); } @@ -146,14 +153,17 @@ export async function evalProgram( throw new Error(stderr); } - const { size, value, arrayCount } = parsePartialJSONFile(projectResultsFile + ppi.id, 100_000); + const { size, value, arrayCount } = parsePartialJSONFile( + projectResultsFile + ppi.id, + 100_000 + ); return { skipWrite: true, value, stdout: out, - size, - arrayCount, + size, + arrayCount, }; } catch (e) { const resultsFileRE = new RegExp( diff --git a/ui/style.css b/ui/style.css index 7d84f3734..21769fe02 100644 --- a/ui/style.css +++ b/ui/style.css @@ -42,8 +42,8 @@ a:hover { } .main-body { - flex: 1; - overflow-y: auto; + flex: 1; + overflow-y: auto; } header { From 6f86e9d49e4d88418bf203548d93e2df8cc80de2 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 22:34:22 +0000 Subject: [PATCH 11/18] Add support for writing large arrays in JavaScript --- shared/languages/javascript.ts | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/shared/languages/javascript.ts b/shared/languages/javascript.ts index b0a030a5e..7c4322cb2 100644 --- a/shared/languages/javascript.ts +++ b/shared/languages/javascript.ts @@ -38,7 +38,21 @@ function DM_getPanel(i) { } function DM_setPanel(v) { const fs = require('fs'); - fs.writeFileSync('${resultsFile + panelId}', JSON.stringify(v)); + const fd = fs.openFileSync('${resultsFile + panelId}', 'w'); + if (Array.isArray(v)) { + fd.writeSync(fd, '['); + for (let i = 0; i < v.length; i++) { + const row = v[i]; + let rowJSON = JSON.stringify(row); + if (i < v.length - 1) { + rowJSON += ','; + } + fd.writeSync(rowJSON); + } + fd.writeSync(fd, ']'); + } else { + fs.writeSync(fd, JSON.stringify(v)); + } }`; } From ce809ac5f057fec40b127faa8e54b64ae7901625 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 17:36:56 -0500 Subject: [PATCH 12/18] OpenSync not openfilesync --- shared/languages/javascript.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/languages/javascript.ts b/shared/languages/javascript.ts index 7c4322cb2..fcbd864d9 100644 --- a/shared/languages/javascript.ts +++ b/shared/languages/javascript.ts @@ -38,7 +38,7 @@ function DM_getPanel(i) { } function DM_setPanel(v) { const fs = require('fs'); - const fd = fs.openFileSync('${resultsFile + panelId}', 'w'); + const fd = fs.openSync('${resultsFile + panelId}', 'w'); if (Array.isArray(v)) { fd.writeSync(fd, '['); for (let i = 0; i < v.length; i++) { From 8cf3b516cdb2ca05b7bc65132db75b77876b3c09 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sat, 4 Dec 2021 17:38:01 -0500 Subject: [PATCH 13/18] fs.write --- shared/languages/javascript.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/shared/languages/javascript.ts b/shared/languages/javascript.ts index fcbd864d9..bc37fce27 100644 --- a/shared/languages/javascript.ts +++ b/shared/languages/javascript.ts @@ -40,16 +40,16 @@ function DM_setPanel(v) { const fs = require('fs'); const fd = fs.openSync('${resultsFile + panelId}', 'w'); if (Array.isArray(v)) { - fd.writeSync(fd, '['); + fs.writeSync(fd, '['); for (let i = 0; i < v.length; i++) { const row = v[i]; let rowJSON = JSON.stringify(row); if (i < v.length - 1) { rowJSON += ','; } - fd.writeSync(rowJSON); + fs.writeSync(rowJSON); } - fd.writeSync(fd, ']'); + fs.writeSync(fd, ']'); } else { fs.writeSync(fd, JSON.stringify(v)); } From e242a6b816f53709ad92ce4c0a645e96c66e0a00 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sun, 5 Dec 2021 01:18:11 +0000 Subject: [PATCH 14/18] Tests for escaped quotes --- desktop/panel/program.test.js | 29 +++++++++++++++++++++++++++++ desktop/panel/program.ts | 21 ++++++++++++++++++--- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/desktop/panel/program.test.js b/desktop/panel/program.test.js index 99909d6e4..9f5792f28 100644 --- a/desktop/panel/program.test.js +++ b/desktop/panel/program.test.js @@ -188,4 +188,33 @@ describe('parsePartialJSONFile', function parsePartialJSONFileTest() { f.cleanup(); } }); + + test('handles open-close in string', async function handlesOpenCloseInString() { + for (const c of ['{', '}', '[', ']']) { + const f = await makeTmpFile(); + try { + const whole = `[{"foo": "${c}bar"}, {"big": "bad"}]`; + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ foo: `${c}bar` }]); + } finally { + f.cleanup(); + } + } + }); + + test('handles escaped quotes in string', async function handlesEscapedQuotesInString() { + const f = await makeTmpFile(); + try { + const whole = `[{"foo":"bar\\" { "},{"big":"bad"}]`; + expect(JSON.stringify(JSON.parse(whole))).toEqual(whole); + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ foo: `bar" { ` }]); + } finally { + f.cleanup(); + } + }); }); diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 2d3ec6bff..84f5017b3 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -34,17 +34,29 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { let done = false; let f = ''; const incomplete = []; + let inString = false; while (!done) { const bufferSize = 1024; const b = Buffer.alloc(bufferSize); - fs.readSync(fd, b); + const bytesRead = fs.readSync(fd, b); // To be able to iterate over code points let bs = Array.from(b.toString()); outer: for (let i = 0; i < bs.length; i++) { const c = bs[i]; - switch (c) { + if (c !== '"' && inString) { + continue; + } + + switch (c) { + case '"': + const previous = (i + bs.length) === 0 ? '' : (i > 0 ? bs[i-1] : f.charAt(f.length-1)); + const isEscaped = previous === '\\'; + if (!isEscaped) { + inString = !inString; + } + break; case '{': case '[': incomplete.push(c); @@ -65,6 +77,9 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { } f += bs.join(''); + if (bytesRead < bufferSize) { + break; + } } while (incomplete.length) { @@ -75,7 +90,7 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { } } - const value = JSON.parse(f.toString()); + const value = JSON.parse(f); return { size, From df2bbf29ae0e4c233910e23ea5c49892b2a5faba Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sun, 5 Dec 2021 02:22:02 +0000 Subject: [PATCH 15/18] Fixes for format --- desktop/panel/program.test.js | 12 ++++++------ desktop/panel/program.ts | 31 ++++++++++++++++++------------- 2 files changed, 24 insertions(+), 19 deletions(-) diff --git a/desktop/panel/program.test.js b/desktop/panel/program.test.js index 9f5792f28..a956fefde 100644 --- a/desktop/panel/program.test.js +++ b/desktop/panel/program.test.js @@ -193,13 +193,13 @@ describe('parsePartialJSONFile', function parsePartialJSONFileTest() { for (const c of ['{', '}', '[', ']']) { const f = await makeTmpFile(); try { - const whole = `[{"foo": "${c}bar"}, {"big": "bad"}]`; - fs.writeFileSync(f.path, whole); - const { value, size } = parsePartialJSONFile(f.path, 3); - expect(size).toBe(whole.length); - expect(value).toStrictEqual([{ foo: `${c}bar` }]); + const whole = `[{"foo": "${c}bar"}, {"big": "bad"}]`; + fs.writeFileSync(f.path, whole); + const { value, size } = parsePartialJSONFile(f.path, 3); + expect(size).toBe(whole.length); + expect(value).toStrictEqual([{ foo: `${c}bar` }]); } finally { - f.cleanup(); + f.cleanup(); } } }); diff --git a/desktop/panel/program.ts b/desktop/panel/program.ts index 84f5017b3..0c482da29 100644 --- a/desktop/panel/program.ts +++ b/desktop/panel/program.ts @@ -45,18 +45,23 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { let bs = Array.from(b.toString()); outer: for (let i = 0; i < bs.length; i++) { const c = bs[i]; - if (c !== '"' && inString) { - continue; - } - - switch (c) { - case '"': - const previous = (i + bs.length) === 0 ? '' : (i > 0 ? bs[i-1] : f.charAt(f.length-1)); - const isEscaped = previous === '\\'; - if (!isEscaped) { - inString = !inString; - } - break; + if (c !== '"' && inString) { + continue; + } + + switch (c) { + case '"': + const previous = + i + bs.length === 0 + ? '' + : i > 0 + ? bs[i - 1] + : f.charAt(f.length - 1); + const isEscaped = previous === '\\'; + if (!isEscaped) { + inString = !inString; + } + break; case '{': case '[': incomplete.push(c); @@ -78,7 +83,7 @@ export function parsePartialJSONFile(file: string, maxBytesToRead: number) { f += bs.join(''); if (bytesRead < bufferSize) { - break; + break; } } From f68fe279853b22384fbebff30b5113b22ab6d04a Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sun, 5 Dec 2021 02:22:18 +0000 Subject: [PATCH 16/18] Drop console logs --- ui/app.test.jsx | 1 - 1 file changed, 1 deletion(-) diff --git a/ui/app.test.jsx b/ui/app.test.jsx index d455e615f..074c14953 100644 --- a/ui/app.test.jsx +++ b/ui/app.test.jsx @@ -96,7 +96,6 @@ test( for (let i = 0; i < connectors.length; i++) { const c = connectors.at(i); act(() => { - console.log(c.debug()); c.find({ 'data-testid': 'show-hide-connector', icon: true, From 2af7a5094f693d688555e44cbdc8abec0d995fe2 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sun, 5 Dec 2021 03:01:23 +0000 Subject: [PATCH 17/18] Uncomment tests --- desktop/panel/program.test.js | 306 +++++++++++++++++----------------- 1 file changed, 153 insertions(+), 153 deletions(-) diff --git a/desktop/panel/program.test.js b/desktop/panel/program.test.js index a956fefde..9a288e9f2 100644 --- a/desktop/panel/program.test.js +++ b/desktop/panel/program.test.js @@ -10,170 +10,170 @@ const { makeEvalHandler } = require('./eval'); const { inPath, withSavedPanels } = require('./testutil'); const { parsePartialJSONFile } = require('./program'); -// const TESTS = [ -// { -// type: 'javascript', -// content: -// 'const prev = DM_getPanel(0); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);', -// condition: true, -// }, -// { -// type: 'javascript', -// content: `const prev = DM_getPanel('Raw Data'); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);`, -// condition: true, -// }, -// { -// type: 'sql', -// content: 'SELECT name, age::INT + 10 AS age FROM DM_getPanel(0)', -// condition: true, -// }, -// { -// type: 'sql', -// content: `SELECT name, age::INT + 10 AS age FROM DM_getPanel('Raw Data')`, -// condition: true, -// }, -// // Rest are only mandatory-tested on Linux to make CI easier for now -// { -// type: 'python', -// content: -// 'prev = DM_getPanel(0)\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)', -// condition: process.platform === 'linux' || inPath('python3'), -// }, -// { -// type: 'python', -// content: `prev = DM_getPanel('Raw Data')\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)`, -// condition: process.platform === 'linux' || inPath('python3'), -// }, -// { -// type: 'ruby', -// content: -// 'prev = DM_getPanel(0)\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)', -// condition: process.platform === 'linux' || inPath('ruby'), -// }, -// { -// type: 'ruby', -// content: `prev = DM_getPanel('Raw Data')\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)`, -// condition: process.platform === 'linux' || inPath('ruby'), -// }, -// { -// type: 'julia', -// content: -// 'prev = DM_getPanel(0)\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)', -// condition: process.platform === 'linux' || inPath('julia'), -// }, -// { -// type: 'julia', -// content: `prev = DM_getPanel("Raw Data")\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)`, -// condition: process.platform === 'linux' || inPath('julia'), -// }, -// { -// type: 'r', -// content: -// 'prev = DM_getPanel(0)\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)', -// condition: process.platform === 'linux' || inPath('Rscript'), -// }, -// { -// type: 'r', -// content: `prev = DM_getPanel('Raw Data')\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)`, -// condition: process.platform === 'linux' || inPath('Rscript'), -// }, -// ]; +const TESTS = [ + { + type: 'javascript', + content: + 'const prev = DM_getPanel(0); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);', + condition: true, + }, + { + type: 'javascript', + content: `const prev = DM_getPanel('Raw Data'); const next = prev.map((row) => ({ ...row, "age": +row.age + 10 })); DM_setPanel(next);`, + condition: true, + }, + { + type: 'sql', + content: 'SELECT name, age::INT + 10 AS age FROM DM_getPanel(0)', + condition: true, + }, + { + type: 'sql', + content: `SELECT name, age::INT + 10 AS age FROM DM_getPanel('Raw Data')`, + condition: true, + }, + // Rest are only mandatory-tested on Linux to make CI easier for now + { + type: 'python', + content: + 'prev = DM_getPanel(0)\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)', + condition: process.platform === 'linux' || inPath('python3'), + }, + { + type: 'python', + content: `prev = DM_getPanel('Raw Data')\nnext = [{ **row, "age": int(row["age"]) + 10 } for row in prev]\nDM_setPanel(next)`, + condition: process.platform === 'linux' || inPath('python3'), + }, + { + type: 'ruby', + content: + 'prev = DM_getPanel(0)\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)', + condition: process.platform === 'linux' || inPath('ruby'), + }, + { + type: 'ruby', + content: `prev = DM_getPanel('Raw Data')\npanel = prev.map { |row| { name: row["name"], age: row["age"].to_i + 10 } }\nDM_setPanel(panel)`, + condition: process.platform === 'linux' || inPath('ruby'), + }, + { + type: 'julia', + content: + 'prev = DM_getPanel(0)\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)', + condition: process.platform === 'linux' || inPath('julia'), + }, + { + type: 'julia', + content: `prev = DM_getPanel("Raw Data")\nfor row in prev\n row["age"] = parse(Int64, row["age"]) + 10\nend\nDM_setPanel(prev)`, + condition: process.platform === 'linux' || inPath('julia'), + }, + { + type: 'r', + content: + 'prev = DM_getPanel(0)\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)', + condition: process.platform === 'linux' || inPath('Rscript'), + }, + { + type: 'r', + content: `prev = DM_getPanel('Raw Data')\nfor (i in 1:length(prev)) {\n prev[[i]]$age = strtoi(prev[[i]]$age) + 10\n}\nDM_setPanel(prev)`, + condition: process.platform === 'linux' || inPath('Rscript'), + }, +]; -// for (const t of TESTS) { -// if (!t.condition) { -// continue; -// } +for (const t of TESTS) { + if (!t.condition) { + continue; + } -// describe(t.type, () => { -// // First pass runs in process, second pass runs in subprocess -// for (const subprocessName of [ -// undefined, -// path.join(CODE_ROOT, 'build', 'desktop_runner.js'), -// ]) { -// test(`runs ${t.type} programs to perform addition via ${ -// subprocessName ? subprocessName : 'same-process' -// }`, async () => { -// const lp = new LiteralPanelInfo({ -// contentTypeInfo: { type: 'text/csv' }, -// content: 'age,name\n12,Kev\n18,Nyra', -// name: 'Raw Data', -// }); + describe(t.type, () => { + // First pass runs in process, second pass runs in subprocess + for (const subprocessName of [ + undefined, + path.join(CODE_ROOT, 'build', 'desktop_runner.js'), + ]) { + test(`runs ${t.type} programs to perform addition via ${ + subprocessName ? subprocessName : 'same-process' + }`, async () => { + const lp = new LiteralPanelInfo({ + contentTypeInfo: { type: 'text/csv' }, + content: 'age,name\n12,Kev\n18,Nyra', + name: 'Raw Data', + }); -// const pp = new ProgramPanelInfo({ -// type: t.type, -// content: t.content, -// }); + const pp = new ProgramPanelInfo({ + type: t.type, + content: t.content, + }); -// let finished = false; -// const panels = [lp, pp]; -// await withSavedPanels( -// panels, -// async (project) => { -// const panelValueBuffer = fs.readFileSync( -// getProjectResultsFile(project.projectName) + pp.id -// ); -// expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual([ -// { name: 'Kev', age: 22 }, -// { name: 'Nyra', age: 28 }, -// ]); + let finished = false; + const panels = [lp, pp]; + await withSavedPanels( + panels, + async (project) => { + const panelValueBuffer = fs.readFileSync( + getProjectResultsFile(project.projectName) + pp.id + ); + expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual([ + { name: 'Kev', age: 22 }, + { name: 'Nyra', age: 28 }, + ]); -// finished = true; -// }, -// { evalPanels: true, subprocessName } -// ); + finished = true; + }, + { evalPanels: true, subprocessName } + ); -// if (!finished) { -// throw new Error('Callback did not finish'); -// } -// }, 300_000); -// } + if (!finished) { + throw new Error('Callback did not finish'); + } + }, 300_000); + } -// for (const n of [0, 1]) { -// test(`${t.type} default content ${ -// n === 0 ? 'first' : 'second' -// } panel`, async () => { -// const lp = new LiteralPanelInfo(); -// lp.literal.contentTypeInfo = { type: 'text/csv' }; -// lp.content = 'age,name\n12,Kev\n18,Nyra'; + for (const n of [0, 1]) { + test(`${t.type} default content ${ + n === 0 ? 'first' : 'second' + } panel`, async () => { + const lp = new LiteralPanelInfo(); + lp.literal.contentTypeInfo = { type: 'text/csv' }; + lp.content = 'age,name\n12,Kev\n18,Nyra'; -// const pp = new ProgramPanelInfo(); -// pp.program.type = t.type; -// pp.content = LANGUAGES[t.type].defaultContent(n); + const pp = new ProgramPanelInfo(); + pp.program.type = t.type; + pp.content = LANGUAGES[t.type].defaultContent(n); -// let finished = false; -// const panels = [lp, pp]; -// await withSavedPanels( -// panels, -// async (project) => { -// const panelValueBuffer = fs.readFileSync( -// getProjectResultsFile(project.projectName) + pp.id -// ); -// // defaultContent(0) returns [] and defaultContent(!0) returns the previous panel -// expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual( -// n === 0 -// ? t.type === 'r' -// ? null -// : t.type === 'sql' -// ? [{ NULL: null }] -// : [] -// : [ -// { name: 'Kev', age: '12' }, -// { name: 'Nyra', age: '18' }, -// ] -// ); + let finished = false; + const panels = [lp, pp]; + await withSavedPanels( + panels, + async (project) => { + const panelValueBuffer = fs.readFileSync( + getProjectResultsFile(project.projectName) + pp.id + ); + // defaultContent(0) returns [] and defaultContent(!0) returns the previous panel + expect(JSON.parse(panelValueBuffer.toString())).toStrictEqual( + n === 0 + ? t.type === 'r' + ? null + : t.type === 'sql' + ? [{ NULL: null }] + : [] + : [ + { name: 'Kev', age: '12' }, + { name: 'Nyra', age: '18' }, + ] + ); -// finished = true; -// }, -// { evalPanels: true } -// ); + finished = true; + }, + { evalPanels: true } + ); -// if (!finished) { -// throw new Error('Callback did not finish'); -// } -// }, 300_000); -// } -// }); -// } + if (!finished) { + throw new Error('Callback did not finish'); + } + }, 300_000); + } + }); +} describe('parsePartialJSONFile', function parsePartialJSONFileTest() { test('correctly fills out partial file', async function fillsOutPartial() { From f30379009a96d1613a910b1bb83fda08eae86243 Mon Sep 17 00:00:00 2001 From: Phil Eaton Date: Sun, 5 Dec 2021 21:45:09 +0000 Subject: [PATCH 18/18] Fix for JavaScript write --- shared/languages/javascript.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/shared/languages/javascript.ts b/shared/languages/javascript.ts index bc37fce27..ff6d9adbc 100644 --- a/shared/languages/javascript.ts +++ b/shared/languages/javascript.ts @@ -47,7 +47,7 @@ function DM_setPanel(v) { if (i < v.length - 1) { rowJSON += ','; } - fs.writeSync(rowJSON); + fs.writeSync(fd, rowJSON); } fs.writeSync(fd, ']'); } else {