From e06f1a9dc6499d93ddc192e720c46c1e3af2da7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=B6ran=20Sander?= Date: Tue, 24 Oct 2023 07:15:24 +0000 Subject: [PATCH] fix(task-import): Importing tasks from CSV file no longer gives "premature close" Fixes #323 --- .vscode/launch.json | 100 +++++++++++++++++++------------------- src/lib/cmd/importtask.js | 61 ++++++++++++++++++++--- 2 files changed, 103 insertions(+), 58 deletions(-) diff --git a/.vscode/launch.json b/.vscode/launch.json index baafb65..388a916 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -58,70 +58,32 @@ // ------------------------------------ // Import tasks from Excel file // ------------------------------------ - "args": [ - "task-import", - "--auth-type", - "cert", - "--host", - "192.168.100.109", - "--auth-cert-file", - "./cert/client.pem", - "--auth-cert-key-file", - "./cert/client_key.pem", - "--auth-user-dir", - "LAB", - "--auth-user-id", - "goran", - - "--file-type", - "excel", - - "--file-name", - "testdata/tasks.xlsx", - "--sheet-name", - "4" - - // "--import-app", - // "--import-app-sheet-name", - // "App import", - - // "--qvf-overwrite", - // "no", - - // "--limit-import-count", - // "2", - - // "--sleep-app-upload", - // "500", - - // "--dry-run" - ] - - // ------------------------------------ - // Import tasks from CSV file - // ------------------------------------ // "args": [ // "task-import", // "--auth-type", // "cert", // "--host", // "192.168.100.109", - // // "--auth-cert-file", - // // "./cert/client.pem", - // // "--auth-cert-key-file", - // // "./cert/client_key.pem", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", // "--auth-user-dir", // "LAB", // "--auth-user-id", // "goran", // "--file-type", - // "csv", + // "excel", // "--file-name", - // // "tasks2source.csv", - // // "task-chain.csv", - // "reload-tasks.csv", + // "testdata/tasks.xlsx", + // "--sheet-name", + // "4" + + // // "--import-app", + // // "--import-app-sheet-name", + // // "App import", // // "--qvf-overwrite", // // "no", @@ -129,9 +91,47 @@ // // "--limit-import-count", // // "2", + // // "--sleep-app-upload", + // // "500", + // // "--dry-run" // ] + // ------------------------------------ + // Import tasks from CSV file + // ------------------------------------ + "args": [ + "task-import", + "--auth-type", + "cert", + "--host", + "192.168.100.109", + // "--auth-cert-file", + // "./cert/client.pem", + // "--auth-cert-key-file", + // "./cert/client_key.pem", + "--auth-user-dir", + "LAB", + "--auth-user-id", + "goran", + + "--file-type", + "csv", + + "--file-name", + // "tasks2source.csv", + // "task-chain.csv", + "testdata/reload-tasks.csv", + + // "--qvf-overwrite", + // "no", + + // "--limit-import-count", + // "2", + + // "--dry-run" + ] + // ------------------------------------ // Export apps to QVF files // ------------------------------------ diff --git a/src/lib/cmd/importtask.js b/src/lib/cmd/importtask.js index 5295574..551cfaa 100644 --- a/src/lib/cmd/importtask.js +++ b/src/lib/cmd/importtask.js @@ -1,8 +1,10 @@ -// import { csvParse } from 'csv-parse'; - const xlsx = require('node-xlsx').default; const { parse } = require('csv-parse'); +// const { parse } = require('csv-parse/lib/sync'); const fs = require('fs'); +// const fsp = require('fs').promises; + +const { finished } = require('stream/promises'); const { logger, setLoggingLevel, isPkg, execPath, verifyFileExists, isNumeric } = require('../../globals'); const { QlikSenseTasks } = require('../task/class_alltasks'); @@ -11,27 +13,70 @@ const { getTaskColumnPosFromHeaderRow } = require('../util/lookups'); const { getTagsFromQseow } = require('../util/tag'); const { getCustomPropertiesFromQseow } = require('../util/customproperties'); -const processCsvFile = async (options) => { - // First get header row - let parser = fs.createReadStream(options.fileName).pipe( +const getHeaders = async (options) => { + const records = []; + const parser = fs.createReadStream(options.fileName).pipe( parse({ info: true, to_line: 1, }) ); + parser.on('readable', () => { + let record; + while ((record = parser.read()) !== null) { + // Work with each record + records.push(record); + } + }); + await finished(parser); + return records; +}; + +const processCsvFile = async (options) => { + // First get header row + + // const parser = parse({ + // delimiter: ',', + // info: true, + // // to_line: 1, + // }); + + // const parser = fs.createReadStream(options.fileName).pipe( + // parse({ + // info: true, + // to_line: 1, + // }).pipe(process.stdout) + // ); + + const headers = await getHeaders(options); + + // const headers = fs.createReadStream(options.fileName).pipe(parser).pipe(process.stdout); + + // const content = await fsp.readFile(options.fileName); + + // Parse the CSV content + // const headers = parse(content, { bom: true, to_line: 1 }); const headerRow = []; + + // Push all column headers to array // eslint-disable-next-line no-restricted-syntax - for await (const record of parser) { + for (const record of headers) { // Get each column header text headerRow.push(record.record); } + // // eslint-disable-next-line no-restricted-syntax + // for await (const record of headers) { + // // for await (const record of parser) { + // // Get each column header text + // headerRow.push(record.record); + // } // Get positions of column headers const colHeaders = getTaskColumnPosFromHeaderRow(headerRow[0]); const records = []; - parser = fs.createReadStream(options.fileName).pipe( + const parser = fs.createReadStream(options.fileName).pipe( parse({ info: true, skip_empty_lines: true, @@ -302,7 +347,7 @@ const importTaskFromFile = async (options) => { // Get all custom properties const cpExisting = await getCustomPropertiesFromQseow(options); - // Verify file exists + // Verify task definitions file exists const taskFileExists = await verifyFileExists(options.fileName); if (taskFileExists === false) { logger.error(`Missing task file "${options.fileName}". Aborting`);