GH-37657: [JS] Run bin scripts with ts-node

dev/archery/archery/integration/tester_js.py

@@ -23,10 +23,10 @@
 
 
 _EXE_PATH = os.path.join(ARROW_ROOT_DEFAULT, 'js/bin')
-_VALIDATE = os.path.join(_EXE_PATH, 'integration.js')
-_JSON_TO_ARROW = os.path.join(_EXE_PATH, 'json-to-arrow.js')
-_STREAM_TO_FILE = os.path.join(_EXE_PATH, 'stream-to-file.js')
-_FILE_TO_STREAM = os.path.join(_EXE_PATH, 'file-to-stream.js')
+_VALIDATE = os.path.join(_EXE_PATH, 'integration.ts')
+_JSON_TO_ARROW = os.path.join(_EXE_PATH, 'json-to-arrow.ts')
+_STREAM_TO_FILE = os.path.join(_EXE_PATH, 'stream-to-file.ts')
+_FILE_TO_STREAM = os.path.join(_EXE_PATH, 'file-to-stream.ts')
 
 
 class JSTester(Tester):

js/.vscode/launch.json

@@ -31,7 +31,7 @@
       "args": {
         "cwd": "${workspaceFolder}",
         "description": "Select a file to debug",
-        "command": "./node_modules/.bin/jest --listTests | sed -r \"s@$PWD/test/@@g\"",
+        "command": "node_modules/.bin/jest --listTests | sed -r \"s@$PWD/test/@@g\"",
       }
     },
     {
@@ -98,25 +98,23 @@
       "request": "launch",
       "name": "Debug Integration Tests",
       "cwd": "${workspaceFolder}",
-      "program": "${workspaceFolder}/bin/integration.js",
+      "program": "${workspaceFolder}/bin/integration.ts",
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "env": {
         "NODE_NO_WARNINGS": "1",
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "args": [
         "--mode",
         "VALIDATE",
         "-j", "test/data/json/unions.json",
-        "-a", "./test/data/cpp/stream/struct_example.arrow"
+        "-a", "test/data/cpp/stream/unions.arrow"
       ]
     },
     {
@@ -140,8 +138,7 @@
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "runtimeArgs": [
-        "--loader",
-        "ts-node/esm/transpile-only"
+        "--loader", "ts-node/esm/transpile-only"
       ]
     },
     {
@@ -150,12 +147,10 @@
       "name": "Debug bin/arrow2csv",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "console": "integratedTerminal",
       "skipFiles": [
@@ -165,7 +160,7 @@
       "args": [
         "${workspaceFolder}/src/bin/arrow2csv.ts",
         "-f",
-        "./test/data/cpp/stream/simple.arrow"
+        "test/data/cpp/stream/simple.arrow"
       ]
     },
     {
@@ -174,21 +169,19 @@
       "name": "Debug bin/file-to-stream",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/file-to-stream.js",
-        "./test/data/cpp/file/struct_example.arrow",
-        "./struct_example-stream-out.arrow",
+        "${workspaceFolder}/bin/file-to-stream.ts",
+        "test/data/cpp/file/struct_example.arrow",
+        "struct_example-stream-out.arrow",
       ]
     },
     {
@@ -197,21 +190,19 @@
       "name": "Debug bin/stream-to-file",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/stream-to-file.js",
-        "./test/data/cpp/stream/struct_example.arrow",
-        "./struct_example-file-out.arrow",
+        "${workspaceFolder}/bin/stream-to-file.ts",
+        "test/data/cpp/stream/struct_example.arrow",
+        "struct_example-file-out.arrow",
       ]
     },
     {
@@ -220,23 +211,21 @@
       "name": "Debug bin/json-to-arrow",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/json-to-arrow.js",
+        "${workspaceFolder}/bin/json-to-arrow.ts",
         "-j",
-        "./test/data/json/struct_example.json",
+        "test/data/json/struct_example.json",
         "-a",
-        "./struct_example-stream-out.arrow",
+        "struct_example-stream-out.arrow",
         "-f",
         "stream"
       ]
@@ -247,20 +236,18 @@
       "name": "Debug bin/print-buffer-alignment",
       "cwd": "${workspaceFolder}",
       "env": {
-        "ARROW_JS_DEBUG": "src",
         "TS_NODE_CACHE": "false"
       },
       "runtimeArgs": [
-        "-r",
-        "ts-node/register"
+        "--loader", "ts-node/esm/transpile-only"
       ],
       "skipFiles": [
         "<node_internals>/**/*.js",
         "${workspaceFolder}/node_modules/**/*.js"
       ],
       "args": [
-        "${workspaceFolder}/bin/print-buffer-alignment.js",
-        "./test/data/cpp/stream/struct_example.arrow"
+        "${workspaceFolder}/bin/print-buffer-alignment.ts",
+        "test/data/cpp/stream/struct_example.arrow"
       ]
     },
     {

js/bin/file-to-stream.js → js/bin/file-to-stream.ts

@@ -1,4 +1,4 @@
-#! /usr/bin/env node
+#! /usr/bin/env -S node --loader ts-node/esm/transpile-only
 
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
@@ -19,16 +19,15 @@
 
 // @ts-check
 
-const fs = require('fs');
-const path = require('path');
-const eos = require('util').promisify(require('stream').finished);
-const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '.cjs';
-const { RecordBatchReader, RecordBatchStreamWriter } = require(`../index${extension}`);
+import * as fs from 'fs';
+import * as Path from 'path';
+import { finished as eos } from 'stream/promises';
+import { RecordBatchReader, RecordBatchStreamWriter } from '../index.ts';
 
 (async () => {
 
-    const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(path.resolve(process.argv[2]));
-    const writable = process.argv.length < 4 ? process.stdout : fs.createWriteStream(path.resolve(process.argv[3]));
+    const readable = process.argv.length < 3 ? process.stdin : fs.createReadStream(Path.resolve(process.argv[2]));
+    const writable = process.argv.length < 4 ? process.stdout : fs.createWriteStream(Path.resolve(process.argv[3]));
 
     const fileToStream = readable
         .pipe(RecordBatchReader.throughNode())

js/bin/integration.js → js/bin/integration.ts

@@ -1,4 +1,4 @@
-#! /usr/bin/env node
+#! /usr/bin/env -S node --loader ts-node/esm/transpile-only
 
 // Licensed to the Apache Software Foundation (ASF) under one
 // or more contributor license agreements.  See the NOTICE file
@@ -17,23 +17,28 @@
 // specific language governing permissions and limitations
 // under the License.
 
-// @ts-nocheck
+import * as fs from 'fs';
+import * as Path from 'path';
+import { glob } from 'glob';
+import { zip } from 'ix/iterable/zip.js';
+import commandLineArgs from 'command-line-args';
+// @ts-ignore
+import { parse as bignumJSONParse } from 'json-bignum';
 
-const fs = require('fs');
-const Path = require('path');
-const { glob } = require('glob');
-const { zip } = require('ix/iterable/zip');
-const { parse: bignumJSONParse } = require('json-bignum');
-const argv = require(`command-line-args`)(cliOpts(), { partial: true });
-const extension = process.env.ARROW_JS_DEBUG === 'src' ? '.ts' : '.cjs';
-const {
+import {
     Table,
+    Vector,
+    RecordBatch,
+    ArrowJSONLike,
     RecordBatchReader,
     RecordBatchStreamWriter,
-    util: { createElementComparator }
-} = require(`../index${extension}`);
+    util,
+} from '../index.ts';
 
-const exists = async (p) => {
+const { createElementComparator } = util;
+const argv = commandLineArgs(cliOpts(), { partial: true });
+
+const exists = async (p: string) => {
     try {
         return !!(await fs.promises.stat(p));
     } catch (e) { return false; }
@@ -60,7 +65,7 @@ const exists = async (p) => {
             for (let [jsonPath, arrowPath] of zip(jsonPaths, arrowPaths)) {
                 try {
                     await validate(jsonPath, arrowPath);
-                } catch (e) {
+                } catch (e: any) {
                     threw = true;
                     e && process.stderr.write(`${e?.stack || e}\n`);
                 }
@@ -108,7 +113,7 @@ function print_usage() {
         {
             header: 'Synopsis',
             content: [
-                '$ integration.js -j file.json -a file.arrow --mode validate'
+                '$ integration.ts -j file.json -a file.arrow --mode validate'
             ]
         },
         {
@@ -125,7 +130,7 @@ function print_usage() {
     return 1;
 }
 
-async function validate(jsonPath, arrowPath) {
+async function validate(jsonPath: string, arrowPath: string) {
 
     const files = await Promise.all([
         fs.promises.readFile(arrowPath),
@@ -147,65 +152,65 @@ async function validate(jsonPath, arrowPath) {
     validateTableToBuffersIntegration('binary', 'file')(jsonData, arrowData);
 }
 
-function validateReaderIntegration(jsonData, arrowBuffer) {
+function validateReaderIntegration(jsonData: ArrowJSONLike, arrowBuffer: Uint8Array) {
     const msg = `json and arrow record batches report the same values`;
     try {
         const jsonReader = RecordBatchReader.from(jsonData);
         const binaryReader = RecordBatchReader.from(arrowBuffer);
         for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonReader, binaryReader)) {
             compareTableIsh(jsonRecordBatch, binaryRecordBatch);
         }
-    } catch (e) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); }
+    } catch (e: any) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); }
     process.stdout.write(`${msg}: pass\n`);
 }
 
-function validateTableFromBuffersIntegration(jsonData, arrowBuffer) {
+function validateTableFromBuffersIntegration(jsonData: ArrowJSONLike, arrowBuffer: Uint8Array) {
     const msg = `json and arrow tables report the same values`;
     try {
         const jsonTable = new Table(RecordBatchReader.from(jsonData));
         const binaryTable = new Table(RecordBatchReader.from(arrowBuffer));
         compareTableIsh(jsonTable, binaryTable);
-    } catch (e) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); }
+    } catch (e: any) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); }
     process.stdout.write(`${msg}: pass\n`);
 }
 
-function validateTableToBuffersIntegration(srcFormat, arrowFormat) {
+function validateTableToBuffersIntegration(srcFormat: 'json' | 'binary', arrowFormat: 'file' | 'stream') {
     const refFormat = srcFormat === `json` ? `binary` : `json`;
-    return function testTableToBuffersIntegration(jsonData, arrowBuffer) {
+    return function testTableToBuffersIntegration(jsonData: ArrowJSONLike, arrowBuffer: Uint8Array) {
         const msg = `serialized ${srcFormat} ${arrowFormat} reports the same values as the ${refFormat} ${arrowFormat}`;
         try {
-            const refTable = new Table(RecordBatchReader.from(refFormat === `json` ? jsonData : arrowBuffer));
-            const srcTable = new Table(RecordBatchReader.from(srcFormat === `json` ? jsonData : arrowBuffer));
+            const refTable = new Table(refFormat === `json` ? RecordBatchReader.from(jsonData) : RecordBatchReader.from(arrowBuffer));
+            const srcTable = new Table(srcFormat === `json` ? RecordBatchReader.from(jsonData) : RecordBatchReader.from(arrowBuffer));
             const dstTable = new Table(RecordBatchReader.from(RecordBatchStreamWriter.writeAll(srcTable).toUint8Array(true)));
             compareTableIsh(dstTable, refTable);
-        } catch (e) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); }
+        } catch (e: any) { throw new Error(`${msg}: fail \n ${e?.stack || e}`); }
         process.stdout.write(`${msg}: pass\n`);
     };
 }
 
-function compareTableIsh(actual, expected) {
-    if (actual.length !== expected.length) {
-        throw new Error(`length: ${actual.length} !== ${expected.length}`);
+function compareTableIsh(actual: Table | RecordBatch, expected: Table | RecordBatch) {
+    if (actual.numRows !== expected.numRows) {
+        throw new Error(`numRows: ${actual.numRows} !== ${expected.numRows}`);
     }
     if (actual.numCols !== expected.numCols) {
         throw new Error(`numCols: ${actual.numCols} !== ${expected.numCols}`);
     }
     (() => {
         for (let i = -1, n = actual.numCols; ++i < n;) {
-            const v1 = actual.getChildAt(i);
-            const v2 = expected.getChildAt(i);
+            const v1 = actual.getChildAt(i)!;
+            const v2 = expected.getChildAt(i)!;
             compareVectors(v1, v2);
         }
     })();
 }
 
-function compareVectors(actual, expected) {
+function compareVectors(actual: Vector, expected: Vector) {
 
     if ((actual == null && expected != null) || (expected == null && actual != null)) {
         throw new Error(`${actual == null ? `actual` : `expected`} is null, was expecting ${actual ?? expected} to be that also`);
     }
 
-    const props = ['type', 'length', 'nullCount'];
+    const props = ['type', 'length', 'nullCount'] as (keyof Vector & string)[];
 
     (() => {
         for (let i = -1, n = props.length; ++i < n;) {
@@ -236,7 +241,7 @@ function compareVectors(actual, expected) {
     })();
 }
 
-async function loadLocalJSONAndArrowPathsForDebugging(jsonPaths, arrowPaths) {
+async function loadLocalJSONAndArrowPathsForDebugging(jsonPaths: string[], arrowPaths: string[]) {
 
     const sourceJSONPaths = await glob(Path.resolve(__dirname, `../test/data/json/`, `*.json`));
 
@@ -254,7 +259,13 @@ async function loadLocalJSONAndArrowPathsForDebugging(jsonPaths, arrowPaths) {
 
     return [jsonPaths, arrowPaths];
 
-    async function loadJSONAndArrowPaths(sourceJSONPaths, jsonPaths, arrowPaths, source, format) {
+    async function loadJSONAndArrowPaths(
+        sourceJSONPaths: string[],
+        jsonPaths: string[],
+        arrowPaths: string[],
+        source: 'cpp' | 'java',
+        format: 'file' | 'stream'
+    ) {
         for (const jsonPath of sourceJSONPaths) {
             const { name } = Path.parse(jsonPath);
             const arrowPath = Path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);