Skip to content

Commit

Permalink
make it easier to run all integration tests from local data
Browse files Browse the repository at this point in the history
  • Loading branch information
@trxcllnt
trxcllnt committed Jan 24, 2018
1 parent c0fd2f9 commit e537789
Show file tree
Hide file tree
Showing 5 changed files with 106 additions and 45 deletions.
56 changes: 50 additions & 6 deletions js/bin/integration.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,8 @@
// specific language governing permissions and limitations
// under the License.

var fs = require('fs');
var glob = require('glob');
var path = require('path');
var gulp = require.resolve(path.join(`..`, `node_modules/gulp/bin/gulp.js`));
var child_process = require(`child_process`);
Expand All @@ -29,12 +31,14 @@ var optionList = [
{
type: String,
name: 'arrow', alias: 'a',
description: 'The Arrow file to read/write'
multiple: true, defaultValue: [],
description: 'The Arrow file[s] to read/write'
},
{
type: String,
name: 'json', alias: 'j',
description: 'The JSON file to read/write'
multiple: true, defaultValue: [],
description: 'The JSON file[s] to read/write'
}
];

Expand Down Expand Up @@ -66,20 +70,60 @@ function print_usage() {
process.exit(1);
}

if (!argv.arrow || !argv.json || !argv.mode) {
let jsonPaths = argv.json;
let arrowPaths = argv.arrow;

if (!argv.mode) {
return print_usage();
}

let mode = argv.mode.toUpperCase();
if (mode === 'VALIDATE' && !jsonPaths.length) {
jsonPaths = glob.sync(path.resolve(__dirname, `../test/data/json/`, `*.json`));
if (!arrowPaths.length) {
[jsonPaths, arrowPaths] = jsonPaths.reduce(([jsonPaths, arrowPaths], jsonPath) => {
const { name } = path.parse(jsonPath);
for (const source of ['cpp', 'java']) {
for (const format of ['file', 'stream']) {
const arrowPath = path.resolve(__dirname, `../test/data/${source}/${format}/${name}.arrow`);
if (fs.existsSync(arrowPath)) {
jsonPaths.push(jsonPath);
arrowPaths.push(arrowPath);
console.log('-j', jsonPath, '-a', arrowPath, '\\');
}
}
}
return [jsonPaths, arrowPaths];
}, [[], []]);
}
} else if (!jsonPaths.length) {
return print_usage();
}

switch (argv.mode.toUpperCase()) {
switch (mode) {
case 'VALIDATE':
const args = [`test`, `-i`].concat(argv._unknown || []);
jsonPaths.forEach((p, i) => {
args.push('-j', p, '-a', arrowPaths[i]);
});
child_process.spawnSync(
gulp,
[`test`, `-i`].concat(process.argv.slice(2)),
gulp, args,
{
cwd: path.resolve(__dirname, '..'),
stdio: ['ignore', 'inherit', 'inherit']
}
);
// for (let i = -1, n = jsonPaths.length; ++i < n;) {
// const jsonPath = jsonPaths[i];
// const arrowPath = arrowPaths[i];
// child_process.spawnSync(
// gulp, args.concat(['-j', jsonPath, '-a', arrowPath]),
// {
// cwd: path.resolve(__dirname, '..'),
// stdio: ['ignore', 'inherit', 'inherit']
// }
// );
// }
break;
default:
print_usage();
Expand Down
6 changes: 2 additions & 4 deletions js/gulp/argv.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,11 @@ const argv = require(`command-line-args`)([
{ name: `target`, type: String, defaultValue: `` },
{ name: `module`, type: String, defaultValue: `` },
{ name: `coverage`, type: Boolean, defaultValue: false },
{ name: `json_file`, alias: `j`, type: String, defaultValue: null },
{ name: `arrow_file`, alias: `a`, type: String, defaultValue: null },
{ name: `integration`, alias: `i`, type: Boolean, defaultValue: false },
{ name: `targets`, alias: `t`, type: String, multiple: true, defaultValue: [] },
{ name: `modules`, alias: `m`, type: String, multiple: true, defaultValue: [] },
{ name: `sources`, alias: `s`, type: String, multiple: true, defaultValue: [`cpp`, `java`] },
{ name: `formats`, alias: `f`, type: String, multiple: true, defaultValue: [`file`, `stream`] },
{ name: `json_files`, alias: `j`, type: String, multiple: true, defaultValue: [] },
{ name: `arrow_files`, alias: `a`, type: String, multiple: true, defaultValue: [] },
], { partial: true });

const { targets, modules } = argv;
Expand Down
6 changes: 2 additions & 4 deletions js/gulp/test-task.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,11 +48,9 @@ const testTask = ((cache, execArgv, testOptions) => memoizeTask(cache, function
opts.env = { ...opts.env,
TEST_TARGET: target,
TEST_MODULE: format,
JSON_PATH: argv.json_file,
ARROW_PATH: argv.arrow_file,
TEST_TS_SOURCE: !!argv.coverage,
TEST_SOURCES: JSON.stringify(Array.isArray(argv.sources) ? argv.sources : [argv.sources]),
TEST_FORMATS: JSON.stringify(Array.isArray(argv.formats) ? argv.formats : [argv.formats]),
JSON_PATHS: JSON.stringify(Array.isArray(argv.json_files) ? argv.json_files : [argv.json_files]),
ARROW_PATHS: JSON.stringify(Array.isArray(argv.arrow_files) ? argv.arrow_files : [argv.arrow_files]),
};
return !debug ?
child_process.spawn(jest, args, opts) :
Expand Down
8 changes: 4 additions & 4 deletions js/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,8 +62,8 @@
"devDependencies": {
"@std/esm": "0.19.7",
"@types/flatbuffers": "1.6.5",
"@types/glob": "5.0.34",
"@types/jest": "22.0.1",
"@types/glob": "5.0.35",
"@types/jest": "22.1.0",
"@types/node": "9.3.0",
"ast-types": "0.10.1",
"benchmark": "2.1.4",
Expand All @@ -78,9 +78,9 @@
"gulp-rename": "1.2.2",
"gulp-sourcemaps": "2.6.3",
"gulp-transform-js-ast": "1.0.2",
"gulp-typescript": "3.2.3",
"gulp-typescript": "3.2.4",
"ix": "2.3.4",
"jest": "22.1.3",
"jest": "22.1.4",
"jest-environment-node-debug": "2.0.0",
"json": "9.0.6",
"lerna": "2.7.1",
Expand Down
75 changes: 48 additions & 27 deletions js/test/integration/validate-tests.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,41 @@
import * as fs from 'fs';
import * as path from 'path';

if (!process.env.JSON_PATH || !process.env.ARROW_PATH) {
throw new Error('Integration tests need paths to both json and arrow files');
}
import Arrow from '../Arrow';
import { zip } from 'ix/iterable/zip';
import { toArray } from 'ix/iterable/toarray';

const jsonPath = path.resolve(process.env.JSON_PATH + '');
const arrowPath = path.resolve(process.env.ARROW_PATH + '');
/* tslint:disable */
const { parse: bignumJSONParse } = require('json-bignum');

if (!fs.existsSync(jsonPath) || !fs.existsSync(arrowPath)) {
throw new Error('Integration tests need both json and arrow files to exist');
}
const { Table, read } = Arrow;

/* tslint:disable */
const { parse } = require('json-bignum');
if (!process.env.JSON_PATHS || !process.env.ARROW_PATHS) {
throw new Error('Integration tests need paths to both json and arrow files');
}

const jsonData = parse(fs.readFileSync(jsonPath, 'utf8'));
const arrowBuffers: Uint8Array[] = [fs.readFileSync(arrowPath)];
function resolvePathArgs(paths: string) {
let pathsArray = JSON.parse(paths) as string | string[];
return (Array.isArray(pathsArray) ? pathsArray : [pathsArray])
.map((p) => path.resolve(p))
.map((p) => {
if (fs.existsSync(p)) {
return p;
}
console.warn(`Could not find file "${p}"`);
return undefined;
});
}

import Arrow from '../Arrow';
import { zip } from 'ix/iterable/zip';
import { toArray } from 'ix/iterable/toarray';
const getOrReadFileBuffer = ((cache: any) => function getFileBuffer(path: string, ...args: any[]) {
return cache[path] || (cache[path] = fs.readFileSync(path, ...args));
})({});

const { Table, read } = Arrow;
const jsonAndArrowPaths = toArray(zip(
resolvePathArgs(process.env.JSON_PATHS!),
resolvePathArgs(process.env.ARROW_PATHS!)
))
.filter(([p1, p2]) => p1 !== undefined && p2 !== undefined) as [string, string][];

expect.extend({
toEqualVector(v1: any, v2: any) {
Expand All @@ -66,7 +79,7 @@ expect.extend({

for (let i = -1, n = props.length; ++i < n;) {
const prop = props[i];
if (this.utils.stringify(v1[prop]) !== this.utils.stringify(v2[prop])) {
if (`${v1[prop]}` !== `${v2[prop]}`) {
propsFailures.push(`${prop}: ${format(v1[prop], v2[prop], ' !== ')}`);
}
}
Expand Down Expand Up @@ -98,35 +111,43 @@ expect.extend({
});

describe(`Integration`, () => {
testReaderIntegration();
testTableFromBuffersIntegration();
for (const [jsonFilePath, arrowFilePath] of jsonAndArrowPaths) {
let { name, dir } = path.parse(arrowFilePath);
dir = dir.split(path.sep).slice(-2).join(path.sep);
const json = bignumJSONParse(getOrReadFileBuffer(jsonFilePath, 'utf8'));
const arrowBuffer = getOrReadFileBuffer(arrowFilePath) as Uint8Array;
describe(path.join(dir, name), () => {
testReaderIntegration(json, arrowBuffer);
testTableFromBuffersIntegration(json, arrowBuffer);
});
}
});

function testReaderIntegration() {
test(`json and arrow buffers report the same values`, () => {
debugger;
function testReaderIntegration(jsonData: any, arrowBuffer: Uint8Array) {
test(`json and arrow record batches report the same values`, () => {
expect.hasAssertions();
const jsonRecordBatches = toArray(read(jsonData));
const binaryRecordBatches = toArray(read(arrowBuffers));
const binaryRecordBatches = toArray(read(arrowBuffer));
for (const [jsonRecordBatch, binaryRecordBatch] of zip(jsonRecordBatches, binaryRecordBatches)) {
expect(jsonRecordBatch.length).toEqual(binaryRecordBatch.length);
expect(jsonRecordBatch.numCols).toEqual(binaryRecordBatch.numCols);
for (let i = -1, n = jsonRecordBatch.numCols; ++i < n;) {
(jsonRecordBatch.columns[i] as any).name = jsonRecordBatch.schema.fields[i].name;
(expect(jsonRecordBatch.columns[i]) as any).toEqualVector(binaryRecordBatch.columns[i]);
}
}
});
}

function testTableFromBuffersIntegration() {
test(`json and arrow buffers report the same values`, () => {
debugger;
function testTableFromBuffersIntegration(jsonData: any, arrowBuffer: Uint8Array) {
test(`json and arrow tables report the same values`, () => {
expect.hasAssertions();
const jsonTable = Table.from(jsonData);
const binaryTable = Table.from(arrowBuffers);
const binaryTable = Table.from(arrowBuffer);
expect(jsonTable.length).toEqual(binaryTable.length);
expect(jsonTable.numCols).toEqual(binaryTable.numCols);
for (let i = -1, n = jsonTable.numCols; ++i < n;) {
(jsonTable.columns[i] as any).name = jsonTable.schema.fields[i].name;
(expect(jsonTable.columns[i]) as any).toEqualVector(binaryTable.columns[i]);
}
});
Expand Down

0 comments on commit e537789

Please sign in to comment.