Skip to content
Permalink
Browse files

feat: parallel testing (denoland#224)

  • Loading branch information...
chiefbiiko authored and ry committed Mar 4, 2019
1 parent e2fd507 commit 41bdd096f0b300056c58a04392d109bf11c1ce8e
Showing with 190 additions and 29 deletions.
  1. +18 −0 testing/bench.ts
  2. +1 −1 testing/format_test.ts
  3. +167 −26 testing/mod.ts
  4. +4 −2 testing/test.ts
@@ -0,0 +1,18 @@
import { bench, runBenchmarks } from "./../benching/mod.ts";
import { runTests } from "./mod.ts";

import "./test.ts";

bench(async function testingSerial(b) {
b.start();
await runTests();
b.stop();
});

bench(async function testingParallel(b) {
b.start();
await runTests({ parallel: true });
b.stop();
});

runBenchmarks({ only: /testing/ });
@@ -56,7 +56,7 @@ test({
});

test({
name: "prints empty arguments",
name: "prints an empty array",
fn() {
const val: any[] = [];
assertEqual(format(val), "Array []");
@@ -241,55 +241,196 @@ function green_ok() {
return green("ok");
}

export async function runTests() {
let passed = 0;
let failed = 0;

console.log("running", tests.length, "tests");
for (let i = 0; i < tests.length; i++) {
const { fn, name } = tests[i];
let result = green_ok();
interface TestStats {
filtered: number;
ignored: number;
measured: number;
passed: number;
failed: number;
}

interface TestResult {
name: string;
error: Error;
ok: boolean;
printed: boolean;
}

interface TestResults {
keys: Map<string, number>;
cases: Map<number, TestResult>;
}

function createTestResults(tests: Array<TestDefinition>): TestResults {
return tests.reduce(
(acc: TestResults, { name }: TestDefinition, i: number): TestResults => {
acc.keys.set(name, i);
acc.cases.set(i, { name, printed: false, ok: false, error: null });
return acc;
},
{ cases: new Map(), keys: new Map() }
);
}

function report(result: TestResult): void {
if (result.ok) {
console.log(`test ${result.name} ... ${green_ok()}`);
} else if (result.error) {
console.error(
`test ${result.name} ... ${red_failed()}\n${result.error.stack}`
);
} else {
console.log(`test ${result.name} ... unresolved`);
}
result.printed = true;
}

function printResults(
stats: TestStats,
results: TestResults,
flush: boolean
): void {
if (flush) {
for (const result of results.cases.values()) {
if (!result.printed) {
report(result);
if (result.error && exitOnFail) {
break;
}
}
}
}
// Attempting to match the output of Rust's test runner.
console.log(
`\ntest result: ${stats.failed ? red_failed() : green_ok()}. ` +
`${stats.passed} passed; ${stats.failed} failed; ` +
`${stats.ignored} ignored; ${stats.measured} measured; ` +
`${stats.filtered} filtered out\n`
);
}

function previousPrinted(name: string, results: TestResults): boolean {
const curIndex: number = results.keys.get(name);
if (curIndex === 0) {
return true;
}
return results.cases.get(curIndex - 1).printed;
}

async function createTestCase(
stats: TestStats,
results: TestResults,
{ fn, name }: TestDefinition
): Promise<void> {
const result: TestResult = results.cases.get(results.keys.get(name));
try {
await fn();
stats.passed++;
result.ok = true;
} catch (err) {
stats.failed++;
result.error = err;
if (exitOnFail) {
throw err;
}
}
if (previousPrinted(name, results)) {
report(result);
}
}

function initTestCases(
stats: TestStats,
results: TestResults,
tests: Array<TestDefinition>
): Array<Promise<void>> {
return tests.map(createTestCase.bind(null, stats, results));
}

async function runTestsParallel(
stats: TestStats,
results: TestResults,
tests: Array<TestDefinition>
): Promise<void> {
try {
await Promise.all(initTestCases(stats, results, tests));
} catch (_) {
// The error was thrown to stop awaiting all promises if exitOnFail === true
// stats.failed has been incremented and the error stored in results
}
}

async function runTestsSerial(
stats: TestStats,
tests: Array<TestDefinition>
): Promise<void> {
for (const { fn, name } of tests) {
// See https://github.com/denoland/deno/pull/1452
// about this usage of groupCollapsed
console.groupCollapsed(`test ${name} `);
try {
await fn();
passed++;
console.log("...", result);
stats.passed++;
console.log("...", green_ok());
console.groupEnd();
} catch (e) {
result = red_failed();
console.log("...", result);
} catch (err) {
console.log("...", red_failed());
console.groupEnd();
console.error(e);
failed++;
console.error(err.stack);
stats.failed++;
if (exitOnFail) {
break;
}
}
}
}

// Attempting to match the output of Rust's test runner.
const result = failed > 0 ? red_failed() : green_ok();
console.log(
`\ntest result: ${result}. ${passed} passed; ${failed} failed; ` +
`${ignored} ignored; ${measured} measured; ${filtered} filtered out\n`
);
/** Defines options for controlling execution details of a test suite. */
export interface RunOptions {
parallel?: boolean;
}

if (failed === 0) {
// All good.
/**
* Runs specified test cases.
* Parallel execution can be enabled via the boolean option; default: serial.
*/
export async function runTests({ parallel = false }: RunOptions = {}): Promise<
void
> {
const stats: TestStats = {
measured: 0,
ignored: 0,
filtered: filtered,
passed: 0,
failed: 0
};
const results: TestResults = createTestResults(tests);
console.log(`running ${tests.length} tests`);
if (parallel) {
await runTestsParallel(stats, results, tests);
} else {
await runTestsSerial(stats, tests);
}
printResults(stats, results, parallel);
if (stats.failed) {
// Use setTimeout to avoid the error being ignored due to unhandled
// promise rejections being swallowed.
setTimeout(() => {
console.error(`There were ${failed} test failures.`);
console.error(`There were ${stats.failed} test failures.`);
Deno.exit(1);
}, 0);
}
}

export async function runIfMain(meta: ImportMeta) {
/**
* Runs specified test cases if the enclosing script is main.
* Execution mode is toggleable via opts.parallel, defaults to false.
*/
export async function runIfMain(
meta: ImportMeta,
opts?: RunOptions
): Promise<void> {
if (meta.main) {
runTests();
return runTests(opts);
}
}
@@ -1,6 +1,6 @@
// Copyright 2018-2019 the Deno authors. All rights reserved. MIT license.

import { test, assert, assertEqual, equal } from "./mod.ts";
import { test, assert, assertEqual, equal, runIfMain } from "./mod.ts";
import "./format_test.ts";
import "./diff_test.ts";
import "./pretty_test.ts";
@@ -267,7 +267,7 @@ test(async function testingThrowsAsyncMsgIncludes() {
assert(count === 1);
});

test(async function testingThrowsMsgNotIncludes() {
test(async function testingThrowsAsyncMsgNotIncludes() {
let count = 0;
let didThrow = false;
try {
@@ -289,3 +289,5 @@ test(async function testingThrowsMsgNotIncludes() {
assert(count === 1);
assert(didThrow);
});

runIfMain(import.meta, { parallel: true });

0 comments on commit 41bdd09

Please sign in to comment.
You can’t perform that action at this time.
You signed in with another tab or window. Reload to refresh your session. You signed out in another tab or window. Reload to refresh your session.