Merge pull request #2359 from NomicFoundation/parallel-tests

Add `--parallel`, `--bail` and `--grep` params to the test task

.changeset/hip-planes-invent.md

@@ -0,0 +1,7 @@
+---
+"hardhat": minor
+---
+
+The `test` task now supports a `--parallel` flag to run tests in parallel. There are also two other new flags: `--bail`, to stop the execution after the first test failure, and `--grep`, to filter which tests should be run.
+
+To support running tests in parallel, the version of `mocha` used by Hardhat was upgraded to its latest version. This should be a mostly backward-compatible change, but there could be some edge cases where this breaks existing tests.

config/eslint/eslintrc.js

@@ -162,6 +162,7 @@ module.exports = {
       "error",
       {
         groups: [
+          "type",
           "object",
           ["builtin", "external"],
           "parent",

docs/.vuepress/config.js

@@ -101,6 +101,7 @@ module.exports = {
             ["/guides/compile-contracts.md", "Compiling your contracts", 0],
             ["/guides/waffle-testing.md", "Testing with ethers.js & Waffle", 0],
             ["/guides/truffle-testing.md", "Testing with Web3.js & Truffle", 0],
+            ["/guides/parallel-tests.md", "Running tests in parallel", 0],
             ["/guides/truffle-migration.md", "Migrating from Truffle", 0],
             ["/guides/deploying.md", "Deploying your contracts", 0],
             ["/guides/scripts.md", "Writing scripts", 0],

docs/guides/parallel-tests.md

@@ -0,0 +1,16 @@
+# Running tests in parallel
+
+You can run your tests in parallel by using the `--parallel` flag:
+
+```
+$ npx hardhat test --parallel
+```
+
+Most of the time, running your tests serially or in parallel should produce the same results, but there are some scenarios where tests run in parallel will behave differently:
+
+- In serial mode all the test files share the same instance of the [Hardhat Runtime Environment](/advanced/hardhat-runtime-environment.html), but in parallel mode this is not always the case. Mocha uses a pool of workers to execute the tests, and each worker starts with its own instance of the HRE. This means that if one test file deploys a contract, then that deployment will exist in some of the other test files and it won't in others.
+- The `.only` modifier doesn't work in parallel mode. As an alternative, you can use [`--grep`](https://mochajs.org/#-grep-regexp-g-regexp) to run specific tests.
+- Because parallel mode uses more system resources, the duration of individual tests might be longer, so there's a chance that some tests start timing out for that reason. If you run into this problem, you can increase the tests timeout in the [Mocha section of your Hardhat config](/config/#mocha-configuration) or using [`this.timeout()`](https://mochajs.org/#timeouts) in your tests.
+- The order in which tests are executed is non-deterministic.
+
+There are some other limitations related to parallel mode. You can read more about them in [Mocha's docs](https://mochajs.org/#parallel-tests). And if you are running into some issue when using parallel mode, you can check their [Troubleshooting parallel mode](https://mochajs.org/#troubleshooting-parallel-mode) section.

packages/e2e/package.json

@@ -21,7 +21,7 @@
   "dependencies": {
     "@types/chai": "^4.2.0",
     "@types/fs-extra": "^5.1.0",
-    "@types/mocha": "^9.0.0",
+    "@types/mocha": "^9.1.0",
     "@types/node": "^12.0.0",
     "@types/shelljs": "^0.8.6",
     "@typescript-eslint/eslint-plugin": "4.29.2",
@@ -32,7 +32,7 @@
     "eslint-plugin-import": "2.24.1",
     "eslint-plugin-prettier": "3.4.0",
     "fs-extra": "^7.0.1",
-    "mocha": "^7.2.0",
+    "mocha": "^9.2.0",
     "prettier": "2.4.1",
     "rimraf": "^3.0.2",
     "shelljs": "^0.8.5",

packages/e2e/test/fixture-projects/basic-project/test/another-test.js

@@ -0,0 +1,3 @@
+describe("simple", () => {
+  it("should pass", () => {});
+});

packages/e2e/test/index.ts

@@ -54,7 +54,7 @@ describe("e2e tests", function () {
       // check stdout
 
       // check we get passing runs
-      assert.match(stdout, /1 passing/);
+      assert.match(stdout, /2 passing/);
       // check we get no runs without tests
       assert.notMatch(
         stdout,
@@ -84,6 +84,25 @@ describe("e2e tests", function () {
       );
       assert.equal(testRunCode2, 0);
     });
+
+    it("should run tests in parallel", function () {
+      // hh clean
+      const { code: hhCleanCode1 } = shell.exec(`${hardhatBinary} clean`);
+      assert.equal(hhCleanCode1, 0);
+
+      // hh test --parallel
+      const { code: hhCompileCode, stdout } = shell.exec(
+        `${hardhatBinary} test --parallel`
+      );
+      assert.equal(hhCompileCode, 0);
+
+      // check we get passing runs
+      assert.match(stdout, /2 passing/);
+
+      // hh clean
+      const { code: hhCleanCode2 } = shell.exec(`${hardhatBinary} clean`);
+      assert.equal(hhCleanCode2, 0);
+    });
   });
 
   describe("sample projects", function () {

packages/hardhat-core/package.json

@@ -63,7 +63,7 @@
     "@types/fs-extra": "^5.1.0",
     "@types/glob": "^7.1.1",
     "@types/lodash": "^4.14.123",
-    "@types/mocha": "^9.0.0",
+    "@types/mocha": "^9.1.0",
     "@types/node": "^12.0.0",
     "@types/node-fetch": "^2.3.7",
     "@types/qs": "^6.5.3",
@@ -81,7 +81,7 @@
     "eslint-plugin-import": "2.24.1",
     "eslint-plugin-prettier": "3.4.0",
     "ethers": "^5.0.0",
-    "mocha": "^7.2.0",
+    "mocha": "^9.2.0",
     "prettier": "2.4.1",
     "proxy": "^1.0.2",
     "rimraf": "^3.0.2",
@@ -124,7 +124,7 @@
     "lodash": "^4.17.11",
     "merkle-patricia-tree": "^4.2.2",
     "mnemonist": "^0.38.0",
-    "mocha": "^7.2.0",
+    "mocha": "^9.2.0",
     "node-fetch": "^2.6.0",
     "qs": "^6.7.0",
     "raw-body": "^2.4.1",

packages/hardhat-core/src/builtin-tasks/node.ts

@@ -1,6 +1,7 @@
+import type EthereumjsUtilT from "ethereumjs-util";
+
 import chalk from "chalk";
 import debug from "debug";
-import type EthereumjsUtilT from "ethereumjs-util";
 import fsExtra from "fs-extra";
 
 import { HARDHAT_NETWORK_NAME } from "../internal/constants";

packages/hardhat-core/src/builtin-tasks/test.ts

@@ -1,3 +1,5 @@
+import type { MochaOptions } from "mocha";
+
 import chalk from "chalk";
 import path from "path";
 
@@ -47,24 +49,61 @@ subtask(TASK_TEST_GET_TEST_FILES)
 subtask(TASK_TEST_SETUP_TEST_ENVIRONMENT, async () => {});
 
 subtask(TASK_TEST_RUN_MOCHA_TESTS)
+  .addFlag("parallel", "Run tests in parallel")
+  .addFlag("bail", "Stop running tests after the first test failure")
+  .addOptionalParam(
+    "grep",
+    "Only run tests matching the given string or regexp"
+  )
   .addOptionalVariadicPositionalParam(
     "testFiles",
     "An optional list of files to test",
     []
   )
-  .setAction(async ({ testFiles }: { testFiles: string[] }, { config }) => {
-    const { default: Mocha } = await import("mocha");
-    const mocha = new Mocha(config.mocha);
-    testFiles.forEach((file) => mocha.addFile(file));
+  .setAction(
+    async (
+      taskArgs: {
+        bail: boolean;
+        parallel: boolean;
+        testFiles: string[];
+        grep?: string;
+      },
+      { config }
+    ) => {
+      const { default: Mocha } = await import("mocha");
 
-    const testFailures = await new Promise<number>((resolve) => {
-      mocha.run(resolve);
-    });
+      const mochaConfig: MochaOptions = {
+        ...config.mocha,
+        grep: taskArgs.grep,
+      };
 
-    mocha.dispose();
+      if (taskArgs.bail) {
+        mochaConfig.bail = true;
+      }
+      if (taskArgs.parallel) {
+        mochaConfig.parallel = true;
+      }
 
-    return testFailures;
-  });
+      if (mochaConfig.parallel === true) {
+        const mochaRequire = mochaConfig.require ?? [];
+        if (!mochaRequire.includes("hardhat/register")) {
+          mochaRequire.push("hardhat/register");
+        }
+        mochaConfig.require = mochaRequire;
+      }
+
+      const mocha = new Mocha(mochaConfig);
+      taskArgs.testFiles.forEach((file) => mocha.addFile(file));
+
+      const testFailures = await new Promise<number>((resolve) => {
+        mocha.run(resolve);
+      });
+
+      mocha.dispose();
+
+      return testFailures;
+    }
+  );
 
 subtask(TASK_TEST_RUN_SHOW_FORK_RECOMMENDATIONS).setAction(
   async (_, { config, network }) => {
@@ -84,14 +123,26 @@ task(TASK_TEST, "Runs mocha tests")
     []
   )
   .addFlag("noCompile", "Don't compile before running this task")
+  .addFlag("parallel", "Run tests in parallel")
+  .addFlag("bail", "Stop running tests after the first test failure")
+  .addOptionalParam(
+    "grep",
+    "Only run tests matching the given string or regexp"
+  )
   .setAction(
     async (
       {
         testFiles,
         noCompile,
+        parallel,
+        bail,
+        grep,
       }: {
         testFiles: string[];
         noCompile: boolean;
+        parallel: boolean;
+        bail: boolean;
+        grep?: string;
       },
       { run, network }
     ) => {
@@ -107,6 +158,9 @@ task(TASK_TEST, "Runs mocha tests")
 
       const testFailures = await run(TASK_TEST_RUN_MOCHA_TESTS, {
         testFiles: files,
+        parallel,
+        bail,
+        grep,
       });
 
       if (network.name === HARDHAT_NETWORK_NAME) {

packages/hardhat-core/src/builtin-tasks/utils/solidity-files-cache.ts

@@ -1,11 +1,12 @@
+import type { LoDashStatic } from "lodash";
+import type { ProjectPathsConfig, SolcConfig } from "../../types";
+
 import debug from "debug";
 import fsExtra from "fs-extra";
 import * as t from "io-ts";
-import type { LoDashStatic } from "lodash";
 import * as path from "path";
 
 import { SOLIDITY_FILES_CACHE_FILENAME } from "../../internal/constants";
-import type { ProjectPathsConfig, SolcConfig } from "../../types";
 
 const log = debug("hardhat:core:tasks:compile:cache");
 

packages/hardhat-core/src/internal/core/config/config-loading.ts

@@ -1,9 +1,10 @@
+import type StackTraceParserT from "stacktrace-parser";
+
 import chalk from "chalk";
 import debug from "debug";
 import fsExtra from "fs-extra";
 import path from "path";
 import semver from "semver";
-import type StackTraceParserT from "stacktrace-parser";
 
 import { HardhatArguments, HardhatConfig } from "../../../types";
 import { HardhatContext } from "../../context";

packages/hardhat-core/src/internal/core/providers/construction.ts

@@ -11,6 +11,7 @@ import type {
   NetworkConfig,
   ProjectPathsConfig,
 } from "../../../types";
+
 import { HARDHAT_NETWORK_NAME } from "../../constants";
 import { ModulesLogger } from "../../hardhat-network/provider/modules/logger";
 import {

packages/hardhat-core/src/internal/hardhat-network/jsonrpc/server.ts

@@ -1,7 +1,8 @@
+import type WsT from "ws";
+
 import debug from "debug";
 import http, { Server } from "http";
 import { AddressInfo } from "net";
-import type WsT from "ws";
 
 import {
   EIP1193Provider,

packages/hardhat-core/src/internal/hardhat-network/provider/node-types.ts

@@ -1,3 +1,5 @@
+import type { ReturnData } from "./return-data";
+
 import { Block } from "@ethereumjs/block";
 import { RunBlockResult } from "@ethereumjs/vm/dist/runBlock";
 import { BN } from "ethereumjs-util";
@@ -6,8 +8,6 @@ import { HARDHAT_MEMPOOL_SUPPORTED_ORDERS } from "../../constants";
 import { BuildInfo, HardhatNetworkChainsConfig } from "../../../types";
 import { MessageTrace } from "../stack-traces/message-trace";
 
-import type { ReturnData } from "./return-data";
-
 export type NodeConfig = LocalNodeConfig | ForkedNodeConfig;
 
 export function isForkedNodeConfig(

packages/hardhat-core/src/internal/hardhat-network/provider/provider.ts

@@ -1,11 +1,3 @@
-import Common from "@ethereumjs/common";
-import chalk from "chalk";
-import debug from "debug";
-import { BN } from "ethereumjs-util";
-import { EventEmitter } from "events";
-import fsExtra from "fs-extra";
-import semver from "semver";
-
 import type {
   Artifacts,
   BoundExperimentalHardhatNetworkMessageTraceHook,
@@ -14,6 +6,15 @@ import type {
   HardhatNetworkChainsConfig,
   RequestArguments,
 } from "../../../types";
+
+import Common from "@ethereumjs/common";
+import chalk from "chalk";
+import debug from "debug";
+import { BN } from "ethereumjs-util";
+import { EventEmitter } from "events";
+import fsExtra from "fs-extra";
+import semver from "semver";
+
 import {
   HARDHAT_NETWORK_RESET_EVENT,
   HARDHAT_NETWORK_REVERT_SNAPSHOT_EVENT,

packages/hardhat-core/src/internal/hardhat-network/stack-traces/debug.ts

@@ -182,7 +182,7 @@ function flattenSourceReference(sourceReference?: SourceReference) {
 
   return {
     ...sourceReference,
-    file: sourceReference.file.sourceName,
+    file: sourceReference.sourceName,
   };
 }