Do not run flaky cli tests locally by default (#15928)

.github/workflows/ci.yml

@@ -75,6 +75,8 @@ jobs:
         run: |
           make -j test-ci-coverage
           yarn test:esm
+        env:
+          BABEL_CLI_FLAKY_TESTS: true
       - name: Upload coverage report
         uses: codecov/codecov-action@v3
 
@@ -100,6 +102,7 @@ jobs:
         env:
           BABEL_ENV: test
           USE_ESM: true
+          BABEL_CLI_FLAKY_TESTS: trues
 
   build:
     name: Build Babel Artifacts
@@ -247,6 +250,7 @@ jobs:
         run: |
           BABEL_ENV=test node --max-old-space-size=4096 ./node_modules/.bin/jest --ci
         env:
+          BABEL_CLI_FLAKY_TESTS: true
           TEST_FUZZ: "${{ (matrix.node-version == '6' || matrix.node-version == '8' || matrix.node-version == '10') && 'false' || 'true' }}"
       - name: Use Node.js latest # For `yarn version` in post actions of the first actions/setup-node
         if: matrix.node-version == '6' || matrix.node-version == '8' || matrix.node-version == '10'
@@ -327,6 +331,7 @@ jobs:
           BABEL_ENV: test
           BABEL_8_BREAKING: true
           BABEL_TYPES_8_BREAKING: true
+          BABEL_CLI_FLAKY_TESTS: true
 
   test-windows:
     name: Test on Windows
@@ -352,6 +357,7 @@ jobs:
         run: yarn jest --ci
         env:
           BABEL_ENV: test
+          BABEL_CLI_FLAKY_TESTS: true
 
   external-parser-tests:
     name: Third-party Parser Tests

CONTRIBUTING.md

@@ -219,6 +219,9 @@ In case you're not able to reproduce an error on CI locally, it may be due to
 
 - Node Version: Travis CI runs the tests against all major node versions. If your tests use JavaScript features unsupported by lower versions of node, then use [minNodeVersion option](#writing-tests) in options.json.
 - Timeout: Check the CI log and if the only errors are timeout errors and you are sure that it's not related to the changes you made, ask someone in the slack channel to trigger rebuild on the CI build and it might be resolved
+- Some `@babel/cli` tests are known to be flaky and thus we do not run them by
+  default locally. You can run them by setting the `BABEL_CLI_FLAKY_TESTS=true`
+  env variable.
 
 In case you're locally getting errors which are not on the CI, it may be due to
 

packages/babel-cli/test/fixtures/babel/dir --out-dir --watch multiple dir/options.json

@@ -2,5 +2,6 @@
   "args": ["*/src", "--out-dir", "../lib", "--relative", "--watch"],
   "noBabelrc": true,
   "noDefaultPlugins": true,
-  "minNodeVersion": 10
+  "minNodeVersion": 10,
+  "flaky": true
 }

packages/babel-cli/test/index.js

@@ -316,8 +316,13 @@ fs.readdirSync(fixtureLoc).forEach(function (binName) {
       }
 
       const skip =
-        opts.minNodeVersion &&
-        parseInt(process.versions.node, 10) < opts.minNodeVersion;
+        (opts.minNodeVersion &&
+          parseInt(process.versions.node, 10) < opts.minNodeVersion) ||
+        (opts.flaky && !process.env.BABEL_CLI_FLAKY_TESTS);
+
+      if (opts.flaky) {
+        testName += " (flaky)";
+      }
 
       // eslint-disable-next-line jest/valid-title
       (skip ? it.skip : it)(