feat: BuildPulse flaky test detection (#11967)

Co-authored-by: Jason Rudolph <jason@jasonrudolph.com>
prisma · Feb 24, 2022 · 3d07eb6 · 3d07eb6
1 parent 52eed07
commit 3d07eb6
Show file tree

Hide file tree

Showing 21 changed files with 264 additions and 26 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -557,14 +557,14 @@ jobs:
           name: generator-helper-${{ matrix.os }}
 
   #
-  # Run all tests on macOS and Windows.
+  # Run (almost) all tests on macOS and Windows.
   #
   # Unlike the other jobs, this job doesn't use Docker (and thus skips some
   # tests that require dependencies not easily installable without Docker).
   #
-  # It also runs the tests for different packages sequentially to prevent the
-  # combinatorial explosion of the number of parallel jobs, to minimize the
-  # number of times we need to install MySQL using the package manager
+  # It also runs most tests for different packages sequentially (except client,
+  # see below) to prevent the combinatorial explosion of the number of parallel jobs,
+  # to minimize the number of times we need to install MySQL using the package manager
   # (PostgreSQL and MongoDB are provided by GitHub Actions out of the box), and
   # minimize the time spent waiting for a free runner.
   #
@@ -577,7 +577,6 @@ jobs:
       ${{
         contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') ||
         contains(needs.detect_jobs_to_run.outputs.jobs, '-sdk-') ||
-        contains(needs.detect_jobs_to_run.outputs.jobs, '-client-') ||
         contains(needs.detect_jobs_to_run.outputs.jobs, '-migrate-') ||
         contains(needs.detect_jobs_to_run.outputs.jobs, '-cli-')
       }}
@@ -589,6 +588,9 @@ jobs:
         node: [16]
         queryEngine: ['library', 'binary']
 
+    env:
+      JEST_JUNIT_SUITE_NAME: '${{ matrix.os }}/node-${{ matrix.node }}/${{ matrix.queryEngine }}/{title}'
+
     steps:
       - uses: actions/checkout@v2
 
@@ -639,23 +641,6 @@ jobs:
           flags: sdk,${{ matrix.os }},${{ matrix.queryEngine }}
           name: sdk-${{ matrix.os }}-${{ matrix.queryEngine }}
 
-      - name: Test packages/client
-        if: ${{ contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') || contains(needs.detect_jobs_to_run.outputs.jobs, '-client-') }}
-        run: pnpm run test-notypes
-        working-directory: packages/client
-        env:
-          CI: true
-          SKIP_GIT: true
-          GITHUB_CONTEXT: ${{ toJson(github) }}
-          NODE_OPTIONS: '--max-old-space-size=8096'
-
-      - uses: codecov/codecov-action@v2
-        if: ${{ contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') || contains(needs.detect_jobs_to_run.outputs.jobs, '-client-') }}
-        with:
-          files: ./packages/client/src/__tests__/coverage/clover.xml
-          flags: client,${{ matrix.os }},${{ matrix.queryEngine }}
-          name: client-${{ matrix.os }}-${{ matrix.queryEngine }}
-
       - name: Test packages/migrate
         if: ${{ contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') || contains(needs.detect_jobs_to_run.outputs.jobs, '-migrate-') }}
         run: pnpm run test
@@ -735,3 +720,103 @@ jobs:
           files: ./packages/generator-helper/src/__tests__/coverage/clover.xml
           flags: generator-helper,${{ matrix.os }},library,binary
           name: generator-helper-${{ matrix.os }}
+
+      - name: Upload test results to BuildPulse for flaky test detection
+        # Only run this step for branches where where we have access to secrets.
+        # Only run this step on macOS for now.
+        # Run this step even when the tests fail. Skip if the workflow is cancelled.
+        if: startsWith(github.repository, 'prisma/') && runner.os == 'macOS' && !cancelled()
+        env:
+          BUILDPULSE_ACCESS_KEY_ID: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          BUILDPULSE_SECRET_ACCESS_KEY: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
+        run: |
+          curl -fsSL https://get.buildpulse.io/test-reporter-darwin-amd64 > ./buildpulse-test-reporter
+          chmod +x ./buildpulse-test-reporter
+          ./buildpulse-test-reporter submit packages --account-id 17219288 --repository-id 192925833
+
+  #
+  # Run Client tests on macOS and Windows. (Rasoning see above)
+  #
+  no-docker-client:
+    timeout-minutes: 40
+    runs-on: ${{ matrix.os }}
+    needs: detect_jobs_to_run
+
+    if: |
+      ${{
+        contains(needs.detect_jobs_to_run.outputs.jobs, '-client-')
+      }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [macos-11, windows-latest]
+        node: [16]
+        queryEngine: ['library', 'binary']
+
+    env:
+      JEST_JUNIT_SUITE_NAME: '${{ matrix.os }}/node-${{ matrix.node }}/${{ matrix.queryEngine }}/{title}'
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - name: Prerequisites
+        shell: bash
+        run: |
+          echo "TEST_SKIP_MSSQL=true" >> $GITHUB_ENV
+          echo "TEST_SKIP_MONGODB=true" >> $GITHUB_ENV
+          echo "TEST_SKIP_COCKROACHDB=true" >> $GITHUB_ENV
+          echo "TEST_MONGO_URI_MIGRATE=mongodb://localhost:27017/tests-migrate" >> $GITHUB_ENV
+          echo "PRISMA_CLI_QUERY_ENGINE_TYPE=${{ matrix.queryEngine }}" >> $GITHUB_ENV
+          echo "PRISMA_CLIENT_ENGINE_TYPE=${{ matrix.queryEngine }}" >> $GITHUB_ENV
+
+      - name: Setup PostgreSQL
+        run: bash .github/workflows/setup-postgres.sh
+
+      - name: Setup MySQL
+        run: bash .github/workflows/setup-mysql.sh
+
+      - uses: pnpm/action-setup@v2.1.0
+        with:
+          version: latest
+
+      - uses: actions/setup-node@v2
+        with:
+          cache: 'pnpm'
+          node-version: ${{ matrix.node }}
+
+      - run: bash .github/workflows/setup.sh
+        env:
+          CI: true
+          SKIP_GIT: true
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+
+      - name: Test packages/client
+        if: ${{ contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') || contains(needs.detect_jobs_to_run.outputs.jobs, '-client-') }}
+        run: pnpm run test-notypes
+        working-directory: packages/client
+        env:
+          CI: true
+          SKIP_GIT: true
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+          NODE_OPTIONS: '--max-old-space-size=8096'
+
+      - uses: codecov/codecov-action@v2
+        if: ${{ contains(needs.detect_jobs_to_run.outputs.jobs, '-all-') || contains(needs.detect_jobs_to_run.outputs.jobs, '-client-') }}
+        with:
+          files: ./packages/client/src/__tests__/coverage/clover.xml
+          flags: client,${{ matrix.os }},${{ matrix.queryEngine }}
+          name: client-${{ matrix.os }}-${{ matrix.queryEngine }}
+
+      - name: Upload test results to BuildPulse for flaky test detection
+        # Only run this step for branches where where we have access to secrets.
+        # Only run this step on macOS for now.
+        # Run this step even when the tests fail. Skip if the workflow is cancelled.
+        if: startsWith(github.repository, 'prisma/') && runner.os == 'macOS' && !cancelled()
+        env:
+          BUILDPULSE_ACCESS_KEY_ID: ${{ secrets.BUILDPULSE_ACCESS_KEY_ID }}
+          BUILDPULSE_SECRET_ACCESS_KEY: ${{ secrets.BUILDPULSE_SECRET_ACCESS_KEY }}
+        run: |
+          curl -fsSL https://get.buildpulse.io/test-reporter-darwin-amd64 > ./buildpulse-test-reporter
+          chmod +x ./buildpulse-test-reporter
+          ./buildpulse-test-reporter submit packages/client --account-id 17219288 --repository-id 192925833
diff --git a/package.json b/package.json
@@ -63,6 +63,7 @@
     "graphviz-mit": "0.0.9",
     "husky": "7.0.4",
     "is-ci": "3.0.1",
+    "jest-junit": "13.0.0",
     "lint-staged": "12.3.4",
     "node-fetch": "2.6.7",
     "p-map": "4.0.0",

diff --git a/packages/cli/jest.config.js b/packages/cli/jest.config.js
@@ -7,4 +7,16 @@ module.exports = {
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
   snapshotSerializers: ['@prisma/sdk/src/utils/jestSnapshotSerializer'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/cli/package.json b/packages/cli/package.json
@@ -92,6 +92,7 @@
     "global-dirs": "3.0.0",
     "is-installed-globally": "0.4.0",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "line-replace": "2.0.1",
     "log-update": "4.0.0",
     "make-dir": "3.1.0",

diff --git a/packages/client/jest.config.js b/packages/client/jest.config.js
@@ -28,4 +28,16 @@ module.exports = {
   snapshotSerializers: ['@prisma/sdk/src/utils/jestSnapshotSerializer'],
   testTimeout: 90000,
   setupFiles: ['./helpers/jestSetup.js'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/client/package.json b/packages/client/package.json
@@ -89,6 +89,7 @@
     "is-obj": "2.0.0",
     "is-regexp": "2.1.0",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "js-levenshtein": "1.1.6",
     "klona": "2.0.5",
     "lz-string": "1.4.4",

diff --git a/packages/debug/jest.config.js b/packages/debug/jest.config.js
@@ -1,9 +1,21 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
+  testMatch: ['**/src/__tests__/**/*.test.ts'],
   collectCoverage: process.env.CI ? true : false,
   coverageReporters: ['clover'],
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
-  testMatch: ['**/src/__tests__/**/*.test.ts'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/debug/package.json b/packages/debug/package.json
@@ -22,6 +22,7 @@
     "@types/node": "12.20.46",
     "esbuild": "0.13.14",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "ts-jest": "27.1.3",
     "typescript": "4.5.4"
   },

diff --git a/packages/engine-core/jest.config.js b/packages/engine-core/jest.config.js
@@ -1,9 +1,21 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
+  testMatch: ['**/src/__tests__/**/*.test.ts'],
   collectCoverage: process.env.CI ? true : false,
   coverageReporters: ['clover'],
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
-  testMatch: ['**/src/__tests__/**/*.test.ts'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/engine-core/package.json b/packages/engine-core/package.json
@@ -23,6 +23,7 @@
     "@types/node": "12.20.46",
     "esbuild": "0.13.14",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "ts-jest": "27.1.3",
     "typescript": "4.5.4"
   },

diff --git a/packages/generator-helper/jest.config.js b/packages/generator-helper/jest.config.js
@@ -6,4 +6,16 @@ module.exports = {
   coverageReporters: ['clover'],
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/generator-helper/package.json b/packages/generator-helper/package.json
@@ -36,6 +36,7 @@
     "@types/node": "12.20.46",
     "esbuild": "0.13.14",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "ts-jest": "27.1.3",
     "ts-node": "10.4.0",
     "typescript": "4.5.4"

diff --git a/packages/integration-tests/jest.config.js b/packages/integration-tests/jest.config.js
@@ -7,4 +7,16 @@ module.exports = {
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
   snapshotSerializers: ['@prisma/sdk/src/utils/jestSnapshotSerializer'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/integration-tests/package.json b/packages/integration-tests/package.json
@@ -25,6 +25,7 @@
     "execa": "5.1.1",
     "fs-jetpack": "4.3.0",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "mariadb": "2.5.5",
     "mssql": "8.0.1",
     "pg": "8.7.1",

diff --git a/packages/migrate/jest.config.js b/packages/migrate/jest.config.js
@@ -6,6 +6,7 @@ module.exports = {
   coverageReporters: ['clover'],
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
+  snapshotSerializers: ['@prisma/sdk/src/utils/jestSnapshotSerializer'],
   coveragePathIgnorePatterns: [
     'bin.ts',
     'setupMysql.ts',
@@ -14,7 +15,18 @@ module.exports = {
     'test-handlePanic.ts',
     'test-interactivelyCreateDatabase.ts',
   ],
-  snapshotSerializers: ['@prisma/sdk/src/utils/jestSnapshotSerializer'],
   // to get rid of "jest-haste-map: Haste module naming collision: package name"
   modulePathIgnorePatterns: ['<rootDir>/src/__tests__/fixtures/'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }
diff --git a/packages/migrate/package.json b/packages/migrate/package.json
@@ -31,6 +31,7 @@
     "esbuild": "0.13.14",
     "fs-jetpack": "4.3.0",
     "jest": "27.5.1",
+    "jest-junit": "13.0.0",
     "make-dir": "3.1.0",
     "mock-stdin": "1.0.0",
     "tempy": "1.0.1",

diff --git a/packages/react-prisma/jest.config.js b/packages/react-prisma/jest.config.js
@@ -1,9 +1,21 @@
 module.exports = {
   preset: 'ts-jest',
   testEnvironment: 'node',
+  testMatch: ['**/src/__tests__/**/*.test.ts'],
   collectCoverage: process.env.CI ? true : false,
   coverageReporters: ['clover'],
   coverageDirectory: 'src/__tests__/coverage',
   collectCoverageFrom: ['src/**/*.ts', '!**/__tests__/**/*'],
-  testMatch: ['**/src/__tests__/**/*.test.ts'],
+  reporters: [
+    'default',
+    [
+      'jest-junit',
+      {
+        addFileAttribute: 'true',
+        ancestorSeparator: ' › ',
+        classNameTemplate: '{classname}',
+        titleTemplate: '{title}',
+      },
+    ],
+  ],
 }