Split stability tests into a separate Github Actions workflow (#5600)

* added separate workflow for stability tests * upload artifacts on success of last step only * update job name
open-telemetry · Oct 6, 2021 · 3a278c1 · 3a278c1
1 parent 23fd505
commit 3a278c1
Show file tree

Hide file tree

Showing 2 changed files with 96 additions and 45 deletions.
diff --git a/.github/workflows/build-and-test.yml b/.github/workflows/build-and-test.yml
@@ -82,9 +82,7 @@ jobs:
       - name: Split Loadtest Jobs
         id: splitloadtest
         run: ./.github/workflows/scripts/setup_e2e_tests.sh
-      - name: Split Stability Jobs
-        id: splitstabilitytest
-        run: ./.github/workflows/scripts/setup_stability_tests.sh
+
   lint:
     runs-on: ubuntu-latest
     needs: [setup-environment]
@@ -263,48 +261,6 @@ jobs:
       - name: Correctness
         run: make -C testbed run-correctness-metrics-tests
 
-  stability-tests:
-    # Stability tests disabled in the makefile. We disable them here to avoid
-    # consuming workers just to run a noop make target.
-    # uncomment the following line to enable stability tests.
-    # if: ${{ github.ref == 'ref/head/main' }}
-    if: ${{ false }}
-    runs-on: ubuntu-latest
-    needs: [setup-environment]
-    strategy:
-      matrix: ${{ fromJson(needs.setup-environment.outputs.stabilitytest_matrix) }}
-    steps:
-      - name: Checkout Repo
-        uses: actions/checkout@v2
-      - name: Setup Go
-        uses: actions/setup-go@v2.1.4
-        with:
-          go-version: 1.17
-      - name: Setup Go Environment
-        run: |
-          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
-          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
-      - name: Cache Go
-        id: module-cache
-        uses: actions/cache@v2
-        env:
-          cache-name: cache-go-modules
-        with:
-          path: |
-            /home/runner/go/pkg/mod
-            /home/runner/.cache/go-build
-          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
-      - name: Run Stability Tests
-        run: |
-          echo "Running ${matrix.test}..."
-          TEST_ARGS="-test.run=${matrix.test}" make stability-tests
-      - name: Upload Unit Test Results
-        if: always()
-        uses: actions/upload-artifact@v2
-        with:
-          name: unittests-results
-          path: test-results/junit/results.xml
-
   build-examples:
     runs-on: ubuntu-latest
     needs: [setup-environment]

diff --git a/.github/workflows/stability-tests.yml b/.github/workflows/stability-tests.yml
@@ -0,0 +1,95 @@
+name: stability-tests
+on:
+  push:
+    branches: [ main ]
+    tags:
+      - 'v[0-9]+.[0-9]+.[0-9]+*'
+  pull_request:
+
+jobs:
+  setup-environment:
+    runs-on: ubuntu-latest
+    outputs:
+      stabilitytest_matrix: ${{ steps.splitstabilitytest.outputs.stabilitytest_matrix}}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2.1.4
+        with:
+          go-version: 1.17
+      - name: Setup Go Environment
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+      - name: Cache Go
+        id: module-cache
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-go-modules
+        with:
+          path: |
+            /home/runner/go/pkg/mod
+            /home/runner/.cache/go-build
+          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
+      - name: Cache Tools
+        id: tool-cache
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-tool-binaries
+        with:
+          path: /home/runner/go/bin
+          key: v1-tools-${{ runner.os }}-${{ hashFiles('./internal/tools/go.mod') }}
+      - name: Install dependencies
+        if: steps.module-cache.outputs.hit != 'true'
+        run: make gomoddownload
+      - name: Install Tools
+        if: steps.tool-cache.outputs.cache-hit != 'true'
+        run: make install-tools
+      - name: Split Stability Jobs
+        id: splitstabilitytest
+        run: ./.github/workflows/scripts/setup_stability_tests.sh
+
+  stability-test:
+    # Stability tests disabled in the makefile, this job is currently running a noop make target
+    runs-on: ubuntu-latest
+    needs: [setup-environment]
+    strategy:
+      fail-fast: false
+      matrix: ${{ fromJson(needs.setup-environment.outputs.stabilitytest_matrix) }}
+    steps:
+      - name: Checkout Repo
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2.1.4
+        with:
+          go-version: 1.17
+      - name: Setup Go Environment
+        run: |
+          echo "GOPATH=$(go env GOPATH)" >> $GITHUB_ENV
+          echo "$(go env GOPATH)/bin" >> $GITHUB_PATH
+      - name: Cache Go
+        id: module-cache
+        uses: actions/cache@v2
+        env:
+          cache-name: cache-go-modules
+        with:
+          path: |
+            /home/runner/go/pkg/mod
+            /home/runner/.cache/go-build
+          key: v1-go-pkg-mod-${{ runner.os }}-${{ hashFiles('**/go.mod', '**/go.sum') }}
+      - name: Run Stability Tests
+        run: make stability-tests
+        env:
+          TEST_ARGS: "-test.run=${{ matrix.test }}"
+      - name: Create Test Result Archive
+        if: ${{ failure() || success() }}
+        continue-on-error: true
+        run: tar -cvf stabilitytest_results.tar testbed/stabilitytests/results
+      - name: Upload Test Results
+        if: ${{ success() }}
+        continue-on-error: true
+        uses: actions/upload-artifact@v2
+        with:
+          name: stability-test-results
+          path: stabilitytest_results.tar