Merging to release-5.0.12: [SYSE-358 release-5-lts] April template ap…

…plication (#6247) [SYSE-358 release-5-lts] April template application (#6247) Skip signing when building a snapshot to allow dependabot PRs to build. Use larger runners for build and test. --------- Co-authored-by: Gromit <policy@gromit>
TykTechnologies · Apr 29, 2024 · 1c3c7f0 · 1c3c7f0
1 parent 023bc6b
commit 1c3c7f0
Show file tree

Hide file tree

Showing 2 changed files with 75 additions and 29 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -24,7 +24,7 @@ env:
 jobs:
   goreleaser:
     name: '${{ matrix.golang_cross }}'
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m
     permissions:
       id-token: write # AWS OIDC JWT
       contents: read # actions/checkout
@@ -33,18 +33,12 @@ jobs:
       matrix:
         golang_cross:
           - 1.16
-          - 1.16-el7
         include:
-          - golang_cross: 1.16-el7
-            goreleaser: 'ci/goreleaser/goreleaser-el7.yml'
-            rpmvers: 'el/7 amazon/2'
-            debvers: 'ubuntu/xenial ubuntu/bionic debian/jessie'
-            cgo: 0
           - golang_cross: 1.16
             goreleaser: 'ci/goreleaser/goreleaser.yml'
             cgo: 1
-            rpmvers: 'el/8 el/9 amazon/2023'
-            debvers: 'ubuntu/bionic ubuntu/focal ubuntu/jammy debian/buster debian/bullseye debian/bookworm'
+            rpmvers: 'el/7 el/8 el/9 amazon/2 amazon/2023'
+            debvers: 'ubuntu/xenial ubuntu/bionic ubuntu/focal ubuntu/jammy debian/jessie debian/buster debian/bullseye debian/bookworm'
     outputs:
       tags: ${{ steps.metadata.outputs.tags }}
       commit_author: ${{ steps.fetch-author.outputs.commit_author}}
@@ -88,20 +82,20 @@ jobs:
           echo '#!/bin/sh
           ci/bin/unlock-agent.sh
           mkdir -p /go/src
-          go mod tidy
-          go mod vendor
+          GO111MODULE=on go mod tidy
+          GO111MODULE=on go mod vendor
           cp -r -f vendor/* /go/src
           mkdir -p /go/src/github.com/TykTechnologies/tyk
           cp -r ./* /go/src/github.com/TykTechnologies/tyk
           find /go/src -name vendor | xargs --no-run-if-empty -d'\n' rm -rf
           rm -rf vendor
           git config --global url."https://${{ secrets.ORG_GH_TOKEN }}@github.com".insteadOf "https://github.com"
           git config --global --add safe.directory /go/src/github.com/TykTechnologies/tyk
-          goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot' || '' }}' | tee /tmp/build.sh
+          goreleaser release --clean -f ${{ matrix.goreleaser }} ${{ !startsWith(github.ref, 'refs/tags/') && ' --snapshot --skip-sign' || '' }}' | tee /tmp/build.sh
           chmod +x /tmp/build.sh
           docker run --rm --privileged -e GITHUB_TOKEN=${{ github.token }} \
           -e GOPRIVATE=github.com/TykTechnologies                                \
-          -e GO111MODULE=on \
+          -e GO111MODULE=off                                                     \
           -e DEBVERS='${{ matrix.debvers }}'                               \
           -e RPMVERS='${{ matrix.rpmvers }}'                               \
           -e CGO_ENABLED=${{ matrix.cgo }}                                 \
@@ -178,7 +172,7 @@ jobs:
             !dist/*PAYG*.rpm
   test-controller-api:
     needs: goreleaser
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m-2
     container: tykio/gromit:v1.7
     outputs:
       conf: ${{ steps.params.outputs.api_conf }}
@@ -187,6 +181,7 @@ jobs:
       sink: ${{ steps.params.outputs.sink }}
       gd_tag: ${{ steps.params.outputs.gd_tag }}
       versions: ${{ steps.params.outputs.versions }}
+      exclude: ${{ steps.params.outputs.exclude }}
     steps:
       - name: set params
         id: params
@@ -201,7 +196,10 @@ jobs:
         run: gromit policy controller --loglevel debug | tee -a "$GITHUB_OUTPUT"
   api-tests:
     needs: [goreleaser, test-controller-api]
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-latest-m-2
+    env:
+      METADATA_REPORT_PATH: /tmp/metadata.toml
+      XUNIT_REPORT_PATH: ${{ github.workspace }}/reports/pytest-report.xml
     permissions:
       id-token: write # This is required for requesting the Github JWT
       contents: read # This is required for actions/checkout
@@ -215,6 +213,7 @@ jobs:
         include:
           - db: postgres15
             markers: "and not sql"
+        exclude: ${{ fromJson(needs.test-controller-api.outputs.exclude) }}
     steps:
       - uses: aws-actions/configure-aws-credentials@v4
         with:
@@ -249,23 +248,70 @@ jobs:
           TYK_DB_LICENSEKEY: ${{ secrets.DASH_LICENSE }}
           TYK_MDCB_LICENSE: ${{ secrets.MDCB_LICENSE }}
           ECR: ${{ steps.ecr.outputs.registry }}
-          VERSIONS: ${{ needs.test-controller-api.outputs.versions}}
-        run: "echo \"ECR=${ECR}\n$VERSIONS\ntyk_pump_image=${{matrix.pump}} \ntyk_sink_image=${{matrix.sink}} \n# base dir for config files\nconfs_dir=./pro-ha\n# pick database to use\nenv_file=local-${{ matrix.db }}.env\n\" > versions.env\nprintf '::group::versions\\n%s\\n::endgroup::' \"$(cat versions.env)\"\n# Add Tyk component config variations to $env_file\ncat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env\n# bring up env, the project name is important\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d\n./dash-bootstrap.sh http://localhost:3000\ndocker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d\n"
+          VERSIONS: ${{ needs.test-controller-api.outputs.versions }}
+        run: |
+          echo "ECR=${ECR}
+          $VERSIONS
+          tyk_pump_image=${{matrix.pump}}
+          tyk_sink_image=${{matrix.sink}}
+          confs_dir=./pro-ha
+          env_file=local-${{ matrix.db }}.env" > versions.env
+          echo "::group::versions"
+          cat versions.env
+          echo "::endgroup::"
+          # Add Tyk component config variations to $env_file
+          cat confs/${{ matrix.conf }}.env >> local-${{ matrix.db }}.env
+          # bring up env, the project name is important
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile master-datacenter up --quiet-pull -d
+          ./dash-bootstrap.sh http://localhost:3000
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile slave-datacenter up --quiet-pull -d
       - name: Run tests
         working-directory: auto
         id: test_execution
         run: |
+          # Generate report id
+          echo "id=$(date +%s%N)" >> $GITHUB_OUTPUT
+          # Run tests
           set -o pipefail
           echo "### API tests ${{ matrix.db }} ${{ matrix.conf }}" >> $GITHUB_STEP_SUMMARY
           if docker run --rm --network auto_default --env-file pytest.env -v ${{ github.workspace }}/reports:/app/reports \
-             ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ needs.test-controller-api.outputs.gd_tag }} \
-             pytest -c pytest_ci.ini --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then
-               echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
+            ${{ steps.ecr.outputs.registry }}/tyk-automated-tests:${{ needs.test-controller-api.outputs.gd_tag }} \
+            pytest -c pytest_ci.ini --junitxml=./${XUNIT_REPORT_PATH#"${{ github.workspace }}"} --ci -m "not local and not dind ${{ matrix.markers }}" | tee tests.out; then
+              echo "All tests passed!" >> $GITHUB_STEP_SUMMARY
           else
-               echo "::error title=API tests ${{ matrix.db }} ${{ matrix.conf }}::Test execution failed"
-               cat tests.out >> $GITHUB_STEP_SUMMARY
-               exit 1
+            echo "::error title=API tests ${{ matrix.db }} ${{ matrix.conf }}::Test execution failed"
+            cat tests.out >> $GITHUB_STEP_SUMMARY
+            exit 1
+          fi
+      - name: Generate metadata
+        if: always() && steps.test_execution.outcome != 'skipped'
+        id: metadata_report
+        env:
+          REPORT_NAME: ${{ github.repository }}_${{ github.run_id }}_${{ github.run_attempt }}-${{steps.test_execution.outputs.id}}
+        run: |
+          # Generate metadata report
+          echo "[metadata]
+          repo = ${{ github.repository }}
+          branch = ${{ github.ref }}
+          commit = ${{ github.sha }}
+          test_suite_version = ${{ needs.test-controller-api.outputs.gd_tag }}
+          test_suite_name = ${{ github.job }}
+          test_suite_run = ${{ github.run_id }}-${{ github.run_attempt }}
+          db = ${{ matrix.db }}
+          conf = ${{ matrix.conf }}
+          pump_compatibility = ${{ matrix.pump }}
+          sink_compatibility = ${{ matrix.sink }}
+          " > ${METADATA_REPORT_PATH}
+          # Print metadata report
+          if [[ "${{ runner.debug }}" == "1" ]]; then
+            echo "::group::metadata report"
+            cat ${METADATA_REPORT_PATH}
+            echo "::endgroup::"
           fi
+          #Upload xunit report
+          aws s3 cp ${XUNIT_REPORT_PATH}  s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.xml
+          #Upload metadata report
+          aws s3 cp ${METADATA_REPORT_PATH} s3://assets.dev.tyk.technology/testreports/${REPORT_NAME#*/}.metadata.toml
       - name: Docker logs for all components
         if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
         working-directory: auto
@@ -277,16 +323,16 @@ jobs:
           ECR: ${{ steps.ecr.outputs.registry }}
           VERSIONS: ${{ needs.test-controller-api.outputs.versions }}
         run: |
-          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose-api.log
+          docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${{ matrix.db }}.yml --env-file versions.env --profile all logs | sort > ${{ github.workspace }}/docker-compose.log
           echo "::group::DockerLogs"
-          cat ${{ github.workspace }}/docker-compose-api.log
+          cat ${{ github.workspace }}/docker-compose.log
           echo "::endgroup::"
       - name: Upload Artifact
         uses: actions/upload-artifact@v4
         if: failure() && (steps.test_execution.outcome != 'success' || steps.env_up.outcome != 'success')
         with:
-          name: docker-compose-logs-api-${{ matrix.db }}-${{ matrix.conf }}-${{ github.run_id }}
-          path: ${{ github.workspace }}/docker-compose-api.log
+          name: docker-compose-logs-${{ github.job }}-${{ matrix.db }}-${{ matrix.conf }}-${{ github.run_id }}
+          path: ${{ github.workspace }}/docker-compose.log
           retention-days: 3
           overwrite: true
       - name: Archive Integration tests report
@@ -387,7 +433,7 @@ jobs:
           - registry.access.redhat.com/ubi8/ubi
           - registry.access.redhat.com/ubi9/ubi
           - amazonlinux:2
-          - registry.access.redhat.com/ubi7/ubi
+          - registry.access.redhat.com/ubi7/ubi          
     steps:
       - uses: actions/checkout@v4
         with:

diff --git a/ci/Dockerfile.std b/ci/Dockerfile.std
@@ -18,7 +18,7 @@ RUN rm -fv /usr/bin/passwd /usr/sbin/adduser || true
 RUN rm -rf /root/.cache \
     && apt-get -y autoremove \
     && apt-get clean \
-    && rm -rf /usr/include/* \
+    && rm -rf /usr/include/* /var/cache/apt/archives /var/lib/{apt,dpkg,cache,log} \
     && find /usr/lib -type f -name '*.a' -o -name '*.o' -delete
 
 # Comment this to test in dev