From 4d0a11c23a2785478579be78773dd273791f0793 Mon Sep 17 00:00:00 2001 From: Thomas Gummerer Date: Wed, 24 Jan 2024 17:26:08 +0100 Subject: [PATCH] reduce the number of partitions for macos runners I was watching the merge queue quite closely today, and it's continually stuck waiting to run jobs on MacOS runners. However when the jobs are running on them they tend to be faster than e.g. Windows runners, often running the tests assigned to them in 2 min, vs. 8 min on windows. In addition we're spending quite a bit of time to setup the actual running of tests, and by spreading the tests out over multiple runners that time multiplies. Given the macos runners are the most expensive ones that probably also adds up. Given all that, we can reduce the number of MacOS runners we're using, while not trading that off against longer overall CI times, as those are limited by slower jobs. Unfortunately this results in some duplication in the ci job, but I think the tradeoff here might be worth it. --- .github/workflows/ci.yml | 95 ++++++++++++++++++++++++++++++---------- 1 file changed, 72 insertions(+), 23 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3b74198f4df3..d782b5ca9333 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -180,20 +180,39 @@ jobs: --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8 ) - ACCEPTANCE_TEST_MATRIX=$( - ./scripts/get-job-matrix.py \ - -vvv \ - generate-matrix \ - --kind acceptance-test \ - "$CODEGEN_TESTS_FLAG" \ - --tags all xplatform_acceptance \ - --platform "${ACCEPTANCE_PLATFORMS[@]}" \ - --version-set current \ - --partition-module pkg 1 \ - --partition-module sdk 1 \ - --partition-module tests 1 \ - --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8 - ) + if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]${windows-latest}[[:space:]] ]]; then + ACCEPTANCE_TEST_MATRIX_WIN=$( + ./scripts/get-job-matrix.py \ + -vvv \ + generate-matrix \ + --kind acceptance-test \ + "$CODEGEN_TESTS_FLAG" \ + --tags all xplatform_acceptance \ + --platform windows-latest" \ + --version-set current \ + --partition-module pkg 1 \ + --partition-module sdk 1 \ + --partition-module tests 1 \ + --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8 + ) + fi + + if [[ " ${ACCEPTANCE_PLATFORMS[*]} " =~ [[:space:]]${macos-latest}[[:space:]] ]]; then + ACCEPTANCE_TEST_MATRIX_MACOS=$( + ./scripts/get-job-matrix.py \ + -vvv \ + generate-matrix \ + --kind acceptance-test \ + "$CODEGEN_TESTS_FLAG" \ + --tags all xplatform_acceptance \ + --platform macos-latest" \ + --version-set current \ + --partition-module pkg 1 \ + --partition-module sdk 1 \ + --partition-module tests 1 \ + --partition-package github.com/pulumi/pulumi/tests/integration tests/integration 8 + ) + fi echo "::endgroup::" echo "::group::Version set variable" @@ -209,8 +228,11 @@ jobs: echo "::group::Integration test matrix" echo "$INTEGRATION_TEST_MATRIX" | yq -P '.' echo "::endgroup::" - echo "::group::acceptance test matrix" - echo "$ACCEPTANCE_TEST_MATRIX" | yq -P '.' + echo "::group::acceptance test matrix windows" + echo "$ACCEPTANCE_TEST_MATRIX_WIN" | yq -P '.' + echo "::endgroup::" + echo "::group::acceptance test matrix macos" + echo "$ACCEPTANCE_TEST_MATRIX_MACOS" | yq -P '.' echo "::endgroup::" echo "::group::Version set" echo "$VERSION_SET" | yq -P '.' @@ -219,14 +241,16 @@ jobs: echo "::group::Set outputs" ./.github/scripts/set-output unit-test-matrix "${UNIT_TEST_MATRIX}" ./.github/scripts/set-output integration-test-matrix "${INTEGRATION_TEST_MATRIX}" - ./.github/scripts/set-output acceptance-test-matrix "${ACCEPTANCE_TEST_MATRIX}" + ./.github/scripts/set-output acceptance-test-matrix-win "${ACCEPTANCE_TEST_MATRIX_WIN}" + ./.github/scripts/set-output acceptance-test-matrix-macos "${ACCEPTANCE_TEST_MATRIX_MACOS}}" ./.github/scripts/set-output version-set "${VERSION_SET}" ./.github/scripts/set-output build-targets "${BUILD_TARGETS}" echo "::endgroup::" outputs: unit-test-matrix: "${{ fromJson(steps.matrix.outputs.unit-test-matrix) }}" integration-test-matrix: "${{ fromJson(steps.matrix.outputs.integration-test-matrix) }}" - acceptance-test-matrix: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix) }}" + acceptance-test-matrix-win: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-win) }}" + acceptance-test-matrix-macos: "${{ fromJson(steps.matrix.outputs.acceptance-test-matrix-macos) }}" version-set: "${{ fromJson(steps.matrix.outputs.version-set) }}" build-targets: "${{ fromJson(steps.matrix.outputs.build-targets) }}" @@ -323,17 +347,42 @@ jobs: version-set: ${{ toJson(matrix.version-set) }} secrets: inherit - # Tests that depend on builds, but a smaller subset against Windows & MacOS platforms. - acceptance-test: + # Tests that depend on builds, but a smaller subset against Windows platform. + acceptance-test-win: + # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from + # appearing in the rendered title of the job name. See: unit test. + name: Acceptance Test${{ matrix.platform && '' }} + needs: [matrix, build-binaries, build-sdks] + if: ${{ needs.matrix.outputs.acceptance-test-matrix-win != '{}' }} + # alow jobs to fail if the platform contains windows + strategy: + matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-win) }} + uses: ./.github/workflows/ci-run-test.yml + with: + ref: ${{ inputs.ref }} + version: ${{ inputs.version }} + platform: ${{ matrix.platform }} + + test-name: ${{ matrix.test-suite.name || matrix.test-suite.command }} on ${{ matrix.platform }}/${{ matrix.version-set.name }} + test-command: ${{ matrix.test-suite.command }} + is-integration-test: true # TODO: set to true here + enable-coverage: ${{ inputs.enable-coverage }} + # require-build: false # TODO, remove ${{ matrix.require-build || false }} + + version-set: ${{ toJson(matrix.version-set) }} + secrets: inherit + + # Tests that depend on builds, but a smaller subset against MacOS platform. + acceptance-test-win: # By putting a variable in the name, we remove GitHub's auto-generated matrix parameters from # appearing in the rendered title of the job name. See: unit test. name: Acceptance Test${{ matrix.platform && '' }} needs: [matrix, build-binaries, build-sdks] - if: ${{ needs.matrix.outputs.acceptance-test-matrix != '{}' }} + if: ${{ needs.matrix.outputs.acceptance-test-matrix-macos != '{}' }} # alow jobs to fail if the platform contains windows strategy: - fail-fast: ${{ contains(needs.matrix.outputs.acceptance-test-matrix, 'macos') }} - matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix) }} + fail-fast: true + matrix: ${{ fromJson(needs.matrix.outputs.acceptance-test-matrix-macos) }} uses: ./.github/workflows/ci-run-test.yml with: ref: ${{ inputs.ref }}