diff --git a/.github/workflows/run-matrix.yml b/.github/workflows/run-matrix.yml new file mode 100644 index 000000000..02b7aae1e --- /dev/null +++ b/.github/workflows/run-matrix.yml @@ -0,0 +1,37 @@ +name: run-matrix + +on: + workflow_call: + inputs: + include: + required: true + description: Matrix include JSON string + type: string + +jobs: + docker: + name: "docker (version: ${{ matrix.version }}, framework: ${{ matrix.framework }})" + runs-on: ubuntu-latest + strategy: + fail-fast: false + max-parallel: 5 + matrix: + include: ${{ fromJSON(inputs.include) }} + steps: + - uses: actions/checkout@v3 + - name: Run tests + run: ./tests/scripts/docker/run_tests.sh ${{ matrix.version }} ${{ matrix.framework }} + env: + LOCALSTACK_VOLUME_DIR: localstack_data + - if: success() || failure() + name: Upload JUnit Test Results + uses: actions/upload-artifact@v3 + with: + name: test-results + path: "**/*-python-agent-junit.xml" + - if: success() || failure() + name: Upload Coverage Reports + uses: actions/upload-artifact@v3 + with: + name: coverage-reports + path: "**/.coverage*" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9a91305c7..2738f9269 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -20,6 +20,8 @@ jobs: runs-on: ubuntu-latest outputs: matrix: ${{ steps.generate.outputs.matrix }} + data: ${{ steps.split.outputs.data }} + chunks: ${{ steps.split.outputs.chunks }} steps: - uses: actions/checkout@v3 - id: generate @@ -30,34 +32,53 @@ jobs: # Use .ci/.matrix_framework_full.yml if it's a scheduled workflow, otherwise use .ci/.matrix_framework.yml frameworksFile: .ci/.matrix_framework${{ github.event_name == 'schedule' && '_full' || '' }}.yml excludedFile: .ci/.matrix_exclude.yml + - name: Split matrix + shell: python + id: split + run: | + import os + import json + + def split(lst, n): + return [lst[i::n] for i in range(n)] + + matrix = json.loads(os.environ['GENERATED_MATRIX']) + + # Using the number 4 because the full matrix has roughly 400+ items + # Hence, it is split into chunks of size ~100 + # We are doing this because the matrix in GH actions has a max limit of 256 + chunks = split(matrix['include'], 4) + chunks_json = json.dumps(chunks) + + with open(os.environ['GITHUB_OUTPUT'], 'a') as f: + print(f'chunks={chunks_json}', file=f) - docker: - name: "docker (version: ${{ matrix.version }}, framework: ${{ matrix.framework }})" - needs: create-matrix - runs-on: ubuntu-latest - strategy: - fail-fast: false - max-parallel: 10 - matrix: ${{ fromJson(needs.create-matrix.outputs.matrix) }} - steps: - - uses: actions/checkout@v3 - - name: Run tests - run: ./tests/scripts/docker/run_tests.sh ${{ matrix.version }} ${{ matrix.framework }} env: - LOCALSTACK_VOLUME_DIR: localstack_data - - if: success() || failure() - name: Upload JUnit Test Results - uses: actions/upload-artifact@v3 - with: - name: test-results - path: "**/*-python-agent-junit.xml" - - if: success() || failure() - name: Upload Coverage Reports - uses: actions/upload-artifact@v3 - with: - name: coverage-reports - path: "**/.coverage*" + GENERATED_MATRIX: ${{ steps.generate.outputs.matrix }} + chunks-0: + needs: create-matrix + uses: ./.github/workflows/run-matrix.yml + with: + include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[0]) }} + + chunks-1: + needs: create-matrix + uses: ./.github/workflows/run-matrix.yml + with: + include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[1]) }} + + chunks-2: + needs: create-matrix + uses: ./.github/workflows/run-matrix.yml + with: + include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[2]) }} + + chunks-3: + needs: create-matrix + uses: ./.github/workflows/run-matrix.yml + with: + include: ${{ toJSON(fromJSON(needs.create-matrix.outputs.chunks)[3]) }} windows: name: "windows (version: ${{ matrix.version }}, framework: ${{ matrix.framework }}, asyncio: ${{ matrix.asyncio }})"