Skip to content

Support large input collections for Multi-instance #6896

Support large input collections for Multi-instance

Support large input collections for Multi-instance #6896

Workflow file for this run

name: CI
on:
push:
branches:
- main
- stable/*
- release-*
- trying
- staging
pull_request: { }
merge_group: { }
workflow_dispatch: { }
workflow_call: { }
defaults:
run:
# use bash shell by default to ensure pipefail behavior is the default
# see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#exit-codes-and-error-action-preference
shell: bash
env:
DOCKER_PLATFORMS: "linux/amd64,linux/arm64"
jobs:
integration-tests:
name: "[IT] ${{ matrix.name }}"
timeout-minutes: 20
runs-on: [ self-hosted, linux, amd64, "16" ]
strategy:
fail-fast: false
matrix:
group: [ modules, qa-integration, qa-update ]
include:
- group: modules
name: "Module Integration Tests"
maven-modules: "'!qa/integration-tests,!qa/update-tests'"
maven-build-threads: 2
maven-test-fork-count: 7
tcc-enabled: 'false'
- group: qa-integration
name: "QA Integration Tests"
maven-modules: "qa/integration-tests"
maven-build-threads: 1
maven-test-fork-count: 10
tcc-enabled: 'true'
tcc-concurrency: 3
- group: qa-update
name: "QA Update Tests"
maven-modules: "qa/update-tests"
maven-build-threads: 1
maven-test-fork-count: 10
tcc-enabled: 'true'
tcc-concurrency: 2
env:
TC_CLOUD_LOGS_VERBOSE: true
TC_CLOUD_TOKEN: ${{ matrix.tcc-enabled == 'true' && secrets.TC_CLOUD_TOKEN || '' }}
TC_CLOUD_CONCURRENCY: ${{ matrix.tcc-concurrency }}
ZEEBE_TEST_DOCKER_IMAGE: localhost:5000/camunda/zeebe:current-test
services:
registry:
image: registry:2
ports:
- 5000:5000
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
maven-cache: 'true'
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
id: build-zeebe
with:
maven-extra-args: -T1C
- uses: ./.github/actions/build-docker
with:
repository: localhost:5000/camunda/zeebe
version: current-test
push: true
distball: ${{ steps.build-zeebe.outputs.distball }}
- name: Prepare Testcontainers Cloud agent
if: env.TC_CLOUD_TOKEN != ''
run: |
curl -L -o agent https://app.testcontainers.cloud/download/testcontainers-cloud-agent_linux_x86-64
chmod +x agent
./agent --private-registry-url=http://localhost:5000 '--private-registry-allowed-image-name-globs=*,*/*' > .testcontainers-agent.log 2>&1 &
./agent wait
- name: Create build output log file
run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
- name: Maven Test Build
run: >
./mvnw -B -T ${{ matrix.maven-build-threads }} --no-snapshot-updates
-D forkCount=${{ matrix.maven-test-fork-count }}
-D maven.javadoc.skip=true
-D skipUTs -D skipChecks
-D failsafe.rerunFailingTestsCount=3 -D flaky.test.reportDir=failsafe-reports
-P parallel-tests,extract-flaky-tests
-pl ${{ matrix.maven-modules }}
verify
| tee "${BUILD_OUTPUT_FILE_PATH}"
- name: Duplicate Test Check
uses: ./.github/actions/check-duplicate-tests
with:
buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
- name: Upload test artifacts
uses: ./.github/actions/collect-test-artifacts
if: failure()
with:
name: "[IT] ${{ matrix.name }}"
unit-tests:
name: Unit tests
runs-on: [ self-hosted, linux, amd64, "16" ]
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
go: false
maven-cache: 'true'
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
with:
go: false
maven-extra-args: -T1C
- name: Create build output log file
run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
- name: Maven Test Build
# we use the verify goal here as flaky test extraction is bound to the post-integration-test
# phase of Maven https://maven.apache.org/guides/introduction/introduction-to-the-lifecycle.html#default-lifecycle
run: >
./mvnw -T2 -B --no-snapshot-updates
-D skipITs -D skipChecks -D surefire.rerunFailingTestsCount=3
-D junitThreadCount=16
-P skip-random-tests,parallel-tests,extract-flaky-tests
verify
| tee "${BUILD_OUTPUT_FILE_PATH}"
- name: Normalize artifact name
run: echo "ARTIFACT_NAME=$(echo ${{ matrix.project }} | sed 's/\//-/g')" >> $GITHUB_ENV
- name: Duplicate Test Check
uses: ./.github/actions/check-duplicate-tests
with:
buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
- name: Upload test artifacts
uses: ./.github/actions/collect-test-artifacts
if: failure()
with:
name: "unit tests"
smoke-tests:
name: "[Smoke] ${{ matrix.os }} with ${{ matrix.arch }}"
timeout-minutes: 20
runs-on: ${{ matrix.runner }}
strategy:
fail-fast: false
matrix:
os: [ macos, windows, linux ]
arch: [ amd64 ]
include:
- os: macos
runner: macos-latest
- os: windows
runner: windows-latest
- os: linux
runner: [ self-hosted, linux, amd64 ]
- os: linux
runner: [ self-hosted, linux, amd64 ]
arch: arm64
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
go: false
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
id: build-zeebe
with:
go: false
maven-extra-args: -T1C
- uses: ./.github/actions/build-docker
id: build-docker
# Currently only Linux runners support building docker images without further ado
if: ${{ runner.os == 'Linux' }}
with:
version: current-test
distball: ${{ steps.build-zeebe.outputs.distball }}
platforms: linux/${{ matrix.arch }}
push: false
- name: Run smoke test on ${{ matrix.arch }}
env:
DOCKER_DEFAULT_PLATFORM: linux/${{ matrix.arch }}
# For non Linux runners there is no container available for testing, see build-docker job
EXCLUDED_TEST_GROUPS: ${{ runner.os != 'Linux' && 'container' }}
run: >
./mvnw -B --no-snapshot-updates
-DskipUTs -DskipChecks -Dsurefire.rerunFailingTestsCount=3
-pl qa/integration-tests
-P smoke-test,extract-flaky-tests
-D excludedGroups=$EXCLUDED_TEST_GROUPS
verify
- name: Upload test artifacts
uses: ./.github/actions/collect-test-artifacts
if: failure()
with:
name: "[Smoke] ${{ matrix.os }} with ${{ matrix.arch }}"
property-tests:
name: Property Tests
runs-on: [ self-hosted, linux, amd64, "16" ]
timeout-minutes: 30
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
go: false
maven-cache: 'true'
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
with:
go: false
maven-extra-args: -T1C
- name: Create build output log file
run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
- name: Maven Test Build
run: >
./mvnw -T1C -B --no-snapshot-updates
-P parallel-tests,include-random-tests
-D junitThreadCount=16
-D skipChecks
test
| tee "${BUILD_OUTPUT_FILE_PATH}"
- name: Duplicate Test Check
uses: ./.github/actions/check-duplicate-tests
with:
buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
- name: Upload test artifacts
uses: ./.github/actions/collect-test-artifacts
if: failure()
with:
name: Property Tests
go-client:
name: Go client tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
id: build-zeebe
# Once we're on Go 1.18, use the official gorelease to do this
- name: Check backwards compatibility
working-directory: clients/go/
run: |
go install github.com/smola/gocompat/...@v0.3.0
PREFIX=github.com/camunda/zeebe/clients/go/v8
EXCLUDE=""
for file in {internal,cmd/zbctl/internal}/*; do
EXCLUDE="$EXCLUDE --exclude-package $PREFIX/$file"
done
gocompat compare --go1compat ${EXCLUDE} ./...
- uses: ./.github/actions/build-docker
id: build-docker
with:
repository: camunda/zeebe
version: current-test
distball: ${{ steps.build-zeebe.outputs.distball }}
- name: Run Go tests
working-directory: clients/go
run: go test -mod=vendor -v ./...
java-client:
name: Java client tests
runs-on: ubuntu-latest
timeout-minutes: 20
steps:
- uses: actions/checkout@v3
# First package the complete application
- uses: ./.github/actions/setup-zeebe
with:
go: false
maven-cache: 'true'
maven-cache-key-modifier: java-client
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
with:
go: false
maven-extra-args: -am -pl clients/java -T1C
# This is a workaround for java 8, which does not support the --add-exports options
- run: rm .mvn/jvm.config
# Then run client tests with JDK 8
- uses: actions/setup-java@v3.11.0
with:
java-version: '8'
distribution: 'temurin'
- name: Create build output log file
run: echo "BUILD_OUTPUT_FILE_PATH=$(mktemp)" >> $GITHUB_ENV
- name: Maven Test Build
run: >
./mvnw -B --no-snapshot-updates
-P disableCheckstyle,extract-flaky-tests,'!autoFormat'
-D skipChecks -D skipITs
-D surefire.rerunFailingTestsCount=3
-pl clients/java
verify
| tee "${BUILD_OUTPUT_FILE_PATH}"
- name: Duplicate Test Check
uses: ./.github/actions/check-duplicate-tests
with:
buildOutputFilePath: ${{ env.BUILD_OUTPUT_FILE_PATH }}
- name: Upload test artifacts
uses: ./.github/actions/collect-test-artifacts
if: failure()
with:
name: Java 8 Client
codeql:
name: CodeQL
runs-on: [ self-hosted, linux, amd64, "16" ]
permissions:
security-events: write
steps:
- name: Checkout repository
uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
go: false
maven-cache: 'true'
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- name: Initialize CodeQL
uses: github/codeql-action/init@v2
with:
languages: java
queries: +security-and-quality
- uses: ./.github/actions/build-zeebe
with:
maven-extra-args: -T1C
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v2
with:
upload: False
output: sarif-results
- name: Remove results for generated code
uses: advanced-security/filter-sarif@main
with:
patterns: |
+**/*.java
-**/generated-sources/**/*.java
-**/generated-test-sources/**/*.java
input: sarif-results/java.sarif
output: sarif-results/java.sarif
- name: Upload CodeQL Results
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: sarif-results/java.sarif
go-lint:
name: Go linting
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
java: false
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- name: golangci-lint
uses: golangci/golangci-lint-action@v3
with:
# fixed to avoid triggering false positive; see https://github.com/golangci/golangci-lint-action/issues/535
version: v1.47.3
working-directory: clients/go
java-checks:
name: Java checks
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
- uses: ./.github/actions/setup-zeebe
with:
go: false
maven-cache: 'true'
maven-cache-key-modifier: java-checks
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- run: ./mvnw -T1C -B -D skipTests -P !autoFormat,checkFormat,spotbugs verify
docker-checks:
name: Docker checks
runs-on: ubuntu-latest
services:
# local registry is used as this job needs to push as it builds multi-platform images
registry:
image: registry:2
ports:
- 5000:5000
env:
LOCAL_DOCKER_IMAGE: localhost:5000/camunda/zeebe
steps:
- uses: actions/checkout@v3
- uses: hadolint/hadolint-action@v3.1.0
with:
config: ./.hadolint.yaml
dockerfile: ./Dockerfile
format: sarif
output-file: ./hadolint.sarif
no-color: true
verbose: true
- name: Upload Hadolint Results
if: always()
uses: github/codeql-action/upload-sarif@v2
with:
sarif_file: ./hadolint.sarif
- uses: ./.github/actions/setup-zeebe
with:
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
id: build-zeebe
- uses: ./.github/actions/build-docker
id: build-docker
with:
# we use a local registry for pushing
repository: ${{ env.LOCAL_DOCKER_IMAGE }}
distball: ${{ steps.build-zeebe.outputs.distball }}
platforms: ${{ env.DOCKER_PLATFORMS }}
# push is needed for multi-arch images as buildkit does not support loading them locally
push: true
- name: Verify Docker image
uses: ./.github/actions/verify-zeebe-docker
with:
imageName: ${{ env.LOCAL_DOCKER_IMAGE }}
date: ${{ steps.build-docker.outputs.date }}
revision: ${{ github.sha }}
version: ${{ steps.build-docker.outputs.version }}
platforms: ${{ env.DOCKER_PLATFORMS }}
test-summary:
# Used by bors to check all tests, including the unit test matrix.
# New test jobs must be added to the `needs` lists!
# This name is hard-referenced from bors.toml; remember to update that if this name changes
name: Test summary
runs-on: ubuntu-latest
needs:
- integration-tests
- unit-tests
- smoke-tests
- property-tests
- go-client
- java-client
- codeql
- java-checks
- go-lint
- docker-checks
steps:
- run: exit 0
event_file:
# We need to upload the event file as an artifact in order to support
# publishing the results of forked repositories
# https://github.com/EnricoMi/publish-unit-test-result-action#support-fork-repositories-and-dependabot-branches
name: "Event File"
runs-on: ubuntu-latest
needs:
- integration-tests
- unit-tests
- smoke-tests
- property-tests
- go-client
- java-client
if: always()
steps:
- name: Upload
uses: actions/upload-artifact@v3
with:
name: Event File
path: ${{ github.event_path }}
retention-days: 1
deploy-snapshots:
name: Deploy snapshot artifacts
needs: [ test-summary ]
runs-on: ubuntu-latest
if: github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
concurrency:
group: deploy-maven-snapshot
cancel-in-progress: false
steps:
- uses: actions/checkout@v3
- name: Import Secrets
id: secrets
uses: hashicorp/vault-action@v2.5.0
with:
url: ${{ secrets.VAULT_ADDR }}
method: approle
roleId: ${{ secrets.VAULT_ROLE_ID }}
secretId: ${{ secrets.VAULT_SECRET_ID }}
secrets: |
secret/data/products/zeebe/ci/zeebe ARTIFACTS_USR;
secret/data/products/zeebe/ci/zeebe ARTIFACTS_PSW;
- uses: actions/setup-java@v3.11.0
with:
distribution: 'temurin'
java-version: '17'
# Use CI Nexus as co-located pull-through cache for Maven artifacts via ~/.m2/settings.xml
- name: 'Create settings.xml'
uses: s4u/maven-settings-action@v2.8.0
with:
githubServer: false
servers: |
[{
"id": "camunda-nexus",
"username": "${{ steps.secrets.outputs.ARTIFACTS_USR }}",
"password": "${{ steps.secrets.outputs.ARTIFACTS_PSW }}"
}]
mirrors: '[{"url": "https://repository.nexus.camunda.cloud/content/groups/internal/", "id": "camunda-nexus", "mirrorOf": "zeebe,zeebe-snapshots", "name": "camunda Nexus"}]'
# compile and generate-sources to ensure that the Javadoc can be properly generated; compile is
# necessary when using annotation preprocessors for code generation, as otherwise the symbols are
# not resolve-able by the Javadoc generator
- run: ./mvnw -B -D skipTests -D skipChecks compile generate-sources source:jar javadoc:jar deploy
env:
MAVEN_USERNAME: ${{ steps.secrets.outputs.ARTIFACTS_USR }}
MAVEN_PASSWORD: ${{ steps.secrets.outputs.ARTIFACTS_PSW }}
deploy-docker-snapshot:
name: Deploy snapshot Docker image
needs: [ test-summary ]
runs-on: ubuntu-latest
if: github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
concurrency:
group: deploy-docker-snapshot
cancel-in-progress: false
steps:
- uses: actions/checkout@v3
- name: Import Secrets
id: secrets
uses: hashicorp/vault-action@v2.5.0
with:
url: ${{ secrets.VAULT_ADDR }}
method: approle
roleId: ${{ secrets.VAULT_ROLE_ID }}
secretId: ${{ secrets.VAULT_SECRET_ID }}
secrets: |
secret/data/products/zeebe/ci/zeebe REGISTRY_HUB_DOCKER_COM_USR;
secret/data/products/zeebe/ci/zeebe REGISTRY_HUB_DOCKER_COM_PSW;
- name: Login to DockerHub
uses: docker/login-action@v2
with:
username: ${{ steps.secrets.outputs.REGISTRY_HUB_DOCKER_COM_USR }}
password: ${{ steps.secrets.outputs.REGISTRY_HUB_DOCKER_COM_PSW }}
- uses: ./.github/actions/setup-zeebe
with:
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- uses: ./.github/actions/build-zeebe
id: build-zeebe
- uses: ./.github/actions/build-docker
id: build-docker
with:
repository: camunda/zeebe
version: SNAPSHOT
platforms: ${{ env.DOCKER_PLATFORMS }}
push: true
distball: ${{ steps.build-zeebe.outputs.distball }}
deploy-benchmark-images:
name: Deploy benchmark images
needs: [ test-summary ]
runs-on: ubuntu-latest
if: github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
concurrency:
group: deploy-benchmark-images
cancel-in-progress: false
permissions:
contents: 'read'
id-token: 'write'
steps:
- uses: actions/checkout@v3
- uses: google-github-actions/auth@v1
id: auth
with:
token_format: 'access_token'
workload_identity_provider: 'projects/628707732411/locations/global/workloadIdentityPools/zeebe-gh-actions/providers/gha-provider'
service_account: 'zeebe-gh-actions@zeebe-io.iam.gserviceaccount.com'
- name: Login to GCR
uses: docker/login-action@v2
with:
registry: gcr.io
username: oauth2accesstoken
password: ${{ steps.auth.outputs.access_token }}
- uses: ./.github/actions/setup-zeebe
with:
secret_vault_secretId: ${{ secrets.VAULT_SECRET_ID }}
secret_vault_address: ${{ secrets.VAULT_ADDR }}
secret_vault_roleId: ${{ secrets.VAULT_ROLE_ID }}
- run: ./mvnw -B -D skipTests -D skipChecks -pl benchmarks/project -am package
- name: Build Starter Image
run: ./mvnw -pl benchmarks/project jib:build -P starter
- name: Build Worker Image
run: ./mvnw -pl benchmarks/project jib:build -P worker
notify-if-failed:
name: Send slack notification on build failure
runs-on: ubuntu-latest
needs: [ test-summary, deploy-snapshots, deploy-docker-snapshot ]
if: failure() && github.repository == 'camunda/zeebe' && github.ref == 'refs/heads/main'
steps:
- id: slack-notify
name: Send slack notification
uses: slackapi/slack-github-action@v1.23.0
with:
# For posting a rich message using Block Kit
payload: |
{
"text": ":alarm: Build on `main` failed! :alarm:\n${{ github.event.head_commit.url }}",
"blocks": [
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": ":alarm: Build on `main` failed! :alarm:"
}
},
{
"type": "section",
"text": {
"type": "mrkdwn",
"text": "Please check the related commit: ${{ github.event.head_commit.url }}\n \\cc @zeebe-medic"
}
}
]
}
env:
SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
SLACK_WEBHOOK_TYPE: INCOMING_WEBHOOK
auto-merge:
# This workflow will auto merge a PR authored by dependabot[bot]. It runs only on open PRs ready for
# review.
#
# It will merge the PR only if: it is authored by dependabot[bot], is a minor or patch semantic
# update, and all CI checks are successful (ignoring the soon-to-be-removed Jenkins check).
#
# The workflow is divided into multiple sequential jobs to allow giving only minimal permissions to
# the GitHub token passed around.
#
# Once we're using the merge queue feature, I think we can simplify this workflow a lot by relying
# on dependabot merging PRs via its commands, as it will always wait for checks to be green before
# merging.
name: Auto-merge dependabot PRs
runs-on: ubuntu-latest
needs: [ test-summary ]
if: github.repository == 'camunda/zeebe' && github.actor == 'dependabot[bot]'
permissions:
checks: read
pull-requests: write
steps:
- uses: actions/checkout@v3
- id: metadata
name: Fetch dependency metadata
uses: dependabot/fetch-metadata@v1.4.0
with:
github-token: "${{ secrets.GITHUB_TOKEN }}"
- id: approve-and-merge
name: Approve and merge PR
if: steps.metadata.outputs.update-type == 'version-update:semver-patch' || steps.metadata.outputs.update-type == 'version-update:semver-minor'
run: gh pr review ${{ github.event.pull_request.number }} --approve -b "bors merge"
env:
GITHUB_TOKEN: "${{secrets.GITHUB_TOKEN}}"