ref: bump sentry-kafka-schemas to 0.1.60 #15530
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: ci | |
| on: | |
| push: | |
| branches: | |
| - master | |
| pull_request: | |
| jobs: | |
| files-changed: | |
| name: detect what files changed | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| # Map a step output to a job output | |
| outputs: | |
| api_changes: ${{ steps.changes.outputs.api_changes }} | |
| steps: | |
| - uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2 | |
| - name: Check for backend file changes | |
| uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2 | |
| id: changes | |
| with: | |
| token: ${{ github.token }} | |
| filters: .github/file-filters.yml | |
| linting: | |
| name: "pre-commit hooks" # (includes Python formatting + linting) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - name: Internal github app token | |
| id: token | |
| uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0 | |
| continue-on-error: true | |
| with: | |
| app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }} | |
| private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }} | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/pre-commit | |
| key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }} | |
| - name: Setup pre-commit | |
| run: make setup-git | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - uses: getsentry/paths-filter@v2 | |
| id: files | |
| with: | |
| # Enable listing of files matching each filter. | |
| # Paths to files will be available in `${FILTER_NAME}_files` output variable. | |
| # Paths will be escaped and space-delimited. | |
| # Output is usable as command line argument list in linux shell | |
| list-files: shell | |
| # It doesn't make sense to lint deleted files. | |
| # Therefore we specify we are only interested in added or modified files. | |
| filters: | | |
| all: | |
| - added|modified: '**/*' | |
| - name: Run pre-commit checks | |
| # Run pre-commit to lint and format check files that were changed (but not deleted) compared to master. | |
| # XXX: there is a very small chance that it'll expand to exceed Linux's limits | |
| # `getconf ARG_MAX` - max # bytes of args + environ for exec() | |
| # we skip the `no-commit-to-branch` because in CI we are in fact on master already | |
| # and we have merged to it | |
| run: | | |
| SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }} | |
| # If working tree is dirty, commit and update if we have a token | |
| - name: Apply any pre-commit fixed files | |
| if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always() | |
| uses: getsentry/action-github-commit@v1.0.0 | |
| with: | |
| github-token: ${{ steps.token.outputs.token }} | |
| rust-linting: | |
| name: "Linting - Rust" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 10 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - name: Run linter | |
| run: | | |
| make lint-rust format-rust-ci | |
| typing: | |
| name: "mypy typing" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 5 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - name: Run mypy | |
| run: | | |
| make backend-typing | |
| config-validation: | |
| name: "Dataset Config Validation" | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 2 | |
| steps: | |
| - uses: actions/checkout@v3 | |
| name: Checkout code | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: '3.11' | |
| - name: Install dependencies | |
| run: | | |
| python -m pip install --upgrade pip | |
| make install-python-dependencies | |
| - name: Validate configs | |
| run: | | |
| make validate-configs | |
| snuba-image: | |
| name: Build snuba CI image | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 20 | |
| outputs: | |
| branch: ${{ steps.branch.outputs.branch }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v3 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Get branch name | |
| id: branch | |
| # strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that | |
| # it can be used as a docker tag | |
| run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT" | |
| # We are only using ghcr here for CI as `setup-gcloud` is a bit slow | |
| # Should revisit this when we move off of google cloud build (we may want to move these to GCR) | |
| - name: Registry login | |
| run: | | |
| echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin | |
| - name: Build snuba docker image for CI | |
| uses: docker/build-push-action@v4 | |
| if: github.repository_owner == 'getsentry' | |
| with: | |
| context: . | |
| # push: true | |
| load: true | |
| target: testing | |
| tags: | | |
| ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| ghcr.io/getsentry/snuba-ci:latest | |
| cache-from: | | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:latest | |
| # cache-to is re-implemented using the next step, to run conditionally for contributors | |
| - name: Publish images for cache | |
| if: steps.branch.outputs.branch == 'master' || github.event.pull_request.head.repo.full_name == github.repository | |
| # outside contributors won't be able to push to the docker registry | |
| # ignore the failures in this step | |
| continue-on-error: ${{ github.event_name == 'pull_request' }} | |
| run: | | |
| # Useful to speed up CI | |
| docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }} | |
| docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then | |
| # The latest tag should only be published on `master` | |
| docker push ghcr.io/getsentry/snuba-ci:latest | |
| fi | |
| tests: | |
| needs: [linting, snuba-image] | |
| name: Tests and code coverage | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| matrix: | |
| snuba_settings: | |
| [ | |
| "test", | |
| "test_rust", | |
| "test_distributed", | |
| "test_distributed_migrations", | |
| ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| name: Checkout code | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build snuba docker image for CI | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| push: false | |
| tags: snuba-test | |
| load: true | |
| target: testing | |
| cache-from: | | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:latest | |
| - name: Docker set up | |
| run: | | |
| docker network create --attachable cloudbuild | |
| - name: Docker Snuba Rust tests | |
| run: | | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test-rust | |
| if: ${{ matrix.snuba_settings == 'test_rust' }} | |
| - name: Docker Snuba tests | |
| run: | | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker-compose -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings == 'test_distributed' }} | |
| - name: Docker Snuba Multi-Node Tests | |
| run: | | |
| SNUBA_SETTINGS=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml up -d | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }} | |
| - name: Docker Snuba Init Tests | |
| run: | | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test | |
| if: ${{ matrix.snuba_settings == 'test' }} | |
| - name: Upload test results to Codecov | |
| if: ${{ !cancelled() }} | |
| uses: codecov/test-results-action@v1 | |
| with: | |
| token: ${{ secrets.CODECOV_TOKEN }} | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| admin-tests: | |
| needs: [linting] | |
| name: Front end tests for snuba admin | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v2 | |
| name: Checkout code | |
| - uses: volta-cli/action@v4 | |
| - name: Set up and run tests through yarn | |
| run: cd snuba/admin && yarn install && yarn run test --coverage | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| sentry: | |
| needs: [snuba-image, files-changed] | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 60 | |
| strategy: | |
| matrix: | |
| instance: [0, 1, 2, 3] | |
| env: | |
| MIGRATIONS_TEST_MIGRATE: 1 | |
| # XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`. | |
| MATRIX_INSTANCE_TOTAL: 4 | |
| TEST_GROUP_STRATEGY: ROUND_ROBIN | |
| steps: | |
| # Checkout codebase | |
| - name: Checkout snuba | |
| uses: actions/checkout@v2 | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build snuba docker image for CI | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| push: false | |
| tags: snuba-test | |
| load: true | |
| target: testing | |
| cache-from: | | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:latest | |
| # Checkout Sentry and run integration tests against latest snuba | |
| # Make sure this is after `docker build`, otherwise we'll break docker cache | |
| - name: Checkout sentry | |
| uses: actions/checkout@v2 | |
| with: | |
| repository: getsentry/sentry | |
| path: sentry | |
| - name: Setup steps | |
| id: setup | |
| run: | | |
| pip install --upgrade pip wheel | |
| # We cannot execute actions that are not placed under .github of the main repo | |
| mkdir -p .github/actions | |
| cp -r sentry/.github/actions/* .github/actions | |
| - name: Setup Sentry | |
| id: setup-sentry | |
| uses: ./.github/actions/setup-sentry | |
| with: | |
| workdir: sentry | |
| snuba: false | |
| kafka: true | |
| clickhouse: true | |
| - name: Start snuba | |
| run: | | |
| docker run -d --rm \ | |
| -p 127.0.0.1:1218:1218 \ | |
| -e PYTHONUNBUFFERED=1 \ | |
| -e SNUBA_SETTINGS=docker \ | |
| -e DEBUG=1 \ | |
| -e DEFAULT_BROKERS=sentry_kafka:9092 \ | |
| -e CLICKHOUSE_HOST=sentry_clickhouse \ | |
| -e CLICKHOUSE_PORT=9000 \ | |
| -e CLICKHOUSE_HTTP_PORT=8123 \ | |
| -e REDIS_HOST=sentry_redis \ | |
| -e REDIS_PORT=6379 \ | |
| -e REDIS_DB=1 \ | |
| --name sentry_snuba \ | |
| --network sentry \ | |
| snuba-test | |
| docker exec sentry_snuba snuba migrations migrate --force | |
| - name: Run snuba tests | |
| if: needs.files-changed.outputs.api_changes == 'false' | |
| working-directory: sentry | |
| run: | | |
| pytest -k 'not __In' tests \ | |
| -m snuba_ci \ | |
| -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" | |
| - name: Run full tests | |
| # TODO: Adjust which changed files we care about | |
| if: needs.files-changed.outputs.api_changes == 'true' | |
| working-directory: sentry | |
| run: | | |
| pytest -k 'not __In' tests/snuba \ | |
| tests/sentry/eventstream/kafka \ | |
| tests/sentry/post_process_forwarder \ | |
| tests/sentry/snuba \ | |
| tests/sentry/search/events \ | |
| tests/sentry/event_manager \ | |
| tests/sentry/api/endpoints/test_organization_ddm_meta.py \ | |
| tests/sentry/api/endpoints/test_organization_profiling_functions.py \ | |
| tests/snuba/test_snql_snuba.py \ | |
| tests/snuba/test_metrics_layer.py \ | |
| -vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml" | |
| - name: Run CI module tests | |
| if: needs.files-changed.outputs.api_changes == 'true' | |
| working-directory: sentry | |
| run: pytest -k 'not __In' tests -vv -m snuba_ci | |
| clickhouse-versions: | |
| needs: [linting, snuba-image] | |
| name: Tests on multiple clickhouse versions | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 30 | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| version: | |
| [ | |
| "21.8.13.1.altinitystable", | |
| "22.8.15.25.altinitystable", | |
| "23.3.13.7.altinitystable", | |
| "23.8.8.21.altinitystable", | |
| ] | |
| steps: | |
| - uses: actions/checkout@v2 | |
| name: Checkout code | |
| - name: Set up Docker Buildx | |
| uses: docker/setup-buildx-action@v2 | |
| - name: Build snuba docker image for CI | |
| uses: docker/build-push-action@v4 | |
| with: | |
| context: . | |
| push: false | |
| tags: snuba-test | |
| load: true | |
| target: testing | |
| cache-from: | | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }} | |
| type=registry,ref=ghcr.io/getsentry/snuba-ci:latest | |
| - name: Docker set up | |
| run: | | |
| docker network create --attachable cloudbuild | |
| - name: Docker Snuba Test other ClickHouse versions | |
| run: | | |
| export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}} | |
| SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test | |
| - name: Upload to codecov | |
| run: | | |
| curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN} | |
| publish-to-dockerhub: | |
| name: Publish Snuba to DockerHub | |
| runs-on: ubuntu-20.04 | |
| if: ${{ (github.ref_name == 'master') }} | |
| steps: | |
| - uses: actions/checkout@v3 # v3.1.0 | |
| - name: Pull the test image | |
| id: image_pull | |
| env: | |
| IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }} | |
| shell: bash | |
| run: | | |
| echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out." | |
| echo "Polling for $IMAGE_URL" | |
| timeout 20m bash -c 'until docker pull "$IMAGE_URL" 2>/dev/null; do sleep 10; done' | |
| - name: Get short SHA for docker tag | |
| id: short_sha | |
| shell: bash | |
| run: | | |
| SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA") | |
| if [[ -z "$SHORT_SHA" ]]; then | |
| echo "Short SHA empty? Re-running rev-parse." | |
| git rev-parse --short "$GITHUB_SHA" | |
| else | |
| echo "sha=$SHORT_SHA" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Push built docker image | |
| shell: bash | |
| env: | |
| SHORT_SHA: ${{ steps.short_sha.outputs.sha }} | |
| IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }} | |
| run: | | |
| # only login if the password is set | |
| if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" | docker login --username=sentrybuilder --password-stdin; fi | |
| # We push 3 tags to Dockerhub: | |
| # first, the full sha of the commit | |
| docker tag ${IMAGE_URL} getsentry/snuba:${GITHUB_SHA} | |
| docker push getsentry/snuba:${GITHUB_SHA} | |
| # second, the short sha of the commit | |
| docker tag ${IMAGE_URL} getsentry/snuba:${SHORT_SHA} | |
| docker push getsentry/snuba:${SHORT_SHA} | |
| # finally, nightly | |
| docker tag ${IMAGE_URL} getsentry/snuba:nightly | |
| docker push getsentry/snuba:nightly |