Skip to content

ref: bump sentry-kafka-schemas to 0.1.60 #15530

ref: bump sentry-kafka-schemas to 0.1.60

ref: bump sentry-kafka-schemas to 0.1.60 #15530

Workflow file for this run

name: ci
on:
push:
branches:
- master
pull_request:
jobs:
files-changed:
name: detect what files changed
runs-on: ubuntu-latest
timeout-minutes: 3
# Map a step output to a job output
outputs:
api_changes: ${{ steps.changes.outputs.api_changes }}
steps:
- uses: actions/checkout@7884fcad6b5d53d10323aee724dc68d8b9096a2e # v2
- name: Check for backend file changes
uses: getsentry/paths-filter@66f7f1844185eb7fb6738ea4ea59d74bb99199e5 # v2
id: changes
with:
token: ${{ github.token }}
filters: .github/file-filters.yml
linting:
name: "pre-commit hooks" # (includes Python formatting + linting)
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v3
name: Checkout code
- name: Internal github app token
id: token
uses: getsentry/action-github-app-token@97c9e23528286821f97fba885c1b1123284b29cc # v2.0.0
continue-on-error: true
with:
app_id: ${{ vars.SENTRY_INTERNAL_APP_ID }}
private_key: ${{ secrets.SENTRY_INTERNAL_APP_PRIVATE_KEY }}
- uses: actions/setup-python@v4
with:
python-version: '3.11'
- uses: actions/cache@v3
with:
path: ~/.cache/pre-commit
key: cache-epoch-1|${{ env.pythonLocation }}|${{ hashFiles('.pre-commit-config.yaml') }}
- name: Setup pre-commit
run: make setup-git
- name: Install dependencies
run: |
python -m pip install --upgrade pip
make install-python-dependencies
- uses: getsentry/paths-filter@v2
id: files
with:
# Enable listing of files matching each filter.
# Paths to files will be available in `${FILTER_NAME}_files` output variable.
# Paths will be escaped and space-delimited.
# Output is usable as command line argument list in linux shell
list-files: shell
# It doesn't make sense to lint deleted files.
# Therefore we specify we are only interested in added or modified files.
filters: |
all:
- added|modified: '**/*'
- name: Run pre-commit checks
# Run pre-commit to lint and format check files that were changed (but not deleted) compared to master.
# XXX: there is a very small chance that it'll expand to exceed Linux's limits
# `getconf ARG_MAX` - max # bytes of args + environ for exec()
# we skip the `no-commit-to-branch` because in CI we are in fact on master already
# and we have merged to it
run: |
SKIP=no-commit-to-branch pre-commit run --files ${{ steps.files.outputs.all_files }}
# If working tree is dirty, commit and update if we have a token
- name: Apply any pre-commit fixed files
if: steps.token.outcome == 'success' && github.ref != 'refs/heads/master' && always()
uses: getsentry/action-github-commit@v1.0.0
with:
github-token: ${{ steps.token.outputs.token }}
rust-linting:
name: "Linting - Rust"
runs-on: ubuntu-latest
timeout-minutes: 10
steps:
- uses: actions/checkout@v3
name: Checkout code
- name: Run linter
run: |
make lint-rust format-rust-ci
typing:
name: "mypy typing"
runs-on: ubuntu-latest
timeout-minutes: 5
steps:
- uses: actions/checkout@v3
name: Checkout code
- uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
make install-python-dependencies
- name: Run mypy
run: |
make backend-typing
config-validation:
name: "Dataset Config Validation"
runs-on: ubuntu-latest
timeout-minutes: 2
steps:
- uses: actions/checkout@v3
name: Checkout code
- uses: actions/setup-python@v4
with:
python-version: '3.11'
- name: Install dependencies
run: |
python -m pip install --upgrade pip
make install-python-dependencies
- name: Validate configs
run: |
make validate-configs
snuba-image:
name: Build snuba CI image
runs-on: ubuntu-latest
timeout-minutes: 20
outputs:
branch: ${{ steps.branch.outputs.branch }}
steps:
- name: Checkout code
uses: actions/checkout@v3
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Get branch name
id: branch
# strip `refs/heads/` from $GITHUB_REF and replace `/` with `-` so that
# it can be used as a docker tag
run: echo "branch=$(echo ${GITHUB_REF#refs/heads/} | tr / -)" >> "$GITHUB_OUTPUT"
# We are only using ghcr here for CI as `setup-gcloud` is a bit slow
# Should revisit this when we move off of google cloud build (we may want to move these to GCR)
- name: Registry login
run: |
echo ${{ secrets.GITHUB_TOKEN }} | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
- name: Build snuba docker image for CI
uses: docker/build-push-action@v4
if: github.repository_owner == 'getsentry'
with:
context: .
# push: true
load: true
target: testing
tags: |
ghcr.io/getsentry/snuba-ci:${{ github.sha }}
ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
ghcr.io/getsentry/snuba-ci:latest
cache-from: |
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
# cache-to is re-implemented using the next step, to run conditionally for contributors
- name: Publish images for cache
if: steps.branch.outputs.branch == 'master' || github.event.pull_request.head.repo.full_name == github.repository
# outside contributors won't be able to push to the docker registry
# ignore the failures in this step
continue-on-error: ${{ github.event_name == 'pull_request' }}
run: |
# Useful to speed up CI
docker push ghcr.io/getsentry/snuba-ci:${{ steps.branch.outputs.branch }}
docker push ghcr.io/getsentry/snuba-ci:${{ github.sha }}
if [ "${{ steps.branch.outputs.branch }}" == 'master' ]; then
# The latest tag should only be published on `master`
docker push ghcr.io/getsentry/snuba-ci:latest
fi
tests:
needs: [linting, snuba-image]
name: Tests and code coverage
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
snuba_settings:
[
"test",
"test_rust",
"test_distributed",
"test_distributed_migrations",
]
steps:
- uses: actions/checkout@v2
name: Checkout code
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build snuba docker image for CI
uses: docker/build-push-action@v4
with:
context: .
push: false
tags: snuba-test
load: true
target: testing
cache-from: |
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
- name: Docker set up
run: |
docker network create --attachable cloudbuild
- name: Docker Snuba Rust tests
run: |
SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test-rust
if: ${{ matrix.snuba_settings == 'test_rust' }}
- name: Docker Snuba tests
run: |
SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=${{ matrix.snuba_settings }} docker-compose -f docker-compose.gcb.yml run --rm snuba-test
if: ${{ matrix.snuba_settings == 'test' || matrix.snuba_settings == 'test_distributed' }}
- name: Docker Snuba Multi-Node Tests
run: |
SNUBA_SETTINGS=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml up -d
SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_distributed_migrations TEST_LOCATION=test_distributed_migrations docker-compose --profile multi_node -f docker-compose.gcb.yml run --rm snuba-test
if: ${{ matrix.snuba_settings == 'test_distributed_migrations' }}
- name: Docker Snuba Init Tests
run: |
SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test_initialization TEST_LOCATION=test_initialization docker-compose -f docker-compose.gcb.yml run --rm snuba-test
if: ${{ matrix.snuba_settings == 'test' }}
- name: Upload test results to Codecov
if: ${{ !cancelled() }}
uses: codecov/test-results-action@v1
with:
token: ${{ secrets.CODECOV_TOKEN }}
- name: Upload to codecov
run: |
curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}
admin-tests:
needs: [linting]
name: Front end tests for snuba admin
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v2
name: Checkout code
- uses: volta-cli/action@v4
- name: Set up and run tests through yarn
run: cd snuba/admin && yarn install && yarn run test --coverage
- name: Upload to codecov
run: |
curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}
sentry:
needs: [snuba-image, files-changed]
runs-on: ubuntu-latest
timeout-minutes: 60
strategy:
matrix:
instance: [0, 1, 2, 3]
env:
MIGRATIONS_TEST_MIGRATE: 1
# XXX: `MATRIX_INSTANCE_TOTAL` must be hardcoded to the length of `strategy.matrix.instance`.
MATRIX_INSTANCE_TOTAL: 4
TEST_GROUP_STRATEGY: ROUND_ROBIN
steps:
# Checkout codebase
- name: Checkout snuba
uses: actions/checkout@v2
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build snuba docker image for CI
uses: docker/build-push-action@v4
with:
context: .
push: false
tags: snuba-test
load: true
target: testing
cache-from: |
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
# Checkout Sentry and run integration tests against latest snuba
# Make sure this is after `docker build`, otherwise we'll break docker cache
- name: Checkout sentry
uses: actions/checkout@v2
with:
repository: getsentry/sentry
path: sentry
- name: Setup steps
id: setup
run: |
pip install --upgrade pip wheel
# We cannot execute actions that are not placed under .github of the main repo
mkdir -p .github/actions
cp -r sentry/.github/actions/* .github/actions
- name: Setup Sentry
id: setup-sentry
uses: ./.github/actions/setup-sentry
with:
workdir: sentry
snuba: false
kafka: true
clickhouse: true
- name: Start snuba
run: |
docker run -d --rm \
-p 127.0.0.1:1218:1218 \
-e PYTHONUNBUFFERED=1 \
-e SNUBA_SETTINGS=docker \
-e DEBUG=1 \
-e DEFAULT_BROKERS=sentry_kafka:9092 \
-e CLICKHOUSE_HOST=sentry_clickhouse \
-e CLICKHOUSE_PORT=9000 \
-e CLICKHOUSE_HTTP_PORT=8123 \
-e REDIS_HOST=sentry_redis \
-e REDIS_PORT=6379 \
-e REDIS_DB=1 \
--name sentry_snuba \
--network sentry \
snuba-test
docker exec sentry_snuba snuba migrations migrate --force
- name: Run snuba tests
if: needs.files-changed.outputs.api_changes == 'false'
working-directory: sentry
run: |
pytest -k 'not __In' tests \
-m snuba_ci \
-vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"
- name: Run full tests
# TODO: Adjust which changed files we care about
if: needs.files-changed.outputs.api_changes == 'true'
working-directory: sentry
run: |
pytest -k 'not __In' tests/snuba \
tests/sentry/eventstream/kafka \
tests/sentry/post_process_forwarder \
tests/sentry/snuba \
tests/sentry/search/events \
tests/sentry/event_manager \
tests/sentry/api/endpoints/test_organization_ddm_meta.py \
tests/sentry/api/endpoints/test_organization_profiling_functions.py \
tests/snuba/test_snql_snuba.py \
tests/snuba/test_metrics_layer.py \
-vv --cov . --cov-report="xml:.artifacts/snuba.coverage.xml"
- name: Run CI module tests
if: needs.files-changed.outputs.api_changes == 'true'
working-directory: sentry
run: pytest -k 'not __In' tests -vv -m snuba_ci
clickhouse-versions:
needs: [linting, snuba-image]
name: Tests on multiple clickhouse versions
runs-on: ubuntu-latest
timeout-minutes: 30
strategy:
fail-fast: false
matrix:
version:
[
"21.8.13.1.altinitystable",
"22.8.15.25.altinitystable",
"23.3.13.7.altinitystable",
"23.8.8.21.altinitystable",
]
steps:
- uses: actions/checkout@v2
name: Checkout code
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v2
- name: Build snuba docker image for CI
uses: docker/build-push-action@v4
with:
context: .
push: false
tags: snuba-test
load: true
target: testing
cache-from: |
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ github.sha }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:${{ needs.snuba-image.outputs.branch }}
type=registry,ref=ghcr.io/getsentry/snuba-ci:latest
- name: Docker set up
run: |
docker network create --attachable cloudbuild
- name: Docker Snuba Test other ClickHouse versions
run: |
export CLICKHOUSE_IMAGE=ghcr.io/getsentry/image-mirror-altinity-clickhouse-server:${{matrix.version}}
SNUBA_IMAGE=snuba-test SNUBA_SETTINGS=test docker-compose -f docker-compose.gcb.yml run --rm snuba-test
- name: Upload to codecov
run: |
curl -Os https://uploader.codecov.io/latest/linux/codecov && chmod +x codecov && ./codecov -t ${CODECOV_TOKEN}
publish-to-dockerhub:
name: Publish Snuba to DockerHub
runs-on: ubuntu-20.04
if: ${{ (github.ref_name == 'master') }}
steps:
- uses: actions/checkout@v3 # v3.1.0
- name: Pull the test image
id: image_pull
env:
IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }}
shell: bash
run: |
echo "We poll for the Docker image that the GCB/GHA build produces until it succeeds or this job times out."
echo "Polling for $IMAGE_URL"
timeout 20m bash -c 'until docker pull "$IMAGE_URL" 2>/dev/null; do sleep 10; done'
- name: Get short SHA for docker tag
id: short_sha
shell: bash
run: |
SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
if [[ -z "$SHORT_SHA" ]]; then
echo "Short SHA empty? Re-running rev-parse."
git rev-parse --short "$GITHUB_SHA"
else
echo "sha=$SHORT_SHA" >> $GITHUB_OUTPUT
fi
- name: Push built docker image
shell: bash
env:
SHORT_SHA: ${{ steps.short_sha.outputs.sha }}
IMAGE_URL: us.gcr.io/sentryio/snuba:${{ github.sha }}
run: |
# only login if the password is set
if [[ "${{ secrets.DOCKER_HUB_RW_TOKEN }}" ]]; then echo "${{ secrets.DOCKER_HUB_RW_TOKEN }}" | docker login --username=sentrybuilder --password-stdin; fi
# We push 3 tags to Dockerhub:
# first, the full sha of the commit
docker tag ${IMAGE_URL} getsentry/snuba:${GITHUB_SHA}
docker push getsentry/snuba:${GITHUB_SHA}
# second, the short sha of the commit
docker tag ${IMAGE_URL} getsentry/snuba:${SHORT_SHA}
docker push getsentry/snuba:${SHORT_SHA}
# finally, nightly
docker tag ${IMAGE_URL} getsentry/snuba:nightly
docker push getsentry/snuba:nightly