From 15151ff295439a0b7694d45bda8f625104f9b89b Mon Sep 17 00:00:00 2001 From: Mark Story Date: Tue, 10 Dec 2024 14:29:14 -0500 Subject: [PATCH 1/5] feat: Switch to devservices instead of sentry devservices Use direnv + devservices to have a smaller dependency footprint. Sentry will also be moving to devservices in the new year and already has opt-in config for using devservices now. Fixes #85 --- .envrc | 14 ++++++++++++++ .github/workflows/ci.yml | 33 +++++++-------------------------- Makefile | 2 +- config/config-sentry-dev.yaml | 2 +- devservices/config.yml | 19 +++++++++++++++++++ python/requirements-dev.txt | 1 + 6 files changed, 43 insertions(+), 28 deletions(-) create mode 100644 .envrc create mode 100644 devservices/config.yml diff --git a/.envrc b/.envrc new file mode 100644 index 00000000..eb1bb7e8 --- /dev/null +++ b/.envrc @@ -0,0 +1,14 @@ +#!/bin/bash +# shellcheck disable=SC1091 +if [[ -f .env ]]; then + dotenv .env +fi + +if [ ! -d .venv ]; then + echo "warning: creating virtualenv for the first time" + make setup + source python/.venv/bin/activate +else + source python/.venv/bin/activate + unset PS1 +fi diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 215e4e5c..4a4bdc01 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -26,10 +26,10 @@ jobs: - uses: swatinem/rust-cache@v2 - name: Run Rustfmt - run: cargo fmt --all -- --check + run: make style - name: Run Clippy - run: cargo clippy --workspace --all-features --tests -- -D clippy::all + run: make lint test: name: Tests (ubuntu) @@ -38,27 +38,14 @@ jobs: steps: - uses: actions/checkout@v2 - - name: Checkout sentry - uses: actions/checkout@v4 - with: - repository: getsentry/sentry - path: sentry + - name: Install Python Dependencies + run: | + make install-py-dev - - name: Setup steps + - name: Start services id: setup run: | - # We cannot execute actions that are not placed under .github of the main repo - mkdir -p .github/actions - cp -r sentry/.github/actions/* .github/actions - - - name: Setup Sentry - id: setup-sentry - uses: ./.github/actions/setup-sentry - with: - workdir: sentry - snuba: false - kafka: true - clickhouse: false + devservices up - name: Install protoc uses: arduino/setup-protoc@v3 @@ -79,12 +66,6 @@ jobs: command: test args: --all - - name: Install Python Dependencies - run: | - python -m venv python/.venv - . python/.venv/bin/activate - pip install -r python/requirements-dev.txt - - name: Run Python Integration Tests run: | export PYTEST_ADDOPTS="" diff --git a/Makefile b/Makefile index 38826ecf..6d452b33 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,5 @@ # Development -setup: setup-git ## run setup tasks to create and configure a development environment +setup: setup-git install-py-dev ## run setup tasks to create and configure a development environment .PHONY: setup setup-git: .git/hooks/pre-commit ## Setup git-hooks diff --git a/config/config-sentry-dev.yaml b/config/config-sentry-dev.yaml index 7ba6e0cc..af0354b1 100644 --- a/config/config-sentry-dev.yaml +++ b/config/config-sentry-dev.yaml @@ -1,2 +1,2 @@ upkeep_task_interval_ms: 2000 -kafka_cluster: sentry_kafka:9093 +kafka_cluster: 127.0.0.1:9092 diff --git a/devservices/config.yml b/devservices/config.yml new file mode 100644 index 00000000..253d2554 --- /dev/null +++ b/devservices/config.yml @@ -0,0 +1,19 @@ +# Ignored by docker compose, used by devservices +x-sentry-service-config: + version: 0.1 + service_name: taskbroker + dependencies: + kafka: + description: Shared instance of kafka used by sentry services + remote: + repo_name: sentry-shared-kafka + branch: main + repo_link: git@github.com:getsentry/sentry-shared-kafka.git + modes: + default: [kafka] + containerized: [kafka] + +networks: + devservices: + name: devservices + external: true diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index 9a79f8ce..5b83352f 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -1,4 +1,5 @@ confluent_kafka>=2.3.0 +devservices==1.0.4 orjson>=3.10.10 protobuf>=5.28.3 pytest>=8.3.3 From f1c17fd5c4820249064aecd31d3d2dd32041a43c Mon Sep 17 00:00:00 2001 From: Mark Story Date: Tue, 10 Dec 2024 15:24:35 -0500 Subject: [PATCH 2/5] Fix parameters for clippy --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6d452b33..8e564712 100644 --- a/Makefile +++ b/Makefile @@ -27,7 +27,7 @@ style: ## Run style checking tools (cargo-fmt) lint: ## Run linting tools (cargo-clippy) @rustup component add clippy 2> /dev/null - cargo clippy --workspace --all-targets --all-features --no-deps --allow-dirty -- -D warnings + cargo clippy --workspace --all-targets --all-features --no-deps -- -D warnings .PHONY: format format: ## Run autofix mode for formatting and lint From 6662495d4d6d184ca2f7fc2cfc126ec03fedf9a8 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Tue, 10 Dec 2024 15:32:10 -0500 Subject: [PATCH 3/5] Try https clone url --- devservices/config.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/devservices/config.yml b/devservices/config.yml index 253d2554..b291c208 100644 --- a/devservices/config.yml +++ b/devservices/config.yml @@ -8,7 +8,7 @@ x-sentry-service-config: remote: repo_name: sentry-shared-kafka branch: main - repo_link: git@github.com:getsentry/sentry-shared-kafka.git + repo_link: https://github.com/getsentry/sentry-shared-kafka modes: default: [kafka] containerized: [kafka] From e0b76e176f1b78be87bf26ae24a502faa19808d9 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Tue, 10 Dec 2024 15:36:05 -0500 Subject: [PATCH 4/5] Update integration tests as well --- python/integration_tests/helpers.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/integration_tests/helpers.py b/python/integration_tests/helpers.py index bcc7f491..67129789 100644 --- a/python/integration_tests/helpers.py +++ b/python/integration_tests/helpers.py @@ -18,7 +18,7 @@ def check_topic_exists(topic_name: str) -> bool: check_topic_cmd = [ "docker", "exec", - "sentry_kafka", + "kafka-kafka-1", "kafka-topics", "--bootstrap-server", "localhost:9092", @@ -37,7 +37,7 @@ def create_topic(topic_name: str, num_partitions: int) -> None: create_topic_cmd = [ "docker", "exec", - "sentry_kafka", + "kafka-kafka-1", "kafka-topics", "--bootstrap-server", "localhost:9092", @@ -59,7 +59,7 @@ def update_topic_partitions(topic_name: str, num_partitions: int) -> None: create_topic_cmd = [ "docker", "exec", - "sentry_kafka", + "kafka-kafka-1", "kafka-topics", "--bootstrap-server", "localhost:9092", From 9b5dd2fcd4b8d63ba08658a098cbc6dd0b1a4f85 Mon Sep 17 00:00:00 2001 From: Mark Story Date: Tue, 10 Dec 2024 16:29:19 -0500 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Ian Woodard <17186604+IanWoodard@users.noreply.github.com> --- python/requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/requirements-dev.txt b/python/requirements-dev.txt index 5b83352f..cf17c64e 100644 --- a/python/requirements-dev.txt +++ b/python/requirements-dev.txt @@ -1,5 +1,5 @@ confluent_kafka>=2.3.0 -devservices==1.0.4 +devservices==1.0.5 orjson>=3.10.10 protobuf>=5.28.3 pytest>=8.3.3