From 3642fd55e488d0731f87b0abd9d6fcdf82486c13 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 12:31:28 +0300 Subject: [PATCH 1/6] feat: Leverage health checks for depends_on Expands the work on #1053 to all applicable services. --- CHANGELOG.md | 1 + docker-compose.yml | 50 +++++++++++++++++++++++++++------------------- 2 files changed, 30 insertions(+), 21 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 31d829c830b..edf49c7de4c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ - feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details. - fix: Fix `curl` image to version 7.77.0 - upgrade: docker-compose version to 1.29.2 +- feat: Leverage health checks for depends_on ## 21.7.0 diff --git a/docker-compose.yml b/docker-compose.yml index 78c7547e138..2cc0fe1a2e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,8 @@ version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped +x-depends_on-condition: &depends_on_condition + condition: service_healthy x-healthcheck-defaults: &healthcheck_defaults # Avoid setting the interval too small, as docker uses much more CPU than one would expect. # Related issues: @@ -15,20 +17,23 @@ x-sentry-defaults: &sentry_defaults <<: *restart_policy image: "$SENTRY_IMAGE" depends_on: - - redis - - postgres - - memcached - - smtp - - snuba-api - - snuba-consumer - - snuba-outcomes-consumer - - snuba-sessions-consumer - - snuba-transactions-consumer - - snuba-subscription-consumer-events - - snuba-subscription-consumer-transactions - - snuba-replacer - - symbolicator - - kafka + redis: + <<: *depends_on_condition + kafka: + <<: *depends_on_condition + postgres: + <<: *depends_on_condition + memcached: ~ + smtp: ~ + snuba-api: ~ + snuba-consumer: ~ + snuba-outcomes-consumer: ~ + snuba-sessions-consumer: ~ + snuba-transactions-consumer: ~ + snuba-subscription-consumer-events: ~ + snuba-subscription-consumer-transactions: ~ + snuba-replacer: ~ + symbolicator: ~ entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: @@ -55,11 +60,11 @@ x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: - condition: service_healthy + <<: *depends_on_condition kafka: - condition: service_healthy + <<: *depends_on_condition redis: - condition: service_healthy + <<: *depends_on_condition image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker @@ -143,7 +148,8 @@ services: kafka: <<: *restart_policy depends_on: - - zookeeper + zookeeper: + <<: *depends_on_condition image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" @@ -318,9 +324,11 @@ services: source: ./geoip target: /geoip depends_on: - - kafka - - redis - - web + kafka: + <<: *depends_on_condition + redis: + <<: *depends_on_condition + web: ~ volumes: sentry-data: external: true From c87dfc05e1331499fb162608b9e026a8ab4527b4 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 12:34:01 +0300 Subject: [PATCH 2/6] no nulls --- docker-compose.yml | 36 ++++++++++++++++++++++++------------ 1 file changed, 24 insertions(+), 12 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2cc0fe1a2e3..c163f6ed8dc 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,17 +23,28 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on_condition postgres: <<: *depends_on_condition - memcached: ~ - smtp: ~ - snuba-api: ~ - snuba-consumer: ~ - snuba-outcomes-consumer: ~ - snuba-sessions-consumer: ~ - snuba-transactions-consumer: ~ - snuba-subscription-consumer-events: ~ - snuba-subscription-consumer-transactions: ~ - snuba-replacer: ~ - symbolicator: ~ + memcached: + <<: *depends_on_condition + smtp: + <<: *depends_on_condition + snuba-api: + <<: *depends_on_condition + snuba-consumer: + <<: *depends_on_condition + snuba-outcomes-consumer: + <<: *depends_on_condition + snuba-sessions-consumer: + <<: *depends_on_condition + snuba-transactions-consumer: + <<: *depends_on_condition + snuba-subscription-consumer-events: + <<: *depends_on_condition + snuba-subscription-consumer-transactions: + <<: *depends_on_condition + snuba-replacer: + <<: *depends_on_condition + symbolicator: + <<: *depends_on_condition entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: @@ -328,7 +339,8 @@ services: <<: *depends_on_condition redis: <<: *depends_on_condition - web: ~ + web: + <<: *depends_on_condition volumes: sentry-data: external: true From 115ae565a662d9a3f6d96a5e27c248a2f46c9d86 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 12:43:53 +0300 Subject: [PATCH 3/6] Revert "no nulls" This reverts commit c87dfc05e1331499fb162608b9e026a8ab4527b4. --- docker-compose.yml | 36 ++++++++++++------------------------ 1 file changed, 12 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index c163f6ed8dc..2cc0fe1a2e3 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,28 +23,17 @@ x-sentry-defaults: &sentry_defaults <<: *depends_on_condition postgres: <<: *depends_on_condition - memcached: - <<: *depends_on_condition - smtp: - <<: *depends_on_condition - snuba-api: - <<: *depends_on_condition - snuba-consumer: - <<: *depends_on_condition - snuba-outcomes-consumer: - <<: *depends_on_condition - snuba-sessions-consumer: - <<: *depends_on_condition - snuba-transactions-consumer: - <<: *depends_on_condition - snuba-subscription-consumer-events: - <<: *depends_on_condition - snuba-subscription-consumer-transactions: - <<: *depends_on_condition - snuba-replacer: - <<: *depends_on_condition - symbolicator: - <<: *depends_on_condition + memcached: ~ + smtp: ~ + snuba-api: ~ + snuba-consumer: ~ + snuba-outcomes-consumer: ~ + snuba-sessions-consumer: ~ + snuba-transactions-consumer: ~ + snuba-subscription-consumer-events: ~ + snuba-subscription-consumer-transactions: ~ + snuba-replacer: ~ + symbolicator: ~ entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: @@ -339,8 +328,7 @@ services: <<: *depends_on_condition redis: <<: *depends_on_condition - web: - <<: *depends_on_condition + web: ~ volumes: sentry-data: external: true From 9e50cc368220450f00537f387010fac61a97fc8e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 12:46:54 +0300 Subject: [PATCH 4/6] fix depends on --- docker-compose.yml | 66 +++++++++++++++++++++++++++++----------------- 1 file changed, 42 insertions(+), 24 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 2cc0fe1a2e3..7151a13ed59 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,8 +1,10 @@ version: "3.4" x-restart-policy: &restart_policy restart: unless-stopped -x-depends_on-condition: &depends_on_condition - condition: service_healthy +x-depends_on-healthy: &depends_on-healthy + condition: service_healthy +x-depends_on-default: &depends_on-default + condition: service_started x-healthcheck-defaults: &healthcheck_defaults # Avoid setting the interval too small, as docker uses much more CPU than one would expect. # Related issues: @@ -18,22 +20,33 @@ x-sentry-defaults: &sentry_defaults image: "$SENTRY_IMAGE" depends_on: redis: - <<: *depends_on_condition + <<: *depends_on-healthy kafka: - <<: *depends_on_condition + <<: *depends_on-healthy postgres: - <<: *depends_on_condition - memcached: ~ - smtp: ~ - snuba-api: ~ - snuba-consumer: ~ - snuba-outcomes-consumer: ~ - snuba-sessions-consumer: ~ - snuba-transactions-consumer: ~ - snuba-subscription-consumer-events: ~ - snuba-subscription-consumer-transactions: ~ - snuba-replacer: ~ - symbolicator: ~ + <<: *depends_on-healthy + memcached: + <<: *depends_on-default + smtp: + <<: *depends_on-default + snuba-api: + <<: *depends_on-default + snuba-consumer: + <<: *depends_on-default + snuba-outcomes-consumer: + <<: *depends_on-default + snuba-sessions-consumer: + <<: *depends_on-default + snuba-transactions-consumer: + <<: *depends_on-default + snuba-subscription-consumer-events: + <<: *depends_on-default + snuba-subscription-consumer-transactions: + <<: *depends_on-default + snuba-replacer: + <<: *depends_on-default + symbolicator: + <<: *depends_on-default entrypoint: "/etc/sentry/entrypoint.sh" command: ["run", "web"] environment: @@ -60,11 +73,11 @@ x-snuba-defaults: &snuba_defaults <<: *restart_policy depends_on: clickhouse: - <<: *depends_on_condition + <<: *depends_on-healthy kafka: - <<: *depends_on_condition + <<: *depends_on-healthy redis: - <<: *depends_on_condition + <<: *depends_on-healthy image: "$SNUBA_IMAGE" environment: SNUBA_SETTINGS: docker @@ -149,7 +162,7 @@ services: <<: *restart_policy depends_on: zookeeper: - <<: *depends_on_condition + <<: *depends_on-healthy image: "confluentinc/cp-kafka:5.5.0" environment: KAFKA_ZOOKEEPER_CONNECT: "zookeeper:2181" @@ -190,7 +203,11 @@ services: # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 healthcheck: - test: ["CMD-SHELL", "wget -nv -t1 --spider 'http://localhost:9000/' || exit 1"] + test: + [ + "CMD-SHELL", + "wget -nv -t1 --spider 'http://localhost:9000/' || exit 1", + ] interval: 3s timeout: 600s retries: 200 @@ -325,10 +342,11 @@ services: target: /geoip depends_on: kafka: - <<: *depends_on_condition + <<: *depends_on-healthy redis: - <<: *depends_on_condition - web: ~ + <<: *depends_on-healthy + web: + <<: *depends_on-default volumes: sentry-data: external: true From 3f103ef4993986b07cefe9372ab60b0f97dcbcec Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 14:25:18 +0300 Subject: [PATCH 5/6] add heathcheck for sentry web service too --- docker-compose.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 7151a13ed59..55b3db5a28e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -287,6 +287,13 @@ services: - "sentry-symbolicator:/data" web: <<: *sentry_defaults + healthcheck: + <<: *healthcheck_defaults + test: + - "CMD" + - "/bin/bash" + - '-c' + - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' cron: <<: *sentry_defaults command: run cron @@ -346,7 +353,7 @@ services: redis: <<: *depends_on-healthy web: - <<: *depends_on-default + <<: *depends_on-healthy volumes: sentry-data: external: true From 6b9dd28732aa0432dc68e8910f86c7e8b0e0dc93 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 6 Aug 2021 14:26:10 +0300 Subject: [PATCH 6/6] add ref to the crazy healthcheck solution --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 55b3db5a28e..63b76619e6f 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -293,6 +293,7 @@ services: - "CMD" - "/bin/bash" - '-c' + # Courtesy of https://unix.stackexchange.com/a/234089/108960 - 'exec 3<>/dev/tcp/127.0.0.1/9000 && echo -e "GET /_health/ HTTP/1.1\r\nhost: 127.0.0.1\r\n\r\n" >&3 && grep ok -s -m 1 <&3' cron: <<: *sentry_defaults