From ab60a9857d747cafeaf1375d559738945f18aa1b Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Wed, 9 Dec 2020 08:45:28 -0500 Subject: [PATCH 01/12] Integrate with MaxMind out of the box --- docker-compose.yml | 16 ++++++++++++++++ nginx/nginx.conf | 4 ++-- relay/config.example.yml | 1 + sentry/sentry.conf.example.py | 6 ++++++ 4 files changed, 25 insertions(+), 2 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 5909226ae4..0f53f3ea70 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,6 +30,7 @@ x-sentry-defaults: &sentry_defaults SENTRY_EVENT_RETENTION_DAYS: volumes: - 'sentry-data:/data' + - 'sentry-geoip:/geoip' - './sentry:/etc/sentry' x-snuba-defaults: &snuba_defaults << : *restart_policy @@ -126,6 +127,18 @@ services: # If you have high volume and your search return incomplete results # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 + geoipupdate: + << : *restart_policy + image: 'maxmindinc/geoipupdate:latest' + environment: + GEOIPUPDATE_FREQUENCY: 168 # one week, in hours + GEOIPUPDATE_EDITION_IDS: GeoLite2-City + # Set the following (e.g., via docker-compose.override.yml) to enable IP geolocation. + # See https://develop.sentry.dev/self-hosted/geolocation/. + GEOIPUPDATE_ACCOUNT_ID: + GEOIPUPDATE_LICENSE_KEY: + volumes: + - 'sentry-geoip:/usr/share/GeoIP' snuba-api: << : *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse @@ -233,6 +246,7 @@ services: read_only: true source: ./relay target: /work/.relay + - 'sentry-geoip:/geoip' depends_on: - kafka - redis @@ -251,6 +265,8 @@ volumes: external: true sentry-symbolicator: external: true + sentry-geoip: + external: true sentry-secrets: sentry-smtp: sentry-zookeeper-log: diff --git a/nginx/nginx.conf b/nginx/nginx.conf index bd81a5e8df..e7e242fc97 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -44,8 +44,8 @@ http { # it could be "close" to close a keepalive connection proxy_set_header Connection ''; proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $remote_addr; + proxy_set_header X-Real-IP $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Request-Id $request_id; proxy_read_timeout 30s; diff --git a/relay/config.example.yml b/relay/config.example.yml index 0488ba91a4..5c0f16bc5b 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -10,3 +10,4 @@ processing: - {name: "bootstrap.servers", value: "kafka:9092"} - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 + geoip_path: "/geoip/GeoIP2-City.mmdb" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index ca217d117a..dfddd3150f 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -260,6 +260,12 @@ def get_internal_network(): } ) +####################### +# MaxMind Integration # +###################### + +GEOIP_PATH_MMDB = '/geoip/GeoIP2-City.mmdb' + ######################### # Bitbucket Integration # ######################## From 7d2c36be2c0117f1959c494cfc0482195082aeff Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 10 Dec 2020 09:38:55 -0500 Subject: [PATCH 02/12] Switch to bind mount --- docker-compose.yml | 11 ++++++----- geoip/.gitignore | 2 ++ 2 files changed, 8 insertions(+), 5 deletions(-) create mode 100644 geoip/.gitignore diff --git a/docker-compose.yml b/docker-compose.yml index 0f53f3ea70..351468d3f7 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -30,8 +30,8 @@ x-sentry-defaults: &sentry_defaults SENTRY_EVENT_RETENTION_DAYS: volumes: - 'sentry-data:/data' - - 'sentry-geoip:/geoip' - './sentry:/etc/sentry' + - './geoip:/geoip:ro' x-snuba-defaults: &snuba_defaults << : *restart_policy depends_on: @@ -138,7 +138,7 @@ services: GEOIPUPDATE_ACCOUNT_ID: GEOIPUPDATE_LICENSE_KEY: volumes: - - 'sentry-geoip:/usr/share/GeoIP' + - './geoip:/usr/share/GeoIP' snuba-api: << : *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse @@ -246,7 +246,10 @@ services: read_only: true source: ./relay target: /work/.relay - - 'sentry-geoip:/geoip' + - type: bind + read_only: true + source: ./geoip + target: /geoip depends_on: - kafka - redis @@ -265,8 +268,6 @@ volumes: external: true sentry-symbolicator: external: true - sentry-geoip: - external: true sentry-secrets: sentry-smtp: sentry-zookeeper-log: diff --git a/geoip/.gitignore b/geoip/.gitignore new file mode 100644 index 0000000000..035a20b690 --- /dev/null +++ b/geoip/.gitignore @@ -0,0 +1,2 @@ +GeoLite2-City.mmdb +.geoipupdate.lock From f47e60c0b7c26ae92360c567944041d1eaea2695 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 10 Dec 2020 09:43:11 -0500 Subject: [PATCH 03/12] Slow down re: docker-compose.override.yml --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 351468d3f7..1d8bb16902 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -133,7 +133,7 @@ services: environment: GEOIPUPDATE_FREQUENCY: 168 # one week, in hours GEOIPUPDATE_EDITION_IDS: GeoLite2-City - # Set the following (e.g., via docker-compose.override.yml) to enable IP geolocation. + # Set the following environment variables to enable IP geolocation. # See https://develop.sentry.dev/self-hosted/geolocation/. GEOIPUPDATE_ACCOUNT_ID: GEOIPUPDATE_LICENSE_KEY: From a1716ebb2b8fd21c0ece58f3a0ac0989be398612 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 10 Dec 2020 10:52:16 -0500 Subject: [PATCH 04/12] Reset X-Real-IP It's only X-Forwarded-For that matters. --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index e7e242fc97..e8d56443af 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -44,7 +44,7 @@ http { # it could be "close" to close a keepalive connection proxy_set_header Connection ''; proxy_set_header Host $host; - proxy_set_header X-Real-IP $proxy_add_x_forwarded_for; + proxy_set_header X-Real-IP $remote_addr; proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; proxy_set_header X-Forwarded-Proto $scheme; proxy_set_header X-Request-Id $request_id; From 3404eab4f232f8c6c50cc71d182c1226a215f028 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 10 Dec 2020 17:25:14 -0500 Subject: [PATCH 05/12] Move frequency default to .env to allow overriding --- .env | 1 + docker-compose.yml | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.env b/.env index 3848c8c0eb..27ccc125a7 100644 --- a/.env +++ b/.env @@ -7,3 +7,4 @@ SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly +GEOIPUPDATE_FREQUENCY=168 # one week, in hours diff --git a/docker-compose.yml b/docker-compose.yml index 1d8bb16902..4299df50e2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -131,7 +131,7 @@ services: << : *restart_policy image: 'maxmindinc/geoipupdate:latest' environment: - GEOIPUPDATE_FREQUENCY: 168 # one week, in hours + GEOIPUPDATE_FREQUENCY: # default set in .env GEOIPUPDATE_EDITION_IDS: GeoLite2-City # Set the following environment variables to enable IP geolocation. # See https://develop.sentry.dev/self-hosted/geolocation/. From 1c3a74e3ca7a7dfab2a5b377acdaa58dbc335538 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Thu, 10 Dec 2020 17:25:50 -0500 Subject: [PATCH 06/12] Ensure that a GeoLite2-City.mmdb always exists Without this, relay exits. --- geoip/GeoLite2-City.mmdb.empty | Bin 0 -> 1055 bytes install.sh | 12 ++++++++++++ 2 files changed, 12 insertions(+) create mode 100644 geoip/GeoLite2-City.mmdb.empty diff --git a/geoip/GeoLite2-City.mmdb.empty b/geoip/GeoLite2-City.mmdb.empty new file mode 100644 index 0000000000000000000000000000000000000000..94f6921fd1c885fac56dcb689bf4467b25d9d78d GIT binary patch literal 1055 zcmZ9`*;dm)0LJmpS~uKxRMfh!RKd~H_-}(2OHXUFNupTG?i-Ebo3{a%=fQ7((_5zw6W$#6%J4|Hb5@0E? z8dzq=<-iK|O1jGI`9s$5<664T9CZQ**&Be3X50jH0lR_CX4%5nN=tMb-A;GVopcw? z{&5duFWpD?(*rcWQaA7kD04Xk#K2)-3^>B2haLs)0>^-Bz;P}o=t+8to~CE$S$dA1 zrx$21y+|(s13(|UpI)X{=vA8EFn7=_6-E^pVp#M#y+O0@-((B}w-~qSh&jDu;+~27 zzys3{fl>A&v*Z_e4A@+rP(dA_#&7|TA!+;Ob)dmTnSGow0r+NVGBgk{LbE>wBGcIe z6VolmB=C$eMN>LWpVJrgC4EI-)9g`i7;ou2;6395{YcyQ&*I Date: Fri, 11 Dec 2020 10:05:55 -0500 Subject: [PATCH 07/12] Tie geoipupdate to install.sh --- .env | 1 - docker-compose.yml | 11 ++--------- geoip/.gitignore | 3 ++- install.sh | 10 +--------- install/geoip.sh | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 39 insertions(+), 20 deletions(-) create mode 100755 install/geoip.sh diff --git a/.env b/.env index 27ccc125a7..3848c8c0eb 100644 --- a/.env +++ b/.env @@ -7,4 +7,3 @@ SENTRY_IMAGE=getsentry/sentry:nightly SNUBA_IMAGE=getsentry/snuba:nightly RELAY_IMAGE=getsentry/relay:nightly SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly -GEOIPUPDATE_FREQUENCY=168 # one week, in hours diff --git a/docker-compose.yml b/docker-compose.yml index 4299df50e2..b6de21eaca 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -128,17 +128,10 @@ services: # You might want to change this to a higher value (and ensure your host has enough memory) MAX_MEMORY_USAGE_RATIO: 0.3 geoipupdate: - << : *restart_policy image: 'maxmindinc/geoipupdate:latest' - environment: - GEOIPUPDATE_FREQUENCY: # default set in .env - GEOIPUPDATE_EDITION_IDS: GeoLite2-City - # Set the following environment variables to enable IP geolocation. - # See https://develop.sentry.dev/self-hosted/geolocation/. - GEOIPUPDATE_ACCOUNT_ID: - GEOIPUPDATE_LICENSE_KEY: + entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf'] volumes: - - './geoip:/usr/share/GeoIP' + - './geoip:/sentry' snuba-api: << : *snuba_defaults # Kafka consumer responsible for feeding events into Clickhouse diff --git a/geoip/.gitignore b/geoip/.gitignore index 035a20b690..0b56a5c2e1 100644 --- a/geoip/.gitignore +++ b/geoip/.gitignore @@ -1,2 +1,3 @@ -GeoLite2-City.mmdb +GeoIP.conf +*.mmdb .geoipupdate.lock diff --git a/install.sh b/install.sh index 3d3a062ead..579fb9328b 100755 --- a/install.sh +++ b/install.sh @@ -30,7 +30,6 @@ SENTRY_CONFIG_YML='sentry/config.yml' SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml' RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' -GEOLITE2_CITY_MMDB='geoip/GeoLite2-City.mmdb' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' MINIMIZE_DOWNTIME= @@ -327,14 +326,7 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then fi -if [[ -f "$GEOLITE2_CITY_MMDB" ]]; then - echo "IP address geolocation database already installed." -else - echo "" - echo "Installing (empty) IP address geolocation database ..." - cp "$GEOLITE2_CITY_MMDB.empty" "$GEOLITE2_CITY_MMDB" - echo "See https://develop.sentry.dev/self-hosted/geolocation/ to configure geolocation." -fi +./install/geoipupdate.sh if [[ "$MINIMIZE_DOWNTIME" ]]; then diff --git a/install/geoip.sh b/install/geoip.sh new file mode 100755 index 0000000000..c484c9ae03 --- /dev/null +++ b/install/geoip.sh @@ -0,0 +1,34 @@ +#!/usr/bin/env bash + +if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi + +dc="docker-compose --no-ansi" +dcr="$dc run --rm" + +GEOLITE2_CITY_MMDB='geoip/GeoLite2-City.mmdb' +GEOIP_CONF='geoip/GeoIP.conf' +result='Done' + +echo "Setting up IP address geolocation ..." +if [[ ! -f "$GEOLITE2_CITY_MMDB" ]]; then + echo -n "Installing (empty) IP address geolocation database ... " + cp "$GEOLITE2_CITY_MMDB.empty" "$GEOLITE2_CITY_MMDB" + echo "done." +else + echo "IP address geolocation database already exists." +fi + +if [[ ! -f "$GEOIP_CONF" ]]; then + echo "IP address geolocation is not configured for updates." + echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions." + result='Error' +else + echo "IP address geolocation is configured for updates." + echo "Updating IP address geolocation database ... " + $dcr geoipupdate + if [ $? -gt 0 ]; then + result='Error' + fi + echo "$result updating IP address geolocation database." +fi +echo "$result setting up IP address geolocation." From 68ebb4426f8bb9bf4fbaaf49244be541b6f56d56 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 11 Dec 2020 11:13:40 -0500 Subject: [PATCH 08/12] Oops! Fix subscript invocation in install.sh --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index 579fb9328b..44c4f3e4f4 100755 --- a/install.sh +++ b/install.sh @@ -326,7 +326,7 @@ if [[ ! -f "$RELAY_CREDENTIALS_JSON" ]]; then fi -./install/geoipupdate.sh +./install/geoip.sh if [[ "$MINIMIZE_DOWNTIME" ]]; then From f208f0941ed147eb865a3d736280f0031f872f48 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 11 Dec 2020 11:16:55 -0500 Subject: [PATCH 09/12] Better encapsulate geoip installation --- install/geoip.sh | 51 ++++++++++++++++++++++++++---------------------- 1 file changed, 28 insertions(+), 23 deletions(-) diff --git a/install/geoip.sh b/install/geoip.sh index c484c9ae03..ec37b21660 100755 --- a/install/geoip.sh +++ b/install/geoip.sh @@ -5,30 +5,35 @@ if [ ! -f 'install.sh' ]; then echo 'Where are you?'; exit 1; fi dc="docker-compose --no-ansi" dcr="$dc run --rm" -GEOLITE2_CITY_MMDB='geoip/GeoLite2-City.mmdb' -GEOIP_CONF='geoip/GeoIP.conf' -result='Done' -echo "Setting up IP address geolocation ..." -if [[ ! -f "$GEOLITE2_CITY_MMDB" ]]; then - echo -n "Installing (empty) IP address geolocation database ... " - cp "$GEOLITE2_CITY_MMDB.empty" "$GEOLITE2_CITY_MMDB" - echo "done." -else - echo "IP address geolocation database already exists." -fi +install_geoip() { + local mmdb='geoip/GeoLite2-City.mmdb' + local conf='geoip/GeoIP.conf' + local result='Done' -if [[ ! -f "$GEOIP_CONF" ]]; then - echo "IP address geolocation is not configured for updates." - echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions." - result='Error' -else - echo "IP address geolocation is configured for updates." - echo "Updating IP address geolocation database ... " - $dcr geoipupdate - if [ $? -gt 0 ]; then + echo "Setting up IP address geolocation ..." + if [[ ! -f "$mmdb" ]]; then + echo -n "Installing (empty) IP address geolocation database ... " + cp "$mmdb.empty" "$mmdb" + echo "done." + else + echo "IP address geolocation database already exists." + fi + + if [[ ! -f "$conf" ]]; then + echo "IP address geolocation is not configured for updates." + echo "See https://develop.sentry.dev/self-hosted/geolocation/ for instructions." result='Error' + else + echo "IP address geolocation is configured for updates." + echo "Updating IP address geolocation database ... " + $dcr geoipupdate + if [ $? -gt 0 ]; then + result='Error' + fi + echo "$result updating IP address geolocation database." fi - echo "$result updating IP address geolocation database." -fi -echo "$result setting up IP address geolocation." + echo "$result setting up IP address geolocation." +} + +install_geoip From b78e6ff36d0f2152fca3415f5cc2270702345046 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Fri, 11 Dec 2020 11:50:19 -0500 Subject: [PATCH 10/12] Fix example config --- relay/config.example.yml | 2 +- sentry/sentry.conf.example.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/relay/config.example.yml b/relay/config.example.yml index 5c0f16bc5b..8538bd7d46 100644 --- a/relay/config.example.yml +++ b/relay/config.example.yml @@ -10,4 +10,4 @@ processing: - {name: "bootstrap.servers", value: "kafka:9092"} - {name: "message.max.bytes", value: 50000000} #50MB or bust redis: redis://redis:6379 - geoip_path: "/geoip/GeoIP2-City.mmdb" + geoip_path: "/geoip/GeoLite2-City.mmdb" diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index dfddd3150f..2d89b5677f 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -262,13 +262,13 @@ def get_internal_network(): ####################### # MaxMind Integration # -###################### +####################### -GEOIP_PATH_MMDB = '/geoip/GeoIP2-City.mmdb' +GEOIP_PATH_MMDB = '/geoip/GeoLite2-City.mmdb' ######################### # Bitbucket Integration # -######################## +######################### # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET' From 1364eaf79cab9150e64b91422434a9bf9a0e39be Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 14 Dec 2020 08:57:13 -0500 Subject: [PATCH 11/12] Prefer root .gitignore --- .gitignore | 3 +++ geoip/.gitignore | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) delete mode 100644 geoip/.gitignore diff --git a/.gitignore b/.gitignore index b8ee807d52..707622f425 100644 --- a/.gitignore +++ b/.gitignore @@ -81,3 +81,6 @@ sentry/requirements.txt relay/credentials.json relay/config.yml symbolicator/config.yml +geoip/GeoIP.conf +geoip/*.mmdb +geoip/.geoipupdate.lock diff --git a/geoip/.gitignore b/geoip/.gitignore deleted file mode 100644 index 0b56a5c2e1..0000000000 --- a/geoip/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -GeoIP.conf -*.mmdb -.geoipupdate.lock From 5c2d180a44927bbfc87f7fc59bbd33ba96fa1743 Mon Sep 17 00:00:00 2001 From: Chad Whitacre Date: Mon, 14 Dec 2020 11:25:03 -0500 Subject: [PATCH 12/12] Document geoipupdate entrypoint override --- docker-compose.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docker-compose.yml b/docker-compose.yml index b6de21eaca..160ae6294c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -129,6 +129,9 @@ services: MAX_MEMORY_USAGE_RATIO: 0.3 geoipupdate: image: 'maxmindinc/geoipupdate:latest' + # Override the entrypoint in order to avoid using envvars for config. + # Futz with settings so we can keep mmdb and conf in same dir on host + # (image looks for them in separate dirs by default). entrypoint: ['/usr/bin/geoipupdate', '-d', '/sentry', '-f', '/sentry/GeoIP.conf'] volumes: - './geoip:/sentry'