From 618027e180f846d3d9eff41780ef436ff752091b Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Mon, 30 Mar 2020 12:07:54 +0200 Subject: [PATCH 01/29] WIP - Working version using Relay for event ingestion. --- docker-compose.yml | 29 +++++++++++++- relay_config/config.yml | 16 ++++++++ relay_config/credentials.json | 5 +++ reverse_proxy_config/nginx.conf | 67 +++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 2 deletions(-) create mode 100644 relay_config/config.yml create mode 100644 relay_config/credentials.json create mode 100644 reverse_proxy_config/nginx.conf diff --git a/docker-compose.yml b/docker-compose.yml index ccf134600a4..c3c72d8b6e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -132,14 +132,15 @@ services: - 'sentry-symbolicator:/data' web: << : *sentry_defaults - ports: - - '9000:9000/tcp' cron: << : *sentry_defaults command: run cron worker: << : *sentry_defaults command: run worker + event-consumer: + << : *sentry_defaults + command: run ingest-consumer --all-consumer-types post-process-forwarder: << : *sentry_defaults # Increase `--commit-batch-size 1` below to deal with high-load environments. @@ -152,6 +153,30 @@ services: args: BASE_IMAGE: 'sentry-onpremise-local' command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' + reverse_proxy: + ports: + - '9000:80/tcp' + image: "nginx:1.16.1" + volumes: + - type: bind + source: ./reverse_proxy_config/nginx.conf + target: /etc/nginx/nginx.conf + depends_on: + - web + - relay + relay: + image: "us.gcr.io/sentryio/relay:latest" + ports: + - '3000:3000/tcp' + command: 'run --config /etc/relay' + volumes: + - type: bind + source: ./relay_config + target: /etc/relay + depends_on: + - kafka + - redis + - web volumes: sentry-data: external: true diff --git a/relay_config/config.yml b/relay_config/config.yml new file mode 100644 index 00000000000..f61defe6454 --- /dev/null +++ b/relay_config/config.yml @@ -0,0 +1,16 @@ +--- +relay: + upstream: "http://web:9000/" + host: 0.0.0.0 + port: 3000 +logging: + level: TRACE + enable_backtraces: false +processing: + enabled: true + kafka_config: + # Original kafka configuration was 9093 but it seems that the + # compose configures kafka to listen at 9092 ? + # - {name: "bootstrap.servers", value: "kafka:9093"} + - {name: "bootstrap.servers", value: "kafka:9092"} + redis: redis://redis:6379 diff --git a/relay_config/credentials.json b/relay_config/credentials.json new file mode 100644 index 00000000000..1ebeffa426a --- /dev/null +++ b/relay_config/credentials.json @@ -0,0 +1,5 @@ +{ + "secret_key": "xWWsHUXG6RxHS9L7QWPrP0aDgVLKSa7vcE4Ypa-Vv44", + "public_key": "pC2BzmrmRjRF_p76louELd0LeTRDRhGAkFv7jG6Ittg", + "id": "0893d495-9f0a-4274-a274-7b6b8d2875ce" +} diff --git a/reverse_proxy_config/nginx.conf b/reverse_proxy_config/nginx.conf new file mode 100644 index 00000000000..d8a43ebb08a --- /dev/null +++ b/reverse_proxy_config/nginx.conf @@ -0,0 +1,67 @@ +user nginx; +worker_processes 1; + +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + + +events { + worker_connections 1024; +} + + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + #gzip on; + + upstream relay { + server relay:3000; + } + + upstream sentry { + server web:9000; + } + + server { + listen 80; + resolver 127.0.0.11 ipv6=off; + + location /api/store/ { + proxy_pass http://relay; + } + location ~ ^/api/\d+/store/$ { + proxy_pass http://relay; + } + location ~ ^/api/\d+/minidump/?$ { + proxy_pass http://relay; + } + location ~ ^/api/\d+/unreal/\w+/$ { + proxy_pass http://relay; + } + location ~ ^/api/\d+/security/$ { + proxy_pass http://relay; + } + location ~ ^/api/\d+/csp-report/$ { + proxy_pass http://relay; + } + location ~ ^/api/\d+/events/[\w-]+/attachments/$ { + proxy_pass http://relay; + } + location / { + proxy_pass http://sentry; + } + } +} From 8cc2621ba06fe08314b77b965cd26d72e51cb153 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Fri, 3 Apr 2020 13:19:27 +0200 Subject: [PATCH 02/29] Changes from code review, still needs work on authentication information generation --- docker-compose.yml | 10 ++++------ {reverse_proxy_config => nginx}/nginx.conf | 7 +------ relay_config/config.yml | 9 +++------ 3 files changed, 8 insertions(+), 18 deletions(-) rename {reverse_proxy_config => nginx}/nginx.conf (91%) diff --git a/docker-compose.yml b/docker-compose.yml index c3c72d8b6e0..e99500715d6 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -153,21 +153,20 @@ services: args: BASE_IMAGE: 'sentry-onpremise-local' command: '"0 0 * * * gosu sentry sentry cleanup --days $SENTRY_EVENT_RETENTION_DAYS"' - reverse_proxy: + nginx: ports: - '9000:80/tcp' image: "nginx:1.16.1" volumes: - type: bind - source: ./reverse_proxy_config/nginx.conf - target: /etc/nginx/nginx.conf + read_only: true + source: ./nginx + target: /etc/nginx depends_on: - web - relay relay: image: "us.gcr.io/sentryio/relay:latest" - ports: - - '3000:3000/tcp' command: 'run --config /etc/relay' volumes: - type: bind @@ -176,7 +175,6 @@ services: depends_on: - kafka - redis - - web volumes: sentry-data: external: true diff --git a/reverse_proxy_config/nginx.conf b/nginx/nginx.conf similarity index 91% rename from reverse_proxy_config/nginx.conf rename to nginx/nginx.conf index d8a43ebb08a..a7b736e3113 100644 --- a/reverse_proxy_config/nginx.conf +++ b/nginx/nginx.conf @@ -11,22 +11,16 @@ events { http { - include /etc/nginx/mime.types; default_type application/octet-stream; log_format main '$remote_addr - $remote_user [$time_local] "$request" ' '$status $body_bytes_sent "$http_referer" ' '"$http_user_agent" "$http_x_forwarded_for"'; - access_log /var/log/nginx/access.log main; - sendfile on; - #tcp_nopush on; keepalive_timeout 65; - #gzip on; - upstream relay { server relay:3000; } @@ -37,6 +31,7 @@ http { server { listen 80; + # use the docker DNS server to resolve ips for relay and sentry containers resolver 127.0.0.11 ipv6=off; location /api/store/ { diff --git a/relay_config/config.yml b/relay_config/config.yml index f61defe6454..9daca523c0c 100644 --- a/relay_config/config.yml +++ b/relay_config/config.yml @@ -3,14 +3,11 @@ relay: upstream: "http://web:9000/" host: 0.0.0.0 port: 3000 -logging: - level: TRACE - enable_backtraces: false +#logging: +# # Available logging levels: TRACE, DEBUG, INFO, WARN, ERROR +# level: WARN processing: enabled: true kafka_config: - # Original kafka configuration was 9093 but it seems that the - # compose configures kafka to listen at 9092 ? - # - {name: "bootstrap.servers", value: "kafka:9093"} - {name: "bootstrap.servers", value: "kafka:9092"} redis: redis://redis:6379 From 7c630fa35ba027552fa5912d6b81aa9f6ce205de Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Mon, 6 Apr 2020 12:13:58 +0200 Subject: [PATCH 03/29] generate Relay credentials at install time --- install.sh | 11 +++++++++++ relay_config/credentials.json | 5 ----- 2 files changed, 11 insertions(+), 5 deletions(-) delete mode 100644 relay_config/credentials.json diff --git a/install.sh b/install.sh index 56de70354ac..e2a7abcc143 100755 --- a/install.sh +++ b/install.sh @@ -16,6 +16,8 @@ SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' +RELAY_CREDENTIALS='relay_config/credentials.json' + DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { @@ -164,6 +166,15 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi +echo "" +echo "Generating Relay credentials..." + +$dcr relay --config /etc/relay credentials generate --overwrite +CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' $RELAY_CREDENTIALS) +sed -i .bkp "s//$CREDENTIALS/g" $SENTRY_CONFIG_PY +# on Mac you must specify a backup file so we need to delete it here +rm "${SENTRY_CONFIG_PY}.bkp" + cleanup echo "" diff --git a/relay_config/credentials.json b/relay_config/credentials.json deleted file mode 100644 index 1ebeffa426a..00000000000 --- a/relay_config/credentials.json +++ /dev/null @@ -1,5 +0,0 @@ -{ - "secret_key": "xWWsHUXG6RxHS9L7QWPrP0aDgVLKSa7vcE4Ypa-Vv44", - "public_key": "pC2BzmrmRjRF_p76louELd0LeTRDRhGAkFv7jG6Ittg", - "id": "0893d495-9f0a-4274-a274-7b6b8d2875ce" -} From 2c7052fd49835e837b4922d56bc788f4b852ab50 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Mon, 6 Apr 2020 12:38:40 +0200 Subject: [PATCH 04/29] generate Relay credentials at install time --- sentry/sentry.conf.example.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index a9fe034dd81..e453d2ac81b 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -220,3 +220,10 @@ # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET' + + +# Known internal Relays ( will be replaced at install time with the generated Relay key) +SENTRY_RELAY_WHITELIST_PK = [ + # a list of public keys with Relays that are allowed to + "" +] From 5a1248f33abd4847497a8b4faec864f86d1ed9da Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Mon, 6 Apr 2020 13:31:43 +0200 Subject: [PATCH 05/29] increase travis timeout until the web service responds --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8ec590d44bc..d6cd235287e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,5 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - timeout 120 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh From c239ece3a27435c98496864006140cff6e11e060 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Mon, 6 Apr 2020 13:54:57 +0200 Subject: [PATCH 06/29] update web check to the login url --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index d6cd235287e..b3563126818 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,5 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 120 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000/auth/login/sentry/); do printf '.'; sleep 0.5; done' - ./test.sh From a8dd78dfbf0ce8951dbef190263cf1206937a463 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 10:12:45 +0200 Subject: [PATCH 07/29] set user when generating credentials (to fix linux installations) --- .travis.yml | 2 +- install.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b3563126818..8ec590d44bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,5 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000/auth/login/sentry/); do printf '.'; sleep 0.5; done' + - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh diff --git a/install.sh b/install.sh index e2a7abcc143..f5e64afd10c 100755 --- a/install.sh +++ b/install.sh @@ -169,7 +169,7 @@ fi echo "" echo "Generating Relay credentials..." -$dcr relay --config /etc/relay credentials generate --overwrite +$dcr --user="$UID" relay --config /etc/relay credentials generate --overwrite CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' $RELAY_CREDENTIALS) sed -i .bkp "s//$CREDENTIALS/g" $SENTRY_CONFIG_PY # on Mac you must specify a backup file so we need to delete it here From e1b90477dd03cc602a849e076b03485593671532 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 10:32:19 +0200 Subject: [PATCH 08/29] fix sed command line to work both on Mac & Linux --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index f5e64afd10c..88099252fc0 100755 --- a/install.sh +++ b/install.sh @@ -171,7 +171,7 @@ echo "Generating Relay credentials..." $dcr --user="$UID" relay --config /etc/relay credentials generate --overwrite CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' $RELAY_CREDENTIALS) -sed -i .bkp "s//$CREDENTIALS/g" $SENTRY_CONFIG_PY +sed -i.bkp "s//$CREDENTIALS/g" $SENTRY_CONFIG_PY # on Mac you must specify a backup file so we need to delete it here rm "${SENTRY_CONFIG_PY}.bkp" From a80c0bec5204381b9db4f8f0a2ff946d0b175bc8 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 10:53:26 +0200 Subject: [PATCH 09/29] WIP - back to login url --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8ec590d44bc..b3563126818 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,5 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000/auth/login/sentry/); do printf '.'; sleep 0.5; done' - ./test.sh From f607bf60706f23116c1103799937dac4d0187c9b Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 11:18:29 +0200 Subject: [PATCH 10/29] WIP --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index b3563126818..1492cd3f4b9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,5 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000/auth/login/sentry/); do printf '.'; sleep 0.5; done' + - timeout 200 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh From 391fd1800b6109db1667bc9eae443763f9dddeb4 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 11:33:18 +0200 Subject: [PATCH 11/29] WIP - dump container logs in travis --- .travis.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1492cd3f4b9..d964ac1929f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,5 +14,9 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d + - docker container ls + - docker logs sentry_onpremise_web_1 + - docker logs sentry_onpremise_nginx_1 + - docker logs sentry_onpremise_relay_1 - timeout 200 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh From 41825341b4be531e452b948fe538371e3321f9dc Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 12:00:10 +0200 Subject: [PATCH 12/29] WIP --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index d964ac1929f..1c480e8c620 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,8 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d + - ls -al + - ls -al relay_config - docker container ls - docker logs sentry_onpremise_web_1 - docker logs sentry_onpremise_nginx_1 From 51ba14a23def5c33b60900e069c6a711bd6cfe06 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 13:20:58 +0200 Subject: [PATCH 13/29] WIP --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 1c480e8c620..f35bd82b8ab 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d + - sudo chown -R 10001:10001 ./relay_config - ls -al - ls -al relay_config - docker container ls From 06296aefbae8013cc3cfed5662096e2e538ef325 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 13:36:06 +0200 Subject: [PATCH 14/29] WIP --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f35bd82b8ab..6636fedeff8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - sudo chown -R 10001:10001 ./relay_config + - sudo chmod -R a=rw ./relay_config - ls -al - ls -al relay_config - docker container ls From 61d9479232a3b8d8e6f415006b9bf2011144f738 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Tue, 7 Apr 2020 17:39:34 +0200 Subject: [PATCH 15/29] WIP --- .travis.yml | 5 +---- docker-compose.yml | 6 ++++-- {relay_config => relay}/config.yml | 0 3 files changed, 5 insertions(+), 6 deletions(-) rename {relay_config => relay}/config.yml (100%) diff --git a/.travis.yml b/.travis.yml index 6636fedeff8..236ced72fb3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,12 +14,9 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - sudo chmod -R a=rw ./relay_config - - ls -al - - ls -al relay_config - docker container ls - docker logs sentry_onpremise_web_1 - docker logs sentry_onpremise_nginx_1 - docker logs sentry_onpremise_relay_1 - - timeout 200 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh diff --git a/docker-compose.yml b/docker-compose.yml index e99500715d6..5d195414b1c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -169,8 +169,8 @@ services: image: "us.gcr.io/sentryio/relay:latest" command: 'run --config /etc/relay' volumes: - - type: bind - source: ./relay_config + - type: volume + source: sentry-relay target: /etc/relay depends_on: - kafka @@ -195,3 +195,5 @@ volumes: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: + sentry-relay: + external: true diff --git a/relay_config/config.yml b/relay/config.yml similarity index 100% rename from relay_config/config.yml rename to relay/config.yml From 671bdb4fed3dc77664fc30c7707fa40588af520b Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Wed, 8 Apr 2020 12:56:51 +0200 Subject: [PATCH 16/29] WIP --- install.sh | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 88099252fc0..b099c6f3e4d 100755 --- a/install.sh +++ b/install.sh @@ -16,8 +16,6 @@ SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' -RELAY_CREDENTIALS='relay_config/credentials.json' - DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { @@ -86,6 +84,12 @@ echo "Created $(docker volume create --name=sentry-zookeeper)." echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." +echo "Created $(docker volume create --name=sentry-relay)." + +# copy config data into the relay volume +docker run -v sentry-relay:/data --name copy-helper busybox true +docker cp ./relay/config.yml copy-helper:/data +docker rm copy-helper echo "" ensure_file_from_example $SENTRY_CONFIG_PY @@ -168,9 +172,15 @@ fi echo "" echo "Generating Relay credentials..." +#generate relay credentials +$dcr --user root relay --config /etc/relay credentials generate --overwrite +#copy credential file localy, from the relay volume, in order to extract the public key +docker cp sentry_onpremise_relay_1:/etc/relay/credentials.json ./credentials.json + +#some test + -$dcr --user="$UID" relay --config /etc/relay credentials generate --overwrite -CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' $RELAY_CREDENTIALS) +CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' ./credentials.json) sed -i.bkp "s//$CREDENTIALS/g" $SENTRY_CONFIG_PY # on Mac you must specify a backup file so we need to delete it here rm "${SENTRY_CONFIG_PY}.bkp" From e232d187e8765473fb4d7eecff03a09736b503b1 Mon Sep 17 00:00:00 2001 From: Markus Unterwaditzer Date: Wed, 8 Apr 2020 14:31:23 +0200 Subject: [PATCH 17/29] pairing with radu: move back to binds, fix perms problems --- .gitignore | 1 + docker-compose.yml | 6 ++---- install.sh | 39 +++++++++++++++++++++++---------------- 3 files changed, 26 insertions(+), 20 deletions(-) diff --git a/.gitignore b/.gitignore index 77b24eb843f..3a74861bae9 100644 --- a/.gitignore +++ b/.gitignore @@ -77,3 +77,4 @@ data/ sentry/sentry.conf.py sentry/config.yml sentry/requirements.txt +relay/credentials.json diff --git a/docker-compose.yml b/docker-compose.yml index 5d195414b1c..65ca9fb6598 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -169,8 +169,8 @@ services: image: "us.gcr.io/sentryio/relay:latest" command: 'run --config /etc/relay' volumes: - - type: volume - source: sentry-relay + - type: bind + source: ./relay target: /etc/relay depends_on: - kafka @@ -195,5 +195,3 @@ volumes: sentry-zookeeper-log: sentry-kafka-log: sentry-smtp-log: - sentry-relay: - external: true diff --git a/install.sh b/install.sh index b099c6f3e4d..c5bdf69a207 100755 --- a/install.sh +++ b/install.sh @@ -14,6 +14,8 @@ MIN_RAM=2400 # MB SENTRY_CONFIG_PY='sentry/sentry.conf.py' SENTRY_CONFIG_YML='sentry/config.yml' +RELAY_CONFIG_YML='relay/config.yml' +RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' DID_CLEAN_UP=0 @@ -84,12 +86,11 @@ echo "Created $(docker volume create --name=sentry-zookeeper)." echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." -echo "Created $(docker volume create --name=sentry-relay)." -# copy config data into the relay volume -docker run -v sentry-relay:/data --name copy-helper busybox true -docker cp ./relay/config.yml copy-helper:/data -docker rm copy-helper +## copy config data into the relay volume +#docker run -v sentry-relay:/data --name copy-helper busybox chmod -R 777 /data +#docker cp ./relay/config.yml copy-helper:/data +#docker rm copy-helper echo "" ensure_file_from_example $SENTRY_CONFIG_PY @@ -172,18 +173,24 @@ fi echo "" echo "Generating Relay credentials..." -#generate relay credentials -$dcr --user root relay --config /etc/relay credentials generate --overwrite -#copy credential file localy, from the relay volume, in order to extract the public key -docker cp sentry_onpremise_relay_1:/etc/relay/credentials.json ./credentials.json -#some test - - -CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' ./credentials.json) -sed -i.bkp "s//$CREDENTIALS/g" $SENTRY_CONFIG_PY -# on Mac you must specify a backup file so we need to delete it here -rm "${SENTRY_CONFIG_PY}.bkp" +if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then + #generate relay credentials + $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite + CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") + CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" + + if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then + echo "FAIL: SENTRY_RELAY_WHITELIST_PK already exists in $SENTRY_CONFIG_PY, please replace with:" + echo "" + echo " $CREDENTIALS" + echo "" + exit 1 + fi + + echo "" >> "$SENTRY_CONFIG_PY" + echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" +fi cleanup From f0110546480d7b76d6a0b707b104b2f9e2572a9f Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Wed, 8 Apr 2020 15:39:32 +0200 Subject: [PATCH 18/29] make the credential file readable by everybody so that user 1001 (the user Relay normally runs under) can access it. --- install.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/install.sh b/install.sh index c5bdf69a207..71ed9d8ba14 100755 --- a/install.sh +++ b/install.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -e +set -evim dc="docker-compose --no-ansi" dcr="$dc run --rm" @@ -177,6 +177,7 @@ echo "Generating Relay credentials..." if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then #generate relay credentials $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite + chmod a+r $(RELAY_CREDENTIALS_JSON) CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" From ac0578d5bcc825f706b97fff9dfae6123e7ea99a Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Wed, 8 Apr 2020 17:14:44 +0200 Subject: [PATCH 19/29] fix small problems, add more messages --- docker-compose.yml | 2 +- install.sh | 20 ++++++++++---------- sentry/sentry.conf.example.py | 6 ------ 3 files changed, 11 insertions(+), 17 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 65ca9fb6598..017428e1274 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -138,7 +138,7 @@ services: worker: << : *sentry_defaults command: run worker - event-consumer: + ingest-consumer: << : *sentry_defaults command: run ingest-consumer --all-consumer-types post-process-forwarder: diff --git a/install.sh b/install.sh index 71ed9d8ba14..5a2644ef40a 100755 --- a/install.sh +++ b/install.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -set -evim +set -e dc="docker-compose --no-ansi" dcr="$dc run --rm" @@ -87,11 +87,6 @@ echo "Created $(docker volume create --name=sentry-kafka)." echo "Created $(docker volume create --name=sentry-clickhouse)." echo "Created $(docker volume create --name=sentry-symbolicator)." -## copy config data into the relay volume -#docker run -v sentry-relay:/data --name copy-helper busybox chmod -R 777 /data -#docker cp ./relay/config.yml copy-helper:/data -#docker rm copy-helper - echo "" ensure_file_from_example $SENTRY_CONFIG_PY ensure_file_from_example $SENTRY_CONFIG_YML @@ -171,13 +166,15 @@ if [ "$SENTRY_DATA_NEEDS_MIGRATION" ]; then "mkdir -p /tmp/files; mv /data/* /tmp/files/; mv /tmp/files /data/files; chown -R sentry:sentry /data" fi -echo "" -echo "Generating Relay credentials..." if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then - #generate relay credentials + echo "" + echo "Generating Relay credentials..." + $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite - chmod a+r $(RELAY_CREDENTIALS_JSON) + chmod a+r $RELAY_CREDENTIALS_JSON + # display the contents of the relay directory (for debug purposes) + ls -al ./relay CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" @@ -191,6 +188,9 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "" >> "$SENTRY_CONFIG_PY" echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" +else + echo "" + echo "Relay credentials exist will NOT generate new ones." fi cleanup diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index e453d2ac81b..b640fa83109 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -221,9 +221,3 @@ # BITBUCKET_CONSUMER_KEY = 'YOUR_BITBUCKET_CONSUMER_KEY' # BITBUCKET_CONSUMER_SECRET = 'YOUR_BITBUCKET_CONSUMER_SECRET' - -# Known internal Relays ( will be replaced at install time with the generated Relay key) -SENTRY_RELAY_WHITELIST_PK = [ - # a list of public keys with Relays that are allowed to - "" -] From dd947ca757aa95db0537de565e1ec80807efe718 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Wed, 8 Apr 2020 17:29:26 +0200 Subject: [PATCH 20/29] more messages to verify everything is ok --- .travis.yml | 1 + install.sh | 2 ++ 2 files changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 236ced72fb3..a82913c89b2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,4 +19,5 @@ script: - docker logs sentry_onpremise_nginx_1 - docker logs sentry_onpremise_relay_1 - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - docker logs sentry_onpremise_relay_1 - ./test.sh diff --git a/install.sh b/install.sh index 5a2644ef40a..3304f5babd2 100755 --- a/install.sh +++ b/install.sh @@ -188,6 +188,8 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "" >> "$SENTRY_CONFIG_PY" echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" + + tail -n 5 "$SENTRY_CONFIG_PY" else echo "" echo "Relay credentials exist will NOT generate new ones." From a566d5b141f4b56572135f6083f9926609f5def3 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Wed, 8 Apr 2020 17:44:36 +0200 Subject: [PATCH 21/29] and more messages to verify everything is ok --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index a82913c89b2..5f8de6c06f3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -20,4 +20,5 @@ script: - docker logs sentry_onpremise_relay_1 - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - docker logs sentry_onpremise_relay_1 + - docker logs sentry_onpremise_web_1 - ./test.sh From c89793715513e305ff757af2a9b13079df18203c Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Wed, 8 Apr 2020 17:59:05 +0200 Subject: [PATCH 22/29] minor cleanup --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 5f8de6c06f3..11b4e89f841 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,10 +15,8 @@ script: - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - docker container ls - - docker logs sentry_onpremise_web_1 - - docker logs sentry_onpremise_nginx_1 - - docker logs sentry_onpremise_relay_1 - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - docker logs sentry_onpremise_nginx_1 - docker logs sentry_onpremise_relay_1 - docker logs sentry_onpremise_web_1 - ./test.sh From 70493630778bb27c150dfdfe1d7c5d8efcd54572 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski <5281987+RaduW@users.noreply.github.com> Date: Fri, 24 Apr 2020 11:44:24 +0200 Subject: [PATCH 23/29] Update install.sh output errors to stderr instead of stdout Co-Authored-By: Burak Yigit Kaya --- install.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/install.sh b/install.sh index 3304f5babd2..dddb014d0cc 100755 --- a/install.sh +++ b/install.sh @@ -179,10 +179,10 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then - echo "FAIL: SENTRY_RELAY_WHITELIST_PK already exists in $SENTRY_CONFIG_PY, please replace with:" - echo "" - echo " $CREDENTIALS" - echo "" + >&2 echo "FAIL: SENTRY_RELAY_WHITELIST_PK already exists in $SENTRY_CONFIG_PY, please replace with:" + >&2 echo "" + >&2 echo " $CREDENTIALS" + >&2 echo "" exit 1 fi From b2be9fafb87381f1199c2cb6a46505996afac724 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski <5281987+RaduW@users.noreply.github.com> Date: Fri, 24 Apr 2020 11:55:05 +0200 Subject: [PATCH 24/29] Update nginx/nginx.conf Set max upload size limit to support envelope chunks Co-Authored-By: Burak Yigit Kaya --- nginx/nginx.conf | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index a7b736e3113..abf1bf4e86d 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -33,7 +33,7 @@ http { listen 80; # use the docker DNS server to resolve ips for relay and sentry containers resolver 127.0.0.11 ipv6=off; - + client_max_body_size 100M; location /api/store/ { proxy_pass http://relay; } From 6474eed055d36692d91142906bf5a57b13df7bcf Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Fri, 24 Apr 2020 12:36:54 +0200 Subject: [PATCH 25/29] Forward all store-like endpoints to Relay. Update proxy config in line with: https://github.com/getsentry/sentry/pull/18433/ --- nginx/nginx.conf | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/nginx/nginx.conf b/nginx/nginx.conf index abf1bf4e86d..20ca9a0c9b4 100644 --- a/nginx/nginx.conf +++ b/nginx/nginx.conf @@ -37,22 +37,7 @@ http { location /api/store/ { proxy_pass http://relay; } - location ~ ^/api/\d+/store/$ { - proxy_pass http://relay; - } - location ~ ^/api/\d+/minidump/?$ { - proxy_pass http://relay; - } - location ~ ^/api/\d+/unreal/\w+/$ { - proxy_pass http://relay; - } - location ~ ^/api/\d+/security/$ { - proxy_pass http://relay; - } - location ~ ^/api/\d+/csp-report/$ { - proxy_pass http://relay; - } - location ~ ^/api/\d+/events/[\w-]+/attachments/$ { + location ~ ^/api/[1-9]\d*/ { proxy_pass http://relay; } location / { From 6209b0d77ad25b1db590be07048d0a5587d8ec7b Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Fri, 24 Apr 2020 13:43:11 +0200 Subject: [PATCH 26/29] fix nginx version to 1.16 --- docker-compose.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 46b4b8de8dc..ae12c1ba285 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -165,7 +165,7 @@ services: nginx: ports: - '9000:80/tcp' - image: "nginx:1.16.1" + image: "nginx:1.16" volumes: - type: bind read_only: true From 82554602d0e1acd6575077fc71fd784dcbaaf268 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Fri, 24 Apr 2020 14:05:13 +0200 Subject: [PATCH 27/29] Make public-key regex more flexible --- install.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install.sh b/install.sh index dddb014d0cc..b352c09d03a 100755 --- a/install.sh +++ b/install.sh @@ -175,7 +175,7 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then chmod a+r $RELAY_CREDENTIALS_JSON # display the contents of the relay directory (for debug purposes) ls -al ./relay - CREDENTIALS=$(sed -n 's/^.*"public_key":[[:space:]]"\([a-zA-Z0-9_-]*\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") + CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" if grep -xq SENTRY_RELAY_WHITELIST_PK "$SENTRY_CONFIG_PY"; then From 4fb77459715068ca634fc8fd7c46fe3588255d13 Mon Sep 17 00:00:00 2001 From: Radu Woinaroski Date: Fri, 24 Apr 2020 14:19:09 +0200 Subject: [PATCH 28/29] Remove debug messages --- .travis.yml | 4 ---- install.sh | 4 ---- 2 files changed, 8 deletions(-) diff --git a/.travis.yml b/.travis.yml index 11b4e89f841..8ec590d44bc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,9 +14,5 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - docker container ls - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - - docker logs sentry_onpremise_nginx_1 - - docker logs sentry_onpremise_relay_1 - - docker logs sentry_onpremise_web_1 - ./test.sh diff --git a/install.sh b/install.sh index b352c09d03a..4a122507692 100755 --- a/install.sh +++ b/install.sh @@ -173,8 +173,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then $dcr --user $(id -u) relay --config /etc/relay credentials generate --overwrite chmod a+r $RELAY_CREDENTIALS_JSON - # display the contents of the relay directory (for debug purposes) - ls -al ./relay CREDENTIALS=$(sed -n 's/^.*"public_key"[[:space:]]*:[[:space:]]*"\([a-zA-Z0-9_-]\{1,\}\)".*$/\1/p' "$RELAY_CREDENTIALS_JSON") CREDENTIALS="SENTRY_RELAY_WHITELIST_PK = [\"$CREDENTIALS\"]" @@ -188,8 +186,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "" >> "$SENTRY_CONFIG_PY" echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" - - tail -n 5 "$SENTRY_CONFIG_PY" else echo "" echo "Relay credentials exist will NOT generate new ones." From 63b5e95654ea3958a8a5814e691d6a5c5e396318 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 24 Apr 2020 15:22:09 +0300 Subject: [PATCH 29/29] Remove redundant "credentials not generated" message --- install.sh | 3 --- 1 file changed, 3 deletions(-) diff --git a/install.sh b/install.sh index 4a122507692..4ac616137ff 100755 --- a/install.sh +++ b/install.sh @@ -186,9 +186,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "" >> "$SENTRY_CONFIG_PY" echo "$CREDENTIALS" >> "$SENTRY_CONFIG_PY" -else - echo "" - echo "Relay credentials exist will NOT generate new ones." fi cleanup