From 42988e01cf9a52b1d4a8bf8de684e569301dff55 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Fri, 24 Apr 2020 14:48:52 +0300 Subject: [PATCH 01/11] test(ingest): Add e2e event ingestion test This is a long-needed test that tests the whole pipeline from Nginx, Relay, to Kafka, and Snuba. The final missing pieces is testing the Symbolicator integration. --- install.sh | 23 ++++++++++---- test.sh | 93 +++++++++++++++++++++++++++++++++++++++++++----------- 2 files changed, 92 insertions(+), 24 deletions(-) diff --git a/install.sh b/install.sh index aea57a6814..bdd73971c3 100755 --- a/install.sh +++ b/install.sh @@ -19,17 +19,31 @@ RELAY_CONFIG_YML='relay/config.yml' RELAY_CREDENTIALS_JSON='relay/credentials.json' SENTRY_EXTRA_REQUIREMENTS='sentry/requirements.txt' +# Courtesy of https://stackoverflow.com/a/2183063/90297 +trap_with_arg() { + func="$1" ; shift + for sig ; do + trap "$func $sig" "$sig" + done +} + DID_CLEAN_UP=0 # the cleanup function will be the exit point cleanup () { if [ "$DID_CLEAN_UP" -eq 1 ]; then return 0; fi - echo "Cleaning up..." - $dc stop &> /dev/null DID_CLEAN_UP=1 + + if [ "$1" != "EXIT" ]; then + echo "An error occurred, caught SIG$1"; + echo "Cleaning up..." + fi + + $dc stop &> /dev/null } -trap cleanup ERR INT TERM +trap_with_arg cleanup ERR INT TERM EXIT + echo "Checking minimum requirements..." @@ -246,9 +260,6 @@ if [ ! -f "$RELAY_CREDENTIALS_JSON" ]; then echo "Relay credentials written to $RELAY_CREDENTIALS_JSON" fi - -cleanup - echo "" echo "----------------" echo "You're all done! Run the following command to get Sentry running:" diff --git a/test.sh b/test.sh index 41800796f4..71fcd84edd 100755 --- a/test.sh +++ b/test.sh @@ -5,30 +5,87 @@ SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}" TEST_USER='test@example.com' TEST_PASS='test123TEST' COOKIE_FILE=$(mktemp) -declare -a TEST_STRINGS=( + +# Courtesy of https://stackoverflow.com/a/2183063/90297 +trap_with_arg() { + func="$1" ; shift + for sig ; do + trap "$func $sig" "$sig" + done +} + +DID_CLEAN_UP=0 +# the cleanup function will be the exit point +cleanup () { + if [ "$DID_CLEAN_UP" -eq 1 ]; then + return 0; + fi + DID_CLEAN_UP=1 + + if [ "$1" != "EXIT" ]; then + echo "An error occurred, caught SIG$1"; + fi + + echo "Cleaning up..." + rm $COOKIE_FILE + echo "Done." +} +trap_with_arg cleanup ERR INT TERM EXIT + +get_csrf_token () { awk '$6 == "sc" { print $7 }' $COOKIE_FILE; } +sentry_api_request () { curl -s -H 'Accept: application/json; charset=utf-8' -H "Referer: $SENTRY_TEST_HOST" -H 'Content-Type: application/json' -H "X-CSRFToken: $(get_csrf_token)" -b "$COOKIE_FILE" -c "$COOKIE_FILE" "$SENTRY_TEST_HOST/api/0/$1" ${@:2}; } + +login () { + INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) + if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then + echo "Initial /auth/login/ redirect failed, exiting..." + echo "$INITIAL_AUTH_REDIRECT" + exit -1 + fi + + CSRF_TOKEN_FOR_LOGIN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' + /csrfmiddlewaretoken/ { + print $4 "=" $6; + exit; + }') + + curl -sL --data-urlencode 'op=login' --data-urlencode "username=$TEST_USER" --data-urlencode "password=$TEST_PASS" --data-urlencode "$CSRF_TOKEN_FOR_LOGIN" "$SENTRY_TEST_HOST/auth/login/sentry/" -H "Referer: $SENTRY_TEST_HOST/auth/login/sentry/" -b "$COOKIE_FILE" -c "$COOKIE_FILE"; +} + +LOGIN_RESPONSE=$(login); +declare -a LOGIN_TEST_STRINGS=( '"isAuthenticated":true' - '"username":"test@example.com"' + '"username":"ben@byk.im"' '"isSuperuser":true' ) +for i in "${LOGIN_TEST_STRINGS[@]}" +do + echo "Testing '$i'..." + echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "Pass." +done -INITIAL_AUTH_REDIRECT=$(curl -sL -o /dev/null $SENTRY_TEST_HOST -w %{url_effective}) -if [ "$INITIAL_AUTH_REDIRECT" != "$SENTRY_TEST_HOST/auth/login/sentry/" ]; then - echo "Initial /auth/login/ redirect failed, exiting..." - echo "$INITIAL_AUTH_REDIRECT" - exit -1 -fi - -CSRF_TOKEN=$(curl $SENTRY_TEST_HOST -sL -c "$COOKIE_FILE" | awk -F "'" ' - /csrfmiddlewaretoken/ { - print $4 "=" $6; - exit; - }') -LOGIN_RESPONSE=$(curl -sL -F 'op=login' -F "username=$TEST_USER" -F "password=$TEST_PASS" -F "$CSRF_TOKEN" "$SENTRY_TEST_HOST/auth/login/" -H "Referer: $SENTRY_TEST_HOST/auth/login/" -b "$COOKIE_FILE" -c "$COOKIE_FILE") +# Set up initial/required settings (InstallWizard request) +sentry_api_request -X PUT "internal/options/?query=is:required" --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null -TEST_RESULT=0 -for i in "${TEST_STRINGS[@]}" +SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') + +echo $SENTRY_DSN + +TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-') +echo "Created event $TEST_EVENT_ID. Checking existence..." +sleep 3 # Give it some time to be ingested through the bowels of Sentry + +EVENT_RESPONSE=$(sentry_api_request "projects/sentry/internal/events/$TEST_EVENT_ID/") +declare -a EVENT_TEST_STRINGS=( + '"eventID":"'"$TEST_EVENT_ID"'"' + '"message":"a failure"' + '"title":"a failure"' + '"object":"42"' +) +for i in "${EVENT_TEST_STRINGS[@]}" do echo "Testing '$i'..." - echo "$LOGIN_RESPONSE" | grep "$i[,}]" >& /dev/null + echo "$EVENT_RESPONSE" | grep "$i[,}]" >& /dev/null echo "Pass." done From 5f42965096d5b8b9b2652cb55019d615f3779910 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 14:51:29 +0300 Subject: [PATCH 02/11] fix test username --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index 71fcd84edd..0cc2104d42 100755 --- a/test.sh +++ b/test.sh @@ -55,7 +55,7 @@ login () { LOGIN_RESPONSE=$(login); declare -a LOGIN_TEST_STRINGS=( '"isAuthenticated":true' - '"username":"ben@byk.im"' + '"username":"test@example.com"' '"isSuperuser":true' ) for i in "${LOGIN_TEST_STRINGS[@]}" From 4f22c59f8989b88796a23a6029324666575e5033 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 14:53:16 +0300 Subject: [PATCH 03/11] remove debugging statement --- test.sh | 2 -- 1 file changed, 2 deletions(-) diff --git a/test.sh b/test.sh index 0cc2104d42..f92398f90c 100755 --- a/test.sh +++ b/test.sh @@ -70,8 +70,6 @@ sentry_api_request -X PUT "internal/options/?query=is:required" --data '{"mail.u SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') -echo $SENTRY_DSN - TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-') echo "Created event $TEST_EVENT_ID. Checking existence..." sleep 3 # Give it some time to be ingested through the bowels of Sentry From d967cf6c89480fb51a318800a82fe45925bd014c Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 15:06:45 +0300 Subject: [PATCH 04/11] fix bash function error --- test.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test.sh b/test.sh index f92398f90c..a78cca7446 100755 --- a/test.sh +++ b/test.sh @@ -66,7 +66,7 @@ do done # Set up initial/required settings (InstallWizard request) -sentry_api_request -X PUT "internal/options/?query=is:required" --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null +sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.use-tls":false,"mail.username":"","mail.port":25,"system.admin-email":"ben@byk.im","mail.password":"","mail.from":"root@localhost","system.url-prefix":"'"$SENTRY_TEST_HOST"'","auth.allow-registration":false,"beacon.anonymous":true}' > /dev/null SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') From 068bc86b8094aa9b360ab4b464a1ed2266b412d1 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 16:09:28 +0300 Subject: [PATCH 05/11] fix relay <-> web http/1.1 issues again --- sentry/sentry.conf.example.py | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index 37218405ef..c2d12d8f56 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -185,15 +185,14 @@ def get_internal_network(): SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { - "http-keepalive": True, - "so-keepalive": True, - "http-auto-chunked": True, - "http-chunked-input": True, - # the number of web workers + 'protocol': 'http11', + 'so-keepalive": True, + 'http-keepalive': 5, + 'http-chunked-input': True, + ' the number of web workers 'workers': 3, 'threads': 4, - # Turn off memory reporting - "memory-report": False, + 'memory-report': False, # Some stuff so uwsgi will cycle workers sensibly 'max-requests': 100000, 'max-requests-delta': 500, @@ -203,7 +202,6 @@ def get_internal_network(): 'thunder-lock': True, 'log-x-forwarded-for': False, 'buffer-size': 32768, - # Relay cannot authenticate without the following 'post-buffering': 32768, 'limit-post': 209715200, 'disable-logging': True, From e05a944ccbf11b94650f15fb6a1da1997c4cdf7e Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 16:10:43 +0300 Subject: [PATCH 06/11] remove redundant post-buffering --- sentry/sentry.conf.example.py | 1 - 1 file changed, 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index c2d12d8f56..e313b0b426 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -202,7 +202,6 @@ def get_internal_network(): 'thunder-lock': True, 'log-x-forwarded-for': False, 'buffer-size': 32768, - 'post-buffering': 32768, 'limit-post': 209715200, 'disable-logging': True, 'reload-on-rss': 600, From 0143b5762464b83de175f182d1cf736b4d9dc7ee Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 16:17:50 +0300 Subject: [PATCH 07/11] typo... --- sentry/sentry.conf.example.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index e313b0b426..b9979f7a0e 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -186,7 +186,7 @@ def get_internal_network(): SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { 'protocol': 'http11', - 'so-keepalive": True, + 'so-keepalive': True, 'http-keepalive': 5, 'http-chunked-input': True, ' the number of web workers From 44368bb4f0c4054b9208a5ed911546ce028ea4e3 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 16:36:13 +0300 Subject: [PATCH 08/11] typo... --- sentry/sentry.conf.example.py | 54 +++++++++++++++++------------------ 1 file changed, 27 insertions(+), 27 deletions(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index b9979f7a0e..ccdd6fd9a3 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -12,22 +12,22 @@ def get_internal_network(): import socket import struct - iface = 'eth0' + iface = "eth0" sockfd = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) - ifreq = struct.pack('16sH14s', iface, socket.AF_INET, b'\x00' * 14) + ifreq = struct.pack("16sH14s", iface, socket.AF_INET, b"\x00" * 14) try: ip = struct.unpack( - "!I", struct.unpack('16sH2x4s8x', fcntl.ioctl(sockfd, 0x8915, ifreq))[2] + "!I", struct.unpack("16sH2x4s8x", fcntl.ioctl(sockfd, 0x8915, ifreq))[2] )[0] netmask = socket.ntohl( - struct.unpack('16sH2xI8x', fcntl.ioctl(sockfd, 0x891B, ifreq))[2] + struct.unpack("16sH2xI8x", fcntl.ioctl(sockfd, 0x891B, ifreq))[2] ) except IOError: return () base = socket.inet_ntoa(struct.pack("!I", ip & netmask)) netmask_bits = 32 - int(round(math.log(ctypes.c_uint32(~netmask).value + 1, 2), 1)) - return ('{0:s}/{1:d}'.format(base, netmask_bits),) + return ("{0:s}/{1:d}".format(base, netmask_bits),) INTERNAL_IPS = get_internal_network() @@ -60,7 +60,7 @@ def get_internal_network(): SENTRY_SINGLE_ORGANIZATION = True SENTRY_OPTIONS["system.event-retention-days"] = int( - env('SENTRY_EVENT_RETENTION_DAYS', '90') + env("SENTRY_EVENT_RETENTION_DAYS", "90") ) ######### @@ -185,29 +185,29 @@ def get_internal_network(): SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { - 'protocol': 'http11', - 'so-keepalive': True, - 'http-keepalive': 5, - 'http-chunked-input': True, - ' the number of web workers - 'workers': 3, - 'threads': 4, - 'memory-report': False, + "protocol": "http11", + "so-keepalive": True, + "http-keepalive": 5, + "http-chunked-input": True, + # the number of web workers + "workers": 3, + "threads": 4, + "memory-report": False, # Some stuff so uwsgi will cycle workers sensibly - 'max-requests': 100000, - 'max-requests-delta': 500, - 'max-worker-lifetime': 86400, + "max-requests": 100000, + "max-requests-delta": 500, + "max-worker-lifetime": 86400, # Duplicate options from sentry default just so we don't get # bit by sentry changing a default value that we depend on. - 'thunder-lock': True, - 'log-x-forwarded-for': False, - 'buffer-size': 32768, - 'limit-post': 209715200, - 'disable-logging': True, - 'reload-on-rss': 600, - 'ignore-sigpipe': True, - 'ignore-write-errors': True, - 'disable-write-exception': True, + "thunder-lock": True, + "log-x-forwarded-for": False, + "buffer-size": 32768, + "limit-post": 209715200, + "disable-logging": True, + "reload-on-rss": 600, + "ignore-sigpipe": True, + "ignore-write-errors": True, + "disable-write-exception": True, } ########### @@ -256,7 +256,7 @@ def get_internal_network(): # GitHub Integration # ###################### -GITHUB_EXTENDED_PERMISSIONS = ['repo'] +GITHUB_EXTENDED_PERMISSIONS = ["repo"] ######################### # Bitbucket Integration # From 8a550309016c60c9632d9400948456f1aa93180d Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 21:17:04 +0300 Subject: [PATCH 09/11] More robust waiting mechanism for event checks --- .travis.yml | 2 +- test.sh | 10 ++++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index 761d9986d3..eb61651167 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ script: - ./install.sh - docker-compose run --rm web createuser --superuser --email test@example.com --password test123TEST - docker-compose up -d - - timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' + - printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null http://localhost:9000); do printf '.'; sleep 0.5; done' - ./test.sh after_failure: diff --git a/test.sh b/test.sh index a78cca7446..ea757a9af5 100755 --- a/test.sh +++ b/test.sh @@ -71,8 +71,14 @@ sentry_api_request "internal/options/?query=is:required" -X PUT --data '{"mail.u SENTRY_DSN=$(sentry_api_request "projects/sentry/internal/keys/" | awk 'BEGIN { RS=",|:{\n"; FS="\""; } $2 == "public" { print $4; exit; }') TEST_EVENT_ID=$(docker run --rm --net host -e "SENTRY_DSN=$SENTRY_DSN" -v $(pwd):/work getsentry/sentry-cli send-event -m "a failure" -e task:create-user -e object:42 | tr -d '-') -echo "Created event $TEST_EVENT_ID. Checking existence..." -sleep 3 # Give it some time to be ingested through the bowels of Sentry +echo "Created event $TEST_EVENT_ID." + +EVENT_PATH="projects/sentry/internal/events/$TEST_EVENT_ID/" +export -f sentry_api_request get_csrf_token +export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH +printf "Checking its existence" +timeout 15 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done' +echo ""; EVENT_RESPONSE=$(sentry_api_request "projects/sentry/internal/events/$TEST_EVENT_ID/") declare -a EVENT_TEST_STRINGS=( From 68116df4b5270a5ceb3e6b6cc57f9f20fef6338b Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Sun, 12 Jul 2020 23:01:58 +0300 Subject: [PATCH 10/11] go back to using http plugin --- sentry/sentry.conf.example.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index ccdd6fd9a3..f295af0483 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -185,7 +185,10 @@ def get_internal_network(): SENTRY_WEB_HOST = "0.0.0.0" SENTRY_WEB_PORT = 9000 SENTRY_WEB_OPTIONS = { - "protocol": "http11", + "http": "%s:%s" % (SENTRY_WEB_HOST, SENTRY_WEB_PORT), + "protocol": "uwsgi", + # This is need to prevent https://git.io/fj7Lw + "uwsgi-socket": None, "so-keepalive": True, "http-keepalive": 5, "http-chunked-input": True, From dc9cd1e05461a4c9c58535a00e079aad1060af61 Mon Sep 17 00:00:00 2001 From: Burak Yigit Kaya Date: Mon, 13 Jul 2020 12:19:14 +0300 Subject: [PATCH 11/11] cater to relay keep-alive timeouts --- sentry/sentry.conf.example.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/sentry/sentry.conf.example.py b/sentry/sentry.conf.example.py index f295af0483..13353c1ed9 100644 --- a/sentry/sentry.conf.example.py +++ b/sentry/sentry.conf.example.py @@ -190,7 +190,8 @@ def get_internal_network(): # This is need to prevent https://git.io/fj7Lw "uwsgi-socket": None, "so-keepalive": True, - "http-keepalive": 5, + # Keep this between 15s-75s as that's what Relay supports + "http-keepalive": 15, "http-chunked-input": True, # the number of web workers "workers": 3,