From 5ef717c598588acf0db7e5e42d6f268d91f506e5 Mon Sep 17 00:00:00 2001
From: Kyle Filz <Kfilz@live.ca>
Date: Wed, 30 Jun 2021 23:50:14 -0500
Subject: [PATCH 01/14] feat: Support custom CA roots

Mount a certificate folder to local ca storage in containers,
and add update command to cron image's entrypoint.
---
 .gitignore         | 3 +++
 CHANGELOG.md       | 4 ++++
 cron/entrypoint.sh | 4 ++++
 docker-compose.yml | 1 +
 4 files changed, 12 insertions(+)

diff --git a/.gitignore b/.gitignore
index 8a169049fa5..b4c1199ef07 100644
--- a/.gitignore
+++ b/.gitignore
@@ -87,3 +87,6 @@ geoip/.geoipupdate.lock
 
 # wal2json download
 postgres/wal2json
+
+# custom certificate authorities
+certificates
diff --git a/CHANGELOG.md b/CHANGELOG.md
index f3f25f354d8..611babda868 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,5 +1,9 @@
 # Changelog
 
+## Unreleased
+
+- feat: Support custom CA roots ([#27062](https://github.com/getsentry/sentry/pull/27062)), see the [docs](https://develop.sentry.dev/self-hosted/custom-ca-roots/) for more details.
+
 ## 21.7.0
 
 - No documented changes.
diff --git a/cron/entrypoint.sh b/cron/entrypoint.sh
index baa833a77b8..383c8b29c7e 100755
--- a/cron/entrypoint.sh
+++ b/cron/entrypoint.sh
@@ -1,5 +1,9 @@
 #!/usr/bin/env bash
 
+if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then
+  update-ca-certificates
+fi
+
 # Prior art:
 # - https://git.io/fjNOg
 # - https://blog.knoldus.com/running-a-cron-job-in-docker-container/
diff --git a/docker-compose.yml b/docker-compose.yml
index 046702711a5..bbe67071647 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -42,6 +42,7 @@ x-sentry-defaults: &sentry_defaults
     - "sentry-data:/data"
     - "./sentry:/etc/sentry"
     - "./geoip:/geoip:ro"
+    - "./certificates:/usr/local/share/ca-certificates:ro"
 x-snuba-defaults: &snuba_defaults
   <<: *restart_policy
   depends_on:

From 87a5aebebbe3efa7184890cc509941e9dff7ac57 Mon Sep 17 00:00:00 2001
From: Kyle Filz <Kfilz@live.ca>
Date: Sat, 3 Jul 2021 18:20:57 -0500
Subject: [PATCH 02/14] Pre for test cleanup, move test.sh to subfolder.

---
 .github/workflows/test.yml | 4 ++--
 test.sh => test/core.sh    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)
 rename test.sh => test/core.sh (99%)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 5c239dbd127..811bdce9737 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -43,12 +43,12 @@ jobs:
         run: |
           echo "Testing initial install"
           ./install.sh
-          ./test.sh
+          ./test/core.sh
           echo "Testing in-place upgrade"
           # Also test plugin installation here
           echo "sentry-auth-oidc" >> sentry/requirements.txt
           ./install.sh --minimize-downtime
-          ./test.sh
+          ./test/core.sh
 
       - name: Inspect failure
         if: failure()
diff --git a/test.sh b/test/core.sh
similarity index 99%
rename from test.sh
rename to test/core.sh
index 37d7af56eda..4618f7b7400 100755
--- a/test.sh
+++ b/test/core.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-source "$(dirname $0)/install/_lib.sh" 
+source "$(dirname $0)/../install/_lib.sh" 
 
 echo "${_group}Setting up variables and helpers ..."
 export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"

From 93a3a1a0d45a865b97086dd18e4663e6bbbf34c9 Mon Sep 17 00:00:00 2001
From: Kyle Filz <Kfilz@live.ca>
Date: Sat, 3 Jul 2021 18:49:08 -0500
Subject: [PATCH 03/14] tests: add tests to check customs CAs

---
 .gitignore                      |  7 ++++++
 test/ca-setup.sh                | 39 +++++++++++++++++++++++++++++++++
 test/cert-test.py               | 17 ++++++++++++++
 test/core.sh                    |  5 +++++
 test/docker-compose.ca-test.yml | 14 ++++++++++++
 test/nginx/nginx.conf           | 32 +++++++++++++++++++++++++++
 6 files changed, 114 insertions(+)
 create mode 100755 test/ca-setup.sh
 create mode 100644 test/cert-test.py
 create mode 100644 test/docker-compose.ca-test.yml
 create mode 100644 test/nginx/nginx.conf

diff --git a/.gitignore b/.gitignore
index b4c1199ef07..fc5762d75ff 100644
--- a/.gitignore
+++ b/.gitignore
@@ -90,3 +90,10 @@ postgres/wal2json
 
 # custom certificate authorities
 certificates
+
+# testing for custom CAs
+sentry/cert-test.py
+test/**/*.crt
+test/**/*.key
+test/**/*.req
+test/ca.srl
diff --git a/test/ca-setup.sh b/test/ca-setup.sh
new file mode 100755
index 00000000000..013cabf4763
--- /dev/null
+++ b/test/ca-setup.sh
@@ -0,0 +1,39 @@
+#! /usr/bin/env bash
+set -e
+
+# remove old test certs
+rm -f test/nginx/self.test.* test/ca.* ../certificates/ca.crt || true
+cp test/cert-test.py sentry/
+mkdir -p certificates/
+
+# generate tighly contrained CA
+openssl req -x509 -new -nodes -newkey rsa:2048 -keyout test/ca.key \
+-sha256 -days 1 -out test/ca.crt -batch \
+-subj "/CN=TEST CA *DO NOT TRUST*" \
+-addext "keyUsage = critical, keyCertSign, cRLSign" \
+-addext "nameConstraints = critical, permitted;DNS:self.test"
+
+# openssl x509 -in test/ca.crt -text -noout
+cp test/ca.crt certificates/
+
+# generate server certificate
+openssl req -new -nodes -newkey rsa:2048 -keyout test/nginx/self.test.key \
+-addext "subjectAltName=DNS:self.test" \
+-out test/nginx/self.test.req -batch -subj "/CN=Self Signed with CA Test Server"
+
+# openssl req -in test/nginx/self.test.req -text -noout
+
+openssl x509 -req -in test/nginx/self.test.req -CA test/ca.crt -CAkey test/ca.key \
+-extfile <(printf "subjectAltName=DNS:self.test") \
+-CAcreateserial -out test/nginx/self.test.crt -days 1 -sha256 
+
+# openssl x509 -in test/nginx/self.test.crt -text -noout
+
+# sanity check that signed certificate passes OpenSSL's validation
+openssl verify -CAfile test/ca.crt test/nginx/self.test.crt
+
+# self signed certificate, for sanity check of not just accepting all certs
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout test/nginx/fake.test.key \
+-out test/nginx/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
+
+# openssl x509 -in test/nginx/fake.test.crt -text -noout
diff --git a/test/cert-test.py b/test/cert-test.py
new file mode 100644
index 00000000000..fbad281b3a2
--- /dev/null
+++ b/test/cert-test.py
@@ -0,0 +1,17 @@
+import requests
+try:
+  value = requests.get("https://self.test").text
+  if value != 'ok':
+    print('Got something other than ok: ' + value)
+    exit(1)
+  print('Custom CA worked.')
+except:
+  print('Custom CA cert failed to work.')
+  exit(1)
+try:
+  requests.get("https://fail.test")
+  print('Accepted a self signed cert! That\'s bad.')
+except:
+  print('Self signed cert didn\'t work, which is good.')
+  exit(0)
+exit(1)
diff --git a/test/core.sh b/test/core.sh
index 4618f7b7400..016fb52d0e1 100755
--- a/test/core.sh
+++ b/test/core.sh
@@ -1,6 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
+COMPOSE_FILE=docker-compose.yml:test/docker-compose.ca-test.yml
 source "$(dirname $0)/../install/_lib.sh" 
 
 echo "${_group}Setting up variables and helpers ..."
@@ -119,3 +120,7 @@ echo "${_endgroup}"
 echo "${_group}Ensure cleanup crons are working ..."
 $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+"
 echo "${_endgroup}"
+
+echo "${_group}Test customs CAs work ..."
+$dc exec -T web python3 /etc/sentry/cert-test.py
+echo "${_endgroup}"
diff --git a/test/docker-compose.ca-test.yml b/test/docker-compose.ca-test.yml
new file mode 100644
index 00000000000..3fe5cb934ff
--- /dev/null
+++ b/test/docker-compose.ca-test.yml
@@ -0,0 +1,14 @@
+version: '3.9'
+services:
+  test:
+    image: nginx:1.21.0-alpine 
+    restart: unless-stopped
+    volumes:
+      - ./test/nginx:/etc/nginx:ro
+    networks:
+      default:
+        aliases:
+          - self.test
+          - fail.test
+  geoipupdate:
+    scale: 0
diff --git a/test/nginx/nginx.conf b/test/nginx/nginx.conf
new file mode 100644
index 00000000000..517aea41029
--- /dev/null
+++ b/test/nginx/nginx.conf
@@ -0,0 +1,32 @@
+user nginx;
+worker_processes 1;
+
+error_log /var/log/nginx/error.log warn;
+pid /var/run/nginx.pid;
+
+events {
+	worker_connections 1024;
+}
+
+http {
+  server {
+    listen               443 ssl;
+    server_name          "self.test";
+    ssl_certificate      "/etc/nginx/self.test.crt";
+    ssl_certificate_key  "/etc/nginx/self.test.key";
+    location / {
+      add_header Content-Type text/plain;
+      return 200 'ok';
+    }
+  }
+  server {
+    listen               443 ssl;
+    server_name          "fake.test";
+    ssl_certificate      "/etc/nginx/fake.test.crt";
+    ssl_certificate_key  "/etc/nginx/fake.test.key";
+    location / {
+      add_header Content-Type text/plain;
+      return 200 'bad';
+    }
+  }
+}

From 4545990cf4a5a0ccb4e0c5fb7faa1cb04aae3631 Mon Sep 17 00:00:00 2001
From: Kyle Filz <Kfilz@live.ca>
Date: Sat, 3 Jul 2021 18:54:51 -0500
Subject: [PATCH 04/14] Add test clean up script.

---
 test/cleanup.sh | 2 ++
 1 file changed, 2 insertions(+)
 create mode 100644 test/cleanup.sh

diff --git a/test/cleanup.sh b/test/cleanup.sh
new file mode 100644
index 00000000000..01a617fe271
--- /dev/null
+++ b/test/cleanup.sh
@@ -0,0 +1,2 @@
+#! bin/sh
+rm -f test/ca.* certificates/* test/nginx/*.test.* sentry/cert-test.py 

From 600becb29afed704cd56fcce353c9533c823454e Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chadwhitacre@sentry.io>
Date: Fri, 16 Jul 2021 18:25:12 -0400
Subject: [PATCH 05/14] Refactor

---
 .github/workflows/test.yml                    | 12 +++---
 .gitignore                                    |  8 ++--
 {test => _integration_tests}/cert-test.py     |  3 ++
 .../docker-compose.test.yml                   |  4 +-
 {test => _integration_tests}/nginx/nginx.conf |  0
 test/core.sh => _integration_tests/run.sh     |  5 ++-
 _integration_tests/setup.sh                   | 43 +++++++++++++++++++
 _integration_tests/teardown.sh                |  3 ++
 install/_lib.sh                               |  2 +-
 test/ca-setup.sh                              | 39 -----------------
 test/cleanup.sh                               |  2 -
 11 files changed, 64 insertions(+), 57 deletions(-)
 rename {test => _integration_tests}/cert-test.py (90%)
 rename test/docker-compose.ca-test.yml => _integration_tests/docker-compose.test.yml (76%)
 rename {test => _integration_tests}/nginx/nginx.conf (100%)
 rename test/core.sh => _integration_tests/run.sh (97%)
 create mode 100755 _integration_tests/setup.sh
 create mode 100755 _integration_tests/teardown.sh
 delete mode 100755 test/ca-setup.sh
 delete mode 100644 test/cleanup.sh

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 811bdce9737..97456dae371 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -14,7 +14,7 @@ defaults:
   run:
     shell: bash
 jobs:
-  unit-test:
+  run-unit-tests:
     runs-on: ubuntu-18.04
     name: "unit tests"
     steps:
@@ -25,9 +25,9 @@ jobs:
         working-directory: install
         run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
 
-  integration-test:
+  run-integration-tests:
     runs-on: ubuntu-18.04
-    name: "test"
+    name: "integration tests"
     steps:
       - name: Pin docker-compose
         run: |
@@ -39,16 +39,16 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Integration Test
+      - name: Integration Tests
         run: |
           echo "Testing initial install"
           ./install.sh
-          ./test/core.sh
+          ./_integration_tests/run.sh
           echo "Testing in-place upgrade"
           # Also test plugin installation here
           echo "sentry-auth-oidc" >> sentry/requirements.txt
           ./install.sh --minimize-downtime
-          ./test/core.sh
+          ./_integration_tests/run.sh
 
       - name: Inspect failure
         if: failure()
diff --git a/.gitignore b/.gitignore
index fc5762d75ff..e110eccc622 100644
--- a/.gitignore
+++ b/.gitignore
@@ -91,9 +91,7 @@ postgres/wal2json
 # custom certificate authorities
 certificates
 
-# testing for custom CAs
+# integration testing
 sentry/cert-test.py
-test/**/*.crt
-test/**/*.key
-test/**/*.req
-test/ca.srl
+_integration_tests/tmp/*
+_integration_tests/nginx/*
diff --git a/test/cert-test.py b/_integration_tests/cert-test.py
similarity index 90%
rename from test/cert-test.py
rename to _integration_tests/cert-test.py
index fbad281b3a2..b16e8e17490 100644
--- a/test/cert-test.py
+++ b/_integration_tests/cert-test.py
@@ -1,4 +1,6 @@
 import requests
+import traceback
+
 try:
   value = requests.get("https://self.test").text
   if value != 'ok':
@@ -7,6 +9,7 @@
   print('Custom CA worked.')
 except:
   print('Custom CA cert failed to work.')
+  traceback.print_exc()
   exit(1)
 try:
   requests.get("https://fail.test")
diff --git a/test/docker-compose.ca-test.yml b/_integration_tests/docker-compose.test.yml
similarity index 76%
rename from test/docker-compose.ca-test.yml
rename to _integration_tests/docker-compose.test.yml
index 3fe5cb934ff..15454775e92 100644
--- a/test/docker-compose.ca-test.yml
+++ b/_integration_tests/docker-compose.test.yml
@@ -1,10 +1,10 @@
 version: '3.9'
 services:
-  test:
+  cert-fixture:
     image: nginx:1.21.0-alpine 
     restart: unless-stopped
     volumes:
-      - ./test/nginx:/etc/nginx:ro
+      - ./_integration_tests/nginx:/etc/nginx:ro
     networks:
       default:
         aliases:
diff --git a/test/nginx/nginx.conf b/_integration_tests/nginx/nginx.conf
similarity index 100%
rename from test/nginx/nginx.conf
rename to _integration_tests/nginx/nginx.conf
diff --git a/test/core.sh b/_integration_tests/run.sh
similarity index 97%
rename from test/core.sh
rename to _integration_tests/run.sh
index 016fb52d0e1..a9b9c0fc734 100755
--- a/test/core.sh
+++ b/_integration_tests/run.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-COMPOSE_FILE=docker-compose.yml:test/docker-compose.ca-test.yml
+export COMPOSE_FILE=../docker-compose.yml:docker-compose.test.yml
 source "$(dirname $0)/../install/_lib.sh" 
 
 echo "${_group}Setting up variables and helpers ..."
@@ -121,6 +121,7 @@ echo "${_group}Ensure cleanup crons are working ..."
 $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+"
 echo "${_endgroup}"
 
-echo "${_group}Test customs CAs work ..."
+echo "${_group}Test custom CAs work ..."
+$dc up -d cert-fixture
 $dc exec -T web python3 /etc/sentry/cert-test.py
 echo "${_endgroup}"
diff --git a/_integration_tests/setup.sh b/_integration_tests/setup.sh
new file mode 100755
index 00000000000..3b4b564cc47
--- /dev/null
+++ b/_integration_tests/setup.sh
@@ -0,0 +1,43 @@
+#! /usr/bin/env bash
+set -e
+
+cd $(dirname "$0")
+./teardown.sh
+mkdir -p ../certificates/
+cp cert-test.py ../sentry/
+
+# generate tightly constrained CA
+# NB: `-addext` requires LibreSSL 3.1.0+, or OpenSSL (brew install openssl)
+openssl req -x509 -new -nodes -newkey rsa:2048 -keyout nginx/ca.key \
+-sha256 -days 1 -out nginx/ca.crt -batch \
+-subj "/CN=TEST CA *DO NOT TRUST*" \
+-addext "keyUsage = critical, keyCertSign, cRLSign" \
+-addext "nameConstraints = critical, permitted;DNS:self.test"
+
+## Lines like the following are debug helpers ...
+# openssl x509 -in nginx/ca.crt -text -noout
+
+mkdir -p ../certificates/
+cp nginx/ca.crt ../certificates/integration-test.crt
+
+# generate server certificate
+openssl req -new -nodes -newkey rsa:2048 -keyout nginx/self.test.key \
+-addext "subjectAltName=DNS:self.test" \
+-out nginx/self.test.req -batch -subj "/CN=Self Signed with CA Test Server"
+
+# openssl req -in nginx/self.test.req -text -noout
+
+openssl x509 -req -in nginx/self.test.req -CA nginx/ca.crt -CAkey nginx/ca.key \
+-extfile <(printf "subjectAltName=DNS:self.test") \
+-CAcreateserial -out nginx/self.test.crt -days 1 -sha256 
+
+# openssl x509 -in nginx/self.test.crt -text -noout
+
+# sanity check that signed certificate passes OpenSSL's validation
+openssl verify -CAfile nginx/ca.crt nginx/self.test.crt
+
+# self signed certificate, for sanity check of not just accepting all certs
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout nginx/fake.test.key \
+-out nginx/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
+
+# openssl x509 -in nginx/fake.test.crt -text -noout
diff --git a/_integration_tests/teardown.sh b/_integration_tests/teardown.sh
new file mode 100755
index 00000000000..efe55e1f5b8
--- /dev/null
+++ b/_integration_tests/teardown.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+cd $(dirname "$0")
+rm -f ./tmp/* ../certificates/integration-test.crt ../sentry/cert-test.py
diff --git a/install/_lib.sh b/install/_lib.sh
index 2d7517fdc65..136162b8cfa 100644
--- a/install/_lib.sh
+++ b/install/_lib.sh
@@ -9,7 +9,7 @@ exec &> >(tee -a "$log_file")
 if [[ "$(basename $0)" = "install.sh" || "$(basename $0)" = "test.sh" ]]; then
   cd "$(dirname $0)/install/"
 else
-  cd "$(dirname $0)"  # assume we're a *-test.sh script
+  cd "$(dirname $0)"  # assume we're a test script or some such
 fi
 
 _ENV="$(realpath ../.env)"
diff --git a/test/ca-setup.sh b/test/ca-setup.sh
deleted file mode 100755
index 013cabf4763..00000000000
--- a/test/ca-setup.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#! /usr/bin/env bash
-set -e
-
-# remove old test certs
-rm -f test/nginx/self.test.* test/ca.* ../certificates/ca.crt || true
-cp test/cert-test.py sentry/
-mkdir -p certificates/
-
-# generate tighly contrained CA
-openssl req -x509 -new -nodes -newkey rsa:2048 -keyout test/ca.key \
--sha256 -days 1 -out test/ca.crt -batch \
--subj "/CN=TEST CA *DO NOT TRUST*" \
--addext "keyUsage = critical, keyCertSign, cRLSign" \
--addext "nameConstraints = critical, permitted;DNS:self.test"
-
-# openssl x509 -in test/ca.crt -text -noout
-cp test/ca.crt certificates/
-
-# generate server certificate
-openssl req -new -nodes -newkey rsa:2048 -keyout test/nginx/self.test.key \
--addext "subjectAltName=DNS:self.test" \
--out test/nginx/self.test.req -batch -subj "/CN=Self Signed with CA Test Server"
-
-# openssl req -in test/nginx/self.test.req -text -noout
-
-openssl x509 -req -in test/nginx/self.test.req -CA test/ca.crt -CAkey test/ca.key \
--extfile <(printf "subjectAltName=DNS:self.test") \
--CAcreateserial -out test/nginx/self.test.crt -days 1 -sha256 
-
-# openssl x509 -in test/nginx/self.test.crt -text -noout
-
-# sanity check that signed certificate passes OpenSSL's validation
-openssl verify -CAfile test/ca.crt test/nginx/self.test.crt
-
-# self signed certificate, for sanity check of not just accepting all certs
-openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout test/nginx/fake.test.key \
--out test/nginx/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
-
-# openssl x509 -in test/nginx/fake.test.crt -text -noout
diff --git a/test/cleanup.sh b/test/cleanup.sh
deleted file mode 100644
index 01a617fe271..00000000000
--- a/test/cleanup.sh
+++ /dev/null
@@ -1,2 +0,0 @@
-#! bin/sh
-rm -f test/ca.* certificates/* test/nginx/*.test.* sentry/cert-test.py 

From 1ac17f468ac0843dfcdf70a696f2cd8358fb8b06 Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chadwhitacre@sentry.io>
Date: Mon, 19 Jul 2021 08:29:41 -0400
Subject: [PATCH 06/14] Achieve parity between CI and local

- Match docker-compose file versions - Not getting an error locally with
  docker-compose 1.29, but I am seeing one in CI where we are pinned to
1.24.

- Drop scale usage - `scale` is a v2 construct. Merged v2/v3 support was
  added in docker-compose 1.27, but we're still pinned to 1.24 in CI.
---
 .github/workflows/test.yml                    | 13 +++++++-----
 .gitignore                                    |  5 ++---
 .../custom-ca-roots}/docker-compose.test.yml  |  8 +++-----
 .../custom-ca-roots}/nginx/nginx.conf         |  0
 .../custom-ca-roots}/setup.sh                 |  8 ++++----
 _integration-test/custom-ca-roots/teardown.sh |  3 +++
 _integration-test/custom-ca-roots/test.py     | 20 +++++++++++++++++++
 .../run.sh                                    | 10 ++++++----
 _integration_tests/cert-test.py               | 20 -------------------
 _integration_tests/teardown.sh                |  3 ---
 10 files changed, 46 insertions(+), 44 deletions(-)
 rename {_integration_tests => _integration-test/custom-ca-roots}/docker-compose.test.yml (61%)
 rename {_integration_tests => _integration-test/custom-ca-roots}/nginx/nginx.conf (100%)
 rename {_integration_tests => _integration-test/custom-ca-roots}/setup.sh (91%)
 create mode 100755 _integration-test/custom-ca-roots/teardown.sh
 create mode 100644 _integration-test/custom-ca-roots/test.py
 rename {_integration_tests => _integration-test}/run.sh (95%)
 delete mode 100644 _integration_tests/cert-test.py
 delete mode 100755 _integration_tests/teardown.sh

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 97456dae371..311fdaf8b49 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -25,9 +25,9 @@ jobs:
         working-directory: install
         run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
 
-  run-integration-tests:
+  run-integration-test:
     runs-on: ubuntu-18.04
-    name: "integration tests"
+    name: "integration test"
     steps:
       - name: Pin docker-compose
         run: |
@@ -39,16 +39,19 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Integration Tests
+      - name: Integration Test
         run: |
           echo "Testing initial install"
+          # Create ./certificates here because install.sh will create it with root:root
+          # and then run.sh (-> setup.sh) won't be able to write to it.
+          mkdir certificates
           ./install.sh
-          ./_integration_tests/run.sh
+          ./_integration-test/run.sh
           echo "Testing in-place upgrade"
           # Also test plugin installation here
           echo "sentry-auth-oidc" >> sentry/requirements.txt
           ./install.sh --minimize-downtime
-          ./_integration_tests/run.sh
+          ./_integration-test/run.sh
 
       - name: Inspect failure
         if: failure()
diff --git a/.gitignore b/.gitignore
index e110eccc622..c8967cdfd61 100644
--- a/.gitignore
+++ b/.gitignore
@@ -92,6 +92,5 @@ postgres/wal2json
 certificates
 
 # integration testing
-sentry/cert-test.py
-_integration_tests/tmp/*
-_integration_tests/nginx/*
+_integration-test/custom-ca-roots/nginx/*
+sentry/test-custom-ca-roots.py
diff --git a/_integration_tests/docker-compose.test.yml b/_integration-test/custom-ca-roots/docker-compose.test.yml
similarity index 61%
rename from _integration_tests/docker-compose.test.yml
rename to _integration-test/custom-ca-roots/docker-compose.test.yml
index 15454775e92..2e730493b63 100644
--- a/_integration_tests/docker-compose.test.yml
+++ b/_integration-test/custom-ca-roots/docker-compose.test.yml
@@ -1,14 +1,12 @@
-version: '3.9'
+version: '3.4'
 services:
-  cert-fixture:
+  fixture-custom-ca-roots:
     image: nginx:1.21.0-alpine 
     restart: unless-stopped
     volumes:
-      - ./_integration_tests/nginx:/etc/nginx:ro
+      - ./_integration-test/custom-ca-roots/nginx:/etc/nginx:ro
     networks:
       default:
         aliases:
           - self.test
           - fail.test
-  geoipupdate:
-    scale: 0
diff --git a/_integration_tests/nginx/nginx.conf b/_integration-test/custom-ca-roots/nginx/nginx.conf
similarity index 100%
rename from _integration_tests/nginx/nginx.conf
rename to _integration-test/custom-ca-roots/nginx/nginx.conf
diff --git a/_integration_tests/setup.sh b/_integration-test/custom-ca-roots/setup.sh
similarity index 91%
rename from _integration_tests/setup.sh
rename to _integration-test/custom-ca-roots/setup.sh
index 3b4b564cc47..c2ac474d3b9 100755
--- a/_integration_tests/setup.sh
+++ b/_integration-test/custom-ca-roots/setup.sh
@@ -3,8 +3,6 @@ set -e
 
 cd $(dirname "$0")
 ./teardown.sh
-mkdir -p ../certificates/
-cp cert-test.py ../sentry/
 
 # generate tightly constrained CA
 # NB: `-addext` requires LibreSSL 3.1.0+, or OpenSSL (brew install openssl)
@@ -17,8 +15,8 @@ openssl req -x509 -new -nodes -newkey rsa:2048 -keyout nginx/ca.key \
 ## Lines like the following are debug helpers ...
 # openssl x509 -in nginx/ca.crt -text -noout
 
-mkdir -p ../certificates/
-cp nginx/ca.crt ../certificates/integration-test.crt
+mkdir -p ../../certificates/
+cp nginx/ca.crt ../../certificates/test-custom-ca-roots.crt
 
 # generate server certificate
 openssl req -new -nodes -newkey rsa:2048 -keyout nginx/self.test.key \
@@ -41,3 +39,5 @@ openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout nginx/fake.test.key \
 -out nginx/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
 
 # openssl x509 -in nginx/fake.test.crt -text -noout
+
+cp test.py ../../sentry/test-custom-ca-roots.py
diff --git a/_integration-test/custom-ca-roots/teardown.sh b/_integration-test/custom-ca-roots/teardown.sh
new file mode 100755
index 00000000000..4b1ebc37f0e
--- /dev/null
+++ b/_integration-test/custom-ca-roots/teardown.sh
@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+cd $(dirname "$0")
+rm -f ../../certificates/test-custom-ca-roots.crt ../../sentry/test-custom-ca-roots.py
diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py
new file mode 100644
index 00000000000..effa7820501
--- /dev/null
+++ b/_integration-test/custom-ca-roots/test.py
@@ -0,0 +1,20 @@
+import requests
+import traceback
+
+try:
+    value = requests.get("https://self.test").text
+    if value != 'ok':
+        print('Got something other than ok: ' + value)
+        exit(1)
+    print('Custom CA worked.')
+except:
+    print('Custom CA cert failed to work.')
+    traceback.print_exc()
+    exit(1)
+try:
+    requests.get("https://fail.test")
+    print('Accepted a self signed cert! That\'s bad.')
+except:
+    print('Self signed cert didn\'t work, which is good.')
+    exit(0)
+exit(1)
diff --git a/_integration_tests/run.sh b/_integration-test/run.sh
similarity index 95%
rename from _integration_tests/run.sh
rename to _integration-test/run.sh
index a9b9c0fc734..f21a615d7e3 100755
--- a/_integration_tests/run.sh
+++ b/_integration-test/run.sh
@@ -1,11 +1,11 @@
 #!/usr/bin/env bash
 set -e
 
-export COMPOSE_FILE=../docker-compose.yml:docker-compose.test.yml
 source "$(dirname $0)/../install/_lib.sh" 
 
 echo "${_group}Setting up variables and helpers ..."
 export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"
+export COMPOSE_FILE=../docker-compose.yml:custom-ca-roots/docker-compose.test.yml
 TEST_USER='test@example.com'
 TEST_PASS='test123TEST'
 COOKIE_FILE=$(mktemp)
@@ -100,7 +100,7 @@ export -f sentry_api_request get_csrf_token
 export SENTRY_TEST_HOST COOKIE_FILE EVENT_PATH
 printf "Getting the test event back"
 timeout 30 bash -c 'until $(sentry_api_request "$EVENT_PATH" -Isf -X GET -o /dev/null); do printf '.'; sleep 0.5; done'
-echo "";
+echo " got it!";
 
 EVENT_RESPONSE=$(sentry_api_request "$EVENT_PATH")
 declare -a EVENT_TEST_STRINGS=(
@@ -122,6 +122,8 @@ $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+"
 echo "${_endgroup}"
 
 echo "${_group}Test custom CAs work ..."
-$dc up -d cert-fixture
-$dc exec -T web python3 /etc/sentry/cert-test.py
+./custom-ca-roots/setup.sh
+$dc up -d fixture-custom-ca-roots
+$dc exec -T web python3 /etc/sentry/test-custom-ca-roots.py
+./custom-ca-roots/teardown.sh
 echo "${_endgroup}"
diff --git a/_integration_tests/cert-test.py b/_integration_tests/cert-test.py
deleted file mode 100644
index b16e8e17490..00000000000
--- a/_integration_tests/cert-test.py
+++ /dev/null
@@ -1,20 +0,0 @@
-import requests
-import traceback
-
-try:
-  value = requests.get("https://self.test").text
-  if value != 'ok':
-    print('Got something other than ok: ' + value)
-    exit(1)
-  print('Custom CA worked.')
-except:
-  print('Custom CA cert failed to work.')
-  traceback.print_exc()
-  exit(1)
-try:
-  requests.get("https://fail.test")
-  print('Accepted a self signed cert! That\'s bad.')
-except:
-  print('Self signed cert didn\'t work, which is good.')
-  exit(0)
-exit(1)
diff --git a/_integration_tests/teardown.sh b/_integration_tests/teardown.sh
deleted file mode 100755
index efe55e1f5b8..00000000000
--- a/_integration_tests/teardown.sh
+++ /dev/null
@@ -1,3 +0,0 @@
-#!/usr/bin/env bash
-cd $(dirname "$0")
-rm -f ./tmp/* ../certificates/integration-test.crt ../sentry/cert-test.py

From b85504ade3bc9f8db8df4fcfbc890655ed807ba9 Mon Sep 17 00:00:00 2001
From: Chad Whitacre <chadwhitacre@sentry.io>
Date: Tue, 27 Jul 2021 11:17:45 -0400
Subject: [PATCH 07/14] Chad's changes

---
 .env                                          |  2 +-
 .github/workflows/test.yml                    |  6 ++++--
 .../custom-ca-roots/docker-compose.test.yml   |  2 +-
 _integration-test/custom-ca-roots/setup.sh    |  2 +-
 _integration-test/custom-ca-roots/test.py     |  8 +++++++
 _integration-test/run.sh                      | 21 +++++++++++++++----
 6 files changed, 32 insertions(+), 9 deletions(-)

diff --git a/.env b/.env
index d49dae9c4f8..8c78622b1b2 100644
--- a/.env
+++ b/.env
@@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90
 # You can either use a port number or an IP:PORT combo for SENTRY_BIND
 # See https://docs.docker.com/compose/compose-file/#ports for more
 SENTRY_BIND=9000
-SENTRY_IMAGE=getsentry/sentry:nightly
+SENTRY_IMAGE='us.gcr.io/sentryio/sentry:449412b925fbef18963f3b0bb95133e8caa82841'
 SNUBA_IMAGE=getsentry/snuba:nightly
 RELAY_IMAGE=getsentry/relay:nightly
 SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 311fdaf8b49..3b47bb3d09e 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,11 +10,13 @@ on:
   pull_request:
 env:
   DOCKER_COMPOSE_VERSION: 1.24.1
+  # TODO: DELETE ME BEFORE MERGING
+  SENTRY_IMAGE: 'us.gcr.io/sentryio/sentry:449412b925fbef18963f3b0bb95133e8caa82841'
 defaults:
   run:
     shell: bash
 jobs:
-  run-unit-tests:
+  unit-test:
     runs-on: ubuntu-18.04
     name: "unit tests"
     steps:
@@ -25,7 +27,7 @@ jobs:
         working-directory: install
         run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
 
-  run-integration-test:
+  integration-test:
     runs-on: ubuntu-18.04
     name: "integration test"
     steps:
diff --git a/_integration-test/custom-ca-roots/docker-compose.test.yml b/_integration-test/custom-ca-roots/docker-compose.test.yml
index 2e730493b63..2bc40ba1b16 100644
--- a/_integration-test/custom-ca-roots/docker-compose.test.yml
+++ b/_integration-test/custom-ca-roots/docker-compose.test.yml
@@ -1,7 +1,7 @@
 version: '3.4'
 services:
   fixture-custom-ca-roots:
-    image: nginx:1.21.0-alpine 
+    image: nginx:1.21.0-alpine
     restart: unless-stopped
     volumes:
       - ./_integration-test/custom-ca-roots/nginx:/etc/nginx:ro
diff --git a/_integration-test/custom-ca-roots/setup.sh b/_integration-test/custom-ca-roots/setup.sh
index c2ac474d3b9..1ba6724bfb6 100755
--- a/_integration-test/custom-ca-roots/setup.sh
+++ b/_integration-test/custom-ca-roots/setup.sh
@@ -27,7 +27,7 @@ openssl req -new -nodes -newkey rsa:2048 -keyout nginx/self.test.key \
 
 openssl x509 -req -in nginx/self.test.req -CA nginx/ca.crt -CAkey nginx/ca.key \
 -extfile <(printf "subjectAltName=DNS:self.test") \
--CAcreateserial -out nginx/self.test.crt -days 1 -sha256 
+-CAcreateserial -out nginx/self.test.crt -days 1 -sha256
 
 # openssl x509 -in nginx/self.test.crt -text -noout
 
diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py
index effa7820501..3bb679dfa8c 100644
--- a/_integration-test/custom-ca-roots/test.py
+++ b/_integration-test/custom-ca-roots/test.py
@@ -1,6 +1,14 @@
 import requests
 import traceback
 
+# ===========================================
+import os
+
+for k,v in sorted(os.environ.items()):
+    print(f'{k:<24} {v}')
+# ===========================================
+
+
 try:
     value = requests.get("https://self.test").text
     if value != 'ok':
diff --git a/_integration-test/run.sh b/_integration-test/run.sh
index f21a615d7e3..cdb718beb9b 100755
--- a/_integration-test/run.sh
+++ b/_integration-test/run.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -e
 
-source "$(dirname $0)/../install/_lib.sh" 
+source "$(dirname $0)/../install/_lib.sh"
 
 echo "${_group}Setting up variables and helpers ..."
 export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"
@@ -122,8 +122,21 @@ $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+"
 echo "${_endgroup}"
 
 echo "${_group}Test custom CAs work ..."
+$dc down
 ./custom-ca-roots/setup.sh
-$dc up -d fixture-custom-ca-roots
-$dc exec -T web python3 /etc/sentry/test-custom-ca-roots.py
-./custom-ca-roots/teardown.sh
+$dc up -d
+$dc exec -T web python3 /etc/sentry/test-custom-ca-roots.py || true
+#./custom-ca-roots/teardown.sh
 echo "${_endgroup}"
+
+dref() { docker ps --format "table {{.ID}}\t{{.Names}}" | grep "$1" | cut -d ' ' -f1; }
+dcat() { printf "\n\n\n$2\n\n" && docker container exec -t $(dref $1) cat $2; }
+
+echo "${_group}DEBUGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"
+cat ../certificates/test-custom-ca-roots.crt
+dcat fixture  /etc/nginx/ca.crt
+dcat web      /usr/local/share/ca-certificates/test-custom-ca-roots.crt
+dcat web      /etc/ssl/certs/ca-certificates.crt | tail -n32
+echo "${_endgroup}"
+
+exit 1

From 3f713c27439e8e97cefb5d0a41e3e0ad99a3c0e6 Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Wed, 28 Jul 2021 16:29:04 +0300
Subject: [PATCH 08/14] BYK's changes

---
 .env                                          |  2 +-
 .github/workflows/test.yml                    |  2 --
 _integration-test/custom-ca-roots/setup.sh    | 35 +++++++++++--------
 _integration-test/custom-ca-roots/teardown.sh |  5 +--
 _integration-test/custom-ca-roots/test.py     |  9 +----
 _integration-test/run.sh                      | 22 +++---------
 docker-compose.yml                            |  4 ++-
 install/_lib.sh                               |  2 +-
 sentry/entrypoint.sh                          |  4 +++
 9 files changed, 38 insertions(+), 47 deletions(-)

diff --git a/.env b/.env
index 8c78622b1b2..d49dae9c4f8 100644
--- a/.env
+++ b/.env
@@ -3,7 +3,7 @@ SENTRY_EVENT_RETENTION_DAYS=90
 # You can either use a port number or an IP:PORT combo for SENTRY_BIND
 # See https://docs.docker.com/compose/compose-file/#ports for more
 SENTRY_BIND=9000
-SENTRY_IMAGE='us.gcr.io/sentryio/sentry:449412b925fbef18963f3b0bb95133e8caa82841'
+SENTRY_IMAGE=getsentry/sentry:nightly
 SNUBA_IMAGE=getsentry/snuba:nightly
 RELAY_IMAGE=getsentry/relay:nightly
 SYMBOLICATOR_IMAGE=getsentry/symbolicator:nightly
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 3b47bb3d09e..8e1664cd035 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -10,8 +10,6 @@ on:
   pull_request:
 env:
   DOCKER_COMPOSE_VERSION: 1.24.1
-  # TODO: DELETE ME BEFORE MERGING
-  SENTRY_IMAGE: 'us.gcr.io/sentryio/sentry:449412b925fbef18963f3b0bb95133e8caa82841'
 defaults:
   run:
     shell: bash
diff --git a/_integration-test/custom-ca-roots/setup.sh b/_integration-test/custom-ca-roots/setup.sh
index 1ba6724bfb6..9832869cb3b 100755
--- a/_integration-test/custom-ca-roots/setup.sh
+++ b/_integration-test/custom-ca-roots/setup.sh
@@ -1,13 +1,16 @@
 #! /usr/bin/env bash
 set -e
 
-cd $(dirname "$0")
-./teardown.sh
+export RANDFILE="$RUNNER_TEMP/.rnd"
+export COMPOSE_FILE="../docker-compose.yml:./custom-ca-roots/docker-compose.test.yml"
+
+TEST_NGINX_CONF_PATH="./custom-ca-roots/nginx"
+CUSTOM_CERTS_PATH="../certificates"
 
 # generate tightly constrained CA
 # NB: `-addext` requires LibreSSL 3.1.0+, or OpenSSL (brew install openssl)
-openssl req -x509 -new -nodes -newkey rsa:2048 -keyout nginx/ca.key \
--sha256 -days 1 -out nginx/ca.crt -batch \
+openssl req -x509 -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/ca.key \
+-sha256 -days 1 -out $TEST_NGINX_CONF_PATH/ca.crt -batch \
 -subj "/CN=TEST CA *DO NOT TRUST*" \
 -addext "keyUsage = critical, keyCertSign, cRLSign" \
 -addext "nameConstraints = critical, permitted;DNS:self.test"
@@ -15,29 +18,33 @@ openssl req -x509 -new -nodes -newkey rsa:2048 -keyout nginx/ca.key \
 ## Lines like the following are debug helpers ...
 # openssl x509 -in nginx/ca.crt -text -noout
 
-mkdir -p ../../certificates/
-cp nginx/ca.crt ../../certificates/test-custom-ca-roots.crt
+mkdir -p $CUSTOM_CERTS_PATH
+cp $TEST_NGINX_CONF_PATH/ca.crt $CUSTOM_CERTS_PATH/test-custom-ca-roots.crt
 
 # generate server certificate
-openssl req -new -nodes -newkey rsa:2048 -keyout nginx/self.test.key \
+openssl req -new -nodes -newkey rsa:2048 -keyout $TEST_NGINX_CONF_PATH/self.test.key \
 -addext "subjectAltName=DNS:self.test" \
--out nginx/self.test.req -batch -subj "/CN=Self Signed with CA Test Server"
+-out $TEST_NGINX_CONF_PATH/self.test.req -batch -subj "/CN=Self Signed with CA Test Server"
 
 # openssl req -in nginx/self.test.req -text -noout
 
-openssl x509 -req -in nginx/self.test.req -CA nginx/ca.crt -CAkey nginx/ca.key \
+openssl x509 -req -in $TEST_NGINX_CONF_PATH/self.test.req -CA $TEST_NGINX_CONF_PATH/ca.crt -CAkey $TEST_NGINX_CONF_PATH/ca.key \
 -extfile <(printf "subjectAltName=DNS:self.test") \
--CAcreateserial -out nginx/self.test.crt -days 1 -sha256
+-CAcreateserial -out $TEST_NGINX_CONF_PATH/self.test.crt -days 1 -sha256
 
 # openssl x509 -in nginx/self.test.crt -text -noout
 
 # sanity check that signed certificate passes OpenSSL's validation
-openssl verify -CAfile nginx/ca.crt nginx/self.test.crt
+openssl verify -CAfile $TEST_NGINX_CONF_PATH/ca.crt $TEST_NGINX_CONF_PATH/self.test.crt
 
 # self signed certificate, for sanity check of not just accepting all certs
-openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout nginx/fake.test.key \
--out nginx/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
+openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout $TEST_NGINX_CONF_PATH/fake.test.key \
+-out $TEST_NGINX_CONF_PATH/fake.test.crt -addext "subjectAltName=DNS:fake.test" -subj "/CN=Self Signed Test Server"
 
 # openssl x509 -in nginx/fake.test.crt -text -noout
 
-cp test.py ../../sentry/test-custom-ca-roots.py
+unset RANDFILE
+
+cp ./custom-ca-roots/test.py ../sentry/test-custom-ca-roots.py
+
+$dc up -d fixture-custom-ca-roots
diff --git a/_integration-test/custom-ca-roots/teardown.sh b/_integration-test/custom-ca-roots/teardown.sh
index 4b1ebc37f0e..059f69b93b4 100755
--- a/_integration-test/custom-ca-roots/teardown.sh
+++ b/_integration-test/custom-ca-roots/teardown.sh
@@ -1,3 +1,4 @@
 #!/usr/bin/env bash
-cd $(dirname "$0")
-rm -f ../../certificates/test-custom-ca-roots.crt ../../sentry/test-custom-ca-roots.py
+$dc rm -s -f -v fixture-custom-ca-roots
+rm -f ../certificates/test-custom-ca-roots.crt ../sentry/test-custom-ca-roots.py
+unset COMPOSE_FILE
diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py
index 3bb679dfa8c..5e983b6b380 100644
--- a/_integration-test/custom-ca-roots/test.py
+++ b/_integration-test/custom-ca-roots/test.py
@@ -1,13 +1,6 @@
 import requests
 import traceback
 
-# ===========================================
-import os
-
-for k,v in sorted(os.environ.items()):
-    print(f'{k:<24} {v}')
-# ===========================================
-
 
 try:
     value = requests.get("https://self.test").text
@@ -22,7 +15,7 @@
 try:
     requests.get("https://fail.test")
     print('Accepted a self signed cert! That\'s bad.')
+    exit(1)
 except:
     print('Self signed cert didn\'t work, which is good.')
     exit(0)
-exit(1)
diff --git a/_integration-test/run.sh b/_integration-test/run.sh
index cdb718beb9b..f25a302639e 100755
--- a/_integration-test/run.sh
+++ b/_integration-test/run.sh
@@ -5,7 +5,6 @@ source "$(dirname $0)/../install/_lib.sh"
 
 echo "${_group}Setting up variables and helpers ..."
 export SENTRY_TEST_HOST="${SENTRY_TEST_HOST:-http://localhost:9000}"
-export COMPOSE_FILE=../docker-compose.yml:custom-ca-roots/docker-compose.test.yml
 TEST_USER='test@example.com'
 TEST_PASS='test123TEST'
 COOKIE_FILE=$(mktemp)
@@ -43,6 +42,7 @@ echo 'SENTRY_BEACON=False' >> $SENTRY_CONFIG_PY
 $dcr web createuser --superuser --email $TEST_USER --password $TEST_PASS || true
 $dc up -d
 printf "Waiting for Sentry to be up"; timeout 60 bash -c 'until $(curl -Isf -o /dev/null $SENTRY_TEST_HOST); do printf '.'; sleep 0.5; done'
+echo ""
 echo "${_endgroup}"
 
 echo "${_group}Running tests ..."
@@ -122,21 +122,7 @@ $dc ps | grep -q -- "-cleanup_.\+[[:space:]]\+Up[[:space:]]\+"
 echo "${_endgroup}"
 
 echo "${_group}Test custom CAs work ..."
-$dc down
-./custom-ca-roots/setup.sh
-$dc up -d
-$dc exec -T web python3 /etc/sentry/test-custom-ca-roots.py || true
-#./custom-ca-roots/teardown.sh
+source ./custom-ca-roots/setup.sh
+$dcr --no-deps web python3 /etc/sentry/test-custom-ca-roots.py
+source ./custom-ca-roots/teardown.sh
 echo "${_endgroup}"
-
-dref() { docker ps --format "table {{.ID}}\t{{.Names}}" | grep "$1" | cut -d ' ' -f1; }
-dcat() { printf "\n\n\n$2\n\n" && docker container exec -t $(dref $1) cat $2; }
-
-echo "${_group}DEBUGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGGG"
-cat ../certificates/test-custom-ca-roots.crt
-dcat fixture  /etc/nginx/ca.crt
-dcat web      /usr/local/share/ca-certificates/test-custom-ca-roots.crt
-dcat web      /etc/ssl/certs/ca-certificates.crt | tail -n32
-echo "${_endgroup}"
-
-exit 1
diff --git a/docker-compose.yml b/docker-compose.yml
index bbe67071647..e9b99117e06 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,6 +35,8 @@ x-sentry-defaults: &sentry_defaults
     PYTHONUSERBASE: "/data/custom-packages"
     SENTRY_CONF: "/etc/sentry"
     SNUBA: "http://snuba-api:1218"
+    REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
+    GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: "/etc/ssl/certs/ca-certificates.crt"
     # Leaving the value empty to just pass whatever is set
     # on the host system (or in the .env file)
     SENTRY_EVENT_RETENTION_DAYS:
@@ -42,7 +44,7 @@ x-sentry-defaults: &sentry_defaults
     - "sentry-data:/data"
     - "./sentry:/etc/sentry"
     - "./geoip:/geoip:ro"
-    - "./certificates:/usr/local/share/ca-certificates:ro"
+    - "./certificates:/usr/local/share/ca-certificates"
 x-snuba-defaults: &snuba_defaults
   <<: *restart_policy
   depends_on:
diff --git a/install/_lib.sh b/install/_lib.sh
index 136162b8cfa..0b5417f456a 100644
--- a/install/_lib.sh
+++ b/install/_lib.sh
@@ -6,7 +6,7 @@ log_file="sentry_install_log-`date +'%Y-%m-%d_%H-%M-%S'`.txt"
 exec &> >(tee -a "$log_file")
 
 # Work from /install/ for install.sh, project root otherwise
-if [[ "$(basename $0)" = "install.sh" || "$(basename $0)" = "test.sh" ]]; then
+if [[ "$(basename $0)" = "install.sh"  ]]; then
   cd "$(dirname $0)/install/"
 else
   cd "$(dirname $0)"  # assume we're a test script or some such
diff --git a/sentry/entrypoint.sh b/sentry/entrypoint.sh
index 55c7e4141a6..2f2614a798e 100755
--- a/sentry/entrypoint.sh
+++ b/sentry/entrypoint.sh
@@ -1,6 +1,10 @@
 #!/bin/bash
 set -e
 
+if [ "$(ls -A /usr/local/share/ca-certificates/)" ]; then
+  update-ca-certificates
+fi
+
 req_file="/etc/sentry/requirements.txt"
 plugins_dir="/data/custom-packages"
 checksum_file="$plugins_dir/.checksum"

From 31737f39978111729a78942c680665e8150dcd51 Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Fri, 30 Jul 2021 15:52:31 +0300
Subject: [PATCH 09/14] try going to read-only mount again

---
 docker-compose.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index e9b99117e06..936e6bbf68e 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -44,7 +44,7 @@ x-sentry-defaults: &sentry_defaults
     - "sentry-data:/data"
     - "./sentry:/etc/sentry"
     - "./geoip:/geoip:ro"
-    - "./certificates:/usr/local/share/ca-certificates"
+    - "./certificates:/usr/local/share/ca-certificates:ro"
 x-snuba-defaults: &snuba_defaults
   <<: *restart_policy
   depends_on:

From 6da9e043ec8e93331e78c8fc58eb8e84f0eb9f6a Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Fri, 30 Jul 2021 15:58:03 +0300
Subject: [PATCH 10/14] port tests to unittest framework

---
 _integration-test/custom-ca-roots/test.py | 30 +++++++++--------------
 1 file changed, 12 insertions(+), 18 deletions(-)

diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py
index 5e983b6b380..fc1a17048eb 100644
--- a/_integration-test/custom-ca-roots/test.py
+++ b/_integration-test/custom-ca-roots/test.py
@@ -1,21 +1,15 @@
+import unittest
 import requests
-import traceback
 
 
-try:
-    value = requests.get("https://self.test").text
-    if value != 'ok':
-        print('Got something other than ok: ' + value)
-        exit(1)
-    print('Custom CA worked.')
-except:
-    print('Custom CA cert failed to work.')
-    traceback.print_exc()
-    exit(1)
-try:
-    requests.get("https://fail.test")
-    print('Accepted a self signed cert! That\'s bad.')
-    exit(1)
-except:
-    print('Self signed cert didn\'t work, which is good.')
-    exit(0)
+class CustomCATests(unittest.TestCase):
+    def test_valid_self_signed(self):
+        self.assertEqual(requests.get("https://self.test").text, 'ohk')
+
+    def test_invalid_self_signed(self):
+        with self.assertRaises(requests.exceptions.SSLError):
+            requests.get("https://fail.test")
+
+
+if __name__ == '__main__':
+    unittest.main()
\ No newline at end of file

From 59b0cde37ead0702cb388122add226559423e868 Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Fri, 30 Jul 2021 16:17:28 +0300
Subject: [PATCH 11/14] fix typo

---
 _integration-test/custom-ca-roots/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py
index fc1a17048eb..4bbb64fc3df 100644
--- a/_integration-test/custom-ca-roots/test.py
+++ b/_integration-test/custom-ca-roots/test.py
@@ -4,7 +4,7 @@
 
 class CustomCATests(unittest.TestCase):
     def test_valid_self_signed(self):
-        self.assertEqual(requests.get("https://self.test").text, 'ohk')
+        self.assertEqual(requests.get("https://self.test").text, 'ok')
 
     def test_invalid_self_signed(self):
         with self.assertRaises(requests.exceptions.SSLError):

From d2ecb74f80a07bde858dee0594bbf74a28ea6062 Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Fri, 30 Jul 2021 18:15:46 +0300
Subject: [PATCH 12/14] bring DEFAULT_CA_BUNDLE back

---
 docker-compose.yml | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index 936e6bbf68e..86f3ed44be1 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -35,8 +35,14 @@ x-sentry-defaults: &sentry_defaults
     PYTHONUSERBASE: "/data/custom-packages"
     SENTRY_CONF: "/etc/sentry"
     SNUBA: "http://snuba-api:1218"
-    REQUESTS_CA_BUNDLE: "/etc/ssl/certs/ca-certificates.crt"
-    GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: "/etc/ssl/certs/ca-certificates.crt"
+    # Force everything to use the system CA bundle
+    # This is mostly needed to support installing custom CA certs
+    # This one is used by botocore
+    DEFAULT_CA_BUNDLE: &ca_bundle "/etc/ssl/certs/ca-certificates.crt"
+    # This one is used by requests
+    REQUESTS_CA_BUNDLE: *ca_bundle
+    # This one is used by grpc/google modules
+    GRPC_DEFAULT_SSL_ROOTS_FILE_PATH_ENV_VAR: *ca_bundle
     # Leaving the value empty to just pass whatever is set
     # on the host system (or in the .env file)
     SENTRY_EVENT_RETENTION_DAYS:

From b353472d3fd2177b87f02c0f0dddd3fce69e3ada Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Fri, 30 Jul 2021 18:16:02 +0300
Subject: [PATCH 13/14] add missing final eol

---
 _integration-test/custom-ca-roots/test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/_integration-test/custom-ca-roots/test.py b/_integration-test/custom-ca-roots/test.py
index 4bbb64fc3df..0f9b501f83a 100644
--- a/_integration-test/custom-ca-roots/test.py
+++ b/_integration-test/custom-ca-roots/test.py
@@ -12,4 +12,4 @@ def test_invalid_self_signed(self):
 
 
 if __name__ == '__main__':
-    unittest.main()
\ No newline at end of file
+    unittest.main()

From f7bfd12c2042388c8fb375be16e777b2e736f68e Mon Sep 17 00:00:00 2001
From: Burak Yigit Kaya <ben@byk.im>
Date: Fri, 30 Jul 2021 18:22:24 +0300
Subject: [PATCH 14/14] upgrade ubuntu version on runners for newer openssl

---
 .github/workflows/test.yml                 | 4 ++--
 _integration-test/custom-ca-roots/setup.sh | 3 ---
 2 files changed, 2 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index 8e1664cd035..2a33ea1815d 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -15,7 +15,7 @@ defaults:
     shell: bash
 jobs:
   unit-test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     name: "unit tests"
     steps:
       - name: Checkout
@@ -26,7 +26,7 @@ jobs:
         run: find ./ -type f -name "*-test.sh" -exec "./{}" \;
 
   integration-test:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-20.04
     name: "integration test"
     steps:
       - name: Pin docker-compose
diff --git a/_integration-test/custom-ca-roots/setup.sh b/_integration-test/custom-ca-roots/setup.sh
index 9832869cb3b..a8cb2f16157 100755
--- a/_integration-test/custom-ca-roots/setup.sh
+++ b/_integration-test/custom-ca-roots/setup.sh
@@ -1,7 +1,6 @@
 #! /usr/bin/env bash
 set -e
 
-export RANDFILE="$RUNNER_TEMP/.rnd"
 export COMPOSE_FILE="../docker-compose.yml:./custom-ca-roots/docker-compose.test.yml"
 
 TEST_NGINX_CONF_PATH="./custom-ca-roots/nginx"
@@ -43,8 +42,6 @@ openssl req -x509 -newkey rsa:2048 -nodes -days 1 -keyout $TEST_NGINX_CONF_PATH/
 
 # openssl x509 -in nginx/fake.test.crt -text -noout
 
-unset RANDFILE
-
 cp ./custom-ca-roots/test.py ../sentry/test-custom-ca-roots.py
 
 $dc up -d fixture-custom-ca-roots