test: add required testing on Python 3.8.12

.github/actions/setup-sentry/action.yml

@@ -54,6 +54,9 @@ outputs:
   matrix-instance-number:
     description: "The matrix instance number (starting at 1)"
     value: ${{ steps.config.outputs.matrix-instance-number }}
+  matrix-instance-total:
+    description: "Reexport of MATRIX_INSTANCE_TOTAL."
+    value: ${{ steps.config.outputs.matrix-instance-total }}
 
 runs:
   using: "composite"
@@ -63,7 +66,10 @@ runs:
       env:
         NEED_KAFKA: ${{ inputs.kafka }}
         MATRIX_INSTANCE: ${{ matrix.instance }}
-        MATRIX_INSTANCE_TOTAL: ${{ strategy.job-total }}
+        # XXX: We should be using something like len(strategy.matrix.instance) (not possible atm)
+        # If you have other things like python-version: [foo, bar, baz] then the sharding logic
+        # isn't right because job-total will be 3x larger and you'd never run 2/3 of the tests.
+        # MATRIX_INSTANCE_TOTAL: ${{ strategy.job-total }}
       run: |
         # Only set `MIGRATIONS_TEST_MIGRATE` if it is not already set (or if it's an empty string)
         if [ -z $MIGRATIONS_TEST_MIGRATE ]; then
@@ -81,11 +87,17 @@ runs:
         ### pytest-sentry configuration ###
         echo "PYTEST_SENTRY_DSN=https://6fd5cfea2d4d46b182ad214ac7810508@sentry.io/2423079" >> $GITHUB_ENV
         echo "PYTEST_ADDOPTS=--reruns 5" >> $GITHUB_ENV
+
         # this handles pytest test sharding
         if [ "$MATRIX_INSTANCE" ]; then
+          if ! [ "$MATRIX_INSTANCE_TOTAL" ]; then
+            echo "MATRIX_INSTANCE_TOTAL is required."
+            exit 1
+          fi
           echo "TEST_GROUP=$MATRIX_INSTANCE" >> $GITHUB_ENV
           echo "TOTAL_TEST_GROUPS=$MATRIX_INSTANCE_TOTAL" >> $GITHUB_ENV
         fi
+
         # This records failures on master to sentry in order to detect flakey tests, as it's
         # expected that people have failing tests on their PRs
         [ "$GITHUB_REF" = "refs/heads/master" ] && echo "PYTEST_SENTRY_ALWAYS_REPORT=1" >> $GITHUB_ENV || true
@@ -124,6 +136,7 @@ runs:
       run: |
         echo "::set-output name=yarn-cache-dir::$(yarn cache dir)"
         echo "::set-output name=matrix-instance-number::$(($MATRIX_INSTANCE+1))"
+        echo "::set-output name=matrix-instance-total::$(($MATRIX_INSTANCE_TOTAL))"
         echo "::set-output name=acceptance-dir::.artifacts/visual-snapshots/acceptance"
 
     - name: Install python dependencies

.github/workflows/acceptance.yml

@@ -20,6 +20,7 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
+        # XXX: When updating this, make sure you also update CI_NODE_TOTAL.
         instance: [0, 1]
 
     env:
@@ -55,7 +56,10 @@ jobs:
         env:
           GITHUB_PR_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
           GITHUB_PR_REF: ${{ github.event.pull_request.head.ref || github.ref }}
-          CI_NODE_TOTAL: ${{ strategy.job-total }}
+          # XXX: CI_NODE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+          #      Otherwise, if there are other things in the matrix, using strategy.job-total
+          #      wouldn't be correct.
+          CI_NODE_TOTAL: 2
           CI_NODE_INDEX: ${{ matrix.instance }}
         run: |
           JEST_TESTS=$(yarn -s jest --listTests --json) yarn test-ci --forceExit
@@ -146,10 +150,12 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0, 1, 2, 3]
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 4
       VISUAL_SNAPSHOT_ENABLE: 1
       TEST_GROUP_STRATEGY: roundrobin
 
@@ -182,7 +188,7 @@ jobs:
         run: |
           tar xf dist.tar.gz
 
-      - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run acceptance tests (#${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         if: always()
         run: |
           mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
@@ -209,10 +215,13 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0]
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 1
+      TEST_GROUP_STRATEGY: roundrobin
       VISUAL_SNAPSHOT_ENABLE: 1
 
     steps:
@@ -244,7 +253,7 @@ jobs:
         run: |
           make build-chartcuterie-config
 
-      - name: Run chartcuterie tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run chartcuterie tests (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         run: |
           mkdir -p ${{ steps.setup.outputs.acceptance-dir }}
           make test-chartcuterie

.github/workflows/api-docs-test.yml

@@ -14,8 +14,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/backend-lint.yml

@@ -12,8 +12,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/backend-test.yml

@@ -13,11 +13,13 @@ jobs:
     timeout-minutes: 20
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0, 1, 2]
 
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 3
       MIGRATIONS_TEST_MIGRATE: 1
 
     steps:
@@ -52,7 +54,7 @@ jobs:
           pip-cache-version: ${{ secrets.PIP_CACHE_VERSION }}
           snuba: true
 
-      - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run backend test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         if: steps.changes.outputs.backend == 'true'
         run: |
           # Note: `USE_SNUBA` is not used for backend tests because there are a few failing tests with Snuba enabled.

.github/workflows/backend-typing.yml

@@ -12,7 +12,7 @@ jobs:
     timeout-minutes: 12
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
+        # TODO(joshuarli): Switch to 3.8.12 when we switch over to Python 3.8.
         python-version: [3.6.13]
     steps:
       - uses: actions/checkout@v2

.github/workflows/check-if-migration-is-required.yml

@@ -25,8 +25,7 @@ jobs:
     runs-on: ubuntu-20.04
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     needs: [should-check]
 
     steps:

.github/workflows/command-line-test.yml

@@ -12,8 +12,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/development-environment.yml

@@ -22,8 +22,7 @@ jobs:
         # Using Ubuntu 18 until I figure out this error:
         # -> ImportError: libffi.so.6: cannot open shared object file: No such file or directory
         os: [macos-11.0, ubuntu-18.04]
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
       fail-fast: false
     env:
       PIP_DISABLE_PIP_VERSION_CHECK: on

.github/workflows/migrations.yml

@@ -41,8 +41,7 @@ jobs:
     timeout-minutes: 8
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     needs: did-migration-change
     if: ${{ needs.did-migration-change.outputs.added == 'true' }}
 

.github/workflows/plugins-test.yml

@@ -13,8 +13,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
 

.github/workflows/relay-integration-test.yml

@@ -13,8 +13,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
         with:

.github/workflows/snuba-integration-test.yml

@@ -13,10 +13,12 @@ jobs:
     timeout-minutes: 30
     strategy:
       matrix:
+        python-version: [3.6.13, 3.8.12]
+        # XXX: When updating this, make sure you also update MATRIX_INSTANCE_TOTAL.
         instance: [0, 1]
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
     env:
+      # XXX: MATRIX_INSTANCE_TOTAL must be hardcoded to the length of strategy.matrix.instance.
+      MATRIX_INSTANCE_TOTAL: 2
       USE_SNUBA: 1
       MIGRATIONS_TEST_MIGRATE: 1
       USE_INDEXER: 1
@@ -54,7 +56,7 @@ jobs:
           snuba: true
           kafka: true
 
-      - name: Run snuba test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ strategy.job-total }})
+      - name: Run snuba test (${{ steps.setup.outputs.matrix-instance-number }} of ${{ steps.setup.outputs.matrix-instance-total }})
         if: steps.changes.outputs.backend == 'true'
         run: |
           make test-snuba

.github/workflows/symbolicator-integration-test.yml

@@ -13,8 +13,7 @@ jobs:
     timeout-minutes: 10
     strategy:
       matrix:
-        # TODO(joshuarli): Add 3.8.12.
-        python-version: [3.6.13]
+        python-version: [3.6.13, 3.8.12]
     steps:
       - uses: actions/checkout@v2
         with:

mypy.ini

@@ -80,8 +80,8 @@ disallow_incomplete_defs=True
 check_untyped_defs=True
 disallow_untyped_decorators=True
 no_implicit_optional=True
-warn_redundant_casts=True
 warn_unused_ignores=True
+warn_redundant_casts=True
 warn_return_any=True
 no_implicit_reexport=True
 

src/sentry/utils/pytest/selenium.py

@@ -304,7 +304,8 @@ def snapshot(self, name, mobile_only=False):
         """
         # TODO(dcramer): ideally this would take the executing test package
         # into account for duplicate names
-        if os.environ.get("VISUAL_SNAPSHOT_ENABLE") != "1":
+        # TODO(josh): Change to 3.8 when it's deployed. Much easier to disable here than in GHA yml.
+        if os.environ.get("VISUAL_SNAPSHOT_ENABLE") != "1" or sys.version_info[:2] != (3, 6):
             return self
 
         self.wait_for_images_loaded()