tests: run postgres via pytest (pypi#15365)

Co-authored-by: Ee Durbin <ewdurbin@gmail.com>

.github/workflows/ci.yml

@@ -47,7 +47,7 @@ jobs:
       matrix:
         include:
           - name: Tests
-            command: bin/tests --postgresql-host localhost
+            command: bin/tests
           - name: Lint
             command: bin/lint
           - name: User Documentation
@@ -62,14 +62,6 @@ jobs:
             command: bin/translations
     runs-on: ubuntu-latest
     services:
-      postgres:
-        image: ${{ (matrix.name == 'Tests') && 'postgres:14.4' || '' }}
-        ports:
-          - 5432:5432
-        env:
-          POSTGRES_HOST_AUTH_METHOD: trust  # never do this in production!
-        # Set health checks to wait until postgres has started
-        options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5
       stripe:
         image: ${{ (matrix.name == 'Tests') && 'stripe/stripe-mock:v0.162.0' || '' }}
         ports:

.gitignore

@@ -25,6 +25,7 @@ node_modules/
 dev/example.sql
 dev/prod.sql
 dev/prod.sql.xz
+dev/.coverage
 ./data/
 
 warehouse/.commit

Dockerfile

@@ -212,7 +212,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     && apt-get update \
     && apt-get install --no-install-recommends -y \
         libpq5 libxml2 libxslt1.1 libcurl4  \
-        $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev vim'; fi) \
+        $(if [ "$DEVEL" = "yes" ]; then echo 'bash libjpeg62 postgresql-client postgresql build-essential libffi-dev libxml2-dev libxslt-dev libpq-dev libcurl4-openssl-dev libssl-dev vim'; fi) \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
@@ -224,3 +224,6 @@ COPY --from=static /opt/warehouse/src/warehouse/static/dist/ /opt/warehouse/src/
 COPY --from=static /opt/warehouse/src/warehouse/admin/static/dist/ /opt/warehouse/src/warehouse/admin/static/dist/
 COPY --from=build /opt/warehouse/ /opt/warehouse/
 COPY . /opt/warehouse/src/
+
+# We cannot run `postgres` as root, so add a user to run the application
+USER nobody

Makefile

@@ -61,7 +61,7 @@ debug: .state/docker-build-base
 	docker compose run --rm --service-ports web
 
 tests: .state/docker-build-base
-	docker compose run --rm web bin/tests --postgresql-host db $(T) $(TESTARGS)
+	docker compose run --rm tests bin/tests $(T) $(TESTARGS)
 
 static_tests: .state/docker-build-static
 	docker compose run --rm static bin/static_tests $(T) $(TESTARGS)

bin/tests

@@ -1,38 +1,6 @@
 #!/bin/bash
-set -e
+set -ex
 
-# Click requires us to ensure we have a well configured environment to run
-# our click commands. So we'll set our environment to ensure our locale is
-# correct.
-export LC_ALL="${ENCODING:-en_US.UTF-8}"
-export LANG="${ENCODING:-en_US.UTF-8}"
-
-COMMAND_ARGS="$@"
-
-# Test the postgres connection
-while [ $# -gt 0 ]; do
-  case $1 in
-    "--postgresql-host") POSTGRES_HOST="$2"
-  esac
-  shift
-done
-
-# Test the postgres connection
-ATTEMPTS=0
-until [ $ATTEMPTS -eq 5 ] || pg_isready -t 10 -h $POSTGRES_HOST; do
-  >&2 echo "Postgres is unavailable, sleeping"
-  sleep $(( ATTEMPTS++ ))
-done
-
-if [ $ATTEMPTS -eq 5 ]; then
-  >&2 echo "Postgres is unavailable, exiting"
-  exit 1
-fi
-
-# Print all the following commands
-set -x
-
-# Actually run our tests.
-python -m coverage run -m pytest --strict-markers $COMMAND_ARGS
+python -m coverage run -m pytest --strict-markers "$@"
 python -m coverage html --show-contexts
 python -m coverage report -m --fail-under 100

docker-compose.yml

@@ -80,7 +80,6 @@ services:
       - ./tests:/opt/warehouse/src/tests:z
       - ./htmlcov:/opt/warehouse/src/htmlcov:z
       - ./pyproject.toml:/opt/warehouse/src/pyproject.toml:z
-      - .coveragerc:/opt/warehouse/src/.coveragerc:z
       - packages:/var/opt/warehouse/packages
       - packages-archive:/var/opt/warehouse/packages-archive
       - sponsorlogos:/var/opt/warehouse/sponsorlogos
@@ -113,6 +112,14 @@ services:
       stripe:
         condition: service_started
 
+  tests:
+    image: warehouse:docker-compose
+    pull_policy: never
+    volumes: *base_volumes
+    depends_on:
+      stripe:
+        condition: service_started
+
   files:
     image: warehouse:docker-compose
     pull_policy: never

pyproject.toml

@@ -1,5 +1,6 @@
 [tool.coverage.run]
 branch = true
+data_file = "dev/.coverage"
 dynamic_context = "test_function"
 source = ["warehouse"]
 omit = [

tests/conftest.py

@@ -33,7 +33,6 @@
 from pyramid.static import ManifestCacheBuster
 from pyramid_jinja2 import IJinja2Environment
 from pyramid_mailer.mailer import DummyMailer
-from pytest_postgresql.config import get_config
 from pytest_postgresql.janitor import DatabaseJanitor
 from sqlalchemy import event
 
@@ -249,13 +248,12 @@ def cli():
 
 
 @pytest.fixture(scope="session")
-def database(request):
-    config = get_config(request)
-    pg_host = config.get("host")
-    pg_port = config.get("port") or os.environ.get("PGPORT", 5432)
-    pg_user = config.get("user")
-    pg_db = config.get("db", "tests")
-    pg_version = config.get("version", 14.4)
+def database(postgresql_proc, request):
+    pg_host = postgresql_proc.host
+    pg_port = postgresql_proc.port
+    pg_user = postgresql_proc.user
+    pg_db = postgresql_proc.dbname
+    pg_version = postgresql_proc.version
 
     janitor = DatabaseJanitor(pg_user, pg_host, pg_port, pg_db, pg_version)
 
@@ -313,6 +311,8 @@ def app_config(database):
         "docs.backend": "warehouse.packaging.services.LocalDocsStorage",
         "sponsorlogos.backend": "warehouse.admin.services.LocalSponsorLogoStorage",
         "billing.backend": "warehouse.subscriptions.services.MockStripeBillingService",
+        "billing.api_base": "http://stripe:12111",
+        "billing.api_version": "2020-08-27",
         "mail.backend": "warehouse.email.services.SMTPEmailSender",
         "files.url": "http://localhost:7000/",
         "archive_files.url": "http://localhost:7000/archive",