bug-1885442: Fail platform specific system tests when unexpectedly ex…

…ecuted

.circleci/config.yml

@@ -91,7 +91,8 @@ jobs:
             echo 'CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage' >> my.env
             docker compose run --rm ci shell ./bin/run_setup.sh
             docker compose up --detach --wait --wait-timeout=10 ci-web
-            docker compose run --rm ci shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv'
+            # Use -m "not aws" to select gcp and unmarked tests
+            docker compose run --rm ci shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv -m "not aws"'
             # remove config on last two lines
             sed '$d' -i my.env
             sed '$d' -i my.env

systemtest/README.rst

@@ -18,7 +18,15 @@ Running tests
 In a terminal, run::
 
     $ make shell
-    app@xxx:/app$ ./systemtests/test_env.sh ENV
+    app@xxx:/app$ ./systemtest/test_env.sh ENV
+
+
+Additional arguments will be passed through to pytest. For example, if testing gcp backends, use
+pytest markers to select the tests for the configured cloud provider::
+
+    $ make shell
+    app@xxx:/app$ ./systemtest/test_env.sh ENV -m gcp  # only gcp
+    app@xxx:/app$ ./systemtest/test_env.sh ENV -m 'not aws'  # gcp and unmarked
 
 
 If you're running against ``local``, you'll need to be running antenna

systemtest/conftest.py

@@ -56,6 +56,21 @@ def posturl(config):
     return config("host", default="http://web:8000/").rstrip("/") + "/submit"
 
 
+@pytest.fixture(
+    # define these params once and reference this fixture to prevent undesirable
+    # combinations, i.e. tests marked with aws *and* gcp for pubsub+s3 or sqs+gcs
+    params=[
+        # tests that require a specific cloud provider backend must be marked with that
+        # provider, and non-default backends must be excluded by default, for example
+        # via pytest.ini's addopts, i.e. addopts = -m 'not gcp'
+        pytest.param("aws", marks=pytest.mark.aws),
+        pytest.param("gcp", marks=pytest.mark.gcp),
+    ]
+)
+def cloud_provider(request):
+    return request.param
+
+
 class GcsHelper:
     def __init__(self, bucket):
         self.bucket = bucket
@@ -141,20 +156,21 @@ def list_objects(self, prefix):
         return [obj["Key"] for obj in resp["Contents"]]
 
 
-@pytest.fixture(params=["gcs", "s3"])
-def storage_helper(config, request):
-    """Generate and returns an S3 or GCS helper using env config."""
-    configured_backend = "s3"
+@pytest.fixture
+def storage_helper(config, cloud_provider):
+    """Generate and return a storage helper using env config."""
+    actual_backend = "s3"
     if (
         config("crashmover_crashstorage_class")
         == "antenna.ext.gcs.crashstorage.GcsCrashStorage"
     ):
-        configured_backend = "gcs"
+        actual_backend = "gcs"
 
-    if configured_backend != request.param:
-        pytest.skip(f"test requires {request.param}")
+    expect_backend = "gcs" if cloud_provider == "gcp" else "s3"
+    if actual_backend != expect_backend:
+        pytest.fail(f"test requires {expect_backend} but found {actual_backend}")
 
-    if configured_backend == "gcs":
+    if actual_backend == "gcs":
         return GcsHelper(
             bucket=config("crashmover_crashstorage_bucket_name"),
         )
@@ -261,20 +277,21 @@ def list_crashids(self):
         return crashids
 
 
-@pytest.fixture(params=["pubsub", "sqs"])
-def queue_helper(config, request):
-    """Generate and returns a PubSub or SQS helper using env config."""
-    configured_backend = "sqs"
+@pytest.fixture
+def queue_helper(config, cloud_provider):
+    """Generate and return a queue helper using env config."""
+    actual_backend = "sqs"
     if (
         config("crashmover_crashpublish_class")
         == "antenna.ext.pubsub.crashpublish.PubSubCrashPublish"
     ):
-        configured_backend = "pubsub"
+        actual_backend = "pubsub"
 
-    if configured_backend != request.param:
-        pytest.skip(f"test requires {request.param}")
+    expect_backend = "pubsub" if cloud_provider == "gcp" else "sqs"
+    if actual_backend != expect_backend:
+        pytest.fail(f"test requires {expect_backend} but found {actual_backend}")
 
-    if configured_backend == "pubsub":
+    if actual_backend == "pubsub":
         return PubSubHelper(
             project_id=config("crashmover_crashpublish_project_id", default=""),
             topic_name=config("crashmover_crashpublish_topic_name", default=""),

systemtest/pytest.ini

@@ -0,0 +1,6 @@
+[pytest]
+addopts = -m 'not gcp'
+
+markers =
+    aws: tests that require aws backends to be configured in the environment. this is the default backend.
+    gcp: tests that require gcp to be configured in the environment. skipped unless explicitly requested.

systemtest/test_env.sh

@@ -6,20 +6,15 @@
 
 # This runs the system tests. It expects the following things to exist:
 #
-# * "python3" available in PATH
+# * "pytest" available in PATH
 #
 # To run this from the root of this repository, do this:
 #
-#     $ ./tests/systemtests/run_tests.sh
-#
-# Set POSTURL to the url to post to.
-#
-# Set NONGINX=1 if you're running against a local dev environment. This
-# will skip tests that require nginx.
+#     $ ./systemtest/test_env.sh ENV
 
 set -e
 
-USAGE="Usage: test_env.sh [local|stage]"
+USAGE="Usage: test_env.sh [local|stage] [pytest args]"
 
 if [[ $# -eq 0 ]]; then
     echo "${USAGE}"
@@ -52,4 +47,4 @@ echo "NGINX_TESTS: ${NGINX_TESTS}"
 # make sure to run systemtest even if this script is called from the git root
 cd /app/systemtest
 
-pytest -vv
+pytest -vv "${@:2}"