diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d82dfd1..a0298c0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,7 +91,8 @@ jobs: echo 'CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage' >> my.env docker compose run --rm ci shell ./bin/run_setup.sh docker compose up --detach --wait --wait-timeout=10 ci-web - docker compose run --rm ci shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv' + # Use -m "not aws" to select gcp and unmarked tests + docker compose run --rm ci shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv -m "not aws"' # remove config on last two lines sed '$d' -i my.env sed '$d' -i my.env diff --git a/systemtest/README.rst b/systemtest/README.rst index 3388efd4..d7dda1b3 100644 --- a/systemtest/README.rst +++ b/systemtest/README.rst @@ -18,7 +18,15 @@ Running tests In a terminal, run:: $ make shell - app@xxx:/app$ ./systemtests/test_env.sh ENV + app@xxx:/app$ ./systemtest/test_env.sh ENV + + +Additional arguments will be passed through to pytest. For example, if testing gcp backends, use +pytest markers to select the tests for the configured cloud provider:: + + $ make shell + app@xxx:/app$ ./systemtest/test_env.sh ENV -m gcp # only gcp + app@xxx:/app$ ./systemtest/test_env.sh ENV -m 'not aws' # gcp and unmarked If you're running against ``local``, you'll need to be running antenna diff --git a/systemtest/conftest.py b/systemtest/conftest.py index d8d36d4c..e1b085cb 100644 --- a/systemtest/conftest.py +++ b/systemtest/conftest.py @@ -56,6 +56,21 @@ def posturl(config): return config("host", default="http://web:8000/").rstrip("/") + "/submit" +@pytest.fixture( + # define these params once and reference this fixture to prevent undesirable + # combinations, i.e. tests marked with aws *and* gcp for pubsub+s3 or sqs+gcs + params=[ + # tests that require a specific cloud provider backend must be marked with that + # provider, and non-default backends must be excluded by default, for example + # via pytest.ini's addopts, i.e. addopts = -m 'not gcp' + pytest.param("aws", marks=pytest.mark.aws), + pytest.param("gcp", marks=pytest.mark.gcp), + ] +) +def cloud_provider(request): + return request.param + + class GcsHelper: def __init__(self, bucket): self.bucket = bucket @@ -141,20 +156,21 @@ def list_objects(self, prefix): return [obj["Key"] for obj in resp["Contents"]] -@pytest.fixture(params=["gcs", "s3"]) -def storage_helper(config, request): - """Generate and returns an S3 or GCS helper using env config.""" - configured_backend = "s3" +@pytest.fixture +def storage_helper(config, cloud_provider): + """Generate and return a storage helper using env config.""" + actual_backend = "s3" if ( config("crashmover_crashstorage_class") == "antenna.ext.gcs.crashstorage.GcsCrashStorage" ): - configured_backend = "gcs" + actual_backend = "gcs" - if configured_backend != request.param: - pytest.skip(f"test requires {request.param}") + expect_backend = "gcs" if cloud_provider == "gcp" else "s3" + if actual_backend != expect_backend: + pytest.fail(f"test requires {expect_backend} but found {actual_backend}") - if configured_backend == "gcs": + if actual_backend == "gcs": return GcsHelper( bucket=config("crashmover_crashstorage_bucket_name"), ) @@ -261,20 +277,21 @@ def list_crashids(self): return crashids -@pytest.fixture(params=["pubsub", "sqs"]) -def queue_helper(config, request): - """Generate and returns a PubSub or SQS helper using env config.""" - configured_backend = "sqs" +@pytest.fixture +def queue_helper(config, cloud_provider): + """Generate and return a queue helper using env config.""" + actual_backend = "sqs" if ( config("crashmover_crashpublish_class") == "antenna.ext.pubsub.crashpublish.PubSubCrashPublish" ): - configured_backend = "pubsub" + actual_backend = "pubsub" - if configured_backend != request.param: - pytest.skip(f"test requires {request.param}") + expect_backend = "pubsub" if cloud_provider == "gcp" else "sqs" + if actual_backend != expect_backend: + pytest.fail(f"test requires {expect_backend} but found {actual_backend}") - if configured_backend == "pubsub": + if actual_backend == "pubsub": return PubSubHelper( project_id=config("crashmover_crashpublish_project_id", default=""), topic_name=config("crashmover_crashpublish_topic_name", default=""), diff --git a/systemtest/pytest.ini b/systemtest/pytest.ini new file mode 100644 index 00000000..d29dc007 --- /dev/null +++ b/systemtest/pytest.ini @@ -0,0 +1,6 @@ +[pytest] +addopts = -m 'not gcp' + +markers = + aws: tests that require aws backends to be configured in the environment. this is the default backend. + gcp: tests that require gcp to be configured in the environment. skipped unless explicitly requested. diff --git a/systemtest/test_env.sh b/systemtest/test_env.sh index 2c47273e..37148561 100755 --- a/systemtest/test_env.sh +++ b/systemtest/test_env.sh @@ -6,20 +6,15 @@ # This runs the system tests. It expects the following things to exist: # -# * "python3" available in PATH +# * "pytest" available in PATH # # To run this from the root of this repository, do this: # -# $ ./tests/systemtests/run_tests.sh -# -# Set POSTURL to the url to post to. -# -# Set NONGINX=1 if you're running against a local dev environment. This -# will skip tests that require nginx. +# $ ./systemtest/test_env.sh ENV set -e -USAGE="Usage: test_env.sh [local|stage]" +USAGE="Usage: test_env.sh [local|stage] [pytest args]" if [[ $# -eq 0 ]]; then echo "${USAGE}" @@ -52,4 +47,4 @@ echo "NGINX_TESTS: ${NGINX_TESTS}" # make sure to run systemtest even if this script is called from the git root cd /app/systemtest -pytest -vv +pytest -vv "${@:2}"