diff --git a/.circleci/config.yml b/.circleci/config.yml index 0d82dfd1..a0298c0a 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -91,7 +91,8 @@ jobs: echo 'CRASHMOVER_CRASHSTORAGE_CLASS=antenna.ext.gcs.crashstorage.GcsCrashStorage' >> my.env docker compose run --rm ci shell ./bin/run_setup.sh docker compose up --detach --wait --wait-timeout=10 ci-web - docker compose run --rm ci shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv' + # Use -m "not aws" to select gcp and unmarked tests + docker compose run --rm ci shell bash -c 'cd systemtest && NGINX_TESTS=0 POST_CHECK=1 HOST=http://ci-web:8000 pytest -vv -m "not aws"' # remove config on last two lines sed '$d' -i my.env sed '$d' -i my.env diff --git a/pyproject.toml b/pyproject.toml index 79aca7bb..a0de01f6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -16,6 +16,26 @@ docstring-quotes = "double" "bin/release.py" = ["S310", "S603", "S607"] +[tool.pytest.ini_options] +addopts = "-rsxX --tb=native --showlocals -m 'not gcp'" +norecursedirs = [".git", "docs", "bin"] +testpaths = "tests/unittest/" + +markers = [ + "aws: tests that require aws backends to be configured in the environment. this is the default.", + "gcp: tests that require gcp backends to be configured in the environment. skipped unless explicitly requested.", +] + +filterwarnings = [ + "error", + # Falcon currently uses cgi which is going away in python 3.13 + "ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning:falcon", + # sentry-sdk has a Falcon integration that needs some help and is currently + # using a thing in Falcon that kicks up a DeprecatedWarning + "ignore:The api_helpers module.*:falcon.util.deprecation.DeprecatedWarning", +] + + [tool.paul-mclendahand] github_user = "mozilla-services" github_project = "antenna" diff --git a/setup.cfg b/setup.cfg index 7d39a0e9..223bfabd 100644 --- a/setup.cfg +++ b/setup.cfg @@ -18,19 +18,6 @@ ignore = exclude = docs max-line-length = 88 -[tool:pytest] -addopts = -rsxX --tb=native --showlocals -norecursedirs = .git docs bin -testpaths = tests/unittest/ - -filterwarnings = - error - # Falcon currently uses cgi which is going away in python 3.13 - ignore:'cgi' is deprecated and slated for removal in Python 3.13:DeprecationWarning:falcon - # sentry-sdk has a Falcon integration that needs some help and is currently - # using a thing in Falcon that kicks up a DeprecatedWarning - ignore:The api_helpers module.*:falcon.util.deprecation.DeprecatedWarning - [doc8] max-line-length = 88 ignore-path = docs/_build/ diff --git a/systemtest/README.rst b/systemtest/README.rst index 3388efd4..d7dda1b3 100644 --- a/systemtest/README.rst +++ b/systemtest/README.rst @@ -18,7 +18,15 @@ Running tests In a terminal, run:: $ make shell - app@xxx:/app$ ./systemtests/test_env.sh ENV + app@xxx:/app$ ./systemtest/test_env.sh ENV + + +Additional arguments will be passed through to pytest. For example, if testing gcp backends, use +pytest markers to select the tests for the configured cloud provider:: + + $ make shell + app@xxx:/app$ ./systemtest/test_env.sh ENV -m gcp # only gcp + app@xxx:/app$ ./systemtest/test_env.sh ENV -m 'not aws' # gcp and unmarked If you're running against ``local``, you'll need to be running antenna diff --git a/systemtest/conftest.py b/systemtest/conftest.py index d8d36d4c..e1b085cb 100644 --- a/systemtest/conftest.py +++ b/systemtest/conftest.py @@ -56,6 +56,21 @@ def posturl(config): return config("host", default="http://web:8000/").rstrip("/") + "/submit" +@pytest.fixture( + # define these params once and reference this fixture to prevent undesirable + # combinations, i.e. tests marked with aws *and* gcp for pubsub+s3 or sqs+gcs + params=[ + # tests that require a specific cloud provider backend must be marked with that + # provider, and non-default backends must be excluded by default, for example + # via pytest.ini's addopts, i.e. addopts = -m 'not gcp' + pytest.param("aws", marks=pytest.mark.aws), + pytest.param("gcp", marks=pytest.mark.gcp), + ] +) +def cloud_provider(request): + return request.param + + class GcsHelper: def __init__(self, bucket): self.bucket = bucket @@ -141,20 +156,21 @@ def list_objects(self, prefix): return [obj["Key"] for obj in resp["Contents"]] -@pytest.fixture(params=["gcs", "s3"]) -def storage_helper(config, request): - """Generate and returns an S3 or GCS helper using env config.""" - configured_backend = "s3" +@pytest.fixture +def storage_helper(config, cloud_provider): + """Generate and return a storage helper using env config.""" + actual_backend = "s3" if ( config("crashmover_crashstorage_class") == "antenna.ext.gcs.crashstorage.GcsCrashStorage" ): - configured_backend = "gcs" + actual_backend = "gcs" - if configured_backend != request.param: - pytest.skip(f"test requires {request.param}") + expect_backend = "gcs" if cloud_provider == "gcp" else "s3" + if actual_backend != expect_backend: + pytest.fail(f"test requires {expect_backend} but found {actual_backend}") - if configured_backend == "gcs": + if actual_backend == "gcs": return GcsHelper( bucket=config("crashmover_crashstorage_bucket_name"), ) @@ -261,20 +277,21 @@ def list_crashids(self): return crashids -@pytest.fixture(params=["pubsub", "sqs"]) -def queue_helper(config, request): - """Generate and returns a PubSub or SQS helper using env config.""" - configured_backend = "sqs" +@pytest.fixture +def queue_helper(config, cloud_provider): + """Generate and return a queue helper using env config.""" + actual_backend = "sqs" if ( config("crashmover_crashpublish_class") == "antenna.ext.pubsub.crashpublish.PubSubCrashPublish" ): - configured_backend = "pubsub" + actual_backend = "pubsub" - if configured_backend != request.param: - pytest.skip(f"test requires {request.param}") + expect_backend = "pubsub" if cloud_provider == "gcp" else "sqs" + if actual_backend != expect_backend: + pytest.fail(f"test requires {expect_backend} but found {actual_backend}") - if configured_backend == "pubsub": + if actual_backend == "pubsub": return PubSubHelper( project_id=config("crashmover_crashpublish_project_id", default=""), topic_name=config("crashmover_crashpublish_topic_name", default=""), diff --git a/systemtest/test_env.sh b/systemtest/test_env.sh index 2c47273e..37148561 100755 --- a/systemtest/test_env.sh +++ b/systemtest/test_env.sh @@ -6,20 +6,15 @@ # This runs the system tests. It expects the following things to exist: # -# * "python3" available in PATH +# * "pytest" available in PATH # # To run this from the root of this repository, do this: # -# $ ./tests/systemtests/run_tests.sh -# -# Set POSTURL to the url to post to. -# -# Set NONGINX=1 if you're running against a local dev environment. This -# will skip tests that require nginx. +# $ ./systemtest/test_env.sh ENV set -e -USAGE="Usage: test_env.sh [local|stage]" +USAGE="Usage: test_env.sh [local|stage] [pytest args]" if [[ $# -eq 0 ]]; then echo "${USAGE}" @@ -52,4 +47,4 @@ echo "NGINX_TESTS: ${NGINX_TESTS}" # make sure to run systemtest even if this script is called from the git root cd /app/systemtest -pytest -vv +pytest -vv "${@:2}"