From 8082d93a55a578b52dcc599f319ad0b0fa54e75f Mon Sep 17 00:00:00 2001 From: Daniel Thorn Date: Mon, 4 Mar 2024 17:21:13 -0800 Subject: [PATCH] use gcs emulator for gcs tests --- setup.cfg | 2 +- tests/conftest.py | 100 +++++++++++++++ .../test_gcs_crashstorage_with_emulator.py | 119 ++++++++++++++++++ tests/unittest/conftest.py | 88 +------------ tests/unittest/test_gcs_crashstorage.py | 2 +- 5 files changed, 222 insertions(+), 89 deletions(-) create mode 100644 tests/conftest.py create mode 100644 tests/external/test_gcs_crashstorage_with_emulator.py diff --git a/setup.cfg b/setup.cfg index 7d39a0e9..e5688304 100644 --- a/setup.cfg +++ b/setup.cfg @@ -21,7 +21,7 @@ max-line-length = 88 [tool:pytest] addopts = -rsxX --tb=native --showlocals norecursedirs = .git docs bin -testpaths = tests/unittest/ +testpaths = tests/ filterwarnings = error diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..7d586de9 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,100 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +from pathlib import Path +import sys + +from everett.manager import ConfigManager, ConfigDictEnv, ConfigOSEnv +from falcon.request import Request +from falcon.testing.helpers import create_environ +from falcon.testing.client import TestClient +import markus +import pytest + + +# Add repository root so we can import antenna. +REPO_ROOT = Path(__file__).parent.parent +sys.path.insert(0, str(REPO_ROOT)) + +from antenna.app import get_app, setup_logging # noqa: E402 +from antenna.app import reset_verify_funs # noqa: E402 + + +def pytest_runtest_setup(): + # Make sure we set up logging and metrics to sane default values. + setup_logging(logging_level="DEBUG", debug=True, host_id="", processname="antenna") + markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}]) + + # Wipe any registered verify functions + reset_verify_funs() + + +@pytest.fixture +def request_generator(): + """Returns a Falcon Request generator""" + + def _request_generator(method, path, query_string=None, headers=None, body=None): + env = create_environ( + method=method, + path=path, + query_string=(query_string or ""), + headers=headers, + body=body, + ) + return Request(env) + + return _request_generator + + +class AntennaTestClient(TestClient): + """Test client to ease testing with Antenna API""" + + @classmethod + def build_config(cls, new_config=None): + """Build ConfigManager using environment and overrides.""" + new_config = new_config or {} + config_manager = ConfigManager( + environments=[ConfigDictEnv(new_config), ConfigOSEnv()] + ) + return config_manager + + def rebuild_app(self, new_config): + """Rebuilds the app + + This is helpful if you've changed configuration and need to rebuild the + app so that components pick up the new configuration. + + :arg new_config: dict of configuration to override normal values to build the + new app with + + """ + self.app = get_app(self.build_config(new_config)) + + def get_crashmover(self): + """Retrieves the crashmover from the AntennaApp.""" + return self.app.app.crashmover + + def get_resource_by_name(self, name): + """Retrieves the Falcon API resource by name""" + return self.app.app.get_resource_by_name(name) + + +@pytest.fixture +def client(): + """Test client for the Antenna API + + This creates an app and a test client that uses that app to submit HTTP + GET/POST requests. + + The app that's created uses configuration defaults. If you need it to use + an app with a different configuration, you can rebuild the app with + different configuration:: + + def test_foo(client, tmpdir): + client.rebuild_app({ + 'BASEDIR': str(tmpdir) + }) + + """ + return AntennaTestClient(get_app(AntennaTestClient.build_config())) diff --git a/tests/external/test_gcs_crashstorage_with_emulator.py b/tests/external/test_gcs_crashstorage_with_emulator.py new file mode 100644 index 00000000..c8e98a7a --- /dev/null +++ b/tests/external/test_gcs_crashstorage_with_emulator.py @@ -0,0 +1,119 @@ +# This Source Code Form is subject to the terms of the Mozilla Public +# License, v. 2.0. If a copy of the MPL was not distributed with this +# file, You can obtain one at https://mozilla.org/MPL/2.0/. + +import io +import os +from unittest.mock import Mock, patch + +import pytest +from google.auth.credentials import AnonymousCredentials +from google.cloud.exceptions import NotFound, Unauthorized +from google.cloud import storage + +from testlib.mini_poster import multipart_encode + + +@pytest.fixture(autouse=True) +def mock_generate_test_filepath(): + with patch("antenna.ext.gcs.crashstorage.generate_test_filepath") as gtfp: + gtfp.return_value = "test/testwrite.txt" + yield + + +@pytest.fixture +def gcs_client(): + if os.environ.get("STORAGE_EMULATOR_HOST"): + client = storage.Client( + credentials=AnonymousCredentials(), + project="test", + ) + try: + yield client + finally: + for bucket in client.list_buckets(): + try: + bucket.delete(force=True) + except NotFound: + pass # same difference + else: + pytest.skip("requires gcs emulator") + + +class TestGcsCrashStorageIntegration: + logging_names = ["antenna"] + + @pytest.mark.parametrize("bucket_name", ["fakebucket", "fakebucket-with-periods"]) + def test_crash_storage(self, client, gcs_client, bucket_name): + # clean up bucket left around from previous tests + try: + gcs_client.get_bucket(bucket_name).delete(force=True) + except NotFound: + pass # same difference + gcs_bucket = gcs_client.create_bucket(bucket_name) + + crash_id = "de1bb258-cbbf-4589-a673-34f800160918" + data, headers = multipart_encode( + { + "uuid": crash_id, + "ProductName": "Firefox", + "Version": "1.0", + "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")), + } + ) + + # Rebuild the app the test client is using with relevant configuration. + client.rebuild_app( + { + "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.gcs.crashstorage.GcsCrashStorage", + "CRASHMOVER_CRASHSTORAGE_BUCKET_NAME": gcs_bucket.name, + } + ) + + result = client.simulate_post("/submit", headers=headers, body=data) + + # Verify the collector returns a 200 status code and the crash id + # we fed it. + assert result.status_code == 200 + assert result.content == f"CrashID=bp-{crash_id}\n".encode("utf-8") + + # Assert we uploaded files to gcs + blobs = sorted(gcs_bucket.list_blobs(), key=lambda b: b.name) + + blob_names = [b.name for b in blobs] + assert blob_names == [ + "test/testwrite.txt", + f"v1/dump_names/{crash_id}", + f"v1/raw_crash/20160918/{crash_id}", + f"v1/upload_file_minidump/{crash_id}", + ] + + blob_contents = [ + b.download_as_bytes() + for b in blobs + # ignore the contents of the raw crash + if not b.name.startswith("v1/raw_crash") + ] + assert blob_contents == [ + b"test", + b'["upload_file_minidump"]', + b"abcd1234", + ] + + def test_missing_bucket_halts_startup(self, client, gcs_client): + bucket_name = "missingbucket" + # ensure bucket is actually missing + with pytest.raises(NotFound): + gcs_client.get_bucket(bucket_name) + + with pytest.raises(NotFound) as excinfo: + # Rebuild the app the test client is using with relevant + # configuration. This calls .verify_write_to_bucket() which fails. + client.rebuild_app( + { + "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.gcs.crashstorage.GcsCrashStorage", + "CRASHMOVER_CRASHSTORAGE_BUCKET_NAME": bucket_name + } + ) + + assert f"b/{bucket_name}" in excinfo.value.args[0] diff --git a/tests/unittest/conftest.py b/tests/unittest/conftest.py index 36b0379e..6cb04d3e 100644 --- a/tests/unittest/conftest.py +++ b/tests/unittest/conftest.py @@ -7,103 +7,17 @@ import sys from unittest import mock -from everett.manager import ConfigManager, ConfigDictEnv, ConfigOSEnv -from falcon.request import Request -from falcon.testing.helpers import create_environ -from falcon.testing.client import TestClient -import markus from markus.testing import MetricsMock import pytest -# Add repository root so we can import antenna and testlib. +# Add repository root so we can import testlib. REPO_ROOT = Path(__file__).parent.parent.parent sys.path.insert(0, str(REPO_ROOT)) -from antenna.app import get_app, setup_logging # noqa -from antenna.app import reset_verify_funs # noqa from testlib.s3mock import S3Mock # noqa -def pytest_runtest_setup(): - # Make sure we set up logging and metrics to sane default values. - setup_logging(logging_level="DEBUG", debug=True, host_id="", processname="antenna") - markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}]) - - # Wipe any registered verify functions - reset_verify_funs() - - -@pytest.fixture -def request_generator(): - """Returns a Falcon Request generator""" - - def _request_generator(method, path, query_string=None, headers=None, body=None): - env = create_environ( - method=method, - path=path, - query_string=(query_string or ""), - headers=headers, - body=body, - ) - return Request(env) - - return _request_generator - - -class AntennaTestClient(TestClient): - """Test client to ease testing with Antenna API""" - - @classmethod - def build_config(cls, new_config=None): - """Build ConfigManager using environment and overrides.""" - new_config = new_config or {} - config_manager = ConfigManager( - environments=[ConfigDictEnv(new_config), ConfigOSEnv()] - ) - return config_manager - - def rebuild_app(self, new_config): - """Rebuilds the app - - This is helpful if you've changed configuration and need to rebuild the - app so that components pick up the new configuration. - - :arg new_config: dict of configuration to override normal values to build the - new app with - - """ - self.app = get_app(self.build_config(new_config)) - - def get_crashmover(self): - """Retrieves the crashmover from the AntennaApp.""" - return self.app.app.crashmover - - def get_resource_by_name(self, name): - """Retrieves the Falcon API resource by name""" - return self.app.app.get_resource_by_name(name) - - -@pytest.fixture -def client(): - """Test client for the Antenna API - - This creates an app and a test client that uses that app to submit HTTP - GET/POST requests. - - The app that's created uses configuration defaults. If you need it to use - an app with a different configuration, you can rebuild the app with - different configuration:: - - def test_foo(client, tmpdir): - client.rebuild_app({ - 'BASEDIR': str(tmpdir) - }) - - """ - return AntennaTestClient(get_app(AntennaTestClient.build_config())) - - @pytest.fixture def s3mock(): """Returns an s3mock context that lets you do S3-related tests diff --git a/tests/unittest/test_gcs_crashstorage.py b/tests/unittest/test_gcs_crashstorage.py index 75d7272f..0a7ae0c5 100644 --- a/tests/unittest/test_gcs_crashstorage.py +++ b/tests/unittest/test_gcs_crashstorage.py @@ -91,7 +91,7 @@ def test_missing_bucket_halts_startup(self, MockStorageClient, client): } ) - assert "404 bucket not found" == str(excinfo.value) + assert ("bucket not found",) == excinfo.value.args mock_client.get_bucket.assert_called_once_with("fakebucket")