use gcs emulator for gcs tests

setup.cfg

@@ -21,7 +21,7 @@ max-line-length = 88
 [tool:pytest]
 addopts = -rsxX --tb=native --showlocals
 norecursedirs = .git docs bin
-testpaths = tests/unittest/
+testpaths = tests/
 
 filterwarnings =
     error

tests/conftest.py

@@ -0,0 +1,100 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+from pathlib import Path
+import sys
+
+from everett.manager import ConfigManager, ConfigDictEnv, ConfigOSEnv
+from falcon.request import Request
+from falcon.testing.helpers import create_environ
+from falcon.testing.client import TestClient
+import markus
+import pytest
+
+
+# Add repository root so we can import antenna.
+REPO_ROOT = Path(__file__).parent.parent
+sys.path.insert(0, str(REPO_ROOT))
+
+from antenna.app import get_app, setup_logging  # noqa: E402
+from antenna.app import reset_verify_funs  # noqa: E402
+
+
+def pytest_runtest_setup():
+    # Make sure we set up logging and metrics to sane default values.
+    setup_logging(logging_level="DEBUG", debug=True, host_id="", processname="antenna")
+    markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
+
+    # Wipe any registered verify functions
+    reset_verify_funs()
+
+
+@pytest.fixture
+def request_generator():
+    """Returns a Falcon Request generator"""
+
+    def _request_generator(method, path, query_string=None, headers=None, body=None):
+        env = create_environ(
+            method=method,
+            path=path,
+            query_string=(query_string or ""),
+            headers=headers,
+            body=body,
+        )
+        return Request(env)
+
+    return _request_generator
+
+
+class AntennaTestClient(TestClient):
+    """Test client to ease testing with Antenna API"""
+
+    @classmethod
+    def build_config(cls, new_config=None):
+        """Build ConfigManager using environment and overrides."""
+        new_config = new_config or {}
+        config_manager = ConfigManager(
+            environments=[ConfigDictEnv(new_config), ConfigOSEnv()]
+        )
+        return config_manager
+
+    def rebuild_app(self, new_config):
+        """Rebuilds the app
+
+        This is helpful if you've changed configuration and need to rebuild the
+        app so that components pick up the new configuration.
+
+        :arg new_config: dict of configuration to override normal values to build the
+            new app with
+
+        """
+        self.app = get_app(self.build_config(new_config))
+
+    def get_crashmover(self):
+        """Retrieves the crashmover from the AntennaApp."""
+        return self.app.app.crashmover
+
+    def get_resource_by_name(self, name):
+        """Retrieves the Falcon API resource by name"""
+        return self.app.app.get_resource_by_name(name)
+
+
+@pytest.fixture
+def client():
+    """Test client for the Antenna API
+
+    This creates an app and a test client that uses that app to submit HTTP
+    GET/POST requests.
+
+    The app that's created uses configuration defaults. If you need it to use
+    an app with a different configuration, you can rebuild the app with
+    different configuration::
+
+        def test_foo(client, tmpdir):
+            client.rebuild_app({
+                'BASEDIR': str(tmpdir)
+            })
+
+    """
+    return AntennaTestClient(get_app(AntennaTestClient.build_config()))

tests/external/test_gcs_crashstorage_with_emulator.py

@@ -0,0 +1,119 @@
+# This Source Code Form is subject to the terms of the Mozilla Public
+# License, v. 2.0. If a copy of the MPL was not distributed with this
+# file, You can obtain one at https://mozilla.org/MPL/2.0/.
+
+import io
+import os
+from unittest.mock import patch
+
+import pytest
+from google.auth.credentials import AnonymousCredentials
+from google.cloud.exceptions import NotFound
+from google.cloud import storage
+
+from testlib.mini_poster import multipart_encode
+
+
+@pytest.fixture(autouse=True)
+def mock_generate_test_filepath():
+    with patch("antenna.ext.gcs.crashstorage.generate_test_filepath") as gtfp:
+        gtfp.return_value = "test/testwrite.txt"
+        yield
+
+
+@pytest.fixture
+def gcs_client():
+    if os.environ.get("STORAGE_EMULATOR_HOST"):
+        client = storage.Client(
+            credentials=AnonymousCredentials(),
+            project="test",
+        )
+        try:
+            yield client
+        finally:
+            for bucket in client.list_buckets():
+                try:
+                    bucket.delete(force=True)
+                except NotFound:
+                    pass  # same difference
+    else:
+        pytest.skip("requires gcs emulator")
+
+
+class TestGcsCrashStorageIntegration:
+    logging_names = ["antenna"]
+
+    def test_crash_storage(self, client, gcs_client):
+        bucket_name = "fakebucket"
+        # clean up bucket left around from previous tests
+        try:
+            gcs_client.get_bucket(bucket_name).delete(force=True)
+        except NotFound:
+            pass  # same difference
+        gcs_bucket = gcs_client.create_bucket(bucket_name)
+
+        crash_id = "de1bb258-cbbf-4589-a673-34f800160918"
+        data, headers = multipart_encode(
+            {
+                "uuid": crash_id,
+                "ProductName": "Firefox",
+                "Version": "1.0",
+                "upload_file_minidump": ("fakecrash.dump", io.BytesIO(b"abcd1234")),
+            }
+        )
+
+        # Rebuild the app the test client is using with relevant configuration.
+        client.rebuild_app(
+            {
+                "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.gcs.crashstorage.GcsCrashStorage",
+                "CRASHMOVER_CRASHSTORAGE_BUCKET_NAME": gcs_bucket.name,
+            }
+        )
+
+        result = client.simulate_post("/submit", headers=headers, body=data)
+
+        # Verify the collector returns a 200 status code and the crash id
+        # we fed it.
+        assert result.status_code == 200
+        assert result.content == f"CrashID=bp-{crash_id}\n".encode("utf-8")
+
+        # Assert we uploaded files to gcs
+        blobs = sorted(gcs_bucket.list_blobs(), key=lambda b: b.name)
+
+        blob_names = [b.name for b in blobs]
+        assert blob_names == [
+            "test/testwrite.txt",
+            f"v1/dump_names/{crash_id}",
+            f"v1/raw_crash/20160918/{crash_id}",
+            f"v1/upload_file_minidump/{crash_id}",
+        ]
+
+        blob_contents = [
+            b.download_as_bytes()
+            for b in blobs
+            # ignore the contents of the raw crash
+            if not b.name.startswith("v1/raw_crash")
+        ]
+        assert blob_contents == [
+            b"test",
+            b'["upload_file_minidump"]',
+            b"abcd1234",
+        ]
+
+    def test_missing_bucket_halts_startup(self, client, gcs_client):
+        bucket_name = "missingbucket"
+        # ensure bucket is actually missing
+        with pytest.raises(NotFound):
+            gcs_client.get_bucket(bucket_name)
+
+        with pytest.raises(NotFound) as excinfo:
+            # Rebuild the app the test client is using with relevant
+            # configuration. This calls .verify_write_to_bucket() which fails.
+            client.rebuild_app(
+                {
+                    "CRASHMOVER_CRASHSTORAGE_CLASS": "antenna.ext.gcs.crashstorage.GcsCrashStorage",
+                    "CRASHMOVER_CRASHSTORAGE_BUCKET_NAME": bucket_name,
+                }
+            )
+
+        assert f"b/{bucket_name}" in excinfo.value.args[0]

tests/unittest/conftest.py

@@ -7,103 +7,17 @@
 import sys
 from unittest import mock
 
-from everett.manager import ConfigManager, ConfigDictEnv, ConfigOSEnv
-from falcon.request import Request
-from falcon.testing.helpers import create_environ
-from falcon.testing.client import TestClient
-import markus
 from markus.testing import MetricsMock
 import pytest
 
 
-# Add repository root so we can import antenna and testlib.
+# Add repository root so we can import testlib.
 REPO_ROOT = Path(__file__).parent.parent.parent
 sys.path.insert(0, str(REPO_ROOT))
 
-from antenna.app import get_app, setup_logging  # noqa
-from antenna.app import reset_verify_funs  # noqa
 from testlib.s3mock import S3Mock  # noqa
 
 
-def pytest_runtest_setup():
-    # Make sure we set up logging and metrics to sane default values.
-    setup_logging(logging_level="DEBUG", debug=True, host_id="", processname="antenna")
-    markus.configure([{"class": "markus.backends.logging.LoggingMetrics"}])
-
-    # Wipe any registered verify functions
-    reset_verify_funs()
-
-
-@pytest.fixture
-def request_generator():
-    """Returns a Falcon Request generator"""
-
-    def _request_generator(method, path, query_string=None, headers=None, body=None):
-        env = create_environ(
-            method=method,
-            path=path,
-            query_string=(query_string or ""),
-            headers=headers,
-            body=body,
-        )
-        return Request(env)
-
-    return _request_generator
-
-
-class AntennaTestClient(TestClient):
-    """Test client to ease testing with Antenna API"""
-
-    @classmethod
-    def build_config(cls, new_config=None):
-        """Build ConfigManager using environment and overrides."""
-        new_config = new_config or {}
-        config_manager = ConfigManager(
-            environments=[ConfigDictEnv(new_config), ConfigOSEnv()]
-        )
-        return config_manager
-
-    def rebuild_app(self, new_config):
-        """Rebuilds the app
-
-        This is helpful if you've changed configuration and need to rebuild the
-        app so that components pick up the new configuration.
-
-        :arg new_config: dict of configuration to override normal values to build the
-            new app with
-
-        """
-        self.app = get_app(self.build_config(new_config))
-
-    def get_crashmover(self):
-        """Retrieves the crashmover from the AntennaApp."""
-        return self.app.app.crashmover
-
-    def get_resource_by_name(self, name):
-        """Retrieves the Falcon API resource by name"""
-        return self.app.app.get_resource_by_name(name)
-
-
-@pytest.fixture
-def client():
-    """Test client for the Antenna API
-
-    This creates an app and a test client that uses that app to submit HTTP
-    GET/POST requests.
-
-    The app that's created uses configuration defaults. If you need it to use
-    an app with a different configuration, you can rebuild the app with
-    different configuration::
-
-        def test_foo(client, tmpdir):
-            client.rebuild_app({
-                'BASEDIR': str(tmpdir)
-            })
-
-    """
-    return AntennaTestClient(get_app(AntennaTestClient.build_config()))
-
-
 @pytest.fixture
 def s3mock():
     """Returns an s3mock context that lets you do S3-related tests