#122: implement google cloud support

Co-authored-by: Jeremy Nauta <jeremypnauta@gmail.com>

.gitignore

@@ -8,7 +8,7 @@
 /static_root/
 /build/
 dist/
-aws-credentials
+storage-credentials
 .mypy_cache
 .idea
 .python-version

Makefile

@@ -1,10 +1,10 @@
 SHELL := /usr/bin/env bash
 
 test:
-	. aws-credentials && ./runtests.py
+	. storage-credentials && ./runtests.py
 
 test-coverage:
-	. aws-credentials && coverage run --source collectfast ./runtests.py
+	. storage-credentials && coverage run --source collectfast ./runtests.py
 
 distribute:
 	pip install --upgrade wheel twine setuptools

README.rst

@@ -138,7 +138,7 @@ open and welcome.
 
 The test suite is built to run against an S3 bucket. To be able to test locally
 you need to provide AWS credentials for a bucket to test against. Add the
-credentials to a file named `aws-credentials` in the root of the project
+credentials to a file named `storage-credentials` in the root of the project
 directory:
 
 .. code:: bash

collectfast/__init__.py

@@ -1 +1 @@
-__version__ = "1.1.1"
+__version__ = "1.2.0"

collectfast/management/commands/collectstatic.py

@@ -117,8 +117,11 @@ def delete_file(self, path, prefixed_path, source_storage):
         if not self.collectfast_enabled:
             return super().delete_file(path, prefixed_path, source_storage)
         if not self.dry_run:
-            self.log("Deleting '%s' on remote storage" % path)
-            self.storage.delete(prefixed_path)
+            try:
+                self.log("Deleting '%s' on remote storage" % path)
+                self.storage.delete(prefixed_path)
+            except self.strategy.delete_not_found_exception:
+                pass
         else:
             self.log("Pretending to delete '%s'" % path)
         return True

collectfast/strategies/base.py

@@ -7,8 +7,10 @@
 from functools import lru_cache
 from io import BytesIO
 from pydoc import locate
+from typing import ClassVar
 from typing import Generic
 from typing import Optional
+from typing import Tuple
 from typing import Type
 from typing import TypeVar
 from typing import Union
@@ -30,6 +32,10 @@
 
 
 class Strategy(abc.ABC, Generic[_RemoteStorage]):
+    # Exceptions raised by storage backend for delete calls to non-existing
+    # objects. The command silently catches these.
+    delete_not_found_exception = ()  # type: ClassVar[Tuple[Type[Exception], ...]]
+
     def __init__(self, remote_storage):
         # type: (_RemoteStorage) -> None
         self.remote_storage = remote_storage

collectfast/strategies/gcloud.py

@@ -0,0 +1,21 @@
+import base64
+import binascii
+from typing import Optional
+
+from google.api_core.exceptions import NotFound
+from storages.backends.gcloud import GoogleCloudStorage
+
+from .base import CachingHashStrategy
+
+
+class GoogleCloudStrategy(CachingHashStrategy[GoogleCloudStorage]):
+    delete_not_found_exception = (NotFound,)
+
+    def get_remote_file_hash(self, prefixed_path):
+        # type: (str) -> Optional[str]
+        normalized_path = prefixed_path.replace("\\", "/")
+        blob = self.remote_storage.bucket.get_blob(normalized_path)
+        if blob is None:
+            return blob
+        md5_base64 = blob._properties["md5Hash"]
+        return binascii.hexlify(base64.urlsafe_b64decode(md5_base64)).decode()

collectfast/tests/settings.py

@@ -1,11 +1,15 @@
 import os
 import pathlib
+import tempfile
+
+from google.oauth2 import service_account
 
 base_path = pathlib.Path.cwd()
 
 # Set USE_TZ to True to work around bug in django-storages
 USE_TZ = True
 
+SECRET_KEY = "nonsense"
 CACHES = {
     "default": {
         "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
@@ -25,10 +29,13 @@
 STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 COLLECTFAST_STRATEGY = "collectfast.strategies.boto3.Boto3Strategy"
 COLLECTFAST_DEBUG = True
+
+GZIP_CONTENT_TYPES = ("text/plain",)
+
+# AWS
 AWS_PRELOAD_METADATA = True
 AWS_STORAGE_BUCKET_NAME = "collectfast"
 AWS_IS_GZIPPED = False
-GZIP_CONTENT_TYPES = ("text/plain",)
 AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "").strip()
 AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "").strip()
 AWS_S3_REGION_NAME = "eu-central-1"
@@ -37,4 +44,16 @@
 AWS_DEFAULT_ACL = None
 S3_USE_SIGV4 = True
 AWS_S3_HOST = "s3.eu-central-1.amazonaws.com"
-SECRET_KEY = "nonsense"
+
+# Google Cloud
+gcloud_credentials_json = os.environ.get("GCLOUD_CREDENTIALS", "").strip()
+if not gcloud_credentials_json:
+    GS_CREDENTIALS = None
+else:
+    with tempfile.NamedTemporaryFile() as file:
+        file.write(gcloud_credentials_json.encode())
+        file.read()
+        GS_CREDENTIALS = service_account.Credentials.from_service_account_file(
+            file.name
+        )
+GS_BUCKET_NAME = "roasted-dufus"

collectfast/tests/test_command.py

@@ -14,17 +14,30 @@
 from .utils import test_many
 from collectfast.management.commands.collectstatic import Command
 
-test_boto_and_boto3 = test_many(
-    boto3=override_django_settings(
+aws_backend_confs = {
+    "boto3": override_django_settings(
         STATICFILES_STORAGE="storages.backends.s3boto3.S3Boto3Storage",
         COLLECTFAST_STRATEGY="collectfast.strategies.boto3.Boto3Strategy",
     ),
-    boto=override_django_settings(
+    "boto": override_django_settings(
         STATICFILES_STORAGE="storages.backends.s3boto.S3BotoStorage",
         COLLECTFAST_STRATEGY="collectfast.strategies.boto.BotoStrategy",
     ),
+}
+# use PEP448-style unpacking instead of copy+update once 3.4 support is dropped
+all_backend_confs = aws_backend_confs.copy()
+all_backend_confs.update(
+    {
+        "google": override_django_settings(
+            STATICFILES_STORAGE="storages.backends.gcloud.GoogleCloudStorage",
+            COLLECTFAST_STRATEGY="collectfast.strategies.gcloud.GoogleCloudStrategy",
+        )
+    }
 )
 
+test_aws_backends = test_many(**aws_backend_confs)
+test_all_backends = test_many(**all_backend_confs)
+
 
 def call_collectstatic(*args, **kwargs):
     # type: (Any, Any) -> str
@@ -35,7 +48,7 @@ def call_collectstatic(*args, **kwargs):
     return out.getvalue()
 
 
-@test_boto_and_boto3
+@test_all_backends
 def test_basics(case):
     # type: (TestCase) -> None
     clean_static_dir()
@@ -45,7 +58,7 @@ def test_basics(case):
     case.assertIn("0 static files copied.", call_collectstatic())
 
 
-@test_boto_and_boto3
+@test_all_backends
 @override_setting("threads", 5)
 def test_threads(case):
     # type: (TestCase) -> None
@@ -89,10 +102,10 @@ def test_dry_run(case):
     case.assertTrue("Pretending to delete", result)
 
 
-@test_boto_and_boto3
+@test_aws_backends
 @override_storage_attr("gzip", True)
 @override_setting("aws_is_gzipped", True)
-def test_is_gzipped(case):
+def test_aws_is_gzipped(case):
     # type: (TestCase) -> None
     clean_static_dir()
     create_static_file()

collectfast/tests/utils.py

@@ -46,7 +46,7 @@ def test(func):
         ...     case.assertEqual(fn(), 1337)
         """
         case_dict = {
-            "%s[%s]" % (func.__name__, mutation_name): mutation(func)
+            "test_%s" % mutation_name: mutation(func)
             for mutation_name, mutation in mutations.items()
         }
 

test-requirements.txt

@@ -5,3 +5,4 @@ coveralls
 django-storages
 boto3
 boto
+google-cloud-storage