#122: implement google cloud support

.gitignore

@@ -12,3 +12,4 @@ aws-credentials
 .mypy_cache
 .idea
 .python-version
+gcloud-credentials.json

collectfast/management/commands/collectstatic.py

@@ -123,8 +123,11 @@ def delete_file(self, path, prefixed_path, source_storage):
         if not self.collectfast_enabled:
             return super().delete_file(path, prefixed_path, source_storage)
         if not self.dry_run:
-            self.log("Deleting '%s' on remote storage" % path)
-            self.storage.delete(prefixed_path)
+            try:
+                self.log("Deleting '%s' on remote storage" % path)
+                self.storage.delete(prefixed_path)
+            except self.strategy.delete_not_found_exception:
+                pass
         else:
             self.log("Pretending to delete '%s'" % path)
         return True

collectfast/strategies/base.py

@@ -6,8 +6,10 @@
 from functools import lru_cache
 from io import BytesIO
 from pydoc import locate
+from typing import ClassVar
 from typing import Generic
 from typing import Optional
+from typing import Tuple
 from typing import Type
 from typing import TypeVar
 from typing import Union
@@ -28,6 +30,10 @@
 
 
 class Strategy(abc.ABC, Generic[_RemoteStorage]):
+    # Exceptions raised by storage backend for delete calls to non-existing
+    # objects. The command silently catches these.
+    delete_not_found_exception = ()  # type: ClassVar[Tuple[Type[Exception], ...]]
+
     def __init__(self, remote_storage):
         # type: (_RemoteStorage) -> None
         self.remote_storage = remote_storage

collectfast/strategies/gcloud.py

@@ -0,0 +1,21 @@
+import base64
+import binascii
+from typing import Optional
+
+from google.api_core.exceptions import NotFound
+from storages.backends.gcloud import GoogleCloudStorage
+
+from .base import CachingHashStrategy
+
+
+class GoogleCloudStrategy(CachingHashStrategy[GoogleCloudStorage]):
+    delete_not_found_exception = (NotFound,)
+
+    def get_remote_file_hash(self, prefixed_path):
+        # type: (str) -> Optional[str]
+        normalized_path = prefixed_path.replace("\\", "/")
+        blob = self.remote_storage.bucket.get_blob(normalized_path)
+        if blob is None:
+            return blob
+        md5_base64 = blob._properties["md5Hash"]
+        return binascii.hexlify(base64.urlsafe_b64decode(md5_base64)).decode()

collectfast/tests/settings.py

@@ -1,11 +1,14 @@
 import os
 import pathlib
 
+from google.oauth2 import service_account
+
 base_path = pathlib.Path.cwd()
 
 # Set USE_TZ to True to work around bug in django-storages
 USE_TZ = True
 
+SECRET_KEY = "nonsense"
 CACHES = {
     "default": {
         "BACKEND": "django.core.cache.backends.locmem.LocMemCache",
@@ -25,10 +28,13 @@
 STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
 COLLECTFAST_STRATEGY = "collectfast.strategies.boto3.Boto3Strategy"
 COLLECTFAST_DEBUG = True
+
+GZIP_CONTENT_TYPES = ("text/plain",)
+
+# AWS
 AWS_PRELOAD_METADATA = True
 AWS_STORAGE_BUCKET_NAME = "collectfast"
 AWS_IS_GZIPPED = False
-GZIP_CONTENT_TYPES = ("text/plain",)
 AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "").strip()
 AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "").strip()
 AWS_S3_REGION_NAME = "eu-central-1"
@@ -37,4 +43,9 @@
 AWS_DEFAULT_ACL = None
 S3_USE_SIGV4 = True
 AWS_S3_HOST = "s3.eu-central-1.amazonaws.com"
-SECRET_KEY = "nonsense"
+
+# Google Cloud
+GS_CREDENTIALS = service_account.Credentials.from_service_account_file(
+    str(base_path / "gcloud-credentials.json")
+)
+GS_BUCKET_NAME = "roasted-dufus"

collectfast/tests/test_command.py

@@ -14,16 +14,26 @@
 from .utils import test_many
 from collectfast.management.commands.collectstatic import Command
 
-test_boto_and_boto3 = test_many(
-    boto3=override_django_settings(
+aws_backend_confs = {
+    "boto3": override_django_settings(
         STATICFILES_STORAGE="storages.backends.s3boto3.S3Boto3Storage",
         COLLECTFAST_STRATEGY="collectfast.strategies.boto3.Boto3Strategy",
     ),
-    boto=override_django_settings(
+    "boto": override_django_settings(
         STATICFILES_STORAGE="storages.backends.s3boto.S3BotoStorage",
         COLLECTFAST_STRATEGY="collectfast.strategies.boto.BotoStrategy",
     ),
-)
+}
+all_backend_confs = {
+    "google": override_django_settings(
+        STATICFILES_STORAGE="storages.backends.gcloud.GoogleCloudStorage",
+        COLLECTFAST_STRATEGY="collectfast.strategies.gcloud.GoogleCloudStrategy",
+    ),
+    **aws_backend_confs,
+}
+
+test_aws_backends = test_many(**aws_backend_confs)
+test_all_backends = test_many(**all_backend_confs)
 
 
 def call_collectstatic(*args, **kwargs):
@@ -35,7 +45,7 @@ def call_collectstatic(*args, **kwargs):
     return out.getvalue()
 
 
-@test_boto_and_boto3
+@test_all_backends
 def test_basics(case):
     # type: (TestCase) -> None
     clean_static_dir()
@@ -45,7 +55,7 @@ def test_basics(case):
     case.assertIn("0 static files copied.", call_collectstatic())
 
 
-@test_boto_and_boto3
+@test_all_backends
 @override_setting("threads", 5)
 def test_threads(case):
     # type: (TestCase) -> None
@@ -89,10 +99,10 @@ def test_dry_run(case):
     case.assertTrue("Pretending to delete", result)
 
 
-@test_boto_and_boto3
+@test_aws_backends
 @override_storage_attr("gzip", True)
 @override_setting("aws_is_gzipped", True)
-def test_is_gzipped(case):
+def test_aws_is_gzipped(case):
     # type: (TestCase) -> None
     clean_static_dir()
     create_static_file()

collectfast/tests/utils.py

@@ -46,7 +46,7 @@ def test(func):
         ...     case.assertEqual(fn(), 1337)
         """
         case_dict = {
-            "%s[%s]" % (func.__name__, mutation_name): mutation(func)
+            "test_%s" % mutation_name: mutation(func)
             for mutation_name, mutation in mutations.items()
         }
 

test-requirements.txt

@@ -5,3 +5,4 @@ coveralls
 django-storages
 boto3
 boto
+google-cloud-storage