diff --git a/.gitignore b/.gitignore index 10f100d..6319548 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,7 @@ /static_root/ /build/ dist/ -aws-credentials +storage-credentials .mypy_cache .idea .python-version diff --git a/Makefile b/Makefile index f99ed3f..46ea9ab 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,10 @@ SHELL := /usr/bin/env bash test: - . aws-credentials && ./runtests.py + . storage-credentials && ./runtests.py test-coverage: - . aws-credentials && coverage run --source collectfast ./runtests.py + . storage-credentials && coverage run --source collectfast ./runtests.py distribute: pip install --upgrade wheel twine setuptools diff --git a/README.rst b/README.rst index 5512cdd..10abd4d 100644 --- a/README.rst +++ b/README.rst @@ -138,7 +138,7 @@ open and welcome. The test suite is built to run against an S3 bucket. To be able to test locally you need to provide AWS credentials for a bucket to test against. Add the -credentials to a file named `aws-credentials` in the root of the project +credentials to a file named `storage-credentials` in the root of the project directory: .. code:: bash diff --git a/collectfast/__init__.py b/collectfast/__init__.py index a82b376..c68196d 100644 --- a/collectfast/__init__.py +++ b/collectfast/__init__.py @@ -1 +1 @@ -__version__ = "1.1.1" +__version__ = "1.2.0" diff --git a/collectfast/management/commands/collectstatic.py b/collectfast/management/commands/collectstatic.py index cf0f57d..456c410 100644 --- a/collectfast/management/commands/collectstatic.py +++ b/collectfast/management/commands/collectstatic.py @@ -117,8 +117,11 @@ def delete_file(self, path, prefixed_path, source_storage): if not self.collectfast_enabled: return super().delete_file(path, prefixed_path, source_storage) if not self.dry_run: - self.log("Deleting '%s' on remote storage" % path) - self.storage.delete(prefixed_path) + try: + self.log("Deleting '%s' on remote storage" % path) + self.storage.delete(prefixed_path) + except self.strategy.delete_not_found_exception: + pass else: self.log("Pretending to delete '%s'" % path) return True diff --git a/collectfast/strategies/base.py b/collectfast/strategies/base.py index 6c4e496..f75848f 100644 --- a/collectfast/strategies/base.py +++ b/collectfast/strategies/base.py @@ -7,8 +7,10 @@ from functools import lru_cache from io import BytesIO from pydoc import locate +from typing import ClassVar from typing import Generic from typing import Optional +from typing import Tuple from typing import Type from typing import TypeVar from typing import Union @@ -30,6 +32,10 @@ class Strategy(abc.ABC, Generic[_RemoteStorage]): + # Exceptions raised by storage backend for delete calls to non-existing + # objects. The command silently catches these. + delete_not_found_exception = () # type: ClassVar[Tuple[Type[Exception], ...]] + def __init__(self, remote_storage): # type: (_RemoteStorage) -> None self.remote_storage = remote_storage diff --git a/collectfast/strategies/gcloud.py b/collectfast/strategies/gcloud.py new file mode 100644 index 0000000..9f3f077 --- /dev/null +++ b/collectfast/strategies/gcloud.py @@ -0,0 +1,21 @@ +import base64 +import binascii +from typing import Optional + +from google.api_core.exceptions import NotFound +from storages.backends.gcloud import GoogleCloudStorage + +from .base import CachingHashStrategy + + +class GoogleCloudStrategy(CachingHashStrategy[GoogleCloudStorage]): + delete_not_found_exception = (NotFound,) + + def get_remote_file_hash(self, prefixed_path): + # type: (str) -> Optional[str] + normalized_path = prefixed_path.replace("\\", "/") + blob = self.remote_storage.bucket.get_blob(normalized_path) + if blob is None: + return blob + md5_base64 = blob._properties["md5Hash"] + return binascii.hexlify(base64.urlsafe_b64decode(md5_base64)).decode() diff --git a/collectfast/tests/settings.py b/collectfast/tests/settings.py index b8f27aa..af5781e 100644 --- a/collectfast/tests/settings.py +++ b/collectfast/tests/settings.py @@ -1,11 +1,15 @@ import os import pathlib +import tempfile + +from google.oauth2 import service_account base_path = pathlib.Path.cwd() # Set USE_TZ to True to work around bug in django-storages USE_TZ = True +SECRET_KEY = "nonsense" CACHES = { "default": { "BACKEND": "django.core.cache.backends.locmem.LocMemCache", @@ -25,10 +29,13 @@ STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" COLLECTFAST_STRATEGY = "collectfast.strategies.boto3.Boto3Strategy" COLLECTFAST_DEBUG = True + +GZIP_CONTENT_TYPES = ("text/plain",) + +# AWS AWS_PRELOAD_METADATA = True AWS_STORAGE_BUCKET_NAME = "collectfast" AWS_IS_GZIPPED = False -GZIP_CONTENT_TYPES = ("text/plain",) AWS_ACCESS_KEY_ID = os.environ.get("AWS_ACCESS_KEY_ID", "").strip() AWS_SECRET_ACCESS_KEY = os.environ.get("AWS_SECRET_ACCESS_KEY", "").strip() AWS_S3_REGION_NAME = "eu-central-1" @@ -37,4 +44,16 @@ AWS_DEFAULT_ACL = None S3_USE_SIGV4 = True AWS_S3_HOST = "s3.eu-central-1.amazonaws.com" -SECRET_KEY = "nonsense" + +# Google Cloud +gcloud_credentials_json = os.environ.get("GCLOUD_CREDENTIALS", "").strip() +if not gcloud_credentials_json: + GS_CREDENTIALS = None +else: + with tempfile.NamedTemporaryFile() as file: + file.write(gcloud_credentials_json.encode()) + file.read() + GS_CREDENTIALS = service_account.Credentials.from_service_account_file( + file.name + ) +GS_BUCKET_NAME = "roasted-dufus" diff --git a/collectfast/tests/strategies/test_boto.py b/collectfast/tests/strategies/test_boto.py deleted file mode 100644 index e69de29..0000000 diff --git a/collectfast/tests/strategies/test_boto3.py b/collectfast/tests/strategies/test_boto3.py deleted file mode 100644 index e69de29..0000000 diff --git a/collectfast/tests/test_command.py b/collectfast/tests/test_command.py index 0d5d1b3..9b8098f 100644 --- a/collectfast/tests/test_command.py +++ b/collectfast/tests/test_command.py @@ -14,17 +14,30 @@ from .utils import test_many from collectfast.management.commands.collectstatic import Command -test_boto_and_boto3 = test_many( - boto3=override_django_settings( +aws_backend_confs = { + "boto3": override_django_settings( STATICFILES_STORAGE="storages.backends.s3boto3.S3Boto3Storage", COLLECTFAST_STRATEGY="collectfast.strategies.boto3.Boto3Strategy", ), - boto=override_django_settings( + "boto": override_django_settings( STATICFILES_STORAGE="storages.backends.s3boto.S3BotoStorage", COLLECTFAST_STRATEGY="collectfast.strategies.boto.BotoStrategy", ), +} +# use PEP448-style unpacking instead of copy+update once 3.4 support is dropped +all_backend_confs = aws_backend_confs.copy() +all_backend_confs.update( + { + "google": override_django_settings( + STATICFILES_STORAGE="storages.backends.gcloud.GoogleCloudStorage", + COLLECTFAST_STRATEGY="collectfast.strategies.gcloud.GoogleCloudStrategy", + ) + } ) +test_aws_backends = test_many(**aws_backend_confs) +test_all_backends = test_many(**all_backend_confs) + def call_collectstatic(*args, **kwargs): # type: (Any, Any) -> str @@ -35,7 +48,7 @@ def call_collectstatic(*args, **kwargs): return out.getvalue() -@test_boto_and_boto3 +@test_all_backends def test_basics(case): # type: (TestCase) -> None clean_static_dir() @@ -45,7 +58,7 @@ def test_basics(case): case.assertIn("0 static files copied.", call_collectstatic()) -@test_boto_and_boto3 +@test_all_backends @override_setting("threads", 5) def test_threads(case): # type: (TestCase) -> None @@ -89,10 +102,10 @@ def test_dry_run(case): case.assertTrue("Pretending to delete", result) -@test_boto_and_boto3 +@test_aws_backends @override_storage_attr("gzip", True) @override_setting("aws_is_gzipped", True) -def test_is_gzipped(case): +def test_aws_is_gzipped(case): # type: (TestCase) -> None clean_static_dir() create_static_file() diff --git a/collectfast/tests/utils.py b/collectfast/tests/utils.py index 14a62e7..95715ab 100644 --- a/collectfast/tests/utils.py +++ b/collectfast/tests/utils.py @@ -46,7 +46,7 @@ def test(func): ... case.assertEqual(fn(), 1337) """ case_dict = { - "%s[%s]" % (func.__name__, mutation_name): mutation(func) + "test_%s" % mutation_name: mutation(func) for mutation_name, mutation in mutations.items() } diff --git a/test-requirements.txt b/test-requirements.txt index 2922586..d079706 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -5,3 +5,4 @@ coveralls django-storages boto3 boto +google-cloud-storage