Merge caace2c into 1e9811d

collectfast/management/commands/collectstatic.py

@@ -9,24 +9,30 @@
 from django.utils.encoding import smart_str
 
 from collectfast import settings
-from collectfast.boto import is_any_boto
-from collectfast.boto import reset_connection
-from collectfast.etag import should_copy_file
+from collectfast.strategies import guess_strategy
+from collectfast.strategies import load_strategy
 
 Task = Tuple[str, str, Storage]
 
+if settings.strategy is not None:
+    strategy = load_strategy(settings.strategy)
+elif settings.staticfiles_storage is not None:
+    warnings.warn(
+        "Falling back to guessing strategy for backwards compatibility. This "
+        "is deprecated and will be removed in a future release. Explicitly "
+        "set COLLECTFAST_STRATEGY to silence this warning."
+    )
+    strategy = load_strategy(guess_strategy(settings.staticfiles_storage))
+else:
+    raise RuntimeError(
+        "No strategy configured, please make sure COLLECTFAST_STRATEGY is set."
+    )
+
 
 class Command(collectstatic.Command):
     def add_arguments(self, parser):
         # type: (CommandParser) -> None
-        super(Command, self).add_arguments(parser)
-        parser.add_argument(
-            "--ignore-etag",
-            action="store_true",
-            dest="ignore_etag",
-            default=False,
-            help="Deprecated since 0.5.0, use --disable-collectfast instead.",
-        )
+        super().add_arguments(parser)
         parser.add_argument(
             "--disable-collectfast",
             action="store_true",
@@ -36,65 +42,45 @@ def add_arguments(self, parser):
         )
 
     def __init__(self, *args, **kwargs):
-        super(Command, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.num_copied_files = 0
         self.tasks = []  # type: List[Task]
         self.collectfast_enabled = settings.enabled
-
-        if is_any_boto(self.storage) and self.storage.preload_metadata is not True:
-            self.storage.preload_metadata = True
-            warnings.warn(
-                "Collectfast does not work properly without `preload_metadata` "
-                "set to `True` on the storage class. Overriding "
-                "`storage.preload_metadata` and continuing."
-            )
+        self.strategy = strategy(self.storage)
 
     def set_options(self, **options):
-        """
-        Set options and handle deprecation.
-        """
-        ignore_etag = options.pop("ignore_etag", False)
+        """Set options and handle deprecation."""
         disable = options.pop("disable_collectfast", False)
-        if ignore_etag:
-            warnings.warn(
-                "--ignore-etag is deprecated since 0.5.0, use "
-                "--disable-collectfast instead."
-            )
-        if ignore_etag or disable:
-            self.collectfast_enabled = False
-        super(Command, self).set_options(**options)
+        self.collectfast_enabled = not disable
+        super().set_options(**options)
 
     def collect(self):
         """
         Override collect to copy files concurrently. The tasks are populated by
         Command.copy_file() which is called by super().collect().
         """
-        ret = super(Command, self).collect()
+        ret = super().collect()
         if settings.threads:
             Pool(settings.threads).map(self.do_copy_file, self.tasks)
         return ret
 
     def handle(self, **options):
-        """
-        Override handle to supress summary output
-        """
-        super(Command, self).handle(**options)
+        """Override handle to suppress summary output."""
+        super().handle(**options)
         return "{} static file{} copied.".format(
             self.num_copied_files, "" if self.num_copied_files == 1 else "s"
         )
 
     def do_copy_file(self, args):
-        """
-        Determine if file should be copied or not and handle exceptions.
-        """
+        """Determine if file should be copied or not and handle exceptions."""
         path, prefixed_path, source_storage = args
 
-        reset_connection(self.storage)
+        self.strategy.pre_should_copy_hook()
 
         if self.collectfast_enabled and not self.dry_run:
             try:
-                if not should_copy_file(
-                    self.storage, path, prefixed_path, source_storage
+                if not self.strategy.should_copy_file(
+                    path, prefixed_path, source_storage
                 ):
                     return False
             except Exception as e:
@@ -109,12 +95,12 @@ def do_copy_file(self, args):
                 )
 
         self.num_copied_files += 1
-        return super(Command, self).copy_file(path, prefixed_path, source_storage)
+        return super().copy_file(path, prefixed_path, source_storage)
 
     def copy_file(self, path, prefixed_path, source_storage):
         # type: (str, str, Storage) -> None
         """
-        Appends path to task queue if threads are enabled, otherwise copies the
+        Append path to task queue if threads are enabled, otherwise copy the
         file with a blocking call.
         """
         args = (path, prefixed_path, source_storage)
@@ -125,15 +111,11 @@ def copy_file(self, path, prefixed_path, source_storage):
 
     def delete_file(self, path, prefixed_path, source_storage):
         # type: (str, str, Storage) -> bool
-        """
-        Override delete_file to skip modified time and exists lookups.
-        """
+        """Override delete_file to skip modified time and exists lookups."""
         if not self.collectfast_enabled:
             # The delete_file method is incorrectly annotated in django-stubs,
             # see https://github.com/typeddjango/django-stubs/issues/130
-            return super(Command, self).delete_file(  # type: ignore
-                path, prefixed_path, source_storage
-            )
+            return super().delete_file(path, prefixed_path, source_storage)
         if not self.dry_run:
             self.log("Deleting '%s'" % path)
             self.storage.delete(prefixed_path)

collectfast/settings.py

@@ -1,13 +1,22 @@
+from typing import Optional
+from typing import Sequence
+from typing import Type
+from typing import Union
+
 from django.conf import settings
+from typing_extensions import Final
+
 
-debug = getattr(settings, "COLLECTFAST_DEBUG", getattr(settings, "DEBUG", False))
+debug = getattr(
+    settings, "COLLECTFAST_DEBUG", getattr(settings, "DEBUG", False)
+)  # type: Final[bool]
 cache_key_prefix = getattr(
     settings, "COLLECTFAST_CACHE_KEY_PREFIX", "collectfast05_asset_"
-)
-cache = getattr(settings, "COLLECTFAST_CACHE", "default")
-threads = getattr(settings, "COLLECTFAST_THREADS", False)
-enabled = getattr(settings, "COLLECTFAST_ENABLED", True)
-is_gzipped = getattr(settings, "AWS_IS_GZIPPED", False)
+)  # type: Final[str]
+cache = getattr(settings, "COLLECTFAST_CACHE", "default")  # type: Final[str]
+threads = getattr(settings, "COLLECTFAST_THREADS", False)  # type: Final[bool]
+enabled = getattr(settings, "COLLECTFAST_ENABLED", True)  # type: Final[bool]
+aws_is_gzipped = getattr(settings, "AWS_IS_GZIPPED", False)  # type: Final[bool]
 gzip_content_types = getattr(
     settings,
     "GZIP_CONTENT_TYPES",
@@ -18,4 +27,10 @@
         "application/x-javascript",
         "image/svg+xml",
     ),
-)
+)  # type: Final[Sequence[str]]
+staticfiles_storage = getattr(
+    settings, "STATICFILES_STORAGE", None
+)  # type: Final[Optional[str]]
+strategy = getattr(
+    settings, "COLLECTFAST_STRATEGY", None
+)  # type: Final[Optional[Union[str, Type]]]

collectfast/strategies/__init__.py

@@ -0,0 +1,4 @@
+from .base import guess_strategy
+from .base import load_strategy
+
+__all__ = ("load_strategy", "guess_strategy")