Merge 55a180b into cf201ce

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 sudo: required
 python:
-  - '3.5'
   - '3.6'
   - '3.7'
   - '3.8'
@@ -13,7 +12,6 @@ install:
   # to fail. Linting and static analysis only happens on the Python 3.8 builds.
   - pip install --upgrade -r lint-requirements.txt || true
 env:
-  - DJANGO=1.11
   - DJANGO=2.2
   - DJANGO=3.0
 before_script:
@@ -23,17 +21,13 @@ script:
   - 'if [[ $(python --version) == "Python 3.8."* ]]; then flake8; fi'
   - 'if [[ $(python --version) == "Python 3.8."* ]]; then black --check .; fi'
   - 'if [[ $(python --version) == "Python 3.8."* ]]; then sorti --check .; fi'
-  - 'if [[ $(python --version) == "Python 3.8."* && "$DJANGO" != "1.11" ]]; then pip install . && mypy .; fi'
+  - 'if [[ $(python --version) == "Python 3.8."* ]]; then pip install . && mypy .; fi'
   - 'if [[ $TRAVIS_REPO_SLUG != "antonagestam/collectfast" ]]; then export SKIP_LIVE_TESTS=true; fi'
   - coverage run --source collectfast -m pytest
 after_script:
   - coveralls
 matrix:
   exclude:
-    - env: DJANGO=3.0
-      python: '3.5'
-    - env: DJANGO=1.11
-      python: '3.8'
     - env: DJANGO=2.2
       python: '3.8'
 notifications:

README.rst

@@ -12,7 +12,6 @@ A faster collectstatic command.
 
 **Supported Storage Backends**
 
-- ``storages.backends.s3boto.S3BotoStorage`` (deprecated, will be removed in 2.0)
 - ``storages.backends.s3boto3.S3Boto3Storage``
 - ``storages.backends.gcloud.GoogleCloudStorage``
 - ``django.core.files.storage.FileSystemStorage``
@@ -58,8 +57,6 @@ Supported Strategies
 +--------------------------------------------------------+-----------------------------------------------+
 |Strategy class                                          |Storage class                                  |
 +========================================================+===============================================+
-|``collectfast.strategies.boto.BotoStrategy``            |``storages.backends.s3boto.S3BotoStorage``     |
-+--------------------------------------------------------+-----------------------------------------------+
 |``collectfast.strategies.boto3.Boto3Strategy``          |``storages.backends.s3boto3.S3Boto3Storage``   |
 +--------------------------------------------------------+-----------------------------------------------+
 |``collectfast.strategies.gcloud.GoogleCloudStrategy``   |``storages.backends.gcloud.GoogleCloudStorage``|

collectfast/management/commands/collectstatic.py

@@ -15,43 +15,34 @@
 from collectfast import __version__
 from collectfast import settings
 from collectfast.strategies import DisabledStrategy
-from collectfast.strategies import guess_strategy
 from collectfast.strategies import load_strategy
 from collectfast.strategies import Strategy
 
 Task = Tuple[str, str, Storage]
 
 
 class Command(collectstatic.Command):
-    def __init__(self, *args, **kwargs):
-        # type: (Any, Any) -> None
+    def __init__(self, *args: Any, **kwargs: Any) -> None:
         super().__init__(*args, **kwargs)
         self.num_copied_files = 0
-        self.tasks = []  # type: List[Task]
+        self.tasks: List[Task] = []
         self.collectfast_enabled = settings.enabled
-        self.strategy = DisabledStrategy(Storage())  # type: Strategy
+        self.strategy: Strategy = DisabledStrategy(Storage())
 
     @staticmethod
-    def _load_strategy():
-        # type: () -> Type[Strategy[Storage]]
+    def _load_strategy() -> Type[Strategy[Storage]]:
         strategy_str = getattr(django_settings, "COLLECTFAST_STRATEGY", None)
         if strategy_str is not None:
             return load_strategy(strategy_str)
 
-        storage_str = getattr(django_settings, "STATICFILES_STORAGE", None)
-        if storage_str is not None:
-            return load_strategy(guess_strategy(storage_str))
-
         raise ImproperlyConfigured(
             "No strategy configured, please make sure COLLECTFAST_STRATEGY is set."
         )
 
-    def get_version(self):
-        # type: () -> str
+    def get_version(self) -> str:
         return __version__
 
-    def add_arguments(self, parser):
-        # type: (CommandParser) -> None
+    def add_arguments(self, parser: CommandParser) -> None:
         super().add_arguments(parser)
         parser.add_argument(
             "--disable-collectfast",
@@ -61,8 +52,7 @@ def add_arguments(self, parser):
             help="Disable Collectfast.",
         )
 
-    def set_options(self, **options):
-        # type: (Any) -> None
+    def set_options(self, **options: Any) -> None:
         """Set options and handle deprecation."""
         self.collectfast_enabled = self.collectfast_enabled and not options.pop(
             "disable_collectfast"
@@ -71,8 +61,7 @@ def set_options(self, **options):
             self.strategy = self._load_strategy()(self.storage)
         super().set_options(**options)
 
-    def collect(self):
-        # type: () -> Dict[str, List[str]]
+    def collect(self) -> Dict[str, List[str]]:
         """
         Override collect to copy files concurrently. The tasks are populated by
         Command.copy_file() which is called by super().collect().
@@ -84,8 +73,7 @@ def collect(self):
             Pool(settings.threads).map(self.maybe_copy_file, self.tasks)
         return ret
 
-    def handle(self, *args, **options):
-        # type: (Any, Any) -> Optional[str]
+    def handle(self, *args: Any, **options: Any) -> Optional[str]:
         """Override handle to suppress summary output."""
         ret = super().handle(**options)
         if not self.collectfast_enabled:
@@ -94,8 +82,7 @@ def handle(self, *args, **options):
             self.num_copied_files, "" if self.num_copied_files == 1 else "s"
         )
 
-    def maybe_copy_file(self, args):
-        # type: (Tuple[str, str, Storage]) -> None
+    def maybe_copy_file(self, args: Task) -> None:
         """Determine if file should be copied or not and handle exceptions."""
         path, prefixed_path, source_storage = args
 
@@ -109,8 +96,7 @@ def maybe_copy_file(self, args):
         self.num_copied_files += 1
         return super().copy_file(path, prefixed_path, source_storage)
 
-    def copy_file(self, path, prefixed_path, source_storage):
-        # type: (str, str, Storage) -> None
+    def copy_file(self, path: str, prefixed_path: str, source_storage: Storage) -> None:
         """
         Append path to task queue if threads are enabled, otherwise copy the
         file with a blocking call.
@@ -121,8 +107,9 @@ def copy_file(self, path, prefixed_path, source_storage):
         else:
             self.maybe_copy_file(args)
 
-    def delete_file(self, path, prefixed_path, source_storage):
-        # type: (str, str, Storage) -> bool
+    def delete_file(
+        self, path: str, prefixed_path: str, source_storage: Storage
+    ) -> bool:
         """Override delete_file to skip modified time and exists lookups."""
         if not self.collectfast_enabled:
             return super().delete_file(path, prefixed_path, source_storage)

collectfast/settings.py

@@ -4,17 +4,17 @@
 from typing_extensions import Final
 
 
-debug = getattr(
+debug: Final[bool] = getattr(
     settings, "COLLECTFAST_DEBUG", getattr(settings, "DEBUG", False)
-)  # type: Final[bool]
-cache_key_prefix = getattr(
+)
+cache_key_prefix: Final[str] = getattr(
     settings, "COLLECTFAST_CACHE_KEY_PREFIX", "collectfast06_asset_"
-)  # type: Final[str]
-cache = getattr(settings, "COLLECTFAST_CACHE", "default")  # type: Final[str]
-threads = getattr(settings, "COLLECTFAST_THREADS", False)  # type: Final[bool]
-enabled = getattr(settings, "COLLECTFAST_ENABLED", True)  # type: Final[bool]
-aws_is_gzipped = getattr(settings, "AWS_IS_GZIPPED", False)  # type: Final[bool]
-gzip_content_types = getattr(
+)
+cache: Final[str] = getattr(settings, "COLLECTFAST_CACHE", "default")
+threads: Final[bool] = getattr(settings, "COLLECTFAST_THREADS", False)
+enabled: Final[bool] = getattr(settings, "COLLECTFAST_ENABLED", True)
+aws_is_gzipped: Final[bool] = getattr(settings, "AWS_IS_GZIPPED", False)
+gzip_content_types: Final[Sequence[str]] = getattr(
     settings,
     "GZIP_CONTENT_TYPES",
     (
@@ -24,4 +24,4 @@
         "application/x-javascript",
         "image/svg+xml",
     ),
-)  # type: Final[Sequence[str]]
+)

collectfast/strategies/__init__.py

@@ -1,6 +1,5 @@
 from .base import DisabledStrategy
-from .base import guess_strategy
 from .base import load_strategy
 from .base import Strategy
 
-__all__ = ("load_strategy", "guess_strategy", "Strategy", "DisabledStrategy")
+__all__ = ("load_strategy", "Strategy", "DisabledStrategy")

collectfast/strategies/base.py

@@ -4,7 +4,6 @@
 import logging
 import mimetypes
 import pydoc
-import warnings
 from functools import lru_cache
 from io import BytesIO
 from typing import ClassVar
@@ -20,7 +19,6 @@
 from django.core.exceptions import ImproperlyConfigured
 from django.core.files.storage import Storage
 from django.utils.encoding import force_bytes
-from typing_extensions import Final
 
 from collectfast import settings
 
@@ -35,15 +33,15 @@
 class Strategy(abc.ABC, Generic[_RemoteStorage]):
     # Exceptions raised by storage backend for delete calls to non-existing
     # objects. The command silently catches these.
-    delete_not_found_exception = ()  # type: ClassVar[Tuple[Type[Exception], ...]]
+    delete_not_found_exception: ClassVar[Tuple[Type[Exception], ...]] = ()
 
-    def __init__(self, remote_storage):
-        # type: (_RemoteStorage) -> None
+    def __init__(self, remote_storage: _RemoteStorage) -> None:
         self.remote_storage = remote_storage
 
     @abc.abstractmethod
-    def should_copy_file(self, path, prefixed_path, local_storage):
-        # type: (str, str, Storage) -> bool
+    def should_copy_file(
+        self, path: str, prefixed_path: str, local_storage: Storage
+    ) -> bool:
         """
         Called for each file before copying happens, this method decides
         whether a file should be copied or not. Return False to indicate that
@@ -52,31 +50,31 @@ def should_copy_file(self, path, prefixed_path, local_storage):
         """
         ...
 
-    def pre_should_copy_hook(self):
-        # type: () -> None
+    def pre_should_copy_hook(self) -> None:
         """Hook called before calling should_copy_file."""
         ...
 
 
 class HashStrategy(Strategy[_RemoteStorage], abc.ABC):
     use_gzip = False
 
-    def should_copy_file(self, path, prefixed_path, local_storage):
-        # type: (str, str, Storage) -> bool
+    def should_copy_file(
+        self, path: str, prefixed_path: str, local_storage: Storage
+    ) -> bool:
         local_hash = self.get_local_file_hash(path, local_storage)
         remote_hash = self.get_remote_file_hash(prefixed_path)
         return local_hash != remote_hash
 
-    def get_gzipped_local_file_hash(self, uncompressed_file_hash, path, contents):
-        # type: (str, str, str) -> str
+    def get_gzipped_local_file_hash(
+        self, uncompressed_file_hash: str, path: str, contents: str
+    ) -> str:
         buffer = BytesIO()
         zf = gzip.GzipFile(mode="wb", compresslevel=6, fileobj=buffer, mtime=0.0)
         zf.write(force_bytes(contents))
         zf.close()
         return hashlib.md5(buffer.getvalue()).hexdigest()
 
-    def get_local_file_hash(self, path, local_storage):
-        # type: (str, Storage) -> str
+    def get_local_file_hash(self, path: str, local_storage: Storage) -> str:
         """Create md5 hash from file contents."""
         contents = local_storage.open(path).read()
         file_hash = hashlib.md5(contents).hexdigest()
@@ -89,24 +87,22 @@ def get_local_file_hash(self, path, local_storage):
         return file_hash
 
     @abc.abstractmethod
-    def get_remote_file_hash(self, prefixed_path):
-        # type: (str) -> Optional[str]
+    def get_remote_file_hash(self, prefixed_path: str) -> Optional[str]:
         ...
 
 
 class CachingHashStrategy(HashStrategy[_RemoteStorage], abc.ABC):
     @lru_cache()
-    def get_cache_key(self, path):
-        # type: (str) -> str
+    def get_cache_key(self, path: str) -> str:
         path_hash = hashlib.md5(path.encode()).hexdigest()
         return settings.cache_key_prefix + path_hash
 
-    def invalidate_cached_hash(self, path):
-        # type: (str) -> None
+    def invalidate_cached_hash(self, path: str) -> None:
         cache.delete(self.get_cache_key(path))
 
-    def should_copy_file(self, path, prefixed_path, local_storage):
-        # type: (str, str, Storage) -> bool
+    def should_copy_file(
+        self, path: str, prefixed_path: str, local_storage: Storage
+    ) -> bool:
         local_hash = self.get_local_file_hash(path, local_storage)
         remote_hash = self.get_cached_remote_file_hash(path, prefixed_path)
         if local_hash != remote_hash:
@@ -116,8 +112,7 @@ def should_copy_file(self, path, prefixed_path, local_storage):
             return True
         return False
 
-    def get_cached_remote_file_hash(self, path, prefixed_path):
-        # type: (str, str) -> str
+    def get_cached_remote_file_hash(self, path: str, prefixed_path: str) -> str:
         """Cache the hash of the remote storage file."""
         cache_key = self.get_cache_key(path)
         hash_ = cache.get(cache_key, False)
@@ -126,8 +121,9 @@ def get_cached_remote_file_hash(self, path, prefixed_path):
             cache.set(cache_key, hash_)
         return str(hash_)
 
-    def get_gzipped_local_file_hash(self, uncompressed_file_hash, path, contents):
-        # type: (str, str, str) -> str
+    def get_gzipped_local_file_hash(
+        self, uncompressed_file_hash: str, path: str, contents: str
+    ) -> str:
         """Cache the hash of the gzipped local file."""
         cache_key = self.get_cache_key("gzip_hash_%s" % uncompressed_file_hash)
         file_hash = cache.get(cache_key, False)
@@ -140,12 +136,12 @@ def get_gzipped_local_file_hash(self, uncompressed_file_hash, path, contents):
 
 
 class DisabledStrategy(Strategy):
-    def should_copy_file(self, path, prefixed_path, local_storage):
-        # type: (str, str, Storage) -> NoReturn
+    def should_copy_file(
+        self, path: str, prefixed_path: str, local_storage: Storage
+    ) -> NoReturn:
         raise NotImplementedError
 
-    def pre_should_copy_hook(self):
-        # type: () -> NoReturn
+    def pre_should_copy_hook(self) -> NoReturn:
         raise NotImplementedError
 
 
@@ -158,42 +154,3 @@ def load_strategy(klass: Union[str, type, object]) -> Type[Strategy[Storage]]:
             % (Strategy.__module__, Strategy.__qualname__)
         )
     return klass
-
-
-_BOTO_STORAGE = "storages.backends.s3boto.S3BotoStorage"  # type: Final
-_BOTO_STRATEGY = "collectfast.strategies.boto.BotoStrategy"  # type: Final
-_BOTO3_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"  # type: Final
-_BOTO3_STRATEGY = "collectfast.strategies.boto3.Boto3Strategy"  # type: Final
-
-
-def _resolves_to_subclass(subclass_ref: str, superclass_ref: str) -> bool:
-    try:
-        subclass = pydoc.locate(subclass_ref)
-        assert isinstance(subclass, type)
-        superclass = pydoc.locate(superclass_ref)
-        assert isinstance(superclass, type)
-    except (ImportError, AssertionError, pydoc.ErrorDuringImport) as e:
-        logger.debug("Failed to import %s: %s" % (superclass_ref, e))
-        return False
-    return issubclass(subclass, superclass)
-
-
-def guess_strategy(storage: str) -> str:
-    warnings.warn(
-        "Falling back to guessing strategy for backwards compatibility. This "
-        "is deprecated and will be removed in a future release. Explicitly "
-        "set COLLECTFAST_STRATEGY to silence this warning.",
-        DeprecationWarning,
-    )
-    if storage == _BOTO_STORAGE:
-        return _BOTO_STRATEGY
-    if storage == _BOTO3_STORAGE:
-        return _BOTO3_STRATEGY
-    if _resolves_to_subclass(storage, _BOTO_STORAGE):
-        return _BOTO_STRATEGY
-    if _resolves_to_subclass(storage, _BOTO3_STORAGE):
-        return _BOTO3_STRATEGY
-    raise ImproperlyConfigured(
-        "Collectfast failed to guess strategy, please make sure "
-        "COLLECTFAST_STRATEGY is set."
-    )