diff --git a/README.rst b/README.rst index f79922a..ea5c379 100644 --- a/README.rst +++ b/README.rst @@ -1,23 +1,25 @@ -Collectfast – A Faster Collectstatic -==================================== +Collectfast +=========== -|Build Status| |Coverage Status| +A faster collectstatic command. -The fast ``collectstatic`` for Django projects with S3 as storage -backend. +|Build Status| |Coverage Status| **Features** -- Comparing and caching of md5 checksums before uploading -- Parallel file uploads using Python's multiprocessing module +- Compares and caches file checksums before uploading +- Parallelizes file uploads using Python's multiprocessing module + +**Supported Storage Backends** -Running Django's ``collectstatic`` command can become really slow as -more and more files are added to a project, especially if heavy -libraries such as jQuery UI are included in the source. This is a custom -management command that compares the md5 sum of files with S3 and -completely ignores ``modified_time``. The results of the hash lookups -are cached locally using your default Django cache. This can make -deploying much faster! +- ``storages.backends.s3boto.S3BotoStorage`` (deprecated) +- ``storages.backends.s3boto3.S3Boto3Storage`` + +Running Django's ``collectstatic`` command can become painfully slow as more +and more files are added to a project, especially when heavy libraries such as +jQuery UI are included in source code. Collectfast customizes the builtin +``collectstatic`` command, adding different optimizations to make uploading +large amounts of files much faster. Installation @@ -29,47 +31,54 @@ Install the app using pip: $ pip install Collectfast -Make sure you have this in your settings file and add ``'collectfast'`` -to your ``INSTALLED_APPS``: +Make sure you have this in your settings file and add ``'collectfast'`` to your +``INSTALLED_APPS``, before ``'django.contrib.staticfiles'``: .. code:: python - STATICFILES_STORAGE = "storages.backends.s3boto.S3BotoStorage" + STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage" + COLLECTFAST_STRATEGY = "collectfast.strategies.boto3.Boto3Strategy" INSTALLED_APPS = ( - # … + # ... 'collectfast', ) -``'collectfast'`` should come before ``'django.contrib.staticfiles'``. -Please note, that failure to do so will cause Django to use -``django.contrib.staticfiles``'s ``collectstatic``. +Collectfast comes with these upload strategies: + +- ``collectfast.strategies.boto.BotoStrategy`` for + ``storages.backends.s3boto.S3BotoStorage`` +- ``collectfast.strategies.boto3.Boto3Strategy`` for + ``storages.backends.s3boto3.S3Boto3Storage`` + +**Note:** ``'collectfast'`` must come before ``'django.contrib.staticfiles'`` in +``INSTALLED_APPS``. -**Note:** ``preload_metadata`` of the storage class will be overwritten as -`True`, see `#30 `_ +**Note:** Boto strategies will set ``preload_metadata`` on the remote storage to +``True``, see `#30 `_. Usage ----- -Collectfast overrides Django's builtin ``collectstatic`` command so just -run ``python manage.py collectstatic`` as normal. You can disable -Collectfast by using the ``--disable-collectfast`` option. +Collectfast overrides Django's builtin ``collectstatic`` command so just run +``python manage.py collectstatic`` as normal. You can disable Collectfast by +using the ``--disable-collectfast`` option. -You can also disable collectfast by setting -``COLLECTFAST_ENABLED = False`` in your settings file. This is useful -when using a local file storage backend for development. +You can also disable collectfast by setting ``COLLECTFAST_ENABLED = False`` in +your settings file. This is useful when using a local file storage backend for +development. Setup Dedicated Cache Backend ----------------------------- -It's recommended to setup a dedicated cache backend for Collectfast. -Every time Collectfast does not find a lookup for a file in the cache it -will trigger a lookup to the storage backend, so it's recommended to -have a fairly high ``TIMEOUT`` setting. +It's recommended to setup a dedicated cache backend for Collectfast. Every +time Collectfast does not find a lookup for a file in the cache it will trigger +a lookup to the storage backend, so it's recommended to have a fairly high +``TIMEOUT`` setting. -Set up your dedicated cache in settings.py with the -``COLLECTFAST_CACHE`` setting: +Set up your dedicated cache in settings.py with the ``COLLECTFAST_CACHE`` +setting: .. code:: python @@ -86,14 +95,13 @@ Set up your dedicated cache in settings.py with the By default Collectfast will use the ``default`` cache. -**Note:** Collectfast will never clean the cache of obsolete files. To -clean out the entire cache, use ``cache.clear()``. `Read more about -Django's cache +**Note:** Collectfast will never clean the cache of obsolete files. To clean +out the entire cache, use ``cache.clear()``. `Read more about Django's cache framework. `_ -**Note:** We recommend you to set the ``MAX_ENTRIES`` setting if you -have more than 300 static files, see -`#47 `_ +**Note:** We recommend you to set the ``MAX_ENTRIES`` setting if you have more +than 300 static files, see `#47 +`_ Enable Parallelization @@ -158,8 +166,7 @@ License ------- Collectfast is licensed under the MIT License, see LICENSE file for more -information. Previous versions of Collectfast was licensed under Creative -Commons Attribution-ShareAlike 3.0 Unported License. +information. .. |Build Status| image:: https://api.travis-ci.org/antonagestam/collectfast.svg?branch=master diff --git a/collectfast/__init__.py b/collectfast/__init__.py index e69de29..5becc17 100644 --- a/collectfast/__init__.py +++ b/collectfast/__init__.py @@ -0,0 +1 @@ +__version__ = "1.0.0" diff --git a/collectfast/management/commands/collectstatic.py b/collectfast/management/commands/collectstatic.py index 1a64712..ec8b86a 100644 --- a/collectfast/management/commands/collectstatic.py +++ b/collectfast/management/commands/collectstatic.py @@ -1,5 +1,7 @@ import warnings from multiprocessing.dummy import Pool +from typing import Any +from typing import Dict from typing import List from typing import Tuple @@ -9,6 +11,7 @@ from django.core.management.base import CommandParser from django.utils.encoding import smart_str +from collectfast import __version__ from collectfast import settings from collectfast.strategies import guess_strategy from collectfast.strategies import load_strategy @@ -31,7 +34,19 @@ class Command(collectstatic.Command): - def add_arguments(self, parser): + def __init__(self, *args, **kwargs): + # type: (Any, Any) -> None + super().__init__(*args, **kwargs) + self.num_copied_files = 0 + self.tasks = [] # type: List[Task] + self.collectfast_enabled = settings.enabled + self.strategy = strategy(self.storage) + + def get_version(self): + # type: () -> str + return __version__ + + def _add_arguments(self, parser): # type: (CommandParser) -> None super().add_arguments(parser) parser.add_argument( @@ -42,27 +57,22 @@ def add_arguments(self, parser): help="Disable Collectfast.", ) - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.num_copied_files = 0 - self.tasks = [] # type: List[Task] - self.collectfast_enabled = settings.enabled - self.strategy = strategy(self.storage) - def set_options(self, **options): + # type: (Any) -> None """Set options and handle deprecation.""" disable = options.pop("disable_collectfast", False) self.collectfast_enabled = not disable super().set_options(**options) def collect(self): + # type: () -> Dict[str, List[str]] """ Override collect to copy files concurrently. The tasks are populated by Command.copy_file() which is called by super().collect(). """ ret = super().collect() if settings.threads: - Pool(settings.threads).map(self.do_copy_file, self.tasks) + Pool(settings.threads).map(self.maybe_copy_file, self.tasks) return ret def handle(self, **options): @@ -72,7 +82,8 @@ def handle(self, **options): self.num_copied_files, "" if self.num_copied_files == 1 else "s" ) - def do_copy_file(self, args): + def maybe_copy_file(self, args): + # type: (Tuple[str, str, Storage]) -> None """Determine if file should be copied or not and handle exceptions.""" path, prefixed_path, source_storage = args @@ -83,7 +94,7 @@ def do_copy_file(self, args): if not self.strategy.should_copy_file( path, prefixed_path, source_storage ): - return False + return except Exception as e: if settings.debug: raise @@ -108,14 +119,12 @@ def copy_file(self, path, prefixed_path, source_storage): if settings.threads: self.tasks.append(args) else: - self.do_copy_file(args) + self.maybe_copy_file(args) def delete_file(self, path, prefixed_path, source_storage): # type: (str, str, Storage) -> bool """Override delete_file to skip modified time and exists lookups.""" if not self.collectfast_enabled: - # The delete_file method is incorrectly annotated in django-stubs, - # see https://github.com/typeddjango/django-stubs/issues/130 return super().delete_file(path, prefixed_path, source_storage) if not self.dry_run: self.log("Deleting '%s'" % path) diff --git a/setup.py b/setup.py index 1625378..dfd666b 100644 --- a/setup.py +++ b/setup.py @@ -3,10 +3,12 @@ from setuptools import find_packages +import collectfast + setup( name="Collectfast", description="A Faster Collectstatic", - version="1.0.0", + version=collectfast.__version__, long_description=open("README.rst").read(), author="Anton Agestam", author_email="msn@antonagestam.se",