Merge 056b012 into debb3c5

.travis.yml

@@ -1,7 +1,6 @@
 language: python
 sudo: required
 python:
-  - '3.5'
   - '3.6'
   - '3.7'
   - '3.8'
@@ -13,7 +12,6 @@ install:
   # to fail. Linting and static analysis only happens on the Python 3.8 builds.
   - pip install --upgrade -r lint-requirements.txt || true
 env:
-  - DJANGO=1.11
   - DJANGO=2.2
   - DJANGO=3.0
 before_script:
@@ -23,17 +21,13 @@ script:
   - 'if [[ $(python --version) == "Python 3.8."* ]]; then flake8; fi'
   - 'if [[ $(python --version) == "Python 3.8."* ]]; then black --check .; fi'
   - 'if [[ $(python --version) == "Python 3.8."* ]]; then sorti --check .; fi'
-  - 'if [[ $(python --version) == "Python 3.8."* && "$DJANGO" != "1.11" ]]; then pip install . && mypy .; fi'
+  - 'if [[ $(python --version) == "Python 3.8."* ]]; then pip install . && mypy .; fi'
   - 'if [[ $TRAVIS_REPO_SLUG != "antonagestam/collectfast" ]]; then export SKIP_LIVE_TESTS=true; fi'
   - coverage run --source collectfast -m pytest
 after_script:
   - coveralls
 matrix:
   exclude:
-    - env: DJANGO=3.0
-      python: '3.5'
-    - env: DJANGO=1.11
-      python: '3.8'
     - env: DJANGO=2.2
       python: '3.8'
 notifications:

Makefile

@@ -9,10 +9,18 @@ test-skip-live:
 test-coverage:
 	. storage-credentials && coverage run --source collectfast -m pytest
 
-distribute:
-	pip install --upgrade wheel twine setuptools
-	python setup.py sdist bdist_wheel
-	twine upload dist/*
+clean:
+	rm -rf Collectfast.egg-info __pycache__ build dist
+
+build: clean
+	python3 -m pip install --upgrade wheel twine setuptools
+	python3 setup.py sdist bdist_wheel
+
+distribute: build
+	python3 -m twine upload dist/*
+
+test-distribute: build
+	python3 -m twine upload --repository-url https://test.pypi.org/legacy/ dist/*
 
 lint:
 	flake8

README.md

@@ -0,0 +1,178 @@
+# Collectfast
+
+A faster collectstatic command.
+
+[![Build Status](https://api.travis-ci.org/antonagestam/collectfast.svg?branch=master)](https://travis-ci.org/antonagestam/collectfast)
+[![Coverage Status](https://coveralls.io/repos/github/antonagestam/collectfast/badge.svg?branch=master)](https://coveralls.io/github/antonagestam/collectfast?branch=master)
+
+**Features**
+
+- Efficiently decide what files to upload using cached checksums
+- Parallel file uploads
+
+**Supported Storage Backends**
+
+- `storages.backends.s3boto3.S3Boto3Storage`
+- `storages.backends.gcloud.GoogleCloudStorage`
+- `django.core.files.storage.FileSystemStorage`
+
+Running Django's `collectstatic` command can become painfully slow as more and
+more files are added to a project, especially when heavy libraries such as
+jQuery UI are included in source code. Collectfast customizes the builtin
+`collectstatic` command, adding different optimizations to make uploading large
+amounts of files much faster.
+
+
+## Installation
+
+Install the app using pip:
+
+```bash
+$ python3 -m pip install Collectfast
+```
+
+Make sure you have this in your settings file and add `'collectfast'` to your
+`INSTALLED_APPS`, before `'django.contrib.staticfiles'`:
+
+```python
+STATICFILES_STORAGE = "storages.backends.s3boto3.S3Boto3Storage"
+COLLECTFAST_STRATEGY = "collectfast.strategies.boto3.Boto3Strategy"
+INSTALLED_APPS = (
+    # ...
+    'collectfast',
+)
+```
+
+**Note:** `'collectfast'` must come before `'django.contrib.staticfiles'` in
+`INSTALLED_APPS`.
+
+**Note:** The boto strategy will set `preload_metadata` on the remote storage
+to `True`, see [#30][issue-30].
+
+[issue-30]: https://github.com/antonagestam/collectfast/issues/30
+
+##### Upload Strategies
+
+Collectfast Strategy|Storage Backend
+---|---
+collectfast.strategies.boto3.Boto3Strategy|storages.backends.s3boto3.S3Boto3Storage
+collectfast.strategies.gcloud.GoogleCloudStrategy|storages.backends.gcloud.GoogleCloudStorage
+collectfast.strategies.filesystem.FileSystemStrategy|django.core.files.storage.FileSystemStorage
+
+Custom strategies can also be made for backends not listed above by
+implementing the `collectfast.strategies.Strategy` ABC.
+
+
+## Usage
+
+Collectfast overrides Django's builtin `collectstatic` command so just run
+`python manage.py collectstatic` as normal.
+
+You can disable Collectfast by using the `--disable-collectfast` option or by
+setting `COLLECTFAST_ENABLED = False` in your settings file.
+
+### Setting Up a Dedicated Cache Backend
+
+It's recommended to setup a dedicated cache backend for Collectfast. Every time
+Collectfast does not find a lookup for a file in the cache it will trigger a
+lookup to the storage backend, so it's recommended to have a fairly high
+`TIMEOUT` setting.
+
+Configure your dedicated cache with the `COLLECTFAST_CACHE` setting:
+
+```python
+CACHES = {
+    'default': {
+        # Your default cache
+    },
+    'collectfast': {
+        # Your dedicated Collectfast cache
+    },
+}
+
+COLLECTFAST_CACHE = 'collectfast'
+```
+
+If `COLLECTFAST_CACHE` isn't set, the `default` cache will be used.
+
+**Note:** Collectfast will never clean the cache of obsolete files. To clean
+out the entire cache, use `cache.clear()`. [See docs for Django's cache
+framework][django-cache].
+
+**Note:** We recommend you to set the `MAX_ENTRIES` setting if you have more
+than 300 static files, see [#47][issue-47].
+
+[django-cache]: https://docs.djangoproject.com/en/stable/topics/cache/
+[issue-47]: https://github.com/antonagestam/collectfast/issues/47
+
+### Enable Parallel Uploads
+
+The parallelization feature enables parallel file uploads using Python's
+multiprocessing module. Enable it by setting the `COLLECTFAST_THREADS` setting.
+
+To enable parallel uploads, a dedicated cache backend must be setup and it must
+use a backend that is thread-safe, i.e. something other than Django's default
+LocMemCache.
+
+```python
+COLLECTFAST_THREADS = 20
+```
+
+
+## Debugging
+
+By default, Collectfast will suppress any exceptions that happens when copying
+and let Django's `collectstatic` handle it. To debug those suppressed errors
+you can set `COLLECTFAST_DEBUG = True` in your Django settings file.
+
+
+## Contribution
+
+Please feel free to contribute by using issues and pull requests. Discussion is
+open and welcome.
+
+### Testing
+
+The test suite is built to run against live S3 and GCloud buckets. You can
+disable live tests by setting `SKIP_LIVE_TESTS=true` or running `make
+test-skip-live`. To run live tests locally you need to provide API credentials
+to test against. Add the credentials to a file named `storage-credentials` in
+the root of the project directory:
+
+```bash
+export AWS_ACCESS_KEY_ID='...'
+export AWS_SECRET_ACCESS_KEY='...'
+export GCLOUD_CREDENTIALS='{...}'  # Google Cloud credentials as JSON
+```
+
+Install test dependencies and target Django version:
+
+```bash
+python3 -m pip install -r test-requirements.txt
+python3 -m pip install django==2.2
+```
+
+Run test suite:
+
+```bash
+make test
+```
+
+Code quality tools are broken out from test requirements because some of them
+only install on Python >= 3.7.
+
+```bash
+python3 -m pip install -r lint-requirements.txt
+```
+
+Run linters and static type check:
+
+```bash
+make lint
+```
+
+
+## License
+
+Collectfast is licensed under the MIT License, see LICENSE file for more
+information.