Hash gzipped files if AWS_IS_GZIPPED is True

.gitignore

@@ -2,3 +2,7 @@
 .coverage
 .DS_Store
 .coveragerc.swp
+.tox
+*.egg-info
+/static/
+/static_root/

collectfast/etag.py

@@ -1,7 +1,12 @@
+import gzip
 import hashlib
 import logging
+import mimetypes
 
 from django.core.cache import caches
+from django.utils.encoding import force_bytes
+from django.utils.six import BytesIO
+
 from storages.utils import safe_join
 
 from collectfast import settings
@@ -73,8 +78,22 @@ def get_file_hash(storage, path):
     Create md5 hash from file contents.
     """
     contents = storage.open(path).read()
-    file_hash = '"%s"' % hashlib.md5(contents).hexdigest()
-    return file_hash
+    file_hash = hashlib.md5(contents).hexdigest()
+
+    # Check if content should be gzipped and hash gzipped content
+    content_type = mimetypes.guess_type(path)[0] or 'application/octet-stream'
+    if settings.is_gzipped and content_type in settings.gzip_content_types:
+        cache_key = get_cache_key('gzip_hash_%s' % file_hash)
+        file_hash = cache.get(cache_key, False)
+        if file_hash is False:
+            buffer = BytesIO()
+            zf = gzip.GzipFile(mode='wb', compresslevel=6, fileobj=buffer, mtime=0.0)
+            zf.write(force_bytes(contents))
+            zf.close()
+            file_hash = hashlib.md5(buffer.getvalue()).hexdigest()
+            cache.set(cache_key, file_hash)
+
+    return '"%s"' % file_hash
 
 
 def has_matching_etag(remote_storage, source_storage, path, prefixed_path):

collectfast/settings.py

@@ -7,3 +7,8 @@
 cache = getattr(settings, "COLLECTFAST_CACHE", "default")
 threads = getattr(settings, "COLLECTFAST_THREADS", False)
 enabled = getattr(settings, "COLLECTFAST_ENABLED", True)
+is_gzipped = getattr(settings, "AWS_IS_GZIPPED", False)
+gzip_content_types = getattr(
+    settings, "GZIP_CONTENT_TYPES", (
+        "text/css", "text/javascript", "application/javascript",
+        "application/x-javascript", "image/svg+xml"))

collectfast/tests/test_command.py

@@ -5,7 +5,7 @@
 from mock import patch
 
 from .utils import test, clean_static_dir, create_static_file, override_setting
-from .utils import with_bucket
+from .utils import with_bucket, override_storage_attr
 
 
 def call_collectstatic(*args, **kwargs):
@@ -80,3 +80,17 @@ def test_ignore_etag_deprecated(case):
         warnings.simplefilter("always")
         call_collectstatic(ignore_etag=True)
         case.assertIn('ignore-etag is deprecated', str(w[0].message))
+
+
+@test
+@override_storage_attr("gzip", True)
+@override_setting("is_gzipped", True)
+@with_bucket
+def test_is_gzipped(case):
+    clean_static_dir()
+    create_static_file()
+    result = call_collectstatic()
+    case.assertIn("1 static file copied.", result)
+    # file state should now be cached
+    result = call_collectstatic()
+    case.assertIn("0 static files copied.", result)

collectfast/tests/utils.py

@@ -5,6 +5,7 @@
 import functools
 
 from django.conf import settings as django_settings
+from django.utils.module_loading import import_string
 
 import boto
 import moto
@@ -62,3 +63,17 @@ def wrapper(*args, **kwargs):
             return ret
         return wrapper
     return decorator
+
+
+def override_storage_attr(name, value):
+    def decorator(fn):
+        @functools.wraps(fn)
+        def wrapper(*args, **kwargs):
+            storage = import_string(getattr(django_settings, 'STATICFILES_STORAGE'))
+            original = getattr(storage, name)
+            setattr(storage, name, value)
+            ret = fn(*args, **kwargs)
+            setattr(storage, name, original)
+            return ret
+        return wrapper
+    return decorator

runtests.py

@@ -72,6 +72,8 @@ def main():
         "MIDDLEWARE_CLASSES": [],
         "AWS_PRELOAD_METADATA": True,
         "AWS_STORAGE_BUCKET_NAME": "collectfast",
+        "AWS_IS_GZIPPED": False,
+        "GZIP_CONTENT_TYPES": ('text/plain',),
     })
 
     if options.TEST_SUITE is not None: