Add CompressedStaticFilesStorage backend

Although the original compression support was written as a mixin it had
become so closely tied to the ManifestStaticFilesStorage class that it
didn't really work with anything else. This patch enables compression
support to be used with other backends, including the Django default.

Closes #194

docs/django.rst

@@ -85,12 +85,23 @@ safely be cached forever. To use it, just add this to your ``settings.py``:
 
    STATICFILES_STORAGE = 'whitenoise.storage.CompressedManifestStaticFilesStorage'
 
-If you need to compress files outside of the static files storage system you can
-use the supplied :ref:`command line utility <cli-utility>`
+This combines automatic compression with the caching behaviour provided by
+Django's ManifestStaticFilesStorage_ backend. If you want to apply compression
+but don't want the caching behaviour then you can use:
+
+.. code-block:: python
+
+   STATICFILES_STORAGE = 'whitenoise.storage.CompressedStaticFilesStorage'
 
 .. note:: If you are having problems after switching to the WhiteNoise storage
    backend please see the :ref:`troubleshooting guide <storage-troubleshoot>`.
 
+.. _ManifestStaticFilesStorage: https://docs.djangoproject.com/en/stable/ref/contrib/staticfiles/#manifeststaticfilesstorage
+
+If you need to compress files outside of the static files storage system you can
+use the supplied :ref:`command line utility <cli-utility>`
+
+
 .. _brotli-compression:
 
 Brotli compression

tests/test_storage.py

@@ -29,7 +29,7 @@ def reset_lazy_object(obj):
 
 
 @override_settings()
-class DjangoWhiteNoiseStorageTest(SimpleTestCase):
+class StorageTestBase(SimpleTestCase):
 
     @classmethod
     def setUpClass(cls):
@@ -38,16 +38,42 @@ def setUpClass(cls):
         cls.tmp = TEXT_TYPE(tempfile.mkdtemp())
         settings.STATICFILES_DIRS = [cls.files.directory]
         settings.STATIC_ROOT = cls.tmp
-        with override_settings(WHITENOISE_KEEP_ONLY_HASHED_FILES=True):
+        with override_settings(**cls.get_settings()):
             call_command('collectstatic', verbosity=0, interactive=False)
-        super(DjangoWhiteNoiseStorageTest, cls).setUpClass()
+        super(StorageTestBase, cls).setUpClass()
 
     @classmethod
     def tearDownClass(cls):
-        super(DjangoWhiteNoiseStorageTest, cls).tearDownClass()
+        super(StorageTestBase, cls).tearDownClass()
         reset_lazy_object(staticfiles_storage)
         shutil.rmtree(cls.tmp)
 
+
+class CompressedStaticFilesStorageTest(StorageTestBase):
+
+    @classmethod
+    def get_settings(self):
+        return {
+            'STATICFILES_STORAGE':
+                'whitenoise.storage.CompressedStaticFilesStorage'
+        }
+
+    def test_compressed_files_are_created(self):
+        for name in ['styles.css.gz', 'styles.css.br']:
+            path = os.path.join(settings.STATIC_ROOT, name)
+            self.assertTrue(os.path.exists(path))
+
+
+class CompressedManifestStaticFilesStorageTest(StorageTestBase):
+
+    @classmethod
+    def get_settings(self):
+        return {
+            'STATICFILES_STORAGE':
+                'whitenoise.storage.CompressedManifestStaticFilesStorage',
+            'WHITENOISE_KEEP_ONLY_HASHED_FILES': True
+        }
+
     def test_make_helpful_exception(self):
         class TriggerException(HashedFilesMixin):
             def exists(self, path):

whitenoise/storage.py

@@ -6,21 +6,117 @@
 import textwrap
 
 from django.conf import settings
-from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
+from django.contrib.staticfiles.storage import (
+        ManifestStaticFilesStorage, StaticFilesStorage)
 
 from .compress import Compressor
 
 
 class CompressedStaticFilesMixin(object):
     """
-    Wraps a StaticFilesStorage instance to create compressed versions of its
-    output files and, optionally, to delete the non-hashed files (i.e. those
-    without the hash in their name)
+    Wraps a StaticFilesStorage instance to compress output files
+    """
+
+    def post_process(self, *args, **kwargs):
+        super_post_process = getattr(
+            super(CompressedStaticFilesMixin, self),
+            'post_process',
+            self.fallback_post_process)
+        files = super_post_process(*args, **kwargs)
+        if not kwargs.get('dry_run'):
+            files = self.post_process_with_compression(files)
+        return files
+
+    # Only used if the class we're wrapping doesn't implement its own
+    # `post_process` method
+    def fallback_post_process(self, paths, dry_run=False, **options):
+        if not dry_run:
+            for path in paths:
+                yield path, None, False
+
+    def post_process_with_compression(self, files):
+        extensions = getattr(settings,
+                             'WHITENOISE_SKIP_COMPRESS_EXTENSIONS', None)
+        compressor = Compressor(extensions=extensions, quiet=True)
+        for name, hashed_name, processed in files:
+            yield name, hashed_name, processed
+            if isinstance(processed, Exception):
+                continue
+            unique_names = set(filter(None, [name, hashed_name]))
+            for name in unique_names:
+                if compressor.should_compress(name):
+                    path = self.path(name)
+                    prefix_len = len(path) - len(name)
+                    for compressed_path in compressor.compress(path):
+                        compressed_name = compressed_path[prefix_len:]
+                        yield name, compressed_name, True
+
+
+class CompressedStaticFilesStorage(
+        CompressedStaticFilesMixin, StaticFilesStorage):
+    pass
+
+
+class HelpfulExceptionMixin(object):
+    """
+    If a CSS file contains references to images, fonts etc that can't be found
+    then Django's `post_process` blows up with a not particularly helpful
+    ValueError that leads people to think WhiteNoise is broken.
+
+    Here we attempt to intercept such errors and reformat them to be more
+    helpful in revealing the source of the problem.
+    """
+
+    ERROR_MSG_RE = re.compile("^The file '(.+)' could not be found")
+
+    ERROR_MSG = textwrap.dedent(u"""\
+        {orig_message}
+
+        The {ext} file '{filename}' references a file which could not be found:
+          {missing}
+
+        Please check the URL references in this {ext} file, particularly any
+        relative paths which might be pointing to the wrong location.
+        """)
+
+    def post_process(self, *args, **kwargs):
+        files = super(HelpfulExceptionMixin, self).post_process(*args, **kwargs)
+        for name, hashed_name, processed in files:
+            if isinstance(processed, Exception):
+                processed = self.make_helpful_exception(processed, name)
+            yield name, hashed_name, processed
+
+    def make_helpful_exception(self, exception, name):
+        if isinstance(exception, ValueError):
+            message = exception.args[0] if len(exception.args) else ''
+            # Stringly typed exceptions. Yay!
+            match = self.ERROR_MSG_RE.search(message)
+            if match:
+                extension = os.path.splitext(name)[1].lstrip('.').upper()
+                message = self.ERROR_MSG.format(
+                        orig_message=message,
+                        filename=name,
+                        missing=match.group(1),
+                        ext=extension)
+                exception = MissingFileError(message)
+        return exception
+
+
+class MissingFileError(ValueError):
+    pass
+
+
+class CompressedManifestStaticFilesStorage(
+        HelpfulExceptionMixin, ManifestStaticFilesStorage):
+    """
+    Extends ManifestStaticFilesStorage instance to create compressed versions
+    of its output files and, optionally, to delete the non-hashed files (i.e.
+    those without the hash in their name)
     """
     _new_files = None
 
     def post_process(self, *args, **kwargs):
-        files = super(CompressedStaticFilesMixin, self).post_process(*args, **kwargs)
+        files = super(CompressedManifestStaticFilesStorage, self).post_process(*args, **kwargs)
         if not kwargs.get('dry_run'):
             files = self.post_process_with_compression(files)
         return files
@@ -52,7 +148,7 @@ def post_process_with_compression(self, files):
             yield name, compressed_name, True
 
     def hashed_name(self, *args, **kwargs):
-        name = super(CompressedStaticFilesMixin, self).hashed_name(*args, **kwargs)
+        name = super(CompressedManifestStaticFilesStorage, self).hashed_name(*args, **kwargs)
         if self._new_files is not None:
             self._new_files.add(self.clean_name(name))
         return name
@@ -86,58 +182,3 @@ def compress_files(self, names):
                 for compressed_path in compressor.compress(path):
                     compressed_name = compressed_path[prefix_len:]
                     yield name, compressed_name
-
-
-class HelpfulExceptionMixin(object):
-    """
-    If a CSS file contains references to images, fonts etc that can't be found
-    then Django's `post_process` blows up with a not particularly helpful
-    ValueError that leads people to think WhiteNoise is broken.
-
-    Here we attempt to intercept such errors and reformat them to be more
-    helpful in revealing the source of the problem.
-    """
-
-    ERROR_MSG_RE = re.compile("^The file '(.+)' could not be found")
-
-    ERROR_MSG = textwrap.dedent(u"""\
-        {orig_message}
-
-        The {ext} file '{filename}' references a file which could not be found:
-          {missing}
-
-        Please check the URL references in this {ext} file, particularly any
-        relative paths which might be pointing to the wrong location.
-        """)
-
-    def post_process(self, *args, **kwargs):
-        files = super(HelpfulExceptionMixin, self).post_process(*args, **kwargs)
-        for name, hashed_name, processed in files:
-            if isinstance(processed, Exception):
-                processed = self.make_helpful_exception(processed, name)
-            yield name, hashed_name, processed
-
-    def make_helpful_exception(self, exception, name):
-        if isinstance(exception, ValueError):
-            message = exception.args[0] if len(exception.args) else ''
-            # Stringly typed exceptions. Yay!
-            match = self.ERROR_MSG_RE.search(message)
-            if match:
-                extension = os.path.splitext(name)[1].lstrip('.').upper()
-                message = self.ERROR_MSG.format(
-                        orig_message=message,
-                        filename=name,
-                        missing=match.group(1),
-                        ext=extension)
-                exception = MissingFileError(message)
-        return exception
-
-
-class MissingFileError(ValueError):
-    pass
-
-
-class CompressedManifestStaticFilesStorage(
-        HelpfulExceptionMixin, CompressedStaticFilesMixin,
-        ManifestStaticFilesStorage):
-    pass