Merge 0489ed1 into 1eca6a5

barbeque/static.py

@@ -1,8 +1,20 @@
+"""ServeStaticFileMiddleware facilitates serving static files on docker.
+
+When serving static files with docker we first serve them through Django,
+it happens only for the first time a static file is requested,
+then static files are cached by nginx.
+
+Another function of the middleware is to maps static files to their hashed names,
+so it is possible to reduce static files to just files with hashed names
+(without keeping the original duplicates).
+"""
+
 import re
 
 from django.conf import settings
 from django.http.response import Http404
 from django.views.static import serve
+from django.utils.module_loading import import_string
 
 
 class ServeStaticFileMiddleware(object):
@@ -11,22 +23,68 @@ def __init__(self, get_response=None):
         self.get_response = get_response
         self.path_regex = re.compile(
             r'^/{0}(.*)$'.format(settings.STATIC_URL.strip('/')))
+        try:
+            self.manifest = self.load_staticfiles_manifest()
+        except AttributeError:
+            self.manifest = None
 
     def __call__(self, request):
         response = self.get_response(request)
         return self.process_response(request, response)
 
+    def serve_response(self, request, file_path):
+        return serve(request, file_path, document_root=settings.STATIC_ROOT)
+
+    def load_staticfiles_manifest(self):
+        """Staticfiles manifest maps original names to names with hash.
+        The method will reise if project storage does not implement load_manifest.
+        """
+        storage_module = import_string(settings.STATICFILES_STORAGE)
+        storage = storage_module()
+        return storage.load_manifest()
+
+    def unhash_file_name(self, requested_path):
+        """Returns file original name (without hash),
+        which is a key in staticfiles manifest
+        """
+        temp_path = re.sub(r'(\.[0-9a-f]{12})\.?(\w+)$', r'.\2', requested_path)
+        return re.sub(r'(\.[0-9a-f]{12})$', r'', temp_path)
+
+    def find_requested_file(self, requested_path):
+        """Returns path to existing file (file path with current hash)"""
+        # manifest = self.load_staticfiles_manifest()
+        if self.manifest is None or len(self.manifest) == 0:
+            return None
+
+        file_name = self.unhash_file_name(requested_path).strip('/')
+        try:
+            return self.manifest[file_name]
+        except KeyError:
+            return None
+
     def process_response(self, request, response):
         if not is_static_request(request, response):
             return response
+
         path = self.path_regex.match(request.path)
         if not path:
             return response
+
+        # Try to serve a file with original name from request
         try:
-            response = serve(
-                request, path.group(1), document_root=settings.STATIC_ROOT)
+            return self.serve_response(request, path.group(1))
         except Http404:
             pass
+
+        # Map requested file to hash and try to serve file with hash
+        requested_path = self.find_requested_file(path.group(1))
+        if requested_path is None:
+            return response
+        try:
+            return self.serve_response(request, requested_path)
+        except Http404:
+            pass
+
         return response
 
 

barbeque/storage.py

@@ -0,0 +1,17 @@
+from django.contrib.staticfiles.storage import ManifestStaticFilesStorage
+from django.contrib.staticfiles.storage import ManifestFilesMixin
+
+
+class CompactManifestStaticFilesStorage(ManifestStaticFilesStorage):
+
+    def post_process(self, *args, **kwargs):
+        """
+        Based on django post_process from ManifestStaticFilesStorage
+        """
+        all_post_processed = super(ManifestFilesMixin,
+                                   self).post_process(*args, **kwargs)
+        for post_processed in all_post_processed:
+            yield post_processed
+        self.save_manifest()
+        for original_file in self.hashed_files:
+            self.delete(original_file)

barbeque/tests/resources/static/staticfiles.json

@@ -0,0 +1 @@
+{"version": "1.0", "paths": {"test_hash.jpg": "test_hash.11aa22bb33cc.jpg"}}

barbeque/tests/test_static.py

@@ -1,5 +1,6 @@
 import os
 import mock
+from collections import OrderedDict
 
 import pytest
 from django.http import HttpResponse, HttpResponseNotFound, HttpResponsePermanentRedirect
@@ -30,6 +31,10 @@ class TestServeStaticFileMiddleware:
     def setup(self, settings):
         settings.ROOT_DIR = os.path.dirname(os.path.dirname(__file__))
         settings.STATIC_ROOT = os.path.join(settings.ROOT_DIR, 'tests', 'resources', 'static')
+        # settings.STATICFILES_STORAGE = (
+        #     'django.contrib.staticfiles.storage.ManifestStaticFilesStorage')
+        settings.STATICFILES_STORAGE = (
+            'barbeque.storage.CompactManifestStaticFilesStorage')
 
     @pytest.fixture
     def patch_settings(self, settings):
@@ -107,3 +112,89 @@ def test_with_client_query_params(self, client, patch_settings):
         response = client.get('/static/test.jpg?v=1')
         assert response.status_code == 200
         assert response['Content-Type'] == 'image/jpeg'
+
+
+class TestServeStaticFileMiddlewareWithHashedFiles:
+
+    @pytest.fixture(autouse=True)
+    def setup(self, settings):
+        settings.ROOT_DIR = os.path.dirname(os.path.dirname(__file__))
+        settings.STATIC_ROOT = os.path.join(settings.ROOT_DIR, 'tests', 'resources', 'static')
+        settings.STATICFILES_STORAGE = (
+            'django.contrib.staticfiles.storage.ManifestStaticFilesStorage')
+
+    @pytest.fixture
+    def patch_settings(self, settings):
+        """
+        Patch settings for tests fith django client
+        """
+        settings.STATICFILES_FINDERS = (
+            'django.contrib.staticfiles.finders.FileSystemFinder',
+            'django.contrib.staticfiles.finders.AppDirectoriesFinder',
+            'compressor.finders.CompressorFinder',
+        )
+        settings.MIDDLEWARE_CLASSES = [
+            'barbeque.static.ServeStaticFileMiddleware',
+        ]
+        settings.INSTALLED_APPS = settings.INSTALLED_APPS + ('django.contrib.staticfiles',)
+        settings.ROOT_URLCONF = 'barbeque.tests.test_static'
+
+    def test_unhash_file_name(self):
+        middleware = ServeStaticFileMiddleware()
+        assert middleware.unhash_file_name(
+            '/static/test_hash.11aa22bb33cc.jpg') == ('/static/test_hash.jpg')
+        assert middleware.unhash_file_name('test_hash.jpg') == 'test_hash.jpg'
+        assert middleware.unhash_file_name(
+            'test_hash.11aa22bb33cc.11aa22bb33cc.jpg') == ('test_hash.11aa22bb33cc.jpg')
+        assert middleware.unhash_file_name('test_hash.11aa22bb33cc') == 'test_hash'
+        assert middleware.unhash_file_name('11aa22bb33cc') == '11aa22bb33cc'
+        assert middleware.unhash_file_name('11aa22bb33cc.jpg') == '11aa22bb33cc.jpg'
+        # assert middleware.unhash_file_name('.11aa22bb33cc.jpg') == '.11aa22bb33cc.jpg'
+
+    def test_hash_file_exists(self, rf):
+        request = rf.get('/static/test_hash.11aa22bb33cc.jpg')
+        middleware = ServeStaticFileMiddleware()
+        response = middleware.process_response(request, HttpResponseNotFound(''))
+        assert response.status_code == 200
+        assert response['Content-Type'] == 'image/jpeg'
+        assert len(response.items()) == 3
+        assert response.has_header('Content-Length')
+        assert response.has_header('Last-Modified')
+
+    def test_hash_file_original_exists(self, rf):
+        request = rf.get('/static/test_hash.jpg')
+        middleware = ServeStaticFileMiddleware()
+        response = middleware.process_response(request, HttpResponseNotFound(''))
+        assert response.status_code == 200
+        assert response['Content-Type'] == 'image/jpeg'
+        assert len(response.items()) == 3
+        assert response.has_header('Content-Length')
+        assert response.has_header('Last-Modified')
+
+    def test_old_hash(self, rf):
+        request = rf.get('/static/test_hash.44dd55ee66ff.jpg')
+        middleware = ServeStaticFileMiddleware()
+        response = middleware.process_response(request, HttpResponseNotFound(''))
+        assert len(response.items()) == 3
+        assert response.has_header('Content-Length')
+        assert response.has_header('Last-Modified')
+
+    def test_hash_file_exists_with_client_hit(self, client, patch_settings):
+        response = client.get('/static/test_hash.11aa22bb33cc.jpg')
+        assert response.status_code == 200
+
+    def test_hash_file_original_exists_with_client_hit(self, client, patch_settings):
+        response = client.get('/static/test_hash.jpg')
+        assert response.status_code == 200
+
+    def test_hash_old_hash_with_client_hit(self, client, patch_settings):
+        response = client.get('/static/test_hash.44dd55ee66ff.jpg')
+        assert response.status_code == 200
+
+    @mock.patch('django.contrib.staticfiles.storage.ManifestStaticFilesStorage.load_manifest')
+    def test_no_staticfiles_manifest(self, manifest_mock, rf):
+        manifest_mock.return_value = OrderedDict()
+        request = rf.get('/static/test_hash.jpg')
+        middleware = ServeStaticFileMiddleware()
+        response = middleware.process_response(request, HttpResponseNotFound(''))
+        assert response.status_code == 404