Merge branch 'release/0.9'

.gitignore

@@ -1,11 +1,13 @@
 build
 compressor/tests/media/CACHE
 compressor/tests/media/custom
-compressor/tests/media/js/3f33b9146e12.js
+compressor/tests/media/js/066cd253eada.js
 dist
 MANIFEST
 *.pyc
 *.egg-info
 .tox/
 *.egg
 docs/_build/
+.coverage
+htmlcov

AUTHORS

@@ -17,6 +17,7 @@ David Ziegler
 Eugene Mirotin
 Fenn Bailey
 Gert Van Gool
+Harro van der Klauw
 Jaap Roes
 Jason Davies
 Jeremy Dunck

compressor/__init__.py

@@ -1,4 +1,4 @@
-VERSION = (0, 8, 0, "f", 0)  # following PEP 386
+VERSION = (0, 9, 0, "f", 0)  # following PEP 386
 DEV_N = None
 
 

compressor/base.py

@@ -1,5 +1,4 @@
 import os
-import socket
 
 from django.core.files.base import ContentFile
 from django.template.loader import render_to_string
@@ -10,7 +9,12 @@
 from compressor.filters import CompilerFilter
 from compressor.storage import default_storage
 from compressor.utils import get_class, staticfiles
-from compressor.utils.cache import cached_property
+from compressor.utils.decorators import cached_property
+
+# Some constants for nicer handling.
+SOURCE_HUNK, SOURCE_FILE = 1, 2
+METHOD_INPUT, METHOD_OUTPUT = 'input', 'output'
+
 
 class Compressor(object):
     """
@@ -55,10 +59,8 @@ def get_filename(self, basename):
         if settings.DEBUG and self.finders:
             filename = self.finders.find(basename)
         # secondly try finding the file in the root
-        else:
-            root_filename = os.path.join(settings.COMPRESS_ROOT, basename)
-            if os.path.exists(root_filename):
-                filename = root_filename
+        elif self.storage.exists(basename):
+            filename = self.storage.path(basename)
         if filename:
             return filename
         # or just raise an exception as the last resort
@@ -79,22 +81,21 @@ def cached_filters(self):
     def mtimes(self):
         return [str(get_mtime(value))
                 for kind, value, basename, elem in self.split_contents()
-                if kind == 'file']
+                if kind == SOURCE_FILE]
 
     @cached_property
     def cachekey(self):
-        key = get_hexdigest(''.join(
+        return get_hexdigest(''.join(
             [self.content] + self.mtimes).encode(self.charset), 12)
-        return "django_compressor.%s.%s" % (socket.gethostname(), key)
 
     @cached_property
     def hunks(self):
         for kind, value, basename, elem in self.split_contents():
-            if kind == "hunk":
-                content = self.filter(value, "input",
+            if kind == SOURCE_HUNK:
+                content = self.filter(value, METHOD_INPUT,
                     elem=elem, kind=kind, basename=basename)
                 yield unicode(content)
-            elif kind == "file":
+            elif kind == SOURCE_FILE:
                 content = ""
                 fd = open(value, 'rb')
                 try:
@@ -104,7 +105,7 @@ def hunks(self):
                         "IOError while processing '%s': %s" % (value, e))
                 finally:
                     fd.close()
-                content = self.filter(content, "input",
+                content = self.filter(content, METHOD_INPUT,
                     filename=value, basename=basename, elem=elem, kind=kind)
                 attribs = self.parser.elem_attribs(elem)
                 charset = attribs.get("charset", self.charset)
@@ -119,22 +120,21 @@ def precompile(self, content, kind=None, elem=None, filename=None, **kwargs):
             return content
         attrs = self.parser.elem_attribs(elem)
         mimetype = attrs.get("type", None)
-        if mimetype is not None:
+        if mimetype:
             command = self.all_mimetypes.get(mimetype)
             if command is None:
                 if mimetype not in ("text/css", "text/javascript"):
-                    error = ("Couldn't find any precompiler in "
-                             "COMPRESS_PRECOMPILERS setting for "
-                             "mimetype '%s'." % mimetype)
-                    raise CompressorError(error)
+                    raise CompressorError("Couldn't find any precompiler in "
+                                          "COMPRESS_PRECOMPILERS setting for "
+                                          "mimetype '%s'." % mimetype)
             else:
-                content = CompilerFilter(content, filter_type=self.type,
-                                         command=command).output(**kwargs)
+                return CompilerFilter(content, filter_type=self.type,
+                    command=command, filename=filename).output(**kwargs)
         return content
 
     def filter(self, content, method, **kwargs):
         # run compiler
-        if method == "input":
+        if method == METHOD_INPUT:
             content = self.precompile(content, **kwargs)
 
         for filter_cls in self.cached_filters:
@@ -149,14 +149,11 @@ def filter(self, content, method, **kwargs):
 
     @cached_property
     def combined(self):
-        return self.filter(self.concat, method="output")
-
-    def hash(self, content):
-        return get_hexdigest(content)[:12]
+        return self.filter(self.concat, method=METHOD_OUTPUT)
 
     def filepath(self, content):
         return os.path.join(settings.COMPRESS_OUTPUT_DIR.strip(os.sep),
-            self.output_prefix, "%s.%s" % (self.hash(content), self.type))
+            self.output_prefix, "%s.%s" % (get_hexdigest(content, 12), self.type))
 
     def output(self, mode='file', forced=False):
         """

compressor/cache.py

@@ -1,29 +1,37 @@
 import os
 import socket
+import time
 
 from django.core.cache import get_cache
 from django.utils.encoding import smart_str
-from django.utils.hashcompat import sha_constructor
+from django.utils.hashcompat import md5_constructor
 
 from compressor.conf import settings
 
 
 def get_hexdigest(plaintext, length=None):
-    digest = sha_constructor(smart_str(plaintext)).hexdigest()
+    digest = md5_constructor(smart_str(plaintext)).hexdigest()
     if length:
         return digest[:length]
     return digest
 
 
+def get_cachekey(key):
+    return ("django_compressor.%s.%s" % (socket.gethostname(), key))
+
+
 def get_mtime_cachekey(filename):
-    return "django_compressor.mtime.%s.%s" % (socket.gethostname(),
-                                              get_hexdigest(filename))
+    return get_cachekey("mtime.%s" % get_hexdigest(filename))
 
 
 def get_offline_cachekey(source):
-    return ("django_compressor.offline.%s.%s" %
-            (socket.gethostname(),
-             get_hexdigest("".join(smart_str(s) for s in source))))
+    return get_cachekey(
+        "offline.%s" % get_hexdigest("".join(smart_str(s) for s in source)))
+
+
+def get_templatetag_cachekey(compressor, mode, kind):
+    return get_cachekey(
+        "templatetag.%s.%s.%s" % (compressor.cachekey, mode, kind))
 
 
 def get_mtime(filename):
@@ -38,9 +46,34 @@ def get_mtime(filename):
 
 
 def get_hashed_mtime(filename, length=12):
-    filename = os.path.realpath(filename)
-    mtime = str(int(get_mtime(filename)))
+    try:
+        filename = os.path.realpath(filename)
+        mtime = str(int(get_mtime(filename)))
+    except OSError:
+        return None
     return get_hexdigest(mtime, length)
 
 
+def cache_get(key):
+    packed_val = cache.get(key)
+    if packed_val is None:
+        return None
+    val, refresh_time, refreshed = packed_val
+    if (time.time() > refresh_time) and not refreshed:
+        # Store the stale value while the cache
+        # revalidates for another MINT_DELAY seconds.
+        cache_set(key, val, refreshed=True,
+            timeout=settings.COMPRESS_MINT_DELAY)
+        return None
+    return val
+
+
+def cache_set(key, val, refreshed=False,
+        timeout=settings.COMPRESS_REBUILD_TIMEOUT):
+    refresh_time = timeout + time.time()
+    real_timeout = timeout + settings.COMPRESS_MINT_DELAY
+    packed_val = (val, refresh_time, refreshed)
+    return cache.set(key, packed_val, real_timeout)
+
+
 cache = get_cache(settings.COMPRESS_CACHE_BACKEND)

compressor/css.py

@@ -1,7 +1,5 @@
-import os
-
 from compressor.conf import settings
-from compressor.base import Compressor
+from compressor.base import Compressor, SOURCE_HUNK, SOURCE_FILE
 from compressor.exceptions import UncompressableFileError
 
 
@@ -26,12 +24,12 @@ def split_contents(self):
                 try:
                     basename = self.get_basename(elem_attribs['href'])
                     filename = self.get_filename(basename)
-                    data = ('file', filename, basename, elem)
+                    data = (SOURCE_FILE, filename, basename, elem)
                 except UncompressableFileError:
                     if settings.DEBUG:
                         raise
             elif elem_name == 'style':
-                data = ('hunk', self.parser.elem_content(elem), None, elem)
+                data = (SOURCE_HUNK, self.parser.elem_content(elem), None, elem)
             if data:
                 self.split_content.append(data)
                 media = elem_attribs.get('media', None)

compressor/filters/base.py

@@ -1,4 +1,3 @@
-import os
 import logging
 import subprocess
 import tempfile
@@ -31,15 +30,18 @@ class CompilerFilter(FilterBase):
     external commands.
     """
     command = None
+    filename = None
     options = {}
 
-    def __init__(self, content, filter_type=None, verbose=0, command=None, **kwargs):
+    def __init__(self, content, filter_type=None, verbose=0, command=None, filename=None, **kwargs):
         super(CompilerFilter, self).__init__(content, filter_type, verbose)
         if command:
             self.command = command
         self.options.update(kwargs)
         if self.command is None:
             raise FilterError("Required command attribute not set")
+        if filename:
+            self.filename = filename
         self.stdout = subprocess.PIPE
         self.stdin = subprocess.PIPE
         self.stderr = subprocess.PIPE
@@ -49,18 +51,21 @@ def output(self, **kwargs):
         outfile = None
         try:
             if "{infile}" in self.command:
-                infile = tempfile.NamedTemporaryFile(mode='w')
-                infile.write(self.content)
-                infile.flush()
-                self.options["infile"] = infile.name
+                if not self.filename:
+                    infile = tempfile.NamedTemporaryFile(mode='w')
+                    infile.write(self.content)
+                    infile.flush()
+                    self.options["infile"] = infile.name
+                else:
+                    self.options["infile"] = self.filename
             if "{outfile}" in self.command:
                 ext = ".%s" % self.type and self.type or ""
                 outfile = tempfile.NamedTemporaryFile(mode='w', suffix=ext)
                 self.options["outfile"] = outfile.name
             cmd = stringformat.FormattableString(self.command).format(**self.options)
             proc = subprocess.Popen(cmd_split(cmd),
                 stdout=self.stdout, stdin=self.stdin, stderr=self.stderr)
-            if infile is not None:
+            if infile is not None or self.filename is not None:
                 filtered, err = proc.communicate()
             else:
                 filtered, err = proc.communicate(self.content)

compressor/filters/css_default.py

@@ -11,40 +11,40 @@
 
 
 class CssAbsoluteFilter(FilterBase):
+
+    def __init__(self, *args, **kwargs):
+        super(CssAbsoluteFilter, self).__init__(*args, **kwargs)
+        self.root = settings.COMPRESS_ROOT
+        self.url = settings.COMPRESS_URL.rstrip('/')
+        self.url_path = self.url
+        self.has_scheme = False
+
     def input(self, filename=None, basename=None, **kwargs):
-        self.root = os.path.normcase(os.path.abspath(settings.COMPRESS_ROOT))
         if filename is not None:
             filename = os.path.normcase(os.path.abspath(filename))
         if (not (filename and filename.startswith(self.root)) and
                 not self.find(basename)):
             return self.content
         self.path = basename.replace(os.sep, '/')
         self.path = self.path.lstrip('/')
-        self.url = settings.COMPRESS_URL.rstrip('/')
-        self.url_path = self.url
-        try:
-            self.mtime = get_hashed_mtime(filename)
-        except OSError:
-            self.mtime = None
-        self.has_http = False
-        if self.url.startswith('http://') or self.url.startswith('https://'):
-            self.has_http = True
+        self.mtime = get_hashed_mtime(filename)
+        if self.url.startswith(('http://', 'https://')):
+            self.has_scheme = True
             parts = self.url.split('/')
             self.url = '/'.join(parts[2:])
             self.url_path = '/%s' % '/'.join(parts[3:])
             self.protocol = '%s/' % '/'.join(parts[:2])
             self.host = parts[2]
-        self.directory_name = '/'.join([self.url, os.path.dirname(self.path)])
-        output = URL_PATTERN.sub(self.url_converter, self.content)
-        return output
+        self.directory_name = '/'.join((self.url, os.path.dirname(self.path)))
+        return URL_PATTERN.sub(self.url_converter, self.content)
 
     def find(self, basename):
         if settings.DEBUG and basename and staticfiles.finders:
             return staticfiles.finders.find(basename)
 
     def guess_filename(self, url):
         local_path = url
-        if self.has_http:
+        if self.has_scheme:
             # COMPRESS_URL had a protocol, remove it and the hostname from our path.
             local_path = local_path.replace(self.protocol + self.host, "", 1)
         # Now, we just need to check if we can find the path from COMPRESS_URL in our url
@@ -59,24 +59,19 @@ def add_mtime(self, url):
         mtime = filename and get_hashed_mtime(filename) or self.mtime
         if mtime is None:
             return url
-        if (url.startswith('http://') or
-            url.startswith('https://') or
-            url.startswith('/')):
+        if url.startswith(('http://', 'https://', '/')):
             if "?" in url:
-                return "%s&%s" % (url, mtime)
-            return "%s?%s" % (url, mtime)
+                url = "%s&%s" % (url, mtime)
+            else:
+                url = "%s?%s" % (url, mtime)
         return url
 
     def url_converter(self, matchobj):
         url = matchobj.group(1)
         url = url.strip(' \'"')
-        if (url.startswith('http://') or
-            url.startswith('https://') or
-            url.startswith('/') or
-            url.startswith('data:')):
+        if url.startswith(('http://', 'https://', '/', 'data:')):
             return "url('%s')" % self.add_mtime(url)
-        full_url = '/'.join([str(self.directory_name), url])
-        full_url = posixpath.normpath(full_url)
-        if self.has_http:
+        full_url = posixpath.normpath('/'.join([self.directory_name, url]))
+        if self.has_scheme:
             full_url = "%s%s" % (self.protocol, full_url)
         return "url('%s')" % self.add_mtime(full_url)

compressor/filters/cssmin/cssmin.py

@@ -248,4 +248,4 @@ def main():
 
 
 if __name__ == '__main__':
-    main()
+    main()