From eff6c8f7d3841fe9736c87c023d65db7c01b65fc Mon Sep 17 00:00:00 2001 From: Harro van der Klauw Date: Thu, 12 May 2011 15:31:32 +0200 Subject: [PATCH 01/25] Made precompiler use the original file if it has a filename --- compressor/base.py | 2 +- compressor/filters/base.py | 15 ++++++++++----- 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index 4741faa70..776c616c6 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -129,7 +129,7 @@ def precompile(self, content, kind=None, elem=None, filename=None, **kwargs): raise CompressorError(error) else: content = CompilerFilter(content, filter_type=self.type, - command=command).output(**kwargs) + command=command, filename=filename).output(**kwargs) return content def filter(self, content, method, **kwargs): diff --git a/compressor/filters/base.py b/compressor/filters/base.py index 3c1a9ca77..2a6e444f7 100644 --- a/compressor/filters/base.py +++ b/compressor/filters/base.py @@ -1,4 +1,3 @@ -import os import logging import subprocess import tempfile @@ -31,15 +30,18 @@ class CompilerFilter(FilterBase): external commands. """ command = None + filename = None options = {} - def __init__(self, content, filter_type=None, verbose=0, command=None, **kwargs): + def __init__(self, content, filter_type=None, verbose=0, command=None, filename=None, **kwargs): super(CompilerFilter, self).__init__(content, filter_type, verbose) if command: self.command = command self.options.update(kwargs) if self.command is None: raise FilterError("Required command attribute not set") + if filename: + self.filename = filename self.stdout = subprocess.PIPE self.stdin = subprocess.PIPE self.stderr = subprocess.PIPE @@ -49,9 +51,12 @@ def output(self, **kwargs): outfile = None try: if "{infile}" in self.command: - infile = tempfile.NamedTemporaryFile(mode='w') - infile.write(self.content) - infile.flush() + if not self.filename: + infile = tempfile.NamedTemporaryFile(mode='w') + infile.write(self.content) + infile.flush() + else: + infile = open(self.filename) self.options["infile"] = infile.name if "{outfile}" in self.command: ext = ".%s" % self.type and self.type or "" From 7afa6cb58ced4951562b6fe4a85c23a31da86fd1 Mon Sep 17 00:00:00 2001 From: Harro van der Klauw Date: Thu, 12 May 2011 17:25:56 +0200 Subject: [PATCH 02/25] Added test and basic precompiler script Also fixed the compiler opening the file without needing it --- compressor/filters/base.py | 6 +++--- compressor/tests/precompiler.py | 19 +++++++++++++++++++ compressor/tests/tests.py | 14 ++++++++++++++ 3 files changed, 36 insertions(+), 3 deletions(-) create mode 100644 compressor/tests/precompiler.py diff --git a/compressor/filters/base.py b/compressor/filters/base.py index 2a6e444f7..e3a75e793 100644 --- a/compressor/filters/base.py +++ b/compressor/filters/base.py @@ -55,9 +55,9 @@ def output(self, **kwargs): infile = tempfile.NamedTemporaryFile(mode='w') infile.write(self.content) infile.flush() + self.options["infile"] = infile.name else: - infile = open(self.filename) - self.options["infile"] = infile.name + self.options["infile"] = self.filename if "{outfile}" in self.command: ext = ".%s" % self.type and self.type or "" outfile = tempfile.NamedTemporaryFile(mode='w', suffix=ext) @@ -65,7 +65,7 @@ def output(self, **kwargs): cmd = stringformat.FormattableString(self.command).format(**self.options) proc = subprocess.Popen(cmd_split(cmd), stdout=self.stdout, stdin=self.stdin, stderr=self.stderr) - if infile is not None: + if infile is not None or self.filename is not None: filtered, err = proc.communicate() else: filtered, err = proc.communicate(self.content) diff --git a/compressor/tests/precompiler.py b/compressor/tests/precompiler.py new file mode 100644 index 000000000..6cc185a86 --- /dev/null +++ b/compressor/tests/precompiler.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +import optparse + +def main(): + p = optparse.OptionParser() + options, arguments = p.parse_args() + + f = open(arguments[0]) + content = f.read() + f.close() + + f = open(arguments[1], 'w') + f.write(content.replace('background:', 'color:')) + f.close() + + +if __name__ == '__main__': + main() + \ No newline at end of file diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 9cd51a44c..afa0eaa6c 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -481,3 +481,17 @@ def test_tidy(self): find_command(settings.COMPRESS_CSSTIDY_BINARY) is None, 'CSStidy binary %r not found' % settings.COMPRESS_CSSTIDY_BINARY )(CssTidyTestCase) + +class PrecompilerTestCase(TestCase): + + def setUp(self): + self.command = 'python %s/precompiler.py {infile} {outfile}' % os.path.dirname(__file__) + self.filename = os.path.join(os.path.dirname(__file__), 'media/css/one.css') + f = open(self.filename, 'r') + self.content = f.read() + f.close() + + def test_precompiler(self): + from compressor.filters.base import CompilerFilter + output = CompilerFilter(content=self.content, filename=self.filename, command=self.command).output() + self.assertEqual(u"body { color:#990; }", output) From c5769d4c0bd1ed59d84b4599078d0ea81bac5c3d Mon Sep 17 00:00:00 2001 From: Harro van der Klauw Date: Fri, 13 May 2011 08:29:50 +0200 Subject: [PATCH 03/25] Improved precompiler.py to also use stdin and/or stdout Made file inputs use options too Added tests for the four differend methods --- compressor/tests/precompiler.py | 28 ++++++++++++++++++++++------ compressor/tests/tests.py | 28 +++++++++++++++++++++++++--- 2 files changed, 47 insertions(+), 9 deletions(-) diff --git a/compressor/tests/precompiler.py b/compressor/tests/precompiler.py index 6cc185a86..9c2ac132d 100644 --- a/compressor/tests/precompiler.py +++ b/compressor/tests/precompiler.py @@ -1,17 +1,33 @@ #!/usr/bin/env python import optparse +import sys def main(): p = optparse.OptionParser() + p.add_option('-f', '--file', action="store", + type="string", dest="filename", + help="File to read from, defaults to stdin", default=None) + p.add_option('-o', '--output', action="store", + type="string", dest="outfile", + help="File to write to, defaults to stdout", default=None) + options, arguments = p.parse_args() - f = open(arguments[0]) - content = f.read() - f.close() + if options.filename: + f = open(options.filename) + content = f.read() + f.close() + else: + content = sys.stdin.read() - f = open(arguments[1], 'w') - f.write(content.replace('background:', 'color:')) - f.close() + content = content.replace('background:', 'color:') + + if options.outfile: + f = open(options.outfile, 'w') + f.write(content) + f.close() + else: + print content if __name__ == '__main__': diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index afa0eaa6c..324fb553b 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -485,13 +485,35 @@ def test_tidy(self): class PrecompilerTestCase(TestCase): def setUp(self): - self.command = 'python %s/precompiler.py {infile} {outfile}' % os.path.dirname(__file__) self.filename = os.path.join(os.path.dirname(__file__), 'media/css/one.css') f = open(self.filename, 'r') self.content = f.read() f.close() - def test_precompiler(self): + def test_precompiler_infile_outfile(self): from compressor.filters.base import CompilerFilter - output = CompilerFilter(content=self.content, filename=self.filename, command=self.command).output() + command = 'python %s/precompiler.py -f {infile} -o {outfile}' % os.path.dirname(__file__) + output = CompilerFilter(content=self.content, filename=self.filename, command=command).output() self.assertEqual(u"body { color:#990; }", output) + + def test_precompiler_stdin_outfile(self): + from compressor.filters.base import CompilerFilter + command = 'python %s/precompiler.py -o {outfile}' % os.path.dirname(__file__) + output = CompilerFilter(content=self.content, filename=None, command=command).output() + self.assertEqual(u"body { color:#990; }", output) + + def test_precompiler_stdin_stdout(self): + from compressor.filters.base import CompilerFilter + command = 'python %s/precompiler.py' % os.path.dirname(__file__) + output = CompilerFilter(content=self.content, filename=None, command=command).output() + self.assertEqual(u"body { color:#990; }\n", output) + + def test_precompiler_infile_stdout(self): + from compressor.filters.base import CompilerFilter + command = 'python %s/precompiler.py -f {infile}' % os.path.dirname(__file__) + output = CompilerFilter(content=self.content, filename=None, command=command).output() + self.assertEqual(u"body { color:#990; }\n", output) + + + + From f9bac59841fecfd6a7f65eff440b2025352031bc Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 10:59:17 +0200 Subject: [PATCH 04/25] Simplified CompilerFilter tests. --- compressor/tests/tests.py | 53 ++++++++++++++++++--------------------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 324fb553b..8176ce29f 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -1,6 +1,8 @@ +from __future__ import with_statement import os import re import socket +import sys from unittest2 import skipIf from BeautifulSoup import BeautifulSoup @@ -32,6 +34,8 @@ from compressor.js import JsCompressor from compressor.management.commands.compress import Command as CompressCommand from compressor.utils import find_command +from compressor.filters.base import CompilerFilter + class CompressorTestCase(TestCase): @@ -483,37 +487,30 @@ def test_tidy(self): )(CssTidyTestCase) class PrecompilerTestCase(TestCase): - + def setUp(self): - self.filename = os.path.join(os.path.dirname(__file__), 'media/css/one.css') - f = open(self.filename, 'r') - self.content = f.read() - f.close() + self.this_dir = os.path.dirname(__file__) + self.filename = os.path.join(self.this_dir, 'media/css/one.css') + self.test_precompiler = os.path.join(self.this_dir, 'precompiler.py') + with open(self.filename, 'r') as f: + self.content = f.read() def test_precompiler_infile_outfile(self): - from compressor.filters.base import CompilerFilter - command = 'python %s/precompiler.py -f {infile} -o {outfile}' % os.path.dirname(__file__) - output = CompilerFilter(content=self.content, filename=self.filename, command=command).output() - self.assertEqual(u"body { color:#990; }", output) - + command = '%s %s -f {infile} -o {outfile}' % (sys.executable, self.test_precompiler) + compiler = CompilerFilter(content=self.content, filename=self.filename, command=command) + self.assertEqual(u"body { color:#990; }", compiler.output()) + def test_precompiler_stdin_outfile(self): - from compressor.filters.base import CompilerFilter - command = 'python %s/precompiler.py -o {outfile}' % os.path.dirname(__file__) - output = CompilerFilter(content=self.content, filename=None, command=command).output() - self.assertEqual(u"body { color:#990; }", output) - + command = '%s %s -o {outfile}' % (sys.executable, self.test_precompiler) + compiler = CompilerFilter(content=self.content, filename=None, command=command) + self.assertEqual(u"body { color:#990; }", compiler.output()) + def test_precompiler_stdin_stdout(self): - from compressor.filters.base import CompilerFilter - command = 'python %s/precompiler.py' % os.path.dirname(__file__) - output = CompilerFilter(content=self.content, filename=None, command=command).output() - self.assertEqual(u"body { color:#990; }\n", output) - + command = '%s %s' % (sys.executable, self.test_precompiler) + compiler = CompilerFilter(content=self.content, filename=None, command=command) + self.assertEqual(u"body { color:#990; }\n", compiler.output()) + def test_precompiler_infile_stdout(self): - from compressor.filters.base import CompilerFilter - command = 'python %s/precompiler.py -f {infile}' % os.path.dirname(__file__) - output = CompilerFilter(content=self.content, filename=None, command=command).output() - self.assertEqual(u"body { color:#990; }\n", output) - - - - + command = '%s %s -f {infile}' % (sys.executable, self.test_precompiler) + compiler = CompilerFilter(content=self.content, filename=None, command=command) + self.assertEqual(u"body { color:#990; }\n", compiler.output()) From 6aed122affbd76333d90fc429e609077b67292b1 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 10:59:36 +0200 Subject: [PATCH 05/25] Added Harro to AUTHORS file. --- AUTHORS | 1 + 1 file changed, 1 insertion(+) diff --git a/AUTHORS b/AUTHORS index a3742f940..50c2be60e 100644 --- a/AUTHORS +++ b/AUTHORS @@ -17,6 +17,7 @@ David Ziegler Eugene Mirotin Fenn Bailey Gert Van Gool +Harro van der Klauw Jaap Roes Jason Davies Jeremy Dunck From f09c85e59a99f442056782a2c1731150ace8efd2 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 10:59:48 +0200 Subject: [PATCH 06/25] Make use of coverage. --- .gitignore | 2 ++ compressor/tests/runtests.py | 21 +++++++++++++++++++-- tox.ini | 24 +++++++++++++++++++++--- 3 files changed, 42 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 9d802fb11..3780c1ed8 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,5 @@ MANIFEST .tox/ *.egg docs/_build/ +.coverage +htmlcov \ No newline at end of file diff --git a/compressor/tests/runtests.py b/compressor/tests/runtests.py index 740f30b68..0b8dea170 100644 --- a/compressor/tests/runtests.py +++ b/compressor/tests/runtests.py @@ -1,8 +1,11 @@ #!/usr/bin/env python import os import sys +import coverage +from os.path import join from django.conf import settings +from django.core.management import call_command TEST_DIR = os.path.dirname(os.path.abspath(__file__)) @@ -28,9 +31,23 @@ def runtests(*test_args): if not test_args: test_args = ['tests'] - parent = os.path.join(TEST_DIR, "..", "..") - sys.path.insert(0, parent) + parent_dir = os.path.join(TEST_DIR, "..", "..") + sys.path.insert(0, parent_dir) + cov = coverage.coverage(branch=True, + include=[ + os.path.join(parent_dir, 'compressor', '*.py') + ], + omit=[ + join(parent_dir, 'compressor', 'tests', '*.py'), + join(parent_dir, 'compressor', 'utils', 'stringformat.py'), + join(parent_dir, 'compressor', 'filters', 'jsmin', 'rjsmin.py'), + join(parent_dir, 'compressor', 'filters', 'cssmin', 'cssmin.py'), + ]) + cov.load() + cov.start() failures = run_tests(test_args, verbosity=1, interactive=True) + cov.stop() + cov.save() sys.exit(failures) diff --git a/tox.ini b/tox.ini index f411f82c3..065a55b7c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,8 +1,17 @@ [testenv] -distribute=false -sitepackages=true +distribute = false +sitepackages = true commands = - python compressor/tests/runtests.py + {envpython} compressor/tests/runtests.py + coverage html -d {envtmpdir}/coverage + +[testenv:docs] +changedir = docs +deps = + Sphinx +commands = + make clean + make html [testenv:py25-1.1.X] basepython = python2.5 @@ -10,6 +19,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.1.4 [testenv:py26-1.1.X] @@ -18,6 +28,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.1.4 [testenv:py27-1.1.X] @@ -26,6 +37,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.1.4 @@ -35,6 +47,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.2.5 [testenv:py26-1.2.X] @@ -43,6 +56,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.2.5 [testenv:py27-1.2.X] @@ -51,6 +65,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.2.5 @@ -60,6 +75,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.3 [testenv:py26-1.3.X] @@ -68,6 +84,7 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.3 [testenv:py27-1.3.X] @@ -76,4 +93,5 @@ deps = unittest2 BeautifulSoup html5lib + coverage django==1.3 From f430422649d21118d65ff5852c9ffb980e468b8a Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:06:51 +0200 Subject: [PATCH 07/25] Moved decorator to proper util module. --- compressor/base.py | 2 +- compressor/parser/beautifulsoup.py | 2 +- compressor/parser/html5lib.py | 2 +- compressor/parser/lxml.py | 2 +- compressor/utils/{cache.py => decorators.py} | 0 5 files changed, 4 insertions(+), 4 deletions(-) rename compressor/utils/{cache.py => decorators.py} (100%) diff --git a/compressor/base.py b/compressor/base.py index 776c616c6..a66652032 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -10,7 +10,7 @@ from compressor.filters import CompilerFilter from compressor.storage import default_storage from compressor.utils import get_class, staticfiles -from compressor.utils.cache import cached_property +from compressor.utils.decorators import cached_property class Compressor(object): """ diff --git a/compressor/parser/beautifulsoup.py b/compressor/parser/beautifulsoup.py index e89660666..498cde800 100644 --- a/compressor/parser/beautifulsoup.py +++ b/compressor/parser/beautifulsoup.py @@ -4,7 +4,7 @@ from compressor.exceptions import ParserError from compressor.parser import ParserBase -from compressor.utils.cache import cached_property +from compressor.utils.decorators import cached_property class BeautifulSoupParser(ParserBase): diff --git a/compressor/parser/html5lib.py b/compressor/parser/html5lib.py index 3a919ab08..52f662d3f 100644 --- a/compressor/parser/html5lib.py +++ b/compressor/parser/html5lib.py @@ -4,7 +4,7 @@ from compressor.exceptions import ParserError from compressor.parser import ParserBase -from compressor.utils.cache import cached_property +from compressor.utils.decorators import cached_property class Html5LibParser(ParserBase): diff --git a/compressor/parser/lxml.py b/compressor/parser/lxml.py index 8fc4bb7cb..b74448b89 100644 --- a/compressor/parser/lxml.py +++ b/compressor/parser/lxml.py @@ -4,7 +4,7 @@ from compressor.exceptions import ParserError from compressor.parser import ParserBase -from compressor.utils.cache import cached_property +from compressor.utils.decorators import cached_property class LxmlParser(ParserBase): diff --git a/compressor/utils/cache.py b/compressor/utils/decorators.py similarity index 100% rename from compressor/utils/cache.py rename to compressor/utils/decorators.py From cee6eaf039608be28e57c62b35135efddafc1c61 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:27:23 +0200 Subject: [PATCH 08/25] Use constants for hunk comparison. --- compressor/base.py | 8 ++++++-- compressor/css.py | 6 +++--- compressor/js.py | 6 +++--- compressor/tests/tests.py | 21 +++++++++++---------- 4 files changed, 23 insertions(+), 18 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index a66652032..0f7ad85af 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -12,6 +12,10 @@ from compressor.utils import get_class, staticfiles from compressor.utils.decorators import cached_property +# Some constants for nicer handling. +SOURCE_HUNK, SOURCE_FILE = 1, 2 + + class Compressor(object): """ Base compressor object to be subclassed for content type @@ -90,11 +94,11 @@ def cachekey(self): @cached_property def hunks(self): for kind, value, basename, elem in self.split_contents(): - if kind == "hunk": + if kind == SOURCE_HUNK: content = self.filter(value, "input", elem=elem, kind=kind, basename=basename) yield unicode(content) - elif kind == "file": + elif kind == SOURCE_FILE: content = "" fd = open(value, 'rb') try: diff --git a/compressor/css.py b/compressor/css.py index 69116f99e..41a484d7b 100644 --- a/compressor/css.py +++ b/compressor/css.py @@ -1,7 +1,7 @@ import os from compressor.conf import settings -from compressor.base import Compressor +from compressor.base import Compressor, SOURCE_HUNK, SOURCE_FILE from compressor.exceptions import UncompressableFileError @@ -26,12 +26,12 @@ def split_contents(self): try: basename = self.get_basename(elem_attribs['href']) filename = self.get_filename(basename) - data = ('file', filename, basename, elem) + data = (SOURCE_FILE, filename, basename, elem) except UncompressableFileError: if settings.DEBUG: raise elif elem_name == 'style': - data = ('hunk', self.parser.elem_content(elem), None, elem) + data = (SOURCE_HUNK, self.parser.elem_content(elem), None, elem) if data: self.split_content.append(data) media = elem_attribs.get('media', None) diff --git a/compressor/js.py b/compressor/js.py index e28b102a1..481478603 100644 --- a/compressor/js.py +++ b/compressor/js.py @@ -1,7 +1,7 @@ import os from compressor.conf import settings -from compressor.base import Compressor +from compressor.base import Compressor, SOURCE_HUNK, SOURCE_FILE from compressor.exceptions import UncompressableFileError @@ -24,11 +24,11 @@ def split_contents(self): basename = self.get_basename(attribs['src']) filename = self.get_filename(basename) self.split_content.append( - ('file', filename, basename, elem)) + (SOURCE_FILE, filename, basename, elem)) except UncompressableFileError: if settings.DEBUG: raise else: content = self.parser.elem_content(elem) - self.split_content.append(('hunk', content, None, elem)) + self.split_content.append((SOURCE_HUNK, content, None, elem)) return self.split_content diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 8176ce29f..df08d5bc5 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -28,6 +28,7 @@ from django.test import TestCase from compressor import base +from compressor.base import SOURCE_HUNK, SOURCE_FILE from compressor.cache import get_hashed_mtime from compressor.conf import settings from compressor.css import CssCompressor @@ -59,9 +60,9 @@ def setUp(self): def test_css_split(self): out = [ - ('file', os.path.join(settings.COMPRESS_ROOT, u'css/one.css'), u'css/one.css', u''), - ('hunk', u'p { border:5px solid green;}', None, u''), - ('file', os.path.join(settings.COMPRESS_ROOT, u'css/two.css'), u'css/two.css', u''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css/one.css'), u'css/one.css', u''), + (SOURCE_HUNK, u'p { border:5px solid green;}', None, u''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css/two.css'), u'css/two.css', u''), ] split = self.css_node.split_contents() split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] @@ -97,8 +98,8 @@ def test_css_return_if_on(self): self.assertEqual(output, self.css_node.output().strip()) def test_js_split(self): - out = [('file', os.path.join(settings.COMPRESS_ROOT, u'js/one.js'), u'js/one.js', ''), - ('hunk', u'obj.value = "value";', None, '') + out = [(SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'js/one.js'), u'js/one.js', ''), + (SOURCE_HUNK, u'obj.value = "value";', None, '') ] split = self.js_node.split_contents() split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] @@ -168,17 +169,17 @@ class Html5LibParserTests(ParserTestCase, CompressorTestCase): def test_css_split(self): out = [ - ('file', os.path.join(settings.COMPRESS_ROOT, u'css/one.css'), u'css/one.css', u''), - ('hunk', u'p { border:5px solid green;}', None, u''), - ('file', os.path.join(settings.COMPRESS_ROOT, u'css/two.css'), u'css/two.css', u''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css/one.css'), u'css/one.css', u''), + (SOURCE_HUNK, u'p { border:5px solid green;}', None, u''), + (SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'css/two.css'), u'css/two.css', u''), ] split = self.css_node.split_contents() split = [(x[0], x[1], x[2], self.css_node.parser.elem_str(x[3])) for x in split] self.assertEqual(out, split) def test_js_split(self): - out = [('file', os.path.join(settings.COMPRESS_ROOT, u'js/one.js'), u'js/one.js', u''), - ('hunk', u'obj.value = "value";', None, u'') + out = [(SOURCE_FILE, os.path.join(settings.COMPRESS_ROOT, u'js/one.js'), u'js/one.js', u''), + (SOURCE_HUNK, u'obj.value = "value";', None, u'') ] split = self.js_node.split_contents() split = [(x[0], x[1], x[2], self.js_node.parser.elem_str(x[3])) for x in split] From 6b3401dc6398a636863a779eba8d95002a90da5c Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:29:08 +0200 Subject: [PATCH 09/25] Minor style fixes. --- compressor/filters/cssmin/cssmin.py | 2 +- compressor/finders.py | 1 + compressor/management/commands/compress.py | 9 ++-- compressor/management/commands/mtime_cache.py | 1 + compressor/templatetags/compress.py | 6 ++- compressor/utils/__init__.py | 42 ++++++++++--------- compressor/utils/stringformat.py | 2 +- 7 files changed, 36 insertions(+), 27 deletions(-) diff --git a/compressor/filters/cssmin/cssmin.py b/compressor/filters/cssmin/cssmin.py index 166835569..b79fc6d6b 100644 --- a/compressor/filters/cssmin/cssmin.py +++ b/compressor/filters/cssmin/cssmin.py @@ -248,4 +248,4 @@ def main(): if __name__ == '__main__': - main() \ No newline at end of file + main() diff --git a/compressor/finders.py b/compressor/finders.py index 33d9485c4..36edf93b6 100644 --- a/compressor/finders.py +++ b/compressor/finders.py @@ -1,6 +1,7 @@ from compressor.utils import staticfiles from compressor.storage import CompressorFileStorage + class CompressorFinder(staticfiles.finders.BaseStorageFinder): """ A staticfiles finder that looks in COMPRESS_ROOT diff --git a/compressor/management/commands/compress.py b/compressor/management/commands/compress.py index b293a4257..8f921dcac 100644 --- a/compressor/management/commands/compress.py +++ b/compressor/management/commands/compress.py @@ -36,6 +36,7 @@ class Command(NoArgsCommand): "can lead to infinite recursion if a link points to a parent " "directory of itself.", dest='follow_links'), ) + def get_loaders(self): from django.template.loader import template_source_loaders if template_source_loaders is None: @@ -113,11 +114,11 @@ def compress(self, log=None, **options): settings.FILE_CHARSET)) finally: template_file.close() - except IOError: # unreadable file -> ignore + except IOError: # unreadable file -> ignore if verbosity > 0: log.write("Unreadable template at: %s\n" % template_name) continue - except TemplateSyntaxError: # broken template -> ignore + except TemplateSyntaxError: # broken template -> ignore if verbosity > 0: log.write("Invalid template at: %s\n" % template_name) continue @@ -159,7 +160,7 @@ def compress(self, log=None, **options): def walk_nodes(self, node): for node in getattr(node, "nodelist", []): if (isinstance(node, CompressorNode) or - node.__class__.__name__ == "CompressorNode"): # for 1.1.X + node.__class__.__name__ == "CompressorNode"): # for 1.1.X yield node else: for node in self.walk_nodes(node): @@ -180,7 +181,7 @@ def handle_extensions(self, extensions=('html',)): """ ext_list = [] for ext in extensions: - ext_list.extend(ext.replace(' ','').split(',')) + ext_list.extend(ext.replace(' ', '').split(',')) for i, ext in enumerate(ext_list): if not ext.startswith('.'): ext_list[i] = '.%s' % ext_list[i] diff --git a/compressor/management/commands/mtime_cache.py b/compressor/management/commands/mtime_cache.py index 8dd0b9554..88deab08d 100644 --- a/compressor/management/commands/mtime_cache.py +++ b/compressor/management/commands/mtime_cache.py @@ -8,6 +8,7 @@ from compressor.conf import settings from compressor.utils import walk + class Command(NoArgsCommand): help = "Add or remove all mtime values from the cache" option_list = NoArgsCommand.option_list + ( diff --git a/compressor/templatetags/compress.py b/compressor/templatetags/compress.py index 734bfd428..44793a4ac 100644 --- a/compressor/templatetags/compress.py +++ b/compressor/templatetags/compress.py @@ -7,6 +7,8 @@ from compressor.conf import settings from compressor.utils import get_class +register = template.Library() + OUTPUT_FILE = 'file' OUTPUT_INLINE = 'inline' OUTPUT_MODES = (OUTPUT_FILE, OUTPUT_INLINE) @@ -15,9 +17,8 @@ "js": settings.COMPRESS_JS_COMPRESSOR, } -register = template.Library() - class CompressorNode(template.Node): + def __init__(self, nodelist, kind=None, mode=OUTPUT_FILE): self.nodelist = nodelist self.kind = kind @@ -105,6 +106,7 @@ def render(self, context, forced=False): # 5. Or don't do anything in production return self.nodelist.render(context) + @register.tag def compress(parser, token): """ diff --git a/compressor/utils/__init__.py b/compressor/utils/__init__.py index 766943c6a..b221a7e93 100644 --- a/compressor/utils/__init__.py +++ b/compressor/utils/__init__.py @@ -1,20 +1,38 @@ # -*- coding: utf-8 -*- import os +import sys from shlex import split as cmd_split from compressor.exceptions import FilterError -try: - any = any - -except NameError: - +if sys.version_info < (2, 5): + # Add any http://docs.python.org/library/functions.html?#any to Python < 2.5 def any(seq): for item in seq: if item: return True return False +else: + any = any + + +if sys.version_info < (2, 6): + def walk(root, topdown=True, onerror=None, followlinks=False): + """ + A version of os.walk that can follow symlinks for Python < 2.6 + """ + for dirpath, dirnames, filenames in os.walk(root, topdown, onerror): + yield (dirpath, dirnames, filenames) + if followlinks: + for d in dirnames: + p = os.path.join(dirpath, d) + if os.path.islink(p): + for link_dirpath, link_dirnames, link_filenames in walk(p): + yield (link_dirpath, link_dirnames, link_filenames) +else: + from os import walk + def get_class(class_string, exception=FilterError): """ @@ -45,20 +63,6 @@ def get_mod_func(callback): return callback[:dot], callback[dot + 1:] -def walk(root, topdown=True, onerror=None, followlinks=False): - """ - A version of os.walk that can follow symlinks for Python < 2.6 - """ - for dirpath, dirnames, filenames in os.walk(root, topdown, onerror): - yield (dirpath, dirnames, filenames) - if followlinks: - for d in dirnames: - p = os.path.join(dirpath, d) - if os.path.islink(p): - for link_dirpath, link_dirnames, link_filenames in walk(p): - yield (link_dirpath, link_dirnames, link_filenames) - - def get_pathext(default_pathext=None): """ Returns the path extensions from environment or a default diff --git a/compressor/utils/stringformat.py b/compressor/utils/stringformat.py index 40c4f8225..9c797b6a6 100644 --- a/compressor/utils/stringformat.py +++ b/compressor/utils/stringformat.py @@ -275,4 +275,4 @@ def selftest(): print 'Test successful' if __name__ == '__main__': - selftest() \ No newline at end of file + selftest() From 463ca0336d27730530411deef37e42c1b689815d Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:51:17 +0200 Subject: [PATCH 10/25] More constants. --- compressor/base.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index 0f7ad85af..cbd838b54 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -14,6 +14,7 @@ # Some constants for nicer handling. SOURCE_HUNK, SOURCE_FILE = 1, 2 +METHOD_INPUT, METHOD_OUTPUT = 'input', 'output' class Compressor(object): @@ -83,7 +84,7 @@ def cached_filters(self): def mtimes(self): return [str(get_mtime(value)) for kind, value, basename, elem in self.split_contents() - if kind == 'file'] + if kind == SOURCE_FILE] @cached_property def cachekey(self): @@ -95,7 +96,7 @@ def cachekey(self): def hunks(self): for kind, value, basename, elem in self.split_contents(): if kind == SOURCE_HUNK: - content = self.filter(value, "input", + content = self.filter(value, METHOD_INPUT, elem=elem, kind=kind, basename=basename) yield unicode(content) elif kind == SOURCE_FILE: @@ -108,7 +109,7 @@ def hunks(self): "IOError while processing '%s': %s" % (value, e)) finally: fd.close() - content = self.filter(content, "input", + content = self.filter(content, METHOD_INPUT, filename=value, basename=basename, elem=elem, kind=kind) attribs = self.parser.elem_attribs(elem) charset = attribs.get("charset", self.charset) @@ -138,7 +139,7 @@ def precompile(self, content, kind=None, elem=None, filename=None, **kwargs): def filter(self, content, method, **kwargs): # run compiler - if method == "input": + if method == METHOD_INPUT: content = self.precompile(content, **kwargs) for filter_cls in self.cached_filters: @@ -153,10 +154,10 @@ def filter(self, content, method, **kwargs): @cached_property def combined(self): - return self.filter(self.concat, method="output") def hash(self, content): return get_hexdigest(content)[:12] + return self.filter(self.concat, method=METHOD_OUTPUT) def filepath(self, content): return os.path.join(settings.COMPRESS_OUTPUT_DIR.strip(os.sep), From c5a369f185cf17d4418e9089c48819024ad5091d Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:59:29 +0200 Subject: [PATCH 11/25] Use the storage directly to get the path. --- compressor/base.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index cbd838b54..addc3fdbc 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -60,10 +60,8 @@ def get_filename(self, basename): if settings.DEBUG and self.finders: filename = self.finders.find(basename) # secondly try finding the file in the root - else: - root_filename = os.path.join(settings.COMPRESS_ROOT, basename) - if os.path.exists(root_filename): - filename = root_filename + elif self.storage.exists(basename): + filename = self.storage.path(basename) if filename: return filename # or just raise an exception as the last resort From 64f5b6df61e281890423aeba51000af7d6442531 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:59:42 +0200 Subject: [PATCH 12/25] Minor style changes. --- compressor/base.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index addc3fdbc..9385231e7 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -122,17 +122,16 @@ def precompile(self, content, kind=None, elem=None, filename=None, **kwargs): return content attrs = self.parser.elem_attribs(elem) mimetype = attrs.get("type", None) - if mimetype is not None: + if mimetype: command = self.all_mimetypes.get(mimetype) if command is None: if mimetype not in ("text/css", "text/javascript"): - error = ("Couldn't find any precompiler in " - "COMPRESS_PRECOMPILERS setting for " - "mimetype '%s'." % mimetype) - raise CompressorError(error) + raise CompressorError("Couldn't find any precompiler in " + "COMPRESS_PRECOMPILERS setting for " + "mimetype '%s'." % mimetype) else: - content = CompilerFilter(content, filter_type=self.type, - command=command, filename=filename).output(**kwargs) + return CompilerFilter(content, filter_type=self.type, + command=command, filename=filename).output(**kwargs) return content def filter(self, content, method, **kwargs): From e7dcf3a03ffd5ff09bad439c5913baf6536839b2 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 11:59:58 +0200 Subject: [PATCH 13/25] Got rid of Compressor.hash method. --- compressor/base.py | 5 +---- compressor/tests/tests.py | 4 ++-- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index 9385231e7..f828d648b 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -151,14 +151,11 @@ def filter(self, content, method, **kwargs): @cached_property def combined(self): - - def hash(self, content): - return get_hexdigest(content)[:12] return self.filter(self.concat, method=METHOD_OUTPUT) def filepath(self, content): return os.path.join(settings.COMPRESS_OUTPUT_DIR.strip(os.sep), - self.output_prefix, "%s.%s" % (self.hash(content), self.type)) + self.output_prefix, "%s.%s" % (get_hexdigest(content, 12), self.type)) def output(self, mode='file', forced=False): """ diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index df08d5bc5..76ff39211 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -29,7 +29,7 @@ from compressor import base from compressor.base import SOURCE_HUNK, SOURCE_FILE -from compressor.cache import get_hashed_mtime +from compressor.cache import get_hashed_mtime, get_hexdigest from compressor.conf import settings from compressor.css import CssCompressor from compressor.js import JsCompressor @@ -91,7 +91,7 @@ def test_cachekey(self): self.assert_(is_cachekey.match(self.css_node.cachekey), "cachekey is returning something that doesn't look like r'django_compressor\.%s\.\w{12}'" % host_name) def test_css_hash(self): - self.assertEqual('666f3aa8eacd', self.css_node.hash(self.css)) + self.assertEqual('666f3aa8eacd', get_hexdigest(self.css, 12)) def test_css_return_if_on(self): output = u'' From 304d719750b4e1243076e51d756a2e2144088915 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 12:06:07 +0200 Subject: [PATCH 14/25] Use MD5 instead of SHA1 to reduce the computational impact. --- .gitignore | 2 +- compressor/cache.py | 4 ++-- compressor/tests/tests.py | 32 ++++++++++++++++---------------- 3 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.gitignore b/.gitignore index 3780c1ed8..4efafd42b 100644 --- a/.gitignore +++ b/.gitignore @@ -1,7 +1,7 @@ build compressor/tests/media/CACHE compressor/tests/media/custom -compressor/tests/media/js/3f33b9146e12.js +compressor/tests/media/js/066cd253eada.js dist MANIFEST *.pyc diff --git a/compressor/cache.py b/compressor/cache.py index e53f4b88e..ebd56893f 100644 --- a/compressor/cache.py +++ b/compressor/cache.py @@ -3,13 +3,13 @@ from django.core.cache import get_cache from django.utils.encoding import smart_str -from django.utils.hashcompat import sha_constructor +from django.utils.hashcompat import md5_constructor from compressor.conf import settings def get_hexdigest(plaintext, length=None): - digest = sha_constructor(smart_str(plaintext)).hexdigest() + digest = md5_constructor(smart_str(plaintext)).hexdigest() if length: return digest[:length] return digest diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 76ff39211..9acabe942 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -91,10 +91,10 @@ def test_cachekey(self): self.assert_(is_cachekey.match(self.css_node.cachekey), "cachekey is returning something that doesn't look like r'django_compressor\.%s\.\w{12}'" % host_name) def test_css_hash(self): - self.assertEqual('666f3aa8eacd', get_hexdigest(self.css, 12)) + self.assertEqual('c618e6846d04', get_hexdigest(self.css, 12)) def test_css_return_if_on(self): - output = u'' + output = u'' self.assertEqual(output, self.css_node.output().strip()) def test_js_split(self): @@ -129,20 +129,20 @@ def test_js_return_if_off(self): settings.COMPRESS_PRECOMPILERS = precompilers def test_js_return_if_on(self): - output = u'' + output = u'' self.assertEqual(output, self.js_node.output()) def test_custom_output_dir(self): try: old_output_dir = settings.COMPRESS_OUTPUT_DIR settings.COMPRESS_OUTPUT_DIR = 'custom' - output = u'' + output = u'' self.assertEqual(output, JsCompressor(self.js).output()) settings.COMPRESS_OUTPUT_DIR = '' - output = u'' + output = u'' self.assertEqual(output, JsCompressor(self.js).output()) settings.COMPRESS_OUTPUT_DIR = '/custom/nested/' - output = u'' + output = u'' self.assertEqual(output, JsCompressor(self.js).output()) finally: settings.COMPRESS_OUTPUT_DIR = old_output_dir @@ -340,7 +340,7 @@ def test_css_tag(self): {% endcompress %} """ context = { 'MEDIA_URL': settings.COMPRESS_URL } - out = u'' + out = u'' self.assertEqual(out, render(template, context)) def test_nonascii_css_tag(self): @@ -350,7 +350,7 @@ def test_nonascii_css_tag(self): {% endcompress %} """ context = { 'MEDIA_URL': settings.COMPRESS_URL } - out = '' + out = '' self.assertEqual(out, render(template, context)) def test_js_tag(self): @@ -360,7 +360,7 @@ def test_js_tag(self): {% endcompress %} """ context = { 'MEDIA_URL': settings.COMPRESS_URL } - out = u'' + out = u'' self.assertEqual(out, render(template, context)) def test_nonascii_js_tag(self): @@ -370,7 +370,7 @@ def test_nonascii_js_tag(self): {% endcompress %} """ context = { 'MEDIA_URL': settings.COMPRESS_URL } - out = u'' + out = u'' self.assertEqual(out, render(template, context)) def test_nonascii_latin1_js_tag(self): @@ -380,7 +380,7 @@ def test_nonascii_latin1_js_tag(self): {% endcompress %} """ context = { 'MEDIA_URL': settings.COMPRESS_URL } - out = u'' + out = u'' self.assertEqual(out, render(template, context)) def test_compress_tag_with_illegal_arguments(self): @@ -419,7 +419,7 @@ def test_css_tag_with_storage(self): {% endcompress %} """ context = { 'MEDIA_URL': settings.COMPRESS_URL } - out = u'' + out = u'' self.assertEqual(out, render(template, context)) @@ -452,8 +452,8 @@ def test_offline(self): count, result = CompressCommand().compress() self.assertEqual(2, count) self.assertEqual([ - u'\n', - u'', + u'\n', + u'', ], result) def test_offline_with_context(self): @@ -464,8 +464,8 @@ def test_offline_with_context(self): count, result = CompressCommand().compress() self.assertEqual(2, count) self.assertEqual([ - u'\n', - u'', + u'\n', + u'', ], result) settings.COMPRESS_OFFLINE_CONTEXT = self._old_offline_context From f9579a2c39f70fbc653b27bea4d574637cb9cca5 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 12:25:33 +0200 Subject: [PATCH 15/25] Moved OSError exception handling to actual function. --- compressor/cache.py | 7 +++++-- compressor/filters/css_default.py | 5 +---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/compressor/cache.py b/compressor/cache.py index ebd56893f..5da78db7c 100644 --- a/compressor/cache.py +++ b/compressor/cache.py @@ -38,8 +38,11 @@ def get_mtime(filename): def get_hashed_mtime(filename, length=12): - filename = os.path.realpath(filename) - mtime = str(int(get_mtime(filename))) + try: + filename = os.path.realpath(filename) + mtime = str(int(get_mtime(filename))) + except OSError: + return None return get_hexdigest(mtime, length) diff --git a/compressor/filters/css_default.py b/compressor/filters/css_default.py index 0aed1d707..1aab20f2b 100644 --- a/compressor/filters/css_default.py +++ b/compressor/filters/css_default.py @@ -22,10 +22,7 @@ def input(self, filename=None, basename=None, **kwargs): self.path = self.path.lstrip('/') self.url = settings.COMPRESS_URL.rstrip('/') self.url_path = self.url - try: - self.mtime = get_hashed_mtime(filename) - except OSError: - self.mtime = None + self.mtime = get_hashed_mtime(filename) self.has_http = False if self.url.startswith('http://') or self.url.startswith('https://'): self.has_http = True From 068a7e24c4aab97d2e1a54cbc64f5652bb9e1e80 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 12:25:59 +0200 Subject: [PATCH 16/25] Normalize settings.COMPRESS_ROOT in setting mangling instead in CSS default filter. --- compressor/filters/css_default.py | 2 +- compressor/settings.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/compressor/filters/css_default.py b/compressor/filters/css_default.py index 1aab20f2b..cf07fd7a8 100644 --- a/compressor/filters/css_default.py +++ b/compressor/filters/css_default.py @@ -12,7 +12,7 @@ class CssAbsoluteFilter(FilterBase): def input(self, filename=None, basename=None, **kwargs): - self.root = os.path.normcase(os.path.abspath(settings.COMPRESS_ROOT)) + self.root = settings.COMPRESS_ROOT if filename is not None: filename = os.path.normcase(os.path.abspath(filename)) if (not (filename and filename.startswith(self.root)) and diff --git a/compressor/settings.py b/compressor/settings.py index fd06e73d8..d838e3700 100644 --- a/compressor/settings.py +++ b/compressor/settings.py @@ -1,3 +1,4 @@ +import os from django import VERSION as DJANGO_VERSION from django.conf import settings from django.core.exceptions import ImproperlyConfigured @@ -66,7 +67,7 @@ def configure_root(self, value): if not value: raise ImproperlyConfigured( "The COMPRESS_ROOT setting must be set.") - return value + return os.path.normcase(os.path.abspath(value)) def configure_url(self, value): # Uses Django 1.3's STATIC_URL by default or falls back to MEDIA_URL From 32b9e8963ef2b6880b4e54fa993a5a04dde45b70 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 12:42:05 +0200 Subject: [PATCH 17/25] Combined cachekey generator in compressor.cache module. --- compressor/base.py | 3 +-- compressor/cache.py | 40 +++++++++++++++++++++++++---- compressor/templatetags/compress.py | 35 +++++-------------------- compressor/tests/tests.py | 6 ++--- 4 files changed, 45 insertions(+), 39 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index f828d648b..edf662a05 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -86,9 +86,8 @@ def mtimes(self): @cached_property def cachekey(self): - key = get_hexdigest(''.join( + return get_hexdigest(''.join( [self.content] + self.mtimes).encode(self.charset), 12) - return "django_compressor.%s.%s" % (socket.gethostname(), key) @cached_property def hunks(self): diff --git a/compressor/cache.py b/compressor/cache.py index 5da78db7c..264cf4c24 100644 --- a/compressor/cache.py +++ b/compressor/cache.py @@ -1,5 +1,6 @@ import os import socket +import time from django.core.cache import get_cache from django.utils.encoding import smart_str @@ -15,15 +16,22 @@ def get_hexdigest(plaintext, length=None): return digest +def get_cachekey(key): + return ("django_compressor.%s.%s" % (socket.gethostname(), key)) + + def get_mtime_cachekey(filename): - return "django_compressor.mtime.%s.%s" % (socket.gethostname(), - get_hexdigest(filename)) + return get_cachekey("mtime.%s" % get_hexdigest(filename)) def get_offline_cachekey(source): - return ("django_compressor.offline.%s.%s" % - (socket.gethostname(), - get_hexdigest("".join(smart_str(s) for s in source)))) + return get_cachekey( + "offline.%s" % get_hexdigest("".join(smart_str(s) for s in source))) + + +def get_templatetag_cachekey(compressor, mode, kind): + return get_cachekey( + "templatetag.%s.%s.%s" % (compressor.cachekey, mode, kind)) def get_mtime(filename): @@ -46,4 +54,26 @@ def get_hashed_mtime(filename, length=12): return get_hexdigest(mtime, length) +def cache_get(key): + packed_val = cache.get(key) + if packed_val is None: + return None + val, refresh_time, refreshed = packed_val + if (time.time() > refresh_time) and not refreshed: + # Store the stale value while the cache + # revalidates for another MINT_DELAY seconds. + cache_set(key, val, refreshed=True, + timeout=settings.COMPRESS_MINT_DELAY) + return None + return val + + +def cache_set(key, val, refreshed=False, + timeout=settings.COMPRESS_REBUILD_TIMEOUT): + refresh_time = timeout + time.time() + real_timeout = timeout + settings.COMPRESS_MINT_DELAY + packed_val = (val, refresh_time, refreshed) + return cache.set(key, packed_val, real_timeout) + + cache = get_cache(settings.COMPRESS_CACHE_BACKEND) diff --git a/compressor/templatetags/compress.py b/compressor/templatetags/compress.py index 44793a4ac..b51adb4b3 100644 --- a/compressor/templatetags/compress.py +++ b/compressor/templatetags/compress.py @@ -1,9 +1,8 @@ -import time - from django import template from django.core.exceptions import ImproperlyConfigured -from compressor.cache import cache, get_offline_cachekey +from compressor.cache import (cache, cache_get, cache_set, + get_offline_cachekey, get_templatetag_cachekey) from compressor.conf import settings from compressor.utils import get_class @@ -26,29 +25,6 @@ def __init__(self, nodelist, kind=None, mode=OUTPUT_FILE): self.compressor_cls = get_class( COMPRESSORS.get(self.kind), exception=ImproperlyConfigured) - def cache_get(self, key): - packed_val = cache.get(key) - if packed_val is None: - return None - val, refresh_time, refreshed = packed_val - if (time.time() > refresh_time) and not refreshed: - # Store the stale value while the cache - # revalidates for another MINT_DELAY seconds. - self.cache_set(key, val, refreshed=True, - timeout=settings.COMPRESS_MINT_DELAY) - return None - return val - - def cache_set(self, key, val, refreshed=False, - timeout=settings.COMPRESS_REBUILD_TIMEOUT): - refresh_time = timeout + time.time() - real_timeout = timeout + settings.COMPRESS_MINT_DELAY - packed_val = (val, refresh_time, refreshed) - return cache.set(key, packed_val, real_timeout) - - def cache_key(self, compressor): - return "%s.%s.%s" % (compressor.cachekey, self.mode, self.kind) - def debug_mode(self, context): if settings.COMPRESS_DEBUG_TOGGLE: # Only check for the debug parameter @@ -72,8 +48,9 @@ def render_cached(self, compressor, forced): and return a tuple of cache key and output """ if settings.COMPRESS_ENABLED and not forced: - cache_key = self.cache_key(compressor) - cache_content = self.cache_get(cache_key) + cache_key = get_templatetag_cachekey( + compressor, self.mode, self.kind) + cache_content = cache_get(cache_key) return cache_key, cache_content return None, None @@ -97,7 +74,7 @@ def render(self, context, forced=False): try: rendered_output = compressor.output(self.mode, forced=forced) if cache_key: - self.cache_set(cache_key, rendered_output) + cache_set(cache_key, rendered_output) return rendered_output except Exception, e: if settings.DEBUG or forced: diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 9acabe942..32903a0f5 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -86,9 +86,9 @@ def test_css_return_if_off(self): self.assertEqual(self.css, self.css_node.output()) def test_cachekey(self): - host_name = socket.gethostname() - is_cachekey = re.compile(r'django_compressor\.%s\.\w{12}' % host_name) - self.assert_(is_cachekey.match(self.css_node.cachekey), "cachekey is returning something that doesn't look like r'django_compressor\.%s\.\w{12}'" % host_name) + is_cachekey = re.compile(r'\w{12}') + self.assertTrue(is_cachekey.match(self.css_node.cachekey), + "cachekey is returning something that doesn't look like r'\w{12}'") def test_css_hash(self): self.assertEqual('c618e6846d04', get_hexdigest(self.css, 12)) From 4147f539221d3a8f4a288f832d12d135594faad1 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 12:42:15 +0200 Subject: [PATCH 18/25] Replaced a stray assert_. --- compressor/tests/tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 32903a0f5..71e6466a3 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -79,7 +79,8 @@ def test_css_output(self): def test_css_mtimes(self): is_date = re.compile(r'^\d{10}[\.\d]+$') for date in self.css_node.mtimes: - self.assert_(is_date.match(str(float(date))), "mtimes is returning something that doesn't look like a date: %s" % date) + self.assertTrue(is_date.match(str(float(date))), + "mtimes is returning something that doesn't look like a date: %s" % date) def test_css_return_if_off(self): settings.COMPRESS_ENABLED = False From 131d53ed6f87fba6cb787742040265c39642c4ff Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 13:19:46 +0200 Subject: [PATCH 19/25] Minor refactoring on default filter. --- compressor/filters/css_default.py | 42 +++++++++++++++---------------- compressor/tests/tests.py | 3 +++ 2 files changed, 23 insertions(+), 22 deletions(-) diff --git a/compressor/filters/css_default.py b/compressor/filters/css_default.py index cf07fd7a8..28acfeac7 100644 --- a/compressor/filters/css_default.py +++ b/compressor/filters/css_default.py @@ -11,8 +11,15 @@ class CssAbsoluteFilter(FilterBase): - def input(self, filename=None, basename=None, **kwargs): + + def __init__(self, *args, **kwargs): + super(CssAbsoluteFilter, self).__init__(*args, **kwargs) self.root = settings.COMPRESS_ROOT + self.url = settings.COMPRESS_URL.rstrip('/') + self.url_path = self.url + self.has_scheme = False + + def input(self, filename=None, basename=None, **kwargs): if filename is not None: filename = os.path.normcase(os.path.abspath(filename)) if (not (filename and filename.startswith(self.root)) and @@ -20,20 +27,16 @@ def input(self, filename=None, basename=None, **kwargs): return self.content self.path = basename.replace(os.sep, '/') self.path = self.path.lstrip('/') - self.url = settings.COMPRESS_URL.rstrip('/') - self.url_path = self.url self.mtime = get_hashed_mtime(filename) - self.has_http = False - if self.url.startswith('http://') or self.url.startswith('https://'): - self.has_http = True + if self.url.startswith(('http://', 'https://')): + self.has_scheme = True parts = self.url.split('/') self.url = '/'.join(parts[2:]) self.url_path = '/%s' % '/'.join(parts[3:]) self.protocol = '%s/' % '/'.join(parts[:2]) self.host = parts[2] - self.directory_name = '/'.join([self.url, os.path.dirname(self.path)]) - output = URL_PATTERN.sub(self.url_converter, self.content) - return output + self.directory_name = '/'.join((self.url, os.path.dirname(self.path))) + return URL_PATTERN.sub(self.url_converter, self.content) def find(self, basename): if settings.DEBUG and basename and staticfiles.finders: @@ -41,7 +44,7 @@ def find(self, basename): def guess_filename(self, url): local_path = url - if self.has_http: + if self.has_scheme: # COMPRESS_URL had a protocol, remove it and the hostname from our path. local_path = local_path.replace(self.protocol + self.host, "", 1) # Now, we just need to check if we can find the path from COMPRESS_URL in our url @@ -56,24 +59,19 @@ def add_mtime(self, url): mtime = filename and get_hashed_mtime(filename) or self.mtime if mtime is None: return url - if (url.startswith('http://') or - url.startswith('https://') or - url.startswith('/')): + if url.startswith(('http://', 'https://', '/')): if "?" in url: - return "%s&%s" % (url, mtime) - return "%s?%s" % (url, mtime) + url = "%s&%s" % (url, mtime) + else: + url = "%s?%s" % (url, mtime) return url def url_converter(self, matchobj): url = matchobj.group(1) url = url.strip(' \'"') - if (url.startswith('http://') or - url.startswith('https://') or - url.startswith('/') or - url.startswith('data:')): + if url.startswith(('http://', 'https://', '/', 'data:')): return "url('%s')" % self.add_mtime(url) - full_url = '/'.join([str(self.directory_name), url]) - full_url = posixpath.normpath(full_url) - if self.has_http: + full_url = posixpath.normpath('/'.join([self.directory_name, url])) + if self.has_scheme: full_url = "%s%s" % (self.protocol, full_url) return "url('%s')" % self.add_mtime(full_url) diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index 71e6466a3..eb0e058ef 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -219,6 +219,7 @@ def test_css_absolute_filter(self): filter = CssAbsoluteFilter(content) self.assertEqual(output, filter.input(filename=filename, basename='css/url/test.css')) settings.COMPRESS_URL = 'http://media.example.com/' + filter = CssAbsoluteFilter(content) filename = os.path.join(settings.COMPRESS_ROOT, 'css/url/test.css') output = "p { background: url('%simages/image.gif?%s') }" % (settings.COMPRESS_URL, get_hashed_mtime(filename)) self.assertEqual(output, filter.input(filename=filename, basename='css/url/test.css')) @@ -231,6 +232,7 @@ def test_css_absolute_filter_https(self): filter = CssAbsoluteFilter(content) self.assertEqual(output, filter.input(filename=filename, basename='css/url/test.css')) settings.COMPRESS_URL = 'https://media.example.com/' + filter = CssAbsoluteFilter(content) filename = os.path.join(settings.COMPRESS_ROOT, 'css/url/test.css') output = "p { background: url('%simages/image.gif?%s') }" % (settings.COMPRESS_URL, get_hashed_mtime(filename)) self.assertEqual(output, filter.input(filename=filename, basename='css/url/test.css')) @@ -243,6 +245,7 @@ def test_css_absolute_filter_relative_path(self): filter = CssAbsoluteFilter(content) self.assertEqual(output, filter.input(filename=filename, basename='css/url/test.css')) settings.COMPRESS_URL = 'https://media.example.com/' + filter = CssAbsoluteFilter(content) output = "p { background: url('%simages/image.gif?%s') }" % (settings.COMPRESS_URL, get_hashed_mtime(filename)) self.assertEqual(output, filter.input(filename=filename, basename='css/url/test.css')) From 282a3c1668451bda8a9534b7312b373d4e39cad7 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 13:33:00 +0200 Subject: [PATCH 20/25] Added [] to tox command call to enable specific runs of tests. --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 065a55b7c..1ac9bfd8d 100644 --- a/tox.ini +++ b/tox.ini @@ -2,7 +2,7 @@ distribute = false sitepackages = true commands = - {envpython} compressor/tests/runtests.py + {envpython} compressor/tests/runtests.py [] coverage html -d {envtmpdir}/coverage [testenv:docs] From fc951bcae04d669eaede9d87cd45666daf0be27c Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 13:36:52 +0200 Subject: [PATCH 21/25] Whitespace cleanup. --- compressor/tests/precompiler.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/compressor/tests/precompiler.py b/compressor/tests/precompiler.py index 9c2ac132d..c15b10310 100644 --- a/compressor/tests/precompiler.py +++ b/compressor/tests/precompiler.py @@ -4,24 +4,24 @@ def main(): p = optparse.OptionParser() - p.add_option('-f', '--file', action="store", - type="string", dest="filename", + p.add_option('-f', '--file', action="store", + type="string", dest="filename", help="File to read from, defaults to stdin", default=None) p.add_option('-o', '--output', action="store", - type="string", dest="outfile", + type="string", dest="outfile", help="File to write to, defaults to stdout", default=None) - + options, arguments = p.parse_args() - + if options.filename: f = open(options.filename) content = f.read() f.close() else: content = sys.stdin.read() - + content = content.replace('background:', 'color:') - + if options.outfile: f = open(options.outfile, 'w') f.write(content) @@ -29,7 +29,6 @@ def main(): else: print content - + if __name__ == '__main__': main() - \ No newline at end of file From eb781f81cf87c8f42f62a1c0a59c541c1ac644dc Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 13:39:20 +0200 Subject: [PATCH 22/25] Updated changelog. --- docs/changelog.txt | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/docs/changelog.txt b/docs/changelog.txt index d613db149..058d2c627 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -1,6 +1,18 @@ Changelog ========= +HEAD +---- + +- Fixed the precompiler support to also use the full file path instead of a + temporarily created file. + +- Enabled test coverage. + +- Refactored caching und other utility code. + +- Switched from SHA1 to MD5 for hash generation to lower the computational impact. + 0.8 --- From d2d331734c9d78a148304d8fb509758b0b74f48d Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Fri, 13 May 2011 16:18:37 +0200 Subject: [PATCH 23/25] Fixed typo. --- docs/changelog.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog.txt b/docs/changelog.txt index 058d2c627..093cd3eca 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -9,7 +9,7 @@ HEAD - Enabled test coverage. -- Refactored caching und other utility code. +- Refactored caching and other utility code. - Switched from SHA1 to MD5 for hash generation to lower the computational impact. From cd0175b66f80e181e39c00437c909bb8d8167617 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 18 May 2011 12:13:42 +0200 Subject: [PATCH 24/25] Removed a few unneeded imports. --- compressor/base.py | 1 - compressor/css.py | 2 -- compressor/js.py | 2 -- compressor/parser/htmlparser.py | 1 - compressor/tests/runtests.py | 1 - compressor/tests/tests.py | 1 - 6 files changed, 8 deletions(-) diff --git a/compressor/base.py b/compressor/base.py index edf662a05..647d49b93 100644 --- a/compressor/base.py +++ b/compressor/base.py @@ -1,5 +1,4 @@ import os -import socket from django.core.files.base import ContentFile from django.template.loader import render_to_string diff --git a/compressor/css.py b/compressor/css.py index 41a484d7b..0ffe31d51 100644 --- a/compressor/css.py +++ b/compressor/css.py @@ -1,5 +1,3 @@ -import os - from compressor.conf import settings from compressor.base import Compressor, SOURCE_HUNK, SOURCE_FILE from compressor.exceptions import UncompressableFileError diff --git a/compressor/js.py b/compressor/js.py index 481478603..53530b9df 100644 --- a/compressor/js.py +++ b/compressor/js.py @@ -1,5 +1,3 @@ -import os - from compressor.conf import settings from compressor.base import Compressor, SOURCE_HUNK, SOURCE_FILE from compressor.exceptions import UncompressableFileError diff --git a/compressor/parser/htmlparser.py b/compressor/parser/htmlparser.py index dbbc76e5f..f3de6830e 100644 --- a/compressor/parser/htmlparser.py +++ b/compressor/parser/htmlparser.py @@ -1,6 +1,5 @@ from HTMLParser import HTMLParser from django.utils.encoding import smart_unicode -from django.utils.datastructures import SortedDict from compressor.exceptions import ParserError from compressor.parser import ParserBase diff --git a/compressor/tests/runtests.py b/compressor/tests/runtests.py index 0b8dea170..9752ec627 100644 --- a/compressor/tests/runtests.py +++ b/compressor/tests/runtests.py @@ -5,7 +5,6 @@ from os.path import join from django.conf import settings -from django.core.management import call_command TEST_DIR = os.path.dirname(os.path.abspath(__file__)) diff --git a/compressor/tests/tests.py b/compressor/tests/tests.py index eb0e058ef..d96505722 100644 --- a/compressor/tests/tests.py +++ b/compressor/tests/tests.py @@ -1,7 +1,6 @@ from __future__ import with_statement import os import re -import socket import sys from unittest2 import skipIf From 69f91c94067f6da47fafb9e09bd7dd096b164c45 Mon Sep 17 00:00:00 2001 From: Jannis Leidel Date: Wed, 18 May 2011 12:21:43 +0200 Subject: [PATCH 25/25] Bumped version to 0.9. --- compressor/__init__.py | 2 +- docs/changelog.txt | 4 ++-- docs/conf.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compressor/__init__.py b/compressor/__init__.py index ad83c1115..974d2e19b 100644 --- a/compressor/__init__.py +++ b/compressor/__init__.py @@ -1,4 +1,4 @@ -VERSION = (0, 8, 0, "f", 0) # following PEP 386 +VERSION = (0, 9, 0, "f", 0) # following PEP 386 DEV_N = None diff --git a/docs/changelog.txt b/docs/changelog.txt index 093cd3eca..4a5d526b6 100644 --- a/docs/changelog.txt +++ b/docs/changelog.txt @@ -1,8 +1,8 @@ Changelog ========= -HEAD ----- +0.9 +--- - Fixed the precompiler support to also use the full file path instead of a temporarily created file. diff --git a/docs/conf.py b/docs/conf.py index b9b810a6b..fb3d6c7ad 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,9 +48,9 @@ # built documents. # # The short X.Y version. -version = '0.8' +version = '0.9' # The full version, including alpha/beta/rc tags. -release = '0.8' +release = '0.9' # The language for content autogenerated by Sphinx. Refer to documentation # for a list of supported languages.