Permalink
Browse files

Merge branch 'cache_urls'

Conflicts:
	filertags/signals.py
  • Loading branch information...
2 parents e491d79 + 665658c commit 9d4c2e600376a35b1f36f9699c44ad2ab028f609 @kux kux committed Nov 18, 2012
Showing with 171 additions and 83 deletions.
  1. +106 −69 filertags/signals.py
  2. +6 −2 filertags/templatetags/filertags.py
  3. +59 −12 filertags/tests.py
View
@@ -1,8 +1,11 @@
+import hashlib
import os.path
import re
import urlparse
from django.core.cache import cache
+from django.core.files.uploadedfile import UploadedFile
+from django.core.files.base import ContentFile
from django.db.models import signals
from filer.models.filemodels import File
@@ -13,10 +16,10 @@
_LOGICAL_URL_TEMPLATE = "/* logicalurl('%s') */"
_RESOURCE_URL_TEMPLATE = "url('%s') " + _LOGICAL_URL_TEMPLATE
-
-_RESOURCE_URL_REGEX = re.compile(r"url\(['\"]?([^'\"\)]+?)['\"]?\)")
+_RESOURCE_URL_REGEX = re.compile(r"\burl\(([^\)]*)\)")
_COMMENT_REGEX = re.compile(r"/\*.*?\*/")
+_ALREADY_PARSED_MARKER = '/* Filer urls already resolved */'
def _is_in_clipboard(filer_file):
@@ -31,19 +34,75 @@ def _get_commented_regions(content):
return [(m.start(), m.end()) for m in re.finditer(_COMMENT_REGEX, content)]
+def _is_in_memory(file_):
+ return isinstance(file_, UploadedFile)
+
+
def _rewrite_file_content(filer_file, new_content):
- old_file = filer_file.file.file
- storage = filer_file.file.storage
- new_file = storage.open(old_file.name, 'w')
- try:
- new_file.write(new_content)
- finally:
- new_file.close()
+ if _is_in_memory(filer_file.file.file):
+ filer_file.file.seek(0)
+ filer_file.file.write(new_content)
+ else:
+ # file_name = filer_file.original_filename
+ storage = filer_file.file.storage
+ fp = ContentFile(new_content, filer_file.file.name)
+ filer_file.file.file = fp
+ filer_file.file.name = storage.save(filer_file.file.name, fp)
+ sha = hashlib.sha1()
+ sha.update(new_content)
+ filer_file.sha1 = sha.hexdigest()
+ filer_file._file_size = len(new_content)
+
+
+def _is_css(filer_file):
+ if filer_file.name:
+ return filer_file.name.endswith('.css')
+ else:
+ return filer_file.original_filename.endswith('.css')
-def _resolve_resource_urls(css_file):
- logical_folder_path = _construct_logical_folder_path(css_file)
+def resolve_resource_urls(instance, **kwargs):
+ """Post save hook for css files uploaded to filer.
+ It's purpose is to resolve the actual urls of resources referenced
+ in css files.
+
+ django-filer has two concepts of urls:
+ * the logical url: media/images/foobar.png
+ * the actual url: filer_public/2012/11/22/foobar.png
+
+ The css as written by the an end user uses logical urls:
+ .button.nice {
+ background: url('../images/misc/foobar.png');
+ -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.5);
+ }
+
+ In order for the resources to be found, the logical urls need to be
+ replaced with the actual urls.
+
+ Whenever a css is saved it parses the content and rewrites all logical
+ urls to their actual urls; the logical url is still being saved
+ as a comment that follows the actual url. This comment is needed for
+ the behaviour described at point 2.
+
+ After url rewriting the above css snippet will look like:
+ .button.nice {
+ background: url('filer_public/2012/11/22/foobar.png') /* logicalurl('media/images/misc/foobar.png') /* ;
+ -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.5);
+ }
+ """
+ if not _is_css(instance):
+ return
+ css_file = instance
+ if _is_in_clipboard(css_file):
+ return
content = css_file.file.read()
+ if content.startswith(_ALREADY_PARSED_MARKER):
+ # this css' resource urls have already been resolved
+ # this happens when moving the css in and out of the clipboard
+ # multiple times
+ return
+
+ logical_folder_path = _construct_logical_folder_path(css_file)
commented_regions = _get_commented_regions(content)
local_cache = {}
@@ -52,7 +111,8 @@ def change_urls(match):
# we don't make any changes to urls that are part of commented regions
if start < match.start() < end or start < match.end() < end:
return match.group()
- url = match.group(1)
+ # strip spaces and quotes
+ url = match.group(1).strip('\'\" ')
parsed_url = urlparse.urlparse(url)
if parsed_url.netloc:
# if the url is absolute, leave it unchaged
@@ -65,79 +125,55 @@ def change_urls(match):
filerfile(logical_file_path), logical_file_path)
return local_cache[logical_file_path]
- new_content = re.sub(_RESOURCE_URL_REGEX, change_urls, content)
+ new_content = '%s\n%s' % (
+ _ALREADY_PARSED_MARKER,
+ re.sub(_RESOURCE_URL_REGEX, change_urls, content))
_rewrite_file_content(css_file, new_content)
-def _update_referencing_css_files(resource_file):
+def update_referencing_css_files(instance, **kwargs):
+ """Post save hook for any resource uploaded to filer that
+ might be referenced by a css.
+ The purpose of this hook is to update the actual url in all css files that
+ reference the resource pointed by 'instance'.
+
+ References are found by looking for comments such as:
+ /* logicalurl('media/images/misc/foobar.png') */
+
+ If the url between parentheses matches the logical url of the resource
+ being saved, the actual url (which percedes the comment)
+ is being updated.
+ """
+ if _is_css(instance):
+ return
+ resource_file = instance
+ if _is_in_clipboard(resource_file):
+ return
+ if resource_file.name:
+ resource_name = resource_file.name
+ else:
+ resource_name = resource_file.original_filename
logical_file_path = os.path.join(
_construct_logical_folder_path(resource_file),
- resource_file.original_filename)
+ resource_name)
css_files = File.objects.filter(original_filename__endswith=".css")
-
for css in css_files:
logical_url_snippet = _LOGICAL_URL_TEMPLATE % logical_file_path
url_updating_regex = "%s %s" % (
_RESOURCE_URL_REGEX.pattern, re.escape(logical_url_snippet))
repl = "url('%s') %s" % (resource_file.url, logical_url_snippet)
try:
- new_content = re.sub(url_updating_regex, repl, css.file.read())
+ content = css.file.read()
+ new_content = re.sub(url_updating_regex, repl, content)
except IOError:
# the filer database might have File entries that reference
# files no longer phisically exist
# TODO: find the root cause of missing filer files
continue
else:
- _rewrite_file_content(css, new_content)
-
-
-def resolve_css_resource_urls(instance, **kwargs):
- """Post save hook for filer resources.
- It's purpose is to resolve the actual urls of resources referenced
- in css files.
-
- django-filer has two concepts of urls:
- * the logical url: media/images/foobar.png
- * the actual url: filer_public/2012/11/22/foobar.png
-
- The css as written by the an end user uses logical urls:
- .button.nice {
- background: url('../images/misc/foobar.png');
- -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.5);
- }
-
- In order for the resources to be found, the logical urls need to be
- replaced with the actual urls.
-
- This post save hook does this in two ways:
- 1) whenever a css is saved it parses the content and rewrites all logical
- urls to their actual urls; the logical url is still being saved
- as a comment that follows the actual url. This comment is needed for
- the behaviour described at point 2.
-
- After url rewriting the above css snippet will look like:
- .button.nice {
- background: url('filer_public/2012/11/22/foobar.png') /* logicalurl('media/images/misc/foobar.png') /* ;
- -moz-box-shadow: inset 0 1px 0 rgba(255,255,255,.5);
- }
-
- 2) when any other kind of resource is saved, all css files are parsed for
- references to the resource being saved. If found, the actual url is
- being rewritten.
-
- References are found by looking for comments such as:
- /* logicalurl('media/images/misc/foobar.png') */
-
- If the url between parentheses matches the logical url of the resource
- being saved, the actual url (which percedes the comment)
- is being updated.
- """
- if _is_in_clipboard(instance):
- return
- if instance.original_filename.endswith('.css'):
- _resolve_resource_urls(instance)
- else:
- _update_referencing_css_files(instance)
+ if content != new_content:
+ _rewrite_file_content(css, new_content)
+ css.save()
def clear_urls_cache(instance, **kwargs):
@@ -149,8 +185,9 @@ def clear_urls_cache(instance, **kwargs):
cache.delete(cache_key)
-signals.post_save.connect(resolve_css_resource_urls, sender=File)
-signals.post_save.connect(resolve_css_resource_urls, sender=Image)
+signals.pre_save.connect(resolve_resource_urls, sender=File)
+signals.post_save.connect(update_referencing_css_files, sender=File)
+signals.post_save.connect(update_referencing_css_files, sender=Image)
signals.post_save.connect(clear_urls_cache, sender=File)
signals.post_save.connect(clear_urls_cache, sender=Image)
@@ -1,10 +1,14 @@
+import logging
+
from django import template
from django.db.models import Q
from django.core.cache import cache
from django.template.defaultfilters import stringfilter, slugify
from filer.models import File, Folder
+logger = logging.getLogger(__name__)
+
def filerthumbnail(path):
parts = path.strip('/').split('/')
@@ -25,8 +29,8 @@ def filerthumbnail(path):
q |= Q(original_filename=file_name, folder=folder, name__isnull=True)
q |= Q(name=file_name, folder=folder)
return File.objects.get(q).file
- except (File.DoesNotExist, Folder.DoesNotExist):
- pass
+ except (File.DoesNotExist, File.MultipleObjectsReturned, Folder.DoesNotExist), e:
+ logger.info('%s on %s' % (e.message, path))
def get_filerfile_cache_key(path):
View
@@ -1,16 +1,63 @@
-"""
-This file demonstrates writing tests using the unittest module. These will pass
-when you run "manage.py test".
-
-Replace this with more appropriate tests for your application.
-"""
+import os
+from django.core.files import File as DjangoFile
from django.test import TestCase
+from filer.models.foldermodels import Folder
+from filer.models.imagemodels import Image
+from filer.models.filemodels import File
+
+from filer.tests.helpers import create_superuser, create_image
+
+from filertags.signals import _ALREADY_PARSED_MARKER
-class SimpleTest(TestCase):
- def test_basic_addition(self):
- """
- Tests that 1 + 1 always equals 2.
- """
- self.assertEqual(1 + 1, 2)
+
+class CssResourceUrlResolvingTest(TestCase):
+
+ def setUp(self):
+ self.superuser = create_superuser()
+ self.client.login(username='admin', password='secret')
+ self.image = create_image()
+ self.image_name = 'test_file.jpg'
+ self.image_filename = os.path.join(
+ os.path.dirname(__file__), 'test_resources', self.image_name)
+ self.image.save(self.image_filename, 'JPEG')
+ self.resource_folder = Folder.objects.create(
+ owner=self.superuser,
+ name='css_resources_test')
+ self.css_folder = Folder.objects.create(
+ owner=self.superuser,
+ name='css',
+ parent=self.resource_folder)
+ self.images_folder = Folder.objects.create(
+ owner=self.superuser,
+ name='images',
+ parent=self.resource_folder)
+ image_file_obj = DjangoFile(open(self.image_filename), name=self.image_name)
+ filer_image = Image(
+ owner=self.superuser,
+ original_filename=self.image_name,
+ folder=self.images_folder,
+ file=image_file_obj)
+ filer_image.save()
+
+ def test_resolve_urls_quoted(self):
+ css_content = """
+.bgimage-single-quotes {
+ background:#CCCCCC url( '../images/test_file.jpg' ) no-repeat center center;
+ background-color: black;
+}
+"""
+ css_path = os.path.join(os.path.dirname(__file__),
+ 'test_resources', 'resources.css')
+ with open(css_path, 'w') as f:
+ f.write(css_content)
+ css_file_obj = DjangoFile(open(css_path), name='resources.css')
+ filer_css = File(owner=self.superuser,
+ original_filename='resources.css',
+ folder=self.css_folder,
+ file=css_file_obj)
+ filer_css.save()
+ with open(filer_css.path) as f:
+ new_content = f.read()
+ self.assertTrue(new_content.startswith(_ALREADY_PARSED_MARKER))

0 comments on commit 9d4c2e6

Please sign in to comment.