Merge pull request #103 from fitodic/feature/modify_img_dir

Customize daguerre's directory path

daguerre/management/commands/_daguerre_clean.py

@@ -1,14 +1,15 @@
 from __future__ import absolute_import
 import os
 
+from django.conf import settings
 from django.core.files.storage import default_storage
 from django.core.management.base import BaseCommand
 from django.db import models
 from django.template.defaultfilters import pluralize
 import six
 
 from daguerre.helpers import IOERRORS
-from daguerre.models import AdjustedImage, Area
+from daguerre.models import AdjustedImage, Area, DEFAULT_ADJUSTED_IMAGE_PATH
 
 
 class Command(BaseCommand):
@@ -127,12 +128,16 @@ def _orphaned_files(self):
         in the database.
 
         """
+        base_dir = getattr(
+            settings, 'DAGUERRE_ADJUSTED_IMAGE_PATH',
+            DEFAULT_ADJUSTED_IMAGE_PATH)
+
         known_paths = set(
             AdjustedImage.objects.values_list('adjusted', flat=True).distinct()
         )
         orphans = []
         for dirpath, dirnames, filenames in self._walk(
-                'daguerre', topdown=False):
+                base_dir, topdown=False):
             for filename in filenames:
                 filepath = os.path.join(dirpath, filename)
                 if filepath not in known_paths:

daguerre/migrations/0004_hash_upload_to_dir.py

@@ -0,0 +1,20 @@
+# -*- coding: utf-8 -*-
+from __future__ import unicode_literals
+
+from django.db import migrations, models
+import daguerre.models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('daguerre', '0003_auto_20160301_2342'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='adjustedimage',
+            name='adjusted',
+            field=models.ImageField(max_length=45, upload_to=daguerre.models.upload_to),
+        ),
+    ]

daguerre/models.py

@@ -1,15 +1,30 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import unicode_literals
+
+import hashlib
 import operator
+import warnings
+from datetime import datetime
 
+from django.conf import settings
 from django.core.exceptions import ValidationError
 from django.core.validators import MinValueValidator
 from django.db import models
 from django.db.models.signals import post_delete, post_save
 from django.dispatch import receiver
+from django.utils.encoding import force_bytes, python_2_unicode_compatible
 from six.moves import reduce
 
 from daguerre.adjustments import registry
 
+# The default image path where the images will be saved to. Can be overriden by
+# defining the DAGUERRE_ADJUSTED_IMAGE_PATH setting in the project's settings.
+# Example: DAGUERRE_ADJUSTED_IMAGE_PATH = 'img'
+DEFAULT_ADJUSTED_IMAGE_PATH = 'dg'
 
+
+@python_2_unicode_compatible
 class Area(models.Model):
     """
     Represents an area of an image. Can be used to specify a crop. Also used
@@ -68,7 +83,7 @@ def serialize(self):
         return dict((f.name, getattr(self, f.name))
                     for f in self._meta.fields)
 
-    def __unicode__(self):
+    def __str__(self):
         if self.name:
             name = self.name
         else:
@@ -99,17 +114,58 @@ def delete_adjusted_images(sender, **kwargs):
     qs.delete()
 
 
+def upload_to(instance, filename):
+    """
+    Construct the directory path where the adjusted images will be saved to
+    using the MD5 hash algorithm.
+
+    Can be customized using the DAGUERRE_PATH setting set in the project's
+    settings. If left unspecified, the default value will be used, i.e. 'dg'.
+    WARNING: The maximum length of the specified string is 13 characters.
+
+    Example:
+    * Default: dg/ce/2b/7014c0bdbedea0e4f4bf.jpeg
+    * DAGUERRE_PATH = 'img': img/ce/2b/7014c0bdbedea0e4f4bf.jpeg
+
+    Known issue:
+    * If the extracted hash string is 'ad', ad blockers will block the image.
+      All occurrences of 'ad' will be replaced with 'ag' since the MD5 hash
+      produces letters from 'a' to 'f'.
+    """
+
+    image_path = getattr(
+        settings, 'DAGUERRE_ADJUSTED_IMAGE_PATH', DEFAULT_ADJUSTED_IMAGE_PATH)
+
+    if len(image_path) > 13:
+        msg = ('The DAGUERRE_PATH value is more than 13 characters long! '
+               'Falling back to the default '
+               'value: "{}".'.format(DEFAULT_ADJUSTED_IMAGE_PATH))
+        warnings.warn(msg)
+        image_path = DEFAULT_ADJUSTED_IMAGE_PATH
+
+    # Avoid TypeError on Py3 by forcing the string to bytestring
+    # https://docs.djangoproject.com/en/dev/_modules/django/contrib/auth/hashers/
+    # https://github.com/django/django/blob/master/django/contrib/auth/hashers.py#L524
+    str_for_hash = force_bytes('{} {}'.format(filename, datetime.utcnow()))
+    # Replace all occurrences of 'ad' with 'ag' to avoid ad blockers
+    hash_for_dir = hashlib.md5(str_for_hash).hexdigest().replace('ad', 'ag')
+    return '{0}/{1}/{2}/{3}'.format(
+        image_path, hash_for_dir[0:2], hash_for_dir[2:4], filename)
+
+
+@python_2_unicode_compatible
 class AdjustedImage(models.Model):
     """Represents a managed image adjustment."""
     storage_path = models.CharField(max_length=200)
     # The image name is a 20-character hash, so the max length with a 4-char
-    # extension (jpeg) is 45.
-    adjusted = models.ImageField(upload_to='daguerre/%Y/%m/%d/',
+    # extension (jpeg) is 45. The maximum length of the
+    # DAGUERRE_ADJUSTED_IMAGE_PATH string is 13.
+    adjusted = models.ImageField(upload_to=upload_to,
                                  max_length=45)
     requested = models.CharField(max_length=100)
 
     class Meta:
         index_together = [['requested', 'storage_path'], ]
 
-    def __unicode__(self):
+    def __str__(self):
         return u"{0}: {1}".format(self.storage_path, self.requested)

daguerre/tests/unit/test_management.py

@@ -101,20 +101,36 @@ def test_duplicate_adjustments(self):
         self.assertTrue(list(duplicates) == [adjusted1] or
                         list(duplicates) == [adjusted2])
 
-    def test_orphaned_files(self):
+    def test_orphaned_files__default_path(self):
         clean = Clean()
         walk_ret = (
-            ('daguerre', ['test'], []),
-            ('daguerre/test', [], ['fake1.png', 'fake2.png', 'fake3.png'])
+            ('dg', ['test'], []),
+            ('dg/test', [], ['fake1.png', 'fake2.png', 'fake3.png'])
         )
         AdjustedImage.objects.create(requested='fit|50|50',
                                      storage_path='whatever.png',
-                                     adjusted='daguerre/test/fake2.png')
+                                     adjusted='dg/test/fake2.png')
         with mock.patch.object(clean, '_walk', return_value=walk_ret) as walk:
             self.assertEqual(clean._orphaned_files(),
-                             ['daguerre/test/fake1.png',
-                              'daguerre/test/fake3.png'])
-            walk.assert_called_once_with('daguerre', topdown=False)
+                             ['dg/test/fake1.png',
+                              'dg/test/fake3.png'])
+            walk.assert_called_once_with('dg', topdown=False)
+
+    @override_settings(DAGUERRE_ADJUSTED_IMAGE_PATH='img')
+    def test_orphaned_files__modified_path(self):
+        clean = Clean()
+        walk_ret = (
+            ('img', ['test'], []),
+            ('img/test', [], ['fake1.png', 'fake2.png', 'fake3.png'])
+        )
+        AdjustedImage.objects.create(requested='fit|50|50',
+                                     storage_path='whatever.png',
+                                     adjusted='img/test/fake2.png')
+        with mock.patch.object(clean, '_walk', return_value=walk_ret) as walk:
+            self.assertEqual(clean._orphaned_files(),
+                             ['img/test/fake1.png',
+                              'img/test/fake3.png'])
+            walk.assert_called_once_with('img', topdown=False)
 
 
 class PreadjustTestCase(BaseTestCase):

daguerre/tests/unit/test_models.py

@@ -1,6 +1,10 @@
-from daguerre.models import AdjustedImage
+import warnings
+
+from daguerre.models import AdjustedImage, upload_to
 from daguerre.tests.base import BaseTestCase
 
+from django.test.utils import override_settings
+
 
 class AreaTestCase(BaseTestCase):
     def test_delete_adjusted_images__save(self):
@@ -50,3 +54,63 @@ def test_delete_adjusted_images__delete(self):
                           AdjustedImage.objects.get,
                           pk=adjusted2.pk)
         AdjustedImage.objects.get(pk=adjusted1.pk)
+
+
+class AdjustedImageUploadToTestCase(BaseTestCase):
+
+    def setUp(self):
+        self.instance = None
+        self.filename = '7014c0bdbedea0e4f4bf.jpeg'
+
+    def test_upload_to__default_upload_dir(self):
+        with warnings.catch_warnings(record=True) as w:
+            hash_path = upload_to(
+                instance=self.instance, filename=self.filename)
+            self.assertTrue(hash_path.startswith('dg/'))
+            self.assertTrue(hash_path.endswith('/{}'.format(self.filename)))
+            self.assertEqual(len(hash_path.split('/')), 4)
+            self.assertTrue(len(hash_path) < 45)
+            self.assertEqual(w, [])
+
+    @override_settings(DAGUERRE_ADJUSTED_IMAGE_PATH='img')
+    def test_upload_to__custom_upload_dir(self):
+        with warnings.catch_warnings(record=True) as w:
+            hash_path = upload_to(
+                instance=self.instance, filename=self.filename)
+            self.assertTrue(hash_path.startswith('img/'))
+            self.assertTrue(hash_path.endswith('/{}'.format(self.filename)))
+            self.assertEqual(len(hash_path.split('/')), 4)
+            self.assertTrue(len(hash_path) < 45)
+            self.assertEqual(w, [])
+
+    @override_settings(DAGUERRE_ADJUSTED_IMAGE_PATH='0123456789123')
+    def test_upload_to__custom_upload_dir_small(self):
+        with warnings.catch_warnings(record=True) as w:
+            hash_path = upload_to(
+                instance=self.instance, filename=self.filename)
+            self.assertTrue(hash_path.startswith('0123456789123/'))
+            self.assertTrue(hash_path.endswith('/{}'.format(self.filename)))
+            self.assertEqual(len(hash_path.split('/')), 4)
+            self.assertTrue(len(hash_path) == 45)
+            self.assertEqual(w, [])
+
+    @override_settings(DAGUERRE_ADJUSTED_IMAGE_PATH='01234567891234')
+    def test_upload_to__custom_upload_dir_big(self):
+        with warnings.catch_warnings(record=True) as w:
+            hash_path = upload_to(
+                instance=self.instance, filename=self.filename)
+            self.assertTrue(hash_path.startswith('dg/'))
+            self.assertTrue(hash_path.endswith('/{}'.format(self.filename)))
+            self.assertEqual(len(hash_path.split('/')), 4)
+            self.assertTrue(len(hash_path) < 45)
+
+            # Test the warning message
+            # https://docs.python.org/2/library/warnings.html#testing-warnings
+            warning_message = ('The DAGUERRE_PATH value is more than 13 '
+                               'characters long! Falling back to the default '
+                               'value: "dg".')
+
+            self.assertEqual(len(w), 1)
+            user_warning = w[0]
+            self.assertEqual(user_warning.category, UserWarning)
+            self.assertEqual(user_warning.message.__str__(), warning_message)

docs/guides/settings.rst

@@ -0,0 +1,25 @@
+Custom settings
+===============
+
+Adjust the image path
++++++++++++++++++++++
+
+The variations are stored under a hashed directory path that starts with the
+``dg`` directory by default (e.g. ``dg/ce/2b/7014c0bdbedea0e4f4bf.jpeg``).
+This setting can be modified in the project's settings by setting the
+``DAGUERRE_ADJUSTED_IMAGE_PATH`` variable.
+
+Example:
+
+.. code-block:: django
+
+    # settings.py
+    DAGUERRE_ADJUSTED_IMAGE_PATH = 'img'
+
+which would produce the following path: ``img/ce/2b/7014c0bdbedea0e4f4bf.jpeg``
+
+
+.. WARNING::
+   The maximum length of the ``DAGUERRE_ADJUSTED_IMAGE_PATH`` string
+   is 13 characters. If the string has more than 13 characters, it will
+   gracefully fall back to the the default value, i.e. ``dg``

docs/index.rst

@@ -6,7 +6,7 @@ Django Daguerre
    :align: right
    :scale: 33 %
    :target: http://en.wikipedia.org/wiki/Louis_Daguerre
-   
+
    Louis Daguerre, Father of Photography
 
 **Django Daguerre** manipulates images on the fly. Use it to scale
@@ -45,6 +45,7 @@ Contents
    guides/template-tags
    guides/areas
    guides/commands
+   guides/settings
 
 
 API Docs