Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

File storage refactoring, adding far more flexibility to Django's fil…

…e handling. The new files.txt document has details of the new features.

This is a backwards-incompatible change; consult BackwardsIncompatibleChanges for details.

Fixes #3567, #3621, #4345, #5361, #5655, #7415.

Many thanks to Marty Alchin who did the vast majority of this work.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@8244 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit 7899568e01fc9c68afe995fa71de915dd9fcdd76 1 parent c49eac7
Jacob Kaplan-Moss authored August 08, 2008

Showing 33 changed files with 1,586 additions and 458 deletions. Show diff stats Hide diff stats

  1. 3  django/conf/global_settings.py
  2. 4  django/contrib/admin/widgets.py
  3. 1  django/core/files/__init__.py
  4. 169  django/core/files/base.py
  5. 42  django/core/files/images.py
  6. 214  django/core/files/storage.py
  7. 59  django/core/files/uploadedfile.py
  8. 1  django/db/models/__init__.py
  9. 123  django/db/models/base.py
  10. 160  django/db/models/fields/__init__.py
  11. 315  django/db/models/fields/files.py
  12. 3  django/db/models/manipulators.py
  13. 23  django/utils/images.py
  14. 39  docs/custom_model_fields.txt
  15. 43  docs/db-api.txt
  16. 388  docs/files.txt
  17. 58  docs/model-api.txt
  18. 10  docs/settings.txt
  19. 25  docs/upload_handling.txt
  20. 1  tests/modeltests/files/__init__.py
  21. 118  tests/modeltests/files/models.py
  22. 69  tests/modeltests/model_forms/models.py
  23. 14  tests/regressiontests/admin_widgets/models.py
  24. 12  tests/regressiontests/bug639/models.py
  25. 2  tests/regressiontests/bug639/tests.py
  26. 1  tests/regressiontests/file_storage/__init__.py
  27. 44  tests/regressiontests/file_storage/models.py
  28. BIN  tests/regressiontests/file_storage/test.png
  29. 66  tests/regressiontests/file_storage/tests.py
  30. 7  tests/regressiontests/file_uploads/models.py
  31. 22  tests/regressiontests/file_uploads/tests.py
  32. 4  tests/regressiontests/serializers_regress/models.py
  33. 4  tests/regressiontests/serializers_regress/tests.py
3  django/conf/global_settings.py
@@ -226,6 +226,9 @@
226 226
 # Path to the "jing" executable -- needed to validate XMLFields
227 227
 JING_PATH = "/usr/bin/jing"
228 228
 
  229
+# Default file storage mechanism that holds media.
  230
+DEFAULT_FILE_STORAGE = 'django.core.files.storage.FileSystemStorage'
  231
+
229 232
 # Absolute path to the directory that holds media.
230 233
 # Example: "/home/media/media.lawrence.com/"
231 234
 MEDIA_ROOT = ''
4  django/contrib/admin/widgets.py
@@ -85,8 +85,8 @@ def __init__(self, attrs={}):
85 85
     def render(self, name, value, attrs=None):
86 86
         output = []
87 87
         if value:
88  
-            output.append('%s <a target="_blank" href="%s%s">%s</a> <br />%s ' % \
89  
-                (_('Currently:'), settings.MEDIA_URL, value, value, _('Change:')))
  88
+            output.append('%s <a target="_blank" href="%s">%s</a> <br />%s ' % \
  89
+                (_('Currently:'), value.url, value, _('Change:')))
90 90
         output.append(super(AdminFileWidget, self).render(name, value, attrs))
91 91
         return mark_safe(u''.join(output))
92 92
 
1  django/core/files/__init__.py
... ...
@@ -0,0 +1 @@
  1
+from django.core.files.base import File
169  django/core/files/base.py
... ...
@@ -0,0 +1,169 @@
  1
+import os
  2
+
  3
+from django.utils.encoding import smart_str, smart_unicode
  4
+
  5
+try:
  6
+    from cStringIO import StringIO
  7
+except ImportError:
  8
+    from StringIO import StringIO
  9
+
  10
+class File(object):
  11
+    DEFAULT_CHUNK_SIZE = 64 * 2**10
  12
+
  13
+    def __init__(self, file):
  14
+        self.file = file
  15
+        self._name = file.name
  16
+        self._mode = file.mode
  17
+        self._closed = False
  18
+
  19
+    def __str__(self):
  20
+        return smart_str(self.name or '')
  21
+
  22
+    def __unicode__(self):
  23
+        return smart_unicode(self.name or u'')
  24
+
  25
+    def __repr__(self):
  26
+        return "<%s: %s>" % (self.__class__.__name__, self or "None")
  27
+
  28
+    def __nonzero__(self):
  29
+        return not not self.name
  30
+
  31
+    def __len__(self):
  32
+        return self.size
  33
+
  34
+    def _get_name(self):
  35
+        return self._name
  36
+    name = property(_get_name)
  37
+
  38
+    def _get_mode(self):
  39
+        return self._mode
  40
+    mode = property(_get_mode)
  41
+
  42
+    def _get_closed(self):
  43
+        return self._closed
  44
+    closed = property(_get_closed)
  45
+
  46
+    def _get_size(self):
  47
+        if not hasattr(self, '_size'):
  48
+            if hasattr(self.file, 'size'):
  49
+                self._size = self.file.size
  50
+            elif os.path.exists(self.file.name):
  51
+                self._size = os.path.getsize(self.file.name)
  52
+            else:
  53
+                raise AttributeError("Unable to determine the file's size.")
  54
+        return self._size
  55
+
  56
+    def _set_size(self, size):
  57
+        self._size = size
  58
+
  59
+    size = property(_get_size, _set_size)
  60
+
  61
+    def chunks(self, chunk_size=None):
  62
+        """
  63
+        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
  64
+        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
  65
+        """
  66
+        if not chunk_size:
  67
+            chunk_size = self.__class__.DEFAULT_CHUNK_SIZE
  68
+
  69
+        if hasattr(self, 'seek'):
  70
+            self.seek(0)
  71
+        # Assume the pointer is at zero...
  72
+        counter = self.size
  73
+
  74
+        while counter > 0:
  75
+            yield self.read(chunk_size)
  76
+            counter -= chunk_size
  77
+
  78
+    def multiple_chunks(self, chunk_size=None):
  79
+        """
  80
+        Returns ``True`` if you can expect multiple chunks.
  81
+
  82
+        NB: If a particular file representation is in memory, subclasses should
  83
+        always return ``False`` -- there's no good reason to read from memory in
  84
+        chunks.
  85
+        """
  86
+        if not chunk_size:
  87
+            chunk_size = self.DEFAULT_CHUNK_SIZE
  88
+        return self.size > chunk_size
  89
+
  90
+    def xreadlines(self):
  91
+        return iter(self)
  92
+
  93
+    def readlines(self):
  94
+        return list(self.xreadlines())
  95
+
  96
+    def __iter__(self):
  97
+        # Iterate over this file-like object by newlines
  98
+        buffer_ = None
  99
+        for chunk in self.chunks():
  100
+            chunk_buffer = StringIO(chunk)
  101
+
  102
+            for line in chunk_buffer:
  103
+                if buffer_:
  104
+                    line = buffer_ + line
  105
+                    buffer_ = None
  106
+
  107
+                # If this is the end of a line, yield
  108
+                # otherwise, wait for the next round
  109
+                if line[-1] in ('\n', '\r'):
  110
+                    yield line
  111
+                else:
  112
+                    buffer_ = line
  113
+
  114
+        if buffer_ is not None:
  115
+            yield buffer_
  116
+
  117
+    def open(self, mode=None):
  118
+        if not self.closed:
  119
+            self.seek(0)
  120
+        elif os.path.exists(self.file.name):
  121
+            self.file = open(self.file.name, mode or self.file.mode)
  122
+        else:
  123
+            raise ValueError("The file cannot be reopened.")
  124
+
  125
+    def seek(self, position):
  126
+        self.file.seek(position)
  127
+
  128
+    def tell(self):
  129
+        return self.file.tell()
  130
+
  131
+    def read(self, num_bytes=None):
  132
+        if num_bytes is None:
  133
+            return self.file.read()
  134
+        return self.file.read(num_bytes)
  135
+
  136
+    def write(self, content):
  137
+        if not self.mode.startswith('w'):
  138
+            raise IOError("File was not opened with write access.")
  139
+        self.file.write(content)
  140
+
  141
+    def flush(self):
  142
+        if not self.mode.startswith('w'):
  143
+            raise IOError("File was not opened with write access.")
  144
+        self.file.flush()
  145
+
  146
+    def close(self):
  147
+        self.file.close()
  148
+        self._closed = True
  149
+
  150
+class ContentFile(File):
  151
+    """
  152
+    A File-like object that takes just raw content, rather than an actual file.
  153
+    """
  154
+    def __init__(self, content):
  155
+        self.file = StringIO(content or '')
  156
+        self.size = len(content or '')
  157
+        self.file.seek(0)
  158
+        self._closed = False
  159
+
  160
+    def __str__(self):
  161
+        return 'Raw content'
  162
+
  163
+    def __nonzero__(self):
  164
+        return True
  165
+
  166
+    def open(self, mode=None):
  167
+        if self._closed:
  168
+            self._closed = False
  169
+        self.seek(0)
42  django/core/files/images.py
... ...
@@ -0,0 +1,42 @@
  1
+"""
  2
+Utility functions for handling images.
  3
+
  4
+Requires PIL, as you might imagine.
  5
+"""
  6
+
  7
+from PIL import ImageFile as PIL
  8
+from django.core.files import File
  9
+
  10
+class ImageFile(File):
  11
+    """
  12
+    A mixin for use alongside django.core.files.base.File, which provides
  13
+    additional features for dealing with images.
  14
+    """
  15
+    def _get_width(self):
  16
+        return self._get_image_dimensions()[0]
  17
+    width = property(_get_width)
  18
+
  19
+    def _get_height(self):
  20
+        return self._get_image_dimensions()[1]
  21
+    height = property(_get_height)
  22
+
  23
+    def _get_image_dimensions(self):
  24
+        if not hasattr(self, '_dimensions_cache'):
  25
+            self._dimensions_cache = get_image_dimensions(self)
  26
+        return self._dimensions_cache
  27
+
  28
+def get_image_dimensions(file_or_path):
  29
+    """Returns the (width, height) of an image, given an open file or a path."""
  30
+    p = PIL.Parser()
  31
+    if hasattr(file_or_path, 'read'):
  32
+        file = file_or_path
  33
+    else:
  34
+        file = open(file_or_path, 'rb')
  35
+    while 1:
  36
+        data = file.read(1024)
  37
+        if not data:
  38
+            break
  39
+        p.feed(data)
  40
+        if p.image:
  41
+            return p.image.size
  42
+    return None
214  django/core/files/storage.py
... ...
@@ -0,0 +1,214 @@
  1
+import os
  2
+import urlparse
  3
+
  4
+from django.conf import settings
  5
+from django.core.exceptions import ImproperlyConfigured, SuspiciousOperation
  6
+from django.utils.encoding import force_unicode, smart_str
  7
+from django.utils.text import force_unicode, get_valid_filename
  8
+from django.utils._os import safe_join
  9
+from django.core.files import locks, File
  10
+
  11
+__all__ = ('Storage', 'FileSystemStorage', 'DefaultStorage', 'default_storage')
  12
+
  13
+class Storage(object):
  14
+    """
  15
+    A base storage class, providing some default behaviors that all other
  16
+    storage systems can inherit or override, as necessary.
  17
+    """
  18
+
  19
+    # The following methods represent a public interface to private methods.
  20
+    # These shouldn't be overridden by subclasses unless absolutely necessary.
  21
+
  22
+    def open(self, name, mode='rb', mixin=None):
  23
+        """
  24
+        Retrieves the specified file from storage, using the optional mixin
  25
+        class to customize what features are available on the File returned.
  26
+        """
  27
+        file = self._open(name, mode)
  28
+        if mixin:
  29
+            # Add the mixin as a parent class of the File returned from storage.
  30
+            file.__class__ = type(mixin.__name__, (mixin, file.__class__), {})
  31
+        return file
  32
+
  33
+    def save(self, name, content):
  34
+        """
  35
+        Saves new content to the file specified by name. The content should be a
  36
+        proper File object, ready to be read from the beginning.
  37
+        """
  38
+        # Check for old-style usage. Warn here first since there are multiple
  39
+        # locations where we need to support both new and old usage.
  40
+        if isinstance(content, basestring):
  41
+            import warnings
  42
+            warnings.warn(
  43
+                message = "Representing files as strings is deprecated." \
  44
+                          "Use django.core.files.base.ContentFile instead.",
  45
+                category = DeprecationWarning,
  46
+                stacklevel = 2
  47
+            )
  48
+            from django.core.files.base import ContentFile
  49
+            content = ContentFile(content)
  50
+
  51
+        # Get the proper name for the file, as it will actually be saved.
  52
+        if name is None:
  53
+            name = content.name
  54
+        name = self.get_available_name(name)
  55
+
  56
+        self._save(name, content)
  57
+
  58
+        # Store filenames with forward slashes, even on Windows
  59
+        return force_unicode(name.replace('\\', '/'))
  60
+
  61
+    # These methods are part of the public API, with default implementations.
  62
+
  63
+    def get_valid_name(self, name):
  64
+        """
  65
+        Returns a filename, based on the provided filename, that's suitable for
  66
+        use in the target storage system.
  67
+        """
  68
+        return get_valid_filename(name)
  69
+
  70
+    def get_available_name(self, name):
  71
+        """
  72
+        Returns a filename that's free on the target storage system, and
  73
+        available for new content to be written to.
  74
+        """
  75
+        # If the filename already exists, keep adding an underscore to the name
  76
+        # of the file until the filename doesn't exist.
  77
+        while self.exists(name):
  78
+            try:
  79
+                dot_index = name.rindex('.')
  80
+            except ValueError: # filename has no dot
  81
+                name += '_'
  82
+            else:
  83
+                name = name[:dot_index] + '_' + name[dot_index:]
  84
+        return name
  85
+
  86
+    def path(self, name):
  87
+        """
  88
+        Returns a local filesystem path where the file can be retrieved using
  89
+        Python's built-in open() function. Storage systems that can't be
  90
+        accessed using open() should *not* implement this method.
  91
+        """
  92
+        raise NotImplementedError("This backend doesn't support absolute paths.")
  93
+
  94
+    # The following methods form the public API for storage systems, but with
  95
+    # no default implementations. Subclasses must implement *all* of these.
  96
+
  97
+    def delete(self, name):
  98
+        """
  99
+        Deletes the specified file from the storage system.
  100
+        """
  101
+        raise NotImplementedError()
  102
+
  103
+    def exists(self, name):
  104
+        """
  105
+        Returns True if a file referened by the given name already exists in the
  106
+        storage system, or False if the name is available for a new file.
  107
+        """
  108
+        raise NotImplementedError()
  109
+
  110
+    def listdir(self, path):
  111
+        """
  112
+        Lists the contents of the specified path, returning a 2-tuple of lists;
  113
+        the first item being directories, the second item being files.
  114
+        """
  115
+        raise NotImplementedError()
  116
+
  117
+    def size(self, name):
  118
+        """
  119
+        Returns the total size, in bytes, of the file specified by name.
  120
+        """
  121
+        raise NotImplementedError()
  122
+
  123
+    def url(self, name):
  124
+        """
  125
+        Returns an absolute URL where the file's contents can be accessed
  126
+        directly by a web browser.
  127
+        """
  128
+        raise NotImplementedError()
  129
+
  130
+class FileSystemStorage(Storage):
  131
+    """
  132
+    Standard filesystem storage
  133
+    """
  134
+
  135
+    def __init__(self, location=settings.MEDIA_ROOT, base_url=settings.MEDIA_URL):
  136
+        self.location = os.path.abspath(location)
  137
+        self.base_url = base_url
  138
+
  139
+    def _open(self, name, mode='rb'):
  140
+        return File(open(self.path(name), mode))
  141
+
  142
+    def _save(self, name, content):
  143
+        full_path = self.path(name)
  144
+
  145
+        directory = os.path.dirname(full_path)
  146
+        if not os.path.exists(directory):
  147
+            os.makedirs(directory)
  148
+        elif not os.path.isdir(directory):
  149
+            raise IOError("%s exists and is not a directory." % directory)
  150
+            
  151
+        if hasattr(content, 'temporary_file_path'):
  152
+            # This file has a file path that we can move.
  153
+            file_move_safe(content.temporary_file_path(), full_path)
  154
+            content.close()
  155
+        else:
  156
+            # This is a normal uploadedfile that we can stream.
  157
+            fp = open(full_path, 'wb')
  158
+            locks.lock(fp, locks.LOCK_EX)
  159
+            for chunk in content.chunks():
  160
+                fp.write(chunk)
  161
+            locks.unlock(fp)
  162
+            fp.close()
  163
+
  164
+    def delete(self, name):
  165
+        name = self.path(name)
  166
+        # If the file exists, delete it from the filesystem.
  167
+        if os.path.exists(name):
  168
+            os.remove(name)
  169
+
  170
+    def exists(self, name):
  171
+        return os.path.exists(self.path(name))
  172
+
  173
+    def listdir(self, path):
  174
+        path = self.path(path)
  175
+        directories, files = [], []
  176
+        for entry in os.listdir(path):
  177
+            if os.path.isdir(os.path.join(path, entry)):
  178
+                directories.append(entry)
  179
+            else:
  180
+                files.append(entry)
  181
+        return directories, files
  182
+
  183
+    def path(self, name):
  184
+        try:
  185
+            path = safe_join(self.location, name)
  186
+        except ValueError:
  187
+            raise SuspiciousOperation("Attempted access to '%s' denied." % name)
  188
+        return os.path.normpath(path)
  189
+
  190
+    def size(self, name):
  191
+        return os.path.getsize(self.path(name))
  192
+
  193
+    def url(self, name):
  194
+        if self.base_url is None:
  195
+            raise ValueError("This file is not accessible via a URL.")
  196
+        return urlparse.urljoin(self.base_url, name).replace('\\', '/')
  197
+
  198
+def get_storage_class(import_path):
  199
+    try:
  200
+        dot = import_path.rindex('.')
  201
+    except ValueError:
  202
+        raise ImproperlyConfigured("%s isn't a storage module." % import_path)
  203
+    module, classname = import_path[:dot], import_path[dot+1:]
  204
+    try:
  205
+        mod = __import__(module, {}, {}, [''])
  206
+    except ImportError, e:
  207
+        raise ImproperlyConfigured('Error importing storage module %s: "%s"' % (module, e))
  208
+    try:
  209
+        return getattr(mod, classname)
  210
+    except AttributeError:
  211
+        raise ImproperlyConfigured('Storage module "%s" does not define a "%s" class.' % (module, classname))
  212
+
  213
+DefaultStorage = get_storage_class(settings.DEFAULT_FILE_STORAGE)
  214
+default_storage = DefaultStorage()
59  django/core/files/uploadedfile.py
@@ -10,6 +10,7 @@
10 10
     from StringIO import StringIO
11 11
 
12 12
 from django.conf import settings
  13
+from django.core.files.base import File
13 14
 
14 15
 from django.core.files import temp as tempfile
15 16
 
@@ -39,7 +40,7 @@ def setter(self, value):
39 40
     else:
40 41
         return property(getter, setter)
41 42
 
42  
-class UploadedFile(object):
  43
+class UploadedFile(File):
43 44
     """
44 45
     A abstract uploaded file (``TemporaryUploadedFile`` and
45 46
     ``InMemoryUploadedFile`` are the built-in concrete subclasses).
@@ -76,23 +77,6 @@ def _set_name(self, name):
76 77
 
77 78
     name = property(_get_name, _set_name)
78 79
 
79  
-    def chunks(self, chunk_size=None):
80  
-        """
81  
-        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
82  
-        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
83  
-        """
84  
-        if not chunk_size:
85  
-            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
86  
-
87  
-        if hasattr(self, 'seek'):
88  
-            self.seek(0)
89  
-        # Assume the pointer is at zero...
90  
-        counter = self.size
91  
-
92  
-        while counter > 0:
93  
-            yield self.read(chunk_size)
94  
-            counter -= chunk_size
95  
-
96 80
     # Deprecated properties
97 81
     filename = deprecated_property(old="filename", new="name")
98 82
     file_name = deprecated_property(old="file_name", new="name")
@@ -108,18 +92,6 @@ def _get_data(self):
108 92
         return self.read()
109 93
     data = property(_get_data)
110 94
 
111  
-    def multiple_chunks(self, chunk_size=None):
112  
-        """
113  
-        Returns ``True`` if you can expect multiple chunks.
114  
-
115  
-        NB: If a particular file representation is in memory, subclasses should
116  
-        always return ``False`` -- there's no good reason to read from memory in
117  
-        chunks.
118  
-        """
119  
-        if not chunk_size:
120  
-            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
121  
-        return self.size > chunk_size
122  
-
123 95
     # Abstract methods; subclasses *must* define read() and probably should
124 96
     # define open/close.
125 97
     def read(self, num_bytes=None):
@@ -131,33 +103,6 @@ def open(self):
131 103
     def close(self):
132 104
         pass
133 105
 
134  
-    def xreadlines(self):
135  
-        return self
136  
-
137  
-    def readlines(self):
138  
-        return list(self.xreadlines())
139  
-
140  
-    def __iter__(self):
141  
-        # Iterate over this file-like object by newlines
142  
-        buffer_ = None
143  
-        for chunk in self.chunks():
144  
-            chunk_buffer = StringIO(chunk)
145  
-
146  
-            for line in chunk_buffer:
147  
-                if buffer_:
148  
-                    line = buffer_ + line
149  
-                    buffer_ = None
150  
-
151  
-                # If this is the end of a line, yield
152  
-                # otherwise, wait for the next round
153  
-                if line[-1] in ('\n', '\r'):
154  
-                    yield line
155  
-                else:
156  
-                    buffer_ = line
157  
-
158  
-        if buffer_ is not None:
159  
-            yield buffer_
160  
-
161 106
     # Backwards-compatible support for uploaded-files-as-dictionaries.
162 107
     def __getitem__(self, key):
163 108
         warnings.warn(
1  django/db/models/__init__.py
@@ -8,6 +8,7 @@
8 8
 from django.db.models.base import Model
9 9
 from django.db.models.fields import *
10 10
 from django.db.models.fields.subclassing import SubfieldBase
  11
+from django.db.models.fields.files import FileField, ImageField
11 12
 from django.db.models.fields.related import ForeignKey, OneToOneField, ManyToManyField, ManyToOneRel, ManyToManyRel, OneToOneRel, TABULAR, STACKED
12 13
 from django.db.models import signals
13 14
 
123  django/db/models/base.py
@@ -3,6 +3,7 @@
3 3
 import sys
4 4
 import os
5 5
 from itertools import izip
  6
+from warnings import warn
6 7
 try:
7 8
     set
8 9
 except NameError:
@@ -12,7 +13,7 @@
12 13
 import django.db.models.manager         # Ditto.
13 14
 from django.core import validators
14 15
 from django.core.exceptions import ObjectDoesNotExist, MultipleObjectsReturned, FieldError
15  
-from django.db.models.fields import AutoField, ImageField
  16
+from django.db.models.fields import AutoField
16 17
 from django.db.models.fields.related import OneToOneRel, ManyToOneRel, OneToOneField
17 18
 from django.db.models.query import delete_objects, Q, CollectedObjects
18 19
 from django.db.models.options import Options
@@ -463,110 +464,42 @@ def _get_next_or_previous_in_order(self, is_next):
463 464
         return getattr(self, cachename)
464 465
 
465 466
     def _get_FIELD_filename(self, field):
466  
-        if getattr(self, field.attname): # Value is not blank.
467  
-            return os.path.normpath(os.path.join(settings.MEDIA_ROOT, getattr(self, field.attname)))
468  
-        return ''
  467
+        warn("instance.get_%s_filename() is deprecated. Use instance.%s.path instead." % \
  468
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
  469
+        try:
  470
+            return getattr(self, field.attname).path
  471
+        except ValueError:
  472
+            return ''
469 473
 
470 474
     def _get_FIELD_url(self, field):
471  
-        if getattr(self, field.attname): # Value is not blank.
472  
-            import urlparse
473  
-            return urlparse.urljoin(settings.MEDIA_URL, getattr(self, field.attname)).replace('\\', '/')
474  
-        return ''
  475
+        warn("instance.get_%s_url() is deprecated. Use instance.%s.url instead." % \
  476
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
  477
+        try:
  478
+            return getattr(self, field.attname).url
  479
+        except ValueError:
  480
+            return ''
475 481
 
476 482
     def _get_FIELD_size(self, field):
477  
-        return os.path.getsize(self._get_FIELD_filename(field))
478  
-
479  
-    def _save_FIELD_file(self, field, filename, raw_field, save=True):
480  
-        # Create the upload directory if it doesn't already exist
481  
-        directory = os.path.join(settings.MEDIA_ROOT, field.get_directory_name())
482  
-        if not os.path.exists(directory):
483  
-            os.makedirs(directory)
484  
-        elif not os.path.isdir(directory):
485  
-            raise IOError('%s exists and is not a directory' % directory)        
486  
-
487  
-        # Check for old-style usage (files-as-dictionaries). Warn here first
488  
-        # since there are multiple locations where we need to support both new
489  
-        # and old usage.
490  
-        if isinstance(raw_field, dict):
491  
-            import warnings
492  
-            warnings.warn(
493  
-                message = "Representing uploaded files as dictionaries is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
494  
-                category = DeprecationWarning,
495  
-                stacklevel = 2
496  
-            )
497  
-            from django.core.files.uploadedfile import SimpleUploadedFile
498  
-            raw_field = SimpleUploadedFile.from_dict(raw_field)
499  
-
500  
-        elif isinstance(raw_field, basestring):
501  
-            import warnings
502  
-            warnings.warn(
503  
-                message = "Representing uploaded files as strings is deprecated. Use django.core.files.uploadedfile.SimpleUploadedFile instead.",
504  
-                category = DeprecationWarning,
505  
-                stacklevel = 2
506  
-            )
507  
-            from django.core.files.uploadedfile import SimpleUploadedFile
508  
-            raw_field = SimpleUploadedFile(filename, raw_field)
509  
-
510  
-        if filename is None:
511  
-            filename = raw_field.file_name
512  
-
513  
-        filename = field.get_filename(filename)
514  
-
515  
-        # If the filename already exists, keep adding an underscore to the name
516  
-        # of the file until the filename doesn't exist.
517  
-        while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
518  
-            try:
519  
-                dot_index = filename.rindex('.')
520  
-            except ValueError: # filename has no dot.
521  
-                filename += '_'
522  
-            else:
523  
-                filename = filename[:dot_index] + '_' + filename[dot_index:]
524  
-
525  
-        # Save the file name on the object and write the file to disk.
526  
-        setattr(self, field.attname, filename)
527  
-        full_filename = self._get_FIELD_filename(field)
528  
-        if hasattr(raw_field, 'temporary_file_path'):
529  
-            # This file has a file path that we can move.
530  
-            file_move_safe(raw_field.temporary_file_path(), full_filename)
531  
-            raw_field.close()
532  
-        else:
533  
-            # This is a normal uploadedfile that we can stream.
534  
-            fp = open(full_filename, 'wb')
535  
-            locks.lock(fp, locks.LOCK_EX)
536  
-            for chunk in raw_field.chunks():
537  
-                fp.write(chunk)
538  
-            locks.unlock(fp)
539  
-            fp.close()
540  
-
541  
-        # Save the width and/or height, if applicable.
542  
-        if isinstance(field, ImageField) and \
543  
-                (field.width_field or field.height_field):
544  
-            from django.utils.images import get_image_dimensions
545  
-            width, height = get_image_dimensions(full_filename)
546  
-            if field.width_field:
547  
-                setattr(self, field.width_field, width)
548  
-            if field.height_field:
549  
-                setattr(self, field.height_field, height)
550  
-
551  
-        # Save the object because it has changed, unless save is False.
552  
-        if save:
553  
-            self.save()
  483
+        warn("instance.get_%s_size() is deprecated. Use instance.%s.size instead." % \
  484
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
  485
+        return getattr(self, field.attname).size
  486
+
  487
+    def _save_FIELD_file(self, field, filename, content, save=True):
  488
+        warn("instance.save_%s_file() is deprecated. Use instance.%s.save() instead." % \
  489
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
  490
+        return getattr(self, field.attname).save(filename, content, save)
554 491
 
555 492
     _save_FIELD_file.alters_data = True
556 493
 
557 494
     def _get_FIELD_width(self, field):
558  
-        return self._get_image_dimensions(field)[0]
  495
+        warn("instance.get_%s_width() is deprecated. Use instance.%s.width instead." % \
  496
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
  497
+        return getattr(self, field.attname).width()
559 498
 
560 499
     def _get_FIELD_height(self, field):
561  
-        return self._get_image_dimensions(field)[1]
562  
-
563  
-    def _get_image_dimensions(self, field):
564  
-        cachename = "__%s_dimensions_cache" % field.name
565  
-        if not hasattr(self, cachename):
566  
-            from django.utils.images import get_image_dimensions
567  
-            filename = self._get_FIELD_filename(field)
568  
-            setattr(self, cachename, get_image_dimensions(filename))
569  
-        return getattr(self, cachename)
  500
+        warn("instance.get_%s_height() is deprecated. Use instance.%s.height instead." % \
  501
+            (field.attname, field.attname), DeprecationWarning, stacklevel=3)
  502
+        return getattr(self, field.attname).height()
570 503
 
571 504
 
572 505
 ############################################
160  django/db/models/fields/__init__.py
@@ -10,6 +10,7 @@
10 10
 from django.db import connection, get_creation_module
11 11
 from django.db.models import signals
12 12
 from django.db.models.query_utils import QueryWrapper
  13
+from django.dispatch import dispatcher
13 14
 from django.conf import settings
14 15
 from django.core import validators
15 16
 from django import oldforms
@@ -757,131 +758,6 @@ def formfield(self, **kwargs):
757 758
         defaults.update(kwargs)
758 759
         return super(EmailField, self).formfield(**defaults)
759 760
 
760  
-class FileField(Field):
761  
-    def __init__(self, verbose_name=None, name=None, upload_to='', **kwargs):
762  
-        self.upload_to = upload_to
763  
-        kwargs['max_length'] = kwargs.get('max_length', 100)
764  
-        Field.__init__(self, verbose_name, name, **kwargs)
765  
-
766  
-    def get_internal_type(self):
767  
-        return "FileField"
768  
-
769  
-    def get_db_prep_value(self, value):
770  
-        "Returns field's value prepared for saving into a database."
771  
-        # Need to convert UploadedFile objects provided via a form to unicode for database insertion
772  
-        if hasattr(value, 'name'):
773  
-            return value.name
774  
-        elif value is None:
775  
-            return None
776  
-        else:
777  
-            return unicode(value)
778  
-
779  
-    def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True):
780  
-        field_list = Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow)
781  
-        if not self.blank:
782  
-            if rel:
783  
-                # This validator makes sure FileFields work in a related context.
784  
-                class RequiredFileField(object):
785  
-                    def __init__(self, other_field_names, other_file_field_name):
786  
-                        self.other_field_names = other_field_names
787  
-                        self.other_file_field_name = other_file_field_name
788  
-                        self.always_test = True
789  
-                    def __call__(self, field_data, all_data):
790  
-                        if not all_data.get(self.other_file_field_name, False):
791  
-                            c = validators.RequiredIfOtherFieldsGiven(self.other_field_names, ugettext_lazy("This field is required."))
792  
-                            c(field_data, all_data)
793  
-                # First, get the core fields, if any.
794  
-                core_field_names = []
795  
-                for f in opts.fields:
796  
-                    if f.core and f != self:
797  
-                        core_field_names.extend(f.get_manipulator_field_names(name_prefix))
798  
-                # Now, if there are any, add the validator to this FormField.
799  
-                if core_field_names:
800  
-                    field_list[0].validator_list.append(RequiredFileField(core_field_names, field_list[1].field_name))
801  
-            else:
802  
-                v = validators.RequiredIfOtherFieldNotGiven(field_list[1].field_name, ugettext_lazy("This field is required."))
803  
-                v.always_test = True
804  
-                field_list[0].validator_list.append(v)
805  
-                field_list[0].is_required = field_list[1].is_required = False
806  
-
807  
-        # If the raw path is passed in, validate it's under the MEDIA_ROOT.
808  
-        def isWithinMediaRoot(field_data, all_data):
809  
-            f = os.path.abspath(os.path.join(settings.MEDIA_ROOT, field_data))
810  
-            if not f.startswith(os.path.abspath(os.path.normpath(settings.MEDIA_ROOT))):
811  
-                raise validators.ValidationError, _("Enter a valid filename.")
812  
-        field_list[1].validator_list.append(isWithinMediaRoot)
813  
-        return field_list
814  
-
815  
-    def contribute_to_class(self, cls, name):
816  
-        super(FileField, self).contribute_to_class(cls, name)
817  
-        setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
818  
-        setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
819  
-        setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
820  
-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
821  
-        signals.post_delete.connect(self.delete_file, sender=cls)
822  
-
823  
-    def delete_file(self, instance, **kwargs):
824  
-        if getattr(instance, self.attname):
825  
-            file_name = getattr(instance, 'get_%s_filename' % self.name)()
826  
-            # If the file exists and no other object of this type references it,
827  
-            # delete it from the filesystem.
828  
-            if os.path.exists(file_name) and \
829  
-                not instance.__class__._default_manager.filter(**{'%s__exact' % self.name: getattr(instance, self.attname)}):
830  
-                os.remove(file_name)
831  
-
832  
-    def get_manipulator_field_objs(self):
833  
-        return [oldforms.FileUploadField, oldforms.HiddenField]
834  
-
835  
-    def get_manipulator_field_names(self, name_prefix):
836  
-        return [name_prefix + self.name + '_file', name_prefix + self.name]
837  
-
838  
-    def save_file(self, new_data, new_object, original_object, change, rel, save=True):
839  
-        upload_field_name = self.get_manipulator_field_names('')[0]
840  
-        if new_data.get(upload_field_name, False):
841  
-            if rel:
842  
-                file = new_data[upload_field_name][0]
843  
-            else:
844  
-                file = new_data[upload_field_name]
845  
-
846  
-            if not file:
847  
-                return
848  
-
849  
-            # Backwards-compatible support for files-as-dictionaries.
850  
-            # We don't need to raise a warning because Model._save_FIELD_file will
851  
-            # do so for us.
852  
-            try:
853  
-                file_name = file.name
854  
-            except AttributeError:
855  
-                file_name = file['filename']
856  
-
857  
-            func = getattr(new_object, 'save_%s_file' % self.name)
858  
-            func(file_name, file, save)
859  
-
860  
-    def get_directory_name(self):
861  
-        return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
862  
-
863  
-    def get_filename(self, filename):
864  
-        from django.utils.text import get_valid_filename
865  
-        f = os.path.join(self.get_directory_name(), get_valid_filename(os.path.basename(filename)))
866  
-        return os.path.normpath(f)
867  
-
868  
-    def save_form_data(self, instance, data):
869  
-        from django.core.files.uploadedfile import UploadedFile
870  
-        if data and isinstance(data, UploadedFile):
871  
-            getattr(instance, "save_%s_file" % self.name)(data.name, data, save=False)
872  
-
873  
-    def formfield(self, **kwargs):
874  
-        defaults = {'form_class': forms.FileField}
875  
-        # If a file has been provided previously, then the form doesn't require
876  
-        # that a new file is provided this time.
877  
-        # The code to mark the form field as not required is used by
878  
-        # form_for_instance, but can probably be removed once form_for_instance
879  
-        # is gone. ModelForm uses a different method to check for an existing file.
880  
-        if 'initial' in kwargs:
881  
-            defaults['required'] = False
882  
-        defaults.update(kwargs)
883  
-        return super(FileField, self).formfield(**defaults)
884  
-
885 761
 class FilePathField(Field):
886 762
     def __init__(self, verbose_name=None, name=None, path='', match=None, recursive=False, **kwargs):
887 763
         self.path, self.match, self.recursive = path, match, recursive
@@ -923,40 +799,6 @@ def formfield(self, **kwargs):
923 799
         defaults.update(kwargs)
924 800
         return super(FloatField, self).formfield(**defaults)
925 801
 
926  
-class ImageField(FileField):
927  
-    def __init__(self, verbose_name=None, name=None, width_field=None, height_field=None, **kwargs):
928  
-        self.width_field, self.height_field = width_field, height_field
929  
-        FileField.__init__(self, verbose_name, name, **kwargs)
930  
-
931  
-    def get_manipulator_field_objs(self):
932  
-        return [oldforms.ImageUploadField, oldforms.HiddenField]
933  
-
934  
-    def contribute_to_class(self, cls, name):
935  
-        super(ImageField, self).contribute_to_class(cls, name)
936  
-        # Add get_BLAH_width and get_BLAH_height methods, but only if the
937  
-        # image field doesn't have width and height cache fields.
938  
-        if not self.width_field:
939  
-            setattr(cls, 'get_%s_width' % self.name, curry(cls._get_FIELD_width, field=self))
940  
-        if not self.height_field:
941  
-            setattr(cls, 'get_%s_height' % self.name, curry(cls._get_FIELD_height, field=self))
942  
-
943  
-    def save_file(self, new_data, new_object, original_object, change, rel, save=True):
944  
-        FileField.save_file(self, new_data, new_object, original_object, change, rel, save)
945  
-        # If the image has height and/or width field(s) and they haven't
946  
-        # changed, set the width and/or height field(s) back to their original
947  
-        # values.
948  
-        if change and (self.width_field or self.height_field) and save:
949  
-            if self.width_field:
950  
-                setattr(new_object, self.width_field, getattr(original_object, self.width_field))
951  
-            if self.height_field:
952  
-                setattr(new_object, self.height_field, getattr(original_object, self.height_field))
953  
-            new_object.save()
954  
-
955  
-    def formfield(self, **kwargs):
956  
-        defaults = {'form_class': forms.ImageField}
957  
-        defaults.update(kwargs)
958  
-        return super(ImageField, self).formfield(**defaults)
959  
-
960 802
 class IntegerField(Field):
961 803
     empty_strings_allowed = False
962 804
     def get_db_prep_value(self, value):
315  django/db/models/fields/files.py
... ...
@@ -0,0 +1,315 @@
  1
+import datetime
  2
+import os
  3
+
  4
+from django.conf import settings
  5
+from django.db.models.fields import Field
  6
+from django.core.files.base import File, ContentFile
  7
+from django.core.files.storage import default_storage
  8
+from django.core.files.images import ImageFile, get_image_dimensions
  9
+from django.core.files.uploadedfile import UploadedFile
  10
+from django.utils.functional import curry
  11
+from django.db.models import signals
  12
+from django.utils.encoding import force_unicode, smart_str
  13
+from django.utils.translation import ugettext_lazy, ugettext as _
  14
+from django import oldforms
  15
+from django import forms
  16
+from django.core import validators
  17
+from django.db.models.loading import cache
  18
+
  19
+class FieldFile(File):
  20
+    def __init__(self, instance, field, name):
  21
+        self.instance = instance
  22
+        self.field = field
  23
+        self.storage = field.storage
  24
+        self._name = name or u''
  25
+        self._closed = False
  26
+
  27
+    def __eq__(self, other):
  28
+        # Older code may be expecting FileField values to be simple strings.
  29
+        # By overriding the == operator, it can remain backwards compatibility.
  30
+        if hasattr(other, 'name'):
  31
+            return self.name == other.name
  32
+        return self.name == other
  33
+
  34
+    # The standard File contains most of the necessary properties, but
  35
+    # FieldFiles can be instantiated without a name, so that needs to
  36
+    # be checked for here.
  37
+
  38
+    def _require_file(self):
  39
+        if not self:
  40
+            raise ValueError("The '%s' attribute has no file associated with it." % self.field.name)
  41
+
  42
+    def _get_file(self):
  43
+        self._require_file()
  44
+        if not hasattr(self, '_file'):
  45
+            self._file = self.storage.open(self.name, 'rb')
  46
+        return self._file
  47
+    file = property(_get_file)
  48
+
  49
+    def _get_path(self):
  50
+        self._require_file()
  51
+        return self.storage.path(self.name)
  52
+    path = property(_get_path)
  53
+
  54
+    def _get_url(self):
  55
+        self._require_file()
  56
+        return self.storage.url(self.name)
  57
+    url = property(_get_url)
  58
+
  59
+    def open(self, mode='rb'):
  60
+        self._require_file()
  61
+        return super(FieldFile, self).open(mode)
  62
+    # open() doesn't alter the file's contents, but it does reset the pointer
  63
+    open.alters_data = True
  64
+
  65
+    # In addition to the standard File API, FieldFiles have extra methods
  66
+    # to further manipulate the underlying file, as well as update the
  67
+    # associated model instance.
  68
+
  69
+    def save(self, name, content, save=True):
  70
+        name = self.field.generate_filename(self.instance, name)
  71
+        self._name = self.storage.save(name, content)
  72
+        setattr(self.instance, self.field.name, self.name)
  73
+
  74
+        # Update the filesize cache
  75
+        self._size = len(content)
  76
+
  77
+        # Save the object because it has changed, unless save is False
  78
+        if save:
  79
+            self.instance.save()
  80
+    save.alters_data = True
  81
+
  82
+    def delete(self, save=True):
  83
+        self.close()
  84
+        self.storage.delete(self.name)
  85
+
  86
+        self._name = None
  87
+        setattr(self.instance, self.field.name, self.name)
  88
+
  89
+        # Delete the filesize cache
  90
+        if hasattr(self, '_size'):
  91
+            del self._size
  92
+
  93
+        if save:
  94
+            self.instance.save()
  95
+    delete.alters_data = True
  96
+
  97
+    def __getstate__(self):
  98
+        # FieldFile needs access to its associated model field and an instance
  99
+        # it's attached to in order to work properly, but the only necessary
  100
+        # data to be pickled is the file's name itself. Everything else will
  101
+        # be restored later, by FileDescriptor below.
  102
+        return {'_name': self.name, '_closed': False}
  103
+
  104
+class FileDescriptor(object):
  105
+    def __init__(self, field):
  106
+        self.field = field
  107
+
  108
+    def __get__(self, instance=None, owner=None):
  109
+        if instance is None:
  110
+            raise AttributeError, "%s can only be accessed from %s instances." % (self.field.name(self.owner.__name__))
  111
+        file = instance.__dict__[self.field.name]
  112
+        if not isinstance(file, FieldFile):
  113
+            # Create a new instance of FieldFile, based on a given file name
  114
+            instance.__dict__[self.field.name] = self.field.attr_class(instance, self.field, file)
  115
+        elif not hasattr(file, 'field'):
  116
+            # The FieldFile was pickled, so some attributes need to be reset.
  117
+            file.instance = instance
  118
+            file.field = self.field
  119
+            file.storage = self.field.storage
  120
+        return instance.__dict__[self.field.name]
  121
+
  122
+    def __set__(self, instance, value):
  123
+        instance.__dict__[self.field.name] = value
  124
+
  125
+class FileField(Field):
  126
+    attr_class = FieldFile
  127
+
  128
+    def __init__(self, verbose_name=None, name=None, upload_to='', storage=None, **kwargs):
  129
+        for arg in ('core', 'primary_key', 'unique'):
  130
+            if arg in kwargs:
  131
+                raise TypeError("'%s' is not a valid argument for %s." % (arg, self.__class__))
  132
+
  133
+        self.storage = storage or default_storage
  134
+        self.upload_to = upload_to
  135
+        if callable(upload_to):
  136
+            self.generate_filename = upload_to
  137
+
  138
+        kwargs['max_length'] = kwargs.get('max_length', 100)
  139
+        super(FileField, self).__init__(verbose_name, name, **kwargs)
  140
+
  141
+    def get_internal_type(self):
  142
+        return "FileField"
  143
+
  144
+    def get_db_prep_lookup(self, lookup_type, value):
  145
+        if hasattr(value, 'name'):
  146
+            value = value.name
  147
+        return super(FileField, self).get_db_prep_lookup(lookup_type, value)
  148
+
  149
+    def get_db_prep_value(self, value):
  150
+        "Returns field's value prepared for saving into a database."
  151
+        # Need to convert File objects provided via a form to unicode for database insertion
  152
+        if value is None:
  153
+            return None
  154
+        return unicode(value)
  155
+
  156
+    def get_manipulator_fields(self, opts, manipulator, change, name_prefix='', rel=False, follow=True):
  157
+        field_list = Field.get_manipulator_fields(self, opts, manipulator, change, name_prefix, rel, follow)
  158
+        if not self.blank:
  159
+            if rel:
  160
+                # This validator makes sure FileFields work in a related context.
  161
+                class RequiredFileField(object):
  162
+                    def __init__(self, other_field_names, other_file_field_name):
  163
+                        self.other_field_names = other_field_names
  164
+                        self.other_file_field_name = other_file_field_name
  165
+                        self.always_test = True
  166
+                    def __call__(self, field_data, all_data):
  167
+                        if not all_data.get(self.other_file_field_name, False):
  168
+                            c = validators.RequiredIfOtherFieldsGiven(self.other_field_names, ugettext_lazy("This field is required."))
  169
+                            c(field_data, all_data)
  170
+                # First, get the core fields, if any.
  171
+                core_field_names = []
  172
+                for f in opts.fields:
  173
+                    if f.core and f != self:
  174
+                        core_field_names.extend(f.get_manipulator_field_names(name_prefix))
  175
+                # Now, if there are any, add the validator to this FormField.
  176
+                if core_field_names:
  177
+                    field_list[0].validator_list.append(RequiredFileField(core_field_names, field_list[1].field_name))
  178
+            else:
  179
+                v = validators.RequiredIfOtherFieldNotGiven(field_list[1].field_name, ugettext_lazy("This field is required."))
  180
+                v.always_test = True
  181
+                field_list[0].validator_list.append(v)
  182
+                field_list[0].is_required = field_list[1].is_required = False