Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse code

Fixed #2070: refactored Django's file upload capabilities.

A description of the new features can be found in the new [http://www.djangoproject.com/documentation/upload_handing/ upload handling documentation]; the executive summary is that Django will now happily handle uploads of large files without issues.

This changes the representation of uploaded files from dictionaries to bona fide objects; see BackwardsIncompatibleChanges for details.


git-svn-id: http://code.djangoproject.com/svn/django/trunk@7814 bcc190cf-cafb-0310-a4f2-bffc1f526a37
  • Loading branch information...
commit d725cc9734272f867d41f7236235c28b3931a1b2 1 parent ef76102
Jacob Kaplan-Moss authored July 01, 2008

Showing 38 changed files with 2,291 additions and 154 deletions. Show diff stats Hide diff stats

  1. 4  AUTHORS
  2. 15  django/conf/global_settings.py
  3. 0  django/core/files/__init__.py
  4. 66  django/core/files/locks.py
  5. 59  django/core/files/move.py
  6. 190  django/core/files/uploadedfile.py
  7. 235  django/core/files/uploadhandler.py
  8. 3  django/core/handlers/modpython.py
  9. 5  django/core/handlers/wsgi.py
  10. 60  django/db/models/base.py
  11. 18  django/db/models/fields/__init__.py
  12. 66  django/http/__init__.py
  13. 658  django/http/multipartparser.py
  14. 62  django/newforms/fields.py
  15. 19  django/oldforms/__init__.py
  16. 26  django/test/client.py
  17. 50  django/utils/datastructures.py
  18. 24  django/utils/text.py
  19. 4  docs/newforms.txt
  20. 27  docs/request_response.txt
  21. 39  docs/settings.txt
  22. 346  docs/upload_handling.txt
  23. 96  tests/modeltests/model_forms/models.py
  24. 11  tests/regressiontests/bug639/tests.py
  25. 25  tests/regressiontests/datastructures/tests.py
  26. 0  tests/regressiontests/file_uploads/__init__.py
  27. 2  tests/regressiontests/file_uploads/models.py
  28. 158  tests/regressiontests/file_uploads/tests.py
  29. 26  tests/regressiontests/file_uploads/uploadhandler.py
  30. 10  tests/regressiontests/file_uploads/urls.py
  31. 70  tests/regressiontests/file_uploads/views.py
  32. 7  tests/regressiontests/forms/error_messages.py
  33. 19  tests/regressiontests/forms/fields.py
  34. 5  tests/regressiontests/forms/forms.py
  35. 12  tests/regressiontests/test_client_regress/models.py
  36. 1  tests/regressiontests/test_client_regress/urls.py
  37. 24  tests/regressiontests/test_client_regress/views.py
  38. 3  tests/urls.py
4  AUTHORS
@@ -59,7 +59,7 @@ answer newbie questions, and generally made Django that much better:
59 59
     Arthur <avandorp@gmail.com>
60 60
     av0000@mail.ru
61 61
     David Avsajanishvili <avsd05@gmail.com>
62  
-    axiak@mit.edu
  62
+    Mike Axiak <axiak@mit.edu>
63 63
     Niran Babalola <niran@niran.org>
64 64
     Morten Bagai <m@bagai.com>
65 65
     Mikaël Barbero <mikael.barbero nospam at nospam free.fr>
@@ -141,7 +141,9 @@ answer newbie questions, and generally made Django that much better:
141 141
     Marc Fargas <telenieko@telenieko.com>
142 142
     Szilveszter Farkas <szilveszter.farkas@gmail.com>
143 143
     favo@exoweb.net
  144
+    fdr <drfarina@gmail.com>
144 145
     Dmitri Fedortchenko <zeraien@gmail.com>
  146
+    Jonathan Feignberg <jdf@pobox.com>
145 147
     Liang Feng <hutuworm@gmail.com>
146 148
     Bill Fenner <fenner@gmail.com>
147 149
     Stefane Fermgier <sf@fermigier.com>
15  django/conf/global_settings.py
@@ -231,6 +231,21 @@
231 231
 # Example: "http://media.lawrence.com"
232 232
 MEDIA_URL = ''
233 233
 
  234
+# List of upload handler classes to be applied in order.
  235
+FILE_UPLOAD_HANDLERS = (
  236
+    'django.core.files.uploadhandler.MemoryFileUploadHandler',
  237
+    'django.core.files.uploadhandler.TemporaryFileUploadHandler',
  238
+)
  239
+
  240
+# Maximum size, in bytes, of a request before it will be streamed to the
  241
+# file system instead of into memory.
  242
+FILE_UPLOAD_MAX_MEMORY_SIZE = 2621440 # i.e. 2.5 MB
  243
+
  244
+# Directory in which upload streamed files will be temporarily saved. A value of
  245
+# `None` will make Django use the operating system's default temporary directory
  246
+# (i.e. "/tmp" on *nix systems).
  247
+FILE_UPLOAD_TEMP_DIR = None
  248
+
234 249
 # Default formatting for date objects. See all available format strings here:
235 250
 # http://www.djangoproject.com/documentation/templates/#now
236 251
 DATE_FORMAT = 'N j, Y'
0  django/core/files/__init__.py
No changes.
66  django/core/files/locks.py
... ...
@@ -0,0 +1,66 @@
  1
+"""
  2
+Portable file locking utilities.
  3
+
  4
+Based partially on example by Jonathan Feignberg <jdf@pobox.com> in the Python
  5
+Cookbook, licensed under the Python Software License.
  6
+
  7
+    http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/65203
  8
+
  9
+Example Usage::
  10
+
  11
+    >>> from django.core.files import locks
  12
+    >>> f = open('./file', 'wb')
  13
+    >>> locks.lock(f, locks.LOCK_EX)
  14
+    >>> f.write('Django')
  15
+    >>> f.close()
  16
+"""
  17
+
  18
+__all__ = ('LOCK_EX','LOCK_SH','LOCK_NB','lock','unlock')
  19
+
  20
+system_type = None
  21
+
  22
+try:
  23
+    import win32con
  24
+    import win32file
  25
+    import pywintypes
  26
+    LOCK_EX = win32con.LOCKFILE_EXCLUSIVE_LOCK
  27
+    LOCK_SH = 0
  28
+    LOCK_NB = win32con.LOCKFILE_FAIL_IMMEDIATELY
  29
+    __overlapped = pywintypes.OVERLAPPED()
  30
+    system_type = 'nt'
  31
+except (ImportError, AttributeError):
  32
+    pass
  33
+
  34
+try:
  35
+    import fcntl
  36
+    LOCK_EX = fcntl.LOCK_EX
  37
+    LOCK_SH = fcntl.LOCK_SH
  38
+    LOCK_NB = fcntl.LOCK_NB
  39
+    system_type = 'posix'
  40
+except (ImportError, AttributeError):
  41
+    pass
  42
+
  43
+if system_type == 'nt':
  44
+    def lock(file, flags):
  45
+        hfile = win32file._get_osfhandle(file.fileno())
  46
+        win32file.LockFileEx(hfile, flags, 0, -0x10000, __overlapped)
  47
+
  48
+    def unlock(file):
  49
+        hfile = win32file._get_osfhandle(file.fileno())
  50
+        win32file.UnlockFileEx(hfile, 0, -0x10000, __overlapped)
  51
+elif system_type == 'posix':
  52
+    def lock(file, flags):
  53
+        fcntl.flock(file.fileno(), flags)
  54
+
  55
+    def unlock(file):
  56
+        fcntl.flock(file.fileno(), fcntl.LOCK_UN)
  57
+else:
  58
+    # File locking is not supported.
  59
+    LOCK_EX = LOCK_SH = LOCK_NB = None
  60
+
  61
+    # Dummy functions that don't do anything.
  62
+    def lock(file, flags):
  63
+        pass
  64
+
  65
+    def unlock(file):
  66
+        pass
59  django/core/files/move.py
... ...
@@ -0,0 +1,59 @@
  1
+"""
  2
+Move a file in the safest way possible::
  3
+
  4
+    >>> from django.core.files.move import file_move_save
  5
+    >>> file_move_save("/tmp/old_file", "/tmp/new_file")
  6
+"""
  7
+
  8
+import os
  9
+from django.core.files import locks
  10
+
  11
+__all__ = ['file_move_safe']
  12
+
  13
+try:
  14
+    import shutil
  15
+    file_move = shutil.move
  16
+except ImportError:
  17
+    file_move = os.rename
  18
+
  19
+def file_move_safe(old_file_name, new_file_name, chunk_size = 1024*64, allow_overwrite=False):
  20
+    """
  21
+    Moves a file from one location to another in the safest way possible.
  22
+
  23
+    First, try using ``shutils.move``, which is OS-dependent but doesn't break
  24
+    if moving across filesystems. Then, try ``os.rename``, which will break
  25
+    across filesystems. Finally, streams manually from one file to another in
  26
+    pure Python.
  27
+
  28
+    If the destination file exists and ``allow_overwrite`` is ``False``, this
  29
+    function will throw an ``IOError``.
  30
+    """
  31
+
  32
+    # There's no reason to move if we don't have to.
  33
+    if old_file_name == new_file_name:
  34
+        return
  35
+
  36
+    if not allow_overwrite and os.path.exists(new_file_name):
  37
+        raise IOError("Cannot overwrite existing file '%s'." % new_file_name)
  38
+
  39
+    try:
  40
+        file_move(old_file_name, new_file_name)
  41
+        return
  42
+    except OSError:
  43
+        # This will happen with os.rename if moving to another filesystem
  44
+        pass
  45
+
  46
+    # If the built-in didn't work, do it the hard way.
  47
+    new_file = open(new_file_name, 'wb')
  48
+    locks.lock(new_file, locks.LOCK_EX)
  49
+    old_file = open(old_file_name, 'rb')
  50
+    current_chunk = None
  51
+
  52
+    while current_chunk != '':
  53
+        current_chunk = old_file.read(chunk_size)
  54
+        new_file.write(current_chunk)
  55
+
  56
+    new_file.close()
  57
+    old_file.close()
  58
+
  59
+    os.remove(old_file_name)
190  django/core/files/uploadedfile.py
... ...
@@ -0,0 +1,190 @@
  1
+"""
  2
+Classes representing uploaded files.
  3
+"""
  4
+
  5
+import os
  6
+try:
  7
+    from cStringIO import StringIO
  8
+except ImportError:
  9
+    from StringIO import StringIO
  10
+
  11
+__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile')
  12
+
  13
+class UploadedFile(object):
  14
+    """
  15
+    A abstract uploadded file (``TemporaryUploadedFile`` and
  16
+    ``InMemoryUploadedFile`` are the built-in concrete subclasses).
  17
+
  18
+    An ``UploadedFile`` object behaves somewhat like a file object and
  19
+    represents some file data that the user submitted with a form.
  20
+    """
  21
+    DEFAULT_CHUNK_SIZE = 64 * 2**10
  22
+
  23
+    def __init__(self, file_name=None, content_type=None, file_size=None, charset=None):
  24
+        self.file_name = file_name
  25
+        self.file_size = file_size
  26
+        self.content_type = content_type
  27
+        self.charset = charset
  28
+
  29
+    def __repr__(self):
  30
+        return "<%s: %s (%s)>" % (self.__class__.__name__, self.file_name, self.content_type)
  31
+
  32
+    def _set_file_name(self, name):
  33
+        # Sanitize the file name so that it can't be dangerous.
  34
+        if name is not None:
  35
+            # Just use the basename of the file -- anything else is dangerous.
  36
+            name = os.path.basename(name)
  37
+            
  38
+            # File names longer than 255 characters can cause problems on older OSes.
  39
+            if len(name) > 255:
  40
+                name, ext = os.path.splitext(name)
  41
+                name = name[:255 - len(ext)] + ext
  42
+                
  43
+        self._file_name = name
  44
+        
  45
+    def _get_file_name(self):
  46
+        return self._file_name
  47
+        
  48
+    file_name = property(_get_file_name, _set_file_name)
  49
+
  50
+    def chunk(self, chunk_size=None):
  51
+        """
  52
+        Read the file and yield chucks of ``chunk_size`` bytes (defaults to
  53
+        ``UploadedFile.DEFAULT_CHUNK_SIZE``).
  54
+        """
  55
+        if not chunk_size:
  56
+            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
  57
+
  58
+        if hasattr(self, 'seek'):
  59
+            self.seek(0)
  60
+        # Assume the pointer is at zero...
  61
+        counter = self.file_size
  62
+
  63
+        while counter > 0:
  64
+            yield self.read(chunk_size)
  65
+            counter -= chunk_size
  66
+
  67
+    def multiple_chunks(self, chunk_size=None):
  68
+        """
  69
+        Returns ``True`` if you can expect multiple chunks.
  70
+
  71
+        NB: If a particular file representation is in memory, subclasses should
  72
+        always return ``False`` -- there's no good reason to read from memory in
  73
+        chunks.
  74
+        """
  75
+        if not chunk_size:
  76
+            chunk_size = UploadedFile.DEFAULT_CHUNK_SIZE
  77
+        return self.file_size < chunk_size
  78
+
  79
+    # Abstract methods; subclasses *must* default read() and probably should
  80
+    # define open/close.
  81
+    def read(self, num_bytes=None):
  82
+        raise NotImplementedError()
  83
+
  84
+    def open(self):
  85
+        pass
  86
+
  87
+    def close(self):
  88
+        pass
  89
+
  90
+    # Backwards-compatible support for uploaded-files-as-dictionaries.
  91
+    def __getitem__(self, key):
  92
+        import warnings
  93
+        warnings.warn(
  94
+            message = "The dictionary access of uploaded file objects is deprecated. Use the new object interface instead.",
  95
+            category = DeprecationWarning,
  96
+            stacklevel = 2
  97
+        )
  98
+        backwards_translate = {
  99
+            'filename': 'file_name',
  100
+            'content-type': 'content_type',
  101
+            }
  102
+
  103
+        if key == 'content':
  104
+            return self.read()
  105
+        elif key == 'filename':
  106
+            return self.file_name
  107
+        elif key == 'content-type':
  108
+            return self.content_type
  109
+        else:
  110
+            return getattr(self, key)
  111
+
  112
+class TemporaryUploadedFile(UploadedFile):
  113
+    """
  114
+    A file uploaded to a temporary location (i.e. stream-to-disk).
  115
+    """
  116
+
  117
+    def __init__(self, file, file_name, content_type, file_size, charset):
  118
+        super(TemporaryUploadedFile, self).__init__(file_name, content_type, file_size, charset)
  119
+        self.file = file
  120
+        self.path = file.name
  121
+        self.file.seek(0)
  122
+
  123
+    def temporary_file_path(self):
  124
+        """
  125
+        Returns the full path of this file.
  126
+        """
  127
+        return self.path
  128
+
  129
+    def read(self, *args, **kwargs):
  130
+        return self.file.read(*args, **kwargs)
  131
+
  132
+    def open(self):
  133
+        self.seek(0)
  134
+
  135
+    def seek(self, *args, **kwargs):
  136
+        self.file.seek(*args, **kwargs)
  137
+
  138
+class InMemoryUploadedFile(UploadedFile):
  139
+    """
  140
+    A file uploaded into memory (i.e. stream-to-memory).
  141
+    """
  142
+    def __init__(self, file, field_name, file_name, content_type, charset, file_size):
  143
+        super(InMemoryUploadedFile, self).__init__(file_name, content_type, charset, file_size)
  144
+        self.file = file
  145
+        self.field_name = field_name
  146
+        self.file.seek(0)
  147
+
  148
+    def seek(self, *args, **kwargs):
  149
+        self.file.seek(*args, **kwargs)
  150
+
  151
+    def open(self):
  152
+        self.seek(0)
  153
+
  154
+    def read(self, *args, **kwargs):
  155
+        return self.file.read(*args, **kwargs)
  156
+
  157
+    def chunk(self, chunk_size=None):
  158
+        self.file.seek(0)
  159
+        yield self.read()
  160
+
  161
+    def multiple_chunks(self, chunk_size=None):
  162
+        # Since it's in memory, we'll never have multiple chunks.
  163
+        return False
  164
+
  165
+class SimpleUploadedFile(InMemoryUploadedFile):
  166
+    """
  167
+    A simple representation of a file, which just has content, size, and a name.
  168
+    """
  169
+    def __init__(self, name, content, content_type='text/plain'):
  170
+        self.file = StringIO(content or '')
  171
+        self.file_name = name
  172
+        self.field_name = None
  173
+        self.file_size = len(content or '')
  174
+        self.content_type = content_type
  175
+        self.charset = None
  176
+        self.file.seek(0)
  177
+
  178
+    def from_dict(cls, file_dict):
  179
+        """
  180
+        Creates a SimpleUploadedFile object from
  181
+        a dictionary object with the following keys:
  182
+           - filename
  183
+           - content-type
  184
+           - content
  185
+        """
  186
+        return cls(file_dict['filename'],
  187
+                   file_dict['content'],
  188
+                   file_dict.get('content-type', 'text/plain'))
  189
+
  190
+    from_dict = classmethod(from_dict)
235  django/core/files/uploadhandler.py
... ...
@@ -0,0 +1,235 @@
  1
+"""
  2
+Base file upload handler classes, and the built-in concrete subclasses
  3
+"""
  4
+import os
  5
+import tempfile
  6
+try:
  7
+    from cStringIO import StringIO
  8
+except ImportError:
  9
+    from StringIO import StringIO
  10
+
  11
+from django.conf import settings
  12
+from django.core.exceptions import ImproperlyConfigured
  13
+from django.core.files.uploadedfile import TemporaryUploadedFile, InMemoryUploadedFile
  14
+
  15
+__all__ = ['UploadFileException','StopUpload', 'SkipFile', 'FileUploadHandler',
  16
+           'TemporaryFileUploadHandler', 'MemoryFileUploadHandler',
  17
+           'load_handler']
  18
+
  19
+class UploadFileException(Exception):
  20
+    """
  21
+    Any error having to do with uploading files.
  22
+    """
  23
+    pass
  24
+
  25
+class StopUpload(UploadFileException):
  26
+    """
  27
+    This exception is raised when an upload must abort.
  28
+    """
  29
+    def __init__(self, connection_reset=False):
  30
+        """
  31
+        If ``connection_reset`` is ``True``, Django knows will halt the upload
  32
+        without consuming the rest of the upload. This will cause the browser to
  33
+        show a "connection reset" error.
  34
+        """
  35
+        self.connection_reset = connection_reset
  36
+
  37
+    def __unicode__(self):
  38
+        if self.connection_reset:
  39
+            return u'StopUpload: Halt current upload.'
  40
+        else:
  41
+            return u'StopUpload: Consume request data, then halt.'
  42
+
  43
+class SkipFile(UploadFileException):
  44
+    """
  45
+    This exception is raised by an upload handler that wants to skip a given file.
  46
+    """
  47
+    pass
  48
+    
  49
+class StopFutureHandlers(UploadFileException):
  50
+    """
  51
+    Upload handers that have handled a file and do not want future handlers to
  52
+    run should raise this exception instead of returning None.
  53
+    """
  54
+    pass
  55
+
  56
+class FileUploadHandler(object):
  57
+    """
  58
+    Base class for streaming upload handlers.
  59
+    """
  60
+    chunk_size = 64 * 2 ** 10 #: The default chunk size is 64 KB.
  61
+
  62
+    def __init__(self, request=None):
  63
+        self.file_name = None
  64
+        self.content_type = None
  65
+        self.content_length = None
  66
+        self.charset = None
  67
+        self.request = request
  68
+
  69
+    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
  70
+        """
  71
+        Handle the raw input from the client.
  72
+
  73
+        Parameters:
  74
+
  75
+            :input_data:
  76
+                An object that supports reading via .read().
  77
+            :META:
  78
+                ``request.META``.
  79
+            :content_length:
  80
+                The (integer) value of the Content-Length header from the
  81
+                client.
  82
+            :boundary: The boundary from the Content-Type header. Be sure to
  83
+                prepend two '--'.
  84
+        """
  85
+        pass
  86
+
  87
+    def new_file(self, field_name, file_name, content_type, content_length, charset=None):
  88
+        """
  89
+        Signal that a new file has been started.
  90
+
  91
+        Warning: As with any data from the client, you should not trust
  92
+        content_length (and sometimes won't even get it).
  93
+        """
  94
+        self.field_name = field_name
  95
+        self.file_name = file_name
  96
+        self.content_type = content_type
  97
+        self.content_length = content_length
  98
+        self.charset = charset
  99
+
  100
+    def receive_data_chunk(self, raw_data, start):
  101
+        """
  102
+        Receive data from the streamed upload parser. ``start`` is the position
  103
+        in the file of the chunk.
  104
+        """
  105
+        raise NotImplementedError()
  106
+
  107
+    def file_complete(self, file_size):
  108
+        """
  109
+        Signal that a file has completed. File size corresponds to the actual
  110
+        size accumulated by all the chunks.
  111
+
  112
+        Subclasses must should return a valid ``UploadedFile`` object.
  113
+        """
  114
+        raise NotImplementedError()
  115
+
  116
+    def upload_complete(self):
  117
+        """
  118
+        Signal that the upload is complete. Subclasses should perform cleanup
  119
+        that is necessary for this handler.
  120
+        """
  121
+        pass
  122
+
  123
+class TemporaryFileUploadHandler(FileUploadHandler):
  124
+    """
  125
+    Upload handler that streams data into a temporary file.
  126
+    """
  127
+    def __init__(self, *args, **kwargs):
  128
+        super(TemporaryFileUploadHandler, self).__init__(*args, **kwargs)
  129
+
  130
+    def new_file(self, file_name, *args, **kwargs):
  131
+        """
  132
+        Create the file object to append to as data is coming in.
  133
+        """
  134
+        super(TemporaryFileUploadHandler, self).new_file(file_name, *args, **kwargs)
  135
+        self.file = TemporaryFile(settings.FILE_UPLOAD_TEMP_DIR)
  136
+        self.write = self.file.write
  137
+
  138
+    def receive_data_chunk(self, raw_data, start):
  139
+        self.write(raw_data)
  140
+
  141
+    def file_complete(self, file_size):
  142
+        self.file.seek(0)
  143
+        return TemporaryUploadedFile(self.file, self.file_name,
  144
+                                     self.content_type, file_size,
  145
+                                     self.charset)
  146
+
  147
+class MemoryFileUploadHandler(FileUploadHandler):
  148
+    """
  149
+    File upload handler to stream uploads into memory (used for small files).
  150
+    """
  151
+
  152
+    def handle_raw_input(self, input_data, META, content_length, boundary, encoding=None):
  153
+        """
  154
+        Use the content_length to signal whether or not this handler should be in use.
  155
+        """
  156
+        # Check the content-length header to see if we should
  157
+        # If the the post is too large, we cannot use the Memory handler.
  158
+        if content_length > settings.FILE_UPLOAD_MAX_MEMORY_SIZE:
  159
+            self.activated = False
  160
+        else:
  161
+            self.activated = True
  162
+
  163
+    def new_file(self, *args, **kwargs):
  164
+        super(MemoryFileUploadHandler, self).new_file(*args, **kwargs)
  165
+        if self.activated:
  166
+            self.file = StringIO()
  167
+            raise StopFutureHandlers()
  168
+
  169
+    def receive_data_chunk(self, raw_data, start):
  170
+        """
  171
+        Add the data to the StringIO file.
  172
+        """
  173
+        if self.activated:
  174
+            self.file.write(raw_data)
  175
+        else:
  176
+            return raw_data
  177
+
  178
+    def file_complete(self, file_size):
  179
+        """
  180
+        Return a file object if we're activated.
  181
+        """
  182
+        if not self.activated:
  183
+            return
  184
+
  185
+        return InMemoryUploadedFile(self.file, self.field_name, self.file_name,
  186
+                                    self.content_type, self.charset, file_size)
  187
+
  188
+class TemporaryFile(object):
  189
+    """
  190
+    A temporary file that tries to delete itself when garbage collected.
  191
+    """
  192
+    def __init__(self, dir):
  193
+        if not dir:
  194
+            dir = tempfile.gettempdir()
  195
+        try:
  196
+            (fd, name) = tempfile.mkstemp(suffix='.upload', dir=dir)
  197
+            self.file = os.fdopen(fd, 'w+b')
  198
+        except (OSError, IOError):
  199
+            raise OSError("Could not create temporary file for uploading, have you set settings.FILE_UPLOAD_TEMP_DIR correctly?")
  200
+        self.name = name
  201
+
  202
+    def __getattr__(self, name):
  203
+        a = getattr(self.__dict__['file'], name)
  204
+        if type(a) != type(0):
  205
+            setattr(self, name, a)
  206
+        return a
  207
+
  208
+    def __del__(self):
  209
+        try:
  210
+            os.unlink(self.name)
  211
+        except OSError:
  212
+            pass
  213
+
  214
+def load_handler(path, *args, **kwargs):
  215
+    """
  216
+    Given a path to a handler, return an instance of that handler.
  217
+
  218
+    E.g.::
  219
+        >>> load_handler('django.core.files.uploadhandler.TemporaryFileUploadHandler', request)
  220
+        <TemporaryFileUploadHandler object at 0x...>
  221
+
  222
+    """
  223
+    i = path.rfind('.')
  224
+    module, attr = path[:i], path[i+1:]
  225
+    try:
  226
+        mod = __import__(module, {}, {}, [attr])
  227
+    except ImportError, e:
  228
+        raise ImproperlyConfigured('Error importing upload handler module %s: "%s"' % (module, e))
  229
+    except ValueError, e:
  230
+        raise ImproperlyConfigured('Error importing upload handler module. Is FILE_UPLOAD_HANDLERS a correctly defined list or tuple?')
  231
+    try:
  232
+        cls = getattr(mod, attr)
  233
+    except AttributeError:
  234
+        raise ImproperlyConfigured('Module "%s" does not define a "%s" upload handler backend' % (module, attr))
  235
+    return cls(*args, **kwargs)
3  django/core/handlers/modpython.py
@@ -53,7 +53,8 @@ def is_secure(self):
53 53
     def _load_post_and_files(self):
54 54
         "Populates self._post and self._files"
55 55
         if 'content-type' in self._req.headers_in and self._req.headers_in['content-type'].startswith('multipart'):
56  
-            self._post, self._files = http.parse_file_upload(self._req.headers_in, self.raw_post_data)
  56
+            self._raw_post_data = ''
  57
+            self._post, self._files = self.parse_file_upload(self.META, self._req)
57 58
         else:
58 59
             self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
59 60
 
5  django/core/handlers/wsgi.py
@@ -112,9 +112,8 @@ def _load_post_and_files(self):
112 112
         # Populates self._post and self._files
113 113
         if self.method == 'POST':
114 114
             if self.environ.get('CONTENT_TYPE', '').startswith('multipart'):
115  
-                header_dict = dict([(k, v) for k, v in self.environ.items() if k.startswith('HTTP_')])
116  
-                header_dict['Content-Type'] = self.environ.get('CONTENT_TYPE', '')
117  
-                self._post, self._files = http.parse_file_upload(header_dict, self.raw_post_data)
  115
+                self._raw_post_data = ''
  116
+                self._post, self._files = self.parse_file_upload(self.META, self.environ['wsgi.input'])
118 117
             else:
119 118
                 self._post, self._files = http.QueryDict(self.raw_post_data, encoding=self._encoding), datastructures.MultiValueDict()
120 119
         else:
60  django/db/models/base.py
@@ -19,6 +19,8 @@
19 19
 from django.utils.datastructures import SortedDict
20 20
 from django.utils.functional import curry
21 21
 from django.utils.encoding import smart_str, force_unicode, smart_unicode
  22
+from django.core.files.move import file_move_safe
  23
+from django.core.files import locks
22 24
 from django.conf import settings
23 25
 
24 26
 try:
@@ -469,16 +471,51 @@ def _get_FIELD_url(self, field):
469 471
     def _get_FIELD_size(self, field):
470 472
         return os.path.getsize(self._get_FIELD_filename(field))
471 473
 
472  
-    def _save_FIELD_file(self, field, filename, raw_contents, save=True):
  474
+    def _save_FIELD_file(self, field, filename, raw_field, save=True):
473 475
         directory = field.get_directory_name()
474 476
         try: # Create the date-based directory if it doesn't exist.
475 477
             os.makedirs(os.path.join(settings.MEDIA_ROOT, directory))
476 478
         except OSError: # Directory probably already exists.
477 479
             pass
  480
+
  481
+        #
  482
+        # Check for old-style usage (files-as-dictionaries). Warn here first
  483
+        # since there are multiple locations where we need to support both new
  484
+        # and old usage.
  485
+        #
  486
+        if isinstance(raw_field, dict):
  487
+            import warnings
  488
+            warnings.warn(
  489
+                message = "Representing uploaded files as dictionaries is"\
  490
+                          " deprected. Use django.core.files.SimpleUploadedFile"\
  491
+                          " instead.",
  492
+                category = DeprecationWarning,
  493
+                stacklevel = 2
  494
+            )
  495
+            from django.core.files.uploadedfile import SimpleUploadedFile
  496
+            raw_field = SimpleUploadedFile.from_dict(raw_field)
  497
+
  498
+        elif isinstance(raw_field, basestring):
  499
+            import warnings
  500
+            warnings.warn(
  501
+                message = "Representing uploaded files as strings is "\
  502
+                          " deprecated. Use django.core.files.SimpleUploadedFile "\
  503
+                          " instead.",
  504
+                category = DeprecationWarning,
  505
+                stacklevel = 2
  506
+            )
  507
+            from django.core.files.uploadedfile import SimpleUploadedFile
  508
+            raw_field = SimpleUploadedFile(filename, raw_field)
  509
+
  510
+        if filename is None:
  511
+            filename = raw_field.file_name
  512
+
478 513
         filename = field.get_filename(filename)
479 514
 
  515
+        #
480 516
         # If the filename already exists, keep adding an underscore to the name of
481 517
         # the file until the filename doesn't exist.
  518
+        #
482 519
         while os.path.exists(os.path.join(settings.MEDIA_ROOT, filename)):
483 520
             try:
484 521
                 dot_index = filename.rindex('.')
@@ -486,14 +523,27 @@ def _save_FIELD_file(self, field, filename, raw_contents, save=True):
486 523
                 filename += '_'
487 524
             else:
488 525
                 filename = filename[:dot_index] + '_' + filename[dot_index:]
  526
+        #
  527
+        # Save the file name on the object and write the file to disk
  528
+        #
489 529
 
490  
-        # Write the file to disk.
491 530
         setattr(self, field.attname, filename)
492 531
 
493 532
         full_filename = self._get_FIELD_filename(field)
494  
-        fp = open(full_filename, 'wb')
495  
-        fp.write(raw_contents)
496  
-        fp.close()
  533
+
  534
+        if hasattr(raw_field, 'temporary_file_path'):
  535
+            # This file has a file path that we can move.
  536
+            raw_field.close()
  537
+            file_move_safe(raw_field.temporary_file_path(), full_filename)
  538
+
  539
+        else:
  540
+            # This is a normal uploadedfile that we can stream.
  541
+            fp = open(full_filename, 'wb')
  542
+            locks.lock(fp, locks.LOCK_EX)
  543
+            for chunk in raw_field.chunk():
  544
+                fp.write(chunk)
  545
+            locks.unlock(fp)
  546
+            fp.close()
497 547
 
498 548
         # Save the width and/or height, if applicable.
499 549
         if isinstance(field, ImageField) and (field.width_field or field.height_field):
18  django/db/models/fields/__init__.py
@@ -811,7 +811,7 @@ def contribute_to_class(self, cls, name):
811 811
         setattr(cls, 'get_%s_filename' % self.name, curry(cls._get_FIELD_filename, field=self))
812 812
         setattr(cls, 'get_%s_url' % self.name, curry(cls._get_FIELD_url, field=self))
813 813
         setattr(cls, 'get_%s_size' % self.name, curry(cls._get_FIELD_size, field=self))
814  
-        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_contents, save=True: instance._save_FIELD_file(self, filename, raw_contents, save))
  814
+        setattr(cls, 'save_%s_file' % self.name, lambda instance, filename, raw_field, save=True: instance._save_FIELD_file(self, filename, raw_field, save))
815 815
         dispatcher.connect(self.delete_file, signal=signals.post_delete, sender=cls)
816 816
 
817 817
     def delete_file(self, instance):
@@ -834,9 +834,19 @@ def save_file(self, new_data, new_object, original_object, change, rel, save=Tru
834 834
         if new_data.get(upload_field_name, False):
835 835
             func = getattr(new_object, 'save_%s_file' % self.name)
836 836
             if rel:
837  
-                func(new_data[upload_field_name][0]["filename"], new_data[upload_field_name][0]["content"], save)
  837
+                file = new_data[upload_field_name][0]
838 838
             else:
839  
-                func(new_data[upload_field_name]["filename"], new_data[upload_field_name]["content"], save)
  839
+                file = new_data[upload_field_name]
  840
+
  841
+            # Backwards-compatible support for files-as-dictionaries.
  842
+            # We don't need to raise a warning because Model._save_FIELD_file will
  843
+            # do so for us.
  844
+            try:
  845
+                file_name = file.file_name
  846
+            except AttributeError:
  847
+                file_name = file['filename']
  848
+
  849
+            func(file_name, file, save)
840 850
 
841 851
     def get_directory_name(self):
842 852
         return os.path.normpath(force_unicode(datetime.datetime.now().strftime(smart_str(self.upload_to))))
@@ -849,7 +859,7 @@ def get_filename(self, filename):
849 859
     def save_form_data(self, instance, data):
850 860
         from django.newforms.fields import UploadedFile
851 861
         if data and isinstance(data, UploadedFile):
852  
-            getattr(instance, "save_%s_file" % self.name)(data.filename, data.content, save=False)
  862
+            getattr(instance, "save_%s_file" % self.name)(data.filename, data.data, save=False)
853 863
 
854 864
     def formfield(self, **kwargs):
855 865
         defaults = {'form_class': forms.FileField}
66  django/http/__init__.py
@@ -9,14 +9,15 @@
9 9
 except ImportError:
10 10
     from cgi import parse_qsl
11 11
 
12  
-from django.utils.datastructures import MultiValueDict, FileDict
  12
+from django.utils.datastructures import MultiValueDict, ImmutableList
13 13
 from django.utils.encoding import smart_str, iri_to_uri, force_unicode
14  
-
  14
+from django.http.multipartparser import MultiPartParser
  15
+from django.conf import settings
  16
+from django.core.files import uploadhandler
15 17
 from utils import *
16 18
 
17 19
 RESERVED_CHARS="!*'();:@&=+$,/?%#[]"
18 20
 
19  
-
20 21
 class Http404(Exception):
21 22
     pass
22 23
 
@@ -25,6 +26,7 @@ class HttpRequest(object):
25 26
 
26 27
     # The encoding used in GET/POST dicts. None means use default setting.
27 28
     _encoding = None
  29
+    _upload_handlers = []
28 30
 
29 31
     def __init__(self):
30 32
         self.GET, self.POST, self.COOKIES, self.META, self.FILES = {}, {}, {}, {}, {}
@@ -102,39 +104,31 @@ def _get_encoding(self):
102 104
 
103 105
     encoding = property(_get_encoding, _set_encoding)
104 106
 
105  
-def parse_file_upload(header_dict, post_data):
106  
-    """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
107  
-    import email, email.Message
108  
-    from cgi import parse_header
109  
-    raw_message = '\r\n'.join(['%s:%s' % pair for pair in header_dict.items()])
110  
-    raw_message += '\r\n\r\n' + post_data
111  
-    msg = email.message_from_string(raw_message)
112  
-    POST = QueryDict('', mutable=True)
113  
-    FILES = MultiValueDict()
114  
-    for submessage in msg.get_payload():
115  
-        if submessage and isinstance(submessage, email.Message.Message):
116  
-            name_dict = parse_header(submessage['Content-Disposition'])[1]
117  
-            # name_dict is something like {'name': 'file', 'filename': 'test.txt'} for file uploads
118  
-            # or {'name': 'blah'} for POST fields
119  
-            # We assume all uploaded files have a 'filename' set.
120  
-            if 'filename' in name_dict:
121  
-                assert type([]) != type(submessage.get_payload()), "Nested MIME messages are not supported"
122  
-                if not name_dict['filename'].strip():
123  
-                    continue
124  
-                # IE submits the full path, so trim everything but the basename.
125  
-                # (We can't use os.path.basename because that uses the server's
126  
-                # directory separator, which may not be the same as the
127  
-                # client's one.)
128  
-                filename = name_dict['filename'][name_dict['filename'].rfind("\\")+1:]
129  
-                FILES.appendlist(name_dict['name'], FileDict({
130  
-                    'filename': filename,
131  
-                    'content-type': 'Content-Type' in submessage and submessage['Content-Type'] or None,
132  
-                    'content': submessage.get_payload(),
133  
-                }))
134  
-            else:
135  
-                POST.appendlist(name_dict['name'], submessage.get_payload())
136  
-    return POST, FILES
137  
-
  107
+    def _initialize_handlers(self):
  108
+        self._upload_handlers = [uploadhandler.load_handler(handler, self)
  109
+                                 for handler in settings.FILE_UPLOAD_HANDLERS]
  110
+
  111
+    def _set_upload_handlers(self, upload_handlers):
  112
+        if hasattr(self, '_files'):
  113
+            raise AttributeError("You cannot set the upload handlers after the upload has been processed.")
  114
+        self._upload_handlers = upload_handlers
  115
+
  116
+    def _get_upload_handlers(self):
  117
+        if not self._upload_handlers:
  118
+            # If thre are no upload handlers defined, initialize them from settings.
  119
+            self._initialize_handlers()
  120
+        return self._upload_handlers
  121
+
  122
+    upload_handlers = property(_get_upload_handlers, _set_upload_handlers)
  123
+
  124
+    def parse_file_upload(self, META, post_data):
  125
+        """Returns a tuple of (POST QueryDict, FILES MultiValueDict)."""
  126
+        self.upload_handlers = ImmutableList(
  127
+            self.upload_handlers,
  128
+            warning = "You cannot alter upload handlers after the upload has been processed."
  129
+        )
  130
+        parser = MultiPartParser(META, post_data, self.upload_handlers, self.encoding)
  131
+        return parser.parse()
138 132
 
139 133
 class QueryDict(MultiValueDict):
140 134
     """
658  django/http/multipartparser.py
... ...
@@ -0,0 +1,658 @@
  1
+"""
  2
+Multi-part parsing for file uploads.
  3
+
  4
+Exposes one class, ``MultiPartParser``, which feeds chunks of uploaded data to
  5
+file upload handlers for processing.
  6
+"""
  7
+import cgi
  8
+from django.conf import settings
  9
+from django.core.exceptions import SuspiciousOperation
  10
+from django.utils.datastructures import MultiValueDict
  11
+from django.utils.encoding import force_unicode
  12
+from django.utils.text import unescape_entities
  13
+from django.core.files.uploadhandler import StopUpload, SkipFile, StopFutureHandlers
  14
+
  15
+__all__ = ('MultiPartParser','MultiPartParserError','InputStreamExhausted')
  16
+
  17
+class MultiPartParserError(Exception):
  18
+    pass
  19
+
  20
+class InputStreamExhausted(Exception):
  21
+    """
  22
+    No more reads are allowed from this device.
  23
+    """
  24
+    pass
  25
+
  26
+RAW = "raw"
  27
+FILE = "file"
  28
+FIELD = "field"
  29
+
  30
+class MultiPartParser(object):
  31
+    """
  32
+    A rfc2388 multipart/form-data parser.
  33
+
  34
+    ``MultiValueDict.parse()`` reads the input stream in ``chunk_size`` chunks
  35
+    and returns a tuple of ``(MultiValueDict(POST), MultiValueDict(FILES))``. If
  36
+    ``file_upload_dir`` is defined files will be streamed to temporary files in
  37
+    that directory.
  38
+    """
  39
+    def __init__(self, META, input_data, upload_handlers, encoding=None):
  40
+        """
  41
+        Initialize the MultiPartParser object.
  42
+
  43
+        :META:
  44
+            The standard ``META`` dictionary in Django request objects.
  45
+        :input_data:
  46
+            The raw post data, as a bytestring.
  47
+        :upload_handler:
  48
+            An UploadHandler instance that performs operations on the uploaded
  49
+            data.
  50
+        :encoding:
  51
+            The encoding with which to treat the incoming data.
  52
+        """
  53
+
  54
+        #
  55
+        # Content-Type should containt multipart and the boundary information.
  56
+        #
  57
+
  58
+        content_type = META.get('HTTP_CONTENT_TYPE', META.get('CONTENT_TYPE', ''))
  59
+        if not content_type.startswith('multipart/'):
  60
+            raise MultiPartParserError('Invalid Content-Type: %s' % content_type)
  61
+
  62
+        # Parse the header to get the boundary to split the parts.
  63
+        ctypes, opts = parse_header(content_type)
  64
+        boundary = opts.get('boundary')
  65
+        if not boundary or not cgi.valid_boundary(boundary):
  66
+            raise MultiPartParserError('Invalid boundary in multipart: %s' % boundary)
  67
+
  68
+
  69
+        #
  70
+        # Content-Length should contain the length of the body we are about
  71
+        # to receive.
  72
+        #
  73
+        try:
  74
+            content_length = int(META.get('HTTP_CONTENT_LENGTH', META.get('CONTENT_LENGTH',0)))
  75
+        except (ValueError, TypeError):
  76
+            # For now set it to 0; we'll try again later on down.
  77
+            content_length = 0
  78
+
  79
+        if content_length <= 0:
  80
+            # This means we shouldn't continue...raise an error.
  81
+            raise MultiPartParserError("Invalid content length: %r" % content_length)
  82
+
  83
+        self._boundary = boundary
  84
+        self._input_data = input_data
  85
+
  86
+        # For compatibility with low-level network APIs (with 32-bit integers),
  87
+        # the chunk size should be < 2^31, but still divisible by 4.
  88
+        self._chunk_size = min(2**31-4, *[x.chunk_size for x in upload_handlers if x.chunk_size])
  89
+
  90
+        self._meta = META
  91
+        self._encoding = encoding or settings.DEFAULT_CHARSET
  92
+        self._content_length = content_length
  93
+        self._upload_handlers = upload_handlers
  94
+
  95
+    def parse(self):
  96
+        """
  97
+        Parse the POST data and break it into a FILES MultiValueDict and a POST
  98
+        MultiValueDict.
  99
+
  100
+        Returns a tuple containing the POST and FILES dictionary, respectively.
  101
+        """
  102
+        # We have to import QueryDict down here to avoid a circular import.
  103
+        from django.http import QueryDict
  104
+
  105
+        encoding = self._encoding
  106
+        handlers = self._upload_handlers
  107
+
  108
+        limited_input_data = LimitBytes(self._input_data, self._content_length)
  109
+
  110
+        # See if the handler will want to take care of the parsing.
  111
+        # This allows overriding everything if somebody wants it.
  112
+        for handler in handlers:
  113
+            result = handler.handle_raw_input(limited_input_data,
  114
+                                              self._meta,
  115
+                                              self._content_length,
  116
+                                              self._boundary,
  117
+                                              encoding)
  118
+            if result is not None:
  119
+                return result[0], result[1]
  120
+
  121
+        # Create the data structures to be used later.
  122
+        self._post = QueryDict('', mutable=True)
  123
+        self._files = MultiValueDict()
  124
+
  125
+        # Instantiate the parser and stream:
  126
+        stream = LazyStream(ChunkIter(limited_input_data, self._chunk_size))
  127
+
  128
+        # Whether or not to signal a file-completion at the beginning of the loop.
  129
+        old_field_name = None
  130
+        counters = [0] * len(handlers)
  131
+
  132
+        try:
  133
+            for item_type, meta_data, field_stream in Parser(stream, self._boundary):
  134
+                if old_field_name:
  135
+                    # We run this at the beginning of the next loop
  136
+                    # since we cannot be sure a file is complete until
  137
+                    # we hit the next boundary/part of the multipart content.
  138
+                    self.handle_file_complete(old_field_name, counters)
  139
+
  140
+                try:
  141
+                    disposition = meta_data['content-disposition'][1]
  142
+                    field_name = disposition['name'].strip()
  143
+                except (KeyError, IndexError, AttributeError):
  144
+                    continue
  145
+
  146
+                transfer_encoding = meta_data.get('content-transfer-encoding')
  147
+                field_name = force_unicode(field_name, encoding, errors='replace')
  148
+
  149
+                if item_type == FIELD:
  150
+                    # This is a post field, we can just set it in the post
  151
+                    if transfer_encoding == 'base64':
  152
+                        raw_data = field_stream.read()
  153
+                        try:
  154
+                            data = str(raw_data).decode('base64')
  155
+                        except:
  156
+                            data = raw_data
  157
+                    else:
  158
+                        data = field_stream.read()
  159
+
  160
+                    self._post.appendlist(field_name,
  161
+                                          force_unicode(data, encoding, errors='replace'))
  162
+                elif item_type == FILE:
  163
+                    # This is a file, use the handler...
  164
+                    file_successful = True
  165
+                    file_name = disposition.get('filename')
  166
+                    if not file_name:
  167
+                        continue
  168
+                    file_name = force_unicode(file_name, encoding, errors='replace')
  169
+                    file_name = self.IE_sanitize(unescape_entities(file_name))
  170
+
  171
+                    content_type = meta_data.get('content-type', ('',))[0].strip()
  172
+                    try:
  173
+                        charset = meta_data.get('content-type', (0,{}))[1].get('charset', None)
  174
+                    except:
  175
+                        charset = None
  176
+
  177
+                    try:
  178
+                        content_length = int(meta_data.get('content-length')[0])
  179
+                    except (IndexError, TypeError, ValueError):
  180
+                        content_length = None
  181
+
  182
+                    counters = [0] * len(handlers)
  183
+                    try:
  184
+                        for handler in handlers:
  185
+                            try:
  186
+                                handler.new_file(field_name, file_name,
  187
+                                                 content_type, content_length,
  188
+                                                 charset)
  189
+                            except StopFutureHandlers:
  190
+                                break
  191
+
  192
+                        for chunk in field_stream:
  193
+                            if transfer_encoding == 'base64':
  194
+                                # We only special-case base64 transfer encoding
  195
+                                try:
  196
+                                    chunk = str(chunk).decode('base64')
  197
+                                except Exception, e:
  198
+                                    # Since this is only a chunk, any error is an unfixable error.
  199
+                                    raise MultiPartParserError("Could not decode base64 data: %r" % e)
  200
+
  201
+                            for i, handler in enumerate(handlers):
  202
+                                chunk_length = len(chunk)
  203
+                                chunk = handler.receive_data_chunk(chunk,
  204
+                                                                   counters[i])
  205
+                                counters[i] += chunk_length
  206
+                                if chunk is None:
  207
+                                    # If the chunk received by the handler is None, then don't continue.
  208
+                                    break
  209
+
  210
+                    except SkipFile, e:
  211
+                        file_successful = False
  212
+                        # Just use up the rest of this file...
  213
+                        exhaust(field_stream)
  214
+                    else:
  215
+                        # Handle file upload completions on next iteration.
  216
+                        old_field_name = field_name
  217
+                else:
  218
+                    # If this is neither a FIELD or a FILE, just exhaust the stream.
  219
+                    exhaust(stream)