Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Fixed #18916 -- Allowed non-ASCII headers.

Thanks Malcolm Tredinnick for the review.
  • Loading branch information...
commit 9b07b5edeb770b037dc735d48dfd6f979422f586 1 parent 1d235b3
@aaugustin aaugustin authored
Showing with 77 additions and 43 deletions.
  1. +35 −21 django/http/__init__.py
  2. +42 −22 tests/regressiontests/httpwrappers/tests.py
View
56 django/http/__init__.py
@@ -2,6 +2,7 @@
import copy
import datetime
+from email.header import Header
import os
import re
import sys
@@ -560,31 +561,44 @@ def serialize(self):
else:
__str__ = serialize
- def _convert_to_ascii(self, *values):
- """Converts all values to ascii strings."""
- for value in values:
- if not isinstance(value, six.string_types):
- value = str(value)
- try:
- if six.PY3:
- # Ensure string only contains ASCII
- value.encode('us-ascii')
+ def _convert_to_charset(self, value, charset, mime_encode=False):
+ """Converts headers key/value to ascii/latin1 native strings.
+
+ `charset` must be 'ascii' or 'latin-1'. If `mime_encode` is True and
+ `value` value can't be represented in the given charset, MIME-encoding
+ is applied.
+ """
+ if not isinstance(value, (bytes, six.text_type)):
+ value = str(value)
+ try:
+ if six.PY3:
+ if isinstance(value, str):
+ # Ensure string is valid in given charset
+ value.encode(charset)
else:
- if isinstance(value, str):
- # Ensure string only contains ASCII
- value.decode('us-ascii')
- else:
- # Convert unicode to an ASCII string
- value = value.encode('us-ascii')
- except UnicodeError as e:
- e.reason += ', HTTP response headers must be in US-ASCII format'
+ # Convert bytestring using given charset
+ value = value.decode(charset)
+ else:
+ if isinstance(value, str):
+ # Ensure string is valid in given charset
+ value.decode(charset)
+ else:
+ # Convert unicode string to given charset
+ value = value.encode(charset)
+ except UnicodeError as e:
+ if mime_encode:
+ # Wrapping in str() is a workaround for #12422 under Python 2.
+ value = str(Header(value, 'utf-8').encode())
+ else:
+ e.reason += ', HTTP response headers must be in %s format' % charset
raise
- if '\n' in value or '\r' in value:
- raise BadHeaderError("Header values can't contain newlines (got %r)" % value)
- yield value
+ if str('\n') in value or str('\r') in value:
+ raise BadHeaderError("Header values can't contain newlines (got %r)" % value)
+ return value
def __setitem__(self, header, value):
- header, value = self._convert_to_ascii(header, value)
+ header = self._convert_to_charset(header, 'ascii')
+ value = self._convert_to_charset(value, 'latin1', mime_encode=True)
self._headers[header.lower()] = (header, value)
def __delitem__(self, header):
View
64 tests/regressiontests/httpwrappers/tests.py
@@ -11,6 +11,7 @@
SimpleCookie, BadHeaderError,
parse_cookie)
from django.test import TestCase
+from django.utils.encoding import smart_str
from django.utils import six
from django.utils import unittest
@@ -228,33 +229,52 @@ def test_non_default_encoding(self):
self.assertEqual(copy.deepcopy(q).encoding, 'iso-8859-15')
class HttpResponseTests(unittest.TestCase):
- def test_unicode_headers(self):
- r = HttpResponse()
-
- # If we insert a unicode value it will be converted to an ascii
- r['value'] = 'test value'
- self.assertTrue(isinstance(r['value'], str))
-
- # An error is raised when a unicode object with non-ascii is assigned.
- self.assertRaises(UnicodeEncodeError, r.__setitem__, 'value', 't\xebst value')
- # An error is raised when a unicode object with non-ASCII format is
- # passed as initial mimetype or content_type.
- self.assertRaises(UnicodeEncodeError, HttpResponse,
- content_type='t\xebst value')
+ def test_headers_type(self):
+ r = HttpResponse()
- # HttpResponse headers must be convertible to ASCII.
- self.assertRaises(UnicodeEncodeError, HttpResponse,
- content_type='t\xebst value')
+ # The following tests explicitly test types in addition to values
+ # because in Python 2 u'foo' == b'foo'.
+
+ # ASCII unicode or bytes values are converted to native strings.
+ r['key'] = 'test'
+ self.assertEqual(r['key'], str('test'))
+ self.assertIsInstance(r['key'], str)
+ r['key'] = 'test'.encode('ascii')
+ self.assertEqual(r['key'], str('test'))
+ self.assertIsInstance(r['key'], str)
+
+ # Latin-1 unicode or bytes values are also converted to native strings.
+ r['key'] = 'café'
+ self.assertEqual(r['key'], smart_str('café', 'latin-1'))
+ self.assertIsInstance(r['key'], str)
+ r['key'] = 'café'.encode('latin-1')
+ self.assertEqual(r['key'], smart_str('café', 'latin-1'))
+ self.assertIsInstance(r['key'], str)
+
+ # Other unicode values are MIME-encoded (there's no way to pass them as bytes).
+ r['key'] = ''
+ self.assertEqual(r['key'], str('=?utf-8?b?4oCg?='))
+ self.assertIsInstance(r['key'], str)
+
+ # The response also converts unicode or bytes keys to strings, but requires
+ # them to contain ASCII
+ r = HttpResponse()
+ r['foo'] = 'bar'
+ l = list(r.items())
+ self.assertEqual(l[0], ('foo', 'bar'))
+ self.assertIsInstance(l[0][0], str)
- # The response also converts unicode keys to strings.)
- r['test'] = 'testing key'
+ r = HttpResponse()
+ r[b'foo'] = 'bar'
l = list(r.items())
- l.sort()
- self.assertEqual(l[1], ('test', 'testing key'))
+ self.assertEqual(l[0], ('foo', 'bar'))
+ self.assertIsInstance(l[0][0], str)
+
+ r = HttpResponse()
+ self.assertRaises(UnicodeError, r.__setitem__, 'føø', 'bar')
+ self.assertRaises(UnicodeError, r.__setitem__, 'føø'.encode('utf-8'), 'bar')
- # It will also raise errors for keys with non-ascii data.
- self.assertRaises(UnicodeEncodeError, r.__setitem__, 't\xebst key', 'value')
def test_newlines_in_headers(self):
# Bug #10188: Do not allow newlines in headers (CR or LF)
Please sign in to comment.
Something went wrong with that request. Please try again.