Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP
Browse files

Merged master changes.

  • Loading branch information...
commit 7e82e83d67ee0871a72e1a3a723afdd214fcefc3 2 parents e29c010 + 39aa890
@freakboy3742 freakboy3742 authored
Showing with 2,279 additions and 876 deletions.
  1. +1 −1  MANIFEST.in
  2. +11 −3 django/conf/__init__.py
  3. +5 −4 django/conf/project_template/project_name/settings.py
  4. +1 −1  django/contrib/admin/templates/admin/change_form.html
  5. +2 −2 django/contrib/admin/util.py
  6. +2 −2 django/contrib/admin/views/main.py
  7. +2 −2 django/contrib/admindocs/utils.py
  8. +4 −4 django/contrib/auth/hashers.py
  9. +16 −9 django/contrib/auth/management/__init__.py
  10. +6 −2 django/contrib/auth/tests/management.py
  11. +6 −6 django/contrib/databrowse/datastructures.py
  12. +1 −2  django/contrib/gis/db/models/query.py
  13. +1 −2  django/contrib/gis/sitemaps/views.py
  14. +9 −0 django/contrib/gis/tests/geoapp/tests.py
  15. +3 −3 django/contrib/markup/templatetags/markup.py
  16. +2 −2 django/contrib/sessions/backends/base.py
  17. +1 −1  django/contrib/staticfiles/management/commands/collectstatic.py
  18. +3 −3 django/contrib/staticfiles/storage.py
  19. +2 −2 django/core/cache/backends/db.py
  20. +2 −2 django/core/cache/backends/filebased.py
  21. +2 −2 django/core/cache/backends/memcached.py
  22. +5 −3 django/core/files/base.py
  23. +9 −2 django/core/files/storage.py
  24. +2 −2 django/core/files/uploadedfile.py
  25. +2 −2 django/core/handlers/wsgi.py
  26. +2 −2 django/core/management/base.py
  27. +77 −39 django/core/management/commands/inspectdb.py
  28. +9 −3 django/core/management/commands/loaddata.py
  29. +16 −37 django/core/management/commands/makemessages.py
  30. +9 −9 django/core/management/commands/shell.py
  31. +4 −2 django/core/management/templates.py
  32. +2 −2 django/core/management/validation.py
  33. +0 −1  django/core/serializers/json.py
  34. +0 −1  django/core/serializers/pyyaml.py
  35. +29 −19 django/core/signing.py
  36. +9 −4 django/core/urlresolvers.py
  37. +3 −3 django/core/validators.py
  38. +4 −4 django/db/backends/__init__.py
  39. +7 −1 django/db/backends/mysql/compiler.py
  40. +17 −5 django/db/backends/mysql/introspection.py
  41. +5 −7 django/db/backends/oracle/base.py
  42. +2 −2 django/db/backends/postgresql_psycopg2/introspection.py
  43. +13 −5 django/db/backends/sqlite3/introspection.py
  44. +2 −2 django/db/backends/util.py
  45. +2 −2 django/db/models/base.py
  46. +3 −2 django/db/models/fields/__init__.py
  47. +2 −2 django/db/models/fields/files.py
  48. +1 −3 django/db/models/query.py
  49. +4 −8 django/db/models/sql/compiler.py
  50. +1 −1  django/db/models/sql/constants.py
  51. +54 −50 django/db/models/sql/query.py
  52. +22 −25 django/forms/formsets.py
  53. +5 −2 django/forms/models.py
  54. +44 −34 django/http/__init__.py
  55. +3 −3 django/template/base.py
  56. +7 −7 django/template/response.py
  57. +3 −3 django/templatetags/cache.py
  58. +5 −5 django/test/client.py
  59. +8 −1 django/test/signals.py
  60. +1 −1  django/test/testcases.py
  61. +3 −3 django/utils/cache.py
  62. +11 −6 django/utils/crypto.py
  63. +3 −3 django/utils/encoding.py
  64. +5 −5 django/utils/formats.py
  65. +0 −1  django/utils/functional.py
  66. +2 −2 django/utils/html.py
  67. +7 −7 django/utils/http.py
  68. +31 −19 django/utils/six.py
  69. +6 −6 django/utils/text.py
  70. +4 −4 django/utils/translation/trans_real.py
  71. +3 −3 django/utils/tzinfo.py
  72. +2 −2 django/views/debug.py
  73. +1 −1  django/views/static.py
  74. +2 −1  docs/conf.py
  75. +1 −1  docs/faq/general.txt
  76. +2 −1  docs/index.txt
  77. +1 −3 docs/internals/committers.txt
  78. +556 −0 docs/ref/class-based-views/flattened-index.txt
  79. +1 −0  docs/ref/class-based-views/index.txt
  80. +4 −2 docs/ref/contrib/admin/index.txt
  81. +3 −1 docs/ref/contrib/comments/index.txt
  82. +104 −43 docs/ref/contrib/csrf.txt
  83. +41 −2 docs/ref/contrib/flatpages.txt
  84. +10 −8 docs/ref/contrib/gis/install.txt
  85. +1 −1  docs/ref/contrib/gis/testing.txt
  86. +11 −0 docs/ref/contrib/gis/tutorial.txt
  87. +8 −4 docs/ref/files/file.txt
  88. +46 −46 docs/ref/models/fields.txt
  89. +3 −3 docs/ref/models/instances.txt
  90. +1 −2  docs/ref/models/querysets.txt
  91. +11 −8 docs/ref/request-response.txt
  92. +3 −3 docs/ref/settings.txt
  93. +8 −0 docs/ref/templates/api.txt
  94. +14 −0 docs/releases/1.3.2.txt
  95. +1 −1  docs/releases/1.3.txt
  96. +14 −0 docs/releases/1.4.1.txt
  97. +14 −0 docs/releases/1.4.2.txt
  98. +60 −4 docs/releases/1.5.txt
  99. +3 −0  docs/releases/index.txt
  100. +11 −10 docs/topics/auth.txt
  101. +1 −1  docs/topics/cache.txt
  102. +33 −116 docs/topics/class-based-views/index.txt
  103. +115 −35 docs/topics/class-based-views/mixins.txt
  104. +13 −15 docs/topics/db/models.txt
  105. +1 −1  docs/topics/db/multi-db.txt
  106. +4 −4 docs/topics/db/queries.txt
  107. +30 −30 docs/topics/forms/media.txt
  108. +3 −1 docs/topics/forms/modelforms.txt
  109. +3 −1 docs/topics/http/file-uploads.txt
  110. +8 −4 docs/topics/http/urls.txt
  111. +4 −0 docs/topics/http/views.txt
  112. +29 −0 docs/topics/logging.txt
  113. +2 −2 docs/topics/python3.txt
  114. +5 −1 docs/topics/templates.txt
  115. +1 −1  docs/topics/testing.txt
  116. +1 −1  setup.cfg
  117. +6 −6 tests/modeltests/files/tests.py
  118. +2 −0  tests/modeltests/validation/models.py
  119. +8 −1 tests/regressiontests/admin_custom_urls/fixtures/actions.json
  120. +29 −9 tests/regressiontests/admin_custom_urls/tests.py
  121. +2 −0  tests/regressiontests/admin_scripts/custom_templates/project_template/ticket-18091-non-ascii-template.txt
  122. +17 −0 tests/regressiontests/admin_scripts/tests.py
  123. +1 −1  tests/regressiontests/admin_util/tests.py
  124. +4 −4 tests/regressiontests/admin_views/tests.py
  125. +45 −24 tests/regressiontests/file_storage/tests.py
  126. +3 −3 tests/regressiontests/file_uploads/tests.py
  127. +2 −2 tests/regressiontests/file_uploads/views.py
  128. +14 −0 tests/regressiontests/fixtures_regress/tests.py
  129. +30 −1 tests/regressiontests/httpwrappers/tests.py
  130. +2 −2 tests/regressiontests/i18n/tests.py
  131. +9 −0 tests/regressiontests/inspectdb/models.py
  132. +28 −12 tests/regressiontests/inspectdb/tests.py
  133. +5 −0 tests/regressiontests/introspection/tests.py
  134. +27 −0 tests/regressiontests/model_fields/models.py
  135. +9 −1 tests/regressiontests/model_fields/tests.py
  136. 0  tests/regressiontests/nested_foreign_keys/__init__.py
  137. +28 −0 tests/regressiontests/nested_foreign_keys/models.py
  138. +166 −0 tests/regressiontests/nested_foreign_keys/tests.py
  139. +15 −0 tests/regressiontests/queries/models.py
  140. +24 −1 tests/regressiontests/queries/tests.py
  141. +10 −0 tests/regressiontests/settings_tests/tests.py
  142. +12 −8 tests/regressiontests/signing/tests.py
  143. +11 −1 tests/regressiontests/urlpatterns_reverse/tests.py
  144. +11 −19 tests/regressiontests/utils/crypto.py
  145. +17 −0 tests/regressiontests/utils/encoding.py
  146. +2 −1  tests/regressiontests/utils/tests.py
View
2  MANIFEST.in
@@ -1,4 +1,4 @@
-include README
+include README.rst
include AUTHORS
include INSTALL
include LICENSE
View
14 django/conf/__init__.py
@@ -152,17 +152,25 @@ def __init__(self, default_settings):
Requests for configuration variables not in this class are satisfied
from the module specified in default_settings (if possible).
"""
+ self.__dict__['_deleted'] = set()
self.default_settings = default_settings
def __getattr__(self, name):
+ if name in self._deleted:
+ raise AttributeError
return getattr(self.default_settings, name)
+ def __setattr__(self, name, value):
+ self._deleted.discard(name)
+ return super(UserSettingsHolder, self).__setattr__(name, value)
+
+ def __delattr__(self, name):
+ self._deleted.add(name)
+ return super(UserSettingsHolder, self).__delattr__(name)
+
def __dir__(self):
return list(self.__dict__) + dir(self.default_settings)
- # For Python < 2.6:
- __members__ = property(lambda self: self.__dir__())
-
settings = LazySettings()
View
9 django/conf/project_template/project_name/settings.py
@@ -13,10 +13,11 @@
'default': {
'ENGINE': 'django.db.backends.', # Add 'postgresql_psycopg2', 'mysql', 'sqlite3' or 'oracle'.
'NAME': '', # Or path to database file if using sqlite3.
- 'USER': '', # Not used with sqlite3.
- 'PASSWORD': '', # Not used with sqlite3.
- 'HOST': '', # Set to empty string for localhost. Not used with sqlite3.
- 'PORT': '', # Set to empty string for default. Not used with sqlite3.
+ # The following settings are not used with sqlite3:
+ 'USER': '',
+ 'PASSWORD': '',
+ 'HOST': '', # Empty for localhost through domain sockets or '127.0.0.1' for localhost through TCP.
+ 'PORT': '', # Set to empty string for default.
}
}
View
2  django/contrib/admin/templates/admin/change_form.html
@@ -29,7 +29,7 @@
{% if change %}{% if not is_popup %}
<ul class="object-tools">
{% block object-tools-items %}
- <li><a href="history/" class="historylink">{% trans "History" %}</a></li>
+ <li><a href="{% url opts|admin_urlname:'history' original.pk %}" class="historylink">{% trans "History" %}</a></li>
{% if has_absolute_url %}<li><a href="{% url 'admin:view_on_site' content_type_id original.pk %}" class="viewsitelink">{% trans "View on site" %}</a></li>{% endif%}
{% endblock %}
</ul>
View
4 django/contrib/admin/util.py
@@ -12,7 +12,7 @@
from django.utils.html import format_html
from django.utils.text import capfirst
from django.utils import timezone
-from django.utils.encoding import force_text, smart_text, smart_bytes
+from django.utils.encoding import force_str, force_text, smart_text
from django.utils import six
from django.utils.translation import ungettext
from django.core.urlresolvers import reverse
@@ -277,7 +277,7 @@ def label_for_field(name, model, model_admin=None, return_attr=False):
label = force_text(model._meta.verbose_name)
attr = six.text_type
elif name == "__str__":
- label = smart_bytes(model._meta.verbose_name)
+ label = force_str(model._meta.verbose_name)
attr = bytes
else:
if callable(name):
View
4 django/contrib/admin/views/main.py
@@ -6,7 +6,7 @@
from django.db import models
from django.db.models.fields import FieldDoesNotExist
from django.utils.datastructures import SortedDict
-from django.utils.encoding import force_text, smart_bytes
+from django.utils.encoding import force_str, force_text
from django.utils.translation import ugettext, ugettext_lazy
from django.utils.http import urlencode
@@ -94,7 +94,7 @@ def get_filters(self, request):
# 'key' will be used as a keyword argument later, so Python
# requires it to be a string.
del lookup_params[key]
- lookup_params[smart_bytes(key)] = value
+ lookup_params[force_str(key)] = value
if not self.model_admin.lookup_allowed(key, value):
raise SuspiciousOperation("Filtering by %s not allowed" % key)
View
4 django/contrib/admindocs/utils.py
@@ -6,7 +6,7 @@
from django.utils.safestring import mark_safe
from django.core.urlresolvers import reverse
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
try:
import docutils.core
import docutils.nodes
@@ -66,7 +66,7 @@ def parse_rst(text, default_reference_context, thing_being_parsed=None):
"link_base" : reverse('django-admindocs-docroot').rstrip('/')
}
if thing_being_parsed:
- thing_being_parsed = smart_bytes("<%s>" % thing_being_parsed)
+ thing_being_parsed = force_bytes("<%s>" % thing_being_parsed)
parts = docutils.core.publish_parts(text, source_path=thing_being_parsed,
destination_path=None, writer_name='html',
settings_overrides=overrides)
View
8 django/contrib/auth/hashers.py
@@ -8,7 +8,7 @@
from django.test.signals import setting_changed
from django.utils import importlib
from django.utils.datastructures import SortedDict
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
from django.core.exceptions import ImproperlyConfigured
from django.utils.crypto import (
pbkdf2, constant_time_compare, get_random_string)
@@ -299,7 +299,7 @@ class SHA1PasswordHasher(BasePasswordHasher):
def encode(self, password, salt):
assert password
assert salt and '$' not in salt
- hash = hashlib.sha1(smart_bytes(salt + password)).hexdigest()
+ hash = hashlib.sha1(force_bytes(salt + password)).hexdigest()
return "%s$%s$%s" % (self.algorithm, salt, hash)
def verify(self, password, encoded):
@@ -327,7 +327,7 @@ class MD5PasswordHasher(BasePasswordHasher):
def encode(self, password, salt):
assert password
assert salt and '$' not in salt
- hash = hashlib.md5(smart_bytes(salt + password)).hexdigest()
+ hash = hashlib.md5(force_bytes(salt + password)).hexdigest()
return "%s$%s$%s" % (self.algorithm, salt, hash)
def verify(self, password, encoded):
@@ -361,7 +361,7 @@ def salt(self):
return ''
def encode(self, password, salt):
- return hashlib.md5(smart_bytes(password)).hexdigest()
+ return hashlib.md5(force_bytes(password)).hexdigest()
def verify(self, password, encoded):
encoded_2 = self.encode(password, '')
View
25 django/contrib/auth/management/__init__.py
@@ -10,6 +10,7 @@
from django.contrib.auth import models as auth_app, get_user_model
from django.core import exceptions
from django.db.models import get_models, signals
+from django.utils import six
from django.utils.six.moves import input
@@ -87,17 +88,23 @@ def get_system_username():
:returns: The username as a unicode string, or an empty string if the
username could not be determined.
"""
- default_locale = locale.getdefaultlocale()[1]
- if default_locale:
+ try:
+ result = getpass.getuser()
+ except (ImportError, KeyError):
+ # KeyError will be raised by os.getpwuid() (called by getuser())
+ # if there is no corresponding entry in the /etc/passwd file
+ # (a very restricted chroot environment, for example).
+ return ''
+ if not six.PY3:
+ default_locale = locale.getdefaultlocale()[1]
+ if not default_locale:
+ return ''
try:
- return getpass.getuser().decode(default_locale)
- except (ImportError, KeyError, UnicodeDecodeError):
- # KeyError will be raised by os.getpwuid() (called by getuser())
- # if there is no corresponding entry in the /etc/passwd file
- # (a very restricted chroot environment, for example).
+ result = result.decode(default_locale)
+ except UnicodeDecodeError:
# UnicodeDecodeError - preventive treatment for non-latin Windows.
- pass
- return ''
+ return ''
+ return result
def get_default_username(check_db=True):
View
8 django/contrib/auth/tests/management.py
@@ -9,16 +9,20 @@
from django.core.management.base import CommandError
from django.test import TestCase
from django.test.utils import override_settings
+from django.utils import six
from django.utils.six import StringIO
class GetDefaultUsernameTestCase(TestCase):
def setUp(self):
- self._getpass_getuser = management.get_system_username
+ self.old_get_system_username = management.get_system_username
def tearDown(self):
- management.get_system_username = self._getpass_getuser
+ management.get_system_username = self.old_get_system_username
+
+ def test_actual_implementation(self):
+ self.assertIsInstance(management.get_system_username(), six.text_type)
def test_simple(self):
management.get_system_username = lambda: 'joe'
View
12 django/contrib/databrowse/datastructures.py
@@ -7,7 +7,7 @@
from django.db import models
from django.utils import formats
from django.utils.text import capfirst
-from django.utils.encoding import smart_text, smart_str, iri_to_uri
+from django.utils.encoding import smart_text, force_str, iri_to_uri
from django.db.models.query import QuerySet
from django.utils.encoding import python_2_unicode_compatible
@@ -23,7 +23,7 @@ def __init__(self, site, model):
self.verbose_name_plural = model._meta.verbose_name_plural
def __repr__(self):
- return smart_str('<EasyModel for %s>' % self.model._meta.object_name)
+ return force_str('<EasyModel for %s>' % self.model._meta.object_name)
def model_databrowse(self):
"Returns the ModelDatabrowse class for this model."
@@ -62,7 +62,7 @@ def __init__(self, easy_model, field):
self.model, self.field = easy_model, field
def __repr__(self):
- return smart_str('<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
+ return force_str('<EasyField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
def choices(self):
for value, label in self.field.choices:
@@ -80,7 +80,7 @@ def __init__(self, easy_model, field, value, label):
self.value, self.label = value, label
def __repr__(self):
- return smart_str('<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
+ return force_str('<EasyChoice for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
def url(self):
return '%s%s/%s/%s/%s/' % (self.model.site.root_url, self.model.model._meta.app_label, self.model.model._meta.module_name, self.field.field.name, iri_to_uri(self.value))
@@ -91,7 +91,7 @@ def __init__(self, easy_model, instance):
self.model, self.instance = easy_model, instance
def __repr__(self):
- return smart_str('<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val()))
+ return force_str('<EasyInstance for %s (%s)>' % (self.model.model._meta.object_name, self.instance._get_pk_val()))
def __str__(self):
val = smart_text(self.instance)
@@ -135,7 +135,7 @@ def __init__(self, easy_model, instance, field):
self.raw_value = getattr(instance.instance, field.name)
def __repr__(self):
- return smart_str('<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
+ return force_str('<EasyInstanceField for %s.%s>' % (self.model.model._meta.object_name, self.field.name))
def values(self):
"""
View
3  django/contrib/gis/db/models/query.py
@@ -1,16 +1,15 @@
from django.db import connections
from django.db.models.query import QuerySet, ValuesQuerySet, ValuesListQuerySet
-from django.utils import six
from django.contrib.gis.db.models import aggregates
from django.contrib.gis.db.models.fields import get_srid_info, PointField, LineStringField
from django.contrib.gis.db.models.sql import AreaField, DistanceField, GeomField, GeoQuery
from django.contrib.gis.geometry.backend import Geometry
from django.contrib.gis.measure import Area, Distance
-from django.utils import six
from django.utils import six
+
class GeoQuerySet(QuerySet):
"The Geographic QuerySet."
View
3  django/contrib/gis/sitemaps/views.py
@@ -8,7 +8,6 @@
from django.contrib.gis.db.models.fields import GeometryField
from django.db import connections, DEFAULT_DB_ALIAS
from django.db.models import get_model
-from django.utils.encoding import smart_bytes
from django.utils import six
from django.utils.translation import ugettext as _
@@ -61,7 +60,7 @@ def sitemap(request, sitemaps, section=None):
raise Http404(_("Page %s empty") % page)
except PageNotAnInteger:
raise Http404(_("No page '%s'") % page)
- xml = smart_bytes(loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls}))
+ xml = loader.render_to_string('gis/sitemaps/geo_sitemap.xml', {'urlset': urls})
return HttpResponse(xml, content_type='application/xml')
def kml(request, label, model, field_name=None, compress=False, using=DEFAULT_DB_ALIAS):
View
9 django/contrib/gis/tests/geoapp/tests.py
@@ -186,6 +186,15 @@ def test_inherited_geofields(self):
self.assertEqual(1, qs.count())
for pc in qs: self.assertEqual(32128, pc.point.srid)
+ def test_raw_sql_query(self):
+ "Testing raw SQL query."
+ cities1 = City.objects.all()
+ # Only PostGIS would support a 'select *' query because of its recognized
+ # HEXEWKB format for geometry fields
+ cities2 = City.objects.raw('select id, name, asText(point) from geoapp_city')
+ self.assertEqual(len(cities1), len(list(cities2)))
+ self.assertTrue(isinstance(cities2[0].point, Point))
+
class GeoLookupTest(TestCase):
View
6 django/contrib/markup/templatetags/markup.py
@@ -13,7 +13,7 @@
from django import template
from django.conf import settings
-from django.utils.encoding import smart_bytes, force_text
+from django.utils.encoding import force_bytes, force_text
from django.utils.safestring import mark_safe
register = template.Library()
@@ -27,7 +27,7 @@ def textile(value):
raise template.TemplateSyntaxError("Error in 'textile' filter: The Python textile library isn't installed.")
return force_text(value)
else:
- return mark_safe(force_text(textile.textile(smart_bytes(value), encoding='utf-8', output='utf-8')))
+ return mark_safe(force_text(textile.textile(force_bytes(value), encoding='utf-8', output='utf-8')))
@register.filter(is_safe=True)
def markdown(value, arg=''):
@@ -80,5 +80,5 @@ def restructuredtext(value):
return force_text(value)
else:
docutils_settings = getattr(settings, "RESTRUCTUREDTEXT_FILTER_SETTINGS", {})
- parts = publish_parts(source=smart_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings)
+ parts = publish_parts(source=force_bytes(value), writer_name="html4css1", settings_overrides=docutils_settings)
return mark_safe(force_text(parts["fragment"]))
View
4 django/contrib/sessions/backends/base.py
@@ -14,7 +14,7 @@
from django.utils.crypto import get_random_string
from django.utils.crypto import salted_hmac
from django.utils import timezone
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
class CreateError(Exception):
"""
@@ -84,7 +84,7 @@ def encode(self, session_dict):
return base64.b64encode(hash.encode() + b":" + pickled).decode('ascii')
def decode(self, session_data):
- encoded_data = base64.b64decode(smart_bytes(session_data))
+ encoded_data = base64.b64decode(force_bytes(session_data))
try:
# could produce ValueError if there is no ':'
hash, pickled = encoded_data.split(b':', 1)
View
2  django/contrib/staticfiles/management/commands/collectstatic.py
@@ -192,7 +192,7 @@ def log(self, msg, level=2):
def clear_dir(self, path):
"""
- Deletes the given relative path using the destinatin storage backend.
+ Deletes the given relative path using the destination storage backend.
"""
dirs, files = self.storage.listdir(path)
for f in files:
View
6 django/contrib/staticfiles/storage.py
@@ -16,7 +16,7 @@
from django.core.files.base import ContentFile
from django.core.files.storage import FileSystemStorage, get_storage_class
from django.utils.datastructures import SortedDict
-from django.utils.encoding import force_text, smart_bytes
+from django.utils.encoding import force_bytes, force_text
from django.utils.functional import LazyObject
from django.utils.importlib import import_module
@@ -118,7 +118,7 @@ def hashed_name(self, name, content=None):
return urlunsplit(unparsed_name)
def cache_key(self, name):
- return 'staticfiles:%s' % hashlib.md5(smart_bytes(name)).hexdigest()
+ return 'staticfiles:%s' % hashlib.md5(force_bytes(name)).hexdigest()
def url(self, name, force=False):
"""
@@ -254,7 +254,7 @@ def post_process(self, paths, dry_run=False, **options):
if hashed_file_exists:
self.delete(hashed_name)
# then save the processed result
- content_file = ContentFile(smart_bytes(content))
+ content_file = ContentFile(force_bytes(content))
saved_name = self._save(hashed_name, content_file)
hashed_name = force_text(saved_name.replace('\\', '/'))
processed = True
View
4 django/core/cache/backends/db.py
@@ -12,7 +12,7 @@
from django.core.cache.backends.base import BaseCache
from django.db import connections, router, transaction, DatabaseError
from django.utils import timezone
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
class Options(object):
@@ -73,7 +73,7 @@ def get(self, key, default=None, version=None):
transaction.commit_unless_managed(using=db)
return default
value = connections[db].ops.process_clob(row[1])
- return pickle.loads(base64.b64decode(smart_bytes(value)))
+ return pickle.loads(base64.b64decode(force_bytes(value)))
def set(self, key, value, timeout=None, version=None):
key = self.make_key(key, version=version)
View
4 django/core/cache/backends/filebased.py
@@ -10,7 +10,7 @@
import pickle
from django.core.cache.backends.base import BaseCache
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
class FileBasedCache(BaseCache):
def __init__(self, dir, params):
@@ -137,7 +137,7 @@ def _key_to_file(self, key):
Thus, a cache key of "foo" gets turnned into a file named
``{cache-dir}ac/bd/18db4cc2f85cedef654fccc4a4d8``.
"""
- path = hashlib.md5(smart_bytes(key)).hexdigest()
+ path = hashlib.md5(force_bytes(key)).hexdigest()
path = os.path.join(path[:2], path[2:4], path[4:])
return os.path.join(self._dir, path)
View
4 django/core/cache/backends/memcached.py
@@ -6,7 +6,7 @@
from django.core.cache.backends.base import BaseCache, InvalidCacheBackendError
from django.utils import six
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
class BaseMemcachedCache(BaseCache):
def __init__(self, server, params, library, value_not_found_exception):
@@ -53,7 +53,7 @@ def _get_memcache_timeout(self, timeout):
def make_key(self, key, version=None):
# Python 2 memcache requires the key to be a byte string.
- return smart_str(super(BaseMemcachedCache, self).make_key(key, version))
+ return force_str(super(BaseMemcachedCache, self).make_key(key, version))
def add(self, key, value, timeout=0, version=None):
key = self.make_key(key, version=version)
View
8 django/core/files/base.py
@@ -1,10 +1,11 @@
from __future__ import unicode_literals
import os
-from io import BytesIO, UnsupportedOperation
+from io import BytesIO, StringIO, UnsupportedOperation
-from django.utils.encoding import smart_bytes, smart_text
+from django.utils.encoding import smart_text
from django.core.files.utils import FileProxyMixin
+from django.utils import six
from django.utils.encoding import python_2_unicode_compatible
@python_2_unicode_compatible
@@ -132,7 +133,8 @@ class ContentFile(File):
"""
def __init__(self, content, name=None):
content = content or b''
- super(ContentFile, self).__init__(BytesIO(content), name=name)
+ stream_class = StringIO if isinstance(content, six.text_type) else BytesIO
+ super(ContentFile, self).__init__(stream_class(content), name=name)
self.size = len(content)
def __str__(self):
View
11 django/core/files/storage.py
@@ -195,11 +195,18 @@ def _save(self, name, content):
fd = os.open(full_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_BINARY', 0))
try:
locks.lock(fd, locks.LOCK_EX)
+ _file = None
for chunk in content.chunks():
- os.write(fd, chunk)
+ if _file is None:
+ mode = 'wb' if isinstance(chunk, bytes) else 'wt'
+ _file = os.fdopen(fd, mode)
+ _file.write(chunk)
finally:
locks.unlock(fd)
- os.close(fd)
+ if _file is not None:
+ _file.close()
+ else:
+ os.close(fd)
except OSError as e:
if e.errno == errno.EEXIST:
# Ooops, the file exists. We need a new file name.
View
4 django/core/files/uploadedfile.py
@@ -8,7 +8,7 @@
from django.conf import settings
from django.core.files.base import File
from django.core.files import temp as tempfile
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
__all__ = ('UploadedFile', 'TemporaryUploadedFile', 'InMemoryUploadedFile',
'SimpleUploadedFile')
@@ -30,7 +30,7 @@ def __init__(self, file=None, name=None, content_type=None, size=None, charset=N
self.charset = charset
def __repr__(self):
- return smart_str("<%s: %s (%s)>" % (
+ return force_str("<%s: %s (%s)>" % (
self.__class__.__name__, self.name, self.content_type))
def _get_name(self):
View
4 django/core/handlers/wsgi.py
@@ -9,7 +9,7 @@
from django.core.handlers import base
from django.core.urlresolvers import set_script_prefix
from django.utils import datastructures
-from django.utils.encoding import force_text, smart_str, iri_to_uri
+from django.utils.encoding import force_str, force_text, iri_to_uri
from django.utils.log import getLogger
logger = getLogger('django.request')
@@ -246,5 +246,5 @@ def __call__(self, environ, start_response):
response_headers = [(str(k), str(v)) for k, v in response.items()]
for c in response.cookies.values():
response_headers.append((str('Set-Cookie'), str(c.output(header=''))))
- start_response(smart_str(status), response_headers)
+ start_response(force_str(status), response_headers)
return response
View
4 django/core/management/base.py
@@ -12,7 +12,7 @@
import django
from django.core.exceptions import ImproperlyConfigured
from django.core.management.color import color_style
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
from django.utils.six import StringIO
@@ -65,7 +65,7 @@ def write(self, msg, style_func=None, ending=None):
msg += ending
style_func = [f for f in (style_func, self.style_func, lambda x:x)
if f is not None][0]
- self._out.write(smart_str(style_func(msg)))
+ self._out.write(force_str(style_func(msg)))
class BaseCommand(object):
View
116 django/core/management/commands/inspectdb.py
@@ -1,4 +1,7 @@
+from __future__ import unicode_literals
+
import keyword
+import re
from optparse import make_option
from django.core.management.base import NoArgsCommand, CommandError
@@ -31,6 +34,7 @@ def handle_inspection(self, options):
table_name_filter = options.get('table_name_filter')
table2model = lambda table_name: table_name.title().replace('_', '').replace(' ', '').replace('-', '')
+ strip_prefix = lambda s: s.startswith("u'") and s[1:] or s
cursor = connection.cursor()
yield "# This is an auto-generated Django model module."
@@ -41,6 +45,7 @@ def handle_inspection(self, options):
yield "#"
yield "# Also note: You'll have to insert the output of 'django-admin.py sqlcustom [appname]'"
yield "# into your database."
+ yield "from __future__ import unicode_literals"
yield ''
yield 'from %s import models' % self.db_module
yield ''
@@ -59,16 +64,19 @@ def handle_inspection(self, options):
indexes = connection.introspection.get_indexes(cursor, table_name)
except NotImplementedError:
indexes = {}
+ used_column_names = [] # Holds column names used in the table so far
for i, row in enumerate(connection.introspection.get_table_description(cursor, table_name)):
- column_name = row[0]
- att_name = column_name.lower()
comment_notes = [] # Holds Field notes, to be displayed in a Python comment.
extra_params = {} # Holds Field parameters such as 'db_column'.
+ column_name = row[0]
+ is_relation = i in relations
+
+ att_name, params, notes = self.normalize_col_name(
+ column_name, used_column_names, is_relation)
+ extra_params.update(params)
+ comment_notes.extend(notes)
- # If the column name can't be used verbatim as a Python
- # attribute, set the "db_column" for this Field.
- if ' ' in att_name or '-' in att_name or keyword.iskeyword(att_name) or column_name != att_name:
- extra_params['db_column'] = column_name
+ used_column_names.append(att_name)
# Add primary_key and unique, if necessary.
if column_name in indexes:
@@ -77,30 +85,12 @@ def handle_inspection(self, options):
elif indexes[column_name]['unique']:
extra_params['unique'] = True
- # Modify the field name to make it Python-compatible.
- if ' ' in att_name:
- att_name = att_name.replace(' ', '_')
- comment_notes.append('Field renamed to remove spaces.')
-
- if '-' in att_name:
- att_name = att_name.replace('-', '_')
- comment_notes.append('Field renamed to remove dashes.')
-
- if column_name != att_name:
- comment_notes.append('Field name made lowercase.')
-
- if i in relations:
+ if is_relation:
rel_to = relations[i][1] == table_name and "'self'" or table2model(relations[i][1])
-
if rel_to in known_models:
field_type = 'ForeignKey(%s' % rel_to
else:
field_type = "ForeignKey('%s'" % rel_to
-
- if att_name.endswith('_id'):
- att_name = att_name[:-3]
- else:
- extra_params['db_column'] = column_name
else:
# Calling `get_field_type` to get the field type string and any
# additional paramters and notes.
@@ -110,16 +100,6 @@ def handle_inspection(self, options):
field_type += '('
- if keyword.iskeyword(att_name):
- att_name += '_field'
- comment_notes.append('Field renamed because it was a Python reserved word.')
-
- if att_name[0].isdigit():
- att_name = 'number_%s' % att_name
- extra_params['db_column'] = six.text_type(column_name)
- comment_notes.append("Field renamed because it wasn't a "
- "valid Python identifier.")
-
# Don't output 'id = meta.AutoField(primary_key=True)', because
# that's assumed if it doesn't exist.
if att_name == 'id' and field_type == 'AutoField(' and extra_params == {'primary_key': True}:
@@ -136,7 +116,9 @@ def handle_inspection(self, options):
if extra_params:
if not field_desc.endswith('('):
field_desc += ', '
- field_desc += ', '.join(['%s=%r' % (k, v) for k, v in extra_params.items()])
+ field_desc += ', '.join([
+ '%s=%s' % (k, strip_prefix(repr(v)))
+ for k, v in extra_params.items()])
field_desc += ')'
if comment_notes:
field_desc += ' # ' + ' '.join(comment_notes)
@@ -144,6 +126,62 @@ def handle_inspection(self, options):
for meta_line in self.get_meta(table_name):
yield meta_line
+ def normalize_col_name(self, col_name, used_column_names, is_relation):
+ """
+ Modify the column name to make it Python-compatible as a field name
+ """
+ field_params = {}
+ field_notes = []
+
+ new_name = col_name.lower()
+ if new_name != col_name:
+ field_notes.append('Field name made lowercase.')
+
+ if is_relation:
+ if new_name.endswith('_id'):
+ new_name = new_name[:-3]
+ else:
+ field_params['db_column'] = col_name
+
+ new_name, num_repl = re.subn(r'\W', '_', new_name)
+ if num_repl > 0:
+ field_notes.append('Field renamed to remove unsuitable characters.')
+
+ if new_name.find('__') >= 0:
+ while new_name.find('__') >= 0:
+ new_name = new_name.replace('__', '_')
+ if col_name.lower().find('__') >= 0:
+ # Only add the comment if the double underscore was in the original name
+ field_notes.append("Field renamed because it contained more than one '_' in a row.")
+
+ if new_name.startswith('_'):
+ new_name = 'field%s' % new_name
+ field_notes.append("Field renamed because it started with '_'.")
+
+ if new_name.endswith('_'):
+ new_name = '%sfield' % new_name
+ field_notes.append("Field renamed because it ended with '_'.")
+
+ if keyword.iskeyword(new_name):
+ new_name += '_field'
+ field_notes.append('Field renamed because it was a Python reserved word.')
+
+ if new_name[0].isdigit():
+ new_name = 'number_%s' % new_name
+ field_notes.append("Field renamed because it wasn't a valid Python identifier.")
+
+ if new_name in used_column_names:
+ num = 0
+ while '%s_%d' % (new_name, num) in used_column_names:
+ num += 1
+ new_name = '%s_%d' % (new_name, num)
+ field_notes.append('Field renamed because of name conflict.')
+
+ if col_name != new_name and field_notes:
+ field_params['db_column'] = col_name
+
+ return new_name, field_params, field_notes
+
def get_field_type(self, connection, table_name, row):
"""
Given the database connection, the table name, and the cursor row
@@ -181,6 +219,6 @@ def get_meta(self, table_name):
to construct the inner Meta class for the model corresponding
to the given database table name.
"""
- return [' class Meta:',
- ' db_table = %r' % table_name,
- '']
+ return [" class Meta:",
+ " db_table = '%s'" % table_name,
+ ""]
View
12 django/core/management/commands/loaddata.py
@@ -196,6 +196,10 @@ def read(self):
loaded_object_count += loaded_objects_in_fixture
fixture_object_count += objects_in_fixture
label_found = True
+ except Exception as e:
+ if not isinstance(e, CommandError):
+ e.args = ("Problem installing fixture '%s': %s" % (full_path, e),)
+ raise
finally:
fixture.close()
@@ -209,7 +213,11 @@ def read(self):
# Since we disabled constraint checks, we must manually check for
# any invalid keys that might have been added
table_names = [model._meta.db_table for model in models]
- connection.check_constraints(table_names=table_names)
+ try:
+ connection.check_constraints(table_names=table_names)
+ except Exception as e:
+ e.args = ("Problem installing fixtures: %s" % e,)
+ raise
except (SystemExit, KeyboardInterrupt):
raise
@@ -217,8 +225,6 @@ def read(self):
if commit:
transaction.rollback(using=using)
transaction.leave_transaction_management(using=using)
- if not isinstance(e, CommandError):
- e.args = ("Problem installing fixture '%s': %s" % (full_path, e),)
raise
# If we found even one object in a fixture, we need to reset the
View
53 django/core/management/commands/makemessages.py
@@ -47,31 +47,27 @@ def _popen(cmd):
output, errors = p.communicate()
return output, errors, p.returncode
-def walk(root, topdown=True, onerror=None, followlinks=False,
- ignore_patterns=None, verbosity=0, stdout=sys.stdout):
+def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False):
"""
- A version of os.walk that can follow symlinks for Python < 2.6
+ Helper function to get all files in the given root.
"""
- if ignore_patterns is None:
- ignore_patterns = []
dir_suffix = '%s*' % os.sep
norm_patterns = [p[:-len(dir_suffix)] if p.endswith(dir_suffix) else p for p in ignore_patterns]
- for dirpath, dirnames, filenames in os.walk(root, topdown, onerror):
- remove_dirs = []
- for dirname in dirnames:
+ all_files = []
+ for dirpath, dirnames, filenames in os.walk(root, topdown=True, followlinks=symlinks):
+ for dirname in dirnames[:]:
if is_ignored(os.path.normpath(os.path.join(dirpath, dirname)), norm_patterns):
- remove_dirs.append(dirname)
- for dirname in remove_dirs:
- dirnames.remove(dirname)
- if verbosity > 1:
- stdout.write('ignoring directory %s\n' % dirname)
- yield (dirpath, dirnames, filenames)
- if followlinks:
- for d in dirnames:
- p = os.path.join(dirpath, d)
- if os.path.islink(p):
- for link_dirpath, link_dirnames, link_filenames in walk(p):
- yield (link_dirpath, link_dirnames, link_filenames)
+ dirnames.remove(dirname)
+ if verbosity > 1:
+ stdout.write('ignoring directory %s\n' % dirname)
+ for filename in filenames:
+ if is_ignored(os.path.normpath(os.path.join(dirpath, filename)), ignore_patterns):
+ if verbosity > 1:
+ stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
+ else:
+ all_files.extend([(dirpath, filename)])
+ all_files.sort()
+ return all_files
def is_ignored(path, ignore_patterns):
"""
@@ -82,23 +78,6 @@ def is_ignored(path, ignore_patterns):
return True
return False
-def find_files(root, ignore_patterns, verbosity, stdout=sys.stdout, symlinks=False):
- """
- Helper function to get all files in the given root.
- """
- all_files = []
- for (dirpath, dirnames, filenames) in walk(root, followlinks=symlinks,
- ignore_patterns=ignore_patterns, verbosity=verbosity, stdout=stdout):
- for filename in filenames:
- norm_filepath = os.path.normpath(os.path.join(dirpath, filename))
- if is_ignored(norm_filepath, ignore_patterns):
- if verbosity > 1:
- stdout.write('ignoring file %s in %s\n' % (filename, dirpath))
- else:
- all_files.extend([(dirpath, filename)])
- all_files.sort()
- return all_files
-
def copy_plural_forms(msgs, locale, domain, verbosity, stdout=sys.stdout):
"""
Copies plural forms header contents from a Django catalog of locale to
View
18 django/core/management/commands/shell.py
@@ -80,14 +80,14 @@ def handle_noargs(self, **options):
readline.parse_and_bind("tab:complete")
# We want to honor both $PYTHONSTARTUP and .pythonrc.py, so follow system
- # conventions and get $PYTHONSTARTUP first then import user.
+ # conventions and get $PYTHONSTARTUP first then .pythonrc.py.
if not use_plain:
- pythonrc = os.environ.get("PYTHONSTARTUP")
- if pythonrc and os.path.isfile(pythonrc):
- try:
- execfile(pythonrc)
- except NameError:
- pass
- # This will import .pythonrc.py as a side-effect
- import user
+ for pythonrc in (os.environ.get("PYTHONSTARTUP"),
+ os.path.expanduser('~/.pythonrc.py')):
+ if pythonrc and os.path.isfile(pythonrc):
+ try:
+ with open(pythonrc) as handle:
+ exec(compile(handle.read(), pythonrc, 'exec'))
+ except NameError:
+ pass
code.interact(local=imported_objects)
View
6 django/core/management/templates.py
@@ -8,6 +8,8 @@
import stat
import sys
import tempfile
+import codecs
+
try:
from urllib.request import urlretrieve
except ImportError: # Python 2
@@ -154,12 +156,12 @@ def handle(self, app_or_project, name, target=None, **options):
# Only render the Python files, as we don't want to
# accidentally render Django templates files
- with open(old_path, 'r') as template_file:
+ with codecs.open(old_path, 'r', 'utf-8') as template_file:
content = template_file.read()
if filename.endswith(extensions) or filename in extra_files:
template = Template(content)
content = template.render(context)
- with open(new_path, 'w') as new_file:
+ with codecs.open(new_path, 'w', 'utf-8') as new_file:
new_file.write(content)
if self.verbosity >= 2:
View
4 django/core/management/validation.py
@@ -1,7 +1,7 @@
import sys
from django.core.management.color import color_style
-from django.utils.encoding import smart_str
+from django.utils.encoding import force_str
from django.utils.itercompat import is_iterable
from django.utils import six
@@ -14,7 +14,7 @@ def __init__(self, outfile=sys.stdout):
def add(self, context, error):
self.errors.append((context, error))
- self.outfile.write(self.style.ERROR(smart_str("%s: %s\n" % (context, error))))
+ self.outfile.write(self.style.ERROR(force_str("%s: %s\n" % (context, error))))
def get_validation_errors(outfile, app=None):
View
1  django/core/serializers/json.py
@@ -12,7 +12,6 @@
from django.core.serializers.base import DeserializationError
from django.core.serializers.python import Serializer as PythonSerializer
from django.core.serializers.python import Deserializer as PythonDeserializer
-from django.utils.encoding import smart_bytes
from django.utils import six
from django.utils.timezone import is_aware
View
1  django/core/serializers/pyyaml.py
@@ -12,7 +12,6 @@
from django.core.serializers.base import DeserializationError
from django.core.serializers.python import Serializer as PythonSerializer
from django.core.serializers.python import Deserializer as PythonDeserializer
-from django.utils.encoding import smart_bytes
from django.utils import six
View
48 django/core/signing.py
@@ -32,6 +32,7 @@
There are 65 url-safe characters: the 64 used by url-safe base64 and the ':'.
These functions make use of all of them.
"""
+
from __future__ import unicode_literals
import base64
@@ -43,7 +44,7 @@
from django.core.exceptions import ImproperlyConfigured
from django.utils import baseconv
from django.utils.crypto import constant_time_compare, salted_hmac
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes, force_str, force_text
from django.utils.importlib import import_module
@@ -62,12 +63,12 @@ class SignatureExpired(BadSignature):
def b64_encode(s):
- return base64.urlsafe_b64encode(smart_bytes(s)).decode('ascii').strip('=')
+ return base64.urlsafe_b64encode(s).strip(b'=')
def b64_decode(s):
- pad = '=' * (-len(s) % 4)
- return base64.urlsafe_b64decode(smart_bytes(s + pad)).decode('ascii')
+ pad = b'=' * (-len(s) % 4)
+ return base64.urlsafe_b64decode(s + pad)
def base64_hmac(salt, value, key):
@@ -116,20 +117,20 @@ def dumps(obj, key=None, salt='django.core.signing', serializer=JSONSerializer,
value or re-using a salt value across different parts of your
application without good cause is a security risk.
"""
- data = serializer().dumps(obj)
+ data = force_bytes(serializer().dumps(obj))
# Flag for if it's been compressed or not
is_compressed = False
if compress:
# Avoid zlib dependency unless compress is being used
- compressed = zlib.compress(smart_bytes(data))
+ compressed = zlib.compress(data)
if len(compressed) < (len(data) - 1):
data = compressed
is_compressed = True
base64d = b64_encode(data)
if is_compressed:
- base64d = '.' + base64d
+ base64d = b'.' + base64d
return TimestampSigner(key, salt=salt).sign(base64d)
@@ -137,37 +138,45 @@ def loads(s, key=None, salt='django.core.signing', serializer=JSONSerializer, ma
"""
Reverse of dumps(), raises BadSignature if signature fails
"""
- base64d = TimestampSigner(key, salt=salt).unsign(s, max_age=max_age)
+ # TimestampSigner.unsign always returns unicode but base64 and zlib
+ # compression operate on bytes.
+ base64d = force_bytes(TimestampSigner(key, salt=salt).unsign(s, max_age=max_age))
decompress = False
- if base64d[0] == '.':
+ if base64d[0] == b'.':
# It's compressed; uncompress it first
base64d = base64d[1:]
decompress = True
data = b64_decode(base64d)
if decompress:
data = zlib.decompress(data)
- return serializer().loads(data)
+ return serializer().loads(force_str(data))
class Signer(object):
+
def __init__(self, key=None, sep=':', salt=None):
- self.sep = sep
- self.key = key or settings.SECRET_KEY
- self.salt = salt or ('%s.%s' %
- (self.__class__.__module__, self.__class__.__name__))
+ # Use of native strings in all versions of Python
+ self.sep = str(sep)
+ self.key = str(key or settings.SECRET_KEY)
+ self.salt = str(salt or
+ '%s.%s' % (self.__class__.__module__, self.__class__.__name__))
def signature(self, value):
- return base64_hmac(self.salt + 'signer', value, self.key)
+ signature = base64_hmac(self.salt + 'signer', value, self.key)
+ # Convert the signature from bytes to str only on Python 3
+ return force_str(signature)
def sign(self, value):
- return '%s%s%s' % (value, self.sep, self.signature(value))
+ value = force_str(value)
+ return str('%s%s%s') % (value, self.sep, self.signature(value))
def unsign(self, signed_value):
+ signed_value = force_str(signed_value)
if not self.sep in signed_value:
raise BadSignature('No "%s" found in value' % self.sep)
value, sig = signed_value.rsplit(self.sep, 1)
if constant_time_compare(sig, self.signature(value)):
- return value
+ return force_text(value)
raise BadSignature('Signature "%s" does not match' % sig)
@@ -177,8 +186,9 @@ def timestamp(self):
return baseconv.base62.encode(int(time.time()))
def sign(self, value):
- value = '%s%s%s' % (value, self.sep, self.timestamp())
- return '%s%s%s' % (value, self.sep, self.signature(value))
+ value = force_str(value)
+ value = str('%s%s%s') % (value, self.sep, self.timestamp())
+ return super(TimestampSigner, self).sign(value)
def unsign(self, value, max_age=None):
result = super(TimestampSigner, self).unsign(value)
View
13 django/core/urlresolvers.py
@@ -14,7 +14,7 @@
from django.http import Http404
from django.core.exceptions import ImproperlyConfigured, ViewDoesNotExist
from django.utils.datastructures import MultiValueDict
-from django.utils.encoding import iri_to_uri, force_text, smart_str
+from django.utils.encoding import force_str, force_text, iri_to_uri
from django.utils.functional import memoize, lazy
from django.utils.importlib import import_module
from django.utils.module_loading import module_has_submodule
@@ -195,7 +195,7 @@ def __init__(self, regex, callback, default_args=None, name=None):
self.name = name
def __repr__(self):
- return smart_str('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))
+ return force_str('<%s %s %s>' % (self.__class__.__name__, self.name, self.regex.pattern))
def add_prefix(self, prefix):
"""
@@ -245,8 +245,13 @@ def __init__(self, regex, urlconf_name, default_kwargs=None, app_name=None, name
self._app_dict = {}
def __repr__(self):
- return smart_str('<%s %s (%s:%s) %s>' % (
- self.__class__.__name__, self.urlconf_name, self.app_name,
+ if isinstance(self.urlconf_name, list) and len(self.urlconf_name):
+ # Don't bother to output the whole list, it can be huge
+ urlconf_repr = '<%s list>' % self.urlconf_name[0].__class__.__name__
+ else:
+ urlconf_repr = repr(self.urlconf_name)
+ return force_str('<%s %s (%s:%s) %s>' % (
+ self.__class__.__name__, urlconf_repr, self.app_name,
self.namespace, self.regex.pattern))
def _populate(self):
View
6 django/core/validators.py
@@ -8,7 +8,7 @@
from django.core.exceptions import ValidationError
from django.utils.translation import ugettext_lazy as _
-from django.utils.encoding import smart_text
+from django.utils.encoding import force_text
from django.utils.ipv6 import is_valid_ipv6_address
from django.utils import six
@@ -37,7 +37,7 @@ def __call__(self, value):
"""
Validates that the input matches the regular expression.
"""
- if not self.regex.search(smart_text(value)):
+ if not self.regex.search(force_text(value)):
raise ValidationError(self.message, code=self.code)
@@ -57,7 +57,7 @@ def __call__(self, value):
except ValidationError as e:
# Trivial case failed. Try for possible IDN domain
if value:
- value = smart_text(value)
+ value = force_text(value)
scheme, netloc, path, query, fragment = urlsplit(value)
try:
netloc = netloc.encode('idna').decode('ascii') # IDN -> ACE
View
8 django/db/backends/__init__.py
@@ -609,7 +609,7 @@ def last_executed_query(self, cursor, sql, params):
exists for database backends to provide a better implementation
according to their own quoting schemes.
"""
- from django.utils.encoding import smart_text, force_text
+ from django.utils.encoding import force_text
# Convert params to contain Unicode values.
to_unicode = lambda s: force_text(s, strings_only=True, errors='replace')
@@ -618,7 +618,7 @@ def last_executed_query(self, cursor, sql, params):
else:
u_params = dict([(to_unicode(k), to_unicode(v)) for k, v in params.items()])
- return smart_text(sql) % u_params
+ return force_text(sql) % u_params
def last_insert_id(self, cursor, table_name, pk_name):
"""
@@ -802,8 +802,8 @@ def tablespace_sql(self, tablespace, inline=False):
def prep_for_like_query(self, x):
"""Prepares a value for use in a LIKE query."""
- from django.utils.encoding import smart_text
- return smart_text(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")
+ from django.utils.encoding import force_text
+ return force_text(x).replace("\\", "\\\\").replace("%", "\%").replace("_", "\_")
# Same as prep_for_like_query(), but called for "iexact" matches, which
# need not necessarily be implemented using "LIKE" in the backend.
View
8 django/db/backends/mysql/compiler.py
@@ -1,10 +1,16 @@
+try:
+ from itertools import zip_longest
+except ImportError:
+ from itertools import izip_longest as zip_longest
+
from django.db.models.sql import compiler
+
class SQLCompiler(compiler.SQLCompiler):
def resolve_columns(self, row, fields=()):
values = []
index_extra_select = len(self.query.extra_select)
- for value, field in map(None, row[index_extra_select:], fields):
+ for value, field in zip_longest(row[index_extra_select:], fields):
if (field and field.get_internal_type() in ("BooleanField", "NullBooleanField") and
value in (0, 1)):
value = bool(value)
View
22 django/db/backends/mysql/introspection.py
@@ -1,8 +1,9 @@
+import re
+from .base import FIELD_TYPE
+
from django.db.backends import BaseDatabaseIntrospection
from django.utils import six
-from MySQLdb import ProgrammingError, OperationalError
-from MySQLdb.constants import FIELD_TYPE
-import re
+
foreign_key_re = re.compile(r"\sCONSTRAINT `[^`]*` FOREIGN KEY \(`([^`]*)`\) REFERENCES `([^`]*)` \(`([^`]*)`\)")
@@ -35,9 +36,20 @@ def get_table_list(self, cursor):
return [row[0] for row in cursor.fetchall()]
def get_table_description(self, cursor, table_name):
- "Returns a description of the table, with the DB-API cursor.description interface."
+ """
+ Returns a description of the table, with the DB-API cursor.description interface."
+ """
+ # varchar length returned by cursor.description is an internal length,
+ # not visible length (#5725), use information_schema database to fix this
+ cursor.execute("""
+ SELECT column_name, character_maximum_length FROM information_schema.columns
+ WHERE table_name = %s AND table_schema = DATABASE()
+ AND character_maximum_length IS NOT NULL""", [table_name])
+ length_map = dict(cursor.fetchall())
+
cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
- return cursor.description
+ return [line[:3] + (length_map.get(line[0], line[3]),) + line[4:]
+ for line in cursor.description]
def _name_to_index(self, cursor, table_name):
"""
View
12 django/db/backends/oracle/base.py
@@ -10,8 +10,6 @@
import sys
import warnings
-from django.utils import six
-
def _setup_environment(environ):
import platform
# Cygwin requires some special voodoo to set the environment variables
@@ -53,7 +51,7 @@ def _setup_environment(environ):
from django.db.backends.oracle.client import DatabaseClient
from django.db.backends.oracle.creation import DatabaseCreation
from django.db.backends.oracle.introspection import DatabaseIntrospection
-from django.utils.encoding import smart_bytes, force_text
+from django.utils.encoding import force_bytes, force_text
from django.utils import six
from django.utils import timezone
@@ -66,7 +64,7 @@ def _setup_environment(environ):
if int(Database.version.split('.', 1)[0]) >= 5 and not hasattr(Database, 'UNICODE'):
convert_unicode = force_text
else:
- convert_unicode = smart_bytes
+ convert_unicode = force_bytes
class DatabaseFeatures(BaseDatabaseFeatures):
@@ -604,9 +602,9 @@ def __init__(self, param, cursor, strings_only=False):
elif param is False:
param = "0"
if hasattr(param, 'bind_parameter'):
- self.smart_bytes = param.bind_parameter(cursor)
+ self.force_bytes = param.bind_parameter(cursor)
else:
- self.smart_bytes = convert_unicode(param, cursor.charset,
+ self.force_bytes = convert_unicode(param, cursor.charset,
strings_only)
if hasattr(param, 'input_size'):
# If parameter has `input_size` attribute, use that.
@@ -685,7 +683,7 @@ def _guess_input_sizes(self, params_list):
self.setinputsizes(*sizes)
def _param_generator(self, params):
- return [p.smart_bytes for p in params]
+ return [p.force_bytes for p in params]
def execute(self, query, params=None):
if params is None:
View
4 django/db/backends/postgresql_psycopg2/introspection.py
@@ -45,8 +45,8 @@ def get_table_description(self, cursor, table_name):
WHERE table_name = %s""", [table_name])
null_map = dict(cursor.fetchall())
cursor.execute("SELECT * FROM %s LIMIT 1" % self.connection.ops.quote_name(table_name))
- return [tuple([item for item in line[:6]] + [null_map[line[0]]=='YES'])
- for line in cursor.description]
+ return [line[:6] + (null_map[line[0]]=='YES',)
+ for line in cursor.description]
def get_relations(self, cursor, table_name):
"""
View
18 django/db/backends/sqlite3/introspection.py
@@ -1,6 +1,14 @@
import re
from django.db.backends import BaseDatabaseIntrospection
+field_size_re = re.compile(r'^\s*(?:var)?char\s*\(\s*(\d+)\s*\)\s*$')
+
+def get_field_size(name):
+ """ Extract the size number from a "varchar(11)" type name """
+ m = field_size_re.search(name)
+ return int(m.group(1)) if m else None
+
+
# This light wrapper "fakes" a dictionary interface, because some SQLite data
# types include variables in them -- e.g. "varchar(30)" -- and can't be matched
# as a simple dictionary lookup.
@@ -32,10 +40,9 @@ def __getitem__(self, key):
try:
return self.base_data_types_reverse[key]
except KeyError:
- import re
- m = re.search(r'^\s*(?:var)?char\s*\(\s*(\d+)\s*\)\s*$', key)
- if m:
- return ('CharField', {'max_length': int(m.group(1))})
+ size = get_field_size(key)
+ if size is not None:
+ return ('CharField', {'max_length': size})
raise KeyError
class DatabaseIntrospection(BaseDatabaseIntrospection):
@@ -53,7 +60,7 @@ def get_table_list(self, cursor):
def get_table_description(self, cursor, table_name):
"Returns a description of the table, with the DB-API cursor.description interface."
- return [(info['name'], info['type'], None, None, None, None,
+ return [(info['name'], info['type'], None, info['size'], None, None,
info['null_ok']) for info in self._table_info(cursor, table_name)]
def get_relations(self, cursor, table_name):
@@ -171,6 +178,7 @@ def _table_info(self, cursor, name):
# cid, name, type, notnull, dflt_value, pk
return [{'name': field[1],
'type': field[2],
+ 'size': get_field_size(field[2]),
'null_ok': not field[3],
'pk': field[5] # undocumented
} for field in cursor.fetchall()]
View
4 django/db/backends/util.py
@@ -6,7 +6,7 @@
from time import time
from django.conf import settings
-from django.utils.encoding import smart_bytes
+from django.utils.encoding import force_bytes
from django.utils.log import getLogger
from django.utils.timezone import utc
@@ -138,7 +138,7 @@ def truncate_name(name, length=None, hash_len=4):
if length is None or len(name) <= length:
return name
- hsh = hashlib.md5(smart_bytes(name)).hexdigest()[:hash_len]
+ hsh = hashlib.md5(force_bytes(name)).hexdigest()[:hash_len]
return '%s%s' % (name[:length-hash_len], hsh)
def format_number(value, max_digits, decimal_places):
View
4 django/db/models/base.py
@@ -23,7 +23,7 @@
from django.db.models.loading import register_models, get_model
from django.utils.translation import ugettext_lazy as _
from django.utils.functional import curry
-from django.utils.encoding import smart_str, force_text
+from django.utils.encoding import force_str, force_text
from django.utils import six
from django.utils.text import get_text_list, capfirst
@@ -407,7 +407,7 @@ def __repr__(self):
u = six.text_type(self)
except (UnicodeEncodeError, UnicodeDecodeError):
u = '[Bad Unicode data]'
- return smart_str('<%s: %s>' % (self.__class__.__name__, u))
+ return force_str('<%s: %s>' % (self.__class__.__name__, u))
def __str__(self):
if not six.PY3 and hasattr(self, '__unicode__'):
View
5 django/db/models/fields/__init__.py
@@ -1047,13 +1047,14 @@ class GenericIPAddressField(Field):
description = _("IP address")
default_error_messages = {}
- def __init__(self, protocol='both', unpack_ipv4=False, *args, **kwargs):
+ def __init__(self, verbose_name=None, name=None, protocol='both',
+ unpack_ipv4=False, *args, **kwargs):
self.unpack_ipv4 = unpack_ipv4
self.default_validators, invalid_error_message = \
validators.ip_address_validators(protocol, unpack_ipv4)
self.default_error_messages['invalid'] = invalid_error_message
kwargs['max_length'] = 39
- Field.__init__(self, *args, **kwargs)
+ Field.__init__(self, verbose_name, name, *args, **kwargs)
def get_internal_type(self):
return "GenericIPAddressField"
View
4 django/db/models/fields/files.py
@@ -8,7 +8,7 @@
from django.core.files.storage import default_storage
from django.core.files.images import ImageFile
from django.db.models import signals
-from django.utils.encoding import force_text, smart_str
+from django.utils.encoding import force_str, force_text
from django.utils import six
from django.utils.translation import ugettext_lazy as _
@@ -280,7 +280,7 @@ def contribute_to_class(self, cls, name):
setattr(cls, self.name, self.descriptor_class(self))
def get_directory_name(self):
- return os.path.normpath(force_text(datetime.datetime.now().strftime(smart_str(self.upload_to))))
+ return os.path.normpath(force_text(datetime.datetime.now().strftime(force_str(self.upload_to))))
def get_filename(self, filename):
return os.path.normpath(self.storage.get_valid_name(os.path.basename(filename)))
View
4 django/db/models/query.py
@@ -498,9 +498,7 @@ def in_bulk(self, id_list):
"Cannot use 'limit' or 'offset' with in_bulk"
if not id_list:
return {}
- qs = self._clone()
- qs.query.add_filter(('pk__in', id_list))
- qs.query.clear_ordering(force_empty=True)
+ qs = self.filter(pk__in=id_list).order_by()
return dict([(obj._get_pk_val(), obj) for obj in qs])
def delete(self):
View
12 django/db/models/sql/compiler.py
@@ -470,9 +470,7 @@ def _setup_joins(self, pieces, opts, alias):
# Must use left outer joins for nullable fields and their relations.
# Ordering or distinct must not affect the returned set, and INNER
# JOINS for nullable fields could do this.
- if joins_to_promote:
- self.query.promote_alias_chain(joins_to_promote,
- self.query.alias_map[joins_to_promote[0]].join_type == self.query.LOUTER)
+ self.query.promote_joins(joins_to_promote)
return field, col, alias, joins, opts
def _final_join_removal(self, col, alias):
@@ -645,8 +643,6 @@ def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
alias_chain.append(alias)
for (dupe_opts, dupe_col) in dupe_set:
self.query.update_dupe_avoidance(dupe_opts, dupe_col, alias)
- if self.query.alias_map[root_alias].join_type == self.query.LOUTER:
- self.query.promote_alias_chain(alias_chain, True)
else:
alias = root_alias
@@ -663,8 +659,6 @@ def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
columns, aliases = self.get_default_columns(start_alias=alias,
opts=f.rel.to._meta, as_pairs=True)
self.query.related_select_cols.extend(columns)
- if self.query.alias_map[alias].join_type == self.query.LOUTER:
- self.query.promote_alias_chain(aliases, True)
self.query.related_select_fields.extend(f.rel.to._meta.fields)
if restricted:
next = requested.get(f.name, {})
@@ -738,7 +732,9 @@ def fill_related_selections(self, opts=None, root_alias=None, cur_depth=1,
self.query.related_select_fields.extend(model._meta.fields)
next = requested.get(f.related_query_name(), {})
- new_nullable = f.null or None
+ # Use True here because we are looking at the _reverse_ side of
+ # the relation, which is always nullable.
+ new_nullable = True
self.fill_related_selections(model._meta, table, cur_depth+1,
used, next, restricted, new_nullable)
View
2  django/db/models/sql/constants.py
@@ -1,7 +1,7 @@
from collections import namedtuple
import re
-# Valid query types (a dictionary is used for speedy lookups).
+# Valid query types (a set is used for speedy lookups).
QUERY_TERMS = set([
'exact', 'iexact', 'contains', 'icontains', 'gt', 'gte', 'lt', 'lte', 'in',
'startswith', 'istartswith', 'endswith', 'iendswith', 'range', 'year',
View
104 django/db/models/sql/query.py
@@ -505,7 +505,7 @@ def combine(self, rhs, connector):
# Again, some of the tables won't have aliases due to
# the trimming of unnecessary tables.
if self.alias_refcount.get(alias) or rhs.alias_refcount.get(alias):
- self.promote_alias(alias, True)
+ self.promote_joins([alias], True)
# Now relabel a copy of the rhs where-clause and add it to the current
# one.
@@ -682,32 +682,38 @@ def unref_alias(self, alias, amount=1):
""" Decreases the reference count for this alias. """
self.alias_refcount[alias] -= amount
- def promote_alias(self, alias, unconditional=False):
- """
- Promotes the join type of an alias to an outer join if it's possible
- for the join to contain NULL values on the left. If 'unconditional' is
- False, the join is only promoted if it is nullable, otherwise it is
- always promoted.
-
- Returns True if the join was promoted by this call.
- """
- if ((unconditional or self.alias_map[alias].nullable) and
- self.alias_map[alias].join_type != self.LOUTER):
- data = self.alias_map[alias]
- data = data._replace(join_type=self.LOUTER)
- self.alias_map[alias] = data
- return True
- return False
-
- def promote_alias_chain(self, chain, must_promote=False):
- """
- Walks along a chain of aliases, promoting the first nullable join and
- any joins following that. If 'must_promote' is True, all the aliases in
- the chain are promoted.
- """
- for alias in chain:
- if self.promote_alias(alias, must_promote):
- must_promote = True
+ def promote_joins(self, aliases, unconditional=False):
+ """
+ Promotes recursively the join type of given aliases and its children to
+ an outer join. If 'unconditional' is False, the join is only promoted if
+ it is nullable or the parent join is an outer join.
+
+ Note about join promotion: When promoting any alias, we make sure all
+ joins which start from that alias are promoted, too. When adding a join
+ in join(), we make sure any join added to already existing LOUTER join
+ is generated as LOUTER. This ensures we don't ever have broken join
+ chains which contain first a LOUTER join, then an INNER JOIN, that is
+ this kind of join should never be generated: a LOUTER b INNER c. The
+ reason for avoiding this type of join chain is that the INNER after
+ the LOUTER will effectively remove any effect the LOUTER had.
+ """
+ aliases = list(aliases)
+ while aliases:
+ alias = aliases.pop(0)
+ parent_alias = self.alias_map[alias].lhs_alias
+ parent_louter = (parent_alias
+ and self.alias_map[parent_alias].join_type == self.LOUTER)
+ already_louter = self.alias_map[alias].join_type == self.LOUTER
+ if ((unconditional or self.alias_map[alias].nullable
+ or parent_louter) and not already_louter):
+ data = self.alias_map[alias]._replace(join_type=self.LOUTER)
+ self.alias_map[alias] = data
+ # Join type of 'alias' changed, so re-examine all aliases that
+ # refer to this one.
+ aliases.extend(
+ join for join in self.alias_map.keys()
+ if (self.alias_map[join].lhs_alias == alias
+ and join not in aliases))
def reset_refcounts(self, to_counts):
"""
@@ -726,19 +732,10 @@ def promote_unused_aliases(self, initial_refcounts, used_aliases):
then and which ones haven't been used and promotes all of those
aliases, plus any children of theirs in the alias tree, to outer joins.
"""
- # FIXME: There's some (a lot of!) overlap with the similar OR promotion
- # in add_filter(). It's not quite identical, but is very similar. So
- # pulling out the common bits is something for later.
- considered = {}
for alias in self.tables:
- if alias not in used_aliases:
- continue
- if (alias not in initial_refcounts or
+ if alias in used_aliases and (alias not in initial_refcounts or
self.alias_refcount[alias] == initial_refcounts[alias]):
- parent = self.alias_map[alias].lhs_alias
- must_promote = considered.get(parent, False)
- promoted = self.promote_alias(alias, must_promote)
- considered[alias] = must_promote or promoted
+ self.promote_joins([alias])
def change_aliases(self, change_map):
"""
@@ -875,6 +872,9 @@ def join(self, connection, always_create=False, exclusions=(),
LOUTER join type. This is used when joining certain types of querysets
and Q-objects together.
+ A join is always created as LOUTER if the lhs alias is LOUTER to make
+ sure we do not generate chains like a LOUTER b INNER c.
+
If 'nullable' is True, the join can potentially involve NULL values and
is a candidate for promotion (to "left outer") when combining querysets.
"""
@@ -900,8 +900,8 @@ def join(self, connection, always_create=False, exclusions=(),
if self.alias_map[alias].lhs_alias != lhs:
continue
self.ref_alias(alias)
- if promote:
- self.promote_alias(alias)
+ if promote or (lhs and self.alias_map[lhs].join_type == self.LOUTER):
+ self.promote_joins([alias])
return alias
# No reuse is possible, so we need a new alias.
@@ -910,7 +910,12 @@ def join(self, connection, always_create=False, exclusions=(),
# Not all tables need to be joined to anything. No join type
# means the later columns are ignored.
join_type = None
- elif promote or outer_if_first:
+ elif (promote or outer_if_first