Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP

Loading…

Fixed #8149 -- Use universal line terminators when iterating `File` obje... #323

Closed
wants to merge 4 commits into from

5 participants

Tai Lee Tim Graham Alex Gaynor Ramiro Morales Aymeric Augustin
Tai Lee

...cts line-by-line.

Tim Graham
Owner

The test doesn't pass on Python 3.

Tim Graham
Owner

Please open a new PR if you are able to update this, thanks!

Tim Graham timgraham closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 10, 2013
  1. Alex Gaynor

    Merge pull request #892 from JonLoy/ticket_20018

    alex authored
    Fixed #20018: Added backtick to fix reference
  2. Ramiro Morales
  3. Aymeric Augustin

    Fixed #20019 -- Ensured HttpRequest.resolver_match always exists.

    aaugustin authored
    Obviously it isn't set until the URL is resolved.
Commits on Mar 11, 2013
  1. Fixed #8149 -- Use universal line terminators when iterating `File` o…

    Tai Lee authored
    …bjects line-by-line.
This page is out of date. Refresh to see the latest.
4 django/core/files/base.py
View
@@ -94,9 +94,7 @@ def __iter__(self):
# Iterate over this file-like object by newlines
buffer_ = None
for chunk in self.chunks():
- chunk_buffer = BytesIO(chunk)
-
- for line in chunk_buffer:
+ for line in chunk.splitlines(True):
if buffer_:
line = buffer_ + line
buffer_ = None
7 django/core/management/commands/compilemessages.py
View
@@ -5,7 +5,7 @@
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
-from django.core.management.utils import popen_wrapper
+from django.core.management.utils import find_command, popen_wrapper
from django.utils._os import npath
def has_bom(fn):
@@ -16,6 +16,10 @@ def has_bom(fn):
sample.startswith(codecs.BOM_UTF16_BE)
def compile_messages(stderr, locale=None):
+ program = 'msgfmt'
+ if find_command(program) is None:
+ raise CommandError("Can't find %s. Make sure you have GNU gettext tools 0.15 or newer installed." % program)
+
basedirs = [os.path.join('conf', 'locale'), 'locale']
if os.environ.get('DJANGO_SETTINGS_MODULE'):
from django.conf import settings
@@ -42,7 +46,6 @@ def compile_messages(stderr, locale=None):
if has_bom(fn):
raise CommandError("The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM." % fn)
pf = os.path.splitext(fn)[0]
- program = 'msgfmt'
args = [program, '--check-format', '-o', npath(pf + '.mo'), npath(pf + '.po')]
output, errors, status = popen_wrapper(args)
if status:
112 django/core/management/commands/makemessages.py
View
@@ -5,11 +5,11 @@
import sys
from itertools import dropwhile
from optparse import make_option
-from subprocess import PIPE, Popen
import django
from django.core.management.base import CommandError, NoArgsCommand
-from django.core.management.utils import handle_extensions
+from django.core.management.utils import (handle_extensions, find_command,
+ popen_wrapper)
from django.utils.functional import total_ordering
from django.utils.text import get_text_list
from django.utils.jslex import prepare_js_for_gettext
@@ -18,6 +18,13 @@
STATUS_OK = 0
+def check_programs(*programs):
+ for program in programs:
+ if find_command(program) is None:
+ raise CommandError("Can't find %s. Make sure you have GNU "
+ "gettext tools 0.15 or newer installed." % program)
+
+
@total_ordering
class TranslatableFile(object):
def __init__(self, dirpath, file_name):
@@ -58,12 +65,24 @@ def process(self, command, potfile, domain, keep_pot=False):
work_file = os.path.join(self.dirpath, thefile)
with open(work_file, "w") as fp:
fp.write(src_data)
- cmd = (
- 'xgettext -d %s -L C %s %s --keyword=gettext_noop '
- '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
- '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
- '--from-code UTF-8 --add-comments=Translators -o - "%s"' %
- (domain, command.wrap, command.location, work_file))
+ args = [
+ 'xgettext',
+ '-d', domain,
+ '--language=C',
+ '--keyword=gettext_noop',
+ '--keyword=gettext_lazy',
+ '--keyword=ngettext_lazy:1,2',
+ '--keyword=pgettext:1c,2',
+ '--keyword=npgettext:1c,2,3',
+ '--from-code=UTF-8',
+ '--add-comments=Translators',
+ '--output=-'
+ ]
+ if command.wrap:
+ args.append(command.wrap)
+ if command.location:
+ args.append(command.location)
+ args.append(work_file)
elif domain == 'django' and (file_ext == '.py' or file_ext in command.extensions):
thefile = self.file
orig_file = os.path.join(self.dirpath, self.file)
@@ -76,18 +95,32 @@ def process(self, command, potfile, domain, keep_pot=False):
with open(os.path.join(self.dirpath, thefile), "w") as fp:
fp.write(content)
work_file = os.path.join(self.dirpath, thefile)
- cmd = (
- 'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
- '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
- '--keyword=ugettext_noop --keyword=ugettext_lazy '
- '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
- '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
- '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
- '--add-comments=Translators -o - "%s"' %
- (domain, command.wrap, command.location, work_file))
+ args = [
+ 'xgettext',
+ '-d', domain,
+ '--language=Python',
+ '--keyword=gettext_noop',
+ '--keyword=gettext_lazy',
+ '--keyword=ngettext_lazy:1,2',
+ '--keyword=ugettext_noop',
+ '--keyword=ugettext_lazy',
+ '--keyword=ungettext_lazy:1,2',
+ '--keyword=pgettext:1c,2',
+ '--keyword=npgettext:1c,2,3',
+ '--keyword=pgettext_lazy:1c,2',
+ '--keyword=npgettext_lazy:1c,2,3',
+ '--from-code=UTF-8',
+ '--add-comments=Translators',
+ '--output=-'
+ ]
+ if command.wrap:
+ args.append(command.wrap)
+ if command.location:
+ args.append(command.location)
+ args.append(work_file)
else:
return
- msgs, errors, status = _popen(cmd)
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
if is_templatized:
@@ -109,15 +142,6 @@ def process(self, command, potfile, domain, keep_pot=False):
if is_templatized:
os.unlink(work_file)
-
-def _popen(cmd):
- """
- Friendly wrapper around Popen for Windows
- """
- p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
- output, errors = p.communicate()
- return output, errors, p.returncode
-
def write_pot_file(potfile, msgs):
"""
Write the :param potfile: POT file with the :param msgs: contents,
@@ -225,8 +249,9 @@ def handle_noargs(self, *args, **options):
"is not created automatically, you have to create it by hand "
"if you want to enable i18n for your project or application.")
+ check_programs('xgettext')
# We require gettext version 0.15 or newer.
- output, errors, status = _popen('xgettext --version')
+ output, errors, status = popen_wrapper(['xgettext', '--version'])
if status != STATUS_OK:
raise CommandError("Error running xgettext. Note that Django "
"internationalization requires GNU gettext 0.15 or newer.")
@@ -248,6 +273,9 @@ def handle_noargs(self, *args, **options):
locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
locales = [os.path.basename(l) for l in locale_dirs]
+ if locales:
+ check_programs('msguniq', 'msgmerge', 'msgattrib')
+
try:
for locale in locales:
if self.verbosity > 0:
@@ -307,8 +335,13 @@ def write_po_file(self, potfile, locale):
Uses mguniq, msgmerge, and msgattrib GNU gettext utilities.
"""
- msgs, errors, status = _popen('msguniq %s %s --to-code=utf-8 "%s"' %
- (self.wrap, self.location, potfile))
+ args = ['msguniq', '--to-code=utf-8']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.append(potfile)
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
@@ -324,8 +357,13 @@ def write_po_file(self, potfile, locale):
if os.path.exists(pofile):
with open(potfile, 'w') as fp:
fp.write(msgs)
- msgs, errors, status = _popen('msgmerge %s %s -q "%s" "%s"' %
- (self.wrap, self.location, pofile, potfile))
+ args = ['msgmerge', '-q']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.extend([pofile, potfile])
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
@@ -340,9 +378,13 @@ def write_po_file(self, potfile, locale):
fp.write(msgs)
if self.no_obsolete:
- msgs, errors, status = _popen(
- 'msgattrib %s %s -o "%s" --no-obsolete "%s"' %
- (self.wrap, self.location, pofile, pofile))
+ args = ['msgattrib', '-o', pofile, '--no-obsolete']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.append(pofile)
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
40 django/core/management/utils.py
View
@@ -1,17 +1,27 @@
+from __future__ import absolute_import
+
import os
from subprocess import PIPE, Popen
+import sys
from django.utils.encoding import force_text, DEFAULT_LOCALE_ENCODING
+from django.utils import six
+
+from .base import CommandError
-def popen_wrapper(args):
+def popen_wrapper(args, os_err_exc_type=CommandError):
"""
Friendly wrapper around Popen.
Returns stdout output, stderr output and OS status code.
"""
- p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
- close_fds=os.name != 'nt', universal_newlines=True)
+ try:
+ p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
+ close_fds=os.name != 'nt', universal_newlines=True)
+ except OSError as e:
+ six.reraise(os_err_exc_type, os_err_exc_type('Error executing %s: %s' %
+ (args[0], e.strerror)), sys.exc_info()[2])
output, errors = p.communicate()
return (
output,
@@ -43,3 +53,27 @@ def handle_extensions(extensions=('html',), ignored=('py',)):
if not ext.startswith('.'):
ext_list[i] = '.%s' % ext_list[i]
return set([x for x in ext_list if x.strip('.') not in ignored])
+
+def find_command(cmd, path=None, pathext=None):
+ if path is None:
+ path = os.environ.get('PATH', []).split(os.pathsep)
+ if isinstance(path, six.string_types):
+ path = [path]
+ # check if there are funny path extensions for executables, e.g. Windows
+ if pathext is None:
+ pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD').split(os.pathsep)
+ # don't use extensions if the command ends with one of them
+ for ext in pathext:
+ if cmd.endswith(ext):
+ pathext = ['']
+ break
+ # check if we find the command on PATH
+ for p in path:
+ f = os.path.join(p, cmd)
+ if os.path.isfile(f):
+ return f
+ for ext in pathext:
+ fext = f + ext
+ if os.path.isfile(fext):
+ return fext
+ return None
1  django/http/request.py
View
@@ -44,6 +44,7 @@ def __init__(self):
self.path = ''
self.path_info = ''
self.method = None
+ self.resolver_match = None
self._post_parse_error = False
def __repr__(self):
9 docs/topics/http/file-uploads.txt
View
@@ -258,10 +258,11 @@ In addition to those inherited from :class:`~django.core.files.File`, all
for line in uploadedfile:
do_something_with(line)
- However, *unlike* standard Python files, :class:`UploadedFile` only
- understands ``\n`` (also known as "Unix-style") line endings. If you know
- that you need to handle uploaded files with different line endings, you'll
- need to do so in your view.
+ .. versionchanged:: 1.6
+
+ In previous versions, :class:`UploadedFile` only understood ``\n`` (also
+ known as "Unix-style") line endings. Now, you can read files line-by-line
+ when they use ``\r`` line endings as well.
Upload Handlers
===============
7 tests/files/tests.py
View
@@ -1,6 +1,7 @@
from __future__ import absolute_import
import gzip
+import io
import shutil
import tempfile
@@ -146,3 +147,9 @@ def test_file_mode(self):
file = SimpleUploadedFile("mode_test.txt", b"content")
self.assertFalse(hasattr(file, 'mode'))
g = gzip.GzipFile(fileobj=file)
+
+ def test_universal_newlines(self):
+ # See #8149 for more information.
+ self.assertEqual(
+ list(File(io.BytesIO(b'1\r2\r3'))),
+ ['1\r', '2\r', '3'])
9 tests/i18n/commands/extraction.py
View
@@ -131,8 +131,13 @@ def test_extraction_warning(self):
os.chdir(self.test_dir)
shutil.copyfile('./code.sample', './code_sample.py')
stdout = StringIO()
- management.call_command('makemessages', locale=LOCALE, stdout=stdout)
- os.remove('./code_sample.py')
+ try:
+ management.call_command('makemessages', locale=LOCALE, stdout=stdout)
+ finally:
+ try:
+ os.remove('./code_sample.py')
+ except OSError:
+ pass
self.assertIn("code_sample.py:4", force_text(stdout.getvalue()))
def test_template_message_context_extractor(self):
26 tests/i18n/commands/tests.py
View
@@ -2,35 +2,11 @@
import re
from subprocess import Popen, PIPE
-from django.utils import six
+from django.core.management.utils import find_command
can_run_extraction_tests = False
can_run_compilation_tests = False
-def find_command(cmd, path=None, pathext=None):
- if path is None:
- path = os.environ.get('PATH', []).split(os.pathsep)
- if isinstance(path, six.string_types):
- path = [path]
- # check if there are funny path extensions for executables, e.g. Windows
- if pathext is None:
- pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD').split(os.pathsep)
- # don't use extensions if the command ends with one of them
- for ext in pathext:
- if cmd.endswith(ext):
- pathext = ['']
- break
- # check if we find the command on PATH
- for p in path:
- f = os.path.join(p, cmd)
- if os.path.isfile(f):
- return f
- for ext in pathext:
- fext = f + ext
- if os.path.isfile(fext):
- return fext
- return None
-
# checks if it can find xgettext on the PATH and
# imports the extraction tests if yes
xgettext_cmd = find_command('xgettext')
6 tests/urlpatterns_reverse/tests.py
View
@@ -9,7 +9,7 @@
from django.core.urlresolvers import (reverse, resolve, get_callable,
get_resolver, NoReverseMatch, Resolver404, ResolverMatch, RegexURLResolver,
RegexURLPattern)
-from django.http import HttpResponseRedirect, HttpResponsePermanentRedirect
+from django.http import HttpRequest, HttpResponseRedirect, HttpResponsePermanentRedirect
from django.shortcuts import redirect
from django.test import TestCase
from django.utils import unittest, six
@@ -529,6 +529,10 @@ def test_resolver_match_on_request(self):
resolver_match = response.resolver_match
self.assertEqual(resolver_match.url_name, 'test-resolver-match')
+ def test_resolver_match_on_request_before_resolution(self):
+ request = HttpRequest()
+ self.assertIsNone(request.resolver_match)
+
class ErroneousViewTests(TestCase):
urls = 'urlpatterns_reverse.erroneous_urls'
13 tests/user_commands/tests.py
View
@@ -1,13 +1,14 @@
import sys
from django.core import management
-from django.core.management.base import CommandError
-from django.test import TestCase
+from django.core.management import CommandError
+from django.core.management.utils import popen_wrapper
+from django.test import SimpleTestCase
from django.utils import translation
from django.utils.six import StringIO
-class CommandTests(TestCase):
+class CommandTests(SimpleTestCase):
def test_command(self):
out = StringIO()
management.call_command('dance', stdout=out)
@@ -58,3 +59,9 @@ def test_configured_locale_preserved(self):
with translation.override('pl'):
management.call_command('leave_locale_alone_true', stdout=out)
self.assertEqual(out.getvalue(), "pl\n")
+
+
+class UtilsTests(SimpleTestCase):
+
+ def test_no_existent_external_program(self):
+ self.assertRaises(CommandError, popen_wrapper, ['a_42_command_that_doesnt_exist_42'])
Something went wrong with that request. Please try again.