Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

Fixed #8149 -- Use universal line terminators when iterating `File` obje... #323

Closed
wants to merge 4 commits into from

5 participants

@mrmachine

...cts line-by-line.

@timgraham
Owner

The test doesn't pass on Python 3.

@timgraham
Owner

Please open a new PR if you are able to update this, thanks!

@timgraham timgraham closed this
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Commits on Mar 10, 2013
  1. @alex

    Merge pull request #892 from JonLoy/ticket_20018

    alex authored
    Fixed #20018: Added backtick to fix reference
  2. @ramiro
  3. @aaugustin

    Fixed #20019 -- Ensured HttpRequest.resolver_match always exists.

    aaugustin authored
    Obviously it isn't set until the URL is resolved.
Commits on Mar 11, 2013
  1. Fixed #8149 -- Use universal line terminators when iterating `File` o…

    Tai Lee authored
    …bjects line-by-line.
This page is out of date. Refresh to see the latest.
View
4 django/core/files/base.py
@@ -94,9 +94,7 @@ def __iter__(self):
# Iterate over this file-like object by newlines
buffer_ = None
for chunk in self.chunks():
- chunk_buffer = BytesIO(chunk)
-
- for line in chunk_buffer:
+ for line in chunk.splitlines(True):
if buffer_:
line = buffer_ + line
buffer_ = None
View
7 django/core/management/commands/compilemessages.py
@@ -5,7 +5,7 @@
from optparse import make_option
from django.core.management.base import BaseCommand, CommandError
-from django.core.management.utils import popen_wrapper
+from django.core.management.utils import find_command, popen_wrapper
from django.utils._os import npath
def has_bom(fn):
@@ -16,6 +16,10 @@ def has_bom(fn):
sample.startswith(codecs.BOM_UTF16_BE)
def compile_messages(stderr, locale=None):
+ program = 'msgfmt'
+ if find_command(program) is None:
+ raise CommandError("Can't find %s. Make sure you have GNU gettext tools 0.15 or newer installed." % program)
+
basedirs = [os.path.join('conf', 'locale'), 'locale']
if os.environ.get('DJANGO_SETTINGS_MODULE'):
from django.conf import settings
@@ -42,7 +46,6 @@ def compile_messages(stderr, locale=None):
if has_bom(fn):
raise CommandError("The %s file has a BOM (Byte Order Mark). Django only supports .po files encoded in UTF-8 and without any BOM." % fn)
pf = os.path.splitext(fn)[0]
- program = 'msgfmt'
args = [program, '--check-format', '-o', npath(pf + '.mo'), npath(pf + '.po')]
output, errors, status = popen_wrapper(args)
if status:
View
112 django/core/management/commands/makemessages.py
@@ -5,11 +5,11 @@
import sys
from itertools import dropwhile
from optparse import make_option
-from subprocess import PIPE, Popen
import django
from django.core.management.base import CommandError, NoArgsCommand
-from django.core.management.utils import handle_extensions
+from django.core.management.utils import (handle_extensions, find_command,
+ popen_wrapper)
from django.utils.functional import total_ordering
from django.utils.text import get_text_list
from django.utils.jslex import prepare_js_for_gettext
@@ -18,6 +18,13 @@
STATUS_OK = 0
+def check_programs(*programs):
+ for program in programs:
+ if find_command(program) is None:
+ raise CommandError("Can't find %s. Make sure you have GNU "
+ "gettext tools 0.15 or newer installed." % program)
+
+
@total_ordering
class TranslatableFile(object):
def __init__(self, dirpath, file_name):
@@ -58,12 +65,24 @@ def process(self, command, potfile, domain, keep_pot=False):
work_file = os.path.join(self.dirpath, thefile)
with open(work_file, "w") as fp:
fp.write(src_data)
- cmd = (
- 'xgettext -d %s -L C %s %s --keyword=gettext_noop '
- '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
- '--keyword=pgettext:1c,2 --keyword=npgettext:1c,2,3 '
- '--from-code UTF-8 --add-comments=Translators -o - "%s"' %
- (domain, command.wrap, command.location, work_file))
+ args = [
+ 'xgettext',
+ '-d', domain,
+ '--language=C',
+ '--keyword=gettext_noop',
+ '--keyword=gettext_lazy',
+ '--keyword=ngettext_lazy:1,2',
+ '--keyword=pgettext:1c,2',
+ '--keyword=npgettext:1c,2,3',
+ '--from-code=UTF-8',
+ '--add-comments=Translators',
+ '--output=-'
+ ]
+ if command.wrap:
+ args.append(command.wrap)
+ if command.location:
+ args.append(command.location)
+ args.append(work_file)
elif domain == 'django' and (file_ext == '.py' or file_ext in command.extensions):
thefile = self.file
orig_file = os.path.join(self.dirpath, self.file)
@@ -76,18 +95,32 @@ def process(self, command, potfile, domain, keep_pot=False):
with open(os.path.join(self.dirpath, thefile), "w") as fp:
fp.write(content)
work_file = os.path.join(self.dirpath, thefile)
- cmd = (
- 'xgettext -d %s -L Python %s %s --keyword=gettext_noop '
- '--keyword=gettext_lazy --keyword=ngettext_lazy:1,2 '
- '--keyword=ugettext_noop --keyword=ugettext_lazy '
- '--keyword=ungettext_lazy:1,2 --keyword=pgettext:1c,2 '
- '--keyword=npgettext:1c,2,3 --keyword=pgettext_lazy:1c,2 '
- '--keyword=npgettext_lazy:1c,2,3 --from-code UTF-8 '
- '--add-comments=Translators -o - "%s"' %
- (domain, command.wrap, command.location, work_file))
+ args = [
+ 'xgettext',
+ '-d', domain,
+ '--language=Python',
+ '--keyword=gettext_noop',
+ '--keyword=gettext_lazy',
+ '--keyword=ngettext_lazy:1,2',
+ '--keyword=ugettext_noop',
+ '--keyword=ugettext_lazy',
+ '--keyword=ungettext_lazy:1,2',
+ '--keyword=pgettext:1c,2',
+ '--keyword=npgettext:1c,2,3',
+ '--keyword=pgettext_lazy:1c,2',
+ '--keyword=npgettext_lazy:1c,2,3',
+ '--from-code=UTF-8',
+ '--add-comments=Translators',
+ '--output=-'
+ ]
+ if command.wrap:
+ args.append(command.wrap)
+ if command.location:
+ args.append(command.location)
+ args.append(work_file)
else:
return
- msgs, errors, status = _popen(cmd)
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
if is_templatized:
@@ -109,15 +142,6 @@ def process(self, command, potfile, domain, keep_pot=False):
if is_templatized:
os.unlink(work_file)
-
-def _popen(cmd):
- """
- Friendly wrapper around Popen for Windows
- """
- p = Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE, close_fds=os.name != 'nt', universal_newlines=True)
- output, errors = p.communicate()
- return output, errors, p.returncode
-
def write_pot_file(potfile, msgs):
"""
Write the :param potfile: POT file with the :param msgs: contents,
@@ -225,8 +249,9 @@ def handle_noargs(self, *args, **options):
"is not created automatically, you have to create it by hand "
"if you want to enable i18n for your project or application.")
+ check_programs('xgettext')
# We require gettext version 0.15 or newer.
- output, errors, status = _popen('xgettext --version')
+ output, errors, status = popen_wrapper(['xgettext', '--version'])
if status != STATUS_OK:
raise CommandError("Error running xgettext. Note that Django "
"internationalization requires GNU gettext 0.15 or newer.")
@@ -248,6 +273,9 @@ def handle_noargs(self, *args, **options):
locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
locales = [os.path.basename(l) for l in locale_dirs]
+ if locales:
+ check_programs('msguniq', 'msgmerge', 'msgattrib')
+
try:
for locale in locales:
if self.verbosity > 0:
@@ -307,8 +335,13 @@ def write_po_file(self, potfile, locale):
Uses mguniq, msgmerge, and msgattrib GNU gettext utilities.
"""
- msgs, errors, status = _popen('msguniq %s %s --to-code=utf-8 "%s"' %
- (self.wrap, self.location, potfile))
+ args = ['msguniq', '--to-code=utf-8']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.append(potfile)
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
@@ -324,8 +357,13 @@ def write_po_file(self, potfile, locale):
if os.path.exists(pofile):
with open(potfile, 'w') as fp:
fp.write(msgs)
- msgs, errors, status = _popen('msgmerge %s %s -q "%s" "%s"' %
- (self.wrap, self.location, pofile, potfile))
+ args = ['msgmerge', '-q']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.extend([pofile, potfile])
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
@@ -340,9 +378,13 @@ def write_po_file(self, potfile, locale):
fp.write(msgs)
if self.no_obsolete:
- msgs, errors, status = _popen(
- 'msgattrib %s %s -o "%s" --no-obsolete "%s"' %
- (self.wrap, self.location, pofile, pofile))
+ args = ['msgattrib', '-o', pofile, '--no-obsolete']
+ if self.wrap:
+ args.append(self.wrap)
+ if self.location:
+ args.append(self.location)
+ args.append(pofile)
+ msgs, errors, status = popen_wrapper(args)
if errors:
if status != STATUS_OK:
raise CommandError(
View
40 django/core/management/utils.py
@@ -1,17 +1,27 @@
+from __future__ import absolute_import
+
import os
from subprocess import PIPE, Popen
+import sys
from django.utils.encoding import force_text, DEFAULT_LOCALE_ENCODING
+from django.utils import six
+
+from .base import CommandError
-def popen_wrapper(args):
+def popen_wrapper(args, os_err_exc_type=CommandError):
"""
Friendly wrapper around Popen.
Returns stdout output, stderr output and OS status code.
"""
- p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
- close_fds=os.name != 'nt', universal_newlines=True)
+ try:
+ p = Popen(args, shell=False, stdout=PIPE, stderr=PIPE,
+ close_fds=os.name != 'nt', universal_newlines=True)
+ except OSError as e:
+ six.reraise(os_err_exc_type, os_err_exc_type('Error executing %s: %s' %
+ (args[0], e.strerror)), sys.exc_info()[2])
output, errors = p.communicate()
return (
output,
@@ -43,3 +53,27 @@ def handle_extensions(extensions=('html',), ignored=('py',)):
if not ext.startswith('.'):
ext_list[i] = '.%s' % ext_list[i]
return set([x for x in ext_list if x.strip('.') not in ignored])
+
+def find_command(cmd, path=None, pathext=None):
+ if path is None:
+ path = os.environ.get('PATH', []).split(os.pathsep)
+ if isinstance(path, six.string_types):
+ path = [path]
+ # check if there are funny path extensions for executables, e.g. Windows
+ if pathext is None:
+ pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD').split(os.pathsep)
+ # don't use extensions if the command ends with one of them
+ for ext in pathext:
+ if cmd.endswith(ext):
+ pathext = ['']
+ break
+ # check if we find the command on PATH
+ for p in path:
+ f = os.path.join(p, cmd)
+ if os.path.isfile(f):
+ return f
+ for ext in pathext:
+ fext = f + ext
+ if os.path.isfile(fext):
+ return fext
+ return None
View
1  django/http/request.py
@@ -44,6 +44,7 @@ def __init__(self):
self.path = ''
self.path_info = ''
self.method = None
+ self.resolver_match = None
self._post_parse_error = False
def __repr__(self):
View
9 docs/topics/http/file-uploads.txt
@@ -258,10 +258,11 @@ In addition to those inherited from :class:`~django.core.files.File`, all
for line in uploadedfile:
do_something_with(line)
- However, *unlike* standard Python files, :class:`UploadedFile` only
- understands ``\n`` (also known as "Unix-style") line endings. If you know
- that you need to handle uploaded files with different line endings, you'll
- need to do so in your view.
+ .. versionchanged:: 1.6
+
+ In previous versions, :class:`UploadedFile` only understood ``\n`` (also
+ known as "Unix-style") line endings. Now, you can read files line-by-line
+ when they use ``\r`` line endings as well.
Upload Handlers
===============
View
7 tests/files/tests.py
@@ -1,6 +1,7 @@
from __future__ import absolute_import
import gzip
+import io
import shutil
import tempfile
@@ -146,3 +147,9 @@ def test_file_mode(self):
file = SimpleUploadedFile("mode_test.txt", b"content")
self.assertFalse(hasattr(file, 'mode'))
g = gzip.GzipFile(fileobj=file)
+
+ def test_universal_newlines(self):
+ # See #8149 for more information.
+ self.assertEqual(
+ list(File(io.BytesIO(b'1\r2\r3'))),
+ ['1\r', '2\r', '3'])
View
9 tests/i18n/commands/extraction.py
@@ -131,8 +131,13 @@ def test_extraction_warning(self):
os.chdir(self.test_dir)
shutil.copyfile('./code.sample', './code_sample.py')
stdout = StringIO()
- management.call_command('makemessages', locale=LOCALE, stdout=stdout)
- os.remove('./code_sample.py')
+ try:
+ management.call_command('makemessages', locale=LOCALE, stdout=stdout)
+ finally:
+ try:
+ os.remove('./code_sample.py')
+ except OSError:
+ pass
self.assertIn("code_sample.py:4", force_text(stdout.getvalue()))
def test_template_message_context_extractor(self):
View
26 tests/i18n/commands/tests.py
@@ -2,35 +2,11 @@
import re
from subprocess import Popen, PIPE
-from django.utils import six
+from django.core.management.utils import find_command
can_run_extraction_tests = False
can_run_compilation_tests = False
-def find_command(cmd, path=None, pathext=None):
- if path is None:
- path = os.environ.get('PATH', []).split(os.pathsep)
- if isinstance(path, six.string_types):
- path = [path]
- # check if there are funny path extensions for executables, e.g. Windows
- if pathext is None:
- pathext = os.environ.get('PATHEXT', '.COM;.EXE;.BAT;.CMD').split(os.pathsep)
- # don't use extensions if the command ends with one of them
- for ext in pathext:
- if cmd.endswith(ext):
- pathext = ['']
- break
- # check if we find the command on PATH
- for p in path:
- f = os.path.join(p, cmd)
- if os.path.isfile(f):
- return f
- for ext in pathext:
- fext = f + ext
- if os.path.isfile(fext):
- return fext
- return None
-
# checks if it can find xgettext on the PATH and
# imports the extraction tests if yes
xgettext_cmd = find_command('xgettext')
View
6 tests/urlpatterns_reverse/tests.py
@@ -9,7 +9,7 @@
from django.core.urlresolvers import (reverse, resolve, get_callable,
get_resolver, NoReverseMatch, Resolver404, ResolverMatch, RegexURLResolver,
RegexURLPattern)
-from django.http import HttpResponseRedirect, HttpResponsePermanentRedirect
+from django.http import HttpRequest, HttpResponseRedirect, HttpResponsePermanentRedirect
from django.shortcuts import redirect
from django.test import TestCase
from django.utils import unittest, six
@@ -529,6 +529,10 @@ def test_resolver_match_on_request(self):
resolver_match = response.resolver_match
self.assertEqual(resolver_match.url_name, 'test-resolver-match')
+ def test_resolver_match_on_request_before_resolution(self):
+ request = HttpRequest()
+ self.assertIsNone(request.resolver_match)
+
class ErroneousViewTests(TestCase):
urls = 'urlpatterns_reverse.erroneous_urls'
View
13 tests/user_commands/tests.py
@@ -1,13 +1,14 @@
import sys
from django.core import management
-from django.core.management.base import CommandError
-from django.test import TestCase
+from django.core.management import CommandError
+from django.core.management.utils import popen_wrapper
+from django.test import SimpleTestCase
from django.utils import translation
from django.utils.six import StringIO
-class CommandTests(TestCase):
+class CommandTests(SimpleTestCase):
def test_command(self):
out = StringIO()
management.call_command('dance', stdout=out)
@@ -58,3 +59,9 @@ def test_configured_locale_preserved(self):
with translation.override('pl'):
management.call_command('leave_locale_alone_true', stdout=out)
self.assertEqual(out.getvalue(), "pl\n")
+
+
+class UtilsTests(SimpleTestCase):
+
+ def test_no_existent_external_program(self):
+ self.assertRaises(CommandError, popen_wrapper, ['a_42_command_that_doesnt_exist_42'])
Something went wrong with that request. Please try again.