[1.5.x] Fixed #19357 -- Allow non-ASCII chars in filesystem paths

Thanks kujiu for the report and Aymeric Augustin for the review.
Backport of c916673 from master.

django/contrib/admindocs/views.py

@@ -14,6 +14,7 @@
 from django.contrib.admindocs import utils
 from django.contrib.sites.models import Site
 from django.utils.importlib import import_module
+from django.utils._os import upath
 from django.utils import six
 from django.utils.translation import ugettext as _
 from django.utils.safestring import mark_safe
@@ -311,7 +312,7 @@ def load_all_installed_template_libraries():
         try:
             libraries = [
                 os.path.splitext(p)[0]
-                for p in os.listdir(os.path.dirname(mod.__file__))
+                for p in os.listdir(os.path.dirname(upath(mod.__file__)))
                 if p.endswith('.py') and p[0].isalpha()
             ]
         except OSError:

django/contrib/auth/tests/context_processors.py

@@ -9,6 +9,7 @@
 from django.db.models import Q
 from django.test import TestCase
 from django.test.utils import override_settings
+from django.utils._os import upath
 
 
 class MockUser(object):
@@ -63,7 +64,7 @@ def test_permlookupdict_in(self):
 @skipIfCustomUser
 @override_settings(
     TEMPLATE_DIRS=(
-            os.path.join(os.path.dirname(__file__), 'templates'),
+            os.path.join(os.path.dirname(upath(__file__)), 'templates'),
         ),
     USE_TZ=False,                           # required for loading the fixture
     PASSWORD_HASHERS=('django.contrib.auth.hashers.SHA1PasswordHasher',),

django/contrib/auth/tests/forms.py

@@ -11,6 +11,7 @@
 from django.test import TestCase
 from django.test.utils import override_settings
 from django.utils.encoding import force_text
+from django.utils._os import upath
 from django.utils import translation
 from django.utils.translation import ugettext as _
 
@@ -331,7 +332,7 @@ def test_cleaned_data(self):
         self.assertEqual(form.cleaned_data['email'], email)
 
     def test_custom_email_subject(self):
-        template_path = os.path.join(os.path.dirname(__file__), 'templates')
+        template_path = os.path.join(os.path.dirname(upath(__file__)), 'templates')
         with self.settings(TEMPLATE_DIRS=(template_path,)):
             data = {'email': 'testclient@example.com'}
             form = PasswordResetForm(data)

django/contrib/auth/tests/views.py

@@ -11,6 +11,7 @@
 from django.utils.encoding import force_text
 from django.utils.html import escape
 from django.utils.http import urlquote
+from django.utils._os import upath
 from django.test import TestCase
 from django.test.utils import override_settings
 
@@ -27,7 +28,7 @@
     LANGUAGE_CODE='en',
     TEMPLATE_LOADERS=global_settings.TEMPLATE_LOADERS,
     TEMPLATE_DIRS=(
-        os.path.join(os.path.dirname(__file__), 'templates'),
+        os.path.join(os.path.dirname(upath(__file__)), 'templates'),
     ),
     USE_TZ=False,
     PASSWORD_HASHERS=('django.contrib.auth.hashers.SHA1PasswordHasher',),

django/contrib/formtools/tests/__init__.py

@@ -14,6 +14,7 @@
 from django.test import TestCase
 from django.test.html import parse_html
 from django.test.utils import override_settings
+from django.utils._os import upath
 from django.utils import unittest
 
 from django.contrib.formtools.tests.wizard import *
@@ -36,7 +37,7 @@ def done(self, request, cleaned_data):
 
 @override_settings(
     TEMPLATE_DIRS=(
-        os.path.join(os.path.dirname(__file__), 'templates'),
+        os.path.join(os.path.dirname(upath(__file__)), 'templates'),
     ),
 )
 class PreviewTests(TestCase):
@@ -214,7 +215,7 @@ def __init__(self, POST=None):
 @override_settings(
     SECRET_KEY="123",
     TEMPLATE_DIRS=(
-        os.path.join(os.path.dirname(__file__), 'templates'),
+        os.path.join(os.path.dirname(upath(__file__)), 'templates'),
     ),
 )
 class WizardTests(TestCase):

django/contrib/formtools/tests/wizard/wizardtests/tests.py

@@ -9,6 +9,7 @@
 from django.contrib.auth.models import User
 from django.contrib.formtools.wizard.views import CookieWizardView
 from django.contrib.formtools.tests.wizard.forms import UserForm, UserFormSet
+from django.utils._os import upath
 
 
 class WizardTests(object):
@@ -86,7 +87,7 @@ def test_form_finish(self):
         self.assertEqual(response.context['wizard']['steps'].current, 'form2')
 
         post_data = self.wizard_step_data[1]
-        post_data['form2-file1'] = open(__file__, 'rb')
+        post_data['form2-file1'] = open(upath(__file__), 'rb')
         response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.context['wizard']['steps'].current, 'form3')
@@ -99,7 +100,7 @@ def test_form_finish(self):
         self.assertEqual(response.status_code, 200)
 
         all_data = response.context['form_list']
-        with open(__file__, 'rb') as f:
+        with open(upath(__file__), 'rb') as f:
             self.assertEqual(all_data[1]['file1'].read(), f.read())
         all_data[1]['file1'].close()
         del all_data[1]['file1']
@@ -118,7 +119,7 @@ def test_cleaned_data(self):
         self.assertEqual(response.status_code, 200)
 
         post_data = self.wizard_step_data[1]
-        with open(__file__, 'rb') as post_file:
+        with open(upath(__file__), 'rb') as post_file:
             post_data['form2-file1'] = post_file
             response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)
@@ -130,7 +131,7 @@ def test_cleaned_data(self):
         self.assertEqual(response.status_code, 200)
 
         all_data = response.context['all_cleaned_data']
-        with open(__file__, 'rb') as f:
+        with open(upath(__file__), 'rb') as f:
             self.assertEqual(all_data['file1'].read(), f.read())
         all_data['file1'].close()
         del all_data['file1']
@@ -150,7 +151,7 @@ def test_manipulated_data(self):
 
         post_data = self.wizard_step_data[1]
         post_data['form2-file1'].close()
-        post_data['form2-file1'] = open(__file__, 'rb')
+        post_data['form2-file1'] = open(upath(__file__), 'rb')
         response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)
 
@@ -178,7 +179,7 @@ def test_form_refresh(self):
 
         post_data = self.wizard_step_data[1]
         post_data['form2-file1'].close()
-        post_data['form2-file1'] = open(__file__, 'rb')
+        post_data['form2-file1'] = open(upath(__file__), 'rb')
         response = self.client.post(self.wizard_url, post_data)
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.context['wizard']['steps'].current, 'form3')
@@ -291,7 +292,7 @@ def setUp(self):
         self.wizard_step_data[0]['form1-user'] = self.testuser.pk
 
     def test_template(self):
-        templates = os.path.join(os.path.dirname(__file__), 'templates')
+        templates = os.path.join(os.path.dirname(upath(__file__)), 'templates')
         with self.settings(
                 TEMPLATE_DIRS=list(settings.TEMPLATE_DIRS) + [templates]):
             response = self.client.get(self.wizard_url)

django/contrib/gis/geometry/test_data.py

@@ -7,13 +7,14 @@
 
 from django.contrib import gis
 from django.utils import six
+from django.utils._os import upath
 
 
 # This global used to store reference geometry data.
 GEOMETRIES = None
 
 # Path where reference test data is located.
-TEST_DATA = os.path.join(os.path.dirname(gis.__file__), 'tests', 'data')
+TEST_DATA = os.path.join(os.path.dirname(upath(gis.__file__)), 'tests', 'data')
 
 
 def tuplize(seq):

django/contrib/gis/tests/geo3d/tests.py

@@ -7,12 +7,13 @@
 from django.contrib.gis.geos import GEOSGeometry, LineString, Point, Polygon
 from django.contrib.gis.utils import LayerMapping, LayerMapError
 from django.test import TestCase
+from django.utils._os import upath
 
 from .models import (City3D, Interstate2D, Interstate3D, InterstateProj2D,
     InterstateProj3D, Point2D, Point3D, MultiPoint3D, Polygon2D, Polygon3D)
 
 
-data_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'data'))
+data_path = os.path.realpath(os.path.join(os.path.dirname(upath(__file__)), '..', 'data'))
 city_file = os.path.join(data_path, 'cities', 'cities.shp')
 vrt_file = os.path.join(data_path, 'test_vrt', 'test_vrt.vrt')
 

django/contrib/gis/tests/geogapp/tests.py

@@ -8,6 +8,7 @@
 from django.contrib.gis import gdal
 from django.contrib.gis.measure import D
 from django.test import TestCase
+from django.utils._os import upath
 
 from .models import City, County, Zipcode
 
@@ -61,7 +62,7 @@ def test05_geography_layermapping(self):
         from django.contrib.gis.utils import LayerMapping
 
         # Getting the shapefile and mapping dictionary.
-        shp_path = os.path.realpath(os.path.join(os.path.dirname(__file__), '..', 'data'))
+        shp_path = os.path.realpath(os.path.join(os.path.dirname(upath(__file__)), '..', 'data'))
         co_shp = os.path.join(shp_path, 'counties', 'counties.shp')
         co_mapping = {'name' : 'Name',
                       'state' : 'State',

django/contrib/gis/tests/layermap/tests.py

@@ -13,13 +13,14 @@
 from django.conf import settings
 from django.test import TestCase
 from django.utils import unittest
+from django.utils._os import upath
 
 from .models import (
     City, County, CountyFeat, Interstate, ICity1, ICity2, Invalid, State,
     city_mapping, co_mapping, cofeat_mapping, inter_mapping)
 
 
-shp_path = os.path.realpath(os.path.join(os.path.dirname(__file__), os.pardir, 'data'))
+shp_path = os.path.realpath(os.path.join(os.path.dirname(upath(__file__)), os.pardir, 'data'))
 city_shp = os.path.join(shp_path, 'cities', 'cities.shp')
 co_shp = os.path.join(shp_path, 'counties', 'counties.shp')
 inter_shp = os.path.join(shp_path, 'interstates', 'interstates.shp')

django/contrib/sitemaps/tests/http.py

@@ -11,6 +11,7 @@
 from django.test.utils import override_settings
 from django.utils.unittest import skipUnless
 from django.utils.formats import localize
+from django.utils._os import upath
 from django.utils.translation import activate, deactivate
 
 from .base import SitemapTestsBase
@@ -29,7 +30,7 @@ def test_simple_sitemap_index(self):
         self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
 
     @override_settings(
-        TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__), 'templates'),)
+        TEMPLATE_DIRS=(os.path.join(os.path.dirname(upath(__file__)), 'templates'),)
     )
     def test_simple_sitemap_custom_index(self):
         "A simple sitemap index can be rendered with a custom template"
@@ -64,7 +65,7 @@ def test_simple_sitemap(self):
         self.assertXMLEqual(response.content.decode('utf-8'), expected_content)
 
     @override_settings(
-        TEMPLATE_DIRS=(os.path.join(os.path.dirname(__file__), 'templates'),)
+        TEMPLATE_DIRS=(os.path.join(os.path.dirname(upath(__file__)), 'templates'),)
     )
     def test_simple_custom_sitemap(self):
         "A simple sitemap can be rendered with a custom template"

django/contrib/staticfiles/storage.py

@@ -19,6 +19,7 @@
 from django.utils.encoding import force_bytes, force_text
 from django.utils.functional import LazyObject
 from django.utils.importlib import import_module
+from django.utils._os import upath
 
 from django.contrib.staticfiles.utils import check_settings, matches_patterns
 
@@ -296,7 +297,7 @@ def __init__(self, app, *args, **kwargs):
         """
         # app is the actual app module
         mod = import_module(app)
-        mod_path = os.path.dirname(mod.__file__)
+        mod_path = os.path.dirname(upath(mod.__file__))
         location = os.path.join(mod_path, self.source_dir)
         super(AppStaticStorage, self).__init__(location, *args, **kwargs)
 

django/core/management/__init__.py

@@ -9,6 +9,7 @@
 from django.core.management.base import BaseCommand, CommandError, handle_default_options
 from django.core.management.color import color_style
 from django.utils.importlib import import_module
+from django.utils._os import upath
 from django.utils import six
 
 # For backwards compatibility: get_version() used to be in this module.
@@ -410,10 +411,10 @@ def setup_environ(settings_mod, original_settings_path=None):
     # Add this project to sys.path so that it's importable in the conventional
     # way. For example, if this file (manage.py) lives in a directory
     # "myproject", this code would add "/path/to/myproject" to sys.path.
-    if '__init__.py' in settings_mod.__file__:
-        p = os.path.dirname(settings_mod.__file__)
+    if '__init__.py' in upath(settings_mod.__file__):
+        p = os.path.dirname(upath(settings_mod.__file__))
     else:
-        p = settings_mod.__file__
+        p = upath(settings_mod.__file__)
     project_directory, settings_filename = os.path.split(p)
     if project_directory == os.curdir or not project_directory:
         project_directory = os.getcwd()

django/core/management/commands/compilemessages.py

@@ -5,6 +5,7 @@
 import sys
 from optparse import make_option
 from django.core.management.base import BaseCommand, CommandError
+from django.utils._os import npath
 
 def has_bom(fn):
     with open(fn, 'rb') as f:
@@ -41,8 +42,8 @@ def compile_messages(stderr, locale=None):
                     # command, so that we can take advantage of shell quoting, to
                     # quote any malicious characters/escaping.
                     # See http://cyberelk.net/tim/articles/cmdline/ar01s02.html
-                    os.environ['djangocompilemo'] = pf + '.mo'
-                    os.environ['djangocompilepo'] = pf + '.po'
+                    os.environ['djangocompilemo'] = npath(pf + '.mo')
+                    os.environ['djangocompilepo'] = npath(pf + '.po')
                     if sys.platform == 'win32': # Different shell-variable syntax
                         cmd = 'msgfmt --check-format -o "%djangocompilemo%" "%djangocompilepo%"'
                     else:

django/core/management/commands/loaddata.py

@@ -15,6 +15,7 @@
       IntegrityError, DatabaseError)
 from django.db.models import get_apps
 from django.utils.encoding import force_text
+from django.utils._os import upath
 from itertools import product
 
 try:
@@ -102,10 +103,10 @@ def read(self):
             if hasattr(app, '__path__'):
                 # It's a 'models/' subpackage
                 for path in app.__path__:
-                    app_module_paths.append(path)
+                    app_module_paths.append(upath(path))
             else:
                 # It's a models.py module
-                app_module_paths.append(app.__file__)
+                app_module_paths.append(upath(app.__file__))
 
         app_fixtures = [os.path.join(os.path.dirname(path), 'fixtures') for path in app_module_paths]
 

django/core/management/commands/makemessages.py

@@ -301,7 +301,7 @@ def make_messages(locale=None, domain='django', verbosity=1, all=False,
 
     locales = []
     if locale is not None:
-        locales.append(locale)
+        locales.append(str(locale))
     elif all:
         locale_dirs = filter(os.path.isdir, glob.glob('%s/*' % localedir))
         locales = [os.path.basename(l) for l in locale_dirs]
@@ -316,8 +316,8 @@ def make_messages(locale=None, domain='django', verbosity=1, all=False,
         if not os.path.isdir(basedir):
             os.makedirs(basedir)
 
-        pofile = os.path.join(basedir, '%s.po' % domain)
-        potfile = os.path.join(basedir, '%s.pot' % domain)
+        pofile = os.path.join(basedir, '%s.po' % str(domain))
+        potfile = os.path.join(basedir, '%s.pot' % str(domain))
 
         if os.path.exists(potfile):
             os.unlink(potfile)

django/core/management/sql.py

@@ -8,6 +8,7 @@
 from django.core.management.base import CommandError
 from django.db import models
 from django.db.models import get_models
+from django.utils._os import upath
 
 
 def sql_create(app, style, connection):
@@ -159,7 +160,7 @@ def _split_statements(content):
 
 def custom_sql_for_model(model, style, connection):
     opts = model._meta
-    app_dir = os.path.normpath(os.path.join(os.path.dirname(models.get_app(model._meta.app_label).__file__), 'sql'))
+    app_dir = os.path.normpath(os.path.join(os.path.dirname(upath(models.get_app(model._meta.app_label).__file__)), 'sql'))
     output = []
 
     # Post-creation SQL should come before any initial SQL data is loaded.

django/core/urlresolvers.py

@@ -251,9 +251,9 @@ def __repr__(self):
             urlconf_repr = '<%s list>' % self.urlconf_name[0].__class__.__name__
         else:
             urlconf_repr = repr(self.urlconf_name)
-        return force_str('<%s %s (%s:%s) %s>' % (
+        return str('<%s %s (%s:%s) %s>') % (
             self.__class__.__name__, urlconf_repr, self.app_name,
-            self.namespace, self.regex.pattern))
+            self.namespace, self.regex.pattern)
 
     def _populate(self):
         lookups = MultiValueDict()