CLN: Clean up of locale testing (#29883)

Author: datapythonista

ci/azure/posix.yml

@@ -19,18 +19,24 @@ jobs:
           ENV_FILE: ci/deps/azure-36-minimum_versions.yaml
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
+
         py36_locale_slow_old_np:
           ENV_FILE: ci/deps/azure-36-locale_slow.yaml
           CONDA_PY: "36"
           PATTERN: "slow"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          # pandas does not use the language (zh_CN), but should support diferent encodings (utf8)
+          # we should test with encodings different than utf8, but doesn't seem like Ubuntu supports any
+          LANG: "zh_CN.utf8"
+          LC_ALL: "zh_CN.utf8"
           EXTRA_APT: "language-pack-zh-hans"
 
         py36_locale:
           ENV_FILE: ci/deps/azure-36-locale.yaml
           CONDA_PY: "36"
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "it_IT.UTF-8"
+          LANG: "it_IT.utf8"
+          LC_ALL: "it_IT.utf8"
+          EXTRA_APT: "language-pack-it"
 
         py36_32bit:
           ENV_FILE: ci/deps/azure-36-32bit.yaml
@@ -42,7 +48,9 @@ jobs:
           ENV_FILE: ci/deps/azure-37-locale.yaml
           CONDA_PY: "37"
           PATTERN: "not slow and not network"
-          LOCALE_OVERRIDE: "zh_CN.UTF-8"
+          LANG: "zh_CN.utf8"
+          LC_ALL: "zh_CN.utf8"
+          EXTRA_APT: "language-pack-zh-hans"
 
         py37_np_dev:
           ENV_FILE: ci/deps/azure-37-numpydev.yaml
@@ -54,10 +62,16 @@ jobs:
 
   steps:
     - script: |
-        if [ "$(uname)" == "Linux" ]; then sudo apt-get install -y libc6-dev-i386 $EXTRA_APT; fi
-        echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
-        echo "Creating Environment"
-        ci/setup_env.sh
+        if [ "$(uname)" == "Linux" ]; then
+          sudo apt-get update
+          sudo apt-get install -y libc6-dev-i386 $EXTRA_APT
+        fi
+      displayName: 'Install extra packages'
+
+    - script: echo '##vso[task.prependpath]$(HOME)/miniconda3/bin'
+      displayName: 'Set conda path'
+
+    - script: ci/setup_env.sh
       displayName: 'Setup environment and build pandas'
 
     - script: |

ci/run_tests.sh

@@ -5,17 +5,6 @@
 # https://github.com/pytest-dev/pytest/issues/1075
 export PYTHONHASHSEED=$(python -c 'import random; print(random.randint(1, 4294967295))')
 
-if [ -n "$LOCALE_OVERRIDE" ]; then
-    export LC_ALL="$LOCALE_OVERRIDE"
-    export LANG="$LOCALE_OVERRIDE"
-    PANDAS_LOCALE=`python -c 'import pandas; pandas.get_option("display.encoding")'`
-    if [[ "$LOCALE_OVERRIDE" != "$PANDAS_LOCALE" ]]; then
-        echo "pandas could not detect the locale. System locale: $LOCALE_OVERRIDE, pandas detected: $PANDAS_LOCALE"
-        # TODO Not really aborting the tests until https://github.com/pandas-dev/pandas/issues/23923 is fixed
-        # exit 1
-    fi
-fi
-
 if [[ "not network" == *"$PATTERN"* ]]; then
     export http_proxy=http://1.2.3.4 https_proxy=http://1.2.3.4;
 fi

ci/setup_env.sh

@@ -1,15 +1,15 @@
 #!/bin/bash -e
 
 # edit the locale file if needed
-if [ -n "$LOCALE_OVERRIDE" ]; then
+if [[ "$(uname)" == "Linux" && -n "$LC_ALL" ]]; then
     echo "Adding locale to the first line of pandas/__init__.py"
     rm -f pandas/__init__.pyc
-    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LOCALE_OVERRIDE')\n"
+    SEDC="3iimport locale\nlocale.setlocale(locale.LC_ALL, '$LC_ALL')\n"
     sed -i "$SEDC" pandas/__init__.py
+
     echo "[head -4 pandas/__init__.py]"
     head -4 pandas/__init__.py
     echo
-    sudo locale-gen "$LOCALE_OVERRIDE"
 fi
 
 MINICONDA_DIR="$HOME/miniconda3"

pandas/tests/config/test_localization.py

@@ -8,6 +8,8 @@
 
 from pandas.compat import is_platform_windows
 
+import pandas as pd
+
 _all_locales = get_locales() or []
 _current_locale = locale.getlocale()
 
@@ -56,21 +58,21 @@ def test_get_locales_prefix():
 
 
 @_skip_if_only_one_locale
-def test_set_locale():
+@pytest.mark.parametrize(
+    "lang,enc",
+    [
+        ("it_CH", "UTF-8"),
+        ("en_US", "ascii"),
+        ("zh_CN", "GB2312"),
+        ("it_IT", "ISO-8859-1"),
+    ],
+)
+def test_set_locale(lang, enc):
     if all(x is None for x in _current_locale):
         # Not sure why, but on some Travis runs with pytest,
         #  getlocale() returned (None, None).
         pytest.skip("Current locale is not set.")
 
-    locale_override = os.environ.get("LOCALE_OVERRIDE", None)
-
-    if locale_override is None:
-        lang, enc = "it_CH", "UTF-8"
-    elif locale_override == "C":
-        lang, enc = "en_US", "ascii"
-    else:
-        lang, enc = locale_override.split(".")
-
     enc = codecs.lookup(enc).name
     new_locale = lang, enc
 
@@ -91,3 +93,13 @@ def test_set_locale():
     # Once we exit the "with" statement, locale should be back to what it was.
     current_locale = locale.getlocale()
     assert current_locale == _current_locale
+
+
+def test_encoding_detected():
+    system_locale = os.environ.get("LC_ALL")
+    system_encoding = system_locale.split(".")[-1] if system_locale else "utf-8"
+
+    assert (
+        codecs.lookup(pd.options.display.encoding).name
+        == codecs.lookup(system_encoding).name
+    )

pandas/tests/io/test_common.py

@@ -146,11 +146,15 @@ def test_read_non_existant(self, reader, module, error_class, fn_ext):
         msg3 = "Expected object or value"
         msg4 = "path_or_buf needs to be a string file path or file-like"
         msg5 = (
-            r"\[Errno 2\] File .+does_not_exist\.{} does not exist:"
-            r" '.+does_not_exist\.{}'"
-        ).format(fn_ext, fn_ext)
+            fr"\[Errno 2\] File .+does_not_exist\.{fn_ext} does not exist:"
+            fr" '.+does_not_exist\.{fn_ext}'"
+        )
+        msg6 = fr"\[Errno 2\] 没有那个文件或目录: '.+does_not_exist\.{fn_ext}'"
+        msg7 = (
+            fr"\[Errno 2\] File o directory non esistente: '.+does_not_exist\.{fn_ext}'"
+        )
         with pytest.raises(
-            error_class, match=r"({}|{}|{}|{}|{})".format(msg1, msg2, msg3, msg4, msg5)
+            error_class, match=fr"({msg1}|{msg2}|{msg3}|{msg4}|{msg5}|{msg6}|{msg7})"
         ):
             reader(path)
 
@@ -177,17 +181,21 @@ def test_read_expands_user_home_dir(
         path = os.path.join("~", "does_not_exist." + fn_ext)
         monkeypatch.setattr(icom, "_expand_user", lambda x: os.path.join("foo", x))
 
-        msg1 = r"File (b')?.+does_not_exist\.{}'? does not exist".format(fn_ext)
+        msg1 = fr"File (b')?.+does_not_exist\.{fn_ext}'? does not exist"
         msg2 = fr"\[Errno 2\] No such file or directory: '.+does_not_exist\.{fn_ext}'"
         msg3 = "Unexpected character found when decoding 'false'"
         msg4 = "path_or_buf needs to be a string file path or file-like"
         msg5 = (
-            r"\[Errno 2\] File .+does_not_exist\.{} does not exist:"
-            r" '.+does_not_exist\.{}'"
-        ).format(fn_ext, fn_ext)
+            fr"\[Errno 2\] File .+does_not_exist\.{fn_ext} does not exist:"
+            fr" '.+does_not_exist\.{fn_ext}'"
+        )
+        msg6 = fr"\[Errno 2\] 没有那个文件或目录: '.+does_not_exist\.{fn_ext}'"
+        msg7 = (
+            fr"\[Errno 2\] File o directory non esistente: '.+does_not_exist\.{fn_ext}'"
+        )
 
         with pytest.raises(
-            error_class, match=r"({}|{}|{}|{}|{})".format(msg1, msg2, msg3, msg4, msg5)
+            error_class, match=fr"({msg1}|{msg2}|{msg3}|{msg4}|{msg5}|{msg6}|{msg7})"
         ):
             reader(path)