Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

8263202: Update Hebrew/Indonesian/Yiddish ISO 639 language codes to current #4069

Closed
wants to merge 8 commits into from
Closed
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2012, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2012, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -156,13 +156,6 @@ String getID() {
return id;
}

String getJavaID() {
// Tweak ISO compatibility for bundle generation
return id.replaceFirst("^he", "iw")
.replaceFirst("^id", "in")
.replaceFirst("^yi", "ji");
}

boolean isRoot() {
return "root".equals(id);
}
@@ -536,7 +536,6 @@ private static void convertBundles(List<Bundle> bundles) throws Exception {
Map<String, Object> targetMap = bundle.getTargetMap();
EnumSet<Bundle.Type> bundleTypes = bundle.getBundleTypes();
var id = bundle.getID();
var javaId = bundle.getJavaID();

if (bundle.isRoot()) {
// Add DateTimePatternChars because CLDR no longer supports localized patterns.
@@ -548,31 +547,31 @@ private static void convertBundles(List<Bundle> bundles) throws Exception {
if (bundleTypes.contains(Bundle.Type.LOCALENAMES)) {
Map<String, Object> localeNamesMap = extractLocaleNames(targetMap, id);
if (!localeNamesMap.isEmpty() || bundle.isRoot()) {
bundleGenerator.generateBundle("util", "LocaleNames", javaId, true, localeNamesMap, BundleType.OPEN);
bundleGenerator.generateBundle("util", "LocaleNames", id, true, localeNamesMap, BundleType.OPEN);
}
}
if (bundleTypes.contains(Bundle.Type.CURRENCYNAMES)) {
Map<String, Object> currencyNamesMap = extractCurrencyNames(targetMap, id, bundle.getCurrencies());
if (!currencyNamesMap.isEmpty() || bundle.isRoot()) {
bundleGenerator.generateBundle("util", "CurrencyNames", javaId, true, currencyNamesMap, BundleType.OPEN);
bundleGenerator.generateBundle("util", "CurrencyNames", id, true, currencyNamesMap, BundleType.OPEN);
}
}
if (bundleTypes.contains(Bundle.Type.TIMEZONENAMES)) {
Map<String, Object> zoneNamesMap = extractZoneNames(targetMap, id);
if (!zoneNamesMap.isEmpty() || bundle.isRoot()) {
bundleGenerator.generateBundle("util", "TimeZoneNames", javaId, true, zoneNamesMap, BundleType.TIMEZONE);
bundleGenerator.generateBundle("util", "TimeZoneNames", id, true, zoneNamesMap, BundleType.TIMEZONE);
}
}
if (bundleTypes.contains(Bundle.Type.CALENDARDATA)) {
Map<String, Object> calendarDataMap = extractCalendarData(targetMap, id);
if (!calendarDataMap.isEmpty() || bundle.isRoot()) {
bundleGenerator.generateBundle("util", "CalendarData", javaId, true, calendarDataMap, BundleType.PLAIN);
bundleGenerator.generateBundle("util", "CalendarData", id, true, calendarDataMap, BundleType.PLAIN);
}
}
if (bundleTypes.contains(Bundle.Type.FORMATDATA)) {
Map<String, Object> formatDataMap = extractFormatData(targetMap, id);
if (!formatDataMap.isEmpty() || bundle.isRoot()) {
bundleGenerator.generateBundle("text", "FormatData", javaId, true, formatDataMap, BundleType.PLAIN);
bundleGenerator.generateBundle("text", "FormatData", id, true, formatDataMap, BundleType.PLAIN);
}
}

@@ -1,5 +1,5 @@
/*
* Copyright (c) 1996, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 1996, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -444,18 +444,26 @@
* in <a href="#special_cases_constructor">Special Cases</a>, only
* for the two cases th_TH_TH and ja_JP_JP.
*
* <h4>Legacy language codes</h4>
* <h4><a id="legacy_language_codes">Legacy language codes</a></h4>
*
* <p>Locale's constructor has always converted three language codes to
* their earlier, obsoleted forms: {@code he} maps to {@code iw},
* {@code yi} maps to {@code ji}, and {@code id} maps to
* {@code in}. This continues to be the case, in order to not break
* backwards compatibility.
* {@code in}. Since Java SE 17, this is no longer the case. Each
* language maps to its new form; {@code iw} maps to {@code he}, {@code ji}
* maps to {@code yi}, and {@code in} maps to {@code id}.
*
* <p>For the backward compatible behavior, the system property
* {@systemProperty java.locale.useOldISOCodes} reverts the behavior
* back to prior to Java SE 17 one. If the system property is set
* to {@code true}, those three current language codes are mapped to their
* backward compatible forms.
*
* <p>The APIs added in 1.7 map between the old and new language codes,

This comment has been minimized.

@JoeWang-Java

JoeWang-Java May 18, 2021
Member

This paragraph needs a rewrite as well it seems, esp. the part that states "getLanguage and toString reflect the old code" is no longer true.

This comment has been minimized.

@naotoj

naotoj May 18, 2021
Author Member

Good catch! In fact, I had modified this paragraph in my preliminary fix, but it slipped away somehow along the fix. Corrected the PR and CSR as well.

* maintaining the old codes internal to Locale (so that
* {@code getLanguage} and {@code toString} reflect the old
* code), but using the new codes in the BCP 47 language tag APIs (so
* maintaining the mapped codes internal to Locale (so that
* {@code getLanguage} and {@code toString} reflect the mapped
* code, which depends on the {@code java.locale.useOldISOCodes} system
* property), but using the new codes in the BCP 47 language tag APIs (so
* that {@code toLanguageTag} reflects the new one). This
* preserves the equivalence between Locales no matter which code or
* API is used to construct them. Java's default resource bundle
@@ -720,13 +728,11 @@ private Locale(BaseLocale baseLocale, LocaleExtensions extensions) {
* Construct a locale from language, country and variant.
* This constructor normalizes the language value to lowercase and
* the country value to uppercase.
* <p>
* <b>Note:</b>
* @implNote
* <ul>
* <li>ISO 639 is not a stable standard; some of the language codes it defines
* (specifically "iw", "ji", and "in") have changed. This constructor accepts both the
* old codes ("iw", "ji", and "in") and the new codes ("he", "yi", and "id"), but all other
* API on Locale will return only the OLD codes.
* <li>Obsolete ISO 639 codes ("iw", "ji", and "in") are mapped to
* their current forms. See <a href="#legacy_language_codes">Legacy language
* codes</a> for more information.
* <li>For backward compatibility reasons, this constructor does not make
* any syntactic checks on the input.
* <li>The two cases ("ja", "JP", "JP") and ("th", "TH", "TH") are handled specially,
@@ -754,13 +760,11 @@ public Locale(String language, String country, String variant) {
* Construct a locale from language and country.
* This constructor normalizes the language value to lowercase and
* the country value to uppercase.
* <p>
* <b>Note:</b>
* @implNote
* <ul>
* <li>ISO 639 is not a stable standard; some of the language codes it defines
* (specifically "iw", "ji", and "in") have changed. This constructor accepts both the
* old codes ("iw", "ji", and "in") and the new codes ("he", "yi", and "id"), but all other
* API on Locale will return only the OLD codes.
* <li>Obsolete ISO 639 codes ("iw", "ji", and "in") are mapped to
* their current forms. See <a href="#legacy_language_codes">Legacy language
* codes</a> for more information.
* <li>For backward compatibility reasons, this constructor does not make
* any syntactic checks on the input.
* </ul>
@@ -779,13 +783,11 @@ public Locale(String language, String country) {
/**
* Construct a locale from a language code.
* This constructor normalizes the language value to lowercase.
* <p>
* <b>Note:</b>
* @implNote
* <ul>
* <li>ISO 639 is not a stable standard; some of the language codes it defines
* (specifically "iw", "ji", and "in") have changed. This constructor accepts both the
* old codes ("iw", "ji", and "in") and the new codes ("he", "yi", and "id"), but all other
* API on Locale will return only the OLD codes.
* <li>Obsolete ISO 639 codes ("iw", "ji", and "in") are mapped to
* their current forms. See <a href="#legacy_language_codes">Legacy language
* codes</a> for more information.
* <li>For backward compatibility reasons, this constructor does not make
* any syntactic checks on the input.
* </ul>
@@ -1208,19 +1210,10 @@ public static synchronized void setDefault(Locale.Category category,
/**
* Returns the language code of this Locale.
*
* <p><b>Note:</b> ISO 639 is not a stable standard&mdash; some languages' codes have changed.
* Locale's constructor recognizes both the new and the old codes for the languages
* whose codes have changed, but this function always returns the old code. If you
* want to check for a specific language whose code has changed, don't do
* <pre>
* if (locale.getLanguage().equals("he")) // BAD!
* ...
* </pre>
* Instead, do
* <pre>
* if (locale.getLanguage().equals(new Locale("he").getLanguage()))
* ...
* </pre>
* @implNote This method returns the new forms for the obsolete ISO 639
* codes ("iw", "ji", and "in"). See <a href="#legacy_language_codes">
* Legacy language codes</a> for more information.
*
* @return The language code, or the empty string if none is defined.
* @see #getDisplayLanguage
*/
@@ -1608,9 +1601,11 @@ public String toLanguageTag() {
*
* <li>The language code "und" is mapped to language "".
*
* <li>The language codes "he", "yi", and "id" are mapped to "iw",
* "ji", and "in" respectively. (This is the same canonicalization
* that's done in Locale's constructors.)
* <li>The language codes "iw", "ji", and "in" are mapped to "he",
* "yi", and "id" respectively. (This is the same canonicalization
* that's done in Locale's constructors.) See
* <a href="#legacy_language_codes">Legacy language codes</a>
* for more information.
*
* <li>The portion of a private use subtag prefixed by "lvariant",
* if any, is removed and appended to the variant field in the
@@ -2397,17 +2392,9 @@ private Object readResolve() throws java.io.ObjectStreamException {

private static String convertOldISOCodes(String language) {
// we accept both the old and the new ISO codes for the languages whose ISO
// codes have changed, but we always store the OLD code, for backward compatibility
language = LocaleUtils.toLowerString(language).intern();
if (language == "he") {
return "iw";
} else if (language == "yi") {
return "ji";
} else if (language == "id") {
return "in";
} else {
return language;
}
// codes have changed, but we always store the NEW code, unless the property
// java.locale.useOldISOCodes is set to "true"
return BaseLocale.convertOldISOCodes(LocaleUtils.toLowerString(language).intern());
}

private static LocaleExtensions getCompatibilityExtensions(String language,
@@ -72,6 +72,8 @@
import sun.security.action.GetPropertyAction;
import sun.util.locale.BaseLocale;
import sun.util.locale.LocaleObjectCache;
import sun.util.resources.Bundles;

import static sun.security.util.SecurityConstants.GET_CLASSLOADER_PERMISSION;


@@ -3098,6 +3100,12 @@ public Locale getFallbackLocale(String baseName, Locale locale) {
* nor {@code "java.properties"}, an
* {@code IllegalArgumentException} is thrown.</li>
*
* <li>If the {@code locale}'s language is one of the
* <a href="./Locale.html#legacy_language_codes">Legacy language
* codes</a>, either old or new, then repeat the loading process
* if needed, with the bundle name with the other language.
* For example, "iw" for "he" and vice versa.
*
* </ul>
*
* @param baseName
@@ -3152,6 +3160,21 @@ public ResourceBundle newBundle(String baseName, Locale locale, String format,
* that is visible to the given loader and accessible to the given caller.
*/
String bundleName = toBundleName(baseName, locale);
var bundle = newBundle0(bundleName, format, loader, reload);
if (bundle == null) {
// Try loading legacy ISO language's other bundles
var otherBundleName = Bundles.toOtherBundleName(baseName, bundleName, locale);
if (!bundleName.equals(otherBundleName)) {
bundle = newBundle0(otherBundleName, format, loader, reload);
}
}

return bundle;
}

private ResourceBundle newBundle0(String bundleName, String format,
ClassLoader loader, boolean reload)
throws IllegalAccessException, InstantiationException, IOException {
ResourceBundle bundle = null;
if (format.equals("java.class")) {
try {
@@ -1,5 +1,5 @@
/*
* Copyright (c) 2015, 2020, Oracle and/or its affiliates. All rights reserved.
* Copyright (c) 2015, 2021, Oracle and/or its affiliates. All rights reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
@@ -27,6 +27,7 @@

import jdk.internal.access.JavaUtilResourceBundleAccess;
import jdk.internal.access.SharedSecrets;
import sun.util.resources.Bundles;

import java.io.IOException;
import java.io.InputStream;
@@ -187,6 +188,17 @@ protected String toBundleName(String baseName, Locale locale) {
public ResourceBundle getBundle(String baseName, Locale locale) {
Module module = this.getClass().getModule();
String bundleName = toBundleName(baseName, locale);
var bundle = getBundle0(module, bundleName);
if (bundle == null) {
var otherBundleName = Bundles.toOtherBundleName(baseName, bundleName, locale);
if (!bundleName.equals(otherBundleName)) {
bundle = getBundle0(module, Bundles.toOtherBundleName(baseName, bundleName, locale));
}
}
return bundle;
}

private ResourceBundle getBundle0(Module module, String bundleName) {
ResourceBundle bundle = null;

for (String format : formats) {
@@ -34,6 +34,7 @@

import jdk.internal.misc.CDS;
import jdk.internal.vm.annotation.Stable;
import sun.security.action.GetPropertyAction;

import java.lang.ref.SoftReference;
import java.util.StringJoiner;
@@ -98,6 +99,13 @@

private volatile int hash;

/**
* Boolean for the old ISO language code compatibility.
*/
private static final boolean OLD_ISO_CODES = GetPropertyAction.privilegedGetProperties()
.getProperty("java.locale.useOldISOCodes", "false")
.equalsIgnoreCase("true");

// This method must be called with normalize = false only when creating the
// Locale.* constants and non-normalized BaseLocale$Keys used for lookup.
private BaseLocale(String language, String script, String region, String variant,
@@ -153,19 +161,22 @@ public static BaseLocale getInstance(String language, String script,

// JDK uses deprecated ISO639.1 language codes for he, yi and id
if (!language.isEmpty()) {
if (language.equals("he")) {
language = "iw";
} else if (language.equals("yi")) {
language = "ji";
} else if (language.equals("id")) {
language = "in";
}
language = convertOldISOCodes(language);
}

Key key = new Key(language, script, region, variant, false);
return Cache.CACHE.get(key);
}

public static String convertOldISOCodes(String language) {
return switch (language) {
case "he", "iw" -> OLD_ISO_CODES ? "iw" : "he";
case "id", "in" -> OLD_ISO_CODES ? "in" : "id";
case "yi", "ji" -> OLD_ISO_CODES ? "ji" : "yi";
default -> language;
};
}

public String getLanguage() {
return language;
}