8275007: Java fails to start with null charset if LC_ALL is set to ce…

…rtain locales

Reviewed-by: rriggs, iris, joehw, alanb

src/java.base/share/classes/java/lang/System.java

@@ -33,7 +33,6 @@
 import java.io.IOException;
 import java.io.InputStream;
 import java.io.PrintStream;
-import java.io.UnsupportedEncodingException;
 import java.lang.annotation.Annotation;
 import java.lang.invoke.MethodHandle;
 import java.lang.invoke.MethodType;
@@ -45,9 +44,9 @@
 import java.lang.reflect.Modifier;
 import java.net.URI;
 import java.net.URL;
-import java.nio.charset.CharacterCodingException;
 import java.nio.channels.Channel;
 import java.nio.channels.spi.SelectorProvider;
+import java.nio.charset.CharacterCodingException;
 import java.nio.charset.Charset;
 import java.security.AccessControlContext;
 import java.security.AccessController;
@@ -84,6 +83,7 @@
 import sun.nio.fs.DefaultFileSystemProvider;
 import sun.reflect.annotation.AnnotationType;
 import sun.nio.ch.Interruptible;
+import sun.nio.cs.UTF_8;
 import sun.security.util.SecurityConstants;
 
 /**
@@ -188,6 +188,11 @@ private System() {
     @SuppressWarnings("removal")
     private static volatile SecurityManager security;   // read by VM
 
+    // `sun.jnu.encoding` if it is not supported. Otherwise null.
+    // It is initialized in `initPhase1()` before any charset providers
+    // are initialized.
+    private static String notSupportedJnuEncoding;
+
     // return true if a security manager is allowed
     private static boolean allowSecurityManager() {
         return (allowSecurityManager != NEVER);
@@ -2017,10 +2022,9 @@ public static void loadLibrary(String libname) {
      * Create PrintStream for stdout/err based on encoding.
      */
     private static PrintStream newPrintStream(FileOutputStream fos, String enc) {
-       if (enc != null) {
-            try {
-                return new PrintStream(new BufferedOutputStream(fos, 128), true, enc);
-            } catch (UnsupportedEncodingException uee) {}
+        if (enc != null) {
+            return new PrintStream(new BufferedOutputStream(fos, 128), true,
+                                   Charset.forName(enc, UTF_8.INSTANCE));
         }
         return new PrintStream(new BufferedOutputStream(fos, 128), true);
     }
@@ -2113,6 +2117,13 @@ private static void initPhase1() {
         VM.saveProperties(tempProps);
         props = createProperties(tempProps);
 
+        // Check if sun.jnu.encoding is supported. If not, replace it with UTF-8.
+        var jnuEncoding = props.getProperty("sun.jnu.encoding");
+        if (jnuEncoding == null || !Charset.isSupported(jnuEncoding)) {
+            notSupportedJnuEncoding = jnuEncoding == null ? "null" : jnuEncoding;
+            props.setProperty("sun.jnu.encoding", "UTF-8");
+        }
+
         StaticProperty.javaHome();          // Load StaticProperty to cache the property values
 
         lineSeparator = props.getProperty("line.separator");
@@ -2141,7 +2152,6 @@ private static void initPhase1() {
         Thread current = Thread.currentThread();
         current.getThreadGroup().add(current);
 
-
         // Subsystems that are invoked during initialization can invoke
         // VM.isBooted() in order to avoid doing things that should
         // wait until the VM is fully initialized. The initialization level
@@ -2248,6 +2258,14 @@ private static void initPhase3() {
                     WARNING: The Security Manager is deprecated and will be removed in a future release""");
         }
 
+        // Emit a warning if `sun.jnu.encoding` is not supported.
+        if (notSupportedJnuEncoding != null) {
+            System.err.println(
+                    "WARNING: The encoding of the underlying platform's" +
+                    " file system is not supported: " +
+                    notSupportedJnuEncoding);
+        }
+
         initialErrStream = System.err;
 
         // initializing the system class loader

src/java.base/share/native/libjava/jni_util.c

@@ -639,21 +639,6 @@ static jmethodID String_getBytes_ID;    /* String.getBytes(enc) */
 static jfieldID String_coder_ID;        /* String.coder */
 static jfieldID String_value_ID;        /* String.value */
 
-static jboolean isJNUEncodingSupported = JNI_FALSE;
-static jboolean jnuEncodingSupported(JNIEnv *env) {
-    jboolean exe;
-    if (isJNUEncodingSupported == JNI_TRUE) {
-        return JNI_TRUE;
-    }
-    isJNUEncodingSupported = (jboolean) JNU_CallStaticMethodByName (
-                                    env, &exe,
-                                    "java/nio/charset/Charset",
-                                    "isSupported",
-                                    "(Ljava/lang/String;)Z",
-                                    jnuEncoding).z;
-    return isJNUEncodingSupported;
-}
-
 /* Create a new string by converting str to a heap-allocated byte array and
  * calling the appropriate String constructor.
  */
@@ -671,22 +656,8 @@ newSizedStringJava(JNIEnv *env, const char *str, const int len)
         jclass strClazz = JNU_ClassString(env);
         CHECK_NULL_RETURN(strClazz, 0);
         (*env)->SetByteArrayRegion(env, bytes, 0, len, (jbyte *)str);
-        if (jnuEncodingSupported(env)) {
-            result = (*env)->NewObject(env, strClazz,
-                                       String_init_ID, bytes, jnuEncoding);
-        } else {
-            /*If the encoding specified in sun.jnu.encoding is not endorsed
-              by "Charset.isSupported" we have to fall back to use String(byte[])
-              explicitly here without specifying the encoding name, in which the
-              StringCoding class will pickup the iso-8859-1 as the fallback
-              converter for us.
-             */
-            jmethodID mid = (*env)->GetMethodID(env, strClazz,
-                                                "<init>", "([B)V");
-            if (mid != NULL) {
-                result = (*env)->NewObject(env, strClazz, mid, bytes);
-            }
-        }
+        result = (*env)->NewObject(env, strClazz,
+                                   String_init_ID, bytes, jnuEncoding);
         (*env)->DeleteLocalRef(env, bytes);
         return result;
     }
@@ -766,11 +737,30 @@ InitializeEncoding(JNIEnv *env, const char *encname)
             strcmp(encname, "utf-16le") == 0) {
             fastEncoding = FAST_CP1252;
         } else {
+            jboolean exe;
             jstring enc = (*env)->NewStringUTF(env, encname);
             if (enc == NULL)
                 return;
-            fastEncoding = NO_FAST_ENCODING;
-            jnuEncoding = (jstring)(*env)->NewGlobalRef(env, enc);
+
+            if ((jboolean) JNU_CallStaticMethodByName (
+                                            env, &exe,
+                                            "java/nio/charset/Charset",
+                                            "isSupported",
+                                            "(Ljava/lang/String;)Z",
+                                            enc).z == JNI_TRUE) {
+                fastEncoding = NO_FAST_ENCODING;
+                jnuEncoding = (jstring)(*env)->NewGlobalRef(env, enc);
+            } else {
+                // jnuEncoding falls back to UTF-8
+                jstring utf8 = (*env)->NewStringUTF(env, "UTF-8");
+                if (utf8 == NULL) {
+                    (*env)->DeleteLocalRef(env, enc);
+                    return;
+                }
+                fastEncoding = FAST_UTF_8;
+                jnuEncoding = (jstring)(*env)->NewGlobalRef(env, utf8);
+                (*env)->DeleteLocalRef(env, utf8);
+            }
             (*env)->DeleteLocalRef(env, enc);
         }
     } else {
@@ -822,32 +812,22 @@ static const char* getStringBytes(JNIEnv *env, jstring jstr) {
     if ((*env)->EnsureLocalCapacity(env, 2) < 0)
         return 0;
 
-    if (jnuEncodingSupported(env)) {
-        hab = (*env)->CallObjectMethod(env, jstr, String_getBytes_ID, jnuEncoding);
-    } else {
-        jmethodID mid;
-        jclass strClazz = JNU_ClassString(env);
-        CHECK_NULL_RETURN(strClazz, 0);
-        mid = (*env)->GetMethodID(env, strClazz,
-                                       "getBytes", "()[B");
-        if (mid != NULL) {
-            hab = (*env)->CallObjectMethod(env, jstr, mid);
+    hab = (*env)->CallObjectMethod(env, jstr, String_getBytes_ID, jnuEncoding);
+    if (hab != 0) {
+        if (!(*env)->ExceptionCheck(env)) {
+            jint len = (*env)->GetArrayLength(env, hab);
+            result = MALLOC_MIN4(len);
+            if (result == 0) {
+                JNU_ThrowOutOfMemoryError(env, 0);
+                (*env)->DeleteLocalRef(env, hab);
+                return 0;
+            }
+            (*env)->GetByteArrayRegion(env, hab, 0, len, (jbyte *)result);
+            result[len] = 0; /* NULL-terminate */
         }
-    }
 
-    if (!(*env)->ExceptionCheck(env)) {
-        jint len = (*env)->GetArrayLength(env, hab);
-        result = MALLOC_MIN4(len);
-        if (result == 0) {
-            JNU_ThrowOutOfMemoryError(env, 0);
-            (*env)->DeleteLocalRef(env, hab);
-            return 0;
-        }
-        (*env)->GetByteArrayRegion(env, hab, 0, len, (jbyte *)result);
-        result[len] = 0; /* NULL-terminate */
+        (*env)->DeleteLocalRef(env, hab);
     }
-
-    (*env)->DeleteLocalRef(env, hab);
     return result;
 }