8366261: Provide utility methods for sun.security.util.Password

Reviewed-by: smarks, weijun

Author: naotoj

src/java.base/share/classes/java/io/Console.java

@@ -25,6 +25,7 @@
 
 package java.io;
 
+import java.lang.annotation.Native;
 import java.util.*;
 import java.nio.charset.Charset;
 import jdk.internal.access.JavaIOAccess;
@@ -550,7 +551,12 @@ private static UnsupportedOperationException newUnsupportedOperationException()
                 "Console class itself does not provide implementation");
     }
 
-    private static final boolean istty = istty();
+    @Native static final int TTY_STDIN_MASK = 0x00000001;
+    @Native static final int TTY_STDOUT_MASK = 0x00000002;
+    @Native static final int TTY_STDERR_MASK = 0x00000004;
+    // ttyStatus() returns bit patterns above, a bit is set if the corresponding file
+    // descriptor is a character device
+    private static final int ttyStatus = ttyStatus();
     private static final Charset STDIN_CHARSET =
         Charset.forName(StaticProperty.stdinEncoding(), UTF_8.INSTANCE);
     private static final Charset STDOUT_CHARSET =
@@ -562,6 +568,9 @@ private static UnsupportedOperationException newUnsupportedOperationException()
             public Console console() {
                 return cons;
             }
+            public boolean isStdinTty() {
+                return Console.isStdinTty();
+            }
         });
     }
 
@@ -583,7 +592,7 @@ private static Console instantiateConsole() {
 
             for (var jcp : ServiceLoader.load(ModuleLayer.boot(), JdkConsoleProvider.class)) {
                 if (consModName.equals(jcp.getClass().getModule().getName())) {
-                    var jc = jcp.console(istty, STDIN_CHARSET, STDOUT_CHARSET);
+                    var jc = jcp.console(isStdinTty() && isStdoutTty(), STDIN_CHARSET, STDOUT_CHARSET);
                     if (jc != null) {
                         c = new ProxyingConsole(jc);
                     }
@@ -594,12 +603,21 @@ private static Console instantiateConsole() {
         }
 
         // If not found, default to built-in Console
-        if (istty && c == null) {
+        if (isStdinTty() && isStdoutTty() && c == null) {
             c = new ProxyingConsole(new JdkConsoleImpl(STDIN_CHARSET, STDOUT_CHARSET));
         }
 
         return c;
     }
 
-    private static native boolean istty();
+    private static boolean isStdinTty() {
+        return (ttyStatus & TTY_STDIN_MASK) != 0;
+    }
+    private static boolean isStdoutTty() {
+        return (ttyStatus & TTY_STDOUT_MASK) != 0;
+    }
+    private static boolean isStderrTty() {
+        return (ttyStatus & TTY_STDERR_MASK) != 0;
+    }
+    private static native int ttyStatus();
 }

src/java.base/share/classes/jdk/internal/access/JavaIOAccess.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -29,4 +29,5 @@
 
 public interface JavaIOAccess {
     Console console();
+    boolean isStdinTty();
 }

src/java.base/share/classes/jdk/internal/io/JdkConsoleImpl.java

@@ -38,10 +38,13 @@
 import java.util.Formatter;
 import java.util.Locale;
 import java.util.Objects;
+import java.util.Optional;
 
 import jdk.internal.access.SharedSecrets;
+import jdk.internal.util.StaticProperty;
 import sun.nio.cs.StreamDecoder;
 import sun.nio.cs.StreamEncoder;
+import sun.nio.cs.UTF_8;
 
 /**
  * JdkConsole implementation based on the platform's TTY.
@@ -103,6 +106,42 @@ public String readLine() {
 
     @Override
     public char[] readPassword(Locale locale, String format, Object ... args) {
+        return readPassword0(false, locale, format, args);
+    }
+
+    // These two methods are intended for sun.security.util.Password, so tools like keytool can
+    // use JdkConsoleImpl even when standard output is redirected. The Password class should first
+    // check if `System.console()` returns a Console instance and use it if available. Otherwise,
+    // it should call this method to obtain a JdkConsoleImpl. This ensures only one Console
+    // instance exists in the Java runtime.
+    private static final StableValue<Optional<JdkConsoleImpl>> INSTANCE = StableValue.of();
+    public static Optional<JdkConsoleImpl> passwordConsole() {
+        return INSTANCE.orElseSet(() -> {
+            // If there's already a proper console, throw an exception
+            if (System.console() != null) {
+                throw new IllegalStateException("Can’t create a dedicated password " +
+                    "console since a real console already exists");
+            }
+
+            // If stdin is NOT redirected, return an Optional containing a JdkConsoleImpl
+            // instance, otherwise an empty Optional.
+            return SharedSecrets.getJavaIOAccess().isStdinTty() ?
+                Optional.of(
+                    new JdkConsoleImpl(
+                        Charset.forName(StaticProperty.stdinEncoding(), UTF_8.INSTANCE),
+                        Charset.forName(StaticProperty.stdoutEncoding(), UTF_8.INSTANCE))) :
+                Optional.empty();
+        });
+    }
+
+    // Dedicated entry for sun.security.util.Password when stdout is redirected.
+    // This method strictly avoids producing any output by using noNewLine = true
+    // and an empty format string.
+    public char[] readPasswordNoNewLine() {
+        return readPassword0(true, Locale.getDefault(Locale.Category.FORMAT), "");
+    }
+
+    private char[] readPassword0(boolean noNewLine, Locale locale, String format, Object ... args) {
         char[] passwd = null;
         synchronized (writeLock) {
             synchronized(readLock) {
@@ -146,7 +185,9 @@ public char[] readPassword(Locale locale, String format, Object ... args) {
                         throw ioe;
                     }
                 }
-                pw.println();
+                if (!noNewLine) {
+                    pw.println();
+                }
             }
         }
         return passwd;

src/java.base/unix/native/libjava/Console_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,8 +31,19 @@
 #include <stdlib.h>
 #include <unistd.h>
 
-JNIEXPORT jboolean JNICALL
-Java_java_io_Console_istty(JNIEnv *env, jclass cls)
+JNIEXPORT jint JNICALL
+Java_java_io_Console_ttyStatus(JNIEnv *env, jclass cls)
 {
-    return isatty(fileno(stdin)) && isatty(fileno(stdout));
+    jint ret = 0;
+
+    if (isatty(fileno(stdin))) {
+        ret |= java_io_Console_TTY_STDIN_MASK;
+    }
+    if (isatty(fileno(stdout))) {
+        ret |= java_io_Console_TTY_STDOUT_MASK;
+    }
+    if (isatty(fileno(stderr))) {
+        ret |= java_io_Console_TTY_STDERR_MASK;
+    }
+    return ret;
 }

src/java.base/windows/native/libjava/Console_md.c

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2005, 2024, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2005, 2025, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -31,21 +31,28 @@
 #include <stdlib.h>
 #include <Wincon.h>
 
-JNIEXPORT jboolean JNICALL
-Java_java_io_Console_istty(JNIEnv *env, jclass cls)
+JNIEXPORT jint JNICALL
+Java_java_io_Console_ttyStatus(JNIEnv *env, jclass cls)
 {
+    jint ret = 0;
     HANDLE hStdOut = GetStdHandle(STD_OUTPUT_HANDLE);
     HANDLE hStdIn = GetStdHandle(STD_INPUT_HANDLE);
+    HANDLE hStdErr = GetStdHandle(STD_ERROR_HANDLE);
 
-    if (hStdIn == INVALID_HANDLE_VALUE ||
-        hStdOut == INVALID_HANDLE_VALUE) {
-        return JNI_FALSE;
+    if (hStdIn != INVALID_HANDLE_VALUE &&
+        GetFileType(hStdIn) == FILE_TYPE_CHAR) {
+        ret |= java_io_Console_TTY_STDIN_MASK;
     }
 
-    if (GetFileType(hStdIn) != FILE_TYPE_CHAR ||
-        GetFileType(hStdOut) != FILE_TYPE_CHAR) {
-        return JNI_FALSE;
+    if (hStdOut != INVALID_HANDLE_VALUE &&
+        GetFileType(hStdOut) == FILE_TYPE_CHAR) {
+        ret |= java_io_Console_TTY_STDOUT_MASK;
     }
 
-    return JNI_TRUE;
+    if (hStdErr != INVALID_HANDLE_VALUE &&
+        GetFileType(hStdErr) == FILE_TYPE_CHAR) {
+        ret |= java_io_Console_TTY_STDERR_MASK;
+    }
+
+    return ret;
 }

test/jdk/java/io/Console/ModuleSelectionTest.java

@@ -21,9 +21,9 @@
  * questions.
  */
 
-/**
+/*
  * @test
- * @bug 8295803 8299689 8351435 8361613
+ * @bug 8295803 8299689 8351435 8361613 8366261
  * @summary Tests System.console() returns correct Console (or null) from the expected
  *          module.
  * @library /test/lib
@@ -92,9 +92,12 @@ public static void main(String... args) throws Throwable {
         var con = System.console();
         var pc = Class.forName("java.io.ProxyingConsole");
         var jdkc = Class.forName("jdk.internal.io.JdkConsole");
-        var istty = (boolean)MethodHandles.privateLookupIn(Console.class, MethodHandles.lookup())
-                .findStatic(Console.class, "istty", MethodType.methodType(boolean.class))
-                .invoke();
+        var lookup = MethodHandles.privateLookupIn(Console.class, MethodHandles.lookup());
+        var istty = (boolean)lookup.findStatic(Console.class, "isStdinTty", MethodType.methodType(boolean.class))
+                        .invoke() &&
+                   (boolean)lookup.findStatic(Console.class, "isStdoutTty", MethodType.methodType(boolean.class))
+                        .invoke();
+
         var impl = con != null ? MethodHandles.privateLookupIn(pc, MethodHandles.lookup())
                 .findGetter(pc, "delegate", jdkc)
                 .invoke(con) : null;