8258855: Two tests sun/security/krb5/auto/ReplayCacheTestProc.java and ReplayCacheTestProcWithMD5.java failed on OL8.3

Reviewed-by: mbalao, serb
Backport-of: 8d780729c65bbfd19f23004723b8786d52f8310a

Author: gnu-andrew

jdk/test/sun/security/krb5/auto/ReplayCacheTestProc.java

@@ -27,7 +27,10 @@
  * @summary More krb5 tests
  * @library ../../../../java/security/testlibrary/ /test/lib
  * @compile -XDignore.symbol.file ReplayCacheTestProc.java
- * @run main/othervm/timeout=300 -Dsun.net.spi.nameservice.provider.1=ns,mock ReplayCacheTestProc
+ * @run main/othervm/timeout=300 -Dsun.net.spi.nameservice.provider.1=ns,mock
+ *      -Dtest.libs=J ReplayCacheTestProc
+ * @run main/othervm/timeout=300 -Dsun.net.spi.nameservice.provider.1=ns,mock
+ *      -Dtest.libs=N ReplayCacheTestProc
  */
 
 import java.io.*;
@@ -43,6 +46,7 @@
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
 
+import jdk.test.lib.Asserts;
 import jdk.test.lib.Platform;
 import sun.security.jgss.GSSUtil;
 import sun.security.krb5.internal.rcache.AuthTime;
@@ -51,9 +55,8 @@
  * This test runs multiple acceptor Procs to mimic AP-REQ replays.
  * These system properties are supported:
  *
- * - test.libs on what types of acceptors to use
+ * - test.libs on what types of acceptors to use. Cannot be null.
  *   Format: CSV of (J|N|N<suffix>=<libname>|J<suffix>=<launcher>)
- *   Default: J,N on Solaris and Linux where N is available, or J
  *   Example: J,N,N14=/krb5-1.14/lib/libgssapi_krb5.so,J8=/java8/bin/java
  *
  * - test.runs on manual runs. If empty, a iterate through all pattern
@@ -117,6 +120,16 @@ public static void main0(String[] args) throws Exception {
                 uid = -1;
             }
 
+            // User-provided libs
+            String userLibs = System.getProperty("test.libs");
+            Asserts.assertNotNull(userLibs, "test.libs property must be provided");
+            libs = userLibs.split(",");
+            if (Arrays.asList(libs).contains("N") && !isNativeLibAvailable()) {
+                // Skip test when native GSS libs are not available in running platform
+                System.out.println("Native mode not available - skipped");
+                return;
+            }
+
             KDC kdc = KDC.create(OneKDC.REALM, HOST, 0, true);
             for (int i=0; i<nc; i++) {
                 kdc.addPrincipal(client(i), OneKDC.PASS);
@@ -129,25 +142,6 @@ public static void main0(String[] args) throws Exception {
             kdc.writeKtab(OneKDC.KTAB);
             KDC.saveConfig(OneKDC.KRB5_CONF, kdc);
 
-            // User-provided libs
-            String userLibs = System.getProperty("test.libs");
-
-            if (userLibs != null) {
-                libs = userLibs.split(",");
-            } else {
-                if (Platform.isOSX() || Platform.isWindows()) {
-                    // macOS uses Heimdal and Windows has no native lib
-                    libs = new String[]{"J"};
-                } else {
-                    if (acceptor("N", "sanity").waitFor() != 0) {
-                        Proc.d("Native mode sanity check failed, only java");
-                        libs = new String[]{"J"};
-                    } else {
-                        libs = new String[]{"J", "N"};
-                    }
-                }
-            }
-
             pi = Proc.create("ReplayCacheTestProc").debug("C")
                     .args("initiator")
                     .start();
@@ -323,6 +317,13 @@ public static void main(String[] args) throws Exception {
         }
     }
 
+    // returns true if native lib is available in running platform
+    // macOS uses Heimdal and Windows has no native lib
+    private static boolean isNativeLibAvailable() throws Exception {
+        return !Platform.isOSX() && !Platform.isWindows()
+                && acceptor("N", "sanity").waitFor() == 0;
+    }
+
     // returns the client name
     private static String client(int p) {
         return "client" + p;

jdk/test/sun/security/krb5/auto/rcache_usemd5.sh

@@ -24,7 +24,9 @@
 # @test
 # @bug 8168518
 # @library ../../../../java/security/testlibrary/ /test/lib
-# @run main/othervm/timeout=300 -Dsun.net.spi.nameservice.provider.1=ns,mock -Djdk.krb5.rcache.useMD5=true ReplayCacheTestProc
+# @run main/othervm/timeout=300 -Dsun.net.spi.nameservice.provider.1=ns,mock
+#           -Djdk.krb5.rcache.useMD5=true
+#           -Dtest.libs=J ReplayCacheTestProc
 # @summary  testing jdk.krb5.rcache.useMD5. This action is put in a separate
 #           test so that ReplayCacheTestProc.java can be launched with special
 #           test.* system properties easily.