8310380: Handle problems in core-related tests on macOS when codesign tool does not work

Reviewed-by: lucy
Backport-of: 39c104df44f17c1d65e35becd4272f73e2c6610c

Author: Andrew Lu

test/hotspot/jtreg/serviceability/sa/TestJmapCoreMetaspace.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2018, 2020, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2018, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -23,7 +23,7 @@
 
 /**
  * @test TestJmapCoreMetaspace
- * @summary Test verifies that jhsdb jmap could generate heap dump from core when metspace is full
+ * @summary Test verifies that jhsdb jmap could generate heap dump from core when metaspace is full
  * @requires vm.hasSA
  * @library /test/lib
  * @run driver/timeout=480 TestJmapCore run metaspace

test/hotspot/jtreg/testlibrary_tests/TestMutuallyExclusivePlatformPredicates.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2014, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2014, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -52,7 +52,7 @@ private static enum MethodGroup {
         MODE("isInt", "isMixed", "isComp"),
         IGNORED("isEmulatedClient", "isDebugBuild", "isFastDebugBuild", "isSlowDebugBuild",
                 "hasSA", "isRoot", "isTieredSupported", "areCustomLoadersSupportedForCDS",
-                "isHardenedOSX");
+                "isHardenedOSX", "hasOSXPlistEntries");
 
         public final List<String> methodNames;
 

test/lib/jdk/test/lib/Platform.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2013, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2013, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -255,28 +255,46 @@ public static boolean hasSA() {
         return true;
     }
 
-    /**
-     * Return true if the test JDK is hardened, otherwise false. Only valid on OSX.
-     */
-    public static boolean isHardenedOSX() throws IOException {
-        // We only care about hardened binaries for 10.14 and later (actually 10.14.5, but
-        // for simplicity we'll also include earlier 10.14 versions).
-        if (getOsVersionMajor() == 10 && getOsVersionMinor() < 14) {
-            return false; // assume not hardened
-        }
-
-        // Find the path to the java binary.
+    private static Process launchCodesignOnJavaBinary() throws IOException {
         String jdkPath = System.getProperty("java.home");
         Path javaPath = Paths.get(jdkPath + "/bin/java");
         String javaFileName = javaPath.toAbsolutePath().toString();
         if (!javaPath.toFile().exists()) {
             throw new FileNotFoundException("Could not find file " + javaFileName);
         }
-
-        // Run codesign on the java binary.
         ProcessBuilder pb = new ProcessBuilder("codesign", "--display", "--verbose", javaFileName);
         pb.redirectErrorStream(true); // redirect stderr to stdout
         Process codesignProcess = pb.start();
+        return codesignProcess;
+    }
+
+    public static boolean hasOSXPlistEntries() throws IOException {
+        Process codesignProcess = launchCodesignOnJavaBinary();
+        BufferedReader is = new BufferedReader(new InputStreamReader(codesignProcess.getInputStream()));
+        String line;
+        while ((line = is.readLine()) != null) {
+            System.out.println("STDOUT: " + line);
+            if (line.indexOf("Info.plist=not bound") != -1) {
+                return false;
+            }
+            if (line.indexOf("Info.plist entries=") != -1) {
+                return true;
+            }
+        }
+        System.out.println("No matching Info.plist entry was found");
+        return false;
+    }
+
+    /**
+     * Return true if the test JDK is hardened, otherwise false. Only valid on OSX.
+     */
+    public static boolean isHardenedOSX() throws IOException {
+        // We only care about hardened binaries for 10.14 and later (actually 10.14.5, but
+        // for simplicity we'll also include earlier 10.14 versions).
+        if (getOsVersionMajor() == 10 && getOsVersionMinor() < 14) {
+            return false; // assume not hardened
+        }
+        Process codesignProcess = launchCodesignOnJavaBinary();
         BufferedReader is = new BufferedReader(new InputStreamReader(codesignProcess.getInputStream()));
         String line;
         boolean isHardened = false;

test/lib/jdk/test/lib/util/CoreUtils.java

@@ -1,5 +1,5 @@
 /*
- * Copyright (c) 2020, 2022, Oracle and/or its affiliates. All rights reserved.
+ * Copyright (c) 2020, 2023, Oracle and/or its affiliates. All rights reserved.
  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
  *
  * This code is free software; you can redistribute it and/or modify it
@@ -110,6 +110,8 @@ public static String getCoreFileLocation(String crashOutputString) throws IOExce
             Asserts.assertGT(new File(coreFileLocation).length(), 0L, "Unexpected core size");
             System.out.println("Found core file: " + coreFileLocation);
             return coreFileLocation; // success!
+        } else {
+            System.out.println("Core file not found. Trying to find a reason why...");
         }
 
         // See if we can figure out the likely reason the core file was not found.
@@ -130,6 +132,11 @@ public static String getCoreFileLocation(String crashOutputString) throws IOExce
                     // We can't generate cores files with hardened binaries on OSX 10.15 and later.
                     throw new SkippedException("Cannot produce core file with hardened binary on OSX 10.15 and later");
                 }
+            } else {
+                // codesign has to add entitlements using the plist. If this is not present we might not generate a core file.
+                if (!Platform.hasOSXPlistEntries()) {
+                    throw new SkippedException("Cannot produce core file with binary having no plist entitlement entries");
+                }
             }
         } else if (Platform.isLinux()) {
             // Check if a crash report tool is installed.