Merge pull request #259 from Karm/macos

macOS port

apps/debug-symbols-smoke/threshold.conf

@@ -1,9 +1,13 @@
 linux.jvm.time.to.finish.threshold.ms=6924
 linux.native.time.to.finish.threshold.ms=14525
+macos.jvm.time.to.finish.threshold.ms=6924
+macos.native.time.to.finish.threshold.ms=14525
 windows.jvm.time.to.finish.threshold.ms=8000
 windows.native.time.to.finish.threshold.ms=17948
 linux.diff_jvm.time.to.finish.threshold.ms=20
 linux.diff_native.time.to.finish.threshold.ms=30
+macos.diff_jvm.time.to.finish.threshold.ms=20
+macos.diff_native.time.to.finish.threshold.ms=38
 windows.diff_jvm.time.to.finish.threshold.ms=20
 windows.diff_native.time.to.finish.threshold.ms=30
 linux.container.jvm.time.to.finish.threshold.ms=6924

apps/helidon-quickstart-se/threshold.conf

@@ -4,3 +4,7 @@ linux.executable.size.threshold.kB=43239
 windows.time.to.first.ok.request.threshold.ms=110
 windows.RSS.threshold.kB=98432
 windows.executable.size.threshold.kB=44239
+# There is something weird going on with Helidon on Macos
+macos.time.to.first.ok.request.threshold.ms=500
+macos.RSS.threshold.kB=58000
+macos.executable.size.threshold.kB=40000

apps/jfr-native-image-performance/threshold.conf

@@ -5,6 +5,13 @@ linux.diff_native.p50.latency=50
 linux.diff_native.mean.latency=50
 linux.diff_native.p90.latency=78
 
+macos.diff_native.time.to.first.ok.request.threshold.ms=40
+macos.diff_native.RSS.threshold.kB=38
+macos.diff_native.executable.size.threshold.kB=0
+macos.diff_native.p50.latency=30
+macos.diff_native.mean.latency=30
+macos.diff_native.p90.latency=30
+
 linux.container.diff_native.time.to.first.ok.request.threshold.ms=80
 linux.container.diff_native.RSS.threshold.kB=50
 linux.container.diff_native.executable.size.threshold.kB=20
@@ -14,4 +21,3 @@ linux.container.diff_native.p90.latency=100
 
 @IfQuarkusVersion(min ="3.7.0")
 linux.container.diff_native.RSS.threshold.kB=57
-

apps/quarkus-full-microprofile/threshold.conf

@@ -4,7 +4,13 @@ linux.executable.size.threshold.kB=79000
 windows.time.to.first.ok.request.threshold.ms=1537
 windows.RSS.threshold.kB=120000
 windows.executable.size.threshold.kB=79000
+# TODO: That is really high. Quarantine?
+macos.time.to.first.ok.request.threshold.ms=850
+macos.RSS.threshold.kB=68352
+macos.executable.size.threshold.kB=79000
 @IfQuarkusVersion(min ="3.7.0")
 linux.executable.size.threshold.kB=79700
+macos.executable.size.threshold.kB=79700
 @IfQuarkusVersion(min ="3.8.0")
 linux.executable.size.threshold.kB=79943
+macos.executable.size.threshold.kB=79943

apps/quarkus-spöklik-encoding/threshold.conf

@@ -6,6 +6,11 @@ linux.RSS.threshold.kB=55000
 #linux.executable.size.threshold.kB=41700
 # Mandrel 21x on RHEL 9; RHEL 8 based build is somewhat smaller.
 linux.executable.size.threshold.kB=44800
+
+macos.time.to.first.ok.request.threshold.ms=1500
+macos.RSS.threshold.kB=55000
+macos.executable.size.threshold.kB=44800
+
 windows.time.to.first.ok.request.threshold.ms=1100
 windows.RSS.threshold.kB=55000
 windows.executable.size.threshold.kB=30478

testsuite/src/it/java/org/graalvm/tests/integration/AppReproducersTest.java

@@ -495,7 +495,7 @@ public void imageioAWTContainerTest(TestInfo testInfo) throws IOException, Inter
 
     @Test
     @Tag("imageio")
-    @DisabledOnOs({OS.WINDOWS}) // AWT support is not there yet
+    @DisabledOnOs({OS.WINDOWS, OS.MAC}) // AWT support is not there yet
     @IfMandrelVersion(min = "21.1")
     public void imageioAWTTest(TestInfo testInfo) throws IOException, InterruptedException {
         imageioAWT(testInfo, Apps.IMAGEIO);

testsuite/src/it/java/org/graalvm/tests/integration/DebugSymbolsTest.java

@@ -117,7 +117,7 @@ public enum DebugOptions {
 
     @Test
     @Tag("debugSymbolsSmoke")
-    @DisabledOnOs({OS.WINDOWS})
+    @DisabledOnOs({OS.WINDOWS, OS.MAC}) // This targets GCC/GDB toolchain specifically.
     public void debugSymbolsSmokeGDB(TestInfo testInfo) throws IOException, InterruptedException {
         final Apps app = Apps.DEBUG_SYMBOLS_SMOKE;
         LOGGER.info("Testing app: " + app);
@@ -212,7 +212,7 @@ private static Map<String, String> getSwitches() {
 
     @Test
     @Tag("debugSymbolsQuarkus")
-    @DisabledOnOs({OS.WINDOWS})
+    @DisabledOnOs({OS.WINDOWS, OS.MAC}) // This targets GCC/GDB toolchain specifically.
     public void debugSymbolsQuarkus(TestInfo testInfo) throws IOException, InterruptedException {
         final Apps app = Apps.DEBUG_QUARKUS_FULL_MICROPROFILE;
         LOGGER.info("Testing app: " + app);
@@ -321,7 +321,7 @@ private boolean applySourcesPatch() {
     @Test
     @Tag("debugSymbolsQuarkus")
     @Tag("builder-image")
-    @DisabledOnOs({ OS.WINDOWS })
+    @DisabledOnOs({ OS.WINDOWS, OS.MAC }) // This targets GCC/GDB toolchain specifically.
     public void debugSymbolsQuarkusContainer(TestInfo testInfo) throws IOException, InterruptedException {
         final Apps app = Apps.DEBUG_QUARKUS_BUILDER_IMAGE_VERTX;
         LOGGER.info("Testing app: " + app);

testsuite/src/it/java/org/graalvm/tests/integration/JFRTest.java

@@ -65,6 +65,7 @@
 import static java.nio.charset.StandardCharsets.UTF_8;
 import static org.graalvm.tests.integration.utils.Commands.BUILDER_IMAGE;
 import static org.graalvm.tests.integration.utils.Commands.CONTAINER_RUNTIME;
+import static org.graalvm.tests.integration.utils.Commands.IS_THIS_MACOS;
 import static org.graalvm.tests.integration.utils.Commands.IS_THIS_WINDOWS;
 import static org.graalvm.tests.integration.utils.Commands.QUARKUS_VERSION;
 import static org.graalvm.tests.integration.utils.Commands.builderRoutine;
@@ -76,9 +77,12 @@
 import static org.graalvm.tests.integration.utils.Commands.findExecutable;
 import static org.graalvm.tests.integration.utils.Commands.getBaseDir;
 import static org.graalvm.tests.integration.utils.Commands.getContainerMemoryKb;
+import static org.graalvm.tests.integration.utils.Commands.getPodmanMachineSSHPort;
 import static org.graalvm.tests.integration.utils.Commands.getRSSkB;
 import static org.graalvm.tests.integration.utils.Commands.getRunCommand;
 import static org.graalvm.tests.integration.utils.Commands.getUnixUIDGID;
+import static org.graalvm.tests.integration.utils.Commands.openSSHTunnel;
+import static org.graalvm.tests.integration.utils.Commands.pidKiller;
 import static org.graalvm.tests.integration.utils.Commands.processStopper;
 import static org.graalvm.tests.integration.utils.Commands.removeContainers;
 import static org.graalvm.tests.integration.utils.Commands.replaceSwitchesInCmd;
@@ -340,13 +344,14 @@ private void startComparisonForBenchmark(Endpoint endpoint, boolean checkThresho
     }
 
     private Map<String, Integer> runBenchmarkForApp(Endpoint endpoint, int trials, Apps app, File appDir, File processLog,
-                                                    String cn, String mn, StringBuilder report, Path measurementsLog,
-                                                    boolean inContainer) throws IOException, InterruptedException {
+            String cn, String mn, StringBuilder report, Path measurementsLog,
+            boolean inContainer) throws IOException, InterruptedException {
 
         Process process = null;
         Process hyperfoilProcess = null;
         int rssSum = 0;
         int startupSum = 0;
+        final long[] tunnelPIDs = new long[] { -1L, -1L };
 
         try {
             for (int i = 0; i < trials; i++) {
@@ -375,6 +380,15 @@ private Map<String, Integer> runBenchmarkForApp(Endpoint endpoint, int trials, A
                 LOGGER.info("Trial " + i + " startup time sum: " + startupSum + " ms, RSS sum: " + rssSum + " KB");
             }
 
+            if (IS_THIS_MACOS) {
+                // TODO: This is obviously way too tailored to our particular macOS podman installation.
+                // We will generalize it as a part of porting to GHA.
+                final String identity = "/Users/tester/.local/share/containers/podman/machine/machine";
+                final int port = getPodmanMachineSSHPort();
+                tunnelPIDs[0] = openSSHTunnel(identity, String.valueOf(port), "core", "localhost", "8080", false);
+                tunnelPIDs[1] = openSSHTunnel(identity, String.valueOf(port), "core", "localhost", "8090", true);
+            }
+
             // Run Hyperfoil controller in container and expose port for test
             final List<String> getAndStartHyperfoil = getRunCommand(app.buildAndRunCmds.cmds[inContainer ? 3 : 2]);
             hyperfoilProcess = runCommand(getAndStartHyperfoil, appDir, processLog, app);
@@ -495,6 +509,13 @@ private Map<String, Integer> runBenchmarkForApp(Endpoint endpoint, int trials, A
                 Files.move(recording,
                         Paths.get(appDir.getAbsolutePath(), "logs", endpoint + "-" + app.name().toLowerCase() + "-flight-native.jfr"));
             }
+            // Bury the tunnels
+            if (IS_THIS_MACOS) {
+                for (long pid : tunnelPIDs) {
+                    if (pid != -1)
+                        pidKiller(pid, true);
+                }
+            }
         }
     }
 

testsuite/src/it/java/org/graalvm/tests/integration/PerfCheckTest.java

@@ -96,7 +96,9 @@
  * @author Michal Karm Babacek <karm@redhat.com>
  */
 @Tag("perfcheck")
-@DisabledOnOs({ OS.WINDOWS }) // We need to replace perf with wmic & Dr.Memory or something
+// Windows: We need to replace perf with wmic & Dr.Memory or something.
+// Mac: We need to figure out what's Mac's "perf".
+@DisabledOnOs({ OS.WINDOWS, OS.MAC})
 public class PerfCheckTest {
 
     private static final Logger LOGGER = Logger.getLogger(PerfCheckTest.class.getName());

testsuite/src/it/java/org/graalvm/tests/integration/utils/BuildAndRunCmds.java

@@ -34,6 +34,7 @@
 import static org.graalvm.tests.integration.utils.Commands.GRAALVM_BUILD_OUTPUT_JSON_FILE;
 import static org.graalvm.tests.integration.utils.Commands.GRAALVM_EXPERIMENTAL_BEGIN;
 import static org.graalvm.tests.integration.utils.Commands.GRAALVM_EXPERIMENTAL_END;
+import static org.graalvm.tests.integration.utils.Commands.IS_THIS_MACOS;
 import static org.graalvm.tests.integration.utils.Commands.IS_THIS_WINDOWS;
 import static org.graalvm.tests.integration.utils.Commands.QUARKUS_VERSION;
 import static org.graalvm.tests.integration.utils.Commands.getUnixUIDGID;
@@ -245,13 +246,13 @@ public enum BuildAndRunCmds {
                     :
                     new String[]{"unzip", "test_data.txt.zip", "-d", "target"},
 
-            new String[]{"native-image", DebugSymbolsTest.DebugOptions.UnlockExperimentalVMOptions_23_1.token,
-                    "-H:GenerateDebugInfo=1", "-H:+PreserveFramePointer", "-H:-DeleteLocalSymbols",
+            new String[] { "native-image", DebugSymbolsTest.DebugOptions.UnlockExperimentalVMOptions_23_1.token,
+                    "-H:GenerateDebugInfo=" + (IS_THIS_MACOS ? "0" : "1"), "-H:+PreserveFramePointer", "-H:-DeleteLocalSymbols",
                     DebugSymbolsTest.DebugOptions.TrackNodeSourcePosition_23_0.token,
                     DebugSymbolsTest.DebugOptions.DebugCodeInfoUseSourceMappings_23_0.token,
                     DebugSymbolsTest.DebugOptions.OmitInlinedMethodDebugLineInfo_23_0.token,
                     DebugSymbolsTest.DebugOptions.LockExperimentalVMOptions_23_1.token,
-                    "-jar", "target/debug-symbols-smoke.jar", "target/debug-symbols-smoke"},
+                    "-jar", "target/debug-symbols-smoke.jar", "target/debug-symbols-smoke" },
             new String[]{"java", "-jar", "./target/debug-symbols-smoke.jar"},
             new String[]{IS_THIS_WINDOWS ? "target\\debug-symbols-smoke.exe" : "./target/debug-symbols-smoke"}
     }),
@@ -269,13 +270,13 @@ public enum BuildAndRunCmds {
                     "-XX:+FlightRecorder",
                     "-XX:StartFlightRecording=settings=" + BASE_DIR + File.separator + "apps" + File.separator + "jfr-native-image-performance/jfr-perf.jfc,filename=logs/flight-native.jfr",
                     "-XX:FlightRecorderLogging=jfr"},
-            new String[]{CONTAINER_RUNTIME, "run", "--name", ContainerNames.HYPERFOIL.name, "--rm", "--network=host", "quay.io/karmkarm/hyperfoil:0.25.2", "standalone"}
+            hyperfoil()
     }),
     PLAINTEXT_PERFORMANCE(new String[][]{
             new String[]{"mvn", "package", "-Pnative", "-Dquarkus.version=" + QUARKUS_VERSION.getVersionString(),
                     "-DfinalName=jfr-plaintext"},
             new String[]{"./target/jfr-plaintext-runner"},
-            new String[]{CONTAINER_RUNTIME, "run", "--name", ContainerNames.HYPERFOIL.name, "--rm", "--network=host", "quay.io/karmkarm/hyperfoil:0.25.2", "standalone"}
+            hyperfoil()
     }),
     JFR_PERFORMANCE_BUILDER_IMAGE(new String[][]{
             new String[]{"mvn", "clean", "package", "-Pnative", "-Dquarkus.native.container-build=true",
@@ -291,7 +292,7 @@ public enum BuildAndRunCmds {
                     "--name", ContainerNames.JFR_PERFORMANCE_BUILDER_IMAGE.name, "jfr-performance-app", "-XX:+FlightRecorder",
                     "-XX:StartFlightRecording=settings=/work/jfr-perf.jfc,filename=/tmp/flight-native.jfr",
                     "-XX:FlightRecorderLogging=jfr"},
-            new String[]{CONTAINER_RUNTIME, "run", "--name", ContainerNames.HYPERFOIL.name, "--rm", "--network=host", "quay.io/karmkarm/hyperfoil:0.25.2", "standalone"}
+            hyperfoil()
     }),
     PLAINTEXT_PERFORMANCE_BUILDER_IMAGE(new String[][]{
             new String[]{"mvn", "clean", "package", "-Pnative", "-Dquarkus.native.container-build=true",
@@ -304,7 +305,7 @@ public enum BuildAndRunCmds {
                     "-t",
                     //"-v", BASE_DIR + File.separator + "apps" + File.separator + "jfr-native-image-performance/logs:/tmp:z",
                     "--name", ContainerNames.JFR_PLAINTEXT_BUILDER_IMAGE.name, "jfr-plaintext-app"},
-            new String[]{CONTAINER_RUNTIME, "run", "--name", ContainerNames.HYPERFOIL.name, "--rm", "--network=host", "quay.io/karmkarm/hyperfoil:0.25.2", "standalone"}
+            hyperfoil()
     }),
     JFR_SMOKE(new String[][]{
             new String[]{"mvn", "package"},
@@ -379,6 +380,21 @@ public enum BuildAndRunCmds {
 
     public final String[][] cmds;
 
+    private static String[] hyperfoil() {
+        if (IS_THIS_MACOS) {
+            // --network=host does not do what you think it does on macOS. It creates a shared network
+            // between your container and the VM running it.
+            // The step that would bring the connection all the way up the stack to your host
+            // is an ssh tunnel. This last step is manual, and we run it separately during the test.
+            return new String[] { CONTAINER_RUNTIME, "run", "--name", ContainerNames.HYPERFOIL.name, "--rm", "--network=host",
+                    "--entrypoint=/bin/bash", "quay.io/karmkarm/hyperfoil:0.25.2", "/deployment/bin/standalone.sh",
+                    "-Dio.hyperfoil.controller.host=localhost" };
+        } else {
+            return new String[] { CONTAINER_RUNTIME, "run", "--name", ContainerNames.HYPERFOIL.name, "--rm", "--network=host",
+                    "quay.io/karmkarm/hyperfoil:0.25.2", "standalone" };
+        }
+    }
+
     BuildAndRunCmds(String[][] cmds) {
         this.cmds = cmds;
     }

testsuite/src/it/java/org/graalvm/tests/integration/utils/Commands.java

@@ -87,6 +87,7 @@ public class Commands {
     public static final FailOnPerfRegressionEnum FAIL_ON_PERF_REGRESSION = FailOnPerfRegressionEnum.valueOf(getProperty("FAIL_ON_PERF_REGRESSION", "true").toUpperCase());
 
     public static final boolean IS_THIS_WINDOWS = System.getProperty("os.name").matches(".*[Ww]indows.*");
+    public static final boolean IS_THIS_MACOS = System.getProperty("os.name").matches(".*[Mm]ac.*");
     private static final Pattern NUM_PATTERN = Pattern.compile("[ \t]*[0-9]+[ \t]*");
     private static final Pattern ALPHANUMERIC_FIRST = Pattern.compile("([a-z0-9]+).*");
     private static final Pattern CONTAINER_STATS_MEMORY = Pattern.compile("(?:table)?[ \t]*([0-9\\.]+)([a-zA-Z]+).*");
@@ -349,6 +350,7 @@ public static Process runCommand(List<String> command, File directory, File logF
     }
 
     public static void pidKiller(long pid, boolean force) {
+        LOGGER.infof("Killing PID: %d, forcefully: %b", pid, force);
         try {
             if (IS_THIS_WINDOWS) {
                 if (!force) {
@@ -443,7 +445,8 @@ public static void stopRunningContainers(String... containerNames) throws Interr
     }
 
     public static void stopRunningContainer(String containerName) throws InterruptedException, IOException {
-        final List<String> cmd = new ArrayList<>(getRunCommand(CONTAINER_RUNTIME, "stop", containerName));
+        // -t 1, just give it a sec and then kill it; we don't care about long graceful shutdowns. Both podman and docker ok.
+        final List<String> cmd = new ArrayList<>(getRunCommand(CONTAINER_RUNTIME, "stop", containerName, "-t", "1"));
         LOGGER.infof("Command: %s", cmd);
         final Process process = Runtime.getRuntime().exec(cmd.toArray(String[]::new));
         process.waitFor(5, TimeUnit.SECONDS);
@@ -607,17 +610,17 @@ public static void processStopper(Process p, boolean force, boolean orderMatters
     }
 
     public static void clearCaches() throws IOException {
-        if (IS_THIS_WINDOWS) {
-            LOGGER.infof("Not implemented for Windows");
+        if (IS_THIS_WINDOWS || IS_THIS_MACOS) {
+            LOGGER.infof("Not implemented for Windows and Mac");
             return;
         }
         final List<String> cmd = getRunCommand("sudo", "bash", "-c", "sync; echo 3 > /proc/sys/vm/drop_caches");
         LOGGER.infof("Command: %s, Output: %s", cmd, runCommand(cmd));
     }
 
     public static void disableTurbo() throws IOException {
-        if (IS_THIS_WINDOWS) {
-            LOGGER.infof("Not implemented for Windows");
+        if (IS_THIS_WINDOWS || IS_THIS_MACOS) {
+            LOGGER.infof("Not implemented for Windows and Mac");
             return;
         }
         final File intel = new File("/sys/devices/system/cpu/intel_pstate/no_turbo");
@@ -636,8 +639,8 @@ public static void disableTurbo() throws IOException {
     }
 
     public static void enableTurbo() throws IOException {
-        if (IS_THIS_WINDOWS) {
-            LOGGER.infof("Not implemented for Windows");
+        if (IS_THIS_WINDOWS || IS_THIS_MACOS) {
+            LOGGER.infof("Not implemented for Windows and Mac");
         }
         final File intel = new File("/sys/devices/system/cpu/intel_pstate/no_turbo");
         if (intel.exists()) {
@@ -960,6 +963,44 @@ public static SerialGCLog parseSerialGCLog(Path path, String statsFor, boolean i
         }
     }
 
+    /**
+     * Open an ssh tunnel.
+     * @return pid - caller is responsible for closing the tunnel
+     */
+    public static long openSSHTunnel(String identity, String sshPort, String user, String host, String port, boolean local) {
+        final List<String> cmd = getRunCommand(
+                "ssh", "-o", "StrictHostKeyChecking=no", "-i", identity, "-p", sshPort,
+                local ? "-L" : "-R", port + ":" + host + ":" + port, user + "@" + host, "-N");
+        LOGGER.infof("Command: %s", cmd);
+        final ProcessBuilder processBuilder = new ProcessBuilder(cmd);
+        final Map<String, String> envA = processBuilder.environment();
+        envA.put("PATH", System.getenv("PATH"));
+        processBuilder.redirectErrorStream(true);
+        try {
+            final Process p = processBuilder.start();
+            return p.pid();
+        } catch (IOException e) {
+            e.printStackTrace();
+        }
+        return -1;
+    }
+
+    /**
+     * @return Podman machine ssh port number
+     */
+    public static int getPodmanMachineSSHPort() throws IOException {
+        final List<String> cmd = getRunCommand(
+                CONTAINER_RUNTIME, "machine", "inspect", "--format", "{{.SSHConfig.Port}}");
+        LOGGER.infof("Command: %s", cmd);
+        final ProcessBuilder pa = new ProcessBuilder(cmd);
+        pa.environment().put("PATH", System.getenv("PATH"));
+        pa.redirectErrorStream(true);
+        final Process p = pa.start();
+        try (InputStream is = p.getInputStream()) {
+            return Integer.parseInt(new String(is.readAllBytes(), US_ASCII).trim());
+        }
+    }
+
     public static String mapToJSON(List<Map<String, String>> maps) {
         final Pattern num = Pattern.compile("\\d+");
         final StringBuilder sb = new StringBuilder();