Gradle integration tests: tackle CI issues / final changes

The major issue was (very likely) that in the `Gradle Tests - JDK 11 Windows` CI job (at least) the Quarkus process running in dev-mode somehow "survived". The observed pattern is that one dev-mode test ran successfully and all following dev-mode tests failed with test timeout exceptions (awaitability). Adding some code to collect the `List` of child `ProcessHandle`s before calling `ProcessHandle.destroy()` plus a (potentially over-cautious) `ProcessHandle.destroyForcibly()` seems to solve the Gradle-Windows-CI issue. A couple of CI runs did not fail.

This change adds "process dumps" (not really useful w/ Windows :( ) from before and after the test run to the failed test outputs.

integration-tests/gradle/src/test/java/io/quarkus/gradle/devmode/QuarkusDevGradleTestBase.java

@@ -7,13 +7,17 @@
 import java.io.File;
 import java.io.IOException;
 import java.nio.file.Files;
+import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.Executors;
 import java.util.concurrent.Future;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
 
 import org.apache.commons.io.FileUtils;
 import org.junit.jupiter.api.Test;
@@ -34,22 +38,53 @@ protected void setupTestCommand() {
 
     @Test
     public void main() throws Exception {
-
         projectDir = getProjectDir();
         beforeQuarkusDev();
         ExecutorService executor = null;
         AtomicReference<BuildResult> buildResult = new AtomicReference<>();
+        List<String> processesBeforeTest = dumpProcesses();
+        List<String> processesAfterTest = Collections.emptyList();
         try {
-            executor = Executors.newSingleThreadExecutor();
-            quarkusDev = executor.submit(() -> {
-                try {
-                    buildResult.set(build());
-                } catch (Exception e) {
-                    throw new IllegalStateException("Failed to build the project", e);
+            try {
+                executor = Executors.newSingleThreadExecutor();
+                quarkusDev = executor.submit(() -> {
+                    try {
+                        buildResult.set(build());
+                    } catch (Exception e) {
+                        throw new IllegalStateException("Failed to build the project", e);
+                    }
+                });
+                testDevMode();
+            } finally {
+                processesAfterTest = dumpProcesses();
+
+                if (quarkusDev != null) {
+                    quarkusDev.cancel(true);
+                }
+                if (executor != null) {
+                    executor.shutdownNow();
+                }
+
+                // Kill all processes that were (indirectly) spawned by the current process.
+                List<ProcessHandle> childProcesses = DevModeTestUtils.killDescendingProcesses();
+
+                DevModeTestUtils.awaitUntilServerDown();
+
+                // sanity: forcefully terminate left-over processes
+                childProcesses.forEach(ProcessHandle::destroyForcibly);
+
+                if (projectDir != null && projectDir.isDirectory()) {
+                    FileUtils.deleteQuietly(projectDir);
                 }
-            });
-            testDevMode();
+            }
         } catch (Exception | AssertionError e) {
+            System.err.println("PROCESSES BEFORE TEST:");
+            processesBeforeTest.forEach(System.err::println);
+            System.err.println("PROCESSES AFTER TEST (BEFORE CLEANUP):");
+            processesAfterTest.forEach(System.err::println);
+            System.err.println("PROCESSES AFTER CLEANUP:");
+            dumpProcesses().forEach(System.err::println);
+
             if (buildResult.get() != null) {
                 System.err.println("BELOW IS THE CAPTURED LOGGING OF THE FAILED GRADLE TEST PROJECT BUILD");
                 System.err.println(buildResult.get().getOutput());
@@ -69,25 +104,22 @@ public void main() throws Exception {
                 }
             }
             throw e;
-        } finally {
-            if (quarkusDev != null) {
-                quarkusDev.cancel(true);
-            }
-            if (executor != null) {
-                executor.shutdownNow();
-            }
-
-            // Kill all processes that were (indirectly) spawned by the current process.
-            DevModeTestUtils.killDescendingProcesses();
-
-            DevModeTestUtils.awaitUntilServerDown();
-
-            if (projectDir != null && projectDir.isDirectory()) {
-                FileUtils.deleteQuietly(projectDir);
-            }
         }
     }
 
+    public static List<String> dumpProcesses() {
+        // ProcessHandle.Info.command()/arguments()/commandLine() are always empty on Windows:
+        // https://bugs.openjdk.java.net/browse/JDK-8176725
+        ProcessHandle current = ProcessHandle.current();
+        return Stream.concat(Stream.of(current), current.descendants()).map(p -> {
+            ProcessHandle.Info i = p.info();
+            return String.format("PID %8d (%8d) started:%s CPU:%s - %s", p.pid(),
+                    p.parent().map(ProcessHandle::pid).orElse(-1L),
+                    i.startInstant().orElse(null), i.totalCpuDuration().orElse(null),
+                    i.commandLine().orElse("<command line not available>"));
+        }).collect(Collectors.toList());
+    }
+
     protected BuildResult build() throws Exception {
         return runGradleWrapper(projectDir, buildArguments());
     }

test-framework/devmode-test-utils/src/main/java/io/quarkus/test/devmode/util/DevModeTestUtils.java

@@ -9,25 +9,36 @@
 import java.net.URL;
 import java.nio.charset.StandardCharsets;
 import java.time.Instant;
+import java.util.List;
 import java.util.Map;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import java.util.concurrent.atomic.AtomicReference;
 import java.util.function.Supplier;
+import java.util.stream.Collectors;
 
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 
 public class DevModeTestUtils {
 
-    public static void killDescendingProcesses() {
+    public static List<ProcessHandle> killDescendingProcesses() {
         // Warning: Do not try to evaluate ProcessHandle.Info.arguments() or .commandLine() as those are always empty on Windows:
         // https://bugs.openjdk.java.net/browse/JDK-8176725
-        ProcessHandle.current().descendants()
+        //
+        // Intentionally collecting the ProcessHandles before calling .destroy(), because it seemed that, at least on
+        // Windows, not all processes were properly killed, leaving (some) processes around, causing following dev-mode
+        // tests to time-out.
+        List<ProcessHandle> childProcesses = ProcessHandle.current().descendants()
                 // destroy younger descendants first
                 .sorted((ph1, ph2) -> ph2.info().startInstant().orElse(Instant.EPOCH)
                         .compareTo(ph1.info().startInstant().orElse(Instant.EPOCH)))
-                .forEach(ProcessHandle::destroy);
+                .collect(Collectors.toList());
+
+        childProcesses.forEach(ProcessHandle::destroy);
+
+        // Returning all child processes for callers that want to do a "kill -9"
+        return childProcesses;
     }
 
     public static void filter(File input, Map<String, String> variables) throws IOException {