Start Ryuk lazily when in the reusable mode (#4938)

* Lazy Ryuk

* Add `ResourceReaper#init`

core/build.gradle

@@ -40,7 +40,12 @@ tasks.japicmp {
 
     classExcludes = []
 
-    methodExcludes = []
+    methodExcludes = [
+        "org.testcontainers.utility.ResourceReaper#start(java.lang.String,com.github.dockerjava.api.DockerClient)",
+        "org.testcontainers.utility.ResourceReaper#start(com.github.dockerjava.api.DockerClient)",
+        "org.testcontainers.utility.ResourceReaper#registerNetworkForCleanup(java.lang.String)",
+        "org.testcontainers.utility.ResourceReaper#removeNetworks(java.lang.String)",
+    ]
 
     fieldExcludes = []
 }

core/src/main/java/org/testcontainers/DockerClientFactory.java

@@ -31,13 +31,15 @@
 import org.testcontainers.utility.ImageNameSubstitutor;
 import org.testcontainers.utility.MountableFile;
 import org.testcontainers.utility.ResourceReaper;
+import org.testcontainers.utility.RyukResourceReaper;
 import org.testcontainers.utility.TestcontainersConfiguration;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.net.URI;
 import java.util.ArrayList;
+import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.Optional;
@@ -61,8 +63,7 @@ public class DockerClientFactory {
     public static final String SESSION_ID = UUID.randomUUID().toString();
 
     public static final Map<String, String> DEFAULT_LABELS = ImmutableMap.of(
-            TESTCONTAINERS_LABEL, "true",
-            TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID
+            TESTCONTAINERS_LABEL, "true"
     );
 
     private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.14");
@@ -73,7 +74,7 @@ public class DockerClientFactory {
     DockerClientProviderStrategy strategy;
 
     @VisibleForTesting
-    DockerClient dockerClient;
+    DockerClient client;
 
     @VisibleForTesting
     RuntimeException cachedClientFailure;
@@ -174,21 +175,20 @@ public String getRemoteDockerUnixSocketPath() {
      */
     @Synchronized
     public DockerClient client() {
-
-        if (dockerClient != null) {
-            return dockerClient;
-        }
-
         // fail-fast if checks have failed previously
         if (cachedClientFailure != null) {
             log.debug("There is a cached checks failure - throwing", cachedClientFailure);
             throw cachedClientFailure;
         }
 
+        if (client != null) {
+            return client;
+        }
+
         final DockerClientProviderStrategy strategy = getOrInitializeStrategy();
 
         log.info("Docker host IP address is {}", strategy.getDockerHostIpAddress());
-        final DockerClient client = new DockerClientDelegate() {
+        client = new DockerClientDelegate() {
 
             @Getter
             final DockerClient dockerClient = strategy.getDockerClient();
@@ -209,25 +209,14 @@ public void close() {
                 "  Operating System: " + dockerInfo.getOperatingSystem() + "\n" +
                 "  Total Memory: " + dockerInfo.getMemTotal() / (1024 * 1024) + " MB");
 
-        final String ryukContainerId;
-
-        boolean useRyuk = !Boolean.parseBoolean(System.getenv("TESTCONTAINERS_RYUK_DISABLED"));
-        if (useRyuk) {
-            log.debug("Ryuk is enabled");
-            try {
-                //noinspection deprecation
-                ryukContainerId = ResourceReaper.start(client);
-            } catch (RuntimeException e) {
-                cachedClientFailure = e;
-                throw e;
-            }
-            log.info("Ryuk started - will monitor and terminate Testcontainers containers on JVM exit");
-        } else {
-            log.debug("Ryuk is disabled");
-            ryukContainerId = null;
-            // best-efforts cleanup at JVM shutdown, without using the Ryuk container
+        final ResourceReaper resourceReaper;
+        try {
+            resourceReaper = ResourceReaper.instance();
             //noinspection deprecation
-            ResourceReaper.instance().setHook();
+            resourceReaper.init();
+        } catch (RuntimeException e) {
+            cachedClientFailure = e;
+            throw e;
         }
 
         boolean checksEnabled = !TestcontainersConfiguration.getInstance().isDisableChecks();
@@ -237,6 +226,12 @@ public void close() {
             try {
                 log.info("Checking the system...");
                 checkDockerVersion(version.getVersion());
+
+                //noinspection deprecation
+                String ryukContainerId = resourceReaper instanceof RyukResourceReaper
+                    ? ((RyukResourceReaper) resourceReaper).getContainerId()
+                    : null;
+
                 if (ryukContainerId != null) {
                     checkDiskSpace(client, ryukContainerId);
                 } else {
@@ -261,8 +256,7 @@ public void close() {
             log.debug("Checks are disabled");
         }
 
-        dockerClient = client;
-        return dockerClient;
+        return client;
     }
 
     private void checkDockerVersion(String dockerVersion) {
@@ -378,8 +372,10 @@ private <T> T runInsideDocker(DockerClient client, Consumer<CreateContainerCmd>
         final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString();
 
         checkAndPullImage(client, tinyImage);
+        HashMap<String, String> labels = new HashMap<>(DEFAULT_LABELS);
+        labels.putAll(ResourceReaper.instance().getLabels());
         CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage)
-                .withLabels(DEFAULT_LABELS);
+                .withLabels(labels);
         createContainerCmdConsumer.accept(createContainerCmd);
         String id = createContainerCmd.exec().getId();
 

core/src/main/java/org/testcontainers/containers/GenericContainer.java

@@ -364,7 +364,7 @@ private void tryStart(Instant startedAt) {
             CreateContainerCmd createCommand = dockerClient.createContainerCmd(dockerImageName);
             applyConfiguration(createCommand);
 
-            createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_LABEL, "true");
+            createCommand.getLabels().putAll(DockerClientFactory.DEFAULT_LABELS);
 
             boolean reused = false;
             final boolean reusable;
@@ -406,7 +406,8 @@ private void tryStart(Instant startedAt) {
             }
 
             if (!reusable) {
-                createCommand.getLabels().put(DockerClientFactory.TESTCONTAINERS_SESSION_ID_LABEL, DockerClientFactory.SESSION_ID);
+                //noinspection deprecation
+                createCommand.getLabels().putAll(ResourceReaper.instance().getLabels());
             }
 
             if (!reused) {

core/src/main/java/org/testcontainers/containers/Network.java

@@ -87,6 +87,8 @@ private String create() {
             Map<String, String> labels = createNetworkCmd.getLabels();
             labels = new HashMap<>(labels != null ? labels : Collections.emptyMap());
             labels.putAll(DockerClientFactory.DEFAULT_LABELS);
+            //noinspection deprecation
+            labels.putAll(ResourceReaper.instance().getLabels());
             createNetworkCmd.withLabels(labels);
 
             return createNetworkCmd.exec().getId();

core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java

@@ -112,6 +112,8 @@ public void onNext(BuildResponseItem item) {
                 labels.putAll(buildImageCmd.getLabels());
             }
             labels.putAll(DockerClientFactory.DEFAULT_LABELS);
+            //noinspection deprecation
+            labels.putAll(ResourceReaper.instance().getLabels());
             buildImageCmd.withLabels(labels);
 
             prePullDependencyImages(dependencyImageNames);

core/src/main/java/org/testcontainers/utility/JVMHookResourceReaper.java

@@ -0,0 +1,57 @@
+package org.testcontainers.utility;
+
+import com.github.dockerjava.api.model.Container;
+import com.github.dockerjava.api.model.PruneType;
+
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+/**
+ * A {@link ResourceReaper} implementation that uses {@link Runtime#addShutdownHook(Thread)}
+ * to cleanup containers.
+ */
+class JVMHookResourceReaper extends ResourceReaper {
+
+    @Override
+    public void init() {
+        setHook();
+    }
+
+    @Override
+    public synchronized void performCleanup() {
+        super.performCleanup();
+        synchronized (DEATH_NOTE) {
+            DEATH_NOTE.forEach(filters -> prune(PruneType.CONTAINERS, filters));
+            DEATH_NOTE.forEach(filters -> prune(PruneType.NETWORKS, filters));
+            DEATH_NOTE.forEach(filters -> prune(PruneType.VOLUMES, filters));
+            DEATH_NOTE.forEach(filters -> prune(PruneType.IMAGES, filters));
+        }
+    }
+
+    private void prune(PruneType pruneType, List<Map.Entry<String, String>> filters) {
+        String[] labels = filters.stream()
+            .filter(it -> "label".equals(it.getKey()))
+            .map(Map.Entry::getValue)
+            .toArray(String[]::new);
+        switch (pruneType) {
+            // Docker only prunes stopped containers, so we have to do it manually
+            case CONTAINERS:
+                List<Container> containers = dockerClient.listContainersCmd()
+                    .withFilter("label", Arrays.asList(labels))
+                    .withShowAll(true)
+                    .exec();
+
+                containers.parallelStream().forEach(container -> {
+                    dockerClient.removeContainerCmd(container.getId())
+                        .withForce(true)
+                        .withRemoveVolumes(true)
+                        .exec();
+                });
+                break;
+            default:
+                dockerClient.pruneCmd(pruneType).withLabelFilter(labels).exec();
+                break;
+        }
+    }
+}