diff --git a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java index 7dec6399b8f..262f9c4839e 100644 --- a/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java +++ b/core/src/main/java/org/testcontainers/containers/DockerComposeContainer.java @@ -163,12 +163,13 @@ private void pullImages() { // (a) as a workaround for https://github.com/docker/compose/issues/5854, which prevents authenticated image pulls being possible when credential helpers are in use // (b) so that credential helper-based auth still works when compose is running from within a container parsedComposeFiles.stream() - .flatMap(it -> it.getServiceImageNames().stream()) + .flatMap(it -> it.getDependencyImageNames().stream()) .forEach(imageName -> { try { + log.info("Preemptively checking local images for '{}', referenced via a compose file or transitive Dockerfile. If not available, it will be pulled.", imageName); DockerClientFactory.instance().checkAndPullImage(dockerClient, imageName); } catch (Exception e) { - log.warn("Failed to pull image '{}'. Exception message was {}", imageName, e.getMessage()); + log.warn("Unable to pre-fetch an image ({}) depended upon by Docker Compose build - startup will continue but may fail. Exception message was: {}", imageName, e.getMessage()); } }); } diff --git a/core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java b/core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java index 838607d8337..8f88724647f 100644 --- a/core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java +++ b/core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java @@ -5,10 +5,13 @@ import lombok.Getter; import lombok.extern.slf4j.Slf4j; import org.apache.commons.io.FileUtils; +import org.testcontainers.images.ParsedDockerfile; import org.yaml.snakeyaml.Yaml; import java.io.File; import java.io.FileInputStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.HashSet; import java.util.Map; import java.util.Set; @@ -23,9 +26,10 @@ class ParsedDockerComposeFile { private final Map composeFileContent; private final String composeFileName; + private final File composeFile; @Getter - private Set serviceImageNames = new HashSet<>(); + private Set dependencyImageNames = new HashSet<>(); ParsedDockerComposeFile(File composeFile) { Yaml yaml = new Yaml(); @@ -35,7 +39,7 @@ class ParsedDockerComposeFile { throw new IllegalArgumentException("Unable to parse YAML file from " + composeFile.getAbsolutePath(), e); } this.composeFileName = composeFile.getAbsolutePath(); - + this.composeFile = composeFile; parseAndValidate(); } @@ -43,6 +47,7 @@ class ParsedDockerComposeFile { ParsedDockerComposeFile(Map testContent) { this.composeFileContent = testContent; this.composeFileName = ""; + this.composeFile = new File("."); parseAndValidate(); } @@ -80,15 +85,61 @@ private void parseAndValidate() { } final Map serviceDefinitionMap = (Map) serviceDefinition; - if (serviceDefinitionMap.containsKey("container_name")) { - throw new IllegalStateException(String.format( - "Compose file %s has 'container_name' property set for service '%s' but this property is not supported by Testcontainers, consider removing it", - composeFileName, - serviceName - )); + + validateNoContainerNameSpecified(serviceName, serviceDefinitionMap); + findServiceImageName(serviceDefinitionMap); + findImageNamesInDockerfile(serviceDefinitionMap); + } + } + + private void validateNoContainerNameSpecified(String serviceName, Map serviceDefinitionMap) { + if (serviceDefinitionMap.containsKey("container_name")) { + throw new IllegalStateException(String.format( + "Compose file %s has 'container_name' property set for service '%s' but this property is not supported by Testcontainers, consider removing it", + composeFileName, + serviceName + )); + } + } + + private void findServiceImageName(Map serviceDefinitionMap) { + if (serviceDefinitionMap.containsKey("image") && serviceDefinitionMap.get("image") instanceof String) { + final String imageName = (String) serviceDefinitionMap.get("image"); + log.debug("Resolved dependency image for Docker Compose in {}: {}", composeFileName, imageName); + dependencyImageNames.add(imageName); + } + } + + private void findImageNamesInDockerfile(Map serviceDefinitionMap) { + final Object buildNode = serviceDefinitionMap.get("build"); + Path dockerfilePath = null; + + if (buildNode instanceof Map) { + final Map buildElement = (Map) buildNode; + final Object dockerfileRelativePath = buildElement.get("dockerfile"); + final Object contextRelativePath = buildElement.get("context"); + if (dockerfileRelativePath instanceof String && contextRelativePath instanceof String) { + dockerfilePath = composeFile + .getParentFile() + .toPath() + .resolve((String) contextRelativePath) + .resolve((String) dockerfileRelativePath) + .normalize(); } - if (serviceDefinitionMap.containsKey("image") && serviceDefinitionMap.get("image") instanceof String) { - serviceImageNames.add((String) serviceDefinitionMap.get("image")); + } else if (buildNode instanceof String) { + dockerfilePath = composeFile + .getParentFile() + .toPath() + .resolve((String) buildNode) + .resolve("./Dockerfile") + .normalize(); + } + + if (dockerfilePath != null && Files.exists(dockerfilePath)) { + Set resolvedImageNames = new ParsedDockerfile(dockerfilePath).getDependencyImageNames(); + if (!resolvedImageNames.isEmpty()) { + log.debug("Resolved Dockerfile dependency images for Docker Compose in {} -> {}: {}", composeFileName, dockerfilePath, resolvedImageNames); + this.dependencyImageNames.addAll(resolvedImageNames); } } } diff --git a/core/src/main/java/org/testcontainers/images/ParsedDockerfile.java b/core/src/main/java/org/testcontainers/images/ParsedDockerfile.java new file mode 100644 index 00000000000..1ad43813757 --- /dev/null +++ b/core/src/main/java/org/testcontainers/images/ParsedDockerfile.java @@ -0,0 +1,67 @@ +package org.testcontainers.images; + +import com.google.common.annotations.VisibleForTesting; +import lombok.Getter; +import lombok.extern.slf4j.Slf4j; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Collections; +import java.util.List; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; +import java.util.stream.Collectors; + +/** + * Representation of a Dockerfile, with partial parsing for extraction of a minimal set of data. + */ +@Slf4j +public class ParsedDockerfile { + + private static final Pattern FROM_LINE_PATTERN = Pattern.compile("FROM ([^\\s]+).*"); + + private final Path dockerFilePath; + + @Getter + private Set dependencyImageNames = Collections.emptySet(); + + public ParsedDockerfile(Path dockerFilePath) { + this.dockerFilePath = dockerFilePath; + parse(read()); + } + + @VisibleForTesting + ParsedDockerfile(List lines) { + this.dockerFilePath = Paths.get("dummy.Dockerfile"); + parse(lines); + } + + private List read() { + if (!Files.exists(dockerFilePath)) { + log.warn("Tried to parse Dockerfile at path {} but none was found", dockerFilePath); + return Collections.emptyList(); + } + + try { + return Files.readAllLines(dockerFilePath); + } catch (IOException e) { + log.warn("Unable to read Dockerfile at path {}", dockerFilePath, e); + return Collections.emptyList(); + } + } + + private void parse(List lines) { + dependencyImageNames = lines.stream() + .map(FROM_LINE_PATTERN::matcher) + .filter(Matcher::matches) + .map(matcher -> matcher.group(1)) + .collect(Collectors.toSet()); + + if (!dependencyImageNames.isEmpty()) { + log.debug("Found dependency images in Dockerfile {}: {}", dockerFilePath, dependencyImageNames); + } + } +} diff --git a/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java b/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java index ded64439a64..1140247960a 100644 --- a/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java +++ b/core/src/main/java/org/testcontainers/images/builder/ImageFromDockerfile.java @@ -2,10 +2,8 @@ import com.github.dockerjava.api.DockerClient; import com.github.dockerjava.api.command.BuildImageCmd; -import com.github.dockerjava.api.exception.DockerClientException; import com.github.dockerjava.api.model.BuildResponseItem; import com.github.dockerjava.core.command.BuildImageResultCallback; -import com.google.common.collect.Sets; import lombok.Cleanup; import lombok.Getter; import lombok.extern.slf4j.Slf4j; @@ -14,6 +12,7 @@ import org.apache.commons.lang.StringUtils; import org.slf4j.Logger; import org.testcontainers.DockerClientFactory; +import org.testcontainers.images.ParsedDockerfile; import org.testcontainers.images.builder.traits.BuildContextBuilderTrait; import org.testcontainers.images.builder.traits.ClasspathTrait; import org.testcontainers.images.builder.traits.DockerfileTrait; @@ -28,6 +27,7 @@ import java.io.PipedInputStream; import java.io.PipedOutputStream; import java.nio.file.Path; +import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; @@ -51,6 +51,7 @@ public class ImageFromDockerfile extends LazyFuture implements private final Map buildArgs = new HashMap<>(); private Optional dockerFilePath = Optional.empty(); private Optional dockerfile = Optional.empty(); + private Set dependencyImageNames = Collections.emptySet(); public ImageFromDockerfile() { this("testcontainers/" + Base58.randomString(16).toLowerCase()); @@ -81,6 +82,17 @@ protected final String resolve() { Logger logger = DockerLoggerFactory.getLogger(dockerImageName); DockerClient dockerClient = DockerClientFactory.instance().client(); + + dependencyImageNames.forEach(imageName -> { + try { + log.info("Pre-emptively checking local images for '{}', referenced via a Dockerfile. If not available, it will be pulled.", imageName); + DockerClientFactory.instance().checkAndPullImage(dockerClient, imageName); + } catch (Exception e) { + log.warn("Unable to pre-fetch an image ({}) depended upon by Dockerfile - image build will continue but may fail. Exception message was: {}", imageName, e.getMessage()); + } + }); + + try { if (deleteOnExit) { ResourceReaper.instance().registerImageForCleanup(dockerImageName); @@ -139,7 +151,11 @@ public void onNext(BuildResponseItem item) { protected void configure(BuildImageCmd buildImageCmd) { buildImageCmd.withTag(this.getDockerImageName()); this.dockerFilePath.ifPresent(buildImageCmd::withDockerfilePath); - this.dockerfile.ifPresent(p -> buildImageCmd.withDockerfile(p.toFile())); + this.dockerfile.ifPresent(p -> { + buildImageCmd.withDockerfile(p.toFile()); + dependencyImageNames = new ParsedDockerfile(p).getDependencyImageNames(); + }); + this.buildArgs.forEach(buildImageCmd::withBuildArg); } diff --git a/core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java b/core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java index c553d250ec9..9a8421086f1 100644 --- a/core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java +++ b/core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java @@ -76,13 +76,27 @@ public void shouldIgnoreUnknownStructure() { public void shouldObtainImageNamesV1() { File file = new File("src/test/resources/docker-compose-imagename-parsing-v1.yml"); ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file); - assertEquals("all defined service names are found", Sets.newHashSet("redis", "mysql"), parsedFile.getServiceImageNames()); + assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "postgres"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, postgres from Dockerfile build } @Test public void shouldObtainImageNamesV2() { File file = new File("src/test/resources/docker-compose-imagename-parsing-v2.yml"); ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file); - assertEquals("all defined service names are found", Sets.newHashSet("redis", "mysql"), parsedFile.getServiceImageNames()); + assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "postgres"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, postgres from Dockerfile build + } + + @Test + public void shouldObtainImageFromDockerfileBuild() { + File file = new File("src/test/resources/docker-compose-imagename-parsing-dockerfile.yml"); + ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file); + assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "alpine:3.2"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, alpine:3.2 from Dockerfile build + } + + @Test + public void shouldObtainImageFromDockerfileBuildWithContext() { + File file = new File("src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml"); + ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file); + assertEquals("all defined images are found", Sets.newHashSet("redis", "mysql", "alpine:3.2"), parsedFile.getDependencyImageNames()); // redis, mysql from compose file, alpine:3.2 from Dockerfile build } } diff --git a/core/src/test/java/org/testcontainers/images/ParsedDockerfileTest.java b/core/src/test/java/org/testcontainers/images/ParsedDockerfileTest.java new file mode 100644 index 00000000000..e2b97f9fab7 --- /dev/null +++ b/core/src/test/java/org/testcontainers/images/ParsedDockerfileTest.java @@ -0,0 +1,54 @@ +package org.testcontainers.images; + +import com.google.common.collect.Sets; +import org.junit.Test; + +import java.nio.file.Paths; + +import static java.util.Arrays.asList; +import static org.junit.Assert.assertEquals; + +public class ParsedDockerfileTest { + + @Test + public void doesSimpleParsing() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage", "RUN something")); + assertEquals("extracts a single image name", Sets.newHashSet("someimage"), parsedDockerfile.getDependencyImageNames()); + } + + @Test + public void handlesTags() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage:tag", "RUN something")); + assertEquals("retains tags in image names", Sets.newHashSet("someimage:tag"), parsedDockerfile.getDependencyImageNames()); + } + + @Test + public void handlesDigests() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage@sha256:abc123", "RUN something")); + assertEquals("retains digests in image names", Sets.newHashSet("someimage@sha256:abc123"), parsedDockerfile.getDependencyImageNames()); + } + + @Test + public void ignoringCommentedFromLines() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage", "#FROM somethingelse")); + assertEquals("ignores commented from lines", Sets.newHashSet("someimage"), parsedDockerfile.getDependencyImageNames()); + } + + @Test + public void ignoringBuildStageNames() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("FROM someimage --as=base", "RUN something", "FROM nextimage", "RUN something")); + assertEquals("ignores build stage names and allows multiple images to be extracted", Sets.newHashSet("someimage", "nextimage"), parsedDockerfile.getDependencyImageNames()); + } + + @Test + public void handlesGracefullyIfNoFromLine() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(asList("RUN something", "# is this even a valid Dockerfile?")); + assertEquals("handles invalid Dockerfiles gracefully", Sets.newHashSet(), parsedDockerfile.getDependencyImageNames()); + } + + @Test + public void handlesGracefullyIfDockerfileNotFound() { + final ParsedDockerfile parsedDockerfile = new ParsedDockerfile(Paths.get("nonexistent.Dockerfile")); + assertEquals("handles missing Dockerfiles gracefully", Sets.newHashSet(), parsedDockerfile.getDependencyImageNames()); + } +} diff --git a/core/src/test/resources/Dockerfile b/core/src/test/resources/Dockerfile new file mode 100644 index 00000000000..d3f2fcb60f7 --- /dev/null +++ b/core/src/test/resources/Dockerfile @@ -0,0 +1 @@ +FROM postgres diff --git a/core/src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml b/core/src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml new file mode 100644 index 00000000000..4e86b83720b --- /dev/null +++ b/core/src/test/resources/docker-compose-imagename-parsing-dockerfile-with-context.yml @@ -0,0 +1,12 @@ +version: "2.1" +services: + redis: + image: redis + mysql: + image: mysql + custom: + build: + context: compose-dockerfile + dockerfile: Dockerfile +networks: + custom_network: {} diff --git a/core/src/test/resources/docker-compose-imagename-parsing-dockerfile.yml b/core/src/test/resources/docker-compose-imagename-parsing-dockerfile.yml new file mode 100644 index 00000000000..46d3da26f4c --- /dev/null +++ b/core/src/test/resources/docker-compose-imagename-parsing-dockerfile.yml @@ -0,0 +1,10 @@ +version: "2.1" +services: + redis: + image: redis + mysql: + image: mysql + custom: + build: compose-dockerfile +networks: + custom_network: {}