Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Image substitution #3102

Merged
merged 18 commits into from
Nov 5, 2020
Merged
Show file tree
Hide file tree
Changes from 5 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 8 additions & 3 deletions core/src/main/java/org/testcontainers/DockerClientFactory.java
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,8 @@
import org.testcontainers.dockerclient.TransportConfig;
import org.testcontainers.images.TimeLimitedLoggedPullImageResultCallback;
import org.testcontainers.utility.ComparableVersion;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.ImageNameSubstitutor;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ResourceReaper;
import org.testcontainers.utility.TestcontainersConfiguration;
Expand Down Expand Up @@ -61,7 +63,7 @@ public class DockerClientFactory {
TESTCONTAINERS_SESSION_ID_LABEL, SESSION_ID
);

private static final String TINY_IMAGE = TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString();
private static final DockerImageName TINY_IMAGE = DockerImageName.parse("alpine:3.5");
private static DockerClientFactory instance;

// Cached client configuration
Expand Down Expand Up @@ -343,8 +345,11 @@ public <T> T runInsideDocker(Consumer<CreateContainerCmd> createContainerCmdCons
}

private <T> T runInsideDocker(DockerClient client, Consumer<CreateContainerCmd> createContainerCmdConsumer, BiFunction<DockerClient, String, T> block) {
checkAndPullImage(client, TINY_IMAGE);
CreateContainerCmd createContainerCmd = client.createContainerCmd(TINY_IMAGE)

final String tinyImage = ImageNameSubstitutor.instance().apply(TINY_IMAGE).asCanonicalNameString();

checkAndPullImage(client, tinyImage);
CreateContainerCmd createContainerCmd = client.createContainerCmd(tinyImage)
.withLabels(DEFAULT_LABELS);
createContainerCmdConsumer.accept(createContainerCmd);
String id = createContainerCmd.exec().getId();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@
import org.testcontainers.utility.AuditLogger;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.CommandLine;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.LogUtils;
import org.testcontainers.utility.MountableFile;
import org.testcontainers.utility.ResourceReaper;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.zeroturnaround.exec.InvalidExitValueException;
import org.zeroturnaround.exec.ProcessExecutor;
import org.zeroturnaround.exec.stream.slf4j.Slf4jStream;
Expand Down Expand Up @@ -608,10 +608,11 @@ interface DockerCompose {
class ContainerisedDockerCompose extends GenericContainer<ContainerisedDockerCompose> implements DockerCompose {

public static final char UNIX_PATH_SEPERATOR = ':';
public static final DockerImageName DEFAULT_IMAGE_NAME = DockerImageName.parse("docker/compose:1.24.1");

public ContainerisedDockerCompose(List<File> composeFiles, String identifier) {

super(TestcontainersConfiguration.getInstance().getDockerComposeDockerImageName());
super(DEFAULT_IMAGE_NAME);
addEnv(ENV_PROJECT_NAME, identifier);

// Map the docker compose file into the container
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -239,13 +239,9 @@ public GenericContainer(@NonNull final RemoteDockerImage image) {
*/
@Deprecated
public GenericContainer() {
this(TestcontainersConfiguration.getInstance().getTinyDockerImageName().asCanonicalNameString());
this(TestcontainersConfiguration.getInstance().getTinyImage());
}

/**
* @deprecated use {@link GenericContainer(DockerImageName)} instead
*/
@Deprecated
public GenericContainer(@NonNull final String dockerImageName) {
this.setDockerImageName(dockerImageName);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,15 +5,15 @@
import lombok.AccessLevel;
import lombok.Getter;
import lombok.SneakyThrows;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.testcontainers.utility.DockerImageName;

import java.time.Duration;
import java.util.AbstractMap;
import java.util.Collections;
import java.util.Map.Entry;
import java.util.Optional;
import java.util.Set;
import java.util.UUID;
import java.util.Map.Entry;
import java.util.concurrent.ConcurrentHashMap;

public enum PortForwardingContainer {
Expand All @@ -29,7 +29,7 @@ public enum PortForwardingContainer {
@SneakyThrows
private Connection createSSHSession() {
String password = UUID.randomUUID().toString();
container = new GenericContainer<>(TestcontainersConfiguration.getInstance().getSSHdDockerImageName())
container = new GenericContainer<>(DockerImageName.parse("testcontainers/sshd:1.0.0"))
.withExposedPorts(22)
.withEnv("PASSWORD", password)
.withCommand(
Expand Down
Original file line number Diff line number Diff line change
@@ -1,12 +1,10 @@
package org.testcontainers.containers;

import org.testcontainers.utility.Base58;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.TestcontainersConfiguration;

import java.util.HashMap;
import java.util.Map;
import java.util.stream.Collectors;
import org.testcontainers.utility.Base58;
import org.testcontainers.utility.DockerImageName;

/**
* A socat container is used as a TCP proxy, enabling any TCP port of another container to be exposed
Expand All @@ -17,7 +15,7 @@ public class SocatContainer extends GenericContainer<SocatContainer> {
private final Map<Integer, String> targets = new HashMap<>();

public SocatContainer() {
this(TestcontainersConfiguration.getInstance().getSocatDockerImageName());
this(DockerImageName.parse("alpine/socat:1.7.3.4-r0"));
}

public SocatContainer(final DockerImageName dockerImageName) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
import lombok.ToString;
import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.testcontainers.containers.wait.strategy.LogMessageWaitStrategy;
import org.testcontainers.utility.TestcontainersConfiguration;
import org.testcontainers.utility.DockerImageName;

import java.io.File;
import java.io.InputStream;
Expand Down Expand Up @@ -52,7 +52,7 @@ public VncRecordingContainer(@NonNull GenericContainer<?> targetContainer) {
* Create a sidekick container and attach it to another container. The VNC output of that container will be recorded.
*/
public VncRecordingContainer(@NonNull Network network, @NonNull String targetNetworkAlias) throws IllegalStateException {
super(TestcontainersConfiguration.getInstance().getVncDockerImageName());
super(DockerImageName.parse("testcontainers/vnc-recorder:1.1.0"));

this.targetNetworkAlias = targetNetworkAlias;
withNetwork(network);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -171,7 +171,7 @@ public static DockerClientProviderStrategy getFirstValidStrategy(List<DockerClie
}

if (strategy.isPersistable()) {
TestcontainersConfiguration.getInstance().updateGlobalConfig("docker.client.strategy", strategy.getClass().getName());
TestcontainersConfiguration.getInstance().updateUserConfig("docker.client.strategy", strategy.getClass().getName());
}

return Stream.of(strategy);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import org.testcontainers.containers.ContainerFetchException;
import org.testcontainers.utility.DockerImageName;
import org.testcontainers.utility.DockerLoggerFactory;
import org.testcontainers.utility.ImageNameSubstitutor;
import org.testcontainers.utility.LazyFuture;

import java.time.Duration;
Expand Down Expand Up @@ -44,12 +45,12 @@ public RemoteDockerImage(DockerImageName dockerImageName) {

@Deprecated
public RemoteDockerImage(String dockerImageName) {
this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(dockerImageName));
this(DockerImageName.parse(dockerImageName));
}

@Deprecated
public RemoteDockerImage(@NonNull String repository, @NonNull String tag) {
this.imageNameFuture = CompletableFuture.completedFuture(DockerImageName.parse(repository).withTag(tag));
this(DockerImageName.parse(repository).withTag(tag));
}

public RemoteDockerImage(@NonNull Future<String> imageFuture) {
Expand Down Expand Up @@ -100,7 +101,10 @@ protected final String resolve() {
}

private DockerImageName getImageName() throws InterruptedException, ExecutionException {
return imageNameFuture.get();
final DockerImageName specifiedImageName = imageNameFuture.get();

// Allow the image name to be substituted
return ImageNameSubstitutor.instance().apply(specifiedImageName);
}

@ToString.Include(name = "imageName", rank = 1)
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,52 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

/**
* {@link ImageNameSubstitutor} which takes replacement image names from configuration.
* See {@link TestcontainersConfiguration} for the subset of image names which can be substituted using this mechanism.
* <p>
* WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader}
bsideup marked this conversation as resolved.
Show resolved Hide resolved
* requires it to be so. Public visibility DOES NOT make it part of the public API.
*/
@Slf4j
public class ConfigurationFileImageNameSubstitutor extends ImageNameSubstitutor {
bsideup marked this conversation as resolved.
Show resolved Hide resolved

private final TestcontainersConfiguration configuration;

public ConfigurationFileImageNameSubstitutor() {
this(TestcontainersConfiguration.getInstance());
}

@VisibleForTesting
ConfigurationFileImageNameSubstitutor(TestcontainersConfiguration configuration) {
this.configuration = configuration;
}

@Override
public DockerImageName apply(final DockerImageName original) {
final DockerImageName result = configuration
.getConfiguredSubstituteImage(original)
.asCompatibleSubstituteFor(original);

if (!result.equals(original)) {
log.warn("Image name {} was substituted by configuration to {}. This approach is deprecated and will be removed in the future",
original,
result
);
}

return result;
}

@Override
protected int getPriority() {
return -2;
}

@Override
protected String getDescription() {
return getClass().getSimpleName();
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

/**
* Testcontainers' default implementation of {@link ImageNameSubstitutor}.
* Delegates to {@link ConfigurationFileImageNameSubstitutor}.
* <p>
* WARNING: this class is not intended to be public, but {@link java.util.ServiceLoader}
* requires it to be so. Public visibility DOES NOT make it part of the public API.
*/
@Slf4j
public class DefaultImageNameSubstitutor extends ImageNameSubstitutor {
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I've made this as a default out-of-the-box substitutor that delegates to both the config file settings and a simple 'prefix' substitutor which can apply a common prefix to all image names.

I suspect this might be enough for 80% of situations where people need a substitutor, which is why I think it's worth including it by default.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

FTR for others looking at this PR:
this comment is outdated, see #3413 for the prefixing substitutor

bsideup marked this conversation as resolved.
Show resolved Hide resolved

private final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor;

public DefaultImageNameSubstitutor() {
configurationFileImageNameSubstitutor = new ConfigurationFileImageNameSubstitutor();
}

@VisibleForTesting
DefaultImageNameSubstitutor(
final ConfigurationFileImageNameSubstitutor configurationFileImageNameSubstitutor
) {
this.configurationFileImageNameSubstitutor = configurationFileImageNameSubstitutor;
}

@Override
public DockerImageName apply(final DockerImageName original) {
return configurationFileImageNameSubstitutor.apply(original);
bsideup marked this conversation as resolved.
Show resolved Hide resolved
}

@Override
protected int getPriority() {
return 0;
}

@Override
protected String getDescription() {
return "DefaultImageNameSubstitutor (delegates to '" + configurationFileImageNameSubstitutor.getDescription() + "')";
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
package org.testcontainers.utility;

import com.google.common.annotations.VisibleForTesting;
import lombok.extern.slf4j.Slf4j;

import java.util.ServiceLoader;
import java.util.function.Function;
import java.util.stream.StreamSupport;

import static java.util.Comparator.comparingInt;

/**
* An image name substitutor converts a Docker image name, as may be specified in code, to an alternative name.
* This is intended to provide a way to override image names, for example to enforce pulling of images from a private
* registry.
*/
@Slf4j
public abstract class ImageNameSubstitutor implements Function<DockerImageName, DockerImageName> {

@VisibleForTesting
static ImageNameSubstitutor instance;

public synchronized static ImageNameSubstitutor instance() {
if (instance == null) {
final ServiceLoader<ImageNameSubstitutor> serviceLoader = ServiceLoader.load(ImageNameSubstitutor.class);

instance = StreamSupport.stream(serviceLoader.spliterator(), false)
.peek(it -> log.debug("Found ImageNameSubstitutor using ServiceLoader: {} (priority {}) ", it, it.getPriority()))
.max(comparingInt(ImageNameSubstitutor::getPriority))
.map(ImageNameSubstitutor::wrapWithLogging)
.orElseThrow(() -> new RuntimeException("Unable to find any ImageNameSubstitutor using ServiceLoader"));

log.info("Using ImageNameSubstitutor: {}", instance);
}

return instance;
}

private static ImageNameSubstitutor wrapWithLogging(final ImageNameSubstitutor wrappedInstance) {
return new LogWrappedImageNameSubstitutor(wrappedInstance);
}

/**
* Substitute a {@link DockerImageName} for another, for example to replace a generic Docker Hub image name with a
* private registry copy of the image.
*
* @param original original name to be replaced
* @return a replacement name, or the original, as appropriate
*/
public abstract DockerImageName apply(DockerImageName original);

/**
* Priority of this {@link ImageNameSubstitutor} compared to other instances that may be found by the service
* loader. The highest priority instance found will always be used.
*
* @return a priority
*/
protected abstract int getPriority();

protected abstract String getDescription();

/**
* Wrapper substitutor which logs which substitutions have been performed.
*/
static class LogWrappedImageNameSubstitutor extends ImageNameSubstitutor {
@VisibleForTesting
final ImageNameSubstitutor wrappedInstance;

public LogWrappedImageNameSubstitutor(final ImageNameSubstitutor wrappedInstance) {
this.wrappedInstance = wrappedInstance;
}

@Override
public DockerImageName apply(final DockerImageName original) {
final String className = wrappedInstance.getClass().getName();
final DockerImageName replacementImage = wrappedInstance.apply(original);

if (!replacementImage.equals(original)) {
log.info("Using {} as a substitute image for {} (using image substitutor: {})", replacementImage.asCanonicalNameString(), original.asCanonicalNameString(), className);
return replacementImage;
} else {
log.debug("Did not find a substitute image for {} (using image substitutor: {})", original.asCanonicalNameString(), className);
return original;
}
}

@Override
protected int getPriority() {
return wrappedInstance.getPriority();
}

@Override
protected String getDescription() {
return wrappedInstance.getDescription();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -72,7 +72,9 @@ private ResourceReaper() {

@SneakyThrows(InterruptedException.class)
public static String start(String hostIpAddress, DockerClient client) {
String ryukImage = TestcontainersConfiguration.getInstance().getRyukDockerImageName().asCanonicalNameString();
String ryukImage = ImageNameSubstitutor.instance()
.apply(DockerImageName.parse("testcontainers/ryuk:0.3.0"))
.asCanonicalNameString();
DockerClientFactory.instance().checkAndPullImage(client, ryukImage);

List<Bind> binds = new ArrayList<>();
Expand Down
Loading