WIP: Docker authentiation using credential store/helpers

.gitignore

@@ -46,3 +46,5 @@ node_modules/
 
 .gradle/
 build/
+out/
+*.class

CHANGELOG.md

@@ -6,10 +6,12 @@ All notable changes to this project will be documented in this file.
 - Fix for setting `ryuk.container.timeout` causes a `ClassCastException` ([\#684](https://github.com/testcontainers/testcontainers-java/issues/684))
 
 ### Changed
+- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
 
 ## [1.7.2] - 2018-04-30
 
 ### Fixed
+- Add support for private repositories using docker credential stores/helpers (fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
 - Retry any exceptions (not just `DockerClientException`) on image pull ([\#662](https://github.com/testcontainers/testcontainers-java/issues/662))
 - Fixed handling of the paths with `+` in them ([\#664](https://github.com/testcontainers/testcontainers-java/issues/664))
 
@@ -24,6 +26,7 @@ All notable changes to this project will be documented in this file.
 - Fixed `HostPortWaitStrategy` throws `NumberFormatException` when port is exposed but not mapped ([\#640](https://github.com/testcontainers/testcontainers-java/issues/640))
 - Fixed log processing: multibyte unicode, linebreaks and ASCII color codes. Color codes can be turned on with `withRemoveAnsiCodes(false)` ([\#643](https://github.com/testcontainers/testcontainers-java/pull/643))
 - Fixed Docker host IP detection within docker container (detect only if not explicitly set) ([\#648](https://github.com/testcontainers/testcontainers-java/pull/648))
+- Add support for private repositories using docker credential stores/helpers ([PR \#647](https://github.com/testcontainers/testcontainers-java/pull/647), fixes [\#567](https://github.com/testcontainers/testcontainers-java/issues/567))
 
 ### Changed
 - Support multiple HTTP status codes for HttpWaitStrategy ([\#630](https://github.com/testcontainers/testcontainers-java/issues/630))

circle.yml

@@ -14,6 +14,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
   modules-no-jdbc-test-no-selenium:
     steps:
       - checkout
@@ -30,6 +32,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
   modules-jdbc-test:
     steps:
       - checkout
@@ -46,6 +50,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
   selenium:
     steps:
       - checkout
@@ -59,6 +65,8 @@ jobs:
           when: always
       - store_test_results:
           path: ~/junit
+      - store_artifacts:
+          path: ~/junit
 
 workflows:
   version: 2

core/build.gradle

@@ -120,7 +120,7 @@ dependencies {
     }
     shaded 'javax.ws.rs:javax.ws.rs-api:2.0.1'
     shaded 'org.rnorth:tcp-unix-socket-proxy:1.0.2'
-    shaded 'org.zeroturnaround:zt-exec:1.8'
+    shaded 'org.zeroturnaround:zt-exec:1.10'
     shaded 'commons-lang:commons-lang:2.6'
     shaded 'commons-io:commons-io:2.5'
     shaded 'commons-codec:commons-codec:1.11'

core/src/main/java/org/testcontainers/images/RemoteDockerImage.java

@@ -3,6 +3,7 @@
 import com.github.dockerjava.api.DockerClient;
 import com.github.dockerjava.api.command.ListImagesCmd;
 import com.github.dockerjava.api.exception.DockerClientException;
+import com.github.dockerjava.api.model.AuthConfig;
 import com.github.dockerjava.api.model.Image;
 import com.github.dockerjava.core.command.PullImageResultCallback;
 import lombok.NonNull;
@@ -14,6 +15,7 @@
 import org.testcontainers.utility.DockerImageName;
 import org.testcontainers.utility.DockerLoggerFactory;
 import org.testcontainers.utility.LazyFuture;
+import org.testcontainers.utility.RegistryAuthLocator;
 
 import java.util.HashSet;
 import java.util.List;
@@ -93,10 +95,14 @@ protected final String resolve() {
 
                 // The image is not available locally - pull it
                 try {
+                    final RegistryAuthLocator authLocator = new RegistryAuthLocator(dockerClient.authConfig());
+                    final AuthConfig effectiveAuthConfig = authLocator.lookupAuthConfig(imageName);
+
                     final PullImageResultCallback callback = new PullImageResultCallback();
                     dockerClient
                         .pullImageCmd(imageName.getUnversionedPart())
                         .withTag(imageName.getVersionPart())
+                        .withAuthConfig(effectiveAuthConfig)
                         .exec(callback);
                     callback.awaitCompletion();
                     AVAILABLE_IMAGE_NAME_CACHE.add(imageName);

core/src/main/java/org/testcontainers/utility/DockerImageName.java

@@ -94,7 +94,11 @@ public String getVersionPart() {
 
     @Override
     public String toString() {
-        return getUnversionedPart() + versioning.getSeparator() + versioning.toString();
+        if (versioning == null) {
+            return getUnversionedPart();
+        } else {
+            return getUnversionedPart() + versioning.getSeparator() + versioning.toString();
+        }
     }
 
     /**
@@ -116,6 +120,10 @@ public void assertValid() {
         }
     }
 
+    public String getRegistry() {
+        return registry;
+    }
+
     private interface Versioning {
         boolean isValid();
         String getSeparator();

core/src/main/java/org/testcontainers/utility/RegistryAuthLocator.java

@@ -0,0 +1,127 @@
+package org.testcontainers.utility;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.github.dockerjava.api.model.AuthConfig;
+import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.zeroturnaround.exec.ProcessExecutor;
+
+import java.io.ByteArrayInputStream;
+import java.io.File;
+import java.util.concurrent.TimeUnit;
+
+import static org.slf4j.LoggerFactory.getLogger;
+
+/**
+ * Utility to look up registry authentication information for an image.
+ */
+public class RegistryAuthLocator {
+
+    private static final Logger log = getLogger(RegistryAuthLocator.class);
+
+    private final AuthConfig defaultAuthConfig;
+    private final File configFile;
+    private final String commandPathPrefix;
+    private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+    @VisibleForTesting
+    RegistryAuthLocator(AuthConfig defaultAuthConfig, File configFile, String commandPathPrefix) {
+        this.defaultAuthConfig = defaultAuthConfig;
+        this.configFile = configFile;
+        this.commandPathPrefix = commandPathPrefix;
+    }
+
+    /**
+     * @param defaultAuthConfig an AuthConfig object that should be returned if there is no overriding authentication
+     *                          available for images that are looked up
+     */
+    public RegistryAuthLocator(AuthConfig defaultAuthConfig) {
+        this.defaultAuthConfig = defaultAuthConfig;
+        final String dockerConfigLocation = System.getenv().getOrDefault("DOCKER_CONFIG",
+            System.getProperty("user.home") + "/.docker");
+        this.configFile = new File(dockerConfigLocation + "/config.json");
+        this.commandPathPrefix = "";
+    }
+
+    /**
+     * Looks up an AuthConfig for a given image name.
+     *
+     * @param dockerImageName image name to be looked up (potentially including a registry URL part)
+     * @return an AuthConfig that is applicable to this specific image OR the defaultAuthConfig that has been set for
+     * this {@link RegistryAuthLocator}.
+     */
+    public AuthConfig lookupAuthConfig(DockerImageName dockerImageName) {
+        log.debug("Looking up auth config for image: {}", dockerImageName);
+
+        log.debug("RegistryAuthLocator has configFile: {} ({}) and commandPathPrefix: {}",
+            configFile,
+            configFile.exists() ? "exists" : "does not exist",
+            commandPathPrefix);
+
+        try {
+            final JsonNode config = OBJECT_MAPPER.readTree(configFile);
+
+            final String reposName = dockerImageName.getRegistry();
+            final JsonNode auths = config.at("/auths/" + reposName);
+
+            if (!auths.isMissingNode() && auths.size() == 0) {
+                // auths/<registry> is an empty dict - use a credential helper
+                return authConfigUsingCredentialsStoreOrHelper(reposName, config);
+            }
+            // otherwise, defaultAuthConfig should already contain any credentials available
+        } catch (Exception e) {
+            log.error("Failure when attempting to lookup auth config (dockerImageName: {}, configFile: {}. " +
+                "Falling back to docker-java default behaviour",
+                dockerImageName,
+                configFile,
+                e);
+        }
+        return defaultAuthConfig;
+    }
+
+    private AuthConfig authConfigUsingCredentialsStoreOrHelper(String hostName, JsonNode config) throws Exception {
+
+        final JsonNode credsStoreName = config.at("/credsStore");
+        final JsonNode credHelper = config.at("/credHelpers/" + hostName);
+
+        if (!credHelper.isMissingNode()) {
+            return runCredentialProvider(hostName, credHelper.asText());
+        } else if (!credsStoreName.isMissingNode()) {
+            return runCredentialProvider(hostName, credsStoreName.asText());
+        } else {
+            throw new IllegalStateException("Unsupported Docker config auths settings!");
+        }
+    }
+
+    private AuthConfig runCredentialProvider(String hostName, String credHelper) throws Exception {
+        final String credentialHelperName = commandPathPrefix + "docker-credential-" + credHelper;
+        String data;
+
+        log.debug("Executing docker credential helper: {} to locate auth config for: {}",
+            credentialHelperName, hostName);
+
+        try {
+            data = new ProcessExecutor()
+                .command(credentialHelperName, "get")
+                .redirectInput(new ByteArrayInputStream(hostName.getBytes()))
+                .readOutput(true)
+                .exitValueNormal()
+                .timeout(30, TimeUnit.SECONDS)
+                .execute()
+                .outputUTF8()
+                .trim();
+        } catch (Exception e) {
+            log.error("Failure running docker credential helper ({})", credentialHelperName);
+            throw e;
+        }
+
+        final JsonNode helperResponse = OBJECT_MAPPER.readTree(data);
+        log.debug("Credential helper provided auth config for: {}", hostName);
+
+        return new AuthConfig()
+            .withRegistryAddress(helperResponse.at("/ServerURL").asText())
+            .withUsername(helperResponse.at("/Username").asText())
+            .withPassword(helperResponse.at("/Secret").asText());
+    }
+}

core/src/test/java/org/testcontainers/dockerclient/ImagePullTest.java

@@ -22,7 +22,7 @@ public static String[] parameters() {
             "gliderlabs/alpine@sha256:a19aa4a17a525c97e5a90a0c53a9f3329d2dc61b0a14df5447757a865671c085",
             "quay.io/testcontainers/ryuk:latest",
             "quay.io/testcontainers/ryuk:0.2.2",
-            "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da"
+            "quay.io/testcontainers/ryuk@sha256:4b606e54c4bba1af4fd814019d342e4664d51e28d3ba2d18d24406edbefd66da",
         };
     }
 

core/src/test/java/org/testcontainers/utility/RegistryAuthLocatorTest.java

@@ -0,0 +1,63 @@
+package org.testcontainers.utility;
+
+import com.github.dockerjava.api.model.AuthConfig;
+import com.google.common.io.Resources;
+import org.apache.commons.lang.SystemUtils;
+import org.jetbrains.annotations.NotNull;
+import org.junit.Assume;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import java.io.File;
+import java.net.URISyntaxException;
+
+import static org.rnorth.visibleassertions.VisibleAssertions.assertEquals;
+import static org.rnorth.visibleassertions.VisibleAssertions.assertNull;
+
+public class RegistryAuthLocatorTest {
+
+    @BeforeClass
+    public static void nonWindowsTest() throws Exception {
+        Assume.assumeFalse(SystemUtils.IS_OS_WINDOWS);
+    }
+
+    @Test
+    public void lookupAuthConfigWithoutCredentials() throws URISyntaxException {
+        final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json");
+
+        final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("unauthenticated.registry.org/org/repo"));
+
+        assertEquals("Default docker registry URL is set on auth config", "https://index.docker.io/v1/", authConfig.getRegistryAddress());
+        assertNull("No username is set", authConfig.getUsername());
+        assertNull("No password is set", authConfig.getPassword());
+    }
+
+    @Test
+    public void lookupAuthConfigUsingStore() throws URISyntaxException {
+        final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-store.json");
+
+        final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo"));
+
+        assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress());
+        assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername());
+        assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword());
+    }
+
+    @Test
+    public void lookupAuthConfigUsingHelper() throws URISyntaxException {
+        final RegistryAuthLocator authLocator = createTestAuthLocator("config-with-helper.json");
+
+        final AuthConfig authConfig = authLocator.lookupAuthConfig(new DockerImageName("registry.example.com/org/repo"));
+
+        assertEquals("Correct server URL is obtained from a credential store", "url", authConfig.getRegistryAddress());
+        assertEquals("Correct username is obtained from a credential store", "username", authConfig.getUsername());
+        assertEquals("Correct secret is obtained from a credential store", "secret", authConfig.getPassword());
+    }
+
+    @NotNull
+    private RegistryAuthLocator createTestAuthLocator(String configName) throws URISyntaxException {
+        final File configFile = new File(Resources.getResource("auth-config/" + configName).toURI());
+        return new RegistryAuthLocator(new AuthConfig(), configFile, configFile.getParentFile().getAbsolutePath() + "/");
+    }
+
+}

core/src/test/resources/auth-config/config-with-helper-and-store.json

@@ -0,0 +1,12 @@
+{
+    "auths": {
+        "registry.example.com": {}
+    },
+    "HttpHeaders": {
+        "User-Agent": "Docker-Client/18.03.0-ce (darwin)"
+    },
+    "credsStore": "fake",
+    "credHelpers": {
+        "registry.example.com": "fake"
+    }
+}

core/src/test/resources/auth-config/config-with-helper.json

@@ -0,0 +1,11 @@
+{
+    "auths": {
+        "registry.example.com": {}
+    },
+    "HttpHeaders": {
+        "User-Agent": "Docker-Client/18.03.0-ce (darwin)"
+    },
+    "credHelpers": {
+        "registry.example.com": "fake"
+    }
+}

core/src/test/resources/auth-config/config-with-store.json

@@ -0,0 +1,9 @@
+{
+    "auths": {
+        "registry.example.com": {}
+    },
+    "HttpHeaders": {
+        "User-Agent": "Docker-Client/18.03.0-ce (darwin)"
+    },
+    "credsStore": "fake"
+}

core/src/test/resources/auth-config/docker-credential-fake

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+if [[ $1 != "get" ]]; then
+    exit 1
+fi
+
+read > /dev/null
+
+echo '{' \
+     '  "ServerURL": "url",' \
+     '  "Username": "username",' \
+     '  "Secret": "secret"' \
+     '}'