chore: address testcontainers vulnerability by replacing with docker-java #1088
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,55 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package aws.smithy.kotlin.runtime.http.test | ||
|
|
||
| import aws.smithy.kotlin.runtime.http.test.util.Docker | ||
| import com.github.dockerjava.api.model.AccessMode | ||
| import com.github.dockerjava.api.model.Bind | ||
| import com.github.dockerjava.api.model.ExposedPort | ||
| import com.github.dockerjava.api.model.Volume | ||
| import java.io.Closeable | ||
|
|
||
| private const val CONTAINER_MOUNT_POINT = "/home/mitmproxy/scripts" | ||
| private const val CONTAINER_PORT = 8080 | ||
| private const val IMAGE_NAME = "mitmproxy/mitmproxy:8.1.0" | ||
| private val PROXY_SCRIPT_ROOT = System.getProperty("MITM_PROXY_SCRIPTS_ROOT") // defined by gradle script | ||
|
|
||
| // Port used for communication with container | ||
| private val exposedPort = ExposedPort.tcp(CONTAINER_PORT) | ||
|
|
||
| class MitmContainer(vararg options: String) : Closeable { | ||
|
|
||
| private val delegate: Docker.Container | ||
|
|
||
| init { | ||
| val cmd = listOf( | ||
| "mitmdump", | ||
| "--flow-detail", | ||
| "2", | ||
| "-s", | ||
| "$CONTAINER_MOUNT_POINT/fakeupstream.py", | ||
| *options, | ||
| ).also { println("Initializing container with command: $it") } | ||
|
|
||
| // Make proxy scripts from host available to container | ||
| val binding = Bind(PROXY_SCRIPT_ROOT, Volume(CONTAINER_MOUNT_POINT), AccessMode.ro) | ||
|
|
||
| delegate = Docker.Instance.createContainer(IMAGE_NAME, cmd, binding, exposedPort) | ||
|
|
||
| try { | ||
| delegate.apply { | ||
| start() | ||
| waitUntilReady() | ||
| } | ||
| } catch (e: Throwable) { | ||
| close() | ||
| throw e | ||
| } | ||
| } | ||
|
|
||
| val hostPort: Int | ||
| get() = delegate.hostPort | ||
|
|
||
| override fun close() = delegate.close() | ||
| } | ||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,173 @@ | ||
| /* | ||
| * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. | ||
| * SPDX-License-Identifier: Apache-2.0 | ||
| */ | ||
| package aws.smithy.kotlin.runtime.http.test.util | ||
|
|
||
| import com.github.dockerjava.api.async.ResultCallback | ||
| import com.github.dockerjava.api.command.AsyncDockerCmd | ||
| import com.github.dockerjava.api.command.SyncDockerCmd | ||
| import com.github.dockerjava.api.model.* | ||
| import com.github.dockerjava.core.DefaultDockerClientConfig | ||
| import com.github.dockerjava.core.DockerClientImpl | ||
| import com.github.dockerjava.zerodep.ZerodepDockerHttpClient | ||
| import java.io.Closeable | ||
| import java.io.IOException | ||
| import java.net.InetAddress | ||
| import java.net.InetSocketAddress | ||
| import java.net.Socket | ||
| import java.net.URI | ||
| import kotlin.time.Duration.Companion.milliseconds | ||
| import kotlin.time.Duration.Companion.seconds | ||
| import kotlin.time.measureTimedValue | ||
|
|
||
| private val DOCKER_HOST = URI.create("unix:///var/run/docker.sock") | ||
| private val MAX_POLL_TIME = 10.seconds | ||
| private const val POLL_CONNECT_TIMEOUT_MS = 100 | ||
| private val POLL_INTERVAL = 250.milliseconds | ||
|
|
||
| /** | ||
| * Wrapper class for the Docker client | ||
| */ | ||
| class Docker { | ||
| companion object { | ||
| val Instance by lazy { Docker() } | ||
| } | ||
|
|
||
| private val client = run { | ||
| val config = DefaultDockerClientConfig.createDefaultConfigBuilder().build() | ||
|
|
||
| val httpClient = ZerodepDockerHttpClient.Builder() | ||
| .dockerHost(DOCKER_HOST) | ||
| .build() | ||
|
|
||
| DockerClientImpl.getInstance(config, httpClient) | ||
| } | ||
|
|
||
| fun createContainer( | ||
| imageName: String, | ||
| cmd: List<String>, | ||
| bind: Bind, | ||
| exposedPort: ExposedPort, | ||
| loggingHandler: (String) -> Unit = ::println, | ||
| ): Container { | ||
| ensureImageExists(imageName, loggingHandler) | ||
|
|
||
| val portBinding = PortBinding(Ports.Binding.empty(), exposedPort) | ||
|
|
||
| val hostConfig = HostConfig | ||
| .newHostConfig() | ||
| .withBinds(bind) | ||
| .withPortBindings(portBinding) | ||
|
|
||
| val id = client | ||
| .createContainerCmd(imageName) | ||
| .withHostConfig(hostConfig) | ||
| .withExposedPorts(exposedPort) | ||
| .withCmd(cmd) | ||
| .execAndMeasure { "Created container ${it.id}" } | ||
| .id | ||
| .substring(0..<12) // Short container IDs are 12 chars vs full container IDs at 64 chars | ||
|
|
||
| val loggerAdapter = LoggerAdapter<Frame>(loggingHandler) { it.payload.decodeToString() } | ||
|
|
||
| client | ||
| .attachContainerCmd(id) | ||
| .withFollowStream(true) | ||
| .withStdOut(true) | ||
| .withStdErr(true) | ||
| .withLogs(true) | ||
| .execAndMeasure(loggerAdapter) { "Attached logger to container $id" } | ||
| .awaitStarted() | ||
|
|
||
| return Container(id, exposedPort) | ||
| } | ||
|
|
||
| private fun ensureImageExists(imageName: String, loggingHandler: (String) -> Unit) { | ||
| val exists = client | ||
| .listImagesCmd() | ||
| .withReferenceFilter(imageName) | ||
| .execAndMeasure { "Checking for $imageName locally (exists = ${it.any()})" } | ||
| .any() | ||
|
|
||
| if (!exists) { | ||
| val loggerAdapter = LoggerAdapter<PullResponseItem>(loggingHandler) { it.status } | ||
|
|
||
| client | ||
| .pullImageCmd(imageName) | ||
| .execAndMeasure(loggerAdapter) { "Started image pull for $imageName" } | ||
| .awaitCompletion() | ||
| } | ||
| } | ||
|
|
||
| inner class Container(val id: String, val exposedPort: ExposedPort) : Closeable { | ||
| private val poller = Poller(MAX_POLL_TIME, POLL_INTERVAL) | ||
|
|
||
| override fun close() { | ||
| client | ||
| .removeContainerCmd(id) | ||
| .withForce(true) | ||
| .exec() | ||
| .also { println("Container $id removed") } | ||
| } | ||
|
|
||
| val hostPort: Int by lazy { | ||
| poller.pollNotNull("Port $exposedPort in container $id") { | ||
| client | ||
| .inspectContainerCmd(id) | ||
| .exec() | ||
| .networkSettings | ||
| .ports | ||
| .bindings[exposedPort] | ||
| ?.first() | ||
| ?.hostPortSpec | ||
| ?.toInt() | ||
| } | ||
| } | ||
|
|
||
| private fun isReady() = | ||
| Socket().use { socket -> | ||
| val endpoint = InetSocketAddress(InetAddress.getLocalHost(), hostPort) | ||
| try { | ||
| socket.connect(endpoint, POLL_CONNECT_TIMEOUT_MS) | ||
| true | ||
| } catch (e: IOException) { | ||
| false | ||
| } | ||
| } | ||
|
|
||
| fun start() { | ||
| client.startContainerCmd(id).execAndMeasure { "Container $id running" } | ||
| } | ||
|
|
||
| fun waitUntilReady() = poller.pollTrue("Socket localHost:$hostPort → $exposedPort on container $id", ::isReady) | ||
| } | ||
| } | ||
|
|
||
| private class LoggerAdapter<I>( | ||
| val handler: (String) -> Unit, | ||
| val converter: (I) -> String?, | ||
| ) : ResultCallback.Adapter<I>() { | ||
| override fun onNext(value: I?) { | ||
| value?.let(converter)?.let(handler) | ||
| } | ||
| } | ||
|
|
||
| private fun <T> SyncDockerCmd<T>.execAndMeasure(msg: (T) -> String): T { | ||
| val (value, duration) = measureTimedValue { | ||
| exec() | ||
| } | ||
| println("${msg(value)} in $duration") | ||
| return value | ||
| } | ||
|
|
||
| private fun <C : AsyncDockerCmd<C, T>?, T, I : ResultCallback<T>> AsyncDockerCmd<C, T>.execAndMeasure( | ||
| input: I, | ||
| msg: (I) -> String, | ||
| ): I { | ||
| val (value, duration) = measureTimedValue { | ||
| exec(input) | ||
| } | ||
| println("${msg(value)} in $duration") | ||
| return value | ||
| } |
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Is this different from
./gradlew dependencies?There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
Yes,
./gradlew dependenciesonly shows the dependencies of the target project (root if otherwise unspecified). I initially ran./gradlew dependenciesto grep for where we transitively consumed the Apache Commons Compress library but couldn't find it because it only showed me the root project dependencies.Running
./gradlew :runtime:protocol:http-client-engines:test-suite:dependenciesshows our use of Apache Commons Compress but then you have to know that :runtime:protocol:http-client-engines:test-suite is where to look. This new task registers an effective alias ofdependenciesfor every subproject in such a way that it can be invoked at the root project and yield the dependencies for root and every subproject.