From bc7e1eadf3ef7e7ee12b7b00cec50d790b889f80 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 28 Sep 2021 16:51:34 -0400 Subject: [PATCH 01/16] feat(rt): bootstrap imds client --- aws-runtime/aws-config/build.gradle.kts | 28 ++ .../kotlin/runtime/config/imds/EC2Metadata.kt | 221 ++++++++++++++++ .../config/imds/ImdsEndpointResolver.kt | 50 ++++ .../sdk/kotlin/runtime/config/imds/Token.kt | 39 +++ .../runtime/config/imds/TokenMiddleware.kt | 95 +++++++ .../runtime/config/imds/EC2MetadataTest.kt | 250 ++++++++++++++++++ .../aws/sdk/kotlin/runtime/AwsSdkSetting.kt | 18 ++ .../sdk/kotlin/runtime/testing/ManualClock.kt | 24 ++ settings.gradle.kts | 1 + 9 files changed, 726 insertions(+) create mode 100644 aws-runtime/aws-config/build.gradle.kts create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt create mode 100644 aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt create mode 100644 aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts new file mode 100644 index 00000000000..96905e9fc52 --- /dev/null +++ b/aws-runtime/aws-config/build.gradle.kts @@ -0,0 +1,28 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +description = "Support for AWS configuration" +extra["moduleName"] = "aws.sdk.kotlin.runtime.config" + +val smithyKotlinVersion: String by project + +kotlin { + sourceSets { + commonMain { + dependencies { + api(project(":aws-runtime:aws-core")) + implementation("aws.smithy.kotlin:logging:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:http:$smithyKotlinVersion") + implementation(project(":aws-runtime:http-client-engine-crt")) + implementation(project(":aws-runtime:protocols:http")) + } + } + commonTest { + dependencies { + implementation(project(":aws-runtime:testing")) + } + } + } +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt new file mode 100644 index 00000000000..474cd218533 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt @@ -0,0 +1,221 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.imds + +import aws.sdk.kotlin.runtime.AwsServiceException +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.client.AwsClientOption +import aws.sdk.kotlin.runtime.endpoint.Endpoint +import aws.sdk.kotlin.runtime.http.ApiMetadata +import aws.sdk.kotlin.runtime.http.AwsUserAgentMetadata +import aws.sdk.kotlin.runtime.http.engine.crt.CrtHttpEngine +import aws.sdk.kotlin.runtime.http.middleware.ServiceEndpointResolver +import aws.sdk.kotlin.runtime.http.middleware.UserAgent +import aws.smithy.kotlin.runtime.client.ExecutionContext +import aws.smithy.kotlin.runtime.client.SdkClientOption +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine +import aws.smithy.kotlin.runtime.http.engine.HttpClientEngineConfig +import aws.smithy.kotlin.runtime.http.operation.* +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.io.Closeable +import aws.smithy.kotlin.runtime.io.middleware.Phase +import aws.smithy.kotlin.runtime.logging.Logger +import aws.smithy.kotlin.runtime.time.Clock +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +/** + * Maximum time allowed by default (6 hours) + */ +internal const val DEFAULT_TOKEN_TTL_SECONDS: Int = 21_600 +internal const val DEFAULT_MAX_RETRIES: UInt = 3u + +private const val SERVICE = "IMDS" + +/** + * IMDSv2 Client + * + * This client supports fetching tokens, retrying failures, and token caching according to the specified TTL. + * + * NOTE: This client ONLY supports IMDSv2. It will not fallback to IMDSv1. + * See [transitioning to IMDSv2](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/configuring-instance-metadata-service.html#instance-metadata-transition-to-version-2) + * for more information. + */ +@OptIn(ExperimentalTime::class) +public class EC2Metadata private constructor(builder: Builder) : Closeable { + public constructor() : this(Builder()) + + private val logger = Logger.getLogger() + + private val maxRetries: UInt = builder.maxRetries + private val endpointOverride: Endpoint? = builder.endpoint + private val endpointModeOverride: EndpointMode? = builder.endpointMode + private val tokenTTL: Duration = builder.tokenTTL + private val clock: Clock = builder.clock + + init { + if (endpointOverride != null && endpointModeOverride != null) { + logger.warn { + "EndpointMode was set in combination with an explicit endpoint. " + + "The mode override will be ignored: endpointMode=$endpointModeOverride, endpoint=$endpointOverride" + } + } + + // validate the override at construction time + if (endpointOverride != null) { + try { + Url.parse(endpointOverride.toUrl().toString()) + } catch (ex: Exception) { + throw ConfigurationException("endpointOverride `$endpointOverride` is not a valid URI", ex) + } + } + } + + // TODO connect/socket timeouts + private val httpClient = sdkHttpClient(builder.engine ?: CrtHttpEngine(HttpClientEngineConfig())) + + // cached middleware instances + private val middleware: List = listOf( + ServiceEndpointResolver.create { + serviceId = SERVICE + resolver = ImdsEndpointResolver(endpointModeOverride, endpointOverride) + }, + UserAgent.create { + metadata = AwsUserAgentMetadata.fromEnvironment(ApiMetadata(SERVICE, "")) + }, + TokenMiddleware.create { + httpClient = this@EC2Metadata.httpClient + ttl = tokenTTL + } + ) + + public companion object { + public operator fun invoke(block: Builder.() -> Unit): EC2Metadata = EC2Metadata(Builder().apply(block)) + } + + /** + * Retrieve information from instance metadata service (IMDS). + * + * This method will combine [path] with the configured endpoint and return the response as a string. + * + * For more information about IMDSv2 methods and functionality, see + * [Instance metadata and user data](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html) + * + * Example: + * + * ```kotlin + * val client = EC2Metadata() + * val amiId = client.get("/latest/meta-data/ami-id") + * ``` + */ + public suspend fun get(path: String): String { + val op = SdkHttpOperation.build { + serializer = UnitSerializer + deserializer = object : HttpDeserialize { + override suspend fun deserialize(context: ExecutionContext, response: HttpResponse): String { + if (response.status.isSuccess()) { + val payload = response.body.readAll() ?: throw EC2MetadataError(response.status.value, "no metadata payload") + return payload.decodeToString() + } else { + throw EC2MetadataError(response.status.value, "error retrieving instance metadata") + } + } + } + context { + operationName = path + service = SERVICE + set(SdkClientOption.Clock, clock) + // artifact of re-using ServiceEndpointResolver middleware + set(AwsClientOption.Region, "not-used") + } + } + middleware.forEach { it.install(op) } + op.execution.mutate.intercept(Phase.Order.Before) { req, next -> + req.subject.url.path = path + next.call(req) + } + + // TODO - retries + return op.roundTrip(httpClient, Unit) + } + + override fun close() { + httpClient.close() + } + + public class Builder { + /** + * The maximum number of retries for fetching tokens and metadata + */ + public var maxRetries: UInt = DEFAULT_MAX_RETRIES + + /** + * The endpoint to make requests to. By default this is determined by the execution environment + */ + public var endpoint: Endpoint? = null + + /** + * The [EndpointMode] to use when connecting to [endpoint] + */ + public var endpointMode: EndpointMode? = null + + /** + * Override the time-to-live for the session token + */ + public var tokenTTL: Duration = Duration.seconds(DEFAULT_TOKEN_TTL_SECONDS) + + /** + * The HTTP engine to use to make requests with. This is here to facilitate testing and can otherwise be ignored + */ + internal var engine: HttpClientEngine? = null + + /** + * The source of time for token refreshes + */ + internal var clock: Clock = Clock.System + } +} + +public enum class EndpointMode(internal val defaultEndpoint: Endpoint) { + /** + * IPv4 mode. This is the default unless otherwise specified + * e.g. `http://169.254.169.254' + */ + IPv4(Endpoint("169.254.169.254", "http")), + + /** + * IPv6 mode + * e.g. `http://[fd00:ec2::254]` + */ + IPv6(Endpoint("[fd00:ec2::254]", "http")); + + public companion object { + public fun fromValue(value: String): EndpointMode = when (value.lowercase()) { + "ipv4" -> IPv4 + "ipv6" -> IPv6 + else -> throw IllegalArgumentException("invalid EndpointMode: $value") + } + } +} + +/** + * Exception thrown when an error occurs retrieving metadata from IMDS + * + * @param statusCode The raw HTTP status code of the response + * @param message The error message + */ +public class EC2MetadataError(statusCode: Int, message: String) : AwsServiceException(message) + +private fun Endpoint.toUrl(): Url { + val endpoint = this + val protocol = Protocol.parse(endpoint.protocol) + return Url( + scheme = protocol, + host = endpoint.hostname, + port = endpoint.port ?: protocol.defaultPort, + ) +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt new file mode 100644 index 00000000000..adaeff4cf8a --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt @@ -0,0 +1,50 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.imds + +import aws.sdk.kotlin.runtime.AwsSdkSetting +import aws.sdk.kotlin.runtime.endpoint.Endpoint +import aws.sdk.kotlin.runtime.endpoint.EndpointResolver +import aws.sdk.kotlin.runtime.resolve +import aws.smithy.kotlin.runtime.http.Url +import aws.smithy.kotlin.runtime.util.Platform + +internal class ImdsEndpointResolver( + private val endpointModeOverride: EndpointMode? = null, + private val endpointOverride: Endpoint? = null +) : EndpointResolver { + // cached endpoint + private var resolvedEndpoint: Endpoint? = null + + override suspend fun resolve(service: String, region: String): Endpoint = resolvedEndpoint ?: doResolveEndpoint() + + private suspend fun doResolveEndpoint(): Endpoint { + val resolved = endpointOverride ?: resolveEndpointFromConfig() + return resolved.also { resolvedEndpoint = it } + } + + private suspend fun resolveEndpointFromConfig(): Endpoint { + // explicit endpoint configured + val endpoint = loadEndpointFromEnv() ?: loadEndpointFromProfile() + if (endpoint != null) return endpoint + + // endpoint default from mode + val endpointMode = endpointModeOverride ?: loadEndpointModeFromEnv() ?: loadEndpointModeFromProfile() ?: EndpointMode.IPv4 + return endpointMode.defaultEndpoint + } + + private suspend fun loadEndpointFromEnv(): Endpoint? { + val uri = AwsSdkSetting.AwsEc2MetadataServiceEndpoint.resolve(Platform) ?: return null + return Url.parse(uri).let { Endpoint(it.host, it.scheme.protocolName) } + } + + private suspend fun loadEndpointFromProfile(): Endpoint? = null + + private suspend fun loadEndpointModeFromEnv(): EndpointMode? = + AwsSdkSetting.AwsEc2MetadataServiceEndpointMode.resolve(Platform)?.let { EndpointMode.fromValue(it) } + + private suspend fun loadEndpointModeFromProfile(): EndpointMode? = null +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt new file mode 100644 index 00000000000..15b9b42c4b0 --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt @@ -0,0 +1,39 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.imds + +import aws.smithy.kotlin.runtime.time.Instant + +/** + * Tokens are cached to remove the need to reload the token between subsequent requests. To ensure + * a request never fails with a 401 (expired token), a buffer window exists during which the token + * is not expired but refreshed anyway to ensure the token doesn't expire during an in-flight operation. + */ +internal const val TOKEN_REFRESH_BUFFER_SECONDS = 120 + +internal const val X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS = "x-aws-ec2-metadata-token-ttl-seconds" +internal const val X_AWS_EC2_METADATA_TOKEN = "x-aws-ec2-metadata-token" + +internal data class Token(val value: ByteArray, val expires: Instant) { + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (other == null || this::class != other::class) return false + + other as Token + + if (!value.contentEquals(other.value)) return false + if (expires != other.expires) return false + + return true + } + + override fun hashCode(): Int { + var result = value.contentHashCode() + result = 31 * result + expires.hashCode() + return result + } +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt new file mode 100644 index 00000000000..3506466fbbb --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt @@ -0,0 +1,95 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.imds + +import aws.smithy.kotlin.runtime.client.SdkClientOption +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.operation.SdkHttpOperation +import aws.smithy.kotlin.runtime.http.operation.SdkHttpRequest +import aws.smithy.kotlin.runtime.http.operation.getLogger +import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder +import aws.smithy.kotlin.runtime.http.request.url +import aws.smithy.kotlin.runtime.http.response.complete +import aws.smithy.kotlin.runtime.time.Clock +import aws.smithy.kotlin.runtime.time.Instant +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +@OptIn(ExperimentalTime::class) +internal class TokenMiddleware(config: Config) : Feature { + private val ttl: Duration = config.ttl + private val httpClient = requireNotNull(config.httpClient) { "SdkHttpClient is required for token middleware to make requests" } + private var cachedToken: Token? = null + + public class Config { + var ttl: Duration = Duration.seconds(DEFAULT_TOKEN_TTL_SECONDS) + var httpClient: SdkHttpClient? = null + } + + public companion object Feature : + HttpClientFeatureFactory { + override val key: FeatureKey = FeatureKey("EC2Metadata_Token_Middleware") + override fun create(block: Config.() -> Unit): TokenMiddleware { + val config = Config().apply(block) + return TokenMiddleware(config) + } + } + + override fun install(operation: SdkHttpOperation) { + operation.execution.mutate.intercept { req, next -> + val clock = req.context.getOrNull(SdkClientOption.Clock) ?: Clock.System + val token = useCachedTokenOrClear(clock) ?: getToken(clock, req).also { cachedToken = it } + req.subject.headers.append(X_AWS_EC2_METADATA_TOKEN, token.value.decodeToString()) + next.call(req) + } + } + + // use the cached token if it's still valid or clear it if expired and return null + private fun useCachedTokenOrClear(clock: Clock): Token? { + val cached = cachedToken ?: return null + val now = clock.now() + val expired = now.epochSeconds >= cached.expires.epochSeconds - TOKEN_REFRESH_BUFFER_SECONDS + if (expired) { + cachedToken = null + } + return cachedToken + } + + private suspend fun getToken(clock: Clock, req: SdkHttpRequest): Token { + val logger = req.context.getLogger("TokenMiddleware") + logger.trace { "refreshing IMDS token" } + + val tokenReq = HttpRequestBuilder().apply { + method = HttpMethod.PUT + headers.append(X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS, ttl.inWholeSeconds.toString()) + req.subject.headers["User-Agent"]?.let { headers.append("User-Agent", it) } + url { + // take endpoint info from original request + scheme = req.subject.url.scheme + host = req.subject.url.host + port = req.subject.url.port + + path = "/latest/api/token" + } + } + + // TODO - retries with custom policy around 400 and 403 + val call = httpClient.call(tokenReq) + return try { + when (call.response.status) { + HttpStatusCode.OK -> { + val ttl = call.response.headers[X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS]?.toLong() ?: throw EC2MetadataError(200, "No TTL provided in IMDS response") + val token = call.response.body.readAll() ?: throw EC2MetadataError(200, "No token provided in IMDS response") + val expires = Instant.fromEpochSeconds(clock.now().epochSeconds + ttl, 0) + Token(token, expires) + } + else -> throw EC2MetadataError(call.response.status.value, "Failed to retrieve IMDS token") + } + } finally { + call.complete() + } + } +} diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt new file mode 100644 index 00000000000..51c169c7bf5 --- /dev/null +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -0,0 +1,250 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.imds + +import aws.sdk.kotlin.runtime.ConfigurationException +import aws.sdk.kotlin.runtime.endpoint.Endpoint +import aws.sdk.kotlin.runtime.testing.ManualClock +import aws.sdk.kotlin.runtime.testing.runSuspendTest +import aws.smithy.kotlin.runtime.http.* +import aws.smithy.kotlin.runtime.http.content.ByteArrayContent +import aws.smithy.kotlin.runtime.http.engine.HttpClientEngineBase +import aws.smithy.kotlin.runtime.http.engine.callContext +import aws.smithy.kotlin.runtime.http.request.HttpRequest +import aws.smithy.kotlin.runtime.http.request.url +import aws.smithy.kotlin.runtime.http.response.HttpCall +import aws.smithy.kotlin.runtime.http.response.HttpResponse +import aws.smithy.kotlin.runtime.time.Instant +import kotlin.test.* +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +@OptIn(ExperimentalTime::class) +class EC2MetadataTest { + + // TODO - move to shared test utils + data class ExpectedHttpRequest(val request: HttpRequest, val response: HttpResponse? = null) + data class ValidateRequest(val expected: HttpRequest, val actual: HttpRequest) { + fun assertRequest() = runSuspendTest { + assertEquals(expected.url.toString(), actual.url.toString(), "URL mismatch") + expected.headers.forEach { name, values -> + values.forEach { + assertTrue(actual.headers.contains(name, it), "Header $name missing value $it") + } + } + + val expectedBody = expected.body.readAll()?.decodeToString() + val actualBody = actual.body.readAll()?.decodeToString() + assertEquals(expectedBody, actualBody, "ValidateRequest HttpBody mismatch") + } + } + + class TestConnection(expected: List = emptyList()) : HttpClientEngineBase("TestConnection") { + private val expected = expected.toMutableList() + // expected is mutated in-flight, store original size + private val expectedCount = expected.size + private var captured = mutableListOf() + + override suspend fun roundTrip(request: HttpRequest): HttpCall { + val next = expected.removeFirstOrNull() ?: error("TestConnection has no remaining expected requests") + captured.add(ValidateRequest(next.request, request)) + + val response = next.response ?: HttpResponse(HttpStatusCode.OK, Headers.Empty, HttpBody.Empty) + val now = Instant.now() + return HttpCall(request, response, now, now, callContext()) + } + + fun requests(): List = captured + fun assertRequests() { + assertEquals(expectedCount, captured.size) + captured.forEach(ValidateRequest::assertRequest) + } + } + + private fun tokenRequest(host: String, ttl: Int): HttpRequest = HttpRequest { + val parsed = Url.parse(host) + url(parsed) + url.path = "/latest/api/token" + headers.append(X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS, ttl.toString()) + } + + private fun tokenResponse(ttl: Int, token: String): HttpResponse = HttpResponse( + HttpStatusCode.OK, + Headers { + append(X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS, ttl.toString()) + }, + ByteArrayContent(token.encodeToByteArray()) + ) + + private fun imdsRequest(url: String, token: String): HttpRequest = HttpRequest { + val parsed = Url.parse(url) + url(parsed) + headers.append(X_AWS_EC2_METADATA_TOKEN, token) + } + + private fun imdsResponse(body: String): HttpResponse = HttpResponse( + HttpStatusCode.OK, + Headers.Empty, + ByteArrayContent(body.encodeToByteArray()) + ) + + @Test + fun testInvalidEndpointOverrideFailsCreation() { + val connection = TestConnection() + assertFailsWith { + EC2Metadata { + engine = connection + endpoint = Endpoint("[foo::254]", protocol = "http") + } + } + } + + @Test + fun testTokensAreCached() = runSuspendTest { + val connection = TestConnection( + listOf( + ExpectedHttpRequest( + tokenRequest("http://169.254.169.254", DEFAULT_TOKEN_TTL_SECONDS), + tokenResponse(DEFAULT_TOKEN_TTL_SECONDS, "TOKEN_A") + ), + ExpectedHttpRequest( + imdsRequest("http://169.254.169.254/latest/metadata", "TOKEN_A"), + imdsResponse("output 1") + ), + ExpectedHttpRequest( + imdsRequest("http://169.254.169.254/latest/metadata", "TOKEN_A"), + imdsResponse("output 2") + ) + ) + ) + + val client = EC2Metadata { engine = connection } + val r1 = client.get("/latest/metadata") + assertEquals("output 1", r1) + + val r2 = client.get("/latest/metadata") + assertEquals("output 2", r2) + connection.assertRequests() + } + + @Test + fun testTokensCanExpire() = runSuspendTest { + val connection = TestConnection( + listOf( + ExpectedHttpRequest( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_A") + ), + ExpectedHttpRequest( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), + imdsResponse("output 1") + ), + ExpectedHttpRequest( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_B") + ), + ExpectedHttpRequest( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_B"), + imdsResponse("output 2") + ) + ) + ) + + val testClock = ManualClock() + + val client = EC2Metadata { + engine = connection + endpointMode = EndpointMode.IPv6 + clock = testClock + tokenTTL = Duration.seconds(600) + } + + val r1 = client.get("/latest/metadata") + assertEquals("output 1", r1) + testClock.advance(Duration.seconds(600)) + + val r2 = client.get("/latest/metadata") + assertEquals("output 2", r2) + connection.assertRequests() + } + + @Test + fun testTokenRefreshBuffer() = runSuspendTest { + // tokens are refreshed up to 120 seconds early to avoid using an expired token + val connection = TestConnection( + listOf( + ExpectedHttpRequest( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_A") + ), + // t = 0 + ExpectedHttpRequest( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), + imdsResponse("output 1") + ), + // t = 400 (no refresh) + ExpectedHttpRequest( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), + imdsResponse("output 2") + ), + // t = 550 (within buffer) + ExpectedHttpRequest( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_B") + ), + ExpectedHttpRequest( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_B"), + imdsResponse("output 3") + ) + ) + ) + + val testClock = ManualClock() + + val client = EC2Metadata { + engine = connection + endpointMode = EndpointMode.IPv6 + clock = testClock + tokenTTL = Duration.seconds(600) + } + + val r1 = client.get("/latest/metadata") + assertEquals("output 1", r1) + testClock.advance(Duration.seconds(400)) + + val r2 = client.get("/latest/metadata") + assertEquals("output 2", r2) + + testClock.advance(Duration.seconds(150)) + val r3 = client.get("/latest/metadata") + assertEquals("output 3", r3) + + connection.assertRequests() + } + + @Test + fun testRetryHttp500() { + fail("not implemented yet") + } + + @Test + fun testRetryTokenFailure() { + // 500 during token acquisition should be retried + fail("not implemented yet") + } + + @Test + fun testNoRetryHttp403() { + // 403 responses from IMDS during token acquisition MUST not be retried + fail("not implemented yet") + } + + @Test + fun testConfig() { + // need to mock various config scenarios + fail("not implemented yet") + } +} diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt index e1d4dfd0671..a1f1a8c29e9 100644 --- a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt +++ b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt @@ -76,6 +76,24 @@ public sealed class AwsSdkSetting( * The name of the default profile that should be loaded from config */ public object AwsProfile : AwsSdkSetting("AWS_PROFILE", "aws.profile", "default") + + /** + * Whether to load information such as credentials, regions from EC2 Metadata instance service. + */ + public object AwsEc2MetadataDisabled : AwsSdkSetting("AWS_EC2_METADATA_DISABLED", "aws.disableEc2Metadata", false) + + /** + * The EC2 instance metadata service endpoint. + * + * This allows a service running in EC2 to automatically load its credentials and region without needing to configure them + * directly. + */ + public object AwsEc2MetadataServiceEndpoint : AwsSdkSetting("AWS_EC2_METADATA_SERVICE_ENDPOINT", "aws.ec2MetadataServiceEndpoint") + + /** + * The endpoint mode to use when connecting to the EC2 metadata service endpoint + */ + public object AwsEc2MetadataServiceEndpointMode : AwsSdkSetting("AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE", "aws.ec2MetadataServiceEndpointMode") } /** diff --git a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt new file mode 100644 index 00000000000..ab2dd457b63 --- /dev/null +++ b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt @@ -0,0 +1,24 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.testing + +import aws.smithy.kotlin.runtime.time.Clock +import aws.smithy.kotlin.runtime.time.Instant +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +public class ManualClock(epoch: Instant = Instant.now()) : Clock { + private var now: Instant = epoch + + @OptIn(ExperimentalTime::class) + public fun advance(duration: Duration) { + now = duration.toComponents { seconds, nanoseconds -> + Instant.fromEpochSeconds(now.epochSeconds + seconds, now.nanosecondsOfSecond + nanoseconds) + } + } + + override fun now(): Instant = now +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 7f0307a6a06..da4d0cc6972 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -29,6 +29,7 @@ include(":codegen:smithy-aws-kotlin-codegen") include(":codegen:protocol-tests") include(":aws-runtime") include(":aws-runtime:aws-core") +include(":aws-runtime:aws-config") include(":aws-runtime:testing") include(":aws-runtime:regions") include(":aws-runtime:auth") From 7d0d16ab60ca7bbb86f75a72b178d4879da41155 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 28 Sep 2021 17:25:21 -0400 Subject: [PATCH 02/16] set unkonwn version --- .../src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt index 474cd218533..456e5d1399d 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt @@ -34,7 +34,7 @@ import kotlin.time.ExperimentalTime internal const val DEFAULT_TOKEN_TTL_SECONDS: Int = 21_600 internal const val DEFAULT_MAX_RETRIES: UInt = 3u -private const val SERVICE = "IMDS" +private const val SERVICE = "imds" /** * IMDSv2 Client @@ -85,7 +85,7 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { resolver = ImdsEndpointResolver(endpointModeOverride, endpointOverride) }, UserAgent.create { - metadata = AwsUserAgentMetadata.fromEnvironment(ApiMetadata(SERVICE, "")) + metadata = AwsUserAgentMetadata.fromEnvironment(ApiMetadata(SERVICE, "unknown")) }, TokenMiddleware.create { httpClient = this@EC2Metadata.httpClient From 2ec0efc40cbee158209ecf8838b2754d2fcdb922 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 29 Sep 2021 11:15:20 -0400 Subject: [PATCH 03/16] refactor to use test utils from smithy-kotlin --- .../runtime/config/imds/EC2MetadataTest.kt | 2 +- .../http-client-engine-crt/build.gradle.kts | 7 +- .../http/engine/crt/AsyncStressTest.kt | 1 + .../crt/BufferedReadChannelByteBufferTest.kt | 2 +- .../engine/crt/BufferedReadChannelTest.kt | 2 +- .../http/engine/crt/TestWithLocalServer.kt | 58 -------------- aws-runtime/testing/build.gradle.kts | 17 +--- .../sdk/kotlin/runtime/testing/ManualClock.kt | 24 ------ .../aws/sdk/kotlin/runtime/testing/runTest.kt | 4 +- .../runtime/testing/ManualDispatchTestBase.kt | 79 ------------------- .../runtime/testing/RandomInputStream.kt | 72 ----------------- .../kotlin/runtime/testing/RandomTempFile.kt | 75 ------------------ .../sdk/kotlin/runtime/testing/runTestJVM.kt | 12 --- 13 files changed, 9 insertions(+), 346 deletions(-) delete mode 100644 aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/TestWithLocalServer.kt delete mode 100644 aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt delete mode 100644 aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/ManualDispatchTestBase.kt delete mode 100644 aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt delete mode 100644 aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt delete mode 100644 aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/runTestJVM.kt diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt index 51c169c7bf5..127f0e83449 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -7,7 +7,6 @@ package aws.sdk.kotlin.runtime.config.imds import aws.sdk.kotlin.runtime.ConfigurationException import aws.sdk.kotlin.runtime.endpoint.Endpoint -import aws.sdk.kotlin.runtime.testing.ManualClock import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.content.ByteArrayContent @@ -18,6 +17,7 @@ import aws.smithy.kotlin.runtime.http.request.url import aws.smithy.kotlin.runtime.http.response.HttpCall import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.time.Instant +import aws.smithy.kotlin.runtime.time.ManualClock import kotlin.test.* import kotlin.time.Duration import kotlin.time.ExperimentalTime diff --git a/aws-runtime/http-client-engine-crt/build.gradle.kts b/aws-runtime/http-client-engine-crt/build.gradle.kts index 80fea13a63c..6514e787163 100644 --- a/aws-runtime/http-client-engine-crt/build.gradle.kts +++ b/aws-runtime/http-client-engine-crt/build.gradle.kts @@ -40,12 +40,7 @@ kotlin { commonTest { dependencies { implementation(project(":aws-runtime:testing")) - } - } - jvmTest { - dependencies { - val ktorServerVersion = "1.5.0" - implementation("io.ktor:ktor-server-cio:$ktorServerVersion") + implementation("aws.smithy.kotlin:http-test:$smithyKotlinVersion") } } } diff --git a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/AsyncStressTest.kt b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/AsyncStressTest.kt index ed8e48c0c07..71953aaa5d3 100644 --- a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/AsyncStressTest.kt +++ b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/AsyncStressTest.kt @@ -14,6 +14,7 @@ import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder import aws.smithy.kotlin.runtime.http.request.url import aws.smithy.kotlin.runtime.http.response.complete import aws.smithy.kotlin.runtime.http.sdkHttpClient +import aws.smithy.kotlin.runtime.httptest.TestWithLocalServer import io.ktor.application.* import io.ktor.response.* import io.ktor.routing.* diff --git a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelByteBufferTest.kt b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelByteBufferTest.kt index b68aad4f870..d2b13286bfd 100644 --- a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelByteBufferTest.kt +++ b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelByteBufferTest.kt @@ -5,7 +5,7 @@ package aws.sdk.kotlin.runtime.http.engine.crt -import aws.sdk.kotlin.runtime.testing.ManualDispatchTestBase +import aws.smithy.kotlin.runtime.testing.ManualDispatchTestBase import kotlinx.coroutines.CancellationException import kotlinx.coroutines.launch import kotlinx.coroutines.yield diff --git a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelTest.kt b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelTest.kt index 75aeb5eba24..07766e1de2c 100644 --- a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelTest.kt +++ b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/BufferedReadChannelTest.kt @@ -6,9 +6,9 @@ package aws.sdk.kotlin.runtime.http.engine.crt import aws.sdk.kotlin.crt.io.byteArrayBuffer -import aws.sdk.kotlin.runtime.testing.ManualDispatchTestBase import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.io.readByte +import aws.smithy.kotlin.runtime.testing.ManualDispatchTestBase import kotlinx.coroutines.* import kotlinx.coroutines.channels.ClosedReceiveChannelException import java.lang.RuntimeException diff --git a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/TestWithLocalServer.kt b/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/TestWithLocalServer.kt deleted file mode 100644 index 14819b48345..00000000000 --- a/aws-runtime/http-client-engine-crt/jvm/test/aws/sdk/kotlin/runtime/http/engine/crt/TestWithLocalServer.kt +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -package aws.sdk.kotlin.runtime.http.engine.crt - -import aws.smithy.kotlin.runtime.logging.Logger -import io.ktor.server.engine.* -import java.net.* -import java.util.concurrent.* -import kotlin.test.AfterTest -import kotlin.test.BeforeTest - -public abstract class TestWithLocalServer { - protected val serverPort: Int = ServerSocket(0).use { it.localPort } - protected val testHost: String = "localhost" - - public abstract val server: ApplicationEngine - - private val logger = Logger.getLogger() - - @BeforeTest - public fun startServer() { - var attempt = 0 - - do { - attempt++ - try { - server.start() - logger.info { "test server listening on: $testHost:$serverPort" } - break - } catch (cause: Throwable) { - if (attempt >= 10) throw cause - Thread.sleep(250L * attempt) - } - } while (true) - - ensureServerRunning() - } - - @AfterTest - public fun stopServer() { - server.stop(0, 0, TimeUnit.SECONDS) - logger.info { "test server stopped" } - } - - private fun ensureServerRunning() { - do { - try { - Socket("localhost", serverPort).close() - break - } catch (_: Throwable) { - Thread.sleep(100) - } - } while (true) - } -} diff --git a/aws-runtime/testing/build.gradle.kts b/aws-runtime/testing/build.gradle.kts index 5937474f3ad..415e3d4ea41 100644 --- a/aws-runtime/testing/build.gradle.kts +++ b/aws-runtime/testing/build.gradle.kts @@ -5,28 +5,13 @@ description = "Internal test utilities" -val kotlinVersion: String by project -val coroutinesVersion: String by project val smithyKotlinVersion: String by project kotlin { sourceSets { commonMain { dependencies { - implementation("org.jetbrains.kotlin:kotlin-test-common:$kotlinVersion") - implementation("org.jetbrains.kotlin:kotlin-test-annotations-common:$kotlinVersion") - } - } - jvmMain{ - dependencies { - implementation("org.jetbrains.kotlin:kotlin-test:$kotlinVersion") - api("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion") - } - } - metadata { - dependencies { - commonMainApi(project(":aws-runtime:aws-core")) - commonMainApi("aws.smithy.kotlin:utils:$smithyKotlinVersion") + api("aws.smithy.kotlin:testing:$smithyKotlinVersion") } } } diff --git a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt deleted file mode 100644 index ab2dd457b63..00000000000 --- a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/ManualClock.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -package aws.sdk.kotlin.runtime.testing - -import aws.smithy.kotlin.runtime.time.Clock -import aws.smithy.kotlin.runtime.time.Instant -import kotlin.time.Duration -import kotlin.time.ExperimentalTime - -public class ManualClock(epoch: Instant = Instant.now()) : Clock { - private var now: Instant = epoch - - @OptIn(ExperimentalTime::class) - public fun advance(duration: Duration) { - now = duration.toComponents { seconds, nanoseconds -> - Instant.fromEpochSeconds(now.epochSeconds + seconds, now.nanosecondsOfSecond + nanoseconds) - } - } - - override fun now(): Instant = now -} diff --git a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/runTest.kt b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/runTest.kt index 66d714c9d15..3ce2eecd6c4 100644 --- a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/runTest.kt +++ b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/runTest.kt @@ -7,8 +7,10 @@ package aws.sdk.kotlin.runtime.testing import kotlinx.coroutines.CoroutineScope import kotlin.coroutines.CoroutineContext import kotlin.coroutines.EmptyCoroutineContext +import aws.smithy.kotlin.runtime.testing.runSuspendTest as runTest +// TODO - migrate unit tests to just use runSuspendTest from smithy-kotlin, for now re-export it to limit # changes /** * MPP compatible runBlocking to run suspend tests in common modules */ -public expect fun runSuspendTest(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> T): T +public fun runSuspendTest(context: CoroutineContext = EmptyCoroutineContext, block: suspend CoroutineScope.() -> T): T = runTest(context, block) diff --git a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/ManualDispatchTestBase.kt b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/ManualDispatchTestBase.kt deleted file mode 100644 index b2da466cf08..00000000000 --- a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/ManualDispatchTestBase.kt +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -package aws.sdk.kotlin.runtime.testing - -import kotlinx.coroutines.ExperimentalCoroutinesApi -import kotlinx.coroutines.test.TestCoroutineScope -import kotlinx.coroutines.test.runBlockingTest -import kotlin.test.assertEquals -import kotlin.test.assertNotEquals - -// modeled after https://github.com/ktorio/ktor/blob/78e36790cdbb30313dfbd23b174bffe805d26dca/ktor-io/common/test/io/ktor/utils/io/ByteChannelTestBase.kt -// but implemented using [kotlinx-coroutines-test](https://kotlin.github.io/kotlinx.coroutines/kotlinx-coroutines-test) -// rather than rolling our own dummy coroutines dispatcher -/** - * Test suspend functions with precise control over coordination. This allows testing that suspension happens at the - * expected point in time using the provided [expect] function, e.g.: - * - * ``` - * class TestFoo : ManualDispatchTestBase() { - * @Test - * fun testFoo() = runTest { - * expect(1) - * launch { - * expect(3) - * someFunctionThatShouldSuspend() - * expect(5) - * } - * - * expect(2) - * yield() - * expect(4) - * unblockSuspendedFunction() - * yield() - * finish(6) - * } - * } - * ``` - * - * Explicitly yielding or hitting a natural suspension point will run the next continuation queued - */ -@OptIn(ExperimentalCoroutinesApi::class) -public abstract class ManualDispatchTestBase { - private var current = 0 - - /** - * Execute a test with the provided (test) coroutine scope. Any calls to `launch` or `async` - * will not be executed immediately and instead be scheduled for dispatch. Explicit calls to `yield()` - * will advance the dispatcher. - */ - protected fun runTest(block: suspend TestCoroutineScope.() -> Unit) { - runBlockingTest { - // ensure launch/async calls are coordinated with yield() points - pauseDispatcher() - block() - } - } - - /** - * Assert the current execution point and increment the count - */ - protected fun expect(n: Int) { - val next = current + 1 - assertNotEquals(0, next, "Already finished") - assertEquals(n, next, "Invalid test state") - current = next - } - - /** - * Assert the current execution point and mark the test finished. Any further - * calls to [expect] will fail. - */ - protected fun finish(n: Int) { - expect(n) - current = -1 - } -} diff --git a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt deleted file mode 100644 index 84d72ee8aac..00000000000 --- a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomInputStream.kt +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ -package aws.sdk.kotlin.runtime.testing - -import java.io.IOException -import java.io.InputStream -import kotlin.random.Random - -/** - * Test utility InputStream implementation that generates random ASCII data when - * read, up to the size specified when constructed. - */ -public class RandomInputStream constructor( - /** The requested amount of data contained in this random stream. */ - private val lengthInBytes: Long, - - /** Flag controlling whether binary or character data is used. */ - private val binaryData: Boolean = false -) : InputStream() { - - /** The number of bytes of data remaining in this random stream. */ - protected var remainingBytes: Long = lengthInBytes - - public val bytesRead: Long - get() = lengthInBytes - remainingBytes - - @Throws(IOException::class) - override fun read(b: ByteArray, off: Int, len: Int): Int { - // Signal that we're out of data if we've hit our limit - if (remainingBytes <= 0) { - return -1 - } - var bytesToRead = len - if (bytesToRead > remainingBytes) { - bytesToRead = remainingBytes.toInt() - } - remainingBytes -= bytesToRead.toLong() - if (binaryData) { - val endExclusive = off + bytesToRead - Random.nextBytes(b, off, endExclusive) - } else { - for (i in 0 until bytesToRead) { - b[off + i] = Random.nextInt(MIN_CHAR_CODE, MAX_CHAR_CODE + 1).toByte() - } - } - return bytesToRead - } - - @Throws(IOException::class) - override fun read(): Int { - // Signal that we're out of data if we've hit our limit - if (remainingBytes <= 0) { - return -1 - } - remainingBytes-- - return if (binaryData) { - Random.nextInt() - } else { - Random.nextInt(MIN_CHAR_CODE, MAX_CHAR_CODE + 1) - } - } - - public companion object { - /** The minimum ASCII code contained in the data in this stream. */ - private const val MIN_CHAR_CODE = 32 - - /** The maximum ASCII code contained in the data in this stream. */ - private const val MAX_CHAR_CODE = 125 - } -} diff --git a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt deleted file mode 100644 index f554c7245cc..00000000000 --- a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/RandomTempFile.kt +++ /dev/null @@ -1,75 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ -package aws.sdk.kotlin.runtime.testing - -import java.io.BufferedOutputStream -import java.io.File -import java.io.FileOutputStream -import java.io.IOException -import java.util.* - -/** - * Extension of File that creates a temporary file with a specified name in - * Java's temporary directory, as declared in the JRE's system properties. The - * file is immediately filled with a specified amount of random ASCII data. - * - * @see RandomInputStream - */ -public class RandomTempFile : File { - /** Flag controlling whether binary or character data is used. */ - private val binaryData: Boolean - - /** - * Creates, and fills, a temp file with a randomly generated name and specified size of random ASCII data. - * - * @param sizeInBytes The amount of random ASCII data, in bytes, for the new temp - * file. - * @throws IOException If any problems were encountered creating the new temp file. - */ - public constructor(sizeInBytes: Long) : this(UUID.randomUUID().toString(), sizeInBytes, false) - - /** - * Creates, and fills, a temp file with the specified name and specified - * size of random data. - * - * @param filename The name for the new temporary file, within the Java temp - * directory as declared in the JRE's system properties. - * @param sizeInBytes The amount of random ASCII data, in bytes, for the new temp - * file. - * @param binaryData Whether to fill the file with binary or character data. - * - * @throws IOException - * If any problems were encountered creating the new temp file. - */ - public constructor(filename: String, sizeInBytes: Long, binaryData: Boolean = false) : super( - TEMP_DIR + separator + System.currentTimeMillis().toString() + "-" + filename - ) { - this.binaryData = binaryData - createFile(sizeInBytes) - } - - @Throws(IOException::class) - public fun createFile(sizeInBytes: Long) { - deleteOnExit() - FileOutputStream(this).use { outputStream -> - BufferedOutputStream(outputStream).use { bufferedOutputStream -> - RandomInputStream(sizeInBytes, binaryData).use { inputStream -> - inputStream.copyTo(bufferedOutputStream) - } - } - } - } - - override fun delete(): Boolean { - if (!super.delete()) { - throw RuntimeException("Could not delete: $absolutePath") - } - return true - } - - public companion object { - private val TEMP_DIR: String = System.getProperty("java.io.tmpdir") - } -} diff --git a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/runTestJVM.kt b/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/runTestJVM.kt deleted file mode 100644 index 1b263cb8c39..00000000000 --- a/aws-runtime/testing/jvm/src/aws/sdk/kotlin/runtime/testing/runTestJVM.kt +++ /dev/null @@ -1,12 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ -package aws.sdk.kotlin.runtime.testing - -import kotlinx.coroutines.CoroutineScope -import kotlinx.coroutines.runBlocking -import kotlin.coroutines.CoroutineContext - -public actual fun runSuspendTest(context: CoroutineContext, block: suspend CoroutineScope.() -> T): T = - runBlocking { block(this) } From 8dbe1932e50ba358adc71458f07721ce6842898d Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 29 Sep 2021 12:14:40 -0400 Subject: [PATCH 04/16] migrate test connection to smithy-kotlin for re-use --- aws-runtime/aws-config/build.gradle.kts | 1 + .../runtime/config/imds/EC2MetadataTest.kt | 159 ++++++------------ 2 files changed, 57 insertions(+), 103 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 96905e9fc52..cd3fbdc87c1 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -22,6 +22,7 @@ kotlin { commonTest { dependencies { implementation(project(":aws-runtime:testing")) + implementation("aws.smithy.kotlin:http-test:$smithyKotlinVersion") } } } diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt index 127f0e83449..7799c804ad3 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -10,13 +10,11 @@ import aws.sdk.kotlin.runtime.endpoint.Endpoint import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.content.ByteArrayContent -import aws.smithy.kotlin.runtime.http.engine.HttpClientEngineBase -import aws.smithy.kotlin.runtime.http.engine.callContext import aws.smithy.kotlin.runtime.http.request.HttpRequest import aws.smithy.kotlin.runtime.http.request.url -import aws.smithy.kotlin.runtime.http.response.HttpCall import aws.smithy.kotlin.runtime.http.response.HttpResponse -import aws.smithy.kotlin.runtime.time.Instant +import aws.smithy.kotlin.runtime.httptest.TestConnection +import aws.smithy.kotlin.runtime.httptest.buildTestConnection import aws.smithy.kotlin.runtime.time.ManualClock import kotlin.test.* import kotlin.time.Duration @@ -25,45 +23,6 @@ import kotlin.time.ExperimentalTime @OptIn(ExperimentalTime::class) class EC2MetadataTest { - // TODO - move to shared test utils - data class ExpectedHttpRequest(val request: HttpRequest, val response: HttpResponse? = null) - data class ValidateRequest(val expected: HttpRequest, val actual: HttpRequest) { - fun assertRequest() = runSuspendTest { - assertEquals(expected.url.toString(), actual.url.toString(), "URL mismatch") - expected.headers.forEach { name, values -> - values.forEach { - assertTrue(actual.headers.contains(name, it), "Header $name missing value $it") - } - } - - val expectedBody = expected.body.readAll()?.decodeToString() - val actualBody = actual.body.readAll()?.decodeToString() - assertEquals(expectedBody, actualBody, "ValidateRequest HttpBody mismatch") - } - } - - class TestConnection(expected: List = emptyList()) : HttpClientEngineBase("TestConnection") { - private val expected = expected.toMutableList() - // expected is mutated in-flight, store original size - private val expectedCount = expected.size - private var captured = mutableListOf() - - override suspend fun roundTrip(request: HttpRequest): HttpCall { - val next = expected.removeFirstOrNull() ?: error("TestConnection has no remaining expected requests") - captured.add(ValidateRequest(next.request, request)) - - val response = next.response ?: HttpResponse(HttpStatusCode.OK, Headers.Empty, HttpBody.Empty) - val now = Instant.now() - return HttpCall(request, response, now, now, callContext()) - } - - fun requests(): List = captured - fun assertRequests() { - assertEquals(expectedCount, captured.size) - captured.forEach(ValidateRequest::assertRequest) - } - } - private fun tokenRequest(host: String, ttl: Int): HttpRequest = HttpRequest { val parsed = Url.parse(host) url(parsed) @@ -104,22 +63,20 @@ class EC2MetadataTest { @Test fun testTokensAreCached() = runSuspendTest { - val connection = TestConnection( - listOf( - ExpectedHttpRequest( - tokenRequest("http://169.254.169.254", DEFAULT_TOKEN_TTL_SECONDS), - tokenResponse(DEFAULT_TOKEN_TTL_SECONDS, "TOKEN_A") - ), - ExpectedHttpRequest( - imdsRequest("http://169.254.169.254/latest/metadata", "TOKEN_A"), - imdsResponse("output 1") - ), - ExpectedHttpRequest( - imdsRequest("http://169.254.169.254/latest/metadata", "TOKEN_A"), - imdsResponse("output 2") - ) + val connection = buildTestConnection { + expect( + tokenRequest("http://169.254.169.254", DEFAULT_TOKEN_TTL_SECONDS), + tokenResponse(DEFAULT_TOKEN_TTL_SECONDS, "TOKEN_A") ) - ) + expect( + imdsRequest("http://169.254.169.254/latest/metadata", "TOKEN_A"), + imdsResponse("output 1") + ) + expect( + imdsRequest("http://169.254.169.254/latest/metadata", "TOKEN_A"), + imdsResponse("output 2") + ) + } val client = EC2Metadata { engine = connection } val r1 = client.get("/latest/metadata") @@ -132,26 +89,24 @@ class EC2MetadataTest { @Test fun testTokensCanExpire() = runSuspendTest { - val connection = TestConnection( - listOf( - ExpectedHttpRequest( - tokenRequest("http://[fd00:ec2::254]", 600), - tokenResponse(600, "TOKEN_A") - ), - ExpectedHttpRequest( - imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), - imdsResponse("output 1") - ), - ExpectedHttpRequest( - tokenRequest("http://[fd00:ec2::254]", 600), - tokenResponse(600, "TOKEN_B") - ), - ExpectedHttpRequest( - imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_B"), - imdsResponse("output 2") - ) + val connection = buildTestConnection { + expect( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_A") + ) + expect( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), + imdsResponse("output 1") + ) + expect( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_B") + ) + expect( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_B"), + imdsResponse("output 2") ) - ) + } val testClock = ManualClock() @@ -174,33 +129,31 @@ class EC2MetadataTest { @Test fun testTokenRefreshBuffer() = runSuspendTest { // tokens are refreshed up to 120 seconds early to avoid using an expired token - val connection = TestConnection( - listOf( - ExpectedHttpRequest( - tokenRequest("http://[fd00:ec2::254]", 600), - tokenResponse(600, "TOKEN_A") - ), - // t = 0 - ExpectedHttpRequest( - imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), - imdsResponse("output 1") - ), - // t = 400 (no refresh) - ExpectedHttpRequest( - imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), - imdsResponse("output 2") - ), - // t = 550 (within buffer) - ExpectedHttpRequest( - tokenRequest("http://[fd00:ec2::254]", 600), - tokenResponse(600, "TOKEN_B") - ), - ExpectedHttpRequest( - imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_B"), - imdsResponse("output 3") - ) + val connection = buildTestConnection { + expect( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_A") + ) + // t = 0 + expect( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), + imdsResponse("output 1") + ) + // t = 400 (no refresh) + expect( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_A"), + imdsResponse("output 2") ) - ) + // t = 550 (within buffer) + expect( + tokenRequest("http://[fd00:ec2::254]", 600), + tokenResponse(600, "TOKEN_B") + ) + expect( + imdsRequest("http://[fd00:ec2::254]/latest/metadata", "TOKEN_B"), + imdsResponse("output 3") + ) + } val testClock = ManualClock() From a816f8773bd05e5b82b8bea1f628cd6232f88667 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 29 Sep 2021 15:06:18 -0400 Subject: [PATCH 05/16] refactor: move profile configuration into aws-config --- aws-runtime/aws-config/build.gradle.kts | 4 ++++ .../src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt | 0 .../src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt | 0 .../common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt | 4 ++-- .../src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt | 0 .../test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt | 0 .../test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt | 0 .../aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt | 0 .../test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt | 0 aws-runtime/aws-core/build.gradle.kts | 4 +--- 10 files changed, 7 insertions(+), 5 deletions(-) rename aws-runtime/{aws-core => aws-config}/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt (100%) rename aws-runtime/{aws-core => aws-config}/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt (100%) rename aws-runtime/{aws-core => aws-config}/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt (93%) rename aws-runtime/{aws-core => aws-config}/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt (100%) rename aws-runtime/{aws-core => aws-config}/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt (100%) rename aws-runtime/{aws-core => aws-config}/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt (100%) rename aws-runtime/{aws-core => aws-config}/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt (100%) rename aws-runtime/{aws-core => aws-config}/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt (100%) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index cd3fbdc87c1..17bfac98c2f 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -23,6 +23,10 @@ kotlin { dependencies { implementation(project(":aws-runtime:testing")) implementation("aws.smithy.kotlin:http-test:$smithyKotlinVersion") + val kotlinxSerializationVersion: String by project + val mockkVersion: String by project + implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime:$kotlinxSerializationVersion") + implementation("io.mockk:mockk:$mockkVersion") } } } diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt similarity index 100% rename from aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt similarity index 100% rename from aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt similarity index 93% rename from aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt index 6e4f0e7aa50..b059a32a010 100644 --- a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt @@ -1,12 +1,12 @@ package aws.sdk.kotlin.runtime.config /** - * The properties and name of the active AWS configuration profile. + * The properties and name of an AWS configuration profile. * * @property name name of profile * @property properties key/value pairs of properties specified by the active profile, accessible via [Map] */ -data class AwsProfile( +public data class AwsProfile( val name: String, private val properties: Map ) : Map by properties diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt similarity index 100% rename from aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt similarity index 100% rename from aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt similarity index 100% rename from aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt similarity index 100% rename from aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt similarity index 100% rename from aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt diff --git a/aws-runtime/aws-core/build.gradle.kts b/aws-runtime/aws-core/build.gradle.kts index 9533f1a0e40..0be647c4c58 100644 --- a/aws-runtime/aws-core/build.gradle.kts +++ b/aws-runtime/aws-core/build.gradle.kts @@ -20,11 +20,9 @@ kotlin { } commonTest { dependencies { - val kotlinxSerializationVersion: String by project + implementation(project(":aws-runtime:testing")) val mockkVersion: String by project - implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime:$kotlinxSerializationVersion") implementation("io.mockk:mockk:$mockkVersion") - implementation(project(":aws-runtime:testing")) } } } From a0647f12abf31320785e9f6ad83db93bc79adf66 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 30 Sep 2021 10:01:26 -0400 Subject: [PATCH 06/16] move file missed in previous refactor --- .../runtime/config}/AWSConfigLoaderFilesystemTest.kt | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) rename aws-runtime/{aws-core/jvm/test => aws-config/jvm/test/aws/sdk/kotlin/runtime/config}/AWSConfigLoaderFilesystemTest.kt (96%) diff --git a/aws-runtime/aws-core/jvm/test/AWSConfigLoaderFilesystemTest.kt b/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/AWSConfigLoaderFilesystemTest.kt similarity index 96% rename from aws-runtime/aws-core/jvm/test/AWSConfigLoaderFilesystemTest.kt rename to aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/AWSConfigLoaderFilesystemTest.kt index 237edb83ab8..aeba3b98529 100644 --- a/aws-runtime/aws-core/jvm/test/AWSConfigLoaderFilesystemTest.kt +++ b/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/AWSConfigLoaderFilesystemTest.kt @@ -1,6 +1,10 @@ -package aws.sdk.kotlin.runtime.auth +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config -import aws.sdk.kotlin.runtime.config.loadActiveAwsProfile import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.util.OperatingSystem import aws.smithy.kotlin.runtime.util.Platform From 61b3c01b566735f1a8f3467ad5b4fe8f22939eca Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 30 Sep 2021 10:56:20 -0400 Subject: [PATCH 07/16] fix type conversion for AwsSdkSetting.resolve --- .../src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt | 16 ++++++++++++++-- .../aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt | 14 +++++++++++--- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt index a1f1a8c29e9..3d6ec607a7c 100644 --- a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt +++ b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt @@ -104,5 +104,17 @@ public sealed class AwsSdkSetting( * @return the value of the [AwsSdkSetting] or null if undefined. */ @InternalSdkApi -public inline fun AwsSdkSetting.resolve(platform: Platform): T? = - (platform.getProperty(jvmProperty) ?: platform.getenv(environmentVariable) ?: defaultValue) as T? +public inline fun AwsSdkSetting.resolve(platform: Platform): T? { + val strValue = platform.getProperty(jvmProperty) ?: platform.getenv(environmentVariable) + if (strValue != null) { + val typed: Any = when (T::class) { + String::class -> strValue + Int::class -> strValue.toInt() + Long::class -> strValue.toLong() + Boolean::class -> strValue.toBoolean() + else -> error("conversion to ${T::class} not implemented for AwsSdkSetting") + } + return typed as? T + } + return defaultValue +} diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt b/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt index ec6df69affb..4ab7ca67fc3 100644 --- a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt +++ b/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt @@ -11,7 +11,7 @@ import kotlin.test.assertNull class AwsSdkSettingTest { @Test - fun itLoadsJVMSettingFirst() { + fun itResolvesJVMSettingFirst() { val testPlatform = mockPlatform(mapOf("AWS_PROFILE" to "env"), mapOf("aws.profile" to "jvm")) val actual = AwsSdkSetting.AwsProfile.resolve(testPlatform) @@ -20,7 +20,7 @@ class AwsSdkSettingTest { } @Test - fun itLoadsEnvSettingSecond() { + fun itResolvesEnvSettingSecond() { val testPlatform = mockPlatform(mapOf("AWS_PROFILE" to "env"), mapOf()) val actual = AwsSdkSetting.AwsProfile.resolve(testPlatform) @@ -29,7 +29,7 @@ class AwsSdkSettingTest { } @Test - fun itLoadsDefaultSettingThird() { + fun itResolvesDefaultSettingThird() { val testPlatform = mockPlatform(mapOf(), mapOf()) val actual = AwsSdkSetting.AwsProfile.resolve(testPlatform) @@ -46,6 +46,14 @@ class AwsSdkSettingTest { assertNull(actual) } + @Test + fun itResolvesType() { + val testPlatform = mockPlatform(mapOf("AWS_EC2_METADATA_DISABLED" to "true"), mapOf()) + + val actual = AwsSdkSetting.AwsEc2MetadataDisabled.resolve(testPlatform) + assertEquals(true, actual) + } + private fun mockPlatform(env: Map, jvmProps: Map): Platform { val testPlatform = mockk() val envKeyParam = slot() From 59d033aa3b47ead6c909333c3df79f0464275da7 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 30 Sep 2021 11:10:42 -0400 Subject: [PATCH 08/16] remove mockk from common; use new interface for platform --- aws-runtime/aws-core/build.gradle.kts | 2 -- .../aws/sdk/kotlin/runtime/AwsSdkSetting.kt | 6 +++--- .../sdk/kotlin/runtime/AwsSdkSettingTest.kt | 20 +++++-------------- 3 files changed, 8 insertions(+), 20 deletions(-) diff --git a/aws-runtime/aws-core/build.gradle.kts b/aws-runtime/aws-core/build.gradle.kts index 0be647c4c58..b687ffdf559 100644 --- a/aws-runtime/aws-core/build.gradle.kts +++ b/aws-runtime/aws-core/build.gradle.kts @@ -21,8 +21,6 @@ kotlin { commonTest { dependencies { implementation(project(":aws-runtime:testing")) - val mockkVersion: String by project - implementation("io.mockk:mockk:$mockkVersion") } } } diff --git a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt index 3d6ec607a7c..cd51b7ce92d 100644 --- a/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt +++ b/aws-runtime/aws-core/common/src/aws/sdk/kotlin/runtime/AwsSdkSetting.kt @@ -5,7 +5,7 @@ package aws.sdk.kotlin.runtime -import aws.smithy.kotlin.runtime.util.Platform +import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider // NOTE: The JVM property names MUST match the ones defined in the Java SDK for any setting added. // see: https://github.com/aws/aws-sdk-java-v2/blob/master/core/sdk-core/src/main/java/software/amazon/awssdk/core/SdkSystemSetting.java @@ -100,11 +100,11 @@ public sealed class AwsSdkSetting( * Read the [AwsSdkSetting] by first checking JVM property, environment variable, and default value. * Property sources not available on a given platform will be ignored. * - * @param platform A singleton that provides platform-specific settings. Exposed as a parameter for testing. + * @param platform A provider of platform-specific settings * @return the value of the [AwsSdkSetting] or null if undefined. */ @InternalSdkApi -public inline fun AwsSdkSetting.resolve(platform: Platform): T? { +public inline fun AwsSdkSetting.resolve(platform: PlatformEnvironProvider): T? { val strValue = platform.getProperty(jvmProperty) ?: platform.getenv(environmentVariable) if (strValue != null) { val typed: Any = when (T::class) { diff --git a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt b/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt index 4ab7ca67fc3..fb926e9b355 100644 --- a/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt +++ b/aws-runtime/aws-core/common/test/aws/sdk/kotlin/runtime/AwsSdkSettingTest.kt @@ -1,9 +1,6 @@ package aws.sdk.kotlin.runtime -import aws.smithy.kotlin.runtime.util.Platform -import io.mockk.every -import io.mockk.mockk -import io.mockk.slot +import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertNull @@ -54,17 +51,10 @@ class AwsSdkSettingTest { assertEquals(true, actual) } - private fun mockPlatform(env: Map, jvmProps: Map): Platform { - val testPlatform = mockk() - val envKeyParam = slot() - val propKeyParam = slot() - - every { testPlatform.getenv(capture(envKeyParam)) } answers { - env[envKeyParam.captured] - } - every { testPlatform.getProperty(capture(propKeyParam)) } answers { - jvmProps[propKeyParam.captured] + private fun mockPlatform(env: Map, jvmProps: Map): PlatformEnvironProvider { + return object : PlatformEnvironProvider { + override fun getenv(key: String): String? = env[key] + override fun getProperty(key: String): String? = jvmProps[key] } - return testPlatform } } From a34e6e213405930841813a508e4f0ad8cbc742e5 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 30 Sep 2021 11:36:14 -0400 Subject: [PATCH 09/16] replace Platform singleton with PlatformProvider interface --- aws-runtime/aws-config/build.gradle.kts | 2 +- .../kotlin/runtime/config/AwsConfigLoader.kt | 12 +++++------ .../kotlin/runtime/config/AwsConfigParser.kt | 4 ++-- .../runtime/config/AwsConfigLoaderTest.kt | 21 ++++++++----------- .../runtime/config/AwsConfigParserTest.kt | 19 +++++++---------- gradle.properties | 2 +- 6 files changed, 26 insertions(+), 34 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 17bfac98c2f..21ee6b7b2ab 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -25,7 +25,7 @@ kotlin { implementation("aws.smithy.kotlin:http-test:$smithyKotlinVersion") val kotlinxSerializationVersion: String by project val mockkVersion: String by project - implementation("org.jetbrains.kotlinx:kotlinx-serialization-runtime:$kotlinxSerializationVersion") + implementation("org.jetbrains.kotlinx:kotlinx-serialization-json:$kotlinxSerializationVersion") implementation("io.mockk:mockk:$mockkVersion") } } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt index eca95afe234..d97e355cf46 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt @@ -4,7 +4,7 @@ import aws.sdk.kotlin.runtime.AwsSdkSetting import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.resolve import aws.smithy.kotlin.runtime.util.OsFamily -import aws.smithy.kotlin.runtime.util.Platform +import aws.smithy.kotlin.runtime.util.PlatformProvider /** * Load the properties of the specified or default AWS configuration profile. This @@ -18,7 +18,7 @@ import aws.smithy.kotlin.runtime.util.Platform * @return an [AwsProfile] regardless if local configuration files are available */ @InternalSdkApi -public suspend fun loadActiveAwsProfile(platform: Platform): AwsProfile { +public suspend fun loadActiveAwsProfile(platform: PlatformProvider): AwsProfile { // Determine active profile and location of configuration files val source = resolveConfigSource(platform) @@ -37,7 +37,7 @@ public suspend fun loadActiveAwsProfile(platform: Platform): AwsProfile { * * @return A map of all profiles, which each are a map of key/value pairs. */ -private suspend fun loadAwsProfiles(platform: Platform, source: AwsConfigurationSource): Map> { +private suspend fun loadAwsProfiles(platform: PlatformProvider, source: AwsConfigurationSource): Map> { // merged AWS configuration based on optional configuration and credential file contents return mergeProfiles( @@ -61,7 +61,7 @@ internal data class AwsConfigurationSource(val profile: String, val configPath: /** * Determine the source of AWS configuration */ -internal fun resolveConfigSource(platform: Platform) = +internal fun resolveConfigSource(platform: PlatformProvider) = AwsConfigurationSource( // If the user does not specify the profile to be used, the default profile must be used by the SDK. // The default profile must be overridable using the AWS_PROFILE environment variable. @@ -81,7 +81,7 @@ internal fun resolveConfigSource(platform: Platform) = * 3. (Windows Platforms) The HOMEDRIVE environment variable prepended to the HOMEPATH environment variable (ie. $HOMEDRIVE$HOMEPATH). * 4. (Optional) A language-specific home path resolution function or variable. */ -internal fun normalizePath(path: String, platform: Platform): String { +internal fun normalizePath(path: String, platform: PlatformProvider): String { if (!path.trim().startsWith('~')) return path val home = resolveHomeDir(platform) ?: error("Unable to determine user home directory") @@ -99,7 +99,7 @@ internal fun normalizePath(path: String, platform: Platform): String { * @param * @return the absolute path of the home directory from which the SDK is running, or null if unspecified by environment. */ -private fun resolveHomeDir(platform: Platform): String? = +private fun resolveHomeDir(platform: PlatformProvider): String? = with(platform) { when (osInfo().family) { OsFamily.Unknown, diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt index 4ddd003fe2b..321f083c3d6 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt @@ -6,7 +6,7 @@ import aws.sdk.kotlin.runtime.InternalSdkApi import aws.sdk.kotlin.runtime.resolve import aws.smithy.kotlin.runtime.logging.Logger import aws.smithy.kotlin.runtime.logging.warn -import aws.smithy.kotlin.runtime.util.Platform +import aws.smithy.kotlin.runtime.util.PlatformProvider // Literal characters used in parsing AWS SDK configuration files internal object Literals { @@ -89,7 +89,7 @@ internal enum class FileType( * Determine the absolute path of the configuration file based on environment and policy * @return the absolute path of the configuration file. This does not imply the file exists or is otherwise valid */ - fun path(platform: Platform): String = + fun path(platform: PlatformProvider): String = setting.resolve(platform)?.trim() ?: pathSegments.joinToString(separator = platform.filePathSeparator) /** diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt index a2468b8fc4f..03f8f546f16 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt @@ -8,10 +8,7 @@ import io.mockk.coEvery import io.mockk.every import io.mockk.mockk import io.mockk.slot -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonLiteral -import kotlinx.serialization.json.JsonObject +import kotlinx.serialization.json.* import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertTrue @@ -20,7 +17,7 @@ class AwsConfigLoaderTest { @Test fun canPassTestSuite() { - val testCases = Json.parseJson(loaderTestSuiteJson).jsonObject["tests"]!!.jsonArray + val testCases = Json.parseToJsonElement(loaderTestSuiteJson).jsonObject["tests"]!!.jsonArray testCases .map { TestCase.fromJson(it.jsonObject) } @@ -107,7 +104,7 @@ class AwsConfigLoaderTest { else -> "/" } every { testPlatform.getenv(capture(envKeyParam)) } answers { - (testCase.environment[envKeyParam.captured] as JsonLiteral?)?.content + (testCase.environment[envKeyParam.captured] as JsonPrimitive?)?.content } every { testPlatform.getProperty(capture(propKeyParam)) } answers { if (propKeyParam.captured == "user.home") testCase.languageSpecificHome else null @@ -128,13 +125,13 @@ class AwsConfigLoaderTest { ) { companion object { fun fromJson(json: JsonObject): TestCase { - val name = (json["name"] as JsonLiteral).content + val name = (json["name"] as JsonPrimitive).content val environment: Map = json["environment"] as JsonObject - val languageSpecificHome = (json["languageSpecificHome"] as JsonLiteral?)?.content - val platformRaw = (json["platform"] as JsonLiteral).content - val profile = (json["profile"] as JsonLiteral?)?.content - val configLocation = (json["configLocation"] as JsonLiteral).content - val credentialsLocation = (json["credentialsLocation"] as JsonLiteral).content + val languageSpecificHome = (json["languageSpecificHome"] as JsonPrimitive?)?.content + val platformRaw = (json["platform"] as JsonPrimitive).content + val profile = (json["profile"] as JsonPrimitive?)?.content + val configLocation = (json["configLocation"] as JsonPrimitive).content + val credentialsLocation = (json["credentialsLocation"] as JsonPrimitive).content val platform = when (platformRaw) { "windows" -> OsFamily.Windows diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt index cb1930173e9..60501b1e4fd 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt @@ -8,12 +8,7 @@ import io.mockk.coEvery import io.mockk.every import io.mockk.mockk import io.mockk.slot -import kotlinx.serialization.json.Json -import kotlinx.serialization.json.JsonElement -import kotlinx.serialization.json.JsonLiteral -import kotlinx.serialization.json.JsonNull -import kotlinx.serialization.json.JsonObject -import kotlinx.serialization.json.JsonPrimitive +import kotlinx.serialization.json.* import kotlin.test.Test import kotlin.test.assertEquals import kotlin.test.assertFails @@ -22,7 +17,7 @@ class AwsProfileParserTest { @Test fun canPassTestSuite() { - val testList = Json.parseJson(parserTestSuiteJson).jsonObject["tests"]!!.jsonArray + val testList = Json.parseToJsonElement(parserTestSuiteJson).jsonObject["tests"]!!.jsonArray testList .map { TestCase.fromJson(it.jsonObject) } @@ -34,7 +29,7 @@ class AwsProfileParserTest { when (testCase) { is TestCase.MatchConfigOutputCase -> { val actual = parse(FileType.CONFIGURATION, testCase.configInput).toJsonElement() - val expectedJson = Json.parseJson(testCase.expectedOutput) + val expectedJson = Json.parseToJsonElement(testCase.expectedOutput) assertEquals(expectedJson, actual, message = "[idx=$index]: $testCase") } is TestCase.MatchCredentialOutputCase -> { @@ -141,11 +136,11 @@ class AwsProfileParserTest { private sealed class TestCase { companion object { fun fromJson(json: JsonObject): TestCase { - val name = (json["name"] as JsonLiteral).content - val configIn = (json["input"]!!.jsonObject["configFile"] as JsonLiteral?)?.content - val credentialIn = (json["input"]!!.jsonObject["credentialsFile"] as JsonLiteral?)?.content + val name = (json["name"] as JsonPrimitive).content + val configIn = (json["input"]!!.jsonObject["configFile"] as JsonPrimitive?)?.content + val credentialIn = (json["input"]!!.jsonObject["credentialsFile"] as JsonPrimitive?)?.content val expected = json["output"]!!.jsonObject["profiles"]?.toString() - val errorContaining = (json["output"]!!.jsonObject["errorContaining"] as JsonLiteral?)?.content + val errorContaining = (json["output"]!!.jsonObject["errorContaining"] as JsonPrimitive?)?.content check(expected != null || errorContaining != null) { "Unexpected output: $json" } check(configIn != null || credentialIn != null) { "Unexpected output: $json" } diff --git a/gradle.properties b/gradle.properties index 6bed1849c6d..0ab89fd3245 100644 --- a/gradle.properties +++ b/gradle.properties @@ -25,7 +25,7 @@ kotlinJVMTargetVersion=1.8 # kotlin libraries coroutinesVersion=1.5.1 atomicFuVersion=0.16.1 -kotlinxSerializationVersion=0.20.0 +kotlinxSerializationVersion=1.3.0 # crt crtKotlinVersion=0.4.1-SNAPSHOT From 7dc3ec5d3ea9eea5dffb4537436731716b59be7b Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 1 Oct 2021 10:28:11 -0400 Subject: [PATCH 10/16] load endpoints from config and add test suite --- .../kotlin/runtime/config/imds/EC2Metadata.kt | 16 +- .../config/imds/ImdsEndpointResolver.kt | 39 +++- .../runtime/config/imds/EC2MetadataTest.kt | 78 +++++++- .../runtime/config/imds/ImdsTestSuite.kt | 172 ++++++++++++++++++ aws-runtime/testing/build.gradle.kts | 1 + .../runtime/testing/TestPlatformProvider.kt | 37 ++++ 6 files changed, 327 insertions(+), 16 deletions(-) create mode 100644 aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsTestSuite.kt create mode 100644 aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt index 456e5d1399d..5b02f78fb0b 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt @@ -25,6 +25,8 @@ import aws.smithy.kotlin.runtime.io.Closeable import aws.smithy.kotlin.runtime.io.middleware.Phase import aws.smithy.kotlin.runtime.logging.Logger import aws.smithy.kotlin.runtime.time.Clock +import aws.smithy.kotlin.runtime.util.Platform +import aws.smithy.kotlin.runtime.util.PlatformProvider import kotlin.time.Duration import kotlin.time.ExperimentalTime @@ -56,6 +58,7 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { private val endpointModeOverride: EndpointMode? = builder.endpointMode private val tokenTTL: Duration = builder.tokenTTL private val clock: Clock = builder.clock + private val platformProvider: PlatformProvider = builder.platformProvider init { if (endpointOverride != null && endpointModeOverride != null) { @@ -82,7 +85,7 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { private val middleware: List = listOf( ServiceEndpointResolver.create { serviceId = SERVICE - resolver = ImdsEndpointResolver(endpointModeOverride, endpointOverride) + resolver = ImdsEndpointResolver(platformProvider, endpointModeOverride, endpointOverride) }, UserAgent.create { metadata = AwsUserAgentMetadata.fromEnvironment(ApiMetadata(SERVICE, "unknown")) @@ -174,9 +177,14 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { internal var engine: HttpClientEngine? = null /** - * The source of time for token refreshes + * The source of time for token refreshes. This is here to facilitate testing and can otherwise be ignored */ internal var clock: Clock = Clock.System + + /** + * The platform provider. This is here to facilitate testing and can otherwise be ignored + */ + internal var platformProvider: PlatformProvider = Platform } } @@ -197,7 +205,7 @@ public enum class EndpointMode(internal val defaultEndpoint: Endpoint) { public fun fromValue(value: String): EndpointMode = when (value.lowercase()) { "ipv4" -> IPv4 "ipv6" -> IPv6 - else -> throw IllegalArgumentException("invalid EndpointMode: $value") + else -> throw IllegalArgumentException("invalid EndpointMode: `$value`") } } } @@ -208,7 +216,7 @@ public enum class EndpointMode(internal val defaultEndpoint: Endpoint) { * @param statusCode The raw HTTP status code of the response * @param message The error message */ -public class EC2MetadataError(statusCode: Int, message: String) : AwsServiceException(message) +public class EC2MetadataError(public val statusCode: Int, message: String) : AwsServiceException(message) private fun Endpoint.toUrl(): Url { val endpoint = this diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt index adaeff4cf8a..40d23ae2965 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt @@ -6,18 +6,25 @@ package aws.sdk.kotlin.runtime.config.imds import aws.sdk.kotlin.runtime.AwsSdkSetting +import aws.sdk.kotlin.runtime.config.AwsProfile +import aws.sdk.kotlin.runtime.config.loadActiveAwsProfile import aws.sdk.kotlin.runtime.endpoint.Endpoint import aws.sdk.kotlin.runtime.endpoint.EndpointResolver import aws.sdk.kotlin.runtime.resolve import aws.smithy.kotlin.runtime.http.Url -import aws.smithy.kotlin.runtime.util.Platform +import aws.smithy.kotlin.runtime.util.PlatformProvider + +internal const val EC2_METADATA_SERVICE_ENDPOINT_PROFILE_KEY = "ec2_metadata_service_endpoint" +internal const val EC2_METADATA_SERVICE_ENDPOINT_MODE_PROFILE_KEY = "ec2_metadata_service_endpoint_mode" internal class ImdsEndpointResolver( + private val platformProvider: PlatformProvider, private val endpointModeOverride: EndpointMode? = null, - private val endpointOverride: Endpoint? = null + private val endpointOverride: Endpoint? = null, ) : EndpointResolver { - // cached endpoint + // cached endpoint and profile private var resolvedEndpoint: Endpoint? = null + private var cachedProfile: AwsProfile? = null override suspend fun resolve(service: String, region: String): Endpoint = resolvedEndpoint ?: doResolveEndpoint() @@ -36,15 +43,27 @@ internal class ImdsEndpointResolver( return endpointMode.defaultEndpoint } - private suspend fun loadEndpointFromEnv(): Endpoint? { - val uri = AwsSdkSetting.AwsEc2MetadataServiceEndpoint.resolve(Platform) ?: return null - return Url.parse(uri).let { Endpoint(it.host, it.scheme.protocolName) } + private fun loadEndpointFromEnv(): Endpoint? = + AwsSdkSetting.AwsEc2MetadataServiceEndpoint.resolve(platformProvider)?.toEndpoint() + + private suspend fun loadEndpointFromProfile(): Endpoint? { + val profile = cachedProfileOrLoad() + return profile[EC2_METADATA_SERVICE_ENDPOINT_PROFILE_KEY]?.toEndpoint() } - private suspend fun loadEndpointFromProfile(): Endpoint? = null + private fun loadEndpointModeFromEnv(): EndpointMode? = + AwsSdkSetting.AwsEc2MetadataServiceEndpointMode.resolve(platformProvider)?.let { EndpointMode.fromValue(it) } - private suspend fun loadEndpointModeFromEnv(): EndpointMode? = - AwsSdkSetting.AwsEc2MetadataServiceEndpointMode.resolve(Platform)?.let { EndpointMode.fromValue(it) } + private suspend fun loadEndpointModeFromProfile(): EndpointMode? { + val profile = cachedProfileOrLoad() + return profile[EC2_METADATA_SERVICE_ENDPOINT_MODE_PROFILE_KEY]?.let { EndpointMode.fromValue(it) } + } + + private suspend fun cachedProfileOrLoad(): AwsProfile = cachedProfile ?: loadActiveAwsProfile(platformProvider).also { cachedProfile = it } +} - private suspend fun loadEndpointModeFromProfile(): EndpointMode? = null +// Parse a string as a URL and convert to an endpoint +internal fun String.toEndpoint(): Endpoint { + val url = Url.parse(this) + return Endpoint(url.host, url.scheme.protocolName, url.port) } diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt index 7799c804ad3..df428fcc36b 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -7,6 +7,7 @@ package aws.sdk.kotlin.runtime.config.imds import aws.sdk.kotlin.runtime.ConfigurationException import aws.sdk.kotlin.runtime.endpoint.Endpoint +import aws.sdk.kotlin.runtime.testing.TestPlatformProvider import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.content.ByteArrayContent @@ -16,6 +17,9 @@ import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestConnection import aws.smithy.kotlin.runtime.httptest.buildTestConnection import aws.smithy.kotlin.runtime.time.ManualClock +import io.kotest.matchers.string.contain +import io.kotest.matchers.string.shouldContain +import kotlinx.serialization.json.* import kotlin.test.* import kotlin.time.Duration import kotlin.time.ExperimentalTime @@ -196,8 +200,78 @@ class EC2MetadataTest { } @Test - fun testConfig() { - // need to mock various config scenarios + fun testHttpConnectTimeouts() { + // Need a 1 sec connect timeout + other timeouts in imds spec fail("not implemented yet") } + + data class ImdsConfigTest( + val name: String, + val env: Map, + val fs: Map, + val endpointOverride: String?, + val modeOverride: String?, + val result: Result, + ) { + companion object { + fun fromJson(element: JsonObject): ImdsConfigTest { + val resultObj = element["result"]!!.jsonObject + // map to success or generic error with the expected message substring of _some_ error that should be thrown + val result = resultObj["Ok"]?.jsonPrimitive?.content?.let { Result.success(it) } + ?: resultObj["Err"]!!.jsonPrimitive.content.let { Result.failure(RuntimeException(it)) } + + return ImdsConfigTest( + element["name"]!!.jsonPrimitive.content, + element["env"]!!.jsonObject.mapValues { it.value.jsonPrimitive.content }, + element["fs"]!!.jsonObject.mapValues { it.value.jsonPrimitive.content }, + element["endpointOverride"]?.jsonPrimitive?.content, + element["modeOverride"]?.jsonPrimitive?.content, + result + ) + } + } + } + + @Test + fun testConfig(): Unit = runSuspendTest { + val tests = Json.parseToJsonElement(imdsTestSuite).jsonObject["tests"]!!.jsonArray.map { ImdsConfigTest.fromJson(it.jsonObject) } + tests.forEach { test -> + val result = runCatching { check(test) } + when { + test.result.isSuccess && result.isSuccess -> {} + test.result.isSuccess && result.isFailure -> fail("expected success but failed; test=${test.name}; result=$result") + test.result.isFailure && result.isSuccess -> fail("expected failure but succeeded; test=${test.name}; result=$result") + test.result.isFailure && result.isFailure -> { + result.exceptionOrNull()!!.message.shouldContain(test.result.exceptionOrNull()!!.message!!) + } + } + } + } + + private suspend fun check(test: ImdsConfigTest) { + val connection = buildTestConnection { + if (test.result.isSuccess) { + val endpoint = test.result.getOrThrow() + expect( + tokenRequest(endpoint, DEFAULT_TOKEN_TTL_SECONDS), + tokenResponse(DEFAULT_TOKEN_TTL_SECONDS, "TOKEN_A") + ) + expect(imdsResponse("output 1")) + } + } + + val client = EC2Metadata { + engine = connection + test.endpointOverride?.let { endpointOverride -> + endpoint = Url.parse(endpointOverride).let { Endpoint(it.host, it.scheme.protocolName) } + } + test.modeOverride?.let { + endpointMode = EndpointMode.fromValue(it) + } + platformProvider = TestPlatformProvider(test.env, fs = test.fs) + } + + client.get("/hello") + connection.assertRequests() + } } diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsTestSuite.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsTestSuite.kt new file mode 100644 index 00000000000..3cbe1ab109e --- /dev/null +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsTestSuite.kt @@ -0,0 +1,172 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.imds + +// language=JSON +internal const val imdsTestSuite = """ +{ + "tests": [ + { + "name": "all fields unset, default endpoint", + "env": {}, + "fs": {}, + "result": { + "Ok": "http://169.254.169.254/latest/api/token" + } + }, + { + "name": "environment variable endpoint override", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT": "http://override:456" + }, + "fs": {}, + "result": { + "Ok": "http://override:456/latest/api/token" + } + }, + { + "name": "profile endpoint override", + "env": { + "AWS_CONFIG_FILE": "config" + }, + "fs": { + "config": "[default]\nec2_metadata_service_endpoint = http://override:456" + }, + "result": { + "Ok": "http://override:456/latest/api/token" + } + }, + { + "name": "environment overrides profile", + "env": { + "AWS_CONFIG_FILE": "config", + "AWS_EC2_METADATA_SERVICE_ENDPOINT": "http://override:456" + }, + "fs": { + "config": "[default]\nec2_metadata_service_endpoint = http://wrong:456" + }, + "result": { + "Ok": "http://override:456/latest/api/token" + } + }, + { + "name": "invalid endpoint mode override", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE": "error" + }, + "fs": { + }, + "result": { + "Err": "invalid EndpointMode: `error`" + } + }, + { + "name": "ipv4 endpoint", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE": "IPv4" + }, + "fs": { + }, + "result": { + "Ok": "http://169.254.169.254/latest/api/token" + } + }, + { + "name": "ipv6 endpoint", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE": "IPv6" + }, + "fs": { + }, + "result": { + "Ok": "http://[fd00:ec2::254]/latest/api/token" + } + }, + { + "name": "case insensitive endpoint", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT_MODE": "ipV6" + }, + "fs": { + }, + "result": { + "Ok": "http://[fd00:ec2::254]/latest/api/token" + } + }, + { + "name": "ipv6 endpoint from config", + "env": { + "AWS_CONFIG_FILE": "config" + }, + "fs": { + "config": "[default]\nec2_metadata_service_endpoint_mode = IPv6" + }, + "result": { + "Ok": "http://[fd00:ec2::254]/latest/api/token" + } + }, + { + "name": "invalid config endpoint mode", + "env": { + "AWS_CONFIG_FILE": "config" + }, + "fs": { + "config": "[default]\nec2_metadata_service_endpoint_mode = IPv7" + }, + "result": { + "Err": "invalid EndpointMode: `IPv7`" + } + }, + { + "name": "explicit endpoint override", + "env": { + "AWS_CONFIG_FILE": "config" + }, + "fs": { + "config": "[default]\nec2_metadata_service_endpoint_mode = IPv6" + }, + "endpointOverride": "https://custom", + "result": { + "Ok": "https://custom/latest/api/token" + } + }, + { + "name": "explicit mode override", + "env": { + "AWS_CONFIG_FILE": "config" + }, + "fs": { + "config": "[default]\nec2_metadata_service_endpoint_mode = IPv4" + }, + "modeOverride": "IPv6", + "result": { + "Ok": "http://[fd00:ec2::254]/latest/api/token" + } + }, + { + "name": "invalid uri", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT": "not a uri" + }, + "fs": {}, + "result": { + "Err": "Illegal character" + } + }, + { + "name": "environment variable overrides endpoint mode override", + "env": { + "AWS_EC2_METADATA_SERVICE_ENDPOINT": "http://169.254.169.200" + }, + "fs": {}, + "modeOverride": "IPv6", + "result": { + "Ok": "http://169.254.169.200/latest/api/token" + } + } + ] +} +""" diff --git a/aws-runtime/testing/build.gradle.kts b/aws-runtime/testing/build.gradle.kts index 415e3d4ea41..7d7fb523d55 100644 --- a/aws-runtime/testing/build.gradle.kts +++ b/aws-runtime/testing/build.gradle.kts @@ -12,6 +12,7 @@ kotlin { commonMain { dependencies { api("aws.smithy.kotlin:testing:$smithyKotlinVersion") + api("aws.smithy.kotlin:utils:$smithyKotlinVersion") } } } diff --git a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt new file mode 100644 index 00000000000..542afe8dfcf --- /dev/null +++ b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt @@ -0,0 +1,37 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.testing + +import aws.smithy.kotlin.runtime.util.Filesystem +import aws.smithy.kotlin.runtime.util.OperatingSystem +import aws.smithy.kotlin.runtime.util.OsFamily +import aws.smithy.kotlin.runtime.util.PlatformProvider + +/** + * An implementation of [PlatformProvider] meant for testing + * @param env Environment variable mappings + * @param props System property mappings + * @param fs Filesystem path to content mappings + * @param os Operating system info to emulate + */ +public class TestPlatformProvider( + env: Map = emptyMap(), + private val props: Map = emptyMap(), + private val fs: Map = emptyMap(), + private val os: OperatingSystem = OperatingSystem(OsFamily.Linux, "test") +) : PlatformProvider, Filesystem by Filesystem.fromMap(fs.mapValues { it.value.encodeToByteArray() }) { + // ensure HOME directory is set for path normalization. this is mostly for AWS config loader behavior + private val env = if (env.containsKey("HOME")) env else env.toMutableMap().apply { put("HOME", "/users/test") } + override val filePathSeparator: String + get() = when (os.family) { + OsFamily.Windows -> "\\" + else -> "/" + } + + override fun osInfo(): OperatingSystem = os + override fun getProperty(key: String): String? = props[key] + override fun getenv(key: String): String? = env[key] +} From e4a27e149cbc8c5f89507ce86db05703f14f5961 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 5 Oct 2021 12:35:49 -0400 Subject: [PATCH 11/16] move profile parsing and types to subpackage --- .../sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt | 4 ++-- .../kotlin/runtime/config/{ => profile}/AwsConfigLoader.kt | 7 ++++++- .../kotlin/runtime/config/{ => profile}/AwsConfigParser.kt | 7 ++++++- .../sdk/kotlin/runtime/config/{ => profile}/AwsProfile.kt | 7 ++++++- .../runtime/config/{ => profile}/ContinuationMerger.kt | 7 ++++++- .../aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt | 1 - .../runtime/config/{ => profile}/AwsConfigLoaderTest.kt | 7 ++++++- .../runtime/config/{ => profile}/AwsConfigParserTest.kt | 7 ++++++- .../runtime/config/{ => profile}/ContinuationMergerTest.kt | 7 ++++++- .../kotlin/runtime/config/{ => profile}/SpecTestSuites.kt | 2 +- .../config/{ => profile}/AWSConfigLoaderFilesystemTest.kt | 2 +- 11 files changed, 46 insertions(+), 12 deletions(-) rename aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/{ => profile}/AwsConfigLoader.kt (96%) rename aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/{ => profile}/AwsConfigParser.kt (99%) rename aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/{ => profile}/AwsProfile.kt (89%) rename aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/{ => profile}/ContinuationMerger.kt (96%) rename aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/{ => profile}/AwsConfigLoaderTest.kt (97%) rename aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/{ => profile}/AwsConfigParserTest.kt (98%) rename aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/{ => profile}/ContinuationMergerTest.kt (91%) rename aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/{ => profile}/SpecTestSuites.kt (99%) rename aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/{ => profile}/AWSConfigLoaderFilesystemTest.kt (98%) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt index 40d23ae2965..9a8efe745ad 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt @@ -6,8 +6,8 @@ package aws.sdk.kotlin.runtime.config.imds import aws.sdk.kotlin.runtime.AwsSdkSetting -import aws.sdk.kotlin.runtime.config.AwsProfile -import aws.sdk.kotlin.runtime.config.loadActiveAwsProfile +import aws.sdk.kotlin.runtime.config.profile.AwsProfile +import aws.sdk.kotlin.runtime.config.profile.loadActiveAwsProfile import aws.sdk.kotlin.runtime.endpoint.Endpoint import aws.sdk.kotlin.runtime.endpoint.EndpointResolver import aws.sdk.kotlin.runtime.resolve diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsConfigLoader.kt similarity index 96% rename from aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsConfigLoader.kt index d97e355cf46..1a5a168e245 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigLoader.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsConfigLoader.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile import aws.sdk.kotlin.runtime.AwsSdkSetting import aws.sdk.kotlin.runtime.InternalSdkApi diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsConfigParser.kt similarity index 99% rename from aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsConfigParser.kt index 321f083c3d6..98695d00a1e 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsConfigParser.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsConfigParser.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile import aws.sdk.kotlin.runtime.AwsSdkSetting import aws.sdk.kotlin.runtime.ConfigurationException diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt similarity index 89% rename from aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt index b059a32a010..48126571488 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/AwsProfile.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/AwsProfile.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile /** * The properties and name of an AWS configuration profile. diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/ContinuationMerger.kt similarity index 96% rename from aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/ContinuationMerger.kt index 99dc263ee90..95f0487016d 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/ContinuationMerger.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/profile/ContinuationMerger.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile /** * Merges property continuation lines diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt index df428fcc36b..82bcb4ef305 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -17,7 +17,6 @@ import aws.smithy.kotlin.runtime.http.response.HttpResponse import aws.smithy.kotlin.runtime.httptest.TestConnection import aws.smithy.kotlin.runtime.httptest.buildTestConnection import aws.smithy.kotlin.runtime.time.ManualClock -import io.kotest.matchers.string.contain import io.kotest.matchers.string.shouldContain import kotlinx.serialization.json.* import kotlin.test.* diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/AwsConfigLoaderTest.kt similarity index 97% rename from aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/AwsConfigLoaderTest.kt index 03f8f546f16..c34be4a5950 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigLoaderTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/AwsConfigLoaderTest.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.util.OperatingSystem diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/AwsConfigParserTest.kt similarity index 98% rename from aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/AwsConfigParserTest.kt index 60501b1e4fd..2ce7e443360 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsConfigParserTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/AwsConfigParserTest.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.util.OperatingSystem diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/ContinuationMergerTest.kt similarity index 91% rename from aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/ContinuationMergerTest.kt index f089a979f62..57566786611 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/ContinuationMergerTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/ContinuationMergerTest.kt @@ -1,4 +1,9 @@ -package aws.sdk.kotlin.runtime.config +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config.profile import kotlin.test.Test import kotlin.test.assertEquals diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt similarity index 99% rename from aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt index a33c2024456..fb5dce90a1b 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/SpecTestSuites.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt @@ -1,4 +1,4 @@ -package aws.sdk.kotlin.runtime.config +package aws.sdk.kotlin.runtime.config.profile /** * This test suite exercises the parser and continuation merger. diff --git a/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/AWSConfigLoaderFilesystemTest.kt b/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/profile/AWSConfigLoaderFilesystemTest.kt similarity index 98% rename from aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/AWSConfigLoaderFilesystemTest.kt rename to aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/profile/AWSConfigLoaderFilesystemTest.kt index aeba3b98529..43e6324d9c8 100644 --- a/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/AWSConfigLoaderFilesystemTest.kt +++ b/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/config/profile/AWSConfigLoaderFilesystemTest.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.runtime.config +package aws.sdk.kotlin.runtime.config.profile import aws.sdk.kotlin.runtime.testing.runSuspendTest import aws.smithy.kotlin.runtime.util.OperatingSystem From fd0c78cbe2ca855f7555debc6cc83031eb5e43fd Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 5 Oct 2021 12:36:02 -0400 Subject: [PATCH 12/16] ignore wip tests --- .../aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt index 82bcb4ef305..4784dcda5e9 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -181,23 +181,27 @@ class EC2MetadataTest { connection.assertRequests() } + @Ignore @Test fun testRetryHttp500() { fail("not implemented yet") } + @Ignore @Test fun testRetryTokenFailure() { // 500 during token acquisition should be retried fail("not implemented yet") } + @Ignore @Test fun testNoRetryHttp403() { // 403 responses from IMDS during token acquisition MUST not be retried fail("not implemented yet") } + @Ignore @Test fun testHttpConnectTimeouts() { // Need a 1 sec connect timeout + other timeouts in imds spec From 1c7a96f6a8dddf6cce895ecf44586cef60634cfd Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 5 Oct 2021 13:10:47 -0400 Subject: [PATCH 13/16] fix erroneous dependency relationship --- aws-runtime/protocols/http/build.gradle.kts | 1 - 1 file changed, 1 deletion(-) diff --git a/aws-runtime/protocols/http/build.gradle.kts b/aws-runtime/protocols/http/build.gradle.kts index d2d0001953d..95d5af42471 100644 --- a/aws-runtime/protocols/http/build.gradle.kts +++ b/aws-runtime/protocols/http/build.gradle.kts @@ -14,7 +14,6 @@ kotlin { commonMain { dependencies { api(project(":aws-runtime:aws-core")) - api(project(":aws-runtime:regions")) api("aws.smithy.kotlin:http:$smithyKotlinVersion") } } From bec67408c8ba4e43364fc1c268891d70643abc6c Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 7 Oct 2021 11:56:34 -0400 Subject: [PATCH 14/16] thread safe cached value + pr feedback --- aws-runtime/aws-config/build.gradle.kts | 1 + .../sdk/kotlin/runtime/config/CachedValue.kt | 64 ++++++++++++ .../kotlin/runtime/config/imds/EC2Metadata.kt | 7 +- .../config/imds/ImdsEndpointResolver.kt | 19 ++-- .../sdk/kotlin/runtime/config/imds/Token.kt | 10 -- .../runtime/config/imds/TokenMiddleware.kt | 34 +++---- .../kotlin/runtime/config/CachedValueTest.kt | 98 +++++++++++++++++++ .../runtime/config/profile/SpecTestSuites.kt | 4 + 8 files changed, 194 insertions(+), 43 deletions(-) create mode 100644 aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/CachedValue.kt create mode 100644 aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/CachedValueTest.kt diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 21ee6b7b2ab..361bcb3f531 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -15,6 +15,7 @@ kotlin { api(project(":aws-runtime:aws-core")) implementation("aws.smithy.kotlin:logging:$smithyKotlinVersion") implementation("aws.smithy.kotlin:http:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:utils:$smithyKotlinVersion") implementation(project(":aws-runtime:http-client-engine-crt")) implementation(project(":aws-runtime:protocols:http")) } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/CachedValue.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/CachedValue.kt new file mode 100644 index 00000000000..76b9c02b4eb --- /dev/null +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/CachedValue.kt @@ -0,0 +1,64 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config + +import aws.smithy.kotlin.runtime.time.Clock +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +/** + * A value with an expiration + */ +internal data class ExpiringValue (val value: T, val expiresAt: Instant) + +/** + * Expiry aware value + * + * @param value The value that expires + * @param bufferTime The amount of time before the actual expiration time when the value is considered expired. By default + * the buffer time is zero meaning the value expires at the expiration time. A non-zero buffer time means the value will + * expire BEFORE the actual expiration. + * @param clock The clock to use for system time + */ +@OptIn(ExperimentalTime::class) +internal class CachedValue ( + private var value: ExpiringValue? = null, + private val bufferTime: Duration = Duration.seconds(0), + private val clock: Clock = Clock.System +) { + constructor(value: T, expiresAt: Instant, bufferTime: Duration = Duration.seconds(0), clock: Clock = Clock.System) : this(ExpiringValue(value, expiresAt), bufferTime, clock) + private val mu = Mutex() + + /** + * Check if the value is expired or not as compared to the time [now] + */ + suspend fun isExpired(): Boolean = mu.withLock { isExpiredUnlocked() } + + private fun isExpiredUnlocked(): Boolean { + val curr = value ?: return true + return clock.now() >= (curr.expiresAt - bufferTime) + } + + /** + * Get the value if it has not expired yet. Returns null if the value has expired + */ + suspend fun get(): T? = mu.withLock { + if (!isExpiredUnlocked()) return value!!.value else null + } + + /** + * Attempt to get the value or refresh it with [initializer] if it is expired + */ + suspend fun getOrLoad(initializer: suspend () -> ExpiringValue): T = mu.withLock { + if (!isExpiredUnlocked()) return@withLock value!!.value + + val refreshed = initializer().also { value = it } + return refreshed.value + } +} diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt index 5b02f78fb0b..6b54c943ac4 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt @@ -15,7 +15,6 @@ import aws.sdk.kotlin.runtime.http.engine.crt.CrtHttpEngine import aws.sdk.kotlin.runtime.http.middleware.ServiceEndpointResolver import aws.sdk.kotlin.runtime.http.middleware.UserAgent import aws.smithy.kotlin.runtime.client.ExecutionContext -import aws.smithy.kotlin.runtime.client.SdkClientOption import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.engine.HttpClientEngine import aws.smithy.kotlin.runtime.http.engine.HttpClientEngineConfig @@ -56,7 +55,7 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { private val maxRetries: UInt = builder.maxRetries private val endpointOverride: Endpoint? = builder.endpoint private val endpointModeOverride: EndpointMode? = builder.endpointMode - private val tokenTTL: Duration = builder.tokenTTL + private val tokenTtl: Duration = builder.tokenTTL private val clock: Clock = builder.clock private val platformProvider: PlatformProvider = builder.platformProvider @@ -92,7 +91,8 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { }, TokenMiddleware.create { httpClient = this@EC2Metadata.httpClient - ttl = tokenTTL + ttl = tokenTtl + clock = this@EC2Metadata.clock } ) @@ -131,7 +131,6 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { context { operationName = path service = SERVICE - set(SdkClientOption.Clock, clock) // artifact of re-using ServiceEndpointResolver middleware set(AwsClientOption.Region, "not-used") } diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt index 9a8efe745ad..fc4fa9e1fae 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt @@ -6,13 +6,13 @@ package aws.sdk.kotlin.runtime.config.imds import aws.sdk.kotlin.runtime.AwsSdkSetting -import aws.sdk.kotlin.runtime.config.profile.AwsProfile import aws.sdk.kotlin.runtime.config.profile.loadActiveAwsProfile import aws.sdk.kotlin.runtime.endpoint.Endpoint import aws.sdk.kotlin.runtime.endpoint.EndpointResolver import aws.sdk.kotlin.runtime.resolve import aws.smithy.kotlin.runtime.http.Url import aws.smithy.kotlin.runtime.util.PlatformProvider +import aws.smithy.kotlin.runtime.util.asyncLazy internal const val EC2_METADATA_SERVICE_ENDPOINT_PROFILE_KEY = "ec2_metadata_service_endpoint" internal const val EC2_METADATA_SERVICE_ENDPOINT_MODE_PROFILE_KEY = "ec2_metadata_service_endpoint_mode" @@ -23,15 +23,12 @@ internal class ImdsEndpointResolver( private val endpointOverride: Endpoint? = null, ) : EndpointResolver { // cached endpoint and profile - private var resolvedEndpoint: Endpoint? = null - private var cachedProfile: AwsProfile? = null + private val resolvedEndpoint = asyncLazy(::doResolveEndpoint) + private val activeProfile = asyncLazy { loadActiveAwsProfile(platformProvider) } - override suspend fun resolve(service: String, region: String): Endpoint = resolvedEndpoint ?: doResolveEndpoint() + override suspend fun resolve(service: String, region: String): Endpoint = resolvedEndpoint.get() - private suspend fun doResolveEndpoint(): Endpoint { - val resolved = endpointOverride ?: resolveEndpointFromConfig() - return resolved.also { resolvedEndpoint = it } - } + private suspend fun doResolveEndpoint(): Endpoint = endpointOverride ?: resolveEndpointFromConfig() private suspend fun resolveEndpointFromConfig(): Endpoint { // explicit endpoint configured @@ -47,7 +44,7 @@ internal class ImdsEndpointResolver( AwsSdkSetting.AwsEc2MetadataServiceEndpoint.resolve(platformProvider)?.toEndpoint() private suspend fun loadEndpointFromProfile(): Endpoint? { - val profile = cachedProfileOrLoad() + val profile = activeProfile.get() return profile[EC2_METADATA_SERVICE_ENDPOINT_PROFILE_KEY]?.toEndpoint() } @@ -55,11 +52,9 @@ internal class ImdsEndpointResolver( AwsSdkSetting.AwsEc2MetadataServiceEndpointMode.resolve(platformProvider)?.let { EndpointMode.fromValue(it) } private suspend fun loadEndpointModeFromProfile(): EndpointMode? { - val profile = cachedProfileOrLoad() + val profile = activeProfile.get() return profile[EC2_METADATA_SERVICE_ENDPOINT_MODE_PROFILE_KEY]?.let { EndpointMode.fromValue(it) } } - - private suspend fun cachedProfileOrLoad(): AwsProfile = cachedProfile ?: loadActiveAwsProfile(platformProvider).also { cachedProfile = it } } // Parse a string as a URL and convert to an endpoint diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt index 15b9b42c4b0..c3ca2bc1f26 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/Token.kt @@ -7,16 +7,6 @@ package aws.sdk.kotlin.runtime.config.imds import aws.smithy.kotlin.runtime.time.Instant -/** - * Tokens are cached to remove the need to reload the token between subsequent requests. To ensure - * a request never fails with a 401 (expired token), a buffer window exists during which the token - * is not expired but refreshed anyway to ensure the token doesn't expire during an in-flight operation. - */ -internal const val TOKEN_REFRESH_BUFFER_SECONDS = 120 - -internal const val X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS = "x-aws-ec2-metadata-token-ttl-seconds" -internal const val X_AWS_EC2_METADATA_TOKEN = "x-aws-ec2-metadata-token" - internal data class Token(val value: ByteArray, val expires: Instant) { override fun equals(other: Any?): Boolean { diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt index 3506466fbbb..c711fbbcc48 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/TokenMiddleware.kt @@ -5,7 +5,8 @@ package aws.sdk.kotlin.runtime.config.imds -import aws.smithy.kotlin.runtime.client.SdkClientOption +import aws.sdk.kotlin.runtime.config.CachedValue +import aws.sdk.kotlin.runtime.config.ExpiringValue import aws.smithy.kotlin.runtime.http.* import aws.smithy.kotlin.runtime.http.operation.SdkHttpOperation import aws.smithy.kotlin.runtime.http.operation.SdkHttpRequest @@ -14,19 +15,30 @@ import aws.smithy.kotlin.runtime.http.request.HttpRequestBuilder import aws.smithy.kotlin.runtime.http.request.url import aws.smithy.kotlin.runtime.http.response.complete import aws.smithy.kotlin.runtime.time.Clock -import aws.smithy.kotlin.runtime.time.Instant import kotlin.time.Duration import kotlin.time.ExperimentalTime +/** + * Tokens are cached to remove the need to reload the token between subsequent requests. To ensure + * a request never fails with a 401 (expired token), a buffer window exists during which the token + * is not expired but refreshed anyway to ensure the token doesn't expire during an in-flight operation. + */ +internal const val TOKEN_REFRESH_BUFFER_SECONDS = 120 + +internal const val X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS = "x-aws-ec2-metadata-token-ttl-seconds" +internal const val X_AWS_EC2_METADATA_TOKEN = "x-aws-ec2-metadata-token" + @OptIn(ExperimentalTime::class) internal class TokenMiddleware(config: Config) : Feature { private val ttl: Duration = config.ttl private val httpClient = requireNotNull(config.httpClient) { "SdkHttpClient is required for token middleware to make requests" } - private var cachedToken: Token? = null + private val clock: Clock = config.clock + private var cachedToken = CachedValue(null, bufferTime = Duration.seconds(TOKEN_REFRESH_BUFFER_SECONDS), clock = clock) public class Config { var ttl: Duration = Duration.seconds(DEFAULT_TOKEN_TTL_SECONDS) var httpClient: SdkHttpClient? = null + var clock: Clock = Clock.System } public companion object Feature : @@ -40,24 +52,12 @@ internal class TokenMiddleware(config: Config) : Feature { override fun install(operation: SdkHttpOperation) { operation.execution.mutate.intercept { req, next -> - val clock = req.context.getOrNull(SdkClientOption.Clock) ?: Clock.System - val token = useCachedTokenOrClear(clock) ?: getToken(clock, req).also { cachedToken = it } + val token = cachedToken.getOrLoad { getToken(clock, req).let { ExpiringValue(it, it.expires) } } req.subject.headers.append(X_AWS_EC2_METADATA_TOKEN, token.value.decodeToString()) next.call(req) } } - // use the cached token if it's still valid or clear it if expired and return null - private fun useCachedTokenOrClear(clock: Clock): Token? { - val cached = cachedToken ?: return null - val now = clock.now() - val expired = now.epochSeconds >= cached.expires.epochSeconds - TOKEN_REFRESH_BUFFER_SECONDS - if (expired) { - cachedToken = null - } - return cachedToken - } - private suspend fun getToken(clock: Clock, req: SdkHttpRequest): Token { val logger = req.context.getLogger("TokenMiddleware") logger.trace { "refreshing IMDS token" } @@ -83,7 +83,7 @@ internal class TokenMiddleware(config: Config) : Feature { HttpStatusCode.OK -> { val ttl = call.response.headers[X_AWS_EC2_METADATA_TOKEN_TTL_SECONDS]?.toLong() ?: throw EC2MetadataError(200, "No TTL provided in IMDS response") val token = call.response.body.readAll() ?: throw EC2MetadataError(200, "No token provided in IMDS response") - val expires = Instant.fromEpochSeconds(clock.now().epochSeconds + ttl, 0) + val expires = clock.now() + Duration.seconds(ttl) Token(token, expires) } else -> throw EC2MetadataError(call.response.status.value, "Failed to retrieve IMDS token") diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/CachedValueTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/CachedValueTest.kt new file mode 100644 index 00000000000..f632ee006aa --- /dev/null +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/CachedValueTest.kt @@ -0,0 +1,98 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.runtime.config + +import aws.sdk.kotlin.runtime.testing.runSuspendTest +import aws.smithy.kotlin.runtime.time.Instant +import aws.smithy.kotlin.runtime.time.ManualClock +import kotlinx.coroutines.async +import kotlinx.coroutines.sync.Mutex +import kotlinx.coroutines.sync.withLock +import kotlinx.coroutines.yield +import kotlin.test.* +import kotlin.time.Duration +import kotlin.time.ExperimentalTime + +@OptIn(ExperimentalTime::class) +class CachedValueTest { + @Test + fun testNull() = runSuspendTest { + val epoch = Instant.fromEpochSeconds(0) + val clock = ManualClock(epoch) + val value = CachedValue(null, clock = clock) + + assertTrue(value.isExpired()) + assertNull(value.get()) + } + + @Test + fun testExpiration() = runSuspendTest { + val epoch = Instant.fromEpochSeconds(0) + val expiresAt = epoch + Duration.seconds(10) + val clock = ManualClock(epoch) + + val value = CachedValue("foo", expiresAt, clock = clock) + + assertFalse(value.isExpired()) + assertEquals("foo", value.get()) + + clock.advance(Duration.seconds(10)) + assertTrue(value.isExpired()) + assertNull(value.get()) + } + + @Test + fun testExpirationBuffer() = runSuspendTest { + val epoch = Instant.fromEpochSeconds(0) + val expiresAt = epoch + Duration.seconds(100) + val clock = ManualClock(epoch) + + val value = CachedValue("foo", expiresAt, bufferTime = Duration.seconds(30), clock = clock) + + assertFalse(value.isExpired()) + assertEquals("foo", value.get()) + + clock.advance(Duration.seconds(70)) + assertTrue(value.isExpired()) + assertNull(value.get()) + } + + @Test + fun testGetOrLoad() = runSuspendTest { + val epoch = Instant.fromEpochSeconds(0) + val expiresAt = epoch + Duration.seconds(100) + val clock = ManualClock(epoch) + + val value = CachedValue("foo", expiresAt, bufferTime = Duration.seconds(30), clock = clock) + + var count = 0 + val mu = Mutex() + val initializer = suspend { + mu.withLock { count++ } + ExpiringValue("bar", expiresAt + Duration.seconds(count * 100)) + } + + assertFalse(value.isExpired()) + assertEquals("foo", value.getOrLoad(initializer)) + assertEquals(0, count) + + // t = 90 + clock.advance(Duration.seconds(90)) + assertEquals("bar", value.getOrLoad(initializer)) + assertFalse(value.isExpired()) + assertEquals(1, count) + + // t = 180 + clock.advance(Duration.seconds(90)) + repeat(10) { + async { + value.getOrLoad(initializer) + } + } + yield() + assertEquals(2, count) + } +} diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt index fb5dce90a1b..15cbc4cd38a 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/profile/SpecTestSuites.kt @@ -1,3 +1,7 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ package aws.sdk.kotlin.runtime.config.profile /** From d8e0cc99ab7a2ac1bf4646f67b97c90d92e72104 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 7 Oct 2021 12:33:07 -0400 Subject: [PATCH 15/16] refactor endpoint configuration --- .../kotlin/runtime/config/imds/EC2Metadata.kt | 45 ++++++++++--------- .../config/imds/ImdsEndpointResolver.kt | 15 ++++--- .../runtime/config/imds/EC2MetadataTest.kt | 14 +++--- 3 files changed, 44 insertions(+), 30 deletions(-) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt index 6b54c943ac4..50405726b26 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt @@ -53,26 +53,19 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { private val logger = Logger.getLogger() private val maxRetries: UInt = builder.maxRetries - private val endpointOverride: Endpoint? = builder.endpoint - private val endpointModeOverride: EndpointMode? = builder.endpointMode + private val endpointConfiguration: EndpointConfiguration = builder.endpointConfiguration private val tokenTtl: Duration = builder.tokenTTL private val clock: Clock = builder.clock private val platformProvider: PlatformProvider = builder.platformProvider init { - if (endpointOverride != null && endpointModeOverride != null) { - logger.warn { - "EndpointMode was set in combination with an explicit endpoint. " + - "The mode override will be ignored: endpointMode=$endpointModeOverride, endpoint=$endpointOverride" - } - } - // validate the override at construction time - if (endpointOverride != null) { + if (endpointConfiguration is EndpointConfiguration.Custom) { + val url = endpointConfiguration.endpoint.toUrl() try { - Url.parse(endpointOverride.toUrl().toString()) + Url.parse(url.toString()) } catch (ex: Exception) { - throw ConfigurationException("endpointOverride `$endpointOverride` is not a valid URI", ex) + throw ConfigurationException("Invalid endpoint configuration: `$url` is not a valid URI", ex) } } } @@ -84,7 +77,7 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { private val middleware: List = listOf( ServiceEndpointResolver.create { serviceId = SERVICE - resolver = ImdsEndpointResolver(platformProvider, endpointModeOverride, endpointOverride) + resolver = ImdsEndpointResolver(platformProvider, endpointConfiguration) }, UserAgent.create { metadata = AwsUserAgentMetadata.fromEnvironment(ApiMetadata(SERVICE, "unknown")) @@ -156,14 +149,9 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { public var maxRetries: UInt = DEFAULT_MAX_RETRIES /** - * The endpoint to make requests to. By default this is determined by the execution environment - */ - public var endpoint: Endpoint? = null - - /** - * The [EndpointMode] to use when connecting to [endpoint] + * The endpoint configuration to use when making requests */ - public var endpointMode: EndpointMode? = null + public var endpointConfiguration: EndpointConfiguration = EndpointConfiguration.Default /** * Override the time-to-live for the session token @@ -187,6 +175,23 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { } } +public sealed class EndpointConfiguration { + /** + * Detected from the execution environment + */ + public object Default : EndpointConfiguration() + + /** + * Override the endpoint to make requests to + */ + public data class Custom(val endpoint: Endpoint) : EndpointConfiguration() + + /** + * Override the [EndpointMode] to use + */ + public data class ModeOverride(val mode: EndpointMode) : EndpointConfiguration() +} + public enum class EndpointMode(internal val defaultEndpoint: Endpoint) { /** * IPv4 mode. This is the default unless otherwise specified diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt index fc4fa9e1fae..ba51583fa6d 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsEndpointResolver.kt @@ -19,8 +19,7 @@ internal const val EC2_METADATA_SERVICE_ENDPOINT_MODE_PROFILE_KEY = "ec2_metadat internal class ImdsEndpointResolver( private val platformProvider: PlatformProvider, - private val endpointModeOverride: EndpointMode? = null, - private val endpointOverride: Endpoint? = null, + private val endpointConfiguration: EndpointConfiguration ) : EndpointResolver { // cached endpoint and profile private val resolvedEndpoint = asyncLazy(::doResolveEndpoint) @@ -28,7 +27,10 @@ internal class ImdsEndpointResolver( override suspend fun resolve(service: String, region: String): Endpoint = resolvedEndpoint.get() - private suspend fun doResolveEndpoint(): Endpoint = endpointOverride ?: resolveEndpointFromConfig() + private suspend fun doResolveEndpoint(): Endpoint = when (endpointConfiguration) { + is EndpointConfiguration.Custom -> endpointConfiguration.endpoint + else -> resolveEndpointFromConfig() + } private suspend fun resolveEndpointFromConfig(): Endpoint { // explicit endpoint configured @@ -36,8 +38,11 @@ internal class ImdsEndpointResolver( if (endpoint != null) return endpoint // endpoint default from mode - val endpointMode = endpointModeOverride ?: loadEndpointModeFromEnv() ?: loadEndpointModeFromProfile() ?: EndpointMode.IPv4 - return endpointMode.defaultEndpoint + val mode = when (endpointConfiguration) { + is EndpointConfiguration.ModeOverride -> endpointConfiguration.mode + else -> loadEndpointModeFromEnv() ?: loadEndpointModeFromProfile() ?: EndpointMode.IPv4 + } + return mode.defaultEndpoint } private fun loadEndpointFromEnv(): Endpoint? = diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt index 4784dcda5e9..eb6063d6158 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt @@ -59,7 +59,9 @@ class EC2MetadataTest { assertFailsWith { EC2Metadata { engine = connection - endpoint = Endpoint("[foo::254]", protocol = "http") + endpointConfiguration = EndpointConfiguration.Custom( + Endpoint("[foo::254]", protocol = "http") + ) } } } @@ -115,7 +117,7 @@ class EC2MetadataTest { val client = EC2Metadata { engine = connection - endpointMode = EndpointMode.IPv6 + endpointConfiguration = EndpointConfiguration.ModeOverride(EndpointMode.IPv6) clock = testClock tokenTTL = Duration.seconds(600) } @@ -162,7 +164,7 @@ class EC2MetadataTest { val client = EC2Metadata { engine = connection - endpointMode = EndpointMode.IPv6 + endpointConfiguration = EndpointConfiguration.ModeOverride(EndpointMode.IPv6) clock = testClock tokenTTL = Duration.seconds(600) } @@ -266,10 +268,12 @@ class EC2MetadataTest { val client = EC2Metadata { engine = connection test.endpointOverride?.let { endpointOverride -> - endpoint = Url.parse(endpointOverride).let { Endpoint(it.host, it.scheme.protocolName) } + val endpoint = Url.parse(endpointOverride).let { Endpoint(it.host, it.scheme.protocolName) } + + endpointConfiguration = EndpointConfiguration.Custom(endpoint) } test.modeOverride?.let { - endpointMode = EndpointMode.fromValue(it) + endpointConfiguration = EndpointConfiguration.ModeOverride(EndpointMode.fromValue(it)) } platformProvider = TestPlatformProvider(test.env, fs = test.fs) } From 4487bc2a2205df60d7f27e272eb33c2cda3ff80b Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 7 Oct 2021 16:25:18 -0400 Subject: [PATCH 16/16] refactor: rename to ImdsClient --- .../config/imds/{EC2Metadata.kt => ImdsClient.kt} | 10 +++++----- .../imds/{EC2MetadataTest.kt => ImdsClientTest.kt} | 12 ++++++------ 2 files changed, 11 insertions(+), 11 deletions(-) rename aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/{EC2Metadata.kt => ImdsClient.kt} (95%) rename aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/{EC2MetadataTest.kt => ImdsClientTest.kt} (97%) diff --git a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsClient.kt similarity index 95% rename from aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt rename to aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsClient.kt index 50405726b26..38cf7eb9374 100644 --- a/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/EC2Metadata.kt +++ b/aws-runtime/aws-config/common/src/aws/sdk/kotlin/runtime/config/imds/ImdsClient.kt @@ -47,10 +47,10 @@ private const val SERVICE = "imds" * for more information. */ @OptIn(ExperimentalTime::class) -public class EC2Metadata private constructor(builder: Builder) : Closeable { +public class ImdsClient private constructor(builder: Builder) : Closeable { public constructor() : this(Builder()) - private val logger = Logger.getLogger() + private val logger = Logger.getLogger() private val maxRetries: UInt = builder.maxRetries private val endpointConfiguration: EndpointConfiguration = builder.endpointConfiguration @@ -83,14 +83,14 @@ public class EC2Metadata private constructor(builder: Builder) : Closeable { metadata = AwsUserAgentMetadata.fromEnvironment(ApiMetadata(SERVICE, "unknown")) }, TokenMiddleware.create { - httpClient = this@EC2Metadata.httpClient + httpClient = this@ImdsClient.httpClient ttl = tokenTtl - clock = this@EC2Metadata.clock + clock = this@ImdsClient.clock } ) public companion object { - public operator fun invoke(block: Builder.() -> Unit): EC2Metadata = EC2Metadata(Builder().apply(block)) + public operator fun invoke(block: Builder.() -> Unit): ImdsClient = ImdsClient(Builder().apply(block)) } /** diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsClientTest.kt similarity index 97% rename from aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt rename to aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsClientTest.kt index eb6063d6158..85b3349a5b0 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/EC2MetadataTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/imds/ImdsClientTest.kt @@ -24,7 +24,7 @@ import kotlin.time.Duration import kotlin.time.ExperimentalTime @OptIn(ExperimentalTime::class) -class EC2MetadataTest { +class ImdsClientTest { private fun tokenRequest(host: String, ttl: Int): HttpRequest = HttpRequest { val parsed = Url.parse(host) @@ -57,7 +57,7 @@ class EC2MetadataTest { fun testInvalidEndpointOverrideFailsCreation() { val connection = TestConnection() assertFailsWith { - EC2Metadata { + ImdsClient { engine = connection endpointConfiguration = EndpointConfiguration.Custom( Endpoint("[foo::254]", protocol = "http") @@ -83,7 +83,7 @@ class EC2MetadataTest { ) } - val client = EC2Metadata { engine = connection } + val client = ImdsClient { engine = connection } val r1 = client.get("/latest/metadata") assertEquals("output 1", r1) @@ -115,7 +115,7 @@ class EC2MetadataTest { val testClock = ManualClock() - val client = EC2Metadata { + val client = ImdsClient { engine = connection endpointConfiguration = EndpointConfiguration.ModeOverride(EndpointMode.IPv6) clock = testClock @@ -162,7 +162,7 @@ class EC2MetadataTest { val testClock = ManualClock() - val client = EC2Metadata { + val client = ImdsClient { engine = connection endpointConfiguration = EndpointConfiguration.ModeOverride(EndpointMode.IPv6) clock = testClock @@ -265,7 +265,7 @@ class EC2MetadataTest { } } - val client = EC2Metadata { + val client = ImdsClient { engine = connection test.endpointOverride?.let { endpointOverride -> val endpoint = Url.parse(endpointOverride).let { Endpoint(it.host, it.scheme.protocolName) }