diff --git a/.changes/16c733de-c490-404e-a5f9-4e1ba917d020.json b/.changes/16c733de-c490-404e-a5f9-4e1ba917d020.json new file mode 100644 index 00000000000..63d6923b3ff --- /dev/null +++ b/.changes/16c733de-c490-404e-a5f9-4e1ba917d020.json @@ -0,0 +1,8 @@ +{ + "id": "16c733de-c490-404e-a5f9-4e1ba917d020", + "type": "feature", + "description": "Add support for detecting custom metadata in system properties (starting with `aws.customMetadata.`) and environment variables (starting with `AWS_CUSTOM_METADATA_`)", + "issues": [ + "awslabs/aws-sdk-kotlin#575" + ] +} \ No newline at end of file diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsSdkSettingTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsSdkSettingTest.kt index 3fd935e05f1..9879777fbf4 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsSdkSettingTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/config/AwsSdkSettingTest.kt @@ -53,7 +53,9 @@ class AwsSdkSettingTest { private fun mockPlatform(env: Map, jvmProps: Map): PlatformEnvironProvider { return object : PlatformEnvironProvider { + override fun getAllEnvVars(): Map = env override fun getenv(key: String): String? = env[key] + override fun getAllProperties(): Map = jvmProps override fun getProperty(key: String): String? = jvmProps[key] } } diff --git a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/region/EnvironmentRegionProviderTest.kt b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/region/EnvironmentRegionProviderTest.kt index 86597b342e9..9d8340a8b75 100644 --- a/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/region/EnvironmentRegionProviderTest.kt +++ b/aws-runtime/aws-config/common/test/aws/sdk/kotlin/runtime/region/EnvironmentRegionProviderTest.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.runtime.region +import aws.smithy.kotlin.runtime.util.EnvironmentProvider import kotlinx.coroutines.ExperimentalCoroutinesApi import kotlinx.coroutines.test.runTest import kotlin.test.Test @@ -13,11 +14,15 @@ import kotlin.test.assertNull @OptIn(ExperimentalCoroutinesApi::class) class EnvironmentRegionProviderTest { + fun Map.asEnvironmentProvider() = object : EnvironmentProvider { + override fun getAllEnvVars(): Map = this@asEnvironmentProvider + override fun getenv(key: String): String? = this@asEnvironmentProvider[key] + } @Test fun noRegion() = runTest { val environ = mapOf() - val provider = EnvironmentRegionProvider { environ[it] } + val provider = EnvironmentRegionProvider(environ.asEnvironmentProvider()) assertNull(provider.getRegion()) } @@ -27,7 +32,7 @@ class EnvironmentRegionProviderTest { "AWS_REGION" to "us-east-1" ) - val provider = EnvironmentRegionProvider { environ[it] } + val provider = EnvironmentRegionProvider(environ.asEnvironmentProvider()) assertEquals("us-east-1", provider.getRegion()) } diff --git a/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/auth/credentials/DefaultChainCredentialsProviderTest.kt b/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/auth/credentials/DefaultChainCredentialsProviderTest.kt index 2002248f3e5..8b2bcc861ce 100644 --- a/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/auth/credentials/DefaultChainCredentialsProviderTest.kt +++ b/aws-runtime/aws-config/jvm/test/aws/sdk/kotlin/runtime/auth/credentials/DefaultChainCredentialsProviderTest.kt @@ -46,7 +46,9 @@ class DefaultChainCredentialsProviderTest { private val fs: Filesystem ) : PlatformProvider, Filesystem by fs { override fun osInfo(): OperatingSystem = OperatingSystem(OsFamily.Linux, "test") + override fun getAllProperties(): Map = mapOf() override fun getProperty(key: String): String? = null + override fun getAllEnvVars(): Map = env override fun getenv(key: String): String? = env[key] } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadata.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadata.kt index 19355bfcafa..3d554d9f577 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadata.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadata.kt @@ -6,9 +6,7 @@ package aws.sdk.kotlin.runtime.http import aws.sdk.kotlin.runtime.InternalSdkApi -import aws.sdk.kotlin.runtime.http.operation.ConfigMetadata import aws.sdk.kotlin.runtime.http.operation.CustomUserAgentMetadata -import aws.sdk.kotlin.runtime.http.operation.FeatureMetadata import aws.smithy.kotlin.runtime.util.* import kotlin.jvm.JvmInline @@ -69,20 +67,23 @@ public data class AwsUserAgentMetadata( ua.add("md/internal") } + // FIXME user agent strings are now too long so commenting out several metadata below. + // Re-enable once user agent strings can be longer. + ua.add("$sdkMetadata") - ua.add("$apiMetadata") + // ua.add("$apiMetadata") ua.add("$osMetadata") ua.add("$languageMetadata") - execEnvMetadata?.let { ua.add("$it") } + // execEnvMetadata?.let { ua.add("$it") } - val features = customMetadata?.typedExtras?.filterIsInstance() - features?.forEach { ua.add("$it") } + // val features = customMetadata?.typedExtras?.filterIsInstance() + // features?.forEach { ua.add("$it") } - val config = customMetadata?.typedExtras?.filterIsInstance() - config?.forEach { ua.add("$it") } + // val config = customMetadata?.typedExtras?.filterIsInstance() + // config?.forEach { ua.add("$it") } frameworkMetadata?.let { ua.add("$it") } - appId?.let { ua.add("app/$it") } + // appId?.let { ua.add("app/$it") } customMetadata?.extras?.let { val wrapper = AdditionalMetadata(it) @@ -107,6 +108,8 @@ internal fun loadAwsUserAgentMetadataFromEnvironment(platform: PlatformProvider, val appId = platform.getProperty(AWS_APP_ID_PROP) ?: platform.getenv(AWS_APP_ID_ENV) val frameworkMetadata = FrameworkMetadata.fromEnvironment(platform) + val customMetadata = CustomUserAgentMetadata.fromEnvironment(platform) + return AwsUserAgentMetadata( sdkMeta, apiMeta, @@ -115,6 +118,7 @@ internal fun loadAwsUserAgentMetadataFromEnvironment(platform: PlatformProvider, detectExecEnv(platform), frameworkMetadata = frameworkMetadata, appId = appId, + customMetadata = customMetadata, ) } @@ -183,10 +187,12 @@ public data class LanguageMetadata( ) { override fun toString(): String = buildString { append("lang/kotlin/$version") + /* FIXME re-enable once user agent strings can be longer if (extras.isNotEmpty()) { val wrapper = AdditionalMetadata(extras) append(" $wrapper") } + */ } } diff --git a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata.kt b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata.kt index b3381479953..2caac386ee6 100644 --- a/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata.kt +++ b/aws-runtime/aws-http/common/src/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadata.kt @@ -8,6 +8,10 @@ package aws.sdk.kotlin.runtime.http.operation import aws.sdk.kotlin.runtime.InternalSdkApi import aws.smithy.kotlin.runtime.client.ExecutionContext import aws.smithy.kotlin.runtime.util.AttributeKey +import aws.smithy.kotlin.runtime.util.PlatformEnvironProvider + +private const val CUSTOM_METADATA_ENV_PREFIX = "AWS_CUSTOM_METADATA_" +private const val CUSTOM_METADATA_PROP_PREFIX = "aws.customMetadata." /** * Operation context element for adding additional metadata to the `User-Agent` header string. @@ -20,6 +24,17 @@ public class CustomUserAgentMetadata { internal companion object { public val ContextKey: AttributeKey = AttributeKey("CustomUserAgentMetadata") + + internal fun fromEnvironment(provider: PlatformEnvironProvider): CustomUserAgentMetadata { + fun Map.findAndStripKeyPrefix(prefix: String) = this + .filterKeys { it.startsWith(prefix) } + .mapKeys { (key, _) -> key.substring(prefix.length) } + + val envVarMap = provider.getAllEnvVars().findAndStripKeyPrefix(CUSTOM_METADATA_ENV_PREFIX) + val propMap = provider.getAllProperties().findAndStripKeyPrefix(CUSTOM_METADATA_PROP_PREFIX) + val allProps = envVarMap + propMap + return CustomUserAgentMetadata().apply { allProps.forEach { (key, value) -> add(key, value) } } + } } /** diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataTest.kt index 51aad546946..167114a05f6 100644 --- a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataTest.kt +++ b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/AwsUserAgentMetadataTest.kt @@ -8,7 +8,7 @@ package aws.sdk.kotlin.runtime.http import aws.sdk.kotlin.runtime.http.operation.CustomUserAgentMetadata import aws.sdk.kotlin.runtime.testing.TestPlatformProvider import aws.smithy.kotlin.runtime.util.OsFamily -import io.kotest.matchers.string.shouldContain +import io.kotest.matchers.string.shouldNotContain import kotlin.test.Test import kotlin.test.assertEquals @@ -31,7 +31,15 @@ class AwsUserAgentMetadataTest { add("foo", "bar") } val ua = AwsUserAgentMetadata(sdkMeta, apiMeta, osMetadata, langMeta, customMetadata = custom) - val expected = "aws-sdk-kotlin/1.2.3 api/test-service/1.2.3 os/linux/ubuntu-20.04 lang/kotlin/1.4.31 md/jvmVersion/1.11 md/foo/bar" + // FIXME re-enable once user agent strings can be longer + val expected = listOf( + "aws-sdk-kotlin/1.2.3", + // "api/test-service/1.2.3", + "os/linux/ubuntu-20.04", + "lang/kotlin/1.4.31", + // "md/jvmVersion/1.11", + "md/foo/bar", + ).joinToString(separator = " ") assertEquals(expected, ua.xAmzUserAgent) } @@ -82,7 +90,10 @@ class AwsUserAgentMetadataTest { ) testEnvironments.forEach { test -> val actual = loadAwsUserAgentMetadataFromEnvironment(test.provider, ApiMetadata("Test Service", "1.2.3")) - actual.xAmzUserAgent.shouldContain(test.expected) + + // FIXME re-enable once user agent strings can be longer + // actual.xAmzUserAgent.shouldContain(test.expected) + actual.xAmzUserAgent.shouldNotContain(test.expected) } } } diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/middleware/UserAgentTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/middleware/UserAgentTest.kt index cd1c479b8b2..0189951e3ee 100644 --- a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/middleware/UserAgentTest.kt +++ b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/middleware/UserAgentTest.kt @@ -60,7 +60,9 @@ class UserAgentTest { assertTrue(request.headers.contains(USER_AGENT)) assertTrue(request.headers.contains(X_AMZ_USER_AGENT)) assertEquals("aws-sdk-kotlin/1.2.3", request.headers[X_AMZ_USER_AGENT]) - assertTrue(request.headers[USER_AGENT]!!.startsWith("aws-sdk-kotlin/1.2.3 api/test-service/1.2.3")) + // FIXME re-enable once user agent strings can be longer + // assertTrue(request.headers[USER_AGENT]!!.startsWith("aws-sdk-kotlin/1.2.3 api/test-service/1.2.3")) + assertTrue(request.headers[USER_AGENT]!!.startsWith("aws-sdk-kotlin/1.2.3")) } @Test diff --git a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadataTest.kt b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadataTest.kt index 11a624959c3..4e55549fa9b 100644 --- a/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadataTest.kt +++ b/aws-runtime/aws-http/common/test/aws/sdk/kotlin/runtime/http/operation/CustomUserAgentMetadataTest.kt @@ -10,6 +10,7 @@ import aws.sdk.kotlin.runtime.http.loadAwsUserAgentMetadataFromEnvironment import aws.sdk.kotlin.runtime.testing.TestPlatformProvider import io.kotest.matchers.string.shouldContain import kotlin.test.Test +import kotlin.test.assertEquals class CustomUserAgentMetadataTest { @Test @@ -30,15 +31,45 @@ class CustomUserAgentMetadataTest { val actual = metadata.copy(customMetadata = customMetadata).xAmzUserAgent + // FIXME re-enable once user agent strings can be longer listOf( "md/foo/bar", "md/truthy", "md/falsey/false", - "cfg/retry-mode/standard", - "ft/s3-transfer/1.2.3", - "ft/waiter" + // "cfg/retry-mode/standard", + // "ft/s3-transfer/1.2.3", + // "ft/waiter", ).forEach { partial -> actual.shouldContain(partial) } } + + @Test + fun testFromEnvironment() { + val props = mapOf( + "irrelevantProp" to "shouldBeIgnored", + "aws.customMetadata" to "shouldBeIgnored", + "aws.customMetadata.foo" to "bar", + "aws.customMetadata.baz" to "qux", + "aws.customMetadata.priority" to "props", + ) + val envVars = mapOf( + "IRRELEVANT_PROP" to "shouldBeIgnored", + "AWS_CUSTOM_METADATA" to "shouldBeIgnored", + "AWS_CUSTOM_METADATA_oof" to "rab", + "AWS_CUSTOM_METADATA_zab" to "xuq", + "AWS_CUSTOM_METADATA_priority" to "envVars", + ) + val provider = TestPlatformProvider(env = envVars, props = props) + val metadata = CustomUserAgentMetadata.fromEnvironment(provider) + + val expected = mapOf( + "foo" to "bar", + "baz" to "qux", + "oof" to "rab", + "zab" to "xuq", + "priority" to "props", // System properties take precedence over env vars + ) + assertEquals(expected, metadata.extras) + } } diff --git a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt index 514c31da1ce..8a11a339529 100644 --- a/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt +++ b/aws-runtime/testing/common/src/aws/sdk/kotlin/runtime/testing/TestPlatformProvider.kt @@ -35,6 +35,8 @@ public class TestPlatformProvider( } override fun osInfo(): OperatingSystem = os + override fun getAllProperties(): Map = props override fun getProperty(key: String): String? = props[key] + override fun getAllEnvVars(): Map = env override fun getenv(key: String): String? = env[key] }