From 937959e364bcceddbd39e6067ac4035650f834ba Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 4 Mar 2022 12:46:58 -0500 Subject: [PATCH 1/5] test: add integration tests for generated event stream code --- build.gradle.kts | 3 +- gradle/jvm.gradle | 3 + settings.gradle.kts | 4 +- tests/codegen/event-stream/build.gradle.kts | 160 +++++++++++++++ .../event-stream-model-template.smithy | 73 +++++++ .../src/test/kotlin/EventStreamTests.kt | 188 ++++++++++++++++++ .../main/kotlin/aws/sdk/kotlin/test/Main.kt | 7 +- 7 files changed, 431 insertions(+), 7 deletions(-) create mode 100644 tests/codegen/event-stream/build.gradle.kts create mode 100644 tests/codegen/event-stream/event-stream-model-template.smithy create mode 100644 tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt diff --git a/build.gradle.kts b/build.gradle.kts index a3fbaa050b2..9b1bd61508a 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -145,7 +145,8 @@ val lintPaths = listOf( "dokka-aws/**/*.kt", "gradle/sdk-plugins/src/**/*.kt", "services/**/*.kt", - "!services/*/generated-src/**/*.kt" + "!services/*/generated-src/**/*.kt", + "tests/**/*.kt" ) tasks.register("ktlint") { diff --git a/gradle/jvm.gradle b/gradle/jvm.gradle index 42a254dbb4b..6f46b99ec4d 100644 --- a/gradle/jvm.gradle +++ b/gradle/jvm.gradle @@ -29,6 +29,9 @@ jvmTest { testLogging { events("passed", "skipped", "failed") showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL } useJUnitPlatform() diff --git a/settings.gradle.kts b/settings.gradle.kts index 6aed9aed72a..7c2eb80b101 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -42,8 +42,8 @@ include(":aws-runtime:protocols:aws-json-protocols") include(":aws-runtime:protocols:aws-xml-protocols") include(":aws-runtime:protocols:aws-event-stream") include(":aws-runtime:crt-util") -// include(":tests") -// include(":tests:codegen:event-stream") +include(":tests") +include(":tests:codegen:event-stream") // generated services fun File.isServiceDir(): Boolean { diff --git a/tests/codegen/event-stream/build.gradle.kts b/tests/codegen/event-stream/build.gradle.kts new file mode 100644 index 00000000000..3f9e57c9172 --- /dev/null +++ b/tests/codegen/event-stream/build.gradle.kts @@ -0,0 +1,160 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin +import software.amazon.smithy.gradle.tasks.SmithyBuild + +plugins { + kotlin("jvm") + id("aws.sdk.kotlin.codegen") +} + +description = "Event stream codegen integration test suite" + +val smithyVersion: String by project +dependencies { + implementation(project(":codegen:smithy-aws-kotlin-codegen")) +} + +data class EventStreamTest( + val projectionName: String, + val protocolName: String, +) { + val model: File + get() = buildDir.resolve("${projectionName}/model.smithy") +} + +val tests = listOf( + EventStreamTest("restJson1", "restJson1") +) + +fun fillInModel(output: File, protocolName: String) { + val template = file("event-stream-model-template.smithy") + val input = template.readText() + val opTraits = when(protocolName) { + "restJson1", "restXml" -> """@http(method: "POST", uri: "/test-eventstream", code: 200)""" + else -> "" + } + val replaced = input.replace("{protocol-name}", protocolName) + .replace("{op-traits}", opTraits) + output.parentFile.mkdirs() + output.writeText(replaced) +} + +val testServiceShapeId = "aws.sdk.kotlin.test.eventstream#TestService" +codegen { + tests.forEach { test -> + + projections.register(test.projectionName) { + imports = listOf(test.model.absolutePath) + transforms = listOf( + """ + { + "name": "includeServices", + "args": { + "services": ["$testServiceShapeId"] + } + } + """ + ) + + smithyKotlinPlugin { + serviceShapeId = testServiceShapeId + packageName = "aws.sdk.kotlin.test.eventstream.${test.protocolName.toLowerCase()}" + packageVersion = "1.0" + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.util.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi" + ) + } + } + } + } +} + +tasks.named("generateSmithyBuildConfig") { + doFirst { + tests.forEach { test -> fillInModel(test.model, test.protocolName) } + } +} + +val generateProjectionsTask = tasks.named("generateSmithyProjections") { + addCompileClasspath = true + + // ensure the generated tests use the same version of the runtime as the aws aws-runtime + val smithyKotlinVersion: String by project + doFirst { + System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinVersion) + } +} + + +val optinAnnotations = listOf( + "kotlin.RequiresOptIn", + "aws.smithy.kotlin.runtime.util.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi", +) +kotlin.sourceSets.all { + optinAnnotations.forEach { languageSettings.optIn(it) } +} + +kotlin.sourceSets.getByName("test") { + codegen.projections.forEach { + val projectedSrcDir = it.projectionRootDir.resolve("src/main/kotlin") + kotlin.srcDir(projectedSrcDir) + } +} + +tasks.withType{ + dependsOn(generateProjectionsTask) + // generated clients have quite a few warnings + kotlinOptions.allWarningsAsErrors = false +} + +tasks.test { + useJUnitPlatform() + testLogging { + events("passed", "skipped", "failed") + showStandardStreams = true + showStackTraces = true + showExceptions = true + exceptionFormat = org.gradle.api.tasks.testing.logging.TestExceptionFormat.FULL + } +} + +dependencies { + val coroutinesVersion: String by project + val smithyKotlinVersion: String by project + + implementation("org.jetbrains.kotlinx:kotlinx-coroutines-core:$coroutinesVersion") + + testImplementation(kotlin("test")) + testImplementation(kotlin("test-junit5")) + testImplementation("org.jetbrains.kotlinx:kotlinx-coroutines-test:$coroutinesVersion") + testImplementation("aws.smithy.kotlin:smithy-test:$smithyKotlinVersion") + + // have to manually add all the dependencies of the generated client(s) + // doing it this way (as opposed to doing what we do for protocol-tests) allows + // the tests to work without a publish to maven-local step at the cost of maintaining + // this set of dependencies manually + // <-- BEGIN GENERATED DEPENDENCY LIST --> + implementation(project(":aws-runtime:protocols:aws-event-stream")) + implementation(project(":aws-runtime:aws-http")) + implementation(project(":aws-runtime:protocols:aws-json-protocols")) + implementation(project(":aws-runtime:http-client-engine-crt")) // because it is hard coded in generated source currently + implementation(project(":aws-runtime:aws-config")) + implementation(project(":aws-runtime:aws-core")) + implementation(project(":aws-runtime:aws-endpoint")) + implementation(project(":aws-runtime:aws-types")) + implementation("aws.smithy.kotlin:runtime-core:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:http:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:io:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:serde:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:serde-json:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:utils:$smithyKotlinVersion") + // <-- END GENERATED DEPENDENCY LIST --> +} \ No newline at end of file diff --git a/tests/codegen/event-stream/event-stream-model-template.smithy b/tests/codegen/event-stream/event-stream-model-template.smithy new file mode 100644 index 00000000000..5564462c7a3 --- /dev/null +++ b/tests/codegen/event-stream/event-stream-model-template.smithy @@ -0,0 +1,73 @@ +namespace aws.sdk.kotlin.test.eventstream + +use aws.protocols#{protocol-name} +use aws.api#service + +@{protocol-name} +@service(sdkId: "EventStreamTest") +service TestService { version: "123", operations: [TestStreamOp] } + +{op-traits} +operation TestStreamOp { + input: TestStreamInputOutput, + output: TestStreamInputOutput, + errors: [SomeError], +} + +structure TestStreamInputOutput { @required value: TestStream } + +@error("client") +structure SomeError { + Message: String, +} + +union TestUnion { + Foo: String, + Bar: Integer, +} + +structure TestStruct { + someString: String, + someInt: Integer, +} + +structure MessageWithBlob { @eventPayload data: Blob } + +structure MessageWithString { @eventPayload data: String } + +structure MessageWithStruct { @eventPayload someStruct: TestStruct } + +structure MessageWithUnion { @eventPayload someUnion: TestUnion } + +structure MessageWithHeaders { + @eventHeader blob: Blob, + @eventHeader boolean: Boolean, + @eventHeader byte: Byte, + @eventHeader int: Integer, + @eventHeader long: Long, + @eventHeader short: Short, + @eventHeader string: String, + @eventHeader timestamp: Timestamp, +} +structure MessageWithHeaderAndPayload { + @eventHeader header: String, + @eventPayload payload: Blob, +} +structure MessageWithNoHeaderPayloadTraits { + someInt: Integer, + someString: String, +} + +@streaming +union TestStream { + MessageWithBlob: MessageWithBlob, + MessageWithString: MessageWithString, + MessageWithStruct: MessageWithStruct, + MessageWithUnion: MessageWithUnion, + MessageWithHeaders: MessageWithHeaders, + MessageWithHeaderAndPayload: MessageWithHeaderAndPayload, + MessageWithNoHeaderPayloadTraits: MessageWithNoHeaderPayloadTraits, + SomeError: SomeError, +} + + diff --git a/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt new file mode 100644 index 00000000000..de3e28dfd0a --- /dev/null +++ b/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt @@ -0,0 +1,188 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +import aws.sdk.kotlin.runtime.auth.credentials.Credentials +import aws.sdk.kotlin.runtime.auth.credentials.StaticCredentialsProvider +import aws.sdk.kotlin.runtime.auth.signing.AwsSignedBodyValue +import aws.sdk.kotlin.runtime.execution.AuthAttributes +import aws.sdk.kotlin.runtime.protocol.eventstream.* +import aws.sdk.kotlin.test.eventstream.restjson1.model.* +import aws.sdk.kotlin.test.eventstream.restjson1.transform.serializeTestStreamOpOperationBody +import aws.smithy.kotlin.runtime.client.ExecutionContext +import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.io.SdkByteBuffer +import aws.smithy.kotlin.runtime.smithy.test.assertJsonStringsEqual +import aws.smithy.kotlin.runtime.time.Instant +import kotlinx.coroutines.ExperimentalCoroutinesApi +import kotlinx.coroutines.flow.flowOf +import kotlinx.coroutines.flow.single +import kotlinx.coroutines.test.runTest +import kotlin.test.Test +import kotlin.test.assertEquals +import kotlin.test.assertIs + +/** + * Integration test suite that checks the generated event stream serialization and deserialization codegen + * works as expected. + */ +@OptIn(ExperimentalCoroutinesApi::class) +class EventStreamTests { + private suspend fun serializedMessage(event: TestStream): Message { + val req = TestStreamOpRequest { + value = flowOf(event) + } + + val testContext = ExecutionContext.build { + attributes[AuthAttributes.SigningRegion] = "us-east-2" + attributes[AuthAttributes.SigningService] = "test" + attributes[AuthAttributes.CredentialsProvider] = StaticCredentialsProvider( + Credentials("fake-access-key", "fake-secret-key") + ) + attributes[AuthAttributes.RequestSignature] = AwsSignedBodyValue.EMPTY_SHA256.encodeToByteArray() + } + + val body = serializeTestStreamOpOperationBody(testContext, req) + assertIs< HttpBody.Streaming>(body) + + val signedMessage = decodeFrames(body.readFrom()).single() + val buffer = SdkByteBuffer.of(signedMessage.payload) + buffer.advance(signedMessage.payload.size.toULong()) + return Message.decode(buffer) + } + + @Test + fun testSerializeMessageWithBlob() = runTest { + val event = TestStream.MessageWithBlob(MessageWithBlob { data = "hello from Kotlin".encodeToByteArray() }) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithBlob", headers[":event-type"]?.expectString()) + assertEquals("application/octet-stream", headers[":content-type"]?.expectString()) + assertEquals("hello from Kotlin", message.payload.decodeToString()) + } + + @Test + fun testSerializeMessageWithString() = runTest { + val event = TestStream.MessageWithString(MessageWithString { data = "hello from Kotlin" }) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithString", headers[":event-type"]?.expectString()) + assertEquals("text/plain", headers[":content-type"]?.expectString()) + assertEquals("hello from Kotlin", message.payload.decodeToString()) + } + + @Test + fun testSerializeMessageWithStruct() = runTest { + val event = TestStream.MessageWithStruct( + MessageWithStruct { + someStruct { + someInt = 2 + someString = "hello struct!" + } + } + ) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithStruct", headers[":event-type"]?.expectString()) + assertEquals("application/json", headers[":content-type"]?.expectString()) + + val expectedBody = """{"someInt":2,"someString":"hello struct!"}""" + assertJsonStringsEqual(expectedBody, message.payload.decodeToString()) + } + + @Test + fun testSerializeMessageWithUnion() = runTest { + val event = TestStream.MessageWithUnion( + MessageWithUnion { + someUnion = TestUnion.Foo("a lep is a ball") + } + ) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithUnion", headers[":event-type"]?.expectString()) + assertEquals("application/json", headers[":content-type"]?.expectString()) + + val expectedBody = """{"Foo":"a lep is a ball"}""" + assertJsonStringsEqual(expectedBody, message.payload.decodeToString()) + } + + @Test + fun testSerializeMessageWithHeaders() = runTest { + val event = TestStream.MessageWithHeaders( + MessageWithHeaders { + blob = "blobby".encodeToByteArray() + boolean = true + byte = 55 + int = 100_000 + short = 16_000 + long = 9_000_000_000L + string = "a tay is a hammer" + timestamp = Instant.fromEpochSeconds(5) + } + ) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithHeaders", headers[":event-type"]?.expectString()) + assertEquals("blobby", headers["blob"]?.expectByteArray()?.decodeToString()) + assertEquals(true, headers["boolean"]?.expectBool()) + assertEquals(55, headers["byte"]?.expectByte()) + assertEquals(16_000, headers["short"]?.expectInt16()) + assertEquals(100_000, headers["int"]?.expectInt32()) + assertEquals(9_000_000_000L, headers["long"]?.expectInt64()) + assertEquals("a tay is a hammer", headers["string"]?.expectString()) + assertEquals(Instant.fromEpochSeconds(5), headers["timestamp"]?.expectTimestamp()) + } + + @Test + fun testSerializeMessageWithHeaderAndPayload() = runTest { + val event = TestStream.MessageWithHeaderAndPayload( + MessageWithHeaderAndPayload { + header = "a korf is a tiger" + payload = "remember a korf is a tiger".encodeToByteArray() + } + ) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithHeaderAndPayload", headers[":event-type"]?.expectString()) + assertEquals("a korf is a tiger", headers["header"]?.expectString()) + assertEquals("remember a korf is a tiger", message.payload.decodeToString()) + } + + @Test + fun testSerializeMessageWithNoTraits() = runTest { + val event = TestStream.MessageWithNoHeaderPayloadTraits( + MessageWithNoHeaderPayloadTraits { + someInt = 2 + someString = "a flix is comb" + } + ) + + val message = serializedMessage(event) + + val headers = message.headers.associate { it.name to it.value } + assertEquals("event", headers[":message-type"]?.expectString()) + assertEquals("MessageWithNoHeaderPayloadTraits", headers[":event-type"]?.expectString()) + assertEquals("application/json", headers[":content-type"]?.expectString()) + val expectedBody = """{"someInt":2,"someString":"a flix is comb"}""" + assertJsonStringsEqual(expectedBody, message.payload.decodeToString()) + } +} diff --git a/tests/integration-tests/ecs-credentials/app/src/main/kotlin/aws/sdk/kotlin/test/Main.kt b/tests/integration-tests/ecs-credentials/app/src/main/kotlin/aws/sdk/kotlin/test/Main.kt index b63537199a2..2d98f494de9 100644 --- a/tests/integration-tests/ecs-credentials/app/src/main/kotlin/aws/sdk/kotlin/test/Main.kt +++ b/tests/integration-tests/ecs-credentials/app/src/main/kotlin/aws/sdk/kotlin/test/Main.kt @@ -4,14 +4,13 @@ */ package aws.sdk.kotlin.test -import kotlinx.coroutines.runBlocking import aws.sdk.kotlin.runtime.auth.credentials.EcsCredentialsProvider import aws.sdk.kotlin.services.sts.StsClient - +import kotlinx.coroutines.runBlocking fun main(): Unit = runBlocking { - StsClient{ + StsClient { region = "us-east-2" credentialsProvider = EcsCredentialsProvider() }.use { client -> @@ -21,4 +20,4 @@ fun main(): Unit = runBlocking { println("UserID: ${resp.userId}") println("ARN: ${resp.arn}") } -} \ No newline at end of file +} From db3245a83b09c35152a20eaa8e98b63b31054485 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 4 Mar 2022 13:25:40 -0500 Subject: [PATCH 2/5] add deserialize tests --- .../src/test/kotlin/EventStreamTests.kt | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt b/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt index de3e28dfd0a..1ac7ba0ea3e 100644 --- a/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt +++ b/tests/codegen/event-stream/src/test/kotlin/EventStreamTests.kt @@ -9,10 +9,13 @@ import aws.sdk.kotlin.runtime.auth.signing.AwsSignedBodyValue import aws.sdk.kotlin.runtime.execution.AuthAttributes import aws.sdk.kotlin.runtime.protocol.eventstream.* import aws.sdk.kotlin.test.eventstream.restjson1.model.* +import aws.sdk.kotlin.test.eventstream.restjson1.transform.deserializeTestStreamOpOperationBody import aws.sdk.kotlin.test.eventstream.restjson1.transform.serializeTestStreamOpOperationBody import aws.smithy.kotlin.runtime.client.ExecutionContext import aws.smithy.kotlin.runtime.http.HttpBody +import aws.smithy.kotlin.runtime.http.content.ByteArrayContent import aws.smithy.kotlin.runtime.io.SdkByteBuffer +import aws.smithy.kotlin.runtime.io.bytes import aws.smithy.kotlin.runtime.smithy.test.assertJsonStringsEqual import aws.smithy.kotlin.runtime.time.Instant import kotlinx.coroutines.ExperimentalCoroutinesApi @@ -44,7 +47,7 @@ class EventStreamTests { } val body = serializeTestStreamOpOperationBody(testContext, req) - assertIs< HttpBody.Streaming>(body) + assertIs(body) val signedMessage = decodeFrames(body.readFrom()).single() val buffer = SdkByteBuffer.of(signedMessage.payload) @@ -52,6 +55,20 @@ class EventStreamTests { return Message.decode(buffer) } + private suspend fun deserializedEvent(message: Message): TestStream { + val buffer = SdkByteBuffer(0U) + message.encode(buffer) + val body = ByteArrayContent(buffer.bytes()) + val builder = TestStreamOpResponse.Builder() + + deserializeTestStreamOpOperationBody(builder, body) + + val resp = builder.build() + checkNotNull(resp.value) + + return resp.value.single() + } + @Test fun testSerializeMessageWithBlob() = runTest { val event = TestStream.MessageWithBlob(MessageWithBlob { data = "hello from Kotlin".encodeToByteArray() }) @@ -63,6 +80,10 @@ class EventStreamTests { assertEquals("MessageWithBlob", headers[":event-type"]?.expectString()) assertEquals("application/octet-stream", headers[":content-type"]?.expectString()) assertEquals("hello from Kotlin", message.payload.decodeToString()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } @Test @@ -76,6 +97,10 @@ class EventStreamTests { assertEquals("MessageWithString", headers[":event-type"]?.expectString()) assertEquals("text/plain", headers[":content-type"]?.expectString()) assertEquals("hello from Kotlin", message.payload.decodeToString()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } @Test @@ -98,6 +123,10 @@ class EventStreamTests { val expectedBody = """{"someInt":2,"someString":"hello struct!"}""" assertJsonStringsEqual(expectedBody, message.payload.decodeToString()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } @Test @@ -117,6 +146,10 @@ class EventStreamTests { val expectedBody = """{"Foo":"a lep is a ball"}""" assertJsonStringsEqual(expectedBody, message.payload.decodeToString()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } @Test @@ -147,6 +180,10 @@ class EventStreamTests { assertEquals(9_000_000_000L, headers["long"]?.expectInt64()) assertEquals("a tay is a hammer", headers["string"]?.expectString()) assertEquals(Instant.fromEpochSeconds(5), headers["timestamp"]?.expectTimestamp()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } @Test @@ -165,6 +202,10 @@ class EventStreamTests { assertEquals("MessageWithHeaderAndPayload", headers[":event-type"]?.expectString()) assertEquals("a korf is a tiger", headers["header"]?.expectString()) assertEquals("remember a korf is a tiger", message.payload.decodeToString()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } @Test @@ -184,5 +225,9 @@ class EventStreamTests { assertEquals("application/json", headers[":content-type"]?.expectString()) val expectedBody = """{"someInt":2,"someString":"a flix is comb"}""" assertJsonStringsEqual(expectedBody, message.payload.decodeToString()) + + val deserialized = deserializedEvent(message) + assertIs(deserialized) + assertEquals(event, deserialized) } } From 047becea77766863f9b5e3701ce58248d281807d Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 7 Mar 2022 09:24:54 -0500 Subject: [PATCH 3/5] add todo --- .../protocols/eventstream/EventStreamSerializerGenerator.kt | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt index 147cfc952c2..7f371c76b04 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt @@ -100,6 +100,10 @@ class EventStreamSerializerGenerator( addStringHeader(":message-type", "event") withBlock("when(input) {", "}") { + // FIXME - this doesn't handle the case when @eventPayload isn't used! + // 2 cases, (1) no header and no payload we can re-use the payloadSerializer; (2) no eventPayload, remaining "unbound" members + // Perhaps: can we have payloadSerializer/Deserializer haven an "inline" option? or could be automatic. The concern is + // overlap if you need to serialize a shape w/all members in one case vs just some members in another. Naming is then hard. streamShape.filterEventStreamErrors(ctx.model) .forEach { member -> withBlock( From 08536a7cdd1ab570014d0853a672be8b3bc64ebe Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 7 Mar 2022 14:48:10 -0500 Subject: [PATCH 4/5] generate payload serde for unbound members --- .../sdk/kotlin/codegen/protocols/RestXml.kt | 16 +++++++++--- .../core/QueryHttpBindingProtocolGenerator.kt | 26 ++++++++++++------- .../eventstream/EventStreamParserGenerator.kt | 8 +++--- .../EventStreamSerializerGenerator.kt | 26 ++++++++++++------- .../AwsHttpBindingProtocolGeneratorTest.kt | 12 +++++++-- 5 files changed, 59 insertions(+), 29 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/RestXml.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/RestXml.kt index 2d1e199de12..fcd122faa17 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/RestXml.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/RestXml.kt @@ -62,13 +62,17 @@ class RestXmlParserGenerator( writer: KotlinWriter ): XmlSerdeDescriptorGenerator = RestXmlSerdeDescriptorGenerator(ctx.toRenderingContext(protocolGenerator, shape, writer), members) - override fun payloadDeserializer(ctx: ProtocolGenerator.GenerationContext, shape: Shape): Symbol { + override fun payloadDeserializer( + ctx: ProtocolGenerator.GenerationContext, + shape: Shape, + members: Collection? + ): Symbol { return if (shape.hasTrait() && shape is MemberShape) { // can't delegate, have to generate a dedicated deserializer because the member xml name is different // from the name of the target shape explicitBoundStructureDeserializer(ctx, shape) } else { - super.payloadDeserializer(ctx, shape) + super.payloadDeserializer(ctx, shape, members) } } @@ -116,13 +120,17 @@ class RestXmlSerializerGenerator( writer: KotlinWriter ): XmlSerdeDescriptorGenerator = RestXmlSerdeDescriptorGenerator(ctx.toRenderingContext(protocolGenerator, shape, writer), members) - override fun payloadSerializer(ctx: ProtocolGenerator.GenerationContext, shape: Shape): Symbol { + override fun payloadSerializer( + ctx: ProtocolGenerator.GenerationContext, + shape: Shape, + members: Collection? + ): Symbol { return if (shape.hasTrait() && shape is MemberShape) { // can't delegate, have to generate a dedicated serializer because the member xml name is different // from the name of the target shape explicitBoundStructureSerializer(ctx, shape) } else { - super.payloadSerializer(ctx, shape) + super.payloadSerializer(ctx, shape, members) } } diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/core/QueryHttpBindingProtocolGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/core/QueryHttpBindingProtocolGenerator.kt index 02805326b28..b832e997cfa 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/core/QueryHttpBindingProtocolGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/core/QueryHttpBindingProtocolGenerator.kt @@ -132,11 +132,14 @@ abstract class AbstractQueryFormUrlSerializerGenerator( writer.write("return serializer.toByteArray()") } - private fun documentSerializer(ctx: ProtocolGenerator.GenerationContext, shape: Shape): Symbol { + private fun documentSerializer( + ctx: ProtocolGenerator.GenerationContext, + shape: Shape, + members: Collection = shape.members() + ): Symbol { val symbol = ctx.symbolProvider.toSymbol(shape) - return symbol.documentSerializer(ctx.settings) { writer -> - val fnName = symbol.documentSerializerName() - writer.openBlock("internal fun #L(serializer: #T, input: #T) {", fnName, RuntimeTypes.Serde.Serializer, symbol) + return shape.documentSerializer(ctx.settings, symbol, members) { writer -> + writer.openBlock("internal fun #identifier.name:L(serializer: #T, input: #T) {", RuntimeTypes.Serde.Serializer, symbol) .call { renderSerializerBody(ctx, shape, shape.members().toList(), writer) } @@ -159,15 +162,20 @@ abstract class AbstractQueryFormUrlSerializerGenerator( } } - override fun payloadSerializer(ctx: ProtocolGenerator.GenerationContext, shape: Shape): Symbol { + override fun payloadSerializer( + ctx: ProtocolGenerator.GenerationContext, + shape: Shape, + members: Collection? + ): Symbol { // re-use document serializer (for the target shape!) val target = shape.targetOrSelf(ctx.model) val symbol = ctx.symbolProvider.toSymbol(shape) - val serializeFn = documentSerializer(ctx, target) - val fnName = symbol.payloadSerializerName() - return symbol.payloadSerializer(ctx.settings) { writer -> + val forMembers = members ?: target.members() + + val serializeFn = documentSerializer(ctx, target, forMembers) + return target.payloadSerializer(ctx.settings, symbol, forMembers) { writer -> addNestedDocumentSerializers(ctx, target, writer) - writer.withBlock("internal fun #L(input: #T): ByteArray {", "}", fnName, symbol) { + writer.withBlock("internal fun #identifier.name:L(input: #T): ByteArray {", "}", symbol) { write("val serializer = #T()", RuntimeTypes.Serde.SerdeFormUrl.FormUrlSerializer) write("#T(serializer, input)", serializeFn) write("return serializer.toByteArray()") diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt index 5f22b115a60..692fb181caf 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt @@ -110,6 +110,7 @@ class EventStreamParserGenerator( val eventHeaderBindings = variant.members().filter { it.hasTrait() } val eventPayloadBinding = variant.members().firstOrNull { it.hasTrait() } + val unbound = variant.members().filterNot { it.hasTrait() || it.hasTrait() } if (eventHeaderBindings.isEmpty() && eventPayloadBinding == null) { // the entire variant can be deserialized from the payload @@ -148,11 +149,10 @@ class EventStreamParserGenerator( if (eventPayloadBinding != null) { renderDeserializeExplicitEventPayloadMember(ctx, eventPayloadBinding, writer) } else { - val members = variant.members().filterNot { it.hasTrait() } - if (members.isNotEmpty()) { + if (unbound.isNotEmpty()) { // all remaining members are bound to payload (but not explicitly bound via @eventPayload) - // use the operation body deserializer - TODO("render unbound event stream payload members") + // FIXME - this would require us to generate a "partial" deserializer like we do for where the builder is passed in + TODO("support for unbound event stream members is not implemented") } } diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt index 7f371c76b04..7d47e7e48d3 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt @@ -89,7 +89,7 @@ class EventStreamSerializerGenerator( definitionFile = "${op.serializerName()}.kt" renderBy = { writer -> - // FIXME - make internal and share across operations? + // TODO - make internal and share across operations? writer.withBlock( "private fun #L(input: #T): #T = #T {", "}", fnName, @@ -100,10 +100,6 @@ class EventStreamSerializerGenerator( addStringHeader(":message-type", "event") withBlock("when(input) {", "}") { - // FIXME - this doesn't handle the case when @eventPayload isn't used! - // 2 cases, (1) no header and no payload we can re-use the payloadSerializer; (2) no eventPayload, remaining "unbound" members - // Perhaps: can we have payloadSerializer/Deserializer haven an "inline" option? or could be automatic. The concern is - // overlap if you need to serialize a shape w/all members in one case vs just some members in another. Naming is then hard. streamShape.filterEventStreamErrors(ctx.model) .forEach { member -> withBlock( @@ -112,11 +108,21 @@ class EventStreamSerializerGenerator( member.unionVariantName() ) { addStringHeader(":event-type", member.memberName) - val target = ctx.model.expectShape(member.target) - target.members().forEach { targetMember -> - when { - targetMember.hasTrait() -> renderSerializeEventHeader(ctx, targetMember, writer) - targetMember.hasTrait() -> renderSerializeEventPayload(ctx, targetMember, writer) + val variant = ctx.model.expectShape(member.target) + + val eventHeaderBindings = variant.members().filter { it.hasTrait() } + val eventPayloadBinding = variant.members().firstOrNull { it.hasTrait() } + val unbound = variant.members().filterNot { it.hasTrait() || it.hasTrait() } + + eventHeaderBindings.forEach { renderSerializeEventHeader(ctx, it, writer) } + + when { + eventPayloadBinding != null -> renderSerializeEventPayload(ctx, eventPayloadBinding, writer) + unbound.isNotEmpty() -> { + writer.addStringHeader(":content-type", payloadContentType) + // get a payload serializer for the given members of the variant + val serializeFn = sdg.payloadSerializer(ctx, variant, unbound) + writer.write("payload = #T(input.value)", serializeFn) } } } diff --git a/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/protocols/core/AwsHttpBindingProtocolGeneratorTest.kt b/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/protocols/core/AwsHttpBindingProtocolGeneratorTest.kt index 68d95ac4480..b010baf21c2 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/protocols/core/AwsHttpBindingProtocolGeneratorTest.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/test/kotlin/aws/sdk/kotlin/codegen/protocols/core/AwsHttpBindingProtocolGeneratorTest.kt @@ -101,7 +101,11 @@ class AwsHttpBindingProtocolGeneratorTest { error("Unneeded for test") } - override fun payloadDeserializer(ctx: ProtocolGenerator.GenerationContext, shape: Shape): Symbol { + override fun payloadDeserializer( + ctx: ProtocolGenerator.GenerationContext, + shape: Shape, + members: Collection? + ): Symbol { error("Unneeded for test") } } @@ -112,7 +116,11 @@ class AwsHttpBindingProtocolGeneratorTest { error("Unneeded for test") } - override fun payloadSerializer(ctx: ProtocolGenerator.GenerationContext, shape: Shape): Symbol { + override fun payloadSerializer( + ctx: ProtocolGenerator.GenerationContext, + shape: Shape, + members: Collection? + ): Symbol { error("Unneeded for test") } } From c70623d7c3bc3ecbd0692b15ef0dbf7a582795c2 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 7 Mar 2022 15:03:19 -0500 Subject: [PATCH 5/5] cleanup --- .../eventstream/EventStreamParserGenerator.kt | 15 +++++++-------- .../eventstream/EventStreamSerializerGenerator.kt | 1 - 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt index 692fb181caf..485b2ee8c2d 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamParserGenerator.kt @@ -37,7 +37,6 @@ class EventStreamParserGenerator( * ``` */ fun responseHandler(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Symbol = - // FIXME - don't use the body deserializer name since we may need to re-use it (albeit with a different signature, we should still be more explicit than this) op.bodyDeserializer(ctx.settings) { writer -> val outputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.output.get())) // we have access to the builder for the output type and the full HttpBody @@ -118,7 +117,7 @@ class EventStreamParserGenerator( writer.write("val e = #T(message.payload)", payloadDeserializeFn) } else { val variantSymbol = ctx.symbolProvider.toSymbol(variant) - writer.write("val builder = #T.Builder()", variantSymbol) + writer.write("val eb = #T.Builder()", variantSymbol) // render members bound to header eventHeaderBindings.forEach { hdrBinding -> @@ -143,7 +142,7 @@ class EventStreamParserGenerator( } else { "" } - writer.write("builder.#L = message.headers.find { it.name == #S }?.value?.#T()$defaultValuePostfix", hdrBinding.defaultName(), hdrBinding.memberName, conversionFn) + writer.write("eb.#L = message.headers.find { it.name == #S }?.value?.#T()$defaultValuePostfix", hdrBinding.defaultName(), hdrBinding.memberName, conversionFn) } if (eventPayloadBinding != null) { @@ -156,7 +155,7 @@ class EventStreamParserGenerator( } } - writer.write("val e = builder.build()") + writer.write("val e = eb.build()") } writer.write("#T.#L(e)", unionSymbol, member.unionVariantName()) @@ -167,15 +166,15 @@ class EventStreamParserGenerator( member: MemberShape, writer: KotlinWriter ) { - // FIXME - check content type for blob and string + // TODO - check content type for blob and string // structure > :test(member > :test(blob, string, structure, union)) val target = ctx.model.expectShape(member.target) when (target.type) { - ShapeType.BLOB -> writer.write("builder.#L = message.payload", member.defaultName()) - ShapeType.STRING -> writer.write("builder.#L = message.payload?.decodeToString()", member.defaultName()) + ShapeType.BLOB -> writer.write("eb.#L = message.payload", member.defaultName()) + ShapeType.STRING -> writer.write("eb.#L = message.payload.decodeToString()", member.defaultName()) ShapeType.STRUCTURE, ShapeType.UNION -> { val payloadDeserializeFn = sdg.payloadDeserializer(ctx, member) - writer.write("builder.#L = #T(message.payload)", member.defaultName(), payloadDeserializeFn) + writer.write("eb.#L = #T(message.payload)", member.defaultName(), payloadDeserializeFn) } else -> throw CodegenException("unsupported shape type `${target.type}` for target: $target; expected blob, string, structure, or union for eventPayload member: $member") } diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt index 7d47e7e48d3..f45e2a518d2 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/protocols/eventstream/EventStreamSerializerGenerator.kt @@ -37,7 +37,6 @@ class EventStreamSerializerGenerator( * ``` */ fun requestHandler(ctx: ProtocolGenerator.GenerationContext, op: OperationShape): Symbol = - // FIXME - don't use the body serializer name since we may need to re-use it (albeit with a different signature, we should still be more explicit than this) op.bodySerializer(ctx.settings) { writer -> val inputSymbol = ctx.symbolProvider.toSymbol(ctx.model.expectShape(op.input.get())) writer.withBlock(