diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CodegenUtils.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CodegenUtils.java index b675aa55521..c6a582ffc55 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CodegenUtils.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CodegenUtils.java @@ -28,6 +28,7 @@ import software.amazon.smithy.model.shapes.Shape; import software.amazon.smithy.model.shapes.StructureShape; import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.typescript.codegen.integration.AddSdkStreamMixinDependency; import software.amazon.smithy.utils.SmithyUnstableApi; /** @@ -75,12 +76,14 @@ public static String getOperationSerializerContextType( /** * Get context type for command deserializer function. + * @param settings The TypeScript settings * @param writer The code writer. * @param model The model for the service containing the given command. * @param operation The operation shape for given command. * @return The TypeScript type for the deserializer context */ public static String getOperationDeserializerContextType( + TypeScriptSettings settings, TypeScriptWriter writer, Model model, OperationShape operation @@ -90,9 +93,15 @@ public static String getOperationDeserializerContextType( // If event stream trait exists, add corresponding serde context type to the intersection type. EventStreamIndex eventStreamIndex = EventStreamIndex.of(model); if (eventStreamIndex.getOutputInfo(operation).isPresent()) { - writer.addImport("EventStreamSerdeContext", "__EventStreamSerdeContext", "@aws-sdk/types"); + writer.addImport("EventStreamSerdeContext", "__EventStreamSerdeContext", + TypeScriptDependency.AWS_SDK_TYPES.packageName); contextInterfaceList.add("__EventStreamSerdeContext"); } + if (AddSdkStreamMixinDependency.hasStreamingBlobDeser(settings, model, operation)) { + writer.addImport("SdkStreamSerdeContext", "__SdkStreamSerdeContext", + TypeScriptDependency.AWS_SDK_TYPES.packageName); + contextInterfaceList.add("__SdkStreamSerdeContext"); + } return String.join(" & ", contextInterfaceList); } @@ -108,7 +117,7 @@ static List getBlobStreamingMembers(Model model, StructureShape sha return shape.getAllMembers().values().stream() .filter(memberShape -> { // Streaming blobs need to have their types modified - // See `writeStreamingMemberType` + // See `writeClientCommandStreamingInputType` Shape target = model.expectShape(memberShape.getTarget()); return target.isBlobShape() && target.hasTrait(StreamingTrait.class); }) @@ -116,12 +125,14 @@ static List getBlobStreamingMembers(Model model, StructureShape sha } /** - * Ease the input streaming member restriction so that users don't need to construct a stream every time. - * This type decoration is allowed in Smithy because it makes input type more permissive than output type - * for the same member. + * Generate the type of the command input of the client sdk given the streaming blob + * member of the shape. The generated type eases the streaming member requirement so that users don't need to + * construct a stream every time. + * This type decoration is allowed in Smithy because it makes, for the same member, the type to be serialized is + * more permissive than the type to be deserialized. * Refer here for more rationales: https://github.com/aws/aws-sdk-js-v3/issues/843 */ - static void writeStreamingMemberType( + static void writeClientCommandStreamingInputType( TypeScriptWriter writer, Symbol containerSymbol, String typeName, @@ -129,16 +140,44 @@ static void writeStreamingMemberType( ) { String memberName = streamingMember.getMemberName(); String optionalSuffix = streamingMember.isRequired() ? "" : "?"; - writer.openBlock("type $LType = Omit<$T, $S> & {", "};", typeName, containerSymbol, memberName, () -> { - writer.writeDocs(String.format("For *`%1$s[\"%2$s\"]`*, see {@link %1$s.%2$s}.", - containerSymbol.getName(), memberName)); - writer.write("$1L$2L: $3T[$1S]|string|Uint8Array|Buffer;", memberName, optionalSuffix, containerSymbol); + writer.openBlock("type $LType = Omit<$T, $S> & {", "};", typeName, + containerSymbol, memberName, () -> { + writer.writeDocs(String.format("For *`%1$s[\"%2$s\"]`*, see {@link %1$s.%2$s}.", + containerSymbol.getName(), memberName)); + writer.write("$1L$2L: $3T[$1S]|string|Uint8Array|Buffer;", memberName, optionalSuffix, + containerSymbol); }); writer.writeDocs(String.format("This interface extends from `%1$s` interface. There are more parameters than" + " `%2$s` defined in {@link %1$s}", containerSymbol.getName(), memberName)); writer.write("export interface $1L extends $1LType {}", typeName); } + /** + * Generate the type of the command output of the client sdk given the streaming blob + * member of the shape. The type marks the streaming blob member to contain the utility methods to transform the + * stream to string, buffer or WHATWG stream API. + */ + static void writeClientCommandStreamingOutputType( + TypeScriptWriter writer, + Symbol containerSymbol, + String typeName, + MemberShape streamingMember + ) { + String memberName = streamingMember.getMemberName(); + String optionalSuffix = streamingMember.isRequired() ? "" : "?"; + writer.addImport("MetadataBearer", "__MetadataBearer", TypeScriptDependency.AWS_SDK_TYPES.packageName); + writer.addImport("SdkStream", "__SdkStream", TypeScriptDependency.AWS_SDK_TYPES.packageName); + writer.addImport("WithSdkStreamMixin", "__WithSdkStreamMixin", TypeScriptDependency.AWS_SDK_TYPES.packageName); + + + writer.write( + "export interface $L extends __WithSdkStreamMixin<$T, $S>, __MetadataBearer {}", + typeName, + containerSymbol, + memberName + ); + } + /** * Returns the list of function parameter key-value pairs to be written for * provided parameters map. diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CommandGenerator.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CommandGenerator.java index 37628672b15..9749f0de3d3 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CommandGenerator.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/CommandGenerator.java @@ -16,7 +16,8 @@ package software.amazon.smithy.typescript.codegen; import static software.amazon.smithy.typescript.codegen.CodegenUtils.getBlobStreamingMembers; -import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeStreamingMemberType; +import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeClientCommandStreamingInputType; +import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeClientCommandStreamingOutputType; import java.nio.file.Paths; import java.util.List; @@ -314,7 +315,8 @@ private void writeInputType(String typeName, Optional inputShape if (blobStreamingMembers.isEmpty()) { writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(input)); } else { - writeStreamingMemberType(writer, symbolProvider.toSymbol(input), typeName, blobStreamingMembers.get(0)); + writeClientCommandStreamingInputType(writer, symbolProvider.toSymbol(input), typeName, + blobStreamingMembers.get(0)); } } else { // If the input is non-existent, then use an empty object. @@ -327,8 +329,15 @@ private void writeOutputType(String typeName, Optional outputSha // to a defined output shape. writer.addImport("MetadataBearer", "__MetadataBearer", TypeScriptDependency.AWS_SDK_TYPES.packageName); if (outputShape.isPresent()) { - writer.write("export interface $L extends $T, __MetadataBearer {}", - typeName, symbolProvider.toSymbol(outputShape.get())); + StructureShape output = outputShape.get(); + List blobStreamingMembers = getBlobStreamingMembers(model, output); + if (blobStreamingMembers.isEmpty()) { + writer.write("export interface $L extends $T, __MetadataBearer {}", + typeName, symbolProvider.toSymbol(outputShape.get())); + } else { + writeClientCommandStreamingOutputType(writer, symbolProvider.toSymbol(output), typeName, + blobStreamingMembers.get(0)); + } } else { writer.write("export interface $L extends __MetadataBearer {}", typeName); } @@ -371,7 +380,8 @@ private void writeSerde() { .write("private deserialize(") .indent() .write("output: $T,", applicationProtocol.getResponseType()) - .write("context: $L", CodegenUtils.getOperationDeserializerContextType(writer, model, operation)) + .write("context: $L", + CodegenUtils.getOperationDeserializerContextType(settings, writer, model, operation)) .dedent() .openBlock("): Promise<$T> {", "}", outputType, () -> writeSerdeDispatcher(false)) .write(""); diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/ServerCommandGenerator.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/ServerCommandGenerator.java index 5e2d7f94bcf..858cdba8ba8 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/ServerCommandGenerator.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/ServerCommandGenerator.java @@ -15,9 +15,6 @@ package software.amazon.smithy.typescript.codegen; -import static software.amazon.smithy.typescript.codegen.CodegenUtils.getBlobStreamingMembers; -import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeStreamingMemberType; - import java.nio.file.Paths; import java.util.Collections; import java.util.Iterator; @@ -31,7 +28,6 @@ import software.amazon.smithy.model.Model; import software.amazon.smithy.model.knowledge.OperationIndex; import software.amazon.smithy.model.knowledge.TopDownIndex; -import software.amazon.smithy.model.shapes.MemberShape; import software.amazon.smithy.model.shapes.OperationShape; import software.amazon.smithy.model.shapes.ServiceShape; import software.amazon.smithy.model.shapes.StructureShape; @@ -98,16 +94,10 @@ private void addInputAndOutputTypes() { writer.write(""); } - // TODO: Flip these so that metadata is attached to input and streaming customization is attached to output. private void writeInputType(String typeName, Optional inputShape) { if (inputShape.isPresent()) { StructureShape input = inputShape.get(); - List blobStreamingMembers = getBlobStreamingMembers(model, input); - if (blobStreamingMembers.isEmpty()) { - writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(input)); - } else { - writeStreamingMemberType(writer, symbolProvider.toSymbol(input), typeName, blobStreamingMembers.get(0)); - } + writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(inputShape.get())); renderNamespace(typeName, input); } else { // If the input is non-existent, then use an empty object. diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/TypeScriptDependency.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/TypeScriptDependency.java index 9112153ff99..23367175294 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/TypeScriptDependency.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/TypeScriptDependency.java @@ -100,6 +100,10 @@ public enum TypeScriptDependency implements SymbolDependencyContainer { XML_PARSER("dependencies", "fast-xml-parser", "4.0.11", false), HTML_ENTITIES("dependencies", "entities", "2.2.0", false), + // Conditionally added when streaming blob response payload exists. + UTIL_STREAM_NODE("dependencies", "@aws-sdk/util-stream-node", false), + UTIL_STREAM_BROWSER("dependencies", "@aws-sdk/util-stream-browser", false), + // Server dependency for SSDKs SERVER_COMMON("dependencies", "@aws-smithy/server-common", "1.0.0-alpha.6", false); diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddDefaultsModeDependency.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddDefaultsModeDependency.java index c1244e98daf..70044d12ca5 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddDefaultsModeDependency.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddDefaultsModeDependency.java @@ -41,6 +41,6 @@ public void addConfigInterfaceFields( writer.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName); writer.writeDocs("The {@link DefaultsMode} that will be used to determine how certain default configuration " + "options are resolved in the SDK."); - writer.write("defaultsMode?: DefaultsMode | Provider;"); + writer.write("defaultsMode?: DefaultsMode | Provider;\n"); } } diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddSdkStreamMixinDependency.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddSdkStreamMixinDependency.java new file mode 100644 index 00000000000..06b908f3b27 --- /dev/null +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/AddSdkStreamMixinDependency.java @@ -0,0 +1,118 @@ +/* + * Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"). + * You may not use this file except in compliance with the License. + * A copy of the License is located at + * + * http://aws.amazon.com/apache2.0 + * + * or in the "license" file accompanying this file. This file is distributed + * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either + * express or implied. See the License for the specific language governing + * permissions and limitations under the License. + */ + +package software.amazon.smithy.typescript.codegen.integration; + +import java.util.Collections; +import java.util.Map; +import java.util.Set; +import java.util.function.Consumer; +import software.amazon.smithy.codegen.core.SymbolProvider; +import software.amazon.smithy.model.Model; +import software.amazon.smithy.model.knowledge.TopDownIndex; +import software.amazon.smithy.model.shapes.BlobShape; +import software.amazon.smithy.model.shapes.MemberShape; +import software.amazon.smithy.model.shapes.OperationShape; +import software.amazon.smithy.model.shapes.ServiceShape; +import software.amazon.smithy.model.shapes.Shape; +import software.amazon.smithy.model.shapes.StructureShape; +import software.amazon.smithy.model.traits.StreamingTrait; +import software.amazon.smithy.typescript.codegen.LanguageTarget; +import software.amazon.smithy.typescript.codegen.TypeScriptDependency; +import software.amazon.smithy.typescript.codegen.TypeScriptSettings; +import software.amazon.smithy.typescript.codegen.TypeScriptWriter; +import software.amazon.smithy.utils.MapUtils; +import software.amazon.smithy.utils.SmithyInternalApi; + +/** + * Add runtime config for injecting utility functions to consume the JavaScript + * runtime-specific stream implementations. + */ +@SmithyInternalApi +public final class AddSdkStreamMixinDependency implements TypeScriptIntegration { + + @Override + public void addConfigInterfaceFields( + TypeScriptSettings settings, + Model model, + SymbolProvider symbolProvider, + TypeScriptWriter writer + ) { + if (!hasStreamingBlobDeser(settings, model)) { + return; + } + + writer.addImport("SdkStreamMixinInjector", "__SdkStreamMixinInjector", + TypeScriptDependency.AWS_SDK_TYPES.packageName); + writer.writeDocs("The internal function that inject utilities to runtime-specific stream to help users" + + " consume the data\n@internal"); + writer.write("sdkStreamMixin?: __SdkStreamMixinInjector;\n"); + } + + @Override + public Map> getRuntimeConfigWriters( + TypeScriptSettings settings, + Model model, + SymbolProvider symbolProvider, + LanguageTarget target + ) { + if (!hasStreamingBlobDeser(settings, model)) { + return Collections.emptyMap(); + } + switch (target) { + case NODE: + return MapUtils.of("sdkStreamMixin", writer -> { + writer.addDependency(TypeScriptDependency.UTIL_STREAM_NODE); + writer.addImport("sdkStreamMixin", "sdkStreamMixin", + TypeScriptDependency.UTIL_STREAM_NODE.packageName); + writer.write("sdkStreamMixin"); + }); + case BROWSER: + return MapUtils.of("sdkStreamMixin", writer -> { + writer.addDependency(TypeScriptDependency.UTIL_STREAM_BROWSER); + writer.addImport("sdkStreamMixin", "sdkStreamMixin", + TypeScriptDependency.UTIL_STREAM_BROWSER.packageName); + writer.write("sdkStreamMixin"); + }); + default: + return Collections.emptyMap(); + } + } + + private static boolean hasStreamingBlobDeser(TypeScriptSettings settings, Model model) { + ServiceShape serviceShape = settings.getService(model); + TopDownIndex topDownIndex = TopDownIndex.of(model); + Set operations = topDownIndex.getContainedOperations(serviceShape); + for (OperationShape operation : operations) { + if (hasStreamingBlobDeser(settings, model, operation)) { + return true; + } + } + return false; + } + + public static boolean hasStreamingBlobDeser(TypeScriptSettings settings, Model model, OperationShape operation) { + StructureShape ioShapeToDeser = (settings.generateServerSdk()) + ? model.expectShape(operation.getInputShape()).asStructureShape().get() + : model.expectShape(operation.getOutputShape()).asStructureShape().get(); + for (MemberShape member : ioShapeToDeser.members()) { + Shape shape = model.expectShape(member.getTarget()); + if (shape instanceof BlobShape && shape.hasTrait(StreamingTrait.class)) { + return true; + } + } + return false; + } +} diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpBindingProtocolGenerator.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpBindingProtocolGenerator.java index 503833fd696..3d1f981a78e 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpBindingProtocolGenerator.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpBindingProtocolGenerator.java @@ -2024,7 +2024,8 @@ private void generateOperationResponseDeserializer( String errorMethodName = methodName + "Error"; // Add the normalized output type. Symbol outputType = symbol.expectProperty("outputType", Symbol.class); - String contextType = CodegenUtils.getOperationDeserializerContextType(writer, context.getModel(), operation); + String contextType = CodegenUtils.getOperationDeserializerContextType(context.getSettings(), writer, + context.getModel(), operation); // Handle the general response. writer.openBlock("export const $L = async(\n" @@ -2320,14 +2321,18 @@ private HttpBinding readPayload( HttpBinding binding ) { TypeScriptWriter writer = context.getWriter(); + boolean isClientSdk = context.getSettings().generateClient(); // There can only be one payload binding. Shape target = context.getModel().expectShape(binding.getMember().getTarget()); // Handle streaming shapes differently. if (target.hasTrait(StreamingTrait.class)) { - // If payload is streaming, return raw low-level stream directly. writer.write("const data: any = output.body;"); + // If payload is streaming blob, return low-level stream with the stream utility functions mixin. + if (isClientSdk && target instanceof BlobShape) { + writer.write("context.sdkStreamMixin(data);"); + } } else if (target instanceof BlobShape) { // If payload is non-streaming Blob, only need to collect stream to binary data (Uint8Array). writer.write("const data: any = await collectBody(output.body, context);"); diff --git a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpRpcProtocolGenerator.java b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpRpcProtocolGenerator.java index fc75dab82ff..865349a4e5d 100644 --- a/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpRpcProtocolGenerator.java +++ b/smithy-typescript-codegen/src/main/java/software/amazon/smithy/typescript/codegen/integration/HttpRpcProtocolGenerator.java @@ -380,8 +380,8 @@ private void generateOperationDeserializer(GenerationContext context, OperationS // e.g., deserializeAws_restJson1_1ExecuteStatement String methodName = ProtocolGenerator.getDeserFunctionName(symbol, getName()); String errorMethodName = methodName + "Error"; - String serdeContextType = CodegenUtils.getOperationDeserializerContextType(writer, context.getModel(), - operation); + String serdeContextType = CodegenUtils.getOperationDeserializerContextType(context.getSettings(), writer, + context.getModel(), operation); // Add the normalized output type. Symbol outputType = symbol.expectProperty("outputType", Symbol.class); diff --git a/smithy-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration b/smithy-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration index 5e305e0300f..e37bfb1b2f0 100644 --- a/smithy-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration +++ b/smithy-typescript-codegen/src/main/resources/META-INF/services/software.amazon.smithy.typescript.codegen.integration.TypeScriptIntegration @@ -3,3 +3,4 @@ software.amazon.smithy.typescript.codegen.integration.AddChecksumRequiredDepende software.amazon.smithy.typescript.codegen.integration.AddDefaultsModeDependency software.amazon.smithy.typescript.codegen.integration.AddHttpApiKeyAuthPlugin software.amazon.smithy.typescript.codegen.integration.AddBaseServiceExceptionClass +software.amazon.smithy.typescript.codegen.integration.AddSdkStreamMixinDependency