Skip to content

Commit

Permalink
Stream improvement serde (#593)
Browse files Browse the repository at this point in the history
* generate util functions to consume response stream

* inject sdk stream utility function to the stream response

* add SDKStreamSerdeContext interface

* add internal trait to the sdkStreamMixin config

* add missing new line for default modes config

* revert SdkStream type in SymbolVisitor, add to command generator

* only mixin stream utils in client SDK

* feat(serde): use type-mapping to apply stream mixin instead of omit

* feat(serde): use ternary

Co-authored-by: Trivikram Kamat <16024985+trivikr@users.noreply.github.com>

* Update AddSdkStreamMixinDependency.java

Co-authored-by: AllanZhengYP <zheallan@amazon.com>
Co-authored-by: Trivikram Kamat <16024985+trivikr@users.noreply.github.com>
  • Loading branch information
3 people committed Oct 12, 2022
1 parent ab6066c commit fd413c5
Show file tree
Hide file tree
Showing 9 changed files with 198 additions and 31 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.shapes.StructureShape;
import software.amazon.smithy.model.traits.StreamingTrait;
import software.amazon.smithy.typescript.codegen.integration.AddSdkStreamMixinDependency;
import software.amazon.smithy.utils.SmithyUnstableApi;

/**
Expand Down Expand Up @@ -75,12 +76,14 @@ public static String getOperationSerializerContextType(

/**
* Get context type for command deserializer function.
* @param settings The TypeScript settings
* @param writer The code writer.
* @param model The model for the service containing the given command.
* @param operation The operation shape for given command.
* @return The TypeScript type for the deserializer context
*/
public static String getOperationDeserializerContextType(
TypeScriptSettings settings,
TypeScriptWriter writer,
Model model,
OperationShape operation
Expand All @@ -90,9 +93,15 @@ public static String getOperationDeserializerContextType(
// If event stream trait exists, add corresponding serde context type to the intersection type.
EventStreamIndex eventStreamIndex = EventStreamIndex.of(model);
if (eventStreamIndex.getOutputInfo(operation).isPresent()) {
writer.addImport("EventStreamSerdeContext", "__EventStreamSerdeContext", "@aws-sdk/types");
writer.addImport("EventStreamSerdeContext", "__EventStreamSerdeContext",
TypeScriptDependency.AWS_SDK_TYPES.packageName);
contextInterfaceList.add("__EventStreamSerdeContext");
}
if (AddSdkStreamMixinDependency.hasStreamingBlobDeser(settings, model, operation)) {
writer.addImport("SdkStreamSerdeContext", "__SdkStreamSerdeContext",
TypeScriptDependency.AWS_SDK_TYPES.packageName);
contextInterfaceList.add("__SdkStreamSerdeContext");
}
return String.join(" & ", contextInterfaceList);
}

Expand All @@ -108,37 +117,67 @@ static List<MemberShape> getBlobStreamingMembers(Model model, StructureShape sha
return shape.getAllMembers().values().stream()
.filter(memberShape -> {
// Streaming blobs need to have their types modified
// See `writeStreamingMemberType`
// See `writeClientCommandStreamingInputType`
Shape target = model.expectShape(memberShape.getTarget());
return target.isBlobShape() && target.hasTrait(StreamingTrait.class);
})
.collect(Collectors.toList());
}

/**
* Ease the input streaming member restriction so that users don't need to construct a stream every time.
* This type decoration is allowed in Smithy because it makes input type more permissive than output type
* for the same member.
* Generate the type of the command input of the client sdk given the streaming blob
* member of the shape. The generated type eases the streaming member requirement so that users don't need to
* construct a stream every time.
* This type decoration is allowed in Smithy because it makes, for the same member, the type to be serialized is
* more permissive than the type to be deserialized.
* Refer here for more rationales: https://github.com/aws/aws-sdk-js-v3/issues/843
*/
static void writeStreamingMemberType(
static void writeClientCommandStreamingInputType(
TypeScriptWriter writer,
Symbol containerSymbol,
String typeName,
MemberShape streamingMember
) {
String memberName = streamingMember.getMemberName();
String optionalSuffix = streamingMember.isRequired() ? "" : "?";
writer.openBlock("type $LType = Omit<$T, $S> & {", "};", typeName, containerSymbol, memberName, () -> {
writer.writeDocs(String.format("For *`%1$s[\"%2$s\"]`*, see {@link %1$s.%2$s}.",
containerSymbol.getName(), memberName));
writer.write("$1L$2L: $3T[$1S]|string|Uint8Array|Buffer;", memberName, optionalSuffix, containerSymbol);
writer.openBlock("type $LType = Omit<$T, $S> & {", "};", typeName,
containerSymbol, memberName, () -> {
writer.writeDocs(String.format("For *`%1$s[\"%2$s\"]`*, see {@link %1$s.%2$s}.",
containerSymbol.getName(), memberName));
writer.write("$1L$2L: $3T[$1S]|string|Uint8Array|Buffer;", memberName, optionalSuffix,
containerSymbol);
});
writer.writeDocs(String.format("This interface extends from `%1$s` interface. There are more parameters than"
+ " `%2$s` defined in {@link %1$s}", containerSymbol.getName(), memberName));
writer.write("export interface $1L extends $1LType {}", typeName);
}

/**
* Generate the type of the command output of the client sdk given the streaming blob
* member of the shape. The type marks the streaming blob member to contain the utility methods to transform the
* stream to string, buffer or WHATWG stream API.
*/
static void writeClientCommandStreamingOutputType(
TypeScriptWriter writer,
Symbol containerSymbol,
String typeName,
MemberShape streamingMember
) {
String memberName = streamingMember.getMemberName();
String optionalSuffix = streamingMember.isRequired() ? "" : "?";
writer.addImport("MetadataBearer", "__MetadataBearer", TypeScriptDependency.AWS_SDK_TYPES.packageName);
writer.addImport("SdkStream", "__SdkStream", TypeScriptDependency.AWS_SDK_TYPES.packageName);
writer.addImport("WithSdkStreamMixin", "__WithSdkStreamMixin", TypeScriptDependency.AWS_SDK_TYPES.packageName);


writer.write(
"export interface $L extends __WithSdkStreamMixin<$T, $S>, __MetadataBearer {}",
typeName,
containerSymbol,
memberName
);
}

/**
* Returns the list of function parameter key-value pairs to be written for
* provided parameters map.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
package software.amazon.smithy.typescript.codegen;

import static software.amazon.smithy.typescript.codegen.CodegenUtils.getBlobStreamingMembers;
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeStreamingMemberType;
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeClientCommandStreamingInputType;
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeClientCommandStreamingOutputType;

import java.nio.file.Paths;
import java.util.List;
Expand Down Expand Up @@ -314,7 +315,8 @@ private void writeInputType(String typeName, Optional<StructureShape> inputShape
if (blobStreamingMembers.isEmpty()) {
writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(input));
} else {
writeStreamingMemberType(writer, symbolProvider.toSymbol(input), typeName, blobStreamingMembers.get(0));
writeClientCommandStreamingInputType(writer, symbolProvider.toSymbol(input), typeName,
blobStreamingMembers.get(0));
}
} else {
// If the input is non-existent, then use an empty object.
Expand All @@ -327,8 +329,15 @@ private void writeOutputType(String typeName, Optional<StructureShape> outputSha
// to a defined output shape.
writer.addImport("MetadataBearer", "__MetadataBearer", TypeScriptDependency.AWS_SDK_TYPES.packageName);
if (outputShape.isPresent()) {
writer.write("export interface $L extends $T, __MetadataBearer {}",
typeName, symbolProvider.toSymbol(outputShape.get()));
StructureShape output = outputShape.get();
List<MemberShape> blobStreamingMembers = getBlobStreamingMembers(model, output);
if (blobStreamingMembers.isEmpty()) {
writer.write("export interface $L extends $T, __MetadataBearer {}",
typeName, symbolProvider.toSymbol(outputShape.get()));
} else {
writeClientCommandStreamingOutputType(writer, symbolProvider.toSymbol(output), typeName,
blobStreamingMembers.get(0));
}
} else {
writer.write("export interface $L extends __MetadataBearer {}", typeName);
}
Expand Down Expand Up @@ -371,7 +380,8 @@ private void writeSerde() {
.write("private deserialize(")
.indent()
.write("output: $T,", applicationProtocol.getResponseType())
.write("context: $L", CodegenUtils.getOperationDeserializerContextType(writer, model, operation))
.write("context: $L",
CodegenUtils.getOperationDeserializerContextType(settings, writer, model, operation))
.dedent()
.openBlock("): Promise<$T> {", "}", outputType, () -> writeSerdeDispatcher(false))
.write("");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,9 +15,6 @@

package software.amazon.smithy.typescript.codegen;

import static software.amazon.smithy.typescript.codegen.CodegenUtils.getBlobStreamingMembers;
import static software.amazon.smithy.typescript.codegen.CodegenUtils.writeStreamingMemberType;

import java.nio.file.Paths;
import java.util.Collections;
import java.util.Iterator;
Expand All @@ -31,7 +28,6 @@
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.knowledge.OperationIndex;
import software.amazon.smithy.model.knowledge.TopDownIndex;
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.OperationShape;
import software.amazon.smithy.model.shapes.ServiceShape;
import software.amazon.smithy.model.shapes.StructureShape;
Expand Down Expand Up @@ -98,16 +94,10 @@ private void addInputAndOutputTypes() {
writer.write("");
}

// TODO: Flip these so that metadata is attached to input and streaming customization is attached to output.
private void writeInputType(String typeName, Optional<StructureShape> inputShape) {
if (inputShape.isPresent()) {
StructureShape input = inputShape.get();
List<MemberShape> blobStreamingMembers = getBlobStreamingMembers(model, input);
if (blobStreamingMembers.isEmpty()) {
writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(input));
} else {
writeStreamingMemberType(writer, symbolProvider.toSymbol(input), typeName, blobStreamingMembers.get(0));
}
writer.write("export interface $L extends $T {}", typeName, symbolProvider.toSymbol(inputShape.get()));
renderNamespace(typeName, input);
} else {
// If the input is non-existent, then use an empty object.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -100,6 +100,10 @@ public enum TypeScriptDependency implements SymbolDependencyContainer {
XML_PARSER("dependencies", "fast-xml-parser", "4.0.11", false),
HTML_ENTITIES("dependencies", "entities", "2.2.0", false),

// Conditionally added when streaming blob response payload exists.
UTIL_STREAM_NODE("dependencies", "@aws-sdk/util-stream-node", false),
UTIL_STREAM_BROWSER("dependencies", "@aws-sdk/util-stream-browser", false),

// Server dependency for SSDKs
SERVER_COMMON("dependencies", "@aws-smithy/server-common", "1.0.0-alpha.6", false);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,6 @@ public void addConfigInterfaceFields(
writer.addImport("Provider", "Provider", TypeScriptDependency.AWS_SDK_TYPES.packageName);
writer.writeDocs("The {@link DefaultsMode} that will be used to determine how certain default configuration "
+ "options are resolved in the SDK.");
writer.write("defaultsMode?: DefaultsMode | Provider<DefaultsMode>;");
writer.write("defaultsMode?: DefaultsMode | Provider<DefaultsMode>;\n");
}
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,118 @@
/*
* Copyright 2022 Amazon.com, Inc. or its affiliates. All Rights Reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License").
* You may not use this file except in compliance with the License.
* A copy of the License is located at
*
* http://aws.amazon.com/apache2.0
*
* or in the "license" file accompanying this file. This file is distributed
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
* express or implied. See the License for the specific language governing
* permissions and limitations under the License.
*/

package software.amazon.smithy.typescript.codegen.integration;

import java.util.Collections;
import java.util.Map;
import java.util.Set;
import java.util.function.Consumer;
import software.amazon.smithy.codegen.core.SymbolProvider;
import software.amazon.smithy.model.Model;
import software.amazon.smithy.model.knowledge.TopDownIndex;
import software.amazon.smithy.model.shapes.BlobShape;
import software.amazon.smithy.model.shapes.MemberShape;
import software.amazon.smithy.model.shapes.OperationShape;
import software.amazon.smithy.model.shapes.ServiceShape;
import software.amazon.smithy.model.shapes.Shape;
import software.amazon.smithy.model.shapes.StructureShape;
import software.amazon.smithy.model.traits.StreamingTrait;
import software.amazon.smithy.typescript.codegen.LanguageTarget;
import software.amazon.smithy.typescript.codegen.TypeScriptDependency;
import software.amazon.smithy.typescript.codegen.TypeScriptSettings;
import software.amazon.smithy.typescript.codegen.TypeScriptWriter;
import software.amazon.smithy.utils.MapUtils;
import software.amazon.smithy.utils.SmithyInternalApi;

/**
* Add runtime config for injecting utility functions to consume the JavaScript
* runtime-specific stream implementations.
*/
@SmithyInternalApi
public final class AddSdkStreamMixinDependency implements TypeScriptIntegration {

@Override
public void addConfigInterfaceFields(
TypeScriptSettings settings,
Model model,
SymbolProvider symbolProvider,
TypeScriptWriter writer
) {
if (!hasStreamingBlobDeser(settings, model)) {
return;
}

writer.addImport("SdkStreamMixinInjector", "__SdkStreamMixinInjector",
TypeScriptDependency.AWS_SDK_TYPES.packageName);
writer.writeDocs("The internal function that inject utilities to runtime-specific stream to help users"
+ " consume the data\n@internal");
writer.write("sdkStreamMixin?: __SdkStreamMixinInjector;\n");
}

@Override
public Map<String, Consumer<TypeScriptWriter>> getRuntimeConfigWriters(
TypeScriptSettings settings,
Model model,
SymbolProvider symbolProvider,
LanguageTarget target
) {
if (!hasStreamingBlobDeser(settings, model)) {
return Collections.emptyMap();
}
switch (target) {
case NODE:
return MapUtils.of("sdkStreamMixin", writer -> {
writer.addDependency(TypeScriptDependency.UTIL_STREAM_NODE);
writer.addImport("sdkStreamMixin", "sdkStreamMixin",
TypeScriptDependency.UTIL_STREAM_NODE.packageName);
writer.write("sdkStreamMixin");
});
case BROWSER:
return MapUtils.of("sdkStreamMixin", writer -> {
writer.addDependency(TypeScriptDependency.UTIL_STREAM_BROWSER);
writer.addImport("sdkStreamMixin", "sdkStreamMixin",
TypeScriptDependency.UTIL_STREAM_BROWSER.packageName);
writer.write("sdkStreamMixin");
});
default:
return Collections.emptyMap();
}
}

private static boolean hasStreamingBlobDeser(TypeScriptSettings settings, Model model) {
ServiceShape serviceShape = settings.getService(model);
TopDownIndex topDownIndex = TopDownIndex.of(model);
Set<OperationShape> operations = topDownIndex.getContainedOperations(serviceShape);
for (OperationShape operation : operations) {
if (hasStreamingBlobDeser(settings, model, operation)) {
return true;
}
}
return false;
}

public static boolean hasStreamingBlobDeser(TypeScriptSettings settings, Model model, OperationShape operation) {
StructureShape ioShapeToDeser = (settings.generateServerSdk())
? model.expectShape(operation.getInputShape()).asStructureShape().get()
: model.expectShape(operation.getOutputShape()).asStructureShape().get();
for (MemberShape member : ioShapeToDeser.members()) {
Shape shape = model.expectShape(member.getTarget());
if (shape instanceof BlobShape && shape.hasTrait(StreamingTrait.class)) {
return true;
}
}
return false;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -2024,7 +2024,8 @@ private void generateOperationResponseDeserializer(
String errorMethodName = methodName + "Error";
// Add the normalized output type.
Symbol outputType = symbol.expectProperty("outputType", Symbol.class);
String contextType = CodegenUtils.getOperationDeserializerContextType(writer, context.getModel(), operation);
String contextType = CodegenUtils.getOperationDeserializerContextType(context.getSettings(), writer,
context.getModel(), operation);

// Handle the general response.
writer.openBlock("export const $L = async(\n"
Expand Down Expand Up @@ -2320,14 +2321,18 @@ private HttpBinding readPayload(
HttpBinding binding
) {
TypeScriptWriter writer = context.getWriter();
boolean isClientSdk = context.getSettings().generateClient();

// There can only be one payload binding.
Shape target = context.getModel().expectShape(binding.getMember().getTarget());

// Handle streaming shapes differently.
if (target.hasTrait(StreamingTrait.class)) {
// If payload is streaming, return raw low-level stream directly.
writer.write("const data: any = output.body;");
// If payload is streaming blob, return low-level stream with the stream utility functions mixin.
if (isClientSdk && target instanceof BlobShape) {
writer.write("context.sdkStreamMixin(data);");
}
} else if (target instanceof BlobShape) {
// If payload is non-streaming Blob, only need to collect stream to binary data (Uint8Array).
writer.write("const data: any = await collectBody(output.body, context);");
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -380,8 +380,8 @@ private void generateOperationDeserializer(GenerationContext context, OperationS
// e.g., deserializeAws_restJson1_1ExecuteStatement
String methodName = ProtocolGenerator.getDeserFunctionName(symbol, getName());
String errorMethodName = methodName + "Error";
String serdeContextType = CodegenUtils.getOperationDeserializerContextType(writer, context.getModel(),
operation);
String serdeContextType = CodegenUtils.getOperationDeserializerContextType(context.getSettings(), writer,
context.getModel(), operation);
// Add the normalized output type.
Symbol outputType = symbol.expectProperty("outputType", Symbol.class);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,3 +3,4 @@ software.amazon.smithy.typescript.codegen.integration.AddChecksumRequiredDepende
software.amazon.smithy.typescript.codegen.integration.AddDefaultsModeDependency
software.amazon.smithy.typescript.codegen.integration.AddHttpApiKeyAuthPlugin
software.amazon.smithy.typescript.codegen.integration.AddBaseServiceExceptionClass
software.amazon.smithy.typescript.codegen.integration.AddSdkStreamMixinDependency

0 comments on commit fd413c5

Please sign in to comment.