chore(codegen): generate and run SSDK protocol tests during CI

This sets up a server-only mode for generate-clients, and a set of yarn targets
that generate and run server protocol tests. The CodeBuild CI for clients uses
Node 10, while SSDKs require Node 14, so this sets up an additional GitHub
action for running the SSDK protocol tests and excludes SSDK protocol tests
from the client targets. When the client is updated to support 14 as a minimum
version, this distinction can be removed.

.github/workflows/server-protocol-tests.yml

@@ -0,0 +1,49 @@
+name: server-protocol-tests
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+
+      - uses: actions/setup-java@v1
+        with:
+          java-version: '11'
+
+      - uses: actions/setup-node@v2
+        with:
+          node-version: '14'
+          cache: 'yarn'
+
+      - name: build and publish smithy-typescript
+        run: |
+          git clone --depth 1 https://github.com/awslabs/smithy-typescript.git
+          cd smithy-typescript
+          ./gradlew clean build publishToMavenLocal
+          cd ..
+
+      - name: install dependencies
+        run: |
+          yarn install
+
+      - name: generate protocol tests
+        run: |
+          yarn generate-server-protocols
+
+      # there are new dependencies as a result of generating server protocol tests
+      - name: install dependencies again
+        run: |
+          yarn install
+
+      - name: run protocol tests
+        run: |
+          yarn test:server-protocols
+
+

codegen/protocol-test-codegen/smithy-build.json

@@ -118,6 +118,11 @@
             "license": "Apache-2.0"
           },
           "private": true
+        },
+        "typescript-ssdk-codegen": {
+          "package": "@aws-sdk/aws-restjson-server",
+          "packageVersion": "1.0.0-alpha.1",
+          "disableDefaultValidation": true
         }
       }
     },

codegen/smithy-aws-typescript-codegen/src/main/java/software/amazon/smithy/aws/typescript/codegen/AwsProtocolUtils.java

@@ -293,6 +293,16 @@ private static boolean filterProtocolTests(
         if (testCase.getId().equals("QueryCustomizedError")) {
             return true;
         }
+        if (testCase.getId().equals("RestJsonStreamingTraitsRequireLengthWithBlob") && settings.generateServerSdk()) {
+            return true;
+        }
+        //TODO: https://github.com/awslabs/smithy/pull/857 and https://github.com/awslabs/smithy-typescript/pull/377
+        if ((testCase.getId().equals("RestJsonAllQueryStringTypes")
+                || testCase.getId().equals("RestJsonQueryStringEscaping"))
+                && settings.generateServerSdk()) {
+            return true;
+        }
+
         return false;
     }
 }

package.json

@@ -7,21 +7,24 @@
   "scripts": {
     "copy-models": "node ./scripts/copy-models",
     "generate-clients": "node ./scripts/generate-clients",
+    "generate-server-protocols": "node ./scripts/generate-clients -s",
     "bootstrap": "yarn",
     "clean": "yarn clear-build-cache && yarn clear-build-info && lerna clean",
     "clear-build-cache": "rimraf ./packages/*/dist ./clients/*/dist ./lib/*/dist ./packages/*/types ./clients/*/types ./lib/*/types",
     "clear-build-info": "rimraf ./packages/**/*.tsbuildinfo ./clients/**/*.tsbuildinfo  ./lib/**/*.tsbuildinfo",
     "remove-documentation": "rimraf ./docs",
     "build:crypto-dependencies": "lerna run --scope '@aws-sdk/{types,util-utf8-browser,util-locate-window,hash-node}' --include-dependencies build",
-    "build:protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/aws-*' --include-dependencies build",
+    "build:protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/aws-*' --ignore '@aws-sdk/*-server' --include-dependencies build",
+    "build:server-protocols": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/*-server' --include-dependencies build",
     "build:all": "yarn build:crypto-dependencies && lerna run build",
     "build-documentation": "yarn remove-documentation && typedoc",
     "pretest:all": "yarn build:all",
     "test:all": "jest --coverage --passWithNoTests && lerna run test --scope '@aws-sdk/{fetch-http-handler,hash-blob-browser}'",
     "test:functional": "jest --config tests/functional/jest.config.js --passWithNoTests",
     "test:integration-legacy": "cucumber-js --fail-fast",
     "test:integration": "jest --config jest.config.integ.js --passWithNoTests",
-    "test:protocols": "yarn build:protocols && lerna run test --scope '@aws-sdk/aws-*'",
+    "test:protocols": "yarn build:protocols && lerna run test --scope '@aws-sdk/aws-*' --ignore '@aws-sdk/*-server'",
+    "test:server-protocols": "yarn build:server-protocols && lerna run test --scope '@aws-sdk/*-server'",
     "pretest:e2e": "yarn build:crypto-dependencies && lerna run --scope '@aws-sdk/{client-cloudformation,karma-credential-loader}' --include-dependencies build",
     "test:e2e": "node ./tests/e2e/index.js",
     "local-publish": "node ./scripts/verdaccio-publish/index.js"
@@ -118,4 +121,4 @@
     ],
     "**/*.{ts,js,md,json}": "prettier --write"
   }
-}
+}

scripts/generate-clients/code-prettify.js

@@ -2,11 +2,16 @@
 const { spawnProcess } = require("../utils/spawn-process");
 const path = require("path");
 
-const prettifyCode = async (dir) =>
-  spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "pprettier"), [
+const prettifyCode = async (dir) => {
+  await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "pprettier"), [
     "--write",
     `${dir}/*/typescript-codegen/**/*.{ts,js,md,json}`,
   ]);
+  await spawnProcess(path.join(__dirname, "..", "..", "node_modules", ".bin", "pprettier"), [
+    "--write",
+    `${dir}/*/typescript-ssdk-codegen/**/*.{ts,js,md,json}`,
+  ]);
+};
 
 module.exports = {
   prettifyCode,

scripts/generate-clients/copy-to-clients.js

@@ -151,6 +151,53 @@ const copyToClients = async (sourceDir, destinationDir) => {
   }
 };
 
+const copyServerTests = async (sourceDir, destinationDir) => {
+  for (const modelName of readdirSync(sourceDir)) {
+    if (modelName === "source") continue;
+
+    const artifactPath = join(sourceDir, modelName, "typescript-ssdk-codegen");
+    const packageManifestPath = join(artifactPath, "package.json");
+    if (!existsSync(packageManifestPath)) {
+      console.error(`${modelName} generates empty server, skip.`);
+      continue;
+    }
+
+    const packageManifest = JSON.parse(readFileSync(packageManifestPath).toString());
+    const packageName = packageManifest.name;
+    const testName = packageName.replace("@aws-sdk/", "");
+
+    console.log(`copying ${packageName} from ${artifactPath} to ${destinationDir}`);
+    const destPath = join(destinationDir, testName);
+    const overwritablePredicate = getOverwritablePredicate(packageName);
+
+    for (const packageSub of readdirSync(artifactPath)) {
+      const packageSubPath = join(artifactPath, packageSub);
+      const destSubPath = join(destPath, packageSub);
+
+      if (packageSub === "package.json") {
+        //copy manifest file
+        const destManifest = existsSync(destSubPath) ? JSON.parse(readFileSync(destSubPath).toString()) : {};
+        const mergedManifest = {
+          ...mergeManifest(packageManifest, destManifest),
+          homepage: `https://github.com/aws/aws-sdk-js-v3/tree/main/protocol_tests/${testName}`,
+          repository: {
+            type: "git",
+            url: "https://github.com/aws/aws-sdk-js-v3.git",
+            directory: `protocol_tests/${testName}`,
+          },
+        };
+        writeFileSync(destSubPath, JSON.stringify(mergedManifest, null, 2).concat(`\n`));
+      } else if (overwritablePredicate(packageSub) || !existsSync(destSubPath)) {
+        if (lstatSync(packageSubPath).isDirectory()) removeSync(destSubPath);
+        copySync(packageSubPath, destSubPath, {
+          overwrite: true,
+        });
+      }
+    }
+  }
+};
+
 module.exports = {
   copyToClients,
+  copyServerTests,
 };

scripts/generate-clients/index.js

@@ -3,7 +3,7 @@ const yargs = require("yargs");
 const path = require("path");
 const { emptyDirSync, rmdirSync } = require("fs-extra");
 const { generateClients, generateProtocolTests } = require("./code-gen");
-const { copyToClients } = require("./copy-to-clients");
+const { copyToClients, copyServerTests } = require("./copy-to-clients");
 const {
   CODE_GEN_SDK_OUTPUT_DIR,
   CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR,
@@ -14,7 +14,12 @@ const { prettifyCode } = require("./code-prettify");
 const SDK_CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "clients"));
 const PROTOCOL_TESTS_CLIENTS_DIR = path.normalize(path.join(__dirname, "..", "..", "protocol_tests"));
 
-const { models, globs, output: clientsDir } = yargs
+const {
+  models,
+  globs,
+  output: clientsDir,
+  s: serverOnly,
+} = yargs
   .alias("m", "models")
   .string("m")
   .describe("m", "The path to directory with models.")
@@ -26,10 +31,27 @@ const { models, globs, output: clientsDir } = yargs
   .string("o")
   .describe("o", "The output directory for built clients")
   .default("o", SDK_CLIENTS_DIR)
+  .alias("s", "server-artifacts")
+  .boolean("s")
+  .describe("s", "Generate server artifacts instead of client ones")
+  .default("s", false)
+  .conflicts("s", ["m", "g"])
   .help().argv;
 
 (async () => {
   try {
+    if (serverOnly) {
+      await generateProtocolTests();
+      await prettifyCode(CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR);
+      await copyServerTests(CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR, PROTOCOL_TESTS_CLIENTS_DIR);
+
+      emptyDirSync(CODE_GEN_PROTOCOL_TESTS_OUTPUT_DIR);
+      emptyDirSync(TEMP_CODE_GEN_INPUT_DIR);
+
+      rmdirSync(TEMP_CODE_GEN_INPUT_DIR);
+      return;
+    }
+
     await generateClients(models || globs);
     await generateProtocolTests();