ci: build all AWS services for PRs to main

.github/scripts/generate-includes.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+"""
+Converts a list of model file paths to a formatted includes list for passing to bootstrap
+"""
+import sys
+import os
+
+not_supported_yet = ["ec2", "location", "elasticbeanstalk", "marketplacecommerceanalytics"]
+
+
+def main():
+    lines = sys.stdin.readlines()
+    services = [os.path.basename(x).split(".")[0] for x in lines]
+    services = ["+" + x for x in services if x not in not_supported_yet]
+    sys.stdout.write(",".join(services))
+
+
+if __name__ == "__main__":
+    main()
+
+

.github/workflows/continuous-integration.yml

@@ -5,10 +5,13 @@ on:
     branches: [ main ]
   pull_request:
     branches: [ main ]
+    paths-ignore:
+      - 'docs/**'
+      - '*.md'
   workflow_dispatch:
 
 env:
-  BUILDER_VERSION: v0.8.0
+  BUILDER_VERSION: v0.8.19
   BUILDER_SOURCE: releases
   # host owned by CRT team to host aws-crt-builder releases. Contact their on-call with any issues
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
@@ -21,21 +24,29 @@ jobs:
     steps:
     - name: Checkout sources
       uses: actions/checkout@v2
+    - uses: actions/cache@v2
+      with:
+        path: |
+          ~/.gradle/caches
+          ~/.gradle/wrapper
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+        restore-keys: |
+          ${{ runner.os }}-gradle-
     - name: Build and Test ${{ env.PACKAGE_NAME }}
       env:
         CI_USER: ${{ secrets.CI_USER}}
         CI_ACCESS_TOKEN: ${{ secrets.CI_ACCESS_TOKEN }}
       run: |
-        echo "${{ secrets.GITHUB_TOKEN }}" | docker login docker.pkg.github.com -u awslabs --password-stdin
-        export DOCKER_IMAGE=docker.pkg.github.com/awslabs/aws-crt-builder/aws-crt-ubuntu-16-x64:${{ env.BUILDER_VERSION }}
-        docker pull $DOCKER_IMAGE
-        docker run --mount type=bind,source=$(pwd),target=/root/${{ env.PACKAGE_NAME }} \
-          --env GITHUB_REF \
-          --env GITHUB_HEAD_REF \
-          --env CI_USER \
-          --env CI_ACCESS_TOKEN \
-          --env GIT_ASKPASS=/root/${{ env.PACKAGE_NAME }}/.github/scripts/git-ci-askpass.sh \
-          $DOCKER_IMAGE build -p ${{ env.PACKAGE_NAME }} --build-dir=/root/${{ env.PACKAGE_NAME }}
+        python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
+        chmod a+x builder.pyz
+        GIT_ASKPASS="$(pwd)/.github/scripts/git-ci-askpass.sh" ./builder.pyz build -p ${{ env.PACKAGE_NAME }}
+    - uses: actions/upload-artifact@v2
+      # uploads local m2 so built runtime artifacts can be re-used
+      name: Upload Published Maven Artifacts
+      with:
+        name: m2-${{ env.name }}-${{ github.sha }}
+        path: ~/.m2
+        retention-days: 2
 
   macos-compat:
     runs-on: macos-latest
@@ -47,7 +58,7 @@ jobs:
         path: |
           ~/.gradle/caches
           ~/.gradle/wrapper
-        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*') }}
+        key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
         restore-keys: |
           ${{ runner.os }}-gradle-
     - name: Build and Test ${{ env.PACKAGE_NAME }}
@@ -58,3 +69,52 @@ jobs:
         python3 -c "from urllib.request import urlretrieve; urlretrieve('${{ env.BUILDER_HOST }}/${{ env.BUILDER_SOURCE }}/${{ env.BUILDER_VERSION }}/builder.pyz?run=${{ env.RUN }}', 'builder.pyz')"
         chmod a+x builder.pyz
         GIT_ASKPASS="$(pwd)/.github/scripts/git-ci-askpass.sh" ./builder.pyz build -p ${{ env.PACKAGE_NAME }}
+
+  build-aws-services-batch:
+    name: Build AWS services parallel
+    runs-on: ubuntu-latest
+    continue-on-error: false
+    needs: linux-compat
+    strategy:
+      matrix:
+        # run jobs in parallel
+        JOB: [1,2,3,4,5,6,7,8,9,10,11,12]
+    env:
+      JOBS: 12
+      JOB: ${{ matrix.JOB }}
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v2
+      - uses: actions/cache@v2
+        with:
+          path: |
+            ~/.gradle/caches
+            ~/.gradle/wrapper
+          key: ${{ runner.os }}-gradle-${{ hashFiles('**/*.gradle*', '**/gradle-wrapper.properties') }}
+          restore-keys: |
+            ${{ runner.os }}-gradle-
+      - uses: actions/download-artifact@v2
+        name: Download Maven Artifacts
+        with:
+          name: m2-${{ env.name }}-${{ github.sha }}
+          path: ~/.m2
+      - name: Generate SDKs
+        run: |
+          ls -lsa ~/.m2/repository
+          find codegen/sdk/aws-models -name '*.json' | sort | awk "NR % $JOBS == $JOB - 1" > services.txt
+          echo "Building the following services"
+          cat services.txt
+          ./gradlew --no-daemon :codegen:sdk:bootstrap -Paws.services=$(cat services.txt | .github/scripts/generate-includes.py)
+      - name: Build AWS Services
+        run: |
+          ./gradlew --parallel build
+
+  build-aws-services:
+    if: ${{ always() }}
+    name: Build all AWS services
+    runs-on: ubuntu-latest
+    needs: build-aws-services-batch
+    steps:
+      - name: Check batch matrix status
+        if: ${{ needs.build-aws-services-batch.result != 'success' }}
+        run: exit 1

README.md

@@ -18,7 +18,7 @@ Generated sources are not checked into the repository, you first have to generat
 
 
 ```sh
-./gradlew :codegen:sdk:bootstrap
+./gradlew --no-daemon :codegen:sdk:bootstrap
 ```
 
 NOTE: This task will respect the AWS services specified by project properties. See options below.
@@ -41,7 +41,7 @@ To see list of all projects run `./gradlew projects`
 See the local.properties definition above to specify this in a config file.
 
 ```sh
-./gradlew -Paws.services=+lambda  :codegen:sdk:bootstrap
+./gradlew --no-daemon -Paws.services=+lambda  :codegen:sdk:bootstrap
 ```
 
 ##### Testing Locally

builder.json

@@ -6,7 +6,7 @@
         "gradlew": "{source_dir}/gradlew -p {source_dir}"
     },
     "build_steps": [
-        "{gradlew} -x test -x jvmTest -x allTests build"
+        "{gradlew} assemble"
     ],
     "test_steps": [
         "{gradlew} publishToMavenLocal",

codegen/protocol-tests/build.gradle.kts

@@ -61,7 +61,8 @@ tasks["clean"].doFirst {
     delete("smithy-build.json")
 }
 
-tasks.create<SmithyBuild>("buildSdk") {
+tasks.create<SmithyBuild>("generateSdk") {
+    group = "codegen"
     // ensure the generated clients use the same version of the runtime as the aws aws-runtime
     val smithyKotlinVersion: String by project
     doFirst {
@@ -70,13 +71,12 @@ tasks.create<SmithyBuild>("buildSdk") {
     addRuntimeClasspath = true
     dependsOn(tasks["generateSmithyBuild"])
     inputs.file(projectDir.resolve("smithy-build.json"))
+    // ensure smithy-aws-kotlin-codegen is up to date
+    inputs.files(configurations.compileClasspath)
 }
 
-// Run the `buildSdk` automatically.
-tasks["build"].finalizedBy(tasks["buildSdk"])
-
 // force rebuild every time while developing
-tasks["buildSdk"].outputs.upToDateWhen { false }
+tasks["generateSdk"].outputs.upToDateWhen { false }
 
 data class ProtocolTest(val projectionName: String, val serviceShapeId: String) {
     val packageName: String
@@ -172,7 +172,7 @@ open class ProtocolTestTask : DefaultTask() {
 
 enabledProtocols.forEach {
     tasks.register<ProtocolTestTask>("testProtocol-${it.projectionName}") {
-        dependsOn(tasks.build)
+        dependsOn(tasks["generateSdk"])
         group = "Verification"
         protocol = it.projectionName
         plugin = "kotlin-codegen"

codegen/sdk/build.gradle.kts

@@ -238,6 +238,8 @@ tasks.create<SmithyBuild>("generateSdk") {
     addRuntimeClasspath = true
     dependsOn(tasks["generateSmithyBuild"])
     inputs.file(projectDir.resolve("smithy-build.json"))
+    // ensure smithy-aws-kotlin-codegen is up to date
+    inputs.files(configurations.compileClasspath)
 }
 
 // Remove generated model file for clean

gradle.properties

@@ -1,10 +1,13 @@
 kotlin.code.style=official
+kkotlin.incremental.js=true
+kotlin.incremental.multiplatform=true
+kotlin.parallel.tasks.in.project=true
 kotlin.mpp.stability.nowarn=true
 kotlin.native.ignoreDisabledTargets=true
 
 # gradle
 # FIXME - see https://github.com/Kotlin/dokka/issues/1405
-org.gradle.jvmargs=-Xmx4g -XX:MaxMetaspaceSize=1G
+org.gradle.jvmargs=-Xmx6g -XX:MaxPermSize=6g -XX:MaxMetaspaceSize=1G
 
 # sdk
 sdkVersion=0.4.0-SNAPSHOT