From f32584ee34bf16232840f46f6a6091477d5d912a Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 17 Nov 2021 16:06:34 -0500 Subject: [PATCH 01/31] poc generate code via plugin --- .gitignore | 2 + aws-runtime/aws-config/build.gradle.kts | 80 ++++ gradle/sdk-plugins/build.gradle.kts | 57 +++ gradle/sdk-plugins/settings.gradle.kts | 5 + .../sdk/kotlin/build/plugin/CodegenPlugin.kt | 341 ++++++++++++++++++ settings.gradle.kts | 2 + 6 files changed, 487 insertions(+) create mode 100644 gradle/sdk-plugins/build.gradle.kts create mode 100644 gradle/sdk-plugins/settings.gradle.kts create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt diff --git a/.gitignore b/.gitignore index 2d0d9a18765..022f474900b 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ build/ .idea/ __pycache__/ local.properties +!gradle/sdk-plugins/src/** +!buildSrc/src/** # ignore generated files services/*/generated-src services/*/build.gradle.kts diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 3ec3e91c229..b3c5dbec7e6 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -2,6 +2,9 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ +plugins { + id("aws.sdk.kotlin.codegen") +} description = "Support for AWS configuration" extra["moduleName"] = "aws.sdk.kotlin.runtime.config" @@ -31,6 +34,15 @@ kotlin { implementation("aws.sdk.kotlin.crt:aws-crt-kotlin:$crtKotlinVersion") implementation(project(":aws-runtime:crt-util")) + + + // generated sts provider + implementation("aws.smithy.kotlin:serde-form-url:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:serde-xml:$smithyKotlinVersion") + implementation("aws.smithy.kotlin:utils:$smithyKotlinVersion") + implementation(project(":aws-runtime:protocols:aws-xml-protocols")) + implementation(project(":aws-runtime:aws-endpoint")) + implementation(project(":aws-runtime:aws-signing")) } } commonTest { @@ -55,3 +67,71 @@ kotlin { } } } + +fun awsModelFile(name: String): String = + rootProject.file("codegen/sdk/aws-models/$name").absolutePath + +codegen { + projection("sts-credentials-provider") { + imports = listOf( + awsModelFile("sts.2011-06-15.json") + ) + + // FIXME - we could make this a typed object if we want or even re-use smithy-kotlin type + // language=JSON + pluginSettings = """ + { + "service": "com.amazonaws.sts#AWSSecurityTokenServiceV20110615", + "package" : { + "name": "aws.sdk.kotlin.runtime.auth.credentials.internal.sts", + "version": "$version", + "description": "Internal STS credentials provider" + }, + "sdkId": "STS", + "build": { + "generateDefaultBuildFiles": false + } + } + """.trimIndent() + } +} + +val codegenTasks = tasks.withType() +tasks.withType { + dependsOn(codegenTasks) +} + +codegen.projections { + // add this projected source dir to the common sourceSet + val projectedSrcDir = projectionRootDir.resolve("src/main/kotlin") + kotlin.sourceSets.commonMain { + println("add $projectedSrcDir to common sourceSet") + kotlin.srcDir(projectedSrcDir) + } +} + + +//tasks.register("generateCredentialProviders") { +// description = "generate STS and SSO credential providers for aws-config to consume" +// projectionName = "sts-credentials-provider" +// +// imports = listOf( +// awsModelFile("sts.2011-06-15.json") +// ) +// +// pluginSettings = """ +// { +// "service": ""com.amazonaws.sts#AWSSecurityTokenServiceV20110615", +// "package" : { +// "name": "aws.sdk.kotlin.runtime.auth.credentials.internal.sts", +// "version": "$version", +// "description": "Internal STS credentials provider" +// }, +// "sdkId": "Sts", +// "build": { +// "generateDefaultBuildFiles": false +// } +// } +// """.trimIndent() +// +//} diff --git a/gradle/sdk-plugins/build.gradle.kts b/gradle/sdk-plugins/build.gradle.kts new file mode 100644 index 00000000000..00fd4891939 --- /dev/null +++ b/gradle/sdk-plugins/build.gradle.kts @@ -0,0 +1,57 @@ + +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +import java.util.Properties +buildscript { + repositories { + mavenCentral() + } +} + +plugins { + `kotlin-dsl` + `java-gradle-plugin` + // FIXME - can we get this to sync with gradle.properties from root project + kotlin("jvm") version "1.5.31" +} + +repositories { + mavenCentral() + gradlePluginPortal() +} + +fun loadVersions() { + val gradleProperties = Properties() + val propertiesFile: File = file("../../gradle.properties") + if (propertiesFile.exists()) { + propertiesFile.inputStream().use { gradleProperties.load(it) } + } + gradleProperties.forEach { + project.ext.set(it.key.toString(), it.value) + } +} +// use the versions from aws-sdk-kotlin/gradle.properties +loadVersions() + +val kotlinVersion: String by project +val smithyVersion: String by project +val smithyGradleVersion: String by project + +dependencies { + implementation("software.amazon.smithy:smithy-gradle-plugin:$smithyGradleVersion") + implementation("software.amazon.smithy:smithy-model:$smithyVersion") + implementation("software.amazon.smithy:smithy-aws-traits:$smithyVersion") + implementation(gradleApi()) +} + +gradlePlugin { + plugins { + val awsCodegenPlugin by creating { + id = "aws.sdk.kotlin.codegen" + implementationClass = "aws.sdk.kotlin.build.plugin.CodegenPlugin" + } + } +} + diff --git a/gradle/sdk-plugins/settings.gradle.kts b/gradle/sdk-plugins/settings.gradle.kts new file mode 100644 index 00000000000..642d47af4cc --- /dev/null +++ b/gradle/sdk-plugins/settings.gradle.kts @@ -0,0 +1,5 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ +rootProject.name = "sdk-plugins" \ No newline at end of file diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt new file mode 100644 index 00000000000..4db10896fc4 --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt @@ -0,0 +1,341 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.build.plugin + +import org.gradle.api.Action +import org.gradle.api.DefaultTask +import org.gradle.api.Plugin +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.provideDelegate +import org.gradle.kotlin.dsl.register +import software.amazon.smithy.gradle.tasks.SmithyBuild + +const val CODEGEN_EXTENSION_NAME = "codegen" +// TODO - create custom transform to include specific operations +// TODO - create custom transform to add `InternalSdkApi` to generated client OR change visibility? +// - e.g. custom trait (@kotlinVisibility("internal")) + +/** + * This plugin handles: + * - applying smithy plugins to the project to generate code + * - providing a [CodegenTask] to generate Kotlin sources from their respective smithy models. + */ +class CodegenPlugin : Plugin { + override fun apply(target: Project):Unit = target.run { + configurePlugins() + installExtension() + } + + private fun Project.configurePlugins() { + // unfortunately all of the tasks provided by smithy rely on the plugin extension, so it also needs applied + plugins.apply("software.amazon.smithy") + tasks.getByName("smithyBuildJar").enabled = false + } + + private fun Project.installExtension(): CodegenExtension { + return extensions.create(CODEGEN_EXTENSION_NAME, CodegenExtension::class.java, project) + } +} + +///** +// * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) +// */ +//class PostProcessSpec { +// +// +//} + + +class ProjectionConfiguration{ + /** + * List of files/directories to import when building the projection + * + * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections + */ + var imports: List? = null + + + /** + * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the + * plugin settings for smithy kotlin. + * + * Example: + * ```json + * { + * "service": , + * "package": { + * "name": , + * "version": + * "description": + * }, + * "sdkId": (Optional: defaults to shape id if not set), + * "build": { } + * } + * ``` + */ + var pluginSettings: String? = null + +// private var postProcessSpec: PostProcessSpec? = null +// fun postProcess(spec: PostProcessSpec.() -> Unit) { +// postProcessSpec = PostProcessSpec().apply(spec) +// } +} + +class KotlinCodegenProjection( + val name: String, + private val project: Project +) { + /** + * Root directory for this projection + */ + val projectionRootDir: java.io.File + get() = project.file("${project.buildDir}/smithyprojections/${project.name}/${name}/kotlin-codegen") + +} + +open class CodegenExtension(private val project: Project) { + private val projections = mutableListOf() + + fun projection(name: String, configure: Action) { + val pc = ProjectionConfiguration() + configure.execute(pc) + // register codegen tasks for projection + project.registerCodegenTasksForProjection(name, pc) + + projections.add(KotlinCodegenProjection(name, project)) + } + + fun projections(action: Action) = projections.forEach { action.execute(it) } + + fun getProjection(name: String): KotlinCodegenProjection? = projections.find { it.name == name } +} + + +private fun Project.registerCodegenTasksForProjection(projectionName: String, configuration: ProjectionConfiguration) { + // generate the projection file for smithy to consume + val smithyBuildConfig = buildDir.resolve("smithy-build-$projectionName.json") + val generateSmithyBuild = tasks.register("$projectionName-smithyBuildJson") { + description = "generate smithy-build.json" + group = "codegen" + outputs.file(smithyBuildConfig) + inputs.property("$projectionName-configuration", configuration.pluginSettings) + doFirst { + if (smithyBuildConfig.exists()) { + smithyBuildConfig.delete() + } + } + doLast { + buildDir.mkdir() + smithyBuildConfig.writeText(generateSmithyBuild(projectionName, configuration)) + } + } + + val codegenConfig = createCodegenConfiguration() + val buildTask = project.tasks.register("$projectionName-smithyBuild") { + dependsOn(generateSmithyBuild) + description = "generate code for $name task" + group = "codegen" + classpath = codegenConfig + smithyBuildConfigs = files(smithyBuildConfig) + inputs.file(smithyBuildConfig) + // outputs.dir(...) + + + // still need to port over from sdk build.gradle.kts + // FIXME - set smithyKotlin runtime version to match + // FIXME - ensure smithy-aws-kotlin codegen is up to date + // FIXME - delete smithy-build.json if it exists? + } + + project.tasks.register("$projectionName-codegen") { + dependsOn(buildTask) + this.projectionName = projectionName + description = "generate code for $projectionName" + } +} + +/** + * Generate the "smithy-build.json" defining the projection + */ +private fun generateSmithyBuild(projectionName: String, configuration: ProjectionConfiguration): String { + val imports = configuration.imports!!.joinToString { "\"$it\"" } + val projection = """ + "$projectionName": { + "imports": [$imports], + "plugins": { + "kotlin-codegen": ${configuration.pluginSettings!!} + } + } + """.trimIndent() + + return """ + { + "version": "1.0", + "projections": { + $projection + } + } + """.trimIndent() +} + + +//data class KotlinCodegenProjection( +// val serviceShapeId: String, +// val packageName: String, +// val packageVersion: String, +// val packageDescription: String, +// val sdkId: String, +// val buildSettings: ?? +//) +private fun Project.createCodegenConfiguration(): Configuration { + val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") + + dependencies { + // depend on aws-kotlin code generation + codegenConfig(project(":codegen:smithy-aws-kotlin-codegen")) + + // smithy plugin requires smithy-cli to be on the classpath, for whatever reason configuring the plugin + // from this plugin doesn't work correctly so we explicitly set it + val smithyVersion: String by project + codegenConfig("software.amazon.smithy:smithy-cli:$smithyVersion") + + // add aws traits to the compile classpath so that the smithy build task can discover them + codegenConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") + } + + return codegenConfig +} + +//abstract class CodegenTask: DefaultTask() { +// @get:Input +// abstract var projectionName: String +// +// @get:Input +// abstract var imports: List +// +// /** +// * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the +// * plugin settings for smithy kotlin. +// * +// * Example: +// * ```json +// * { +// * "service": , +// * "package": { +// * "name": , +// * "version": +// * "description": +// * }, +// * "sdkId": (Optional: defaults to shape id if not set), +// * "build": { } +// * } +// * ``` +// */ +// @get:Input +// abstract var pluginSettings: String +// +//// @get:Input +//// abstract val outputDir: Property +// +// private val smithyBuildTask: SmithyBuild +// +// init { +// group = "codegen" +// description = "Generate code using smithy-kotlin" +// // create backing tasks for this +// smithyBuildTask = project.registerSmithyTasks(name) +// } +// +// // TODO - create custom transform to include specific operations +// +// +// private fun Project.registerSmithyTasks(taskName: String): SmithyBuild { +// // generate the projection file for smithy to consume +// val smithyBuildConfig = buildDir.resolve("smithy-build-$taskName.json") +// val generateSmithyBuild = tasks.create("$taskName-smithyBuildJson") { +// description = "generate smithy-build.json" +// group = "codegen" +// outputs.file(smithyBuildConfig) +// doFirst { +// buildDir.mkdir() +// if (smithyBuildConfig.exists()) { +// smithyBuildConfig.delete() +// } +// } +// doLast { +// smithyBuildConfig.writeText(generateSmithyBuild(projectionName)) +// } +// } +// +// val codegenConfig = createCodegenConfiguration() +// val buildTask = project.tasks.create("$taskName-smithyBuild") { +// dependsOn(generateSmithyBuild) +// description = "generate code for $name task" +// group = "codegen" +// classpath = codegenConfig +// smithyBuildConfigs = files(smithyBuildConfig) +// inputs.file(smithyBuildConfig) +// // outputs.dir(...) +// +// +// // still need to port over from sdk build.gradle.kts +// // FIXME - set smithyKotlin runtime version to match +// // FIXME - ensure smithy-aws-kotlin codegen is up to date +// // FIXME - delete smithy-build.json if it exists? +// } +// return buildTask +// } +// +// @TaskAction +// fun generateCode() { +// logger.info("generating code for projection: $projectionName") +// smithyBuildTask.execute() +// } +// +// +// private fun generateSmithyBuild(projectionName: String): String { +// val projection = """ +// "$projectionName": { +// "imports": [${imports.joinToString()}], +// "plugins": { +// "kotlin-codegen": $pluginSettings +// } +// } +// """.trimIndent() +// +// return """ +// { +// "version": "1.0", +// "projections": { +// $projection +// } +// } +// """.trimIndent() +// } +//} + +abstract class CodegenTask: DefaultTask() { + @get:Input + abstract var projectionName: String + + init { + group = "codegen" + description = "Generate code using smithy-kotlin" + } + + + + @TaskAction + fun generateCode() { + logger.info("generating code for projection: $projectionName") + // NOTE: this task has dependencies on a smithy build task for the projection + } + +} diff --git a/settings.gradle.kts b/settings.gradle.kts index 99c1ab3f9f3..4fb9ab34839 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -23,6 +23,8 @@ pluginManagement { rootProject.name = "aws-sdk-kotlin" +includeBuild("./gradle/sdk-plugins") + include(":dokka-aws") include(":codegen:sdk") include(":codegen:smithy-aws-kotlin-codegen") From e028eaf90bb616e3863688ac091f9c384db3a6e8 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 17 Nov 2021 16:20:54 -0500 Subject: [PATCH 02/31] cleanup --- aws-runtime/aws-config/build.gradle.kts | 30 +- .../kotlin/build/plugin/CodegenExtension.kt | 176 ++++++++++ .../sdk/kotlin/build/plugin/CodegenPlugin.kt | 304 ------------------ .../kotlin/build/plugin/tasks/CodegenTask.kt | 29 ++ 4 files changed, 206 insertions(+), 333 deletions(-) create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index b3c5dbec7e6..7bef8605d1a 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -77,8 +77,6 @@ codegen { awsModelFile("sts.2011-06-15.json") ) - // FIXME - we could make this a typed object if we want or even re-use smithy-kotlin type - // language=JSON pluginSettings = """ { "service": "com.amazonaws.sts#AWSSecurityTokenServiceV20110615", @@ -96,7 +94,7 @@ codegen { } } -val codegenTasks = tasks.withType() +val codegenTasks = tasks.withType() tasks.withType { dependsOn(codegenTasks) } @@ -109,29 +107,3 @@ codegen.projections { kotlin.srcDir(projectedSrcDir) } } - - -//tasks.register("generateCredentialProviders") { -// description = "generate STS and SSO credential providers for aws-config to consume" -// projectionName = "sts-credentials-provider" -// -// imports = listOf( -// awsModelFile("sts.2011-06-15.json") -// ) -// -// pluginSettings = """ -// { -// "service": ""com.amazonaws.sts#AWSSecurityTokenServiceV20110615", -// "package" : { -// "name": "aws.sdk.kotlin.runtime.auth.credentials.internal.sts", -// "version": "$version", -// "description": "Internal STS credentials provider" -// }, -// "sdkId": "Sts", -// "build": { -// "generateDefaultBuildFiles": false -// } -// } -// """.trimIndent() -// -//} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt new file mode 100644 index 00000000000..2584e8c3286 --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt @@ -0,0 +1,176 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.build.plugin + +import aws.sdk.kotlin.build.plugin.tasks.CodegenTask +import org.gradle.api.Action +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.provideDelegate +import org.gradle.kotlin.dsl.register +import software.amazon.smithy.gradle.tasks.SmithyBuild + +open class CodegenExtension(private val project: Project) { + private val projections = mutableListOf() + + fun projection(name: String, configure: Action) { + val pc = ProjectionConfiguration() + configure.execute(pc) + // register codegen tasks for projection + project.registerCodegenTasksForProjection(name, pc) + + projections.add(KotlinCodegenProjection(name, project)) + } + + fun projections(action: Action) = projections.forEach { action.execute(it) } + + fun getProjection(name: String): KotlinCodegenProjection? = projections.find { it.name == name } +} + +///** +// * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) +// */ +//class PostProcessSpec { +// +// +//} + + +class ProjectionConfiguration{ + /** + * List of files/directories to import when building the projection + * + * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections + */ + var imports: List? = null + + + /** + * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the + * plugin settings for smithy kotlin. + * + * Example: + * ```json + * { + * "service": , + * "package": { + * "name": , + * "version": + * "description": + * }, + * "sdkId": (Optional: defaults to shape id if not set), + * "build": { } + * } + * ``` + */ + // FIXME - we could make this a typed object if we want or even re-use smithy-kotlin type + var pluginSettings: String? = null + + // private var postProcessSpec: PostProcessSpec? = null + // fun postProcess(spec: PostProcessSpec.() -> Unit) { + // postProcessSpec = PostProcessSpec().apply(spec) + // } +} + +class KotlinCodegenProjection( + val name: String, + private val project: Project +) { + /** + * Root directory for this projection + */ + val projectionRootDir: java.io.File + get() = project.file("${project.buildDir}/smithyprojections/${project.name}/${name}/kotlin-codegen") + +} + +private fun Project.registerCodegenTasksForProjection(projectionName: String, configuration: ProjectionConfiguration) { + // generate the projection file for smithy to consume + val smithyBuildConfig = buildDir.resolve("smithy-build-$projectionName.json") + val generateSmithyBuild = tasks.register("$projectionName-smithyBuildJson") { + description = "generate smithy-build.json" + group = "codegen" + outputs.file(smithyBuildConfig) + inputs.property("$projectionName-configuration", configuration.pluginSettings) + doFirst { + if (smithyBuildConfig.exists()) { + smithyBuildConfig.delete() + } + } + doLast { + buildDir.mkdir() + smithyBuildConfig.writeText(generateSmithyBuild(projectionName, configuration)) + } + } + + val codegenConfig = createCodegenConfiguration() + val buildTask = project.tasks.register("$projectionName-smithyBuild") { + dependsOn(generateSmithyBuild) + description = "generate code for $name task" + group = "codegen" + classpath = codegenConfig + smithyBuildConfigs = files(smithyBuildConfig) + inputs.file(smithyBuildConfig) + // outputs.dir(...) + + // ensure smithy-aws-kotlin-codegen is up to date + inputs.files(codegenConfig) + } + + project.tasks.register("$projectionName-codegen") { + dependsOn(buildTask) + this.projectionName = projectionName + description = "generate code for $projectionName" + } +} + +/** + * Generate the "smithy-build.json" defining the projection + */ +private fun generateSmithyBuild(projectionName: String, configuration: ProjectionConfiguration): String { + val imports = configuration.imports!!.joinToString { "\"$it\"" } + val projection = """ + "$projectionName": { + "imports": [$imports], + "plugins": { + "kotlin-codegen": ${configuration.pluginSettings!!} + } + } + """.trimIndent() + + return """ + { + "version": "1.0", + "projections": { + $projection + } + } + """.trimIndent() +} + + +// create a configuration (classpath) needed by the SmithyBuild task +private fun Project.createCodegenConfiguration(): Configuration { + val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") + + dependencies { + // depend on aws-kotlin code generation + codegenConfig(project(":codegen:smithy-aws-kotlin-codegen")) + + // smithy plugin requires smithy-cli to be on the classpath, for whatever reason configuring the plugin + // from this plugin doesn't work correctly so we explicitly set it + val smithyVersion: String by project + codegenConfig("software.amazon.smithy:smithy-cli:$smithyVersion") + + // add aws traits to the compile classpath so that the smithy build task can discover them + codegenConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") + } + + return codegenConfig +} + + diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt index 4db10896fc4..446da7209f7 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt @@ -5,17 +5,8 @@ package aws.sdk.kotlin.build.plugin -import org.gradle.api.Action -import org.gradle.api.DefaultTask import org.gradle.api.Plugin import org.gradle.api.Project -import org.gradle.api.artifacts.Configuration -import org.gradle.api.tasks.Input -import org.gradle.api.tasks.TaskAction -import org.gradle.kotlin.dsl.dependencies -import org.gradle.kotlin.dsl.provideDelegate -import org.gradle.kotlin.dsl.register -import software.amazon.smithy.gradle.tasks.SmithyBuild const val CODEGEN_EXTENSION_NAME = "codegen" // TODO - create custom transform to include specific operations @@ -44,298 +35,3 @@ class CodegenPlugin : Plugin { } } -///** -// * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) -// */ -//class PostProcessSpec { -// -// -//} - - -class ProjectionConfiguration{ - /** - * List of files/directories to import when building the projection - * - * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections - */ - var imports: List? = null - - - /** - * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the - * plugin settings for smithy kotlin. - * - * Example: - * ```json - * { - * "service": , - * "package": { - * "name": , - * "version": - * "description": - * }, - * "sdkId": (Optional: defaults to shape id if not set), - * "build": { } - * } - * ``` - */ - var pluginSettings: String? = null - -// private var postProcessSpec: PostProcessSpec? = null -// fun postProcess(spec: PostProcessSpec.() -> Unit) { -// postProcessSpec = PostProcessSpec().apply(spec) -// } -} - -class KotlinCodegenProjection( - val name: String, - private val project: Project -) { - /** - * Root directory for this projection - */ - val projectionRootDir: java.io.File - get() = project.file("${project.buildDir}/smithyprojections/${project.name}/${name}/kotlin-codegen") - -} - -open class CodegenExtension(private val project: Project) { - private val projections = mutableListOf() - - fun projection(name: String, configure: Action) { - val pc = ProjectionConfiguration() - configure.execute(pc) - // register codegen tasks for projection - project.registerCodegenTasksForProjection(name, pc) - - projections.add(KotlinCodegenProjection(name, project)) - } - - fun projections(action: Action) = projections.forEach { action.execute(it) } - - fun getProjection(name: String): KotlinCodegenProjection? = projections.find { it.name == name } -} - - -private fun Project.registerCodegenTasksForProjection(projectionName: String, configuration: ProjectionConfiguration) { - // generate the projection file for smithy to consume - val smithyBuildConfig = buildDir.resolve("smithy-build-$projectionName.json") - val generateSmithyBuild = tasks.register("$projectionName-smithyBuildJson") { - description = "generate smithy-build.json" - group = "codegen" - outputs.file(smithyBuildConfig) - inputs.property("$projectionName-configuration", configuration.pluginSettings) - doFirst { - if (smithyBuildConfig.exists()) { - smithyBuildConfig.delete() - } - } - doLast { - buildDir.mkdir() - smithyBuildConfig.writeText(generateSmithyBuild(projectionName, configuration)) - } - } - - val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register("$projectionName-smithyBuild") { - dependsOn(generateSmithyBuild) - description = "generate code for $name task" - group = "codegen" - classpath = codegenConfig - smithyBuildConfigs = files(smithyBuildConfig) - inputs.file(smithyBuildConfig) - // outputs.dir(...) - - - // still need to port over from sdk build.gradle.kts - // FIXME - set smithyKotlin runtime version to match - // FIXME - ensure smithy-aws-kotlin codegen is up to date - // FIXME - delete smithy-build.json if it exists? - } - - project.tasks.register("$projectionName-codegen") { - dependsOn(buildTask) - this.projectionName = projectionName - description = "generate code for $projectionName" - } -} - -/** - * Generate the "smithy-build.json" defining the projection - */ -private fun generateSmithyBuild(projectionName: String, configuration: ProjectionConfiguration): String { - val imports = configuration.imports!!.joinToString { "\"$it\"" } - val projection = """ - "$projectionName": { - "imports": [$imports], - "plugins": { - "kotlin-codegen": ${configuration.pluginSettings!!} - } - } - """.trimIndent() - - return """ - { - "version": "1.0", - "projections": { - $projection - } - } - """.trimIndent() -} - - -//data class KotlinCodegenProjection( -// val serviceShapeId: String, -// val packageName: String, -// val packageVersion: String, -// val packageDescription: String, -// val sdkId: String, -// val buildSettings: ?? -//) -private fun Project.createCodegenConfiguration(): Configuration { - val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") - - dependencies { - // depend on aws-kotlin code generation - codegenConfig(project(":codegen:smithy-aws-kotlin-codegen")) - - // smithy plugin requires smithy-cli to be on the classpath, for whatever reason configuring the plugin - // from this plugin doesn't work correctly so we explicitly set it - val smithyVersion: String by project - codegenConfig("software.amazon.smithy:smithy-cli:$smithyVersion") - - // add aws traits to the compile classpath so that the smithy build task can discover them - codegenConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") - } - - return codegenConfig -} - -//abstract class CodegenTask: DefaultTask() { -// @get:Input -// abstract var projectionName: String -// -// @get:Input -// abstract var imports: List -// -// /** -// * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the -// * plugin settings for smithy kotlin. -// * -// * Example: -// * ```json -// * { -// * "service": , -// * "package": { -// * "name": , -// * "version": -// * "description": -// * }, -// * "sdkId": (Optional: defaults to shape id if not set), -// * "build": { } -// * } -// * ``` -// */ -// @get:Input -// abstract var pluginSettings: String -// -//// @get:Input -//// abstract val outputDir: Property -// -// private val smithyBuildTask: SmithyBuild -// -// init { -// group = "codegen" -// description = "Generate code using smithy-kotlin" -// // create backing tasks for this -// smithyBuildTask = project.registerSmithyTasks(name) -// } -// -// // TODO - create custom transform to include specific operations -// -// -// private fun Project.registerSmithyTasks(taskName: String): SmithyBuild { -// // generate the projection file for smithy to consume -// val smithyBuildConfig = buildDir.resolve("smithy-build-$taskName.json") -// val generateSmithyBuild = tasks.create("$taskName-smithyBuildJson") { -// description = "generate smithy-build.json" -// group = "codegen" -// outputs.file(smithyBuildConfig) -// doFirst { -// buildDir.mkdir() -// if (smithyBuildConfig.exists()) { -// smithyBuildConfig.delete() -// } -// } -// doLast { -// smithyBuildConfig.writeText(generateSmithyBuild(projectionName)) -// } -// } -// -// val codegenConfig = createCodegenConfiguration() -// val buildTask = project.tasks.create("$taskName-smithyBuild") { -// dependsOn(generateSmithyBuild) -// description = "generate code for $name task" -// group = "codegen" -// classpath = codegenConfig -// smithyBuildConfigs = files(smithyBuildConfig) -// inputs.file(smithyBuildConfig) -// // outputs.dir(...) -// -// -// // still need to port over from sdk build.gradle.kts -// // FIXME - set smithyKotlin runtime version to match -// // FIXME - ensure smithy-aws-kotlin codegen is up to date -// // FIXME - delete smithy-build.json if it exists? -// } -// return buildTask -// } -// -// @TaskAction -// fun generateCode() { -// logger.info("generating code for projection: $projectionName") -// smithyBuildTask.execute() -// } -// -// -// private fun generateSmithyBuild(projectionName: String): String { -// val projection = """ -// "$projectionName": { -// "imports": [${imports.joinToString()}], -// "plugins": { -// "kotlin-codegen": $pluginSettings -// } -// } -// """.trimIndent() -// -// return """ -// { -// "version": "1.0", -// "projections": { -// $projection -// } -// } -// """.trimIndent() -// } -//} - -abstract class CodegenTask: DefaultTask() { - @get:Input - abstract var projectionName: String - - init { - group = "codegen" - description = "Generate code using smithy-kotlin" - } - - - - @TaskAction - fun generateCode() { - logger.info("generating code for projection: $projectionName") - // NOTE: this task has dependencies on a smithy build task for the projection - } - -} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt new file mode 100644 index 00000000000..c00ab198eea --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt @@ -0,0 +1,29 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.build.plugin.tasks + +import org.gradle.api.DefaultTask +import org.gradle.api.tasks.Input +import org.gradle.api.tasks.TaskAction + +abstract class CodegenTask: DefaultTask() { + @get:Input + abstract var projectionName: String + + init { + group = "codegen" + description = "Generate code using smithy-kotlin" + } + + + + @TaskAction + fun generateCode() { + logger.info("generating code for projection: $projectionName") + // NOTE: this task has dependencies on a smithy build task for the projection + } + +} \ No newline at end of file From 773416ce5f5785369b6b58616c92095f424ad1af Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 17 Nov 2021 16:40:32 -0500 Subject: [PATCH 03/31] refactor package --- aws-runtime/aws-config/build.gradle.kts | 8 +++++--- gradle/sdk-plugins/build.gradle.kts | 2 +- .../kotlin/{build/plugin => gradle}/CodegenExtension.kt | 4 ++-- .../sdk/kotlin/{build/plugin => gradle}/CodegenPlugin.kt | 2 +- .../kotlin/{build/plugin => gradle}/tasks/CodegenTask.kt | 2 +- 5 files changed, 10 insertions(+), 8 deletions(-) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/{build/plugin => gradle}/CodegenExtension.kt (98%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/{build/plugin => gradle}/CodegenPlugin.kt (97%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/{build/plugin => gradle}/tasks/CodegenTask.kt (93%) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 7bef8605d1a..7a01380fa5e 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -94,9 +94,11 @@ codegen { } } -val codegenTasks = tasks.withType() -tasks.withType { - dependsOn(codegenTasks) +val codegenTasks = tasks.withType() +tasks.withType { +// dependsOn(codegenTasks) + // FIXME - this isn't working correctly, it generates the code before compile fine but it's recompiling everytime still + codegenTasks.forEach { dependsOn(it) } } codegen.projections { diff --git a/gradle/sdk-plugins/build.gradle.kts b/gradle/sdk-plugins/build.gradle.kts index 00fd4891939..105f74e21e9 100644 --- a/gradle/sdk-plugins/build.gradle.kts +++ b/gradle/sdk-plugins/build.gradle.kts @@ -50,7 +50,7 @@ gradlePlugin { plugins { val awsCodegenPlugin by creating { id = "aws.sdk.kotlin.codegen" - implementationClass = "aws.sdk.kotlin.build.plugin.CodegenPlugin" + implementationClass = "aws.sdk.kotlin.gradle.CodegenPlugin" } } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt similarity index 98% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index 2584e8c3286..f1888f48d1c 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.build.plugin +package aws.sdk.kotlin.gradle -import aws.sdk.kotlin.build.plugin.tasks.CodegenTask +import aws.sdk.kotlin.gradle.tasks.CodegenTask import org.gradle.api.Action import org.gradle.api.Project import org.gradle.api.artifacts.Configuration diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt similarity index 97% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt index 446da7209f7..0a5c48f9271 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.build.plugin +package aws.sdk.kotlin.gradle import org.gradle.api.Plugin import org.gradle.api.Project diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt similarity index 93% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt index c00ab198eea..696f6960bd9 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/build/plugin/tasks/CodegenTask.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.build.plugin.tasks +package aws.sdk.kotlin.gradle.tasks import org.gradle.api.DefaultTask import org.gradle.api.tasks.Input From fa3f0cb8ceeb3c1f6e7ef9777cb7734f5ef434f4 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 17 Nov 2021 16:42:45 -0500 Subject: [PATCH 04/31] add todo --- aws-runtime/aws-config/build.gradle.kts | 2 ++ 1 file changed, 2 insertions(+) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 7a01380fa5e..a91e63361de 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -103,6 +103,8 @@ tasks.withType { codegen.projections { // add this projected source dir to the common sourceSet + // TODO- build.gradle.kts is still being generated, it's NOT used though, we should probably either have a postProcessing spec or a + // plugin setting to not generate it to avoid confusion val projectedSrcDir = projectionRootDir.resolve("src/main/kotlin") kotlin.sourceSets.commonMain { println("add $projectedSrcDir to common sourceSet") From 13b0c805e7328f1c082e8abe38795f6a8e879e49 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 17 Nov 2021 16:48:21 -0500 Subject: [PATCH 05/31] remove erroneous kotlin version --- aws-runtime/aws-config/build.gradle.kts | 3 +++ gradle/sdk-plugins/build.gradle.kts | 3 --- .../main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index a91e63361de..55a79b4a0c6 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -92,6 +92,9 @@ codegen { } """.trimIndent() } + + // TODO - to re-use this infrastracture in say sdk bootstrap or protocol tests it would be useful to + // have a way to completely control the projection } val codegenTasks = tasks.withType() diff --git a/gradle/sdk-plugins/build.gradle.kts b/gradle/sdk-plugins/build.gradle.kts index 105f74e21e9..3bae7d55624 100644 --- a/gradle/sdk-plugins/build.gradle.kts +++ b/gradle/sdk-plugins/build.gradle.kts @@ -13,8 +13,6 @@ buildscript { plugins { `kotlin-dsl` `java-gradle-plugin` - // FIXME - can we get this to sync with gradle.properties from root project - kotlin("jvm") version "1.5.31" } repositories { @@ -35,7 +33,6 @@ fun loadVersions() { // use the versions from aws-sdk-kotlin/gradle.properties loadVersions() -val kotlinVersion: String by project val smithyVersion: String by project val smithyGradleVersion: String by project diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt index 696f6960bd9..f4d5da9e89d 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt @@ -18,12 +18,12 @@ abstract class CodegenTask: DefaultTask() { description = "Generate code using smithy-kotlin" } - - @TaskAction fun generateCode() { logger.info("generating code for projection: $projectionName") // NOTE: this task has dependencies on a smithy build task for the projection + // it doesn't actually do any work (yet) but it does give us our own task to extend _AFTER_ code + // has been generated } } \ No newline at end of file From c0141e00efc5beb48f6668a0bed46a70f092ed48 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 10:22:21 -0500 Subject: [PATCH 06/31] fix dependencies such that we only recompile when necessary --- aws-runtime/aws-config/build.gradle.kts | 28 ++++++++++++++++--- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 21 +++++++++++++- 2 files changed, 44 insertions(+), 5 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 55a79b4a0c6..f9dd058e83f 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -97,11 +97,31 @@ codegen { // have a way to completely control the projection } +// TODO/NOTE - need `compileKotlinJvm` (Type=KotlinCompile), `compileKotlinMetadata` (Type=KotlinCompileCommon), `sourcesJar` and `jvmSourcesJar` (Type=org.gradle.jvm.tasks.Jar) val codegenTasks = tasks.withType() tasks.withType { -// dependsOn(codegenTasks) - // FIXME - this isn't working correctly, it generates the code before compile fine but it's recompiling everytime still - codegenTasks.forEach { dependsOn(it) } + println("kotlin compile task: $name") + codegenTasks.forEach { + println("adding dependency on codegen task: $it") + dependsOn(it) + } +} + +tasks.withType { + println("KotlinCompileCommon task: $name") + codegenTasks.forEach { + println("adding dependency on codegen task: $it") + dependsOn(it) + } +} + + +tasks.withType { + println("jar explicit task: $name") + codegenTasks.forEach { + println("adding dependency on codegen task: $it") + dependsOn(it) + } } codegen.projections { @@ -110,7 +130,7 @@ codegen.projections { // plugin setting to not generate it to avoid confusion val projectedSrcDir = projectionRootDir.resolve("src/main/kotlin") kotlin.sourceSets.commonMain { - println("add $projectedSrcDir to common sourceSet") + println("added $projectedSrcDir to common sourceSet") kotlin.srcDir(projectedSrcDir) } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index f1888f48d1c..3219d914e96 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -17,6 +17,8 @@ import software.amazon.smithy.gradle.tasks.SmithyBuild open class CodegenExtension(private val project: Project) { private val projections = mutableListOf() + // TODO - typed plugin settings and defaults for all projections + fun projection(name: String, configure: Action) { val pc = ProjectionConfiguration() configure.execute(pc) @@ -41,6 +43,8 @@ open class CodegenExtension(private val project: Project) { class ProjectionConfiguration{ + // TODO - add immutable name property, rename to just Projection + /** * List of files/directories to import when building the projection * @@ -76,6 +80,9 @@ class ProjectionConfiguration{ // } } +internal fun Project.projectionRootDir(projectionName: String): java.io.File + = file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen") + class KotlinCodegenProjection( val name: String, private val project: Project @@ -94,6 +101,7 @@ private fun Project.registerCodegenTasksForProjection(projectionName: String, co val generateSmithyBuild = tasks.register("$projectionName-smithyBuildJson") { description = "generate smithy-build.json" group = "codegen" + outputs.file(smithyBuildConfig) inputs.property("$projectionName-configuration", configuration.pluginSettings) doFirst { @@ -114,14 +122,25 @@ private fun Project.registerCodegenTasksForProjection(projectionName: String, co group = "codegen" classpath = codegenConfig smithyBuildConfigs = files(smithyBuildConfig) + inputs.file(smithyBuildConfig) - // outputs.dir(...) + + // register the model file(s) (imports) + configuration.imports?.forEach { importPath -> + val f = project.file(importPath) + if (f.exists()){ + if (f.isDirectory) inputs.dir(f) else inputs.file(f) + } + } // ensure smithy-aws-kotlin-codegen is up to date inputs.files(codegenConfig) + + outputs.dir(project.projectionRootDir(projectionName)) } project.tasks.register("$projectionName-codegen") { + // FIXME - maybe use the name directly? dependsOn(buildTask) this.projectionName = projectionName description = "generate code for $projectionName" From db7a3214a16d4c266d5b353bd4047183f5a23d2a Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 10:33:38 -0500 Subject: [PATCH 07/31] remove one layer --- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 73 +++++++++---------- 1 file changed, 35 insertions(+), 38 deletions(-) diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index 3219d914e96..a9df45cb675 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -15,22 +15,22 @@ import org.gradle.kotlin.dsl.register import software.amazon.smithy.gradle.tasks.SmithyBuild open class CodegenExtension(private val project: Project) { - private val projections = mutableListOf() + private val projections = mutableMapOf() // TODO - typed plugin settings and defaults for all projections - fun projection(name: String, configure: Action) { - val pc = ProjectionConfiguration() - configure.execute(pc) + fun projection(name: String, configure: Action) { + val p = KotlinCodegenProjection(name, project.projectionRootDir(name)) + configure.execute(p) // register codegen tasks for projection - project.registerCodegenTasksForProjection(name, pc) + project.registerCodegenTasksForProjection(p) - projections.add(KotlinCodegenProjection(name, project)) + projections[name] = p } - fun projections(action: Action) = projections.forEach { action.execute(it) } + fun projections(action: Action) = projections.values.forEach { action.execute(it) } - fun getProjection(name: String): KotlinCodegenProjection? = projections.find { it.name == name } + fun getProjectionByName(name: String): KotlinCodegenProjection? = projections[name] } ///** @@ -42,8 +42,17 @@ open class CodegenExtension(private val project: Project) { //} -class ProjectionConfiguration{ - // TODO - add immutable name property, rename to just Projection +class KotlinCodegenProjection( + /** + * The name of the projection + */ + val name: String, + + /** + * Root directory for this projection + */ + val projectionRootDir: java.io.File +){ /** * List of files/directories to import when building the projection @@ -83,27 +92,15 @@ class ProjectionConfiguration{ internal fun Project.projectionRootDir(projectionName: String): java.io.File = file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen") -class KotlinCodegenProjection( - val name: String, - private val project: Project -) { - /** - * Root directory for this projection - */ - val projectionRootDir: java.io.File - get() = project.file("${project.buildDir}/smithyprojections/${project.name}/${name}/kotlin-codegen") - -} - -private fun Project.registerCodegenTasksForProjection(projectionName: String, configuration: ProjectionConfiguration) { +private fun Project.registerCodegenTasksForProjection(projection: KotlinCodegenProjection) { // generate the projection file for smithy to consume - val smithyBuildConfig = buildDir.resolve("smithy-build-$projectionName.json") - val generateSmithyBuild = tasks.register("$projectionName-smithyBuildJson") { + val smithyBuildConfig = buildDir.resolve("smithy-build-${projection.name}.json") + val generateSmithyBuild = tasks.register("${projection.name}-smithyBuildJson") { description = "generate smithy-build.json" group = "codegen" outputs.file(smithyBuildConfig) - inputs.property("$projectionName-configuration", configuration.pluginSettings) + inputs.property("${projection.name}-configuration", projection.pluginSettings) doFirst { if (smithyBuildConfig.exists()) { smithyBuildConfig.delete() @@ -111,12 +108,12 @@ private fun Project.registerCodegenTasksForProjection(projectionName: String, co } doLast { buildDir.mkdir() - smithyBuildConfig.writeText(generateSmithyBuild(projectionName, configuration)) + smithyBuildConfig.writeText(generateSmithyBuild(projection)) } } val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register("$projectionName-smithyBuild") { + val buildTask = project.tasks.register("${projection.name}-smithyBuild") { dependsOn(generateSmithyBuild) description = "generate code for $name task" group = "codegen" @@ -126,7 +123,7 @@ private fun Project.registerCodegenTasksForProjection(projectionName: String, co inputs.file(smithyBuildConfig) // register the model file(s) (imports) - configuration.imports?.forEach { importPath -> + projection.imports?.forEach { importPath -> val f = project.file(importPath) if (f.exists()){ if (f.isDirectory) inputs.dir(f) else inputs.file(f) @@ -136,13 +133,13 @@ private fun Project.registerCodegenTasksForProjection(projectionName: String, co // ensure smithy-aws-kotlin-codegen is up to date inputs.files(codegenConfig) - outputs.dir(project.projectionRootDir(projectionName)) + outputs.dir(project.projectionRootDir(projection.name)) } - project.tasks.register("$projectionName-codegen") { + project.tasks.register("${projection.name}-codegen") { // FIXME - maybe use the name directly? dependsOn(buildTask) - this.projectionName = projectionName + this.projectionName = projection.name description = "generate code for $projectionName" } } @@ -150,13 +147,13 @@ private fun Project.registerCodegenTasksForProjection(projectionName: String, co /** * Generate the "smithy-build.json" defining the projection */ -private fun generateSmithyBuild(projectionName: String, configuration: ProjectionConfiguration): String { - val imports = configuration.imports!!.joinToString { "\"$it\"" } - val projection = """ - "$projectionName": { +private fun generateSmithyBuild(projection: KotlinCodegenProjection): String { + val imports = projection.imports!!.joinToString { "\"$it\"" } + val config = """ + "${projection.name}": { "imports": [$imports], "plugins": { - "kotlin-codegen": ${configuration.pluginSettings!!} + "kotlin-codegen": ${projection.pluginSettings!!} } } """.trimIndent() @@ -165,7 +162,7 @@ private fun generateSmithyBuild(projectionName: String, configuration: Projectio { "version": "1.0", "projections": { - $projection + $config } } """.trimIndent() From 4cbaae98dc6536a753c4ebda5d04ff3b0b9d7cd9 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 11:03:06 -0500 Subject: [PATCH 08/31] generate multiple projections at once using single task --- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 70 +++++++++++++------ .../aws/sdk/kotlin/gradle/CodegenPlugin.kt | 1 + .../sdk/kotlin/gradle/tasks/CodegenTask.kt | 6 +- 3 files changed, 50 insertions(+), 27 deletions(-) diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index a9df45cb675..58a3addbfd2 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -9,27 +9,39 @@ import aws.sdk.kotlin.gradle.tasks.CodegenTask import org.gradle.api.Action import org.gradle.api.Project import org.gradle.api.artifacts.Configuration +import org.gradle.api.plugins.ExtensionAware import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.get import org.gradle.kotlin.dsl.provideDelegate import org.gradle.kotlin.dsl.register import software.amazon.smithy.gradle.tasks.SmithyBuild +/** + * Register and build Smithy projections + */ open class CodegenExtension(private val project: Project) { - private val projections = mutableMapOf() + internal val projections = mutableMapOf() // TODO - typed plugin settings and defaults for all projections + /** + * Configure a new projection + */ fun projection(name: String, configure: Action) { + println("configuring projection $name") val p = KotlinCodegenProjection(name, project.projectionRootDir(name)) configure.execute(p) - // register codegen tasks for projection - project.registerCodegenTasksForProjection(p) - projections[name] = p } + /** + * Execute [action] for each projection + */ fun projections(action: Action) = projections.values.forEach { action.execute(it) } + /** + * Get a projection by name + */ fun getProjectionByName(name: String): KotlinCodegenProjection? = projections[name] } @@ -59,7 +71,7 @@ class KotlinCodegenProjection( * * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections */ - var imports: List? = null + var imports: List = emptyList() /** @@ -92,15 +104,17 @@ class KotlinCodegenProjection( internal fun Project.projectionRootDir(projectionName: String): java.io.File = file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen") -private fun Project.registerCodegenTasksForProjection(projection: KotlinCodegenProjection) { +private val Project.codegenExtension: CodegenExtension + get() = (this as ExtensionAware).extensions[CODEGEN_EXTENSION_NAME] as CodegenExtension + +internal fun Project.registerCodegenTasks() { // generate the projection file for smithy to consume - val smithyBuildConfig = buildDir.resolve("smithy-build-${projection.name}.json") - val generateSmithyBuild = tasks.register("${projection.name}-smithyBuildJson") { + val smithyBuildConfig = buildDir.resolve("smithy-build.json") + val generateSmithyBuild = tasks.register("generateSmithyBuildJson") { description = "generate smithy-build.json" group = "codegen" outputs.file(smithyBuildConfig) - inputs.property("${projection.name}-configuration", projection.pluginSettings) doFirst { if (smithyBuildConfig.exists()) { smithyBuildConfig.delete() @@ -108,22 +122,26 @@ private fun Project.registerCodegenTasksForProjection(projection: KotlinCodegenP } doLast { buildDir.mkdir() - smithyBuildConfig.writeText(generateSmithyBuild(projection)) + val extension = project.codegenExtension + smithyBuildConfig.writeText(generateSmithyBuild(extension.projections.values)) } } val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register("${projection.name}-smithyBuild") { + val buildTask = project.tasks.register("kotlinCodegenSmithyBuild") { dependsOn(generateSmithyBuild) - description = "generate code for $name task" + description = "generate code using smithy-kotlin" group = "codegen" classpath = codegenConfig smithyBuildConfigs = files(smithyBuildConfig) inputs.file(smithyBuildConfig) + val extension = project.codegenExtension + println("registering imports for kotlinCodegenSmithyBuild: ${extension.projections.keys.joinToString()}") // register the model file(s) (imports) - projection.imports?.forEach { importPath -> + val imports = extension.projections.values.flatMap{ it.imports } + imports.forEach { importPath -> val f = project.file(importPath) if (f.exists()){ if (f.isDirectory) inputs.dir(f) else inputs.file(f) @@ -133,23 +151,28 @@ private fun Project.registerCodegenTasksForProjection(projection: KotlinCodegenP // ensure smithy-aws-kotlin-codegen is up to date inputs.files(codegenConfig) - outputs.dir(project.projectionRootDir(projection.name)) + extension.projections.keys.forEach { projectionName -> + outputs.dir(project.projectionRootDir(projectionName)) + } } - project.tasks.register("${projection.name}-codegen") { - // FIXME - maybe use the name directly? + project.tasks.register("kotlinCodegen") { dependsOn(buildTask) - this.projectionName = projection.name - description = "generate code for $projectionName" + description = "generate code for projections" } } /** * Generate the "smithy-build.json" defining the projection */ -private fun generateSmithyBuild(projection: KotlinCodegenProjection): String { - val imports = projection.imports!!.joinToString { "\"$it\"" } - val config = """ +private fun generateSmithyBuild(projections: Collection): String { + val formattedProjections = projections.joinToString(",") { projection -> + // escape windows paths for valid json + val imports = projection.imports + .map { it.replace("\\", "\\\\") } + .joinToString { "\"$it\"" } + + val config = """ "${projection.name}": { "imports": [$imports], "plugins": { @@ -158,11 +181,14 @@ private fun generateSmithyBuild(projection: KotlinCodegenProjection): String { } """.trimIndent() + config + } + return """ { "version": "1.0", "projections": { - $config + $formattedProjections } } """.trimIndent() diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt index 0a5c48f9271..c24022b89f9 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt @@ -22,6 +22,7 @@ class CodegenPlugin : Plugin { override fun apply(target: Project):Unit = target.run { configurePlugins() installExtension() + registerCodegenTasks() } private fun Project.configurePlugins() { diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt index f4d5da9e89d..d5b3233755f 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt @@ -6,13 +6,9 @@ package aws.sdk.kotlin.gradle.tasks import org.gradle.api.DefaultTask -import org.gradle.api.tasks.Input import org.gradle.api.tasks.TaskAction abstract class CodegenTask: DefaultTask() { - @get:Input - abstract var projectionName: String - init { group = "codegen" description = "Generate code using smithy-kotlin" @@ -20,7 +16,7 @@ abstract class CodegenTask: DefaultTask() { @TaskAction fun generateCode() { - logger.info("generating code for projection: $projectionName") + logger.info("generating kotlin code for projections") // NOTE: this task has dependencies on a smithy build task for the projection // it doesn't actually do any work (yet) but it does give us our own task to extend _AFTER_ code // has been generated From 9607aa5a7d990f6986ad1b84992ff3da06582cbe Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 11:12:52 -0500 Subject: [PATCH 09/31] fix up-to-date check for generating the projection file --- .../main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index 58a3addbfd2..f0ce5521b75 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -114,6 +114,13 @@ internal fun Project.registerCodegenTasks() { description = "generate smithy-build.json" group = "codegen" + // set an input property based on a hash of all the plugin settings to get this task's + // up-to-date checks to work correctly (model files are already an input to the actual build task) + val pluginSettingsHash = project.codegenExtension.projections.values.fold(0){ acc, projection -> + acc + (projection.pluginSettings?.hashCode() ?: 0) + } + + inputs.property("pluginSettingsHash", pluginSettingsHash) outputs.file(smithyBuildConfig) doFirst { if (smithyBuildConfig.exists()) { From 2f0a1cfeb44567187052f1935e9626eba1ee4b9b Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 11:21:58 -0500 Subject: [PATCH 10/31] cleanup package structure a bit --- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 184 ------------------ .../aws/sdk/kotlin/gradle/CodegenPlugin.kt | 1 + .../kotlin/gradle/KotlinCodegenProjection.kt | 61 ++++++ .../kotlin/aws/sdk/kotlin/gradle/Utils.kt | 22 +++ .../sdk/kotlin/gradle/tasks/SmithyTasks.kt | 133 +++++++++++++ 5 files changed, 217 insertions(+), 184 deletions(-) create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index f0ce5521b75..7db65d095f8 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -5,16 +5,8 @@ package aws.sdk.kotlin.gradle -import aws.sdk.kotlin.gradle.tasks.CodegenTask import org.gradle.api.Action import org.gradle.api.Project -import org.gradle.api.artifacts.Configuration -import org.gradle.api.plugins.ExtensionAware -import org.gradle.kotlin.dsl.dependencies -import org.gradle.kotlin.dsl.get -import org.gradle.kotlin.dsl.provideDelegate -import org.gradle.kotlin.dsl.register -import software.amazon.smithy.gradle.tasks.SmithyBuild /** * Register and build Smithy projections @@ -45,181 +37,5 @@ open class CodegenExtension(private val project: Project) { fun getProjectionByName(name: String): KotlinCodegenProjection? = projections[name] } -///** -// * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) -// */ -//class PostProcessSpec { -// -// -//} - - -class KotlinCodegenProjection( - /** - * The name of the projection - */ - val name: String, - - /** - * Root directory for this projection - */ - val projectionRootDir: java.io.File -){ - - /** - * List of files/directories to import when building the projection - * - * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections - */ - var imports: List = emptyList() - - - /** - * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the - * plugin settings for smithy kotlin. - * - * Example: - * ```json - * { - * "service": , - * "package": { - * "name": , - * "version": - * "description": - * }, - * "sdkId": (Optional: defaults to shape id if not set), - * "build": { } - * } - * ``` - */ - // FIXME - we could make this a typed object if we want or even re-use smithy-kotlin type - var pluginSettings: String? = null - - // private var postProcessSpec: PostProcessSpec? = null - // fun postProcess(spec: PostProcessSpec.() -> Unit) { - // postProcessSpec = PostProcessSpec().apply(spec) - // } -} - -internal fun Project.projectionRootDir(projectionName: String): java.io.File - = file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen") - -private val Project.codegenExtension: CodegenExtension - get() = (this as ExtensionAware).extensions[CODEGEN_EXTENSION_NAME] as CodegenExtension - -internal fun Project.registerCodegenTasks() { - // generate the projection file for smithy to consume - val smithyBuildConfig = buildDir.resolve("smithy-build.json") - val generateSmithyBuild = tasks.register("generateSmithyBuildJson") { - description = "generate smithy-build.json" - group = "codegen" - - // set an input property based on a hash of all the plugin settings to get this task's - // up-to-date checks to work correctly (model files are already an input to the actual build task) - val pluginSettingsHash = project.codegenExtension.projections.values.fold(0){ acc, projection -> - acc + (projection.pluginSettings?.hashCode() ?: 0) - } - - inputs.property("pluginSettingsHash", pluginSettingsHash) - outputs.file(smithyBuildConfig) - doFirst { - if (smithyBuildConfig.exists()) { - smithyBuildConfig.delete() - } - } - doLast { - buildDir.mkdir() - val extension = project.codegenExtension - smithyBuildConfig.writeText(generateSmithyBuild(extension.projections.values)) - } - } - - val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register("kotlinCodegenSmithyBuild") { - dependsOn(generateSmithyBuild) - description = "generate code using smithy-kotlin" - group = "codegen" - classpath = codegenConfig - smithyBuildConfigs = files(smithyBuildConfig) - - inputs.file(smithyBuildConfig) - - val extension = project.codegenExtension - println("registering imports for kotlinCodegenSmithyBuild: ${extension.projections.keys.joinToString()}") - // register the model file(s) (imports) - val imports = extension.projections.values.flatMap{ it.imports } - imports.forEach { importPath -> - val f = project.file(importPath) - if (f.exists()){ - if (f.isDirectory) inputs.dir(f) else inputs.file(f) - } - } - - // ensure smithy-aws-kotlin-codegen is up to date - inputs.files(codegenConfig) - - extension.projections.keys.forEach { projectionName -> - outputs.dir(project.projectionRootDir(projectionName)) - } - } - - project.tasks.register("kotlinCodegen") { - dependsOn(buildTask) - description = "generate code for projections" - } -} - -/** - * Generate the "smithy-build.json" defining the projection - */ -private fun generateSmithyBuild(projections: Collection): String { - val formattedProjections = projections.joinToString(",") { projection -> - // escape windows paths for valid json - val imports = projection.imports - .map { it.replace("\\", "\\\\") } - .joinToString { "\"$it\"" } - - val config = """ - "${projection.name}": { - "imports": [$imports], - "plugins": { - "kotlin-codegen": ${projection.pluginSettings!!} - } - } - """.trimIndent() - - config - } - - return """ - { - "version": "1.0", - "projections": { - $formattedProjections - } - } - """.trimIndent() -} - - -// create a configuration (classpath) needed by the SmithyBuild task -private fun Project.createCodegenConfiguration(): Configuration { - val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") - - dependencies { - // depend on aws-kotlin code generation - codegenConfig(project(":codegen:smithy-aws-kotlin-codegen")) - - // smithy plugin requires smithy-cli to be on the classpath, for whatever reason configuring the plugin - // from this plugin doesn't work correctly so we explicitly set it - val smithyVersion: String by project - codegenConfig("software.amazon.smithy:smithy-cli:$smithyVersion") - - // add aws traits to the compile classpath so that the smithy build task can discover them - codegenConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") - } - - return codegenConfig -} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt index c24022b89f9..cdaa7e60b98 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.gradle +import aws.sdk.kotlin.gradle.tasks.registerCodegenTasks import org.gradle.api.Plugin import org.gradle.api.Project diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt new file mode 100644 index 00000000000..767ad2ff47a --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt @@ -0,0 +1,61 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.gradle + +///** +// * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) +// */ +//class PostProcessSpec { +// +// +//} + +class KotlinCodegenProjection( + /** + * The name of the projection + */ + val name: String, + + /** + * Root directory for this projection + */ + val projectionRootDir: java.io.File +){ + + /** + * List of files/directories to import when building the projection + * + * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections + */ + var imports: List = emptyList() + + + /** + * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the + * plugin settings for smithy kotlin. + * + * Example: + * ```json + * { + * "service": , + * "package": { + * "name": , + * "version": + * "description": + * }, + * "sdkId": (Optional: defaults to shape id if not set), + * "build": { } + * } + * ``` + */ + // FIXME - we could make this a typed object if we want or even re-use smithy-kotlin type + var pluginSettings: String? = null + + // private var postProcessSpec: PostProcessSpec? = null + // fun postProcess(spec: PostProcessSpec.() -> Unit) { + // postProcessSpec = PostProcessSpec().apply(spec) + // } +} \ No newline at end of file diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt new file mode 100644 index 00000000000..ff633b4fc96 --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt @@ -0,0 +1,22 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.gradle + +import org.gradle.api.Project +import org.gradle.api.plugins.ExtensionAware +import org.gradle.kotlin.dsl.get + +/** + * Get the root directory of the generated kotlin code for a projection + */ +internal fun Project.projectionRootDir(projectionName: String): java.io.File + = file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen") + +/** + * Get the [CodegenExtension] instance configured for the project + */ +internal val Project.codegenExtension: CodegenExtension + get() = ((this as ExtensionAware).extensions[CODEGEN_EXTENSION_NAME] as? CodegenExtension) ?: error("CodegenPlugin has not been applied") diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt new file mode 100644 index 00000000000..95bba1c7090 --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt @@ -0,0 +1,133 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.gradle.tasks + +import aws.sdk.kotlin.gradle.KotlinCodegenProjection +import aws.sdk.kotlin.gradle.codegenExtension +import aws.sdk.kotlin.gradle.projectionRootDir +import org.gradle.api.Project +import org.gradle.api.artifacts.Configuration +import org.gradle.kotlin.dsl.dependencies +import org.gradle.kotlin.dsl.provideDelegate +import org.gradle.kotlin.dsl.register +import software.amazon.smithy.gradle.tasks.SmithyBuild + +internal fun Project.registerCodegenTasks() { + // generate the projection file for smithy to consume + val smithyBuildConfig = buildDir.resolve("smithy-build.json") + val generateSmithyBuild = tasks.register("kotlinCodegenGenerateBuildConfig") { + description = "generate smithy-build.json" + group = "codegen" + + // set an input property based on a hash of all the plugin settings to get this task's + // up-to-date checks to work correctly (model files are already an input to the actual build task) + val pluginSettingsHash = project.codegenExtension.projections.values.fold(0){ acc, projection -> + acc + (projection.pluginSettings?.hashCode() ?: 0) + } + + inputs.property("pluginSettingsHash", pluginSettingsHash) + outputs.file(smithyBuildConfig) + doFirst { + if (smithyBuildConfig.exists()) { + smithyBuildConfig.delete() + } + } + doLast { + buildDir.mkdir() + val extension = project.codegenExtension + smithyBuildConfig.writeText(generateSmithyBuild(extension.projections.values)) + } + } + + val codegenConfig = createCodegenConfiguration() + val buildTask = project.tasks.register("kotlinCodegenSmithyBuild") { + dependsOn(generateSmithyBuild) + description = "generate code using smithy-kotlin" + group = "codegen" + classpath = codegenConfig + smithyBuildConfigs = files(smithyBuildConfig) + + inputs.file(smithyBuildConfig) + + val extension = project.codegenExtension + println("registering imports for kotlinCodegenSmithyBuild: ${extension.projections.keys.joinToString()}") + // register the model file(s) (imports) + val imports = extension.projections.values.flatMap{ it.imports } + imports.forEach { importPath -> + val f = project.file(importPath) + if (f.exists()){ + if (f.isDirectory) inputs.dir(f) else inputs.file(f) + } + } + + // ensure smithy-aws-kotlin-codegen is up to date + inputs.files(codegenConfig) + + extension.projections.keys.forEach { projectionName -> + outputs.dir(project.projectionRootDir(projectionName)) + } + } + + project.tasks.register("kotlinCodegen") { + dependsOn(buildTask) + description = "generate code for projections" + } +} + +/** + * Generate the "smithy-build.json" defining the projection + */ +private fun generateSmithyBuild(projections: Collection): String { + val formattedProjections = projections.joinToString(",") { projection -> + // escape windows paths for valid json + val imports = projection.imports + .map { it.replace("\\", "\\\\") } + .joinToString { "\"$it\"" } + + val config = """ + "${projection.name}": { + "imports": [$imports], + "plugins": { + "kotlin-codegen": ${projection.pluginSettings!!} + } + } + """.trimIndent() + + config + } + + return """ + { + "version": "1.0", + "projections": { + $formattedProjections + } + } + """.trimIndent() +} + + +// create a configuration (classpath) needed by the SmithyBuild task +private fun Project.createCodegenConfiguration(): Configuration { + val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") + + dependencies { + // depend on aws-kotlin code generation + codegenConfig(project(":codegen:smithy-aws-kotlin-codegen")) + + // smithy plugin requires smithy-cli to be on the classpath, for whatever reason configuring the plugin + // from this plugin doesn't work correctly so we explicitly set it + val smithyVersion: String by project + codegenConfig("software.amazon.smithy:smithy-cli:$smithyVersion") + + // add aws traits to the compile classpath so that the smithy build task can discover them + codegenConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") + } + + return codegenConfig +} + + From 6ff978ec9586a02e5ff77180748d448aba2edce1 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 12:10:11 -0500 Subject: [PATCH 11/31] add typed settings and register sso projection --- aws-runtime/aws-config/build.gradle.kts | 82 ++++++++++--------- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 11 ++- .../kotlin/gradle/KotlinCodegenProjection.kt | 61 +++++++++----- .../sdk/kotlin/gradle/tasks/SmithyTasks.kt | 26 +++++- 4 files changed, 121 insertions(+), 59 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index f9dd058e83f..c5832af4848 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -36,13 +36,15 @@ kotlin { - // generated sts provider + // additional dependencies required by generated sts provider implementation("aws.smithy.kotlin:serde-form-url:$smithyKotlinVersion") implementation("aws.smithy.kotlin:serde-xml:$smithyKotlinVersion") - implementation("aws.smithy.kotlin:utils:$smithyKotlinVersion") implementation(project(":aws-runtime:protocols:aws-xml-protocols")) implementation(project(":aws-runtime:aws-endpoint")) implementation(project(":aws-runtime:aws-signing")) + + // additional dependencies required by generated sso provider + implementation(project(":aws-runtime:protocols:aws-json-protocols")) } } commonTest { @@ -72,62 +74,68 @@ fun awsModelFile(name: String): String = rootProject.file("codegen/sdk/aws-models/$name").absolutePath codegen { + val basePackage = "aws.sdk.kotlin.runtime.auth.credentials.internal" + val sharedBuildSettings = mapOf( + "generateDefaultBuildFiles" to false + ) + + // generate an sts client projection("sts-credentials-provider") { imports = listOf( awsModelFile("sts.2011-06-15.json") ) - pluginSettings = """ - { - "service": "com.amazonaws.sts#AWSSecurityTokenServiceV20110615", - "package" : { - "name": "aws.sdk.kotlin.runtime.auth.credentials.internal.sts", - "version": "$version", - "description": "Internal STS credentials provider" - }, - "sdkId": "STS", - "build": { - "generateDefaultBuildFiles": false - } - } - """.trimIndent() + pluginSettings { + serviceShapeId = "com.amazonaws.sts#AWSSecurityTokenServiceV20110615" + packageName = "${basePackage}.sts" + packageVersion = project.version.toString() + packageDescription = "Internal STS credentials provider" + sdkId = "STS" + buildSettings = sharedBuildSettings + } } - // TODO - to re-use this infrastracture in say sdk bootstrap or protocol tests it would be useful to - // have a way to completely control the projection + // generate an sso client + projection("sso-credentials-provider") { + imports = listOf( + awsModelFile("sso.2019-06-10.json") + ) + + pluginSettings { + serviceShapeId = "com.amazonaws.sso#SWBPortalService" + packageName = "${basePackage}.sso" + packageVersion = project.version.toString() + packageDescription = "Internal SSO credentials provider" + sdkId = "SSO" + buildSettings = sharedBuildSettings + } + } } -// TODO/NOTE - need `compileKotlinJvm` (Type=KotlinCompile), `compileKotlinMetadata` (Type=KotlinCompileCommon), `sourcesJar` and `jvmSourcesJar` (Type=org.gradle.jvm.tasks.Jar) +/* +NOTE: We need the following tasks to depend on codegen for gradle caching/up-to-date checks to work correctly: + +* `compileKotlinJvm` (Type=KotlinCompile) +* `compileKotlinMetadata` (Type=KotlinCompileCommon) +* `sourcesJar` and `jvmSourcesJar` (Type=org.gradle.jvm.tasks.Jar) +*/ val codegenTasks = tasks.withType() tasks.withType { - println("kotlin compile task: $name") - codegenTasks.forEach { - println("adding dependency on codegen task: $it") - dependsOn(it) - } + dependsOn(codegenTasks) } tasks.withType { - println("KotlinCompileCommon task: $name") - codegenTasks.forEach { - println("adding dependency on codegen task: $it") - dependsOn(it) - } + dependsOn(codegenTasks) } - tasks.withType { - println("jar explicit task: $name") - codegenTasks.forEach { - println("adding dependency on codegen task: $it") - dependsOn(it) - } + dependsOn(codegenTasks) } codegen.projections { // add this projected source dir to the common sourceSet - // TODO- build.gradle.kts is still being generated, it's NOT used though, we should probably either have a postProcessing spec or a - // plugin setting to not generate it to avoid confusion + // NOTE - build.gradle.kts is still being generated, it's NOT used though + // TODO - we should probably either have a postProcessing spec or a plugin setting to not generate it to avoid confusion val projectedSrcDir = projectionRootDir.resolve("src/main/kotlin") kotlin.sourceSets.commonMain { println("added $projectedSrcDir to common sourceSet") diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index 7db65d095f8..e242dd2acb6 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -14,7 +14,16 @@ import org.gradle.api.Project open class CodegenExtension(private val project: Project) { internal val projections = mutableMapOf() - // TODO - typed plugin settings and defaults for all projections + +// FIXME - not all settings make sense to be defaulted... +// /** +// * Set default plugin settings that each projection uses if no value is set +// */ +// private var defaultPluginSettings: SmithyKotlinPluginSettings? = null +// fun defaultPluginSettings(configure: SmithyKotlinPluginSettings.() -> Unit): Unit { +// if (defaultPluginSettings == null) { defaultPluginSettings = SmithyKotlinPluginSettings() } +// defaultPluginSettings!!.apply(configure) +// } /** * Configure a new projection diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt index 767ad2ff47a..87b99d07e4c 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt @@ -22,7 +22,7 @@ class KotlinCodegenProjection( /** * Root directory for this projection */ - val projectionRootDir: java.io.File + val projectionRootDir: java.io.File, ){ /** @@ -33,29 +33,52 @@ class KotlinCodegenProjection( var imports: List = emptyList() + internal var pluginSettings: SmithyKotlinPluginSettings = SmithyKotlinPluginSettings() + /** - * Smithy Kotlin plugin settings. This *MUST* be a valid JSON object that conforms to the - * plugin settings for smithy kotlin. - * - * Example: - * ```json - * { - * "service": , - * "package": { - * "name": , - * "version": - * "description": - * }, - * "sdkId": (Optional: defaults to shape id if not set), - * "build": { } - * } - * ``` + * Configure smithy-kotlin plugin settings. */ - // FIXME - we could make this a typed object if we want or even re-use smithy-kotlin type - var pluginSettings: String? = null + fun pluginSettings(configure: SmithyKotlinPluginSettings.() -> Unit): Unit { pluginSettings.also(configure) } + // private var postProcessSpec: PostProcessSpec? = null // fun postProcess(spec: PostProcessSpec.() -> Unit) { // postProcessSpec = PostProcessSpec().apply(spec) // } +} + + +class SmithyKotlinPluginSettings { + var serviceShapeId: String? = null + var packageName: String? = null + var packageVersion: String? = null + var packageDescription: String? = null + var sdkId: String? = null + var buildSettings: Map? = null + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SmithyKotlinPluginSettings + + if (serviceShapeId != other.serviceShapeId) return false + if (packageName != other.packageName) return false + if (packageVersion != other.packageVersion) return false + if (packageDescription != other.packageDescription) return false + if (sdkId != other.sdkId) return false + if (buildSettings != other.buildSettings) return false + + return true + } + + override fun hashCode(): Int { + var result = serviceShapeId?.hashCode() ?: 0 + result = 31 * result + (packageName?.hashCode() ?: 0) + result = 31 * result + (packageVersion?.hashCode() ?: 0) + result = 31 * result + (packageDescription?.hashCode() ?: 0) + result = 31 * result + (sdkId?.hashCode() ?: 0) + result = 31 * result + (buildSettings?.hashCode() ?: 0) + return result + } } \ No newline at end of file diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt index 95bba1c7090..c2f9c158552 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt @@ -25,7 +25,7 @@ internal fun Project.registerCodegenTasks() { // set an input property based on a hash of all the plugin settings to get this task's // up-to-date checks to work correctly (model files are already an input to the actual build task) val pluginSettingsHash = project.codegenExtension.projections.values.fold(0){ acc, projection -> - acc + (projection.pluginSettings?.hashCode() ?: 0) + acc + projection.pluginSettings.hashCode() } inputs.property("pluginSettingsHash", pluginSettingsHash) @@ -87,11 +87,33 @@ private fun generateSmithyBuild(projections: Collection .map { it.replace("\\", "\\\\") } .joinToString { "\"$it\"" } + // TODO - probably need some validation in here... + val buildPairs = projection.pluginSettings.buildSettings?.entries?.joinToString( + separator = ",\n", + prefix = "{", + postfix = "}" + ) { + val value = when(it.value) { + is Boolean -> it.value.toString() + else -> "\"${it.value}\"" + } + """"${it.key}": $value""" + } + val config = """ "${projection.name}": { "imports": [$imports], "plugins": { - "kotlin-codegen": ${projection.pluginSettings!!} + "kotlin-codegen": { + "service": "${projection.pluginSettings.serviceShapeId}", + "package": { + "name": "${projection.pluginSettings.packageName}", + "version": "${projection.pluginSettings.packageVersion}", + "description": "${projection.pluginSettings.packageDescription}" + }, + "sdkId": "${projection.pluginSettings.sdkId}", + "build": $buildPairs + } } } """.trimIndent() From 2f61f0118fe76955e35102711d9c235e3ac21072 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 12:12:48 -0500 Subject: [PATCH 12/31] lint codegen plugin --- build.gradle.kts | 1 + .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 4 ---- .../kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt | 3 +-- .../sdk/kotlin/gradle/KotlinCodegenProjection.kt | 15 ++++++--------- .../main/kotlin/aws/sdk/kotlin/gradle/Utils.kt | 4 ++-- .../aws/sdk/kotlin/gradle/tasks/CodegenTask.kt | 5 ++--- .../aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt | 13 +++++-------- 7 files changed, 17 insertions(+), 28 deletions(-) diff --git a/build.gradle.kts b/build.gradle.kts index f0bd2475225..ce74ce3bfad 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -150,6 +150,7 @@ val lintPaths = listOf( "aws-runtime/**/*.kt", "examples/**/*.kt", "dokka-aws/**/*.kt", + "gradle/sdk-plugins/src/**/*.kt", "services/**/*.kt", "!services/*/generated-src/**/*.kt" ) diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index e242dd2acb6..e2dd9f250d7 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -14,7 +14,6 @@ import org.gradle.api.Project open class CodegenExtension(private val project: Project) { internal val projections = mutableMapOf() - // FIXME - not all settings make sense to be defaulted... // /** // * Set default plugin settings that each projection uses if no value is set @@ -45,6 +44,3 @@ open class CodegenExtension(private val project: Project) { */ fun getProjectionByName(name: String): KotlinCodegenProjection? = projections[name] } - - - diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt index cdaa7e60b98..28154747b90 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt @@ -20,7 +20,7 @@ const val CODEGEN_EXTENSION_NAME = "codegen" * - providing a [CodegenTask] to generate Kotlin sources from their respective smithy models. */ class CodegenPlugin : Plugin { - override fun apply(target: Project):Unit = target.run { + override fun apply(target: Project): Unit = target.run { configurePlugins() installExtension() registerCodegenTasks() @@ -36,4 +36,3 @@ class CodegenPlugin : Plugin { return extensions.create(CODEGEN_EXTENSION_NAME, CodegenExtension::class.java, project) } } - diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt index 87b99d07e4c..51076a471ae 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt @@ -5,13 +5,13 @@ package aws.sdk.kotlin.gradle -///** +// /** // * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) // */ -//class PostProcessSpec { +// class PostProcessSpec { // // -//} +// } class KotlinCodegenProjection( /** @@ -23,7 +23,7 @@ class KotlinCodegenProjection( * Root directory for this projection */ val projectionRootDir: java.io.File, -){ +) { /** * List of files/directories to import when building the projection @@ -32,14 +32,12 @@ class KotlinCodegenProjection( */ var imports: List = emptyList() - internal var pluginSettings: SmithyKotlinPluginSettings = SmithyKotlinPluginSettings() /** * Configure smithy-kotlin plugin settings. */ - fun pluginSettings(configure: SmithyKotlinPluginSettings.() -> Unit): Unit { pluginSettings.also(configure) } - + fun pluginSettings(configure: SmithyKotlinPluginSettings.() -> Unit) { pluginSettings.also(configure) } // private var postProcessSpec: PostProcessSpec? = null // fun postProcess(spec: PostProcessSpec.() -> Unit) { @@ -47,7 +45,6 @@ class KotlinCodegenProjection( // } } - class SmithyKotlinPluginSettings { var serviceShapeId: String? = null var packageName: String? = null @@ -81,4 +78,4 @@ class SmithyKotlinPluginSettings { result = 31 * result + (buildSettings?.hashCode() ?: 0) return result } -} \ No newline at end of file +} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt index ff633b4fc96..cbffb7773ae 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt @@ -12,8 +12,8 @@ import org.gradle.kotlin.dsl.get /** * Get the root directory of the generated kotlin code for a projection */ -internal fun Project.projectionRootDir(projectionName: String): java.io.File - = file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen") +internal fun Project.projectionRootDir(projectionName: String): java.io.File = + file("${project.buildDir}/smithyprojections/${project.name}/$projectionName/kotlin-codegen") /** * Get the [CodegenExtension] instance configured for the project diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt index d5b3233755f..4a79df43935 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt @@ -8,7 +8,7 @@ package aws.sdk.kotlin.gradle.tasks import org.gradle.api.DefaultTask import org.gradle.api.tasks.TaskAction -abstract class CodegenTask: DefaultTask() { +abstract class CodegenTask : DefaultTask() { init { group = "codegen" description = "Generate code using smithy-kotlin" @@ -21,5 +21,4 @@ abstract class CodegenTask: DefaultTask() { // it doesn't actually do any work (yet) but it does give us our own task to extend _AFTER_ code // has been generated } - -} \ No newline at end of file +} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt index c2f9c158552..eb352d1c7c5 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt @@ -24,7 +24,7 @@ internal fun Project.registerCodegenTasks() { // set an input property based on a hash of all the plugin settings to get this task's // up-to-date checks to work correctly (model files are already an input to the actual build task) - val pluginSettingsHash = project.codegenExtension.projections.values.fold(0){ acc, projection -> + val pluginSettingsHash = project.codegenExtension.projections.values.fold(0) { acc, projection -> acc + projection.pluginSettings.hashCode() } @@ -55,10 +55,10 @@ internal fun Project.registerCodegenTasks() { val extension = project.codegenExtension println("registering imports for kotlinCodegenSmithyBuild: ${extension.projections.keys.joinToString()}") // register the model file(s) (imports) - val imports = extension.projections.values.flatMap{ it.imports } + val imports = extension.projections.values.flatMap { it.imports } imports.forEach { importPath -> val f = project.file(importPath) - if (f.exists()){ + if (f.exists()) { if (f.isDirectory) inputs.dir(f) else inputs.file(f) } } @@ -93,7 +93,7 @@ private fun generateSmithyBuild(projections: Collection prefix = "{", postfix = "}" ) { - val value = when(it.value) { + val value = when (it.value) { is Boolean -> it.value.toString() else -> "\"${it.value}\"" } @@ -128,10 +128,9 @@ private fun generateSmithyBuild(projections: Collection $formattedProjections } } - """.trimIndent() + """.trimIndent() } - // create a configuration (classpath) needed by the SmithyBuild task private fun Project.createCodegenConfiguration(): Configuration { val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") @@ -151,5 +150,3 @@ private fun Project.createCodegenConfiguration(): Configuration { return codegenConfig } - - From 15533e4cefe038564ea0a27698be7856d31cc29a Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 12:55:42 -0500 Subject: [PATCH 13/31] add transform to only include particular operations --- aws-runtime/aws-config/build.gradle.kts | 31 +++++++++++++++- .../codegen/transforms/IncludeOperations.kt | 36 +++++++++++++++++++ ....amazon.smithy.build.ProjectionTransformer | 1 + .../aws/sdk/kotlin/gradle/CodegenPlugin.kt | 3 -- .../kotlin/gradle/KotlinCodegenProjection.kt | 7 ++++ .../sdk/kotlin/gradle/tasks/SmithyTasks.kt | 3 ++ 6 files changed, 77 insertions(+), 4 deletions(-) create mode 100644 codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt create mode 100644 codegen/smithy-aws-kotlin-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.ProjectionTransformer diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index c5832af4848..4e88a33f267 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -93,6 +93,21 @@ codegen { sdkId = "STS" buildSettings = sharedBuildSettings } + + transforms = listOf( + """ + { + "name": "awsSdkKotlinIncludeServices", + "args": { + "operations": [ + "com.amazonaws.sts#AssumeRoleWithWebIdentity", + "com.amazonaws.sts#AssumeRole" + ] + } + } + """.trimIndent() + + ) } // generate an sso client @@ -101,14 +116,28 @@ codegen { awsModelFile("sso.2019-06-10.json") ) + val serviceShape = "com.amazonaws.sso#SWBPortalService" pluginSettings { - serviceShapeId = "com.amazonaws.sso#SWBPortalService" + serviceShapeId = serviceShape packageName = "${basePackage}.sso" packageVersion = project.version.toString() packageDescription = "Internal SSO credentials provider" sdkId = "SSO" buildSettings = sharedBuildSettings } + + transforms = listOf( + """ + { + "name": "awsSdkKotlinIncludeServices", + "args": { + "operations": [ + "com.amazonaws.sso#GetRoleCredentials" + ] + } + } + """.trimIndent() + ) } } diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt new file mode 100644 index 00000000000..fe841b5c2ce --- /dev/null +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt @@ -0,0 +1,36 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.codegen.transforms + +import software.amazon.smithy.build.TransformContext +import software.amazon.smithy.build.transforms.ConfigurableProjectionTransformer +import software.amazon.smithy.model.Model +import software.amazon.smithy.model.shapes.OperationShape + +/** + * A smithy build [transform](https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#transforms) + * that filters out operations not included in the `operations` list of shape IDs + */ +class IncludeOperations : ConfigurableProjectionTransformer() { + + class Config { +// var service: String = "" + var operations: Set = emptySet() + } + + override fun getName(): String = "awsSdkKotlinIncludeServices" + override fun getConfigType(): Class = Config::class.java + + override fun transformWithConfig(context: TransformContext, config: Config): Model { + check(config.operations.isNotEmpty()) { "no operations provided to IncludeOperations transform!" } + return context.transformer.filterShapes(context.model) { shape -> + when (shape) { + is OperationShape -> shape.id.toString() in config.operations + else -> true + } + } + } +} diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.ProjectionTransformer b/codegen/smithy-aws-kotlin-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.ProjectionTransformer new file mode 100644 index 00000000000..8c906c3195b --- /dev/null +++ b/codegen/smithy-aws-kotlin-codegen/src/main/resources/META-INF/services/software.amazon.smithy.build.ProjectionTransformer @@ -0,0 +1 @@ +aws.sdk.kotlin.codegen.transforms.IncludeOperations \ No newline at end of file diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt index 28154747b90..a98e4a0ddbe 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt @@ -10,9 +10,6 @@ import org.gradle.api.Plugin import org.gradle.api.Project const val CODEGEN_EXTENSION_NAME = "codegen" -// TODO - create custom transform to include specific operations -// TODO - create custom transform to add `InternalSdkApi` to generated client OR change visibility? -// - e.g. custom trait (@kotlinVisibility("internal")) /** * This plugin handles: diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt index 51076a471ae..b1613cd0f33 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt @@ -32,6 +32,13 @@ class KotlinCodegenProjection( */ var imports: List = emptyList() + /** + * A list of transforms to apply + * + * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#transforms + */ + var transforms: List = emptyList() + internal var pluginSettings: SmithyKotlinPluginSettings = SmithyKotlinPluginSettings() /** diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt index eb352d1c7c5..4931cb06d3d 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt @@ -100,9 +100,12 @@ private fun generateSmithyBuild(projections: Collection """"${it.key}": $value""" } + val transforms = projection.transforms.joinToString() + val config = """ "${projection.name}": { "imports": [$imports], + "transforms": [$transforms], "plugins": { "kotlin-codegen": { "service": "${projection.pluginSettings.serviceShapeId}", From 7c6d9dfba4a8583e1123fdb1160cdc1a6f36ab8d Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 14:57:38 -0500 Subject: [PATCH 14/31] use node api from smithy to construct json --- aws-runtime/aws-config/build.gradle.kts | 18 +++--- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 15 ++--- .../kotlin/gradle/KotlinCodegenProjection.kt | 59 ++++++++++++++++++- .../kotlin/aws/sdk/kotlin/gradle/Utils.kt | 28 +++++++++ .../sdk/kotlin/gradle/tasks/SmithyTasks.kt | 57 +++--------------- 5 files changed, 109 insertions(+), 68 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 4e88a33f267..68b4fbbf509 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -75,9 +75,9 @@ fun awsModelFile(name: String): String = codegen { val basePackage = "aws.sdk.kotlin.runtime.auth.credentials.internal" - val sharedBuildSettings = mapOf( - "generateDefaultBuildFiles" to false - ) +// val sharedBuildSettings = mapOf( +// "generateDefaultBuildFiles" to false +// ) // generate an sts client projection("sts-credentials-provider") { @@ -91,7 +91,10 @@ codegen { packageVersion = project.version.toString() packageDescription = "Internal STS credentials provider" sdkId = "STS" - buildSettings = sharedBuildSettings + buildSettings { + generateDefaultBuildFiles = false + generateFullProject = false + } } transforms = listOf( @@ -105,8 +108,7 @@ codegen { ] } } - """.trimIndent() - + """ ) } @@ -123,7 +125,7 @@ codegen { packageVersion = project.version.toString() packageDescription = "Internal SSO credentials provider" sdkId = "SSO" - buildSettings = sharedBuildSettings +// buildSettings = sharedBuildSettings } transforms = listOf( @@ -136,7 +138,7 @@ codegen { ] } } - """.trimIndent() + """ ) } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index e2dd9f250d7..04b93ee362f 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -14,24 +14,17 @@ import org.gradle.api.Project open class CodegenExtension(private val project: Project) { internal val projections = mutableMapOf() -// FIXME - not all settings make sense to be defaulted... -// /** -// * Set default plugin settings that each projection uses if no value is set -// */ -// private var defaultPluginSettings: SmithyKotlinPluginSettings? = null -// fun defaultPluginSettings(configure: SmithyKotlinPluginSettings.() -> Unit): Unit { -// if (defaultPluginSettings == null) { defaultPluginSettings = SmithyKotlinPluginSettings() } -// defaultPluginSettings!!.apply(configure) -// } + // TODO - allow setting default build settings that apply to every projection (or every projection starts with)? /** * Configure a new projection */ fun projection(name: String, configure: Action) { println("configuring projection $name") - val p = KotlinCodegenProjection(name, project.projectionRootDir(name)) + val p = projections.computeIfAbsent(name) { + KotlinCodegenProjection(name, project.projectionRootDir(name)) + } configure.execute(p) - projections[name] = p } /** diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt index b1613cd0f33..c5014758bac 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt @@ -5,6 +5,12 @@ package aws.sdk.kotlin.gradle +import software.amazon.smithy.model.node.ArrayNode +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.node.ObjectNode +import software.amazon.smithy.model.node.ToNode +import java.util.* + // /** // * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) // */ @@ -50,6 +56,38 @@ class KotlinCodegenProjection( // fun postProcess(spec: PostProcessSpec.() -> Unit) { // postProcessSpec = PostProcessSpec().apply(spec) // } + + internal fun toNode(): Node { + // escape windows paths for valid json + val formattedImports = imports + .map { it.replace("\\", "\\\\") } + + val transformNodes = transforms.map { Node.parse(it) } + val obj = ObjectNode.objectNodeBuilder() + .withArrayMember("imports", formattedImports) + .withMember("transforms", ArrayNode.fromNodes(transformNodes)) + .withObjectMember("plugins") { + withMember("kotlin-codegen", pluginSettings.toNode()) + } + return obj.build() + } +} + +class SmithyKotlinBuildSettings : ToNode { + var generateFullProject: Boolean? = null + var generateDefaultBuildFiles: Boolean? = null + var optInAnnotations: List? = null + + override fun toNode(): Node { + val builder = ObjectNode.objectNodeBuilder() + + builder.withOptionalMember("rootProject", generateFullProject) + builder.withOptionalMember("generateDefaultBuildFiles", generateDefaultBuildFiles) + + val optInArrNode = optInAnnotations?.map { Node.from(it) }?.let { ArrayNode.fromNodes(it) } + builder.withOptionalMember("optInAnnotations", Optional.ofNullable(optInArrNode)) + return builder.build() + } } class SmithyKotlinPluginSettings { @@ -58,7 +96,12 @@ class SmithyKotlinPluginSettings { var packageVersion: String? = null var packageDescription: String? = null var sdkId: String? = null - var buildSettings: Map? = null + + internal var buildSettings: SmithyKotlinBuildSettings? = null + fun buildSettings(configure: SmithyKotlinBuildSettings.() -> Unit) { + if (buildSettings == null) buildSettings = SmithyKotlinBuildSettings() + buildSettings!!.apply(configure) + } override fun equals(other: Any?): Boolean { if (this === other) return true @@ -85,4 +128,18 @@ class SmithyKotlinPluginSettings { result = 31 * result + (buildSettings?.hashCode() ?: 0) return result } + + internal fun toNode(): Node { + val obj = ObjectNode.objectNodeBuilder() + .withMember("service", serviceShapeId!!) + .withObjectMember("package") { + withMember("name", packageName!!) + withOptionalMember("version", packageVersion) + withOptionalMember("version", packageDescription) + } + .withOptionalMember("sdkId", sdkId) + .withOptionalMember("build", buildSettings) + + return obj.build() + } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt index cbffb7773ae..2bd3eb8afc7 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt @@ -8,6 +8,11 @@ package aws.sdk.kotlin.gradle import org.gradle.api.Project import org.gradle.api.plugins.ExtensionAware import org.gradle.kotlin.dsl.get +import software.amazon.smithy.model.node.ArrayNode +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.node.ObjectNode +import software.amazon.smithy.model.node.ToNode +import java.util.* /** * Get the root directory of the generated kotlin code for a projection @@ -20,3 +25,26 @@ internal fun Project.projectionRootDir(projectionName: String): java.io.File = */ internal val Project.codegenExtension: CodegenExtension get() = ((this as ExtensionAware).extensions[CODEGEN_EXTENSION_NAME] as? CodegenExtension) ?: error("CodegenPlugin has not been applied") + +internal fun ObjectNode.Builder.withObjectMember(key: String, block: ObjectNode.Builder.() -> Unit): ObjectNode.Builder { + val builder = ObjectNode.objectNodeBuilder() + builder.apply(block) + return withMember(key, builder.build()) +} +internal fun ObjectNode.Builder.withOptionalMember(key: String, member: String?): ObjectNode.Builder = apply { + if (member == null) return this + return withMember(key, member) +} + +internal fun ObjectNode.Builder.withOptionalMember(key: String, member: Boolean?): ObjectNode.Builder = apply { + if (member == null) return this + return withMember(key, member) +} + +internal fun ObjectNode.Builder.withOptionalMember(key: String, member: T?): ObjectNode.Builder = + withOptionalMember(key, Optional.ofNullable(member)) + +internal fun ObjectNode.Builder.withArrayMember(key: String, member: List): ObjectNode.Builder = apply { + val arrNode = member.map { Node.from(it) }.let { ArrayNode.fromNodes(it) } + return withMember(key, arrNode) +} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt index 4931cb06d3d..323e358b630 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt @@ -8,12 +8,14 @@ package aws.sdk.kotlin.gradle.tasks import aws.sdk.kotlin.gradle.KotlinCodegenProjection import aws.sdk.kotlin.gradle.codegenExtension import aws.sdk.kotlin.gradle.projectionRootDir +import aws.sdk.kotlin.gradle.withObjectMember import org.gradle.api.Project import org.gradle.api.artifacts.Configuration import org.gradle.kotlin.dsl.dependencies import org.gradle.kotlin.dsl.provideDelegate import org.gradle.kotlin.dsl.register import software.amazon.smithy.gradle.tasks.SmithyBuild +import software.amazon.smithy.model.node.Node internal fun Project.registerCodegenTasks() { // generate the projection file for smithy to consume @@ -81,57 +83,16 @@ internal fun Project.registerCodegenTasks() { * Generate the "smithy-build.json" defining the projection */ private fun generateSmithyBuild(projections: Collection): String { - val formattedProjections = projections.joinToString(",") { projection -> - // escape windows paths for valid json - val imports = projection.imports - .map { it.replace("\\", "\\\\") } - .joinToString { "\"$it\"" } - - // TODO - probably need some validation in here... - val buildPairs = projection.pluginSettings.buildSettings?.entries?.joinToString( - separator = ",\n", - prefix = "{", - postfix = "}" - ) { - val value = when (it.value) { - is Boolean -> it.value.toString() - else -> "\"${it.value}\"" + val buildConfig = Node.objectNodeBuilder() + .withMember("version", "1.0") + .withObjectMember("projections") { + projections.forEach { projection -> + withMember(projection.name, projection.toNode()) } - """"${it.key}": $value""" } + .build() - val transforms = projection.transforms.joinToString() - - val config = """ - "${projection.name}": { - "imports": [$imports], - "transforms": [$transforms], - "plugins": { - "kotlin-codegen": { - "service": "${projection.pluginSettings.serviceShapeId}", - "package": { - "name": "${projection.pluginSettings.packageName}", - "version": "${projection.pluginSettings.packageVersion}", - "description": "${projection.pluginSettings.packageDescription}" - }, - "sdkId": "${projection.pluginSettings.sdkId}", - "build": $buildPairs - } - } - } - """.trimIndent() - - config - } - - return """ - { - "version": "1.0", - "projections": { - $formattedProjections - } - } - """.trimIndent() + return Node.prettyPrintJson(buildConfig) } // create a configuration (classpath) needed by the SmithyBuild task From bd8504867cbff31219a73e2d32faef0422d29988 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 14:58:35 -0500 Subject: [PATCH 15/31] remove unused code --- aws-runtime/aws-config/build.gradle.kts | 3 --- 1 file changed, 3 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 68b4fbbf509..dc6640bb89f 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -75,9 +75,6 @@ fun awsModelFile(name: String): String = codegen { val basePackage = "aws.sdk.kotlin.runtime.auth.credentials.internal" -// val sharedBuildSettings = mapOf( -// "generateDefaultBuildFiles" to false -// ) // generate an sts client projection("sts-credentials-provider") { From 26be6b7a90a6da4fb4723ef9220ce17ec841cb6e Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 15:02:21 -0500 Subject: [PATCH 16/31] add build settings --- aws-runtime/aws-config/build.gradle.kts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index dc6640bb89f..e28259f803f 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -122,7 +122,10 @@ codegen { packageVersion = project.version.toString() packageDescription = "Internal SSO credentials provider" sdkId = "SSO" -// buildSettings = sharedBuildSettings + buildSettings { + generateDefaultBuildFiles = false + generateFullProject = false + } } transforms = listOf( From 9967458c998aeb22a88a3196f1b50a076e8d488e Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Thu, 18 Nov 2021 16:21:06 -0500 Subject: [PATCH 17/31] fix name --- aws-runtime/aws-config/build.gradle.kts | 4 ++-- .../aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index e28259f803f..d70494e7021 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -97,7 +97,7 @@ codegen { transforms = listOf( """ { - "name": "awsSdkKotlinIncludeServices", + "name": "awsSdkKotlinIncludeOperations", "args": { "operations": [ "com.amazonaws.sts#AssumeRoleWithWebIdentity", @@ -131,7 +131,7 @@ codegen { transforms = listOf( """ { - "name": "awsSdkKotlinIncludeServices", + "name": "awsSdkKotlinIncludeOperations", "args": { "operations": [ "com.amazonaws.sso#GetRoleCredentials" diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt index fe841b5c2ce..5381aa694ad 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt @@ -21,7 +21,7 @@ class IncludeOperations : ConfigurableProjectionTransformer = emptySet() } - override fun getName(): String = "awsSdkKotlinIncludeServices" + override fun getName(): String = "awsSdkKotlinIncludeOperations" override fun getConfigType(): Class = Config::class.java override fun transformWithConfig(context: TransformContext, config: Config): Model { From c274a306eeaf620e48659e4a3dd146704a6a306d Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 19 Nov 2021 09:36:35 -0500 Subject: [PATCH 18/31] add todo --- aws-runtime/aws-config/build.gradle.kts | 1 + 1 file changed, 1 insertion(+) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index d70494e7021..89a0fc4f921 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -94,6 +94,7 @@ codegen { } } + // TODO - could we add a trait such that we change visibility to `internal` or a build setting...? transforms = listOf( """ { From ee2524db57bad8baab15bc5098d11014a0e6e21c Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 19 Nov 2021 11:47:57 -0500 Subject: [PATCH 19/31] refactor to use task avoidance apis correctly and NamedObjectContainer --- aws-runtime/aws-config/build.gradle.kts | 87 ++++++++++--------- .../aws/sdk/kotlin/gradle/CodegenExtension.kt | 24 +---- .../kotlin/gradle/KotlinCodegenProjection.kt | 22 +++++ .../sdk/kotlin/gradle/tasks/SmithyTasks.kt | 37 ++++---- 4 files changed, 87 insertions(+), 83 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 89a0fc4f921..a14b4fab095 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -76,27 +76,28 @@ fun awsModelFile(name: String): String = codegen { val basePackage = "aws.sdk.kotlin.runtime.auth.credentials.internal" - // generate an sts client - projection("sts-credentials-provider") { - imports = listOf( - awsModelFile("sts.2011-06-15.json") - ) - - pluginSettings { - serviceShapeId = "com.amazonaws.sts#AWSSecurityTokenServiceV20110615" - packageName = "${basePackage}.sts" - packageVersion = project.version.toString() - packageDescription = "Internal STS credentials provider" - sdkId = "STS" - buildSettings { - generateDefaultBuildFiles = false - generateFullProject = false + projections { + // generate an sts client + create("sts-credentials-provider") { + imports = listOf( + awsModelFile("sts.2011-06-15.json") + ) + + pluginSettings { + serviceShapeId = "com.amazonaws.sts#AWSSecurityTokenServiceV20110615" + packageName = "${basePackage}.sts" + packageVersion = project.version.toString() + packageDescription = "Internal STS credentials provider" + sdkId = "STS" + buildSettings { + generateDefaultBuildFiles = false + generateFullProject = false + } } - } - // TODO - could we add a trait such that we change visibility to `internal` or a build setting...? - transforms = listOf( - """ + // TODO - could we add a trait such that we change visibility to `internal` or a build setting...? + transforms = listOf( + """ { "name": "awsSdkKotlinIncludeOperations", "args": { @@ -107,30 +108,30 @@ codegen { } } """ - ) - } + ) + } - // generate an sso client - projection("sso-credentials-provider") { - imports = listOf( - awsModelFile("sso.2019-06-10.json") - ) - - val serviceShape = "com.amazonaws.sso#SWBPortalService" - pluginSettings { - serviceShapeId = serviceShape - packageName = "${basePackage}.sso" - packageVersion = project.version.toString() - packageDescription = "Internal SSO credentials provider" - sdkId = "SSO" - buildSettings { - generateDefaultBuildFiles = false - generateFullProject = false + // generate an sso client + create("sso-credentials-provider") { + imports = listOf( + awsModelFile("sso.2019-06-10.json") + ) + + val serviceShape = "com.amazonaws.sso#SWBPortalService" + pluginSettings { + serviceShapeId = serviceShape + packageName = "${basePackage}.sso" + packageVersion = project.version.toString() + packageDescription = "Internal SSO credentials provider" + sdkId = "SSO" + buildSettings { + generateDefaultBuildFiles = false + generateFullProject = false + } } - } - transforms = listOf( - """ + transforms = listOf( + """ { "name": "awsSdkKotlinIncludeOperations", "args": { @@ -140,7 +141,8 @@ codegen { } } """ - ) + ) + } } } @@ -164,7 +166,8 @@ tasks.withType { dependsOn(codegenTasks) } -codegen.projections { + +codegen.projections.all { // add this projected source dir to the common sourceSet // NOTE - build.gradle.kts is still being generated, it's NOT used though // TODO - we should probably either have a postProcessing spec or a plugin setting to not generate it to avoid confusion diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt index 04b93ee362f..9eae4ed551a 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt @@ -5,35 +5,15 @@ package aws.sdk.kotlin.gradle -import org.gradle.api.Action import org.gradle.api.Project /** * Register and build Smithy projections */ open class CodegenExtension(private val project: Project) { - internal val projections = mutableMapOf() // TODO - allow setting default build settings that apply to every projection (or every projection starts with)? - - /** - * Configure a new projection - */ - fun projection(name: String, configure: Action) { - println("configuring projection $name") - val p = projections.computeIfAbsent(name) { - KotlinCodegenProjection(name, project.projectionRootDir(name)) - } - configure.execute(p) + val projections = project.objects.domainObjectContainer(KotlinCodegenProjection::class.java) { name -> + KotlinCodegenProjection(name, project.projectionRootDir(name)) } - - /** - * Execute [action] for each projection - */ - fun projections(action: Action) = projections.values.forEach { action.execute(it) } - - /** - * Get a projection by name - */ - fun getProjectionByName(name: String): KotlinCodegenProjection? = projections[name] } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt index c5014758bac..77fe6199977 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt @@ -71,6 +71,28 @@ class KotlinCodegenProjection( } return obj.build() } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as KotlinCodegenProjection + + if (name != other.name) return false + if (imports != other.imports) return false + if (transforms != other.transforms) return false + if (pluginSettings != other.pluginSettings) return false + + return true + } + + override fun hashCode(): Int { + var result = name.hashCode() + result = 31 * result + imports.hashCode() + result = 31 * result + transforms.hashCode() + result = 31 * result + pluginSettings.hashCode() + return result + } } class SmithyKotlinBuildSettings : ToNode { diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt index 323e358b630..c4e7fe3d3ef 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt @@ -7,7 +7,6 @@ package aws.sdk.kotlin.gradle.tasks import aws.sdk.kotlin.gradle.KotlinCodegenProjection import aws.sdk.kotlin.gradle.codegenExtension -import aws.sdk.kotlin.gradle.projectionRootDir import aws.sdk.kotlin.gradle.withObjectMember import org.gradle.api.Project import org.gradle.api.artifacts.Configuration @@ -24,13 +23,14 @@ internal fun Project.registerCodegenTasks() { description = "generate smithy-build.json" group = "codegen" - // set an input property based on a hash of all the plugin settings to get this task's - // up-to-date checks to work correctly (model files are already an input to the actual build task) - val pluginSettingsHash = project.codegenExtension.projections.values.fold(0) { acc, projection -> - acc + projection.pluginSettings.hashCode() + // set an input property based on a hash of all the projections to get this task's + // up-to-date checks to work correctly (model files are configured as an input to the actual build task) + val projectionHash = project.objects.property(Int::class.java) + projectionHash.set(0) + project.codegenExtension.projections.all { + projectionHash.set(projectionHash.get() + hashCode()) } - - inputs.property("pluginSettingsHash", pluginSettingsHash) + inputs.property("projectionHash", projectionHash) outputs.file(smithyBuildConfig) doFirst { if (smithyBuildConfig.exists()) { @@ -40,7 +40,8 @@ internal fun Project.registerCodegenTasks() { doLast { buildDir.mkdir() val extension = project.codegenExtension - smithyBuildConfig.writeText(generateSmithyBuild(extension.projections.values)) + val projections = extension.projections.asMap + smithyBuildConfig.writeText(generateSmithyBuild(projections.values)) } } @@ -55,22 +56,20 @@ internal fun Project.registerCodegenTasks() { inputs.file(smithyBuildConfig) val extension = project.codegenExtension - println("registering imports for kotlinCodegenSmithyBuild: ${extension.projections.keys.joinToString()}") - // register the model file(s) (imports) - val imports = extension.projections.values.flatMap { it.imports } - imports.forEach { importPath -> - val f = project.file(importPath) - if (f.exists()) { - if (f.isDirectory) inputs.dir(f) else inputs.file(f) + + // every time a projection is added wire up the imports and outputs appropriately for this task + extension.projections.all { + imports.forEach { importPath -> + val f = project.file(importPath) + if (f.exists()) { + if (f.isDirectory) inputs.dir(f) else inputs.file(f) + } } + outputs.dir(projectionRootDir) } // ensure smithy-aws-kotlin-codegen is up to date inputs.files(codegenConfig) - - extension.projections.keys.forEach { projectionName -> - outputs.dir(project.projectionRootDir(projectionName)) - } } project.tasks.register("kotlinCodegen") { From 0be8fa8d02f1f9232c8f1c5169b158382038ef1f Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 19 Nov 2021 14:47:52 -0500 Subject: [PATCH 20/31] refactor the protocol tests build to use new plugin --- codegen/protocol-tests/build.gradle.kts | 182 ++++++++---------------- 1 file changed, 59 insertions(+), 123 deletions(-) diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index 94fcddd378f..a1faf5a7f33 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -5,25 +5,22 @@ import software.amazon.smithy.gradle.tasks.SmithyBuild plugins { - id("software.amazon.smithy") + id("aws.sdk.kotlin.codegen") } description = "Smithy protocol test suite" -buildscript { - val smithyVersion: String by project - dependencies { - classpath("software.amazon.smithy:smithy-cli:$smithyVersion") - } -} - - val smithyVersion: String by project dependencies { implementation("software.amazon.smithy:smithy-aws-protocol-tests:$smithyVersion") - implementation(project(":codegen:smithy-aws-kotlin-codegen")) } +data class ProtocolTest(val projectionName: String, val serviceShapeId: String, val sdkId: String? = null) { + val packageName: String + get() = projectionName.toLowerCase().filter { it.isLetterOrDigit() } +} + + // The following section exposes Smithy protocol test suites as gradle test targets // for the configured protocols in [enabledProtocols]. val enabledProtocols = listOf( @@ -41,157 +38,96 @@ val enabledProtocols = listOf( ProtocolTest("machinelearning", "com.amazonaws.machinelearning#AmazonML_20141212", sdkId = "Machine Learning"), ) -// This project doesn't produce a JAR. -tasks["jar"].enabled = false - -// Run the SmithyBuild task manually since this project needs the built JAR -// from smithy-aws-kotlin-codegen. -tasks["smithyBuildJar"].enabled = false - -task("generateSmithyBuild") { - group = "codegen" - description = "generate smithy-build.json" - val buildFile = projectDir.resolve("smithy-build.json") - doFirst { - buildFile.writeText(generateSmithyBuild(enabledProtocols)) +codegen { + enabledProtocols.forEach { test -> + projections.create(test.projectionName) { + transforms = listOf( + """ + { + "name": "includeServices", + "args": { + "services": ["${test.serviceShapeId}"] + } + } + """ + ) + + pluginSettings { + serviceShapeId = test.serviceShapeId + packageName = "aws.sdk.kotlin.services.${test.packageName}" + packageVersion = "1.0" + sdkId = test.sdkId + buildSettings { + generateFullProject = true + optInAnnotations = listOf( + "aws.smithy.kotlin.runtime.util.InternalApi", + "aws.sdk.kotlin.runtime.InternalSdkApi" + ) + } + } + } } - outputs.file(buildFile) } -// Remove generated model file for clean -tasks["clean"].doFirst { - delete("smithy-build.json") -} +tasks.getByName("kotlinCodegenSmithyBuild") { + // NOTE: The protocol tests are published to maven as a jar, this ensures that + // the aws-protocol-tests dependency is found when generating code such that the `includeServices` transform + // actually works + addCompileClasspath = true -tasks.create("generateSdk") { - group = "codegen" // ensure the generated clients use the same version of the runtime as the aws aws-runtime val smithyKotlinVersion: String by project doFirst { System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinVersion) } - addRuntimeClasspath = true - dependsOn(tasks["generateSmithyBuild"]) - inputs.file(projectDir.resolve("smithy-build.json")) - // ensure smithy-aws-kotlin-codegen is up to date - inputs.files(configurations.compileClasspath) -} - -// force rebuild every time while developing -tasks["generateSdk"].outputs.upToDateWhen { false } - -data class ProtocolTest(val projectionName: String, val serviceShapeId: String, val sdkId: String? = null) { - val packageName: String - get() = projectionName.toLowerCase().filter { it.isLetterOrDigit() } -} - - -// Generates a smithy-build.json file by creating a new projection. -// The generated smithy-build.json file is not committed to git since -// it's rebuilt each time codegen is performed. -fun generateSmithyBuild(tests: List): String { - val projections = tests.joinToString(",") { test -> - val sdkIdEntry = test.sdkId?.let { """"sdkId": "$it",""" } ?: "" - """ - "${test.projectionName}": { - "transforms": [ - { - "name": "includeServices", - "args": { - "services": [ - "${test.serviceShapeId}" - ] - } - } - ], - "plugins": { - "kotlin-codegen": { - "service": "${test.serviceShapeId}", - "package": { - "name": "aws.sdk.kotlin.services.${test.packageName}", - "version": "1.0" - }, - $sdkIdEntry - "build": { - "rootProject": true, - "optInAnnotations": [ - "aws.smithy.kotlin.runtime.util.InternalApi", - "aws.sdk.kotlin.runtime.InternalSdkApi" - ] - } - } - } - } - """ - } - return """ - { - "version": "1.0", - "projections": { - $projections - } - } - """.trimIndent() } open class ProtocolTestTask : DefaultTask() { /** - * The protocol name - */ - @get:Input - var protocol: String = "" - - /** - * The plugin name to use + * The projection */ @get:Input - var plugin: String = "" + var projection: aws.sdk.kotlin.gradle.KotlinCodegenProjection? = null - /** - * The build directory for the task - */ - val generatedBuildDir: File - @OutputDirectory - get() = project.buildDir.resolve("smithyprojections/${project.name}/$protocol/$plugin") @TaskAction fun runTests() { - require(protocol.isNotEmpty()) { "protocol name must be specified" } - require(plugin.isNotEmpty()) { "plugin name must be specified" } - - println("[$protocol] buildDir: $generatedBuildDir") - if (!generatedBuildDir.exists()) { - throw GradleException("$generatedBuildDir does not exist") + val projection = requireNotNull(projection) { "projection is required task input" } + println("[${projection.name}] buildDir: ${projection.projectionRootDir}") + if (!projection.projectionRootDir.exists()) { + throw GradleException("${projection.projectionRootDir} does not exist") } val wrapper = if (System.getProperty("os.name").toLowerCase().contains("windows")) "gradlew.bat" else "gradlew" val gradlew = project.rootProject.file(wrapper).absolutePath // NOTE - this still requires us to publish to maven local. project.exec { - workingDir = generatedBuildDir + workingDir = projection.projectionRootDir executable = gradlew args = listOf("test") } } } -enabledProtocols.forEach { - val protocolName = it.projectionName +val kotlinCodegenTask = tasks.getByName("kotlinCodegen") +codegen.projections.forEach { + val protocolName = it.name - val protocolTestTask = tasks.register("testProtocol-$protocolName") { - dependsOn(tasks["generateSdk"]) + tasks.register("testProtocol-$protocolName") { + dependsOn(kotlinCodegenTask) group = "Verification" - protocol = protocolName - plugin = "kotlin-codegen" - }.get() + projection = it + } // FIXME This is a hack to work around how protocol tests aren't in the actual service model and thus codegen // separately from service customizations. - tasks.create("copyStaticFiles-$protocolName") { + val copyStaticFiles = tasks.register("copyStaticFiles-$protocolName") { + group = "codegen" from(rootProject.projectDir.resolve("services/$protocolName/common/src")) - into(protocolTestTask.generatedBuildDir.resolve("src/main/kotlin/")) - tasks["generateSdk"].finalizedBy(this) + into(it.projectionRootDir.resolve("src/main/kotlin/")) } + + kotlinCodegenTask.finalizedBy(copyStaticFiles) } tasks.register("testAllProtocols") { From 1aa59a89c0eb2c37267dc25dca2d4bb00b53fdac Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Nov 2021 12:43:18 -0500 Subject: [PATCH 21/31] relocate to nested codegen package --- aws-runtime/aws-config/build.gradle.kts | 2 +- codegen/protocol-tests/build.gradle.kts | 2 +- gradle/sdk-plugins/build.gradle.kts | 2 +- .../sdk/kotlin/gradle/{ => codegen}/CodegenExtension.kt | 2 +- .../aws/sdk/kotlin/gradle/{ => codegen}/CodegenPlugin.kt | 4 ++-- .../gradle/{ => codegen}/KotlinCodegenProjection.kt | 2 +- .../kotlin/aws/sdk/kotlin/gradle/{ => codegen}/Utils.kt | 2 +- .../sdk/kotlin/gradle/{ => codegen}/tasks/CodegenTask.kt | 2 +- .../sdk/kotlin/gradle/{ => codegen}/tasks/SmithyTasks.kt | 8 ++++---- 9 files changed, 13 insertions(+), 13 deletions(-) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/{ => codegen}/CodegenExtension.kt (93%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/{ => codegen}/CodegenPlugin.kt (91%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/{ => codegen}/KotlinCodegenProjection.kt (99%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/{ => codegen}/Utils.kt (98%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/{ => codegen}/tasks/CodegenTask.kt (93%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/{ => codegen}/tasks/SmithyTasks.kt (95%) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index a14b4fab095..6c4cf8ad6fa 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -153,7 +153,7 @@ NOTE: We need the following tasks to depend on codegen for gradle caching/up-to- * `compileKotlinMetadata` (Type=KotlinCompileCommon) * `sourcesJar` and `jvmSourcesJar` (Type=org.gradle.jvm.tasks.Jar) */ -val codegenTasks = tasks.withType() +val codegenTasks = tasks.withType() tasks.withType { dependsOn(codegenTasks) } diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index a1faf5a7f33..14759c91e17 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -87,7 +87,7 @@ open class ProtocolTestTask : DefaultTask() { * The projection */ @get:Input - var projection: aws.sdk.kotlin.gradle.KotlinCodegenProjection? = null + var projection: aws.sdk.kotlin.gradle.codegen.KotlinCodegenProjection? = null @TaskAction diff --git a/gradle/sdk-plugins/build.gradle.kts b/gradle/sdk-plugins/build.gradle.kts index 3bae7d55624..cb58a1014ea 100644 --- a/gradle/sdk-plugins/build.gradle.kts +++ b/gradle/sdk-plugins/build.gradle.kts @@ -47,7 +47,7 @@ gradlePlugin { plugins { val awsCodegenPlugin by creating { id = "aws.sdk.kotlin.codegen" - implementationClass = "aws.sdk.kotlin.gradle.CodegenPlugin" + implementationClass = "aws.sdk.kotlin.gradle.codegen.CodegenPlugin" } } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt similarity index 93% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt index 9eae4ed551a..20d419a711c 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle +package aws.sdk.kotlin.gradle.codegen import org.gradle.api.Project diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt similarity index 91% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt index a98e4a0ddbe..53d696db893 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt @@ -3,9 +3,9 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle +package aws.sdk.kotlin.gradle.codegen -import aws.sdk.kotlin.gradle.tasks.registerCodegenTasks +import aws.sdk.kotlin.gradle.codegen.tasks.registerCodegenTasks import org.gradle.api.Plugin import org.gradle.api.Project diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/KotlinCodegenProjection.kt similarity index 99% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/KotlinCodegenProjection.kt index 77fe6199977..a651475e582 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/KotlinCodegenProjection.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle +package aws.sdk.kotlin.gradle.codegen import software.amazon.smithy.model.node.ArrayNode import software.amazon.smithy.model.node.Node diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/Utils.kt similarity index 98% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/Utils.kt index 2bd3eb8afc7..2951b3609c0 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/Utils.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/Utils.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle +package aws.sdk.kotlin.gradle.codegen import org.gradle.api.Project import org.gradle.api.plugins.ExtensionAware diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt similarity index 93% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt index 4a79df43935..d14521fb883 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/CodegenTask.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle.tasks +package aws.sdk.kotlin.gradle.codegen.tasks import org.gradle.api.DefaultTask import org.gradle.api.tasks.TaskAction diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt similarity index 95% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt index c4e7fe3d3ef..7d7f0065e96 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt @@ -3,11 +3,11 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle.tasks +package aws.sdk.kotlin.gradle.codegen.tasks -import aws.sdk.kotlin.gradle.KotlinCodegenProjection -import aws.sdk.kotlin.gradle.codegenExtension -import aws.sdk.kotlin.gradle.withObjectMember +import aws.sdk.kotlin.gradle.codegen.KotlinCodegenProjection +import aws.sdk.kotlin.gradle.codegen.codegenExtension +import aws.sdk.kotlin.gradle.codegen.withObjectMember import org.gradle.api.Project import org.gradle.api.artifacts.Configuration import org.gradle.kotlin.dsl.dependencies From 9e0a5c467559accfaaf7dbb98e93235fdd12aa3b Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Nov 2021 13:53:43 -0500 Subject: [PATCH 22/31] move projection to dsl subpackage and rename --- codegen/protocol-tests/build.gradle.kts | 2 +- .../aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt | 6 ++++-- .../SmithyProjection.kt} | 6 +++--- .../aws/sdk/kotlin/gradle/codegen/{ => dsl}/Utils.kt | 4 +++- .../aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt | 8 ++++---- 5 files changed, 15 insertions(+), 11 deletions(-) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/{KotlinCodegenProjection.kt => dsl/SmithyProjection.kt} (98%) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/{ => dsl}/Utils.kt (92%) diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index 14759c91e17..81da2e87442 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -87,7 +87,7 @@ open class ProtocolTestTask : DefaultTask() { * The projection */ @get:Input - var projection: aws.sdk.kotlin.gradle.codegen.KotlinCodegenProjection? = null + var projection: aws.sdk.kotlin.gradle.codegen.dsl.SmithyProjection? = null @TaskAction diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt index 20d419a711c..93a30cf5bee 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt @@ -5,6 +5,8 @@ package aws.sdk.kotlin.gradle.codegen +import aws.sdk.kotlin.gradle.codegen.dsl.SmithyProjection +import aws.sdk.kotlin.gradle.codegen.dsl.projectionRootDir import org.gradle.api.Project /** @@ -13,7 +15,7 @@ import org.gradle.api.Project open class CodegenExtension(private val project: Project) { // TODO - allow setting default build settings that apply to every projection (or every projection starts with)? - val projections = project.objects.domainObjectContainer(KotlinCodegenProjection::class.java) { name -> - KotlinCodegenProjection(name, project.projectionRootDir(name)) + val projections = project.objects.domainObjectContainer(SmithyProjection::class.java) { name -> + SmithyProjection(name, project.projectionRootDir(name)) } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/KotlinCodegenProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt similarity index 98% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/KotlinCodegenProjection.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt index a651475e582..5b8b08f37be 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/KotlinCodegenProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt @@ -3,7 +3,7 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle.codegen +package aws.sdk.kotlin.gradle.codegen.dsl import software.amazon.smithy.model.node.ArrayNode import software.amazon.smithy.model.node.Node @@ -19,7 +19,7 @@ import java.util.* // // } -class KotlinCodegenProjection( +class SmithyProjection( /** * The name of the projection */ @@ -76,7 +76,7 @@ class KotlinCodegenProjection( if (this === other) return true if (javaClass != other?.javaClass) return false - other as KotlinCodegenProjection + other as SmithyProjection if (name != other.name) return false if (imports != other.imports) return false diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt similarity index 92% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/Utils.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt index 2951b3609c0..6eb1491a1e0 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/Utils.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt @@ -3,8 +3,10 @@ * SPDX-License-Identifier: Apache-2.0. */ -package aws.sdk.kotlin.gradle.codegen +package aws.sdk.kotlin.gradle.codegen.dsl +import aws.sdk.kotlin.gradle.codegen.CODEGEN_EXTENSION_NAME +import aws.sdk.kotlin.gradle.codegen.CodegenExtension import org.gradle.api.Project import org.gradle.api.plugins.ExtensionAware import org.gradle.kotlin.dsl.get diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt index 7d7f0065e96..c9084084905 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt @@ -5,9 +5,9 @@ package aws.sdk.kotlin.gradle.codegen.tasks -import aws.sdk.kotlin.gradle.codegen.KotlinCodegenProjection -import aws.sdk.kotlin.gradle.codegen.codegenExtension -import aws.sdk.kotlin.gradle.codegen.withObjectMember +import aws.sdk.kotlin.gradle.codegen.dsl.SmithyProjection +import aws.sdk.kotlin.gradle.codegen.dsl.codegenExtension +import aws.sdk.kotlin.gradle.codegen.dsl.withObjectMember import org.gradle.api.Project import org.gradle.api.artifacts.Configuration import org.gradle.kotlin.dsl.dependencies @@ -81,7 +81,7 @@ internal fun Project.registerCodegenTasks() { /** * Generate the "smithy-build.json" defining the projection */ -private fun generateSmithyBuild(projections: Collection): String { +private fun generateSmithyBuild(projections: Collection): String { val buildConfig = Node.objectNodeBuilder() .withMember("version", "1.0") .withObjectMember("projections") { From 100652554174f902675c64f25c71269e5eee7551 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Nov 2021 14:17:00 -0500 Subject: [PATCH 23/31] refactor to allow multiple plugins and use extension to configure specific plugin types --- aws-runtime/aws-config/build.gradle.kts | 6 +- codegen/protocol-tests/build.gradle.kts | 5 +- .../gradle/codegen/dsl/SmithyBuildPlugin.kt | 16 +++ .../gradle/codegen/dsl/SmithyKotlinPlugin.kt | 90 ++++++++++++++ .../gradle/codegen/dsl/SmithyProjection.kt | 115 +++--------------- .../gradle/codegen/tasks/SmithyTasks.kt | 4 +- 6 files changed, 134 insertions(+), 102 deletions(-) create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyBuildPlugin.kt create mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPlugin.kt diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index 6c4cf8ad6fa..a5a235919d1 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -1,3 +1,5 @@ +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin + /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. @@ -83,7 +85,7 @@ codegen { awsModelFile("sts.2011-06-15.json") ) - pluginSettings { + smithyKotlinPlugin{ serviceShapeId = "com.amazonaws.sts#AWSSecurityTokenServiceV20110615" packageName = "${basePackage}.sts" packageVersion = project.version.toString() @@ -118,7 +120,7 @@ codegen { ) val serviceShape = "com.amazonaws.sso#SWBPortalService" - pluginSettings { + smithyKotlinPlugin{ serviceShapeId = serviceShape packageName = "${basePackage}.sso" packageVersion = project.version.toString() diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index 81da2e87442..b9c4d326b5c 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -2,6 +2,7 @@ * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import software.amazon.smithy.gradle.tasks.SmithyBuild plugins { @@ -52,7 +53,7 @@ codegen { """ ) - pluginSettings { + smithyKotlinPlugin{ serviceShapeId = test.serviceShapeId packageName = "aws.sdk.kotlin.services.${test.packageName}" packageVersion = "1.0" @@ -69,7 +70,7 @@ codegen { } } -tasks.getByName("kotlinCodegenSmithyBuild") { +tasks.getByName("generateProjections") { // NOTE: The protocol tests are published to maven as a jar, this ensures that // the aws-protocol-tests dependency is found when generating code such that the `includeServices` transform // actually works diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyBuildPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyBuildPlugin.kt new file mode 100644 index 00000000000..9685eb7ea5e --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyBuildPlugin.kt @@ -0,0 +1,16 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.gradle.codegen.dsl + +import software.amazon.smithy.model.node.ToNode + +interface SmithyBuildPlugin : ToNode { + /** + * The name of the build plugin (e.g. `kotlin-codegen`). This is used when generating + * the projection settings for the plugin + */ + val pluginName: String +} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPlugin.kt new file mode 100644 index 00000000000..ceaa1d00082 --- /dev/null +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPlugin.kt @@ -0,0 +1,90 @@ +/* + * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. + * SPDX-License-Identifier: Apache-2.0. + */ + +package aws.sdk.kotlin.gradle.codegen.dsl + +import software.amazon.smithy.model.node.ArrayNode +import software.amazon.smithy.model.node.Node +import software.amazon.smithy.model.node.ObjectNode +import software.amazon.smithy.model.node.ToNode +import java.util.* + +class SmithyKotlinBuildSettings : ToNode { + var generateFullProject: Boolean? = null + var generateDefaultBuildFiles: Boolean? = null + var optInAnnotations: List? = null + + override fun toNode(): Node { + val builder = ObjectNode.objectNodeBuilder() + + builder.withOptionalMember("rootProject", generateFullProject) + builder.withOptionalMember("generateDefaultBuildFiles", generateDefaultBuildFiles) + + val optInArrNode = optInAnnotations?.map { Node.from(it) }?.let { ArrayNode.fromNodes(it) } + builder.withOptionalMember("optInAnnotations", Optional.ofNullable(optInArrNode)) + return builder.build() + } +} + +class SmithyKotlinPluginSettings : SmithyBuildPlugin { + override val pluginName: String = "kotlin-codegen" + + var serviceShapeId: String? = null + var packageName: String? = null + var packageVersion: String? = null + var packageDescription: String? = null + var sdkId: String? = null + + internal var buildSettings: SmithyKotlinBuildSettings? = null + fun buildSettings(configure: SmithyKotlinBuildSettings.() -> Unit) { + if (buildSettings == null) buildSettings = SmithyKotlinBuildSettings() + buildSettings!!.apply(configure) + } + + override fun equals(other: Any?): Boolean { + if (this === other) return true + if (javaClass != other?.javaClass) return false + + other as SmithyKotlinPluginSettings + + if (serviceShapeId != other.serviceShapeId) return false + if (packageName != other.packageName) return false + if (packageVersion != other.packageVersion) return false + if (packageDescription != other.packageDescription) return false + if (sdkId != other.sdkId) return false + if (buildSettings != other.buildSettings) return false + + return true + } + + override fun hashCode(): Int { + var result = serviceShapeId?.hashCode() ?: 0 + result = 31 * result + (packageName?.hashCode() ?: 0) + result = 31 * result + (packageVersion?.hashCode() ?: 0) + result = 31 * result + (packageDescription?.hashCode() ?: 0) + result = 31 * result + (sdkId?.hashCode() ?: 0) + result = 31 * result + (buildSettings?.hashCode() ?: 0) + return result + } + + override fun toNode(): Node { + val obj = ObjectNode.objectNodeBuilder() + .withMember("service", serviceShapeId!!) + .withObjectMember("package") { + withMember("name", packageName!!) + withOptionalMember("version", packageVersion) + withOptionalMember("version", packageDescription) + } + .withOptionalMember("sdkId", sdkId) + .withOptionalMember("build", buildSettings) + + return obj.build() + } +} + +fun SmithyProjection.smithyKotlinPlugin(configure: SmithyKotlinPluginSettings.() -> Unit) { + val p = plugins.computeIfAbsent("kotlin-codegen") { SmithyKotlinPluginSettings() } as SmithyKotlinPluginSettings + p.apply(configure) +} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt index 5b8b08f37be..9df648823d4 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyProjection.kt @@ -8,23 +8,19 @@ package aws.sdk.kotlin.gradle.codegen.dsl import software.amazon.smithy.model.node.ArrayNode import software.amazon.smithy.model.node.Node import software.amazon.smithy.model.node.ObjectNode -import software.amazon.smithy.model.node.ToNode -import java.util.* - -// /** -// * A set of specifications for post-processing the generated files (e.g. remove files, move files around, etc) -// */ -// class PostProcessSpec { -// -// -// } +/** + * A container for settings related to a single Smithy projection. + * + * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections + */ class SmithyProjection( /** * The name of the projection */ val name: String, + // FIXME - technically this is based on plugin. Should the projection root dir be based on plugin as well rather than a single field? /** * Root directory for this projection */ @@ -33,8 +29,6 @@ class SmithyProjection( /** * List of files/directories to import when building the projection - * - * See https://awslabs.github.io/smithy/1.0/guides/building-models/build-config.html#projections */ var imports: List = emptyList() @@ -45,17 +39,10 @@ class SmithyProjection( */ var transforms: List = emptyList() - internal var pluginSettings: SmithyKotlinPluginSettings = SmithyKotlinPluginSettings() - /** - * Configure smithy-kotlin plugin settings. + * Plugin name to plugin settings. Plugins should provide an extension function to configure their own plugin settings */ - fun pluginSettings(configure: SmithyKotlinPluginSettings.() -> Unit) { pluginSettings.also(configure) } - - // private var postProcessSpec: PostProcessSpec? = null - // fun postProcess(spec: PostProcessSpec.() -> Unit) { - // postProcessSpec = PostProcessSpec().apply(spec) - // } + val plugins: MutableMap = mutableMapOf() internal fun toNode(): Node { // escape windows paths for valid json @@ -66,9 +53,14 @@ class SmithyProjection( val obj = ObjectNode.objectNodeBuilder() .withArrayMember("imports", formattedImports) .withMember("transforms", ArrayNode.fromNodes(transformNodes)) - .withObjectMember("plugins") { - withMember("kotlin-codegen", pluginSettings.toNode()) + + if (plugins.isNotEmpty()) { + obj.withObjectMember("plugins") { + plugins.forEach { (pluginName, pluginSettings) -> + withMember(pluginName, pluginSettings.toNode()) + } } + } return obj.build() } @@ -79,89 +71,20 @@ class SmithyProjection( other as SmithyProjection if (name != other.name) return false + if (projectionRootDir != other.projectionRootDir) return false if (imports != other.imports) return false if (transforms != other.transforms) return false - if (pluginSettings != other.pluginSettings) return false + if (plugins != other.plugins) return false return true } override fun hashCode(): Int { var result = name.hashCode() + result = 31 * result + projectionRootDir.hashCode() result = 31 * result + imports.hashCode() result = 31 * result + transforms.hashCode() - result = 31 * result + pluginSettings.hashCode() - return result - } -} - -class SmithyKotlinBuildSettings : ToNode { - var generateFullProject: Boolean? = null - var generateDefaultBuildFiles: Boolean? = null - var optInAnnotations: List? = null - - override fun toNode(): Node { - val builder = ObjectNode.objectNodeBuilder() - - builder.withOptionalMember("rootProject", generateFullProject) - builder.withOptionalMember("generateDefaultBuildFiles", generateDefaultBuildFiles) - - val optInArrNode = optInAnnotations?.map { Node.from(it) }?.let { ArrayNode.fromNodes(it) } - builder.withOptionalMember("optInAnnotations", Optional.ofNullable(optInArrNode)) - return builder.build() - } -} - -class SmithyKotlinPluginSettings { - var serviceShapeId: String? = null - var packageName: String? = null - var packageVersion: String? = null - var packageDescription: String? = null - var sdkId: String? = null - - internal var buildSettings: SmithyKotlinBuildSettings? = null - fun buildSettings(configure: SmithyKotlinBuildSettings.() -> Unit) { - if (buildSettings == null) buildSettings = SmithyKotlinBuildSettings() - buildSettings!!.apply(configure) - } - - override fun equals(other: Any?): Boolean { - if (this === other) return true - if (javaClass != other?.javaClass) return false - - other as SmithyKotlinPluginSettings - - if (serviceShapeId != other.serviceShapeId) return false - if (packageName != other.packageName) return false - if (packageVersion != other.packageVersion) return false - if (packageDescription != other.packageDescription) return false - if (sdkId != other.sdkId) return false - if (buildSettings != other.buildSettings) return false - - return true - } - - override fun hashCode(): Int { - var result = serviceShapeId?.hashCode() ?: 0 - result = 31 * result + (packageName?.hashCode() ?: 0) - result = 31 * result + (packageVersion?.hashCode() ?: 0) - result = 31 * result + (packageDescription?.hashCode() ?: 0) - result = 31 * result + (sdkId?.hashCode() ?: 0) - result = 31 * result + (buildSettings?.hashCode() ?: 0) + result = 31 * result + plugins.hashCode() return result } - - internal fun toNode(): Node { - val obj = ObjectNode.objectNodeBuilder() - .withMember("service", serviceShapeId!!) - .withObjectMember("package") { - withMember("name", packageName!!) - withOptionalMember("version", packageVersion) - withOptionalMember("version", packageDescription) - } - .withOptionalMember("sdkId", sdkId) - .withOptionalMember("build", buildSettings) - - return obj.build() - } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt index c9084084905..5a3d152c183 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt @@ -19,7 +19,7 @@ import software.amazon.smithy.model.node.Node internal fun Project.registerCodegenTasks() { // generate the projection file for smithy to consume val smithyBuildConfig = buildDir.resolve("smithy-build.json") - val generateSmithyBuild = tasks.register("kotlinCodegenGenerateBuildConfig") { + val generateSmithyBuild = tasks.register("generateSmithyBuildConfig") { description = "generate smithy-build.json" group = "codegen" @@ -46,7 +46,7 @@ internal fun Project.registerCodegenTasks() { } val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register("kotlinCodegenSmithyBuild") { + val buildTask = project.tasks.register("generateProjections") { dependsOn(generateSmithyBuild) description = "generate code using smithy-kotlin" group = "codegen" From bbbc81e7f64ff2bcd7643a41f483412a3284c7a7 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Nov 2021 14:37:04 -0500 Subject: [PATCH 24/31] refactor task names; use explicit smithy-cli configuration --- codegen/protocol-tests/build.gradle.kts | 4 +-- .../kotlin/gradle/codegen/CodegenPlugin.kt | 2 ++ ...lugin.kt => SmithyKotlinPluginSettings.kt} | 0 .../gradle/codegen/tasks/SmithyTasks.kt | 25 +++++++++++++------ 4 files changed, 22 insertions(+), 9 deletions(-) rename gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/{SmithyKotlinPlugin.kt => SmithyKotlinPluginSettings.kt} (100%) diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index b9c4d326b5c..4747ea7f4f2 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -41,7 +41,7 @@ val enabledProtocols = listOf( codegen { enabledProtocols.forEach { test -> - projections.create(test.projectionName) { + projections.register(test.projectionName) { transforms = listOf( """ { @@ -70,7 +70,7 @@ codegen { } } -tasks.getByName("generateProjections") { +tasks.named("generateSmithyProjections") { // NOTE: The protocol tests are published to maven as a jar, this ensures that // the aws-protocol-tests dependency is found when generating code such that the `includeServices` transform // actually works diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt index 53d696db893..2aba118cbb4 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt @@ -5,6 +5,7 @@ package aws.sdk.kotlin.gradle.codegen +import aws.sdk.kotlin.gradle.codegen.tasks.createSmithyCliConfiguration import aws.sdk.kotlin.gradle.codegen.tasks.registerCodegenTasks import org.gradle.api.Plugin import org.gradle.api.Project @@ -24,6 +25,7 @@ class CodegenPlugin : Plugin { } private fun Project.configurePlugins() { + createSmithyCliConfiguration() // unfortunately all of the tasks provided by smithy rely on the plugin extension, so it also needs applied plugins.apply("software.amazon.smithy") tasks.getByName("smithyBuildJar").enabled = false diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt similarity index 100% rename from gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPlugin.kt rename to gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt index 5a3d152c183..3aedd698fac 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt @@ -16,10 +16,13 @@ import org.gradle.kotlin.dsl.register import software.amazon.smithy.gradle.tasks.SmithyBuild import software.amazon.smithy.model.node.Node +private const val GENERATE_SMITHY_BUILD_CONFIG_TASK_NAME = "generateSmithyBuildConfig" +private const val GENERATE_SMITHY_PROJECTIONS_TASK_NAME = "generateSmithyProjections" + internal fun Project.registerCodegenTasks() { // generate the projection file for smithy to consume val smithyBuildConfig = buildDir.resolve("smithy-build.json") - val generateSmithyBuild = tasks.register("generateSmithyBuildConfig") { + val generateSmithyBuild = tasks.register(GENERATE_SMITHY_BUILD_CONFIG_TASK_NAME) { description = "generate smithy-build.json" group = "codegen" @@ -46,9 +49,9 @@ internal fun Project.registerCodegenTasks() { } val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register("generateProjections") { + val buildTask = project.tasks.register(GENERATE_SMITHY_PROJECTIONS_TASK_NAME) { dependsOn(generateSmithyBuild) - description = "generate code using smithy-kotlin" + description = "generate projections (code) using Smithy" group = "codegen" classpath = codegenConfig smithyBuildConfigs = files(smithyBuildConfig) @@ -97,19 +100,27 @@ private fun generateSmithyBuild(projections: Collection): Stri // create a configuration (classpath) needed by the SmithyBuild task private fun Project.createCodegenConfiguration(): Configuration { val codegenConfig = configurations.maybeCreate("codegenTaskConfiguration") + codegenConfig.extendsFrom(createSmithyCliConfiguration()) dependencies { // depend on aws-kotlin code generation codegenConfig(project(":codegen:smithy-aws-kotlin-codegen")) + } + + return codegenConfig +} +internal fun Project.createSmithyCliConfiguration(): Configuration { + // see: https://github.com/awslabs/smithy-gradle-plugin/blob/main/src/main/java/software/amazon/smithy/gradle/SmithyPlugin.java#L119 + val smithyCliConfig = configurations.maybeCreate("smithyCli") + dependencies { // smithy plugin requires smithy-cli to be on the classpath, for whatever reason configuring the plugin // from this plugin doesn't work correctly so we explicitly set it val smithyVersion: String by project - codegenConfig("software.amazon.smithy:smithy-cli:$smithyVersion") + smithyCliConfig("software.amazon.smithy:smithy-cli:$smithyVersion") // add aws traits to the compile classpath so that the smithy build task can discover them - codegenConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") + smithyCliConfig("software.amazon.smithy:smithy-aws-traits:$smithyVersion") } - - return codegenConfig + return smithyCliConfig } From d12aa1bc2a0af742cd81cdde7390a1cdb2d1dd6d Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Nov 2021 14:53:16 -0500 Subject: [PATCH 25/31] formatting --- aws-runtime/aws-config/build.gradle.kts | 4 ++-- codegen/protocol-tests/build.gradle.kts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index a5a235919d1..e1470f18698 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -85,7 +85,7 @@ codegen { awsModelFile("sts.2011-06-15.json") ) - smithyKotlinPlugin{ + smithyKotlinPlugin { serviceShapeId = "com.amazonaws.sts#AWSSecurityTokenServiceV20110615" packageName = "${basePackage}.sts" packageVersion = project.version.toString() @@ -120,7 +120,7 @@ codegen { ) val serviceShape = "com.amazonaws.sso#SWBPortalService" - smithyKotlinPlugin{ + smithyKotlinPlugin { serviceShapeId = serviceShape packageName = "${basePackage}.sso" packageVersion = project.version.toString() diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index 4747ea7f4f2..0e4cf893631 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -53,7 +53,7 @@ codegen { """ ) - smithyKotlinPlugin{ + smithyKotlinPlugin { serviceShapeId = test.serviceShapeId packageName = "aws.sdk.kotlin.services.${test.packageName}" packageVersion = "1.0" From ac0db90879d51fe7c44ff640c0b2fe7cebcfc6c6 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Mon, 22 Nov 2021 17:07:58 -0500 Subject: [PATCH 26/31] refactor sdk build script to use codegen plugin --- codegen/sdk/build.gradle.kts | 158 +++++++----------- .../sdk/kotlin/gradle/codegen/dsl/Utils.kt | 2 +- 2 files changed, 57 insertions(+), 103 deletions(-) diff --git a/codegen/sdk/build.gradle.kts b/codegen/sdk/build.gradle.kts index 6d1828550b1..31ba48e04af 100644 --- a/codegen/sdk/build.gradle.kts +++ b/codegen/sdk/build.gradle.kts @@ -6,7 +6,9 @@ // This build file has been adapted from the Go v2 SDK, here: // https://github.com/aws/aws-sdk-go-v2/blob/master/codegen/sdk-codegen/build.gradle.kts -import software.amazon.smithy.gradle.tasks.SmithyBuild +import aws.sdk.kotlin.gradle.codegen.dsl.SmithyProjection +import aws.sdk.kotlin.gradle.codegen.dsl.projectionRootDir +import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin import software.amazon.smithy.model.Model import software.amazon.smithy.model.shapes.ServiceShape import java.util.* @@ -16,27 +18,20 @@ import kotlin.streams.toList description = "AWS SDK codegen tasks" plugins { - id("software.amazon.smithy") + id("aws.sdk.kotlin.codegen") } buildscript { val smithyVersion: String by project dependencies { + classpath("software.amazon.smithy:smithy-model:$smithyVersion") classpath("software.amazon.smithy:smithy-aws-traits:$smithyVersion") - classpath("software.amazon.smithy:smithy-cli:$smithyVersion") } } -dependencies { - implementation(project(":codegen:smithy-aws-kotlin-codegen")) -} - // This project doesn't produce a JAR. tasks["jar"].enabled = false -// Run the SmithyBuild task manually since this project needs the built JAR -tasks["smithyBuildJar"].enabled = false - // get a project property by name if it exists (including from local.properties) fun getProperty(name: String): String? { if (project.hasProperty(name)) { @@ -108,56 +103,49 @@ val disabledServices = setOf( "timestreamquery" ) -// Generates a smithy-build.json file by creating a new projection. -// The generated smithy-build.json file is not committed to git since -// it's rebuilt each time codegen is performed. -fun generateSmithyBuild(services: List): String { - require(services.isNotEmpty()) { - "No services discovered. Verify aws.services and aws.protocols properties in local.build. Aborting." - } - - val projections = services.joinToString(",") { service -> - // escape windows paths for valid json - val absModelPath = service.modelFile.absolutePath.replace("\\", "\\\\") - val importPaths = mutableListOf(absModelPath) - if (file(service.modelExtrasDir).exists()) { - importPaths.add(service.modelExtrasDir.replace("\\", "\\\\")) - } - val imports = importPaths.joinToString { "\"$it\"" } - val transforms = transformsForService(service) - - """ - "${service.projectionName}": { - "imports": [$imports], - "plugins": { - "kotlin-codegen": { - "service": "${service.name}", - "package" : { - "name": "${service.packageName}", - "version": "${service.packageVersion}", - "description": "${service.description}" - }, - "sdkId": "${service.sdkId}", - "build": { - "generateDefaultBuildFiles": false - } +// Manually create the projections rather than using the extension to avoid unnecessary configuration evaluation. +// Otherwise we would be reading the models from disk on every gradle invocation for unrelated projects/tasks +fun awsServiceProjections(): Provider> { + val p = project.provider { + println("AWS service projection provider called") + discoveredServices + }.map { + it.map { service -> + SmithyProjection( + service.projectionName, + project.projectionRootDir(service.projectionName) + ).apply { + val importPaths = mutableListOf(service.modelFile.absolutePath) + if (file(service.modelExtrasDir).exists()) { + importPaths.add(service.modelExtrasDir) + } + imports = importPaths + transforms = transformsForService(service) ?: emptyList() + + smithyKotlinPlugin { + serviceShapeId = service.name + packageName = service.packageName + packageVersion = service.packageVersion + packageDescription = service.description + sdkId = service.sdkId + buildSettings { + generateFullProject = false + generateDefaultBuildFiles = false } - }, - "transforms": $transforms + } } - """ - } - - return """ - { - "version": "1.0", - "projections": { - $projections } } - """.trimIndent() + + // get around class cast issues, listProperty implements what we need to pass this to `NamedObjectContainer` + return project.objects.listProperty().value(p) } +// this will lazily evaluate the provider and only cause the models to be +// mapped if the tasks are actually needed +// NOTE: FYI evaluation still happens if you ask for the list of tasks or rebuild the gradle model in intellij +codegen.projections.addAllLater(awsServiceProjections()) + /** * This function retrieves Smithy transforms associated with the target service to be merged into generated * `smithy-build.json` files. The transform file MUST live in /transforms. @@ -168,11 +156,11 @@ fun generateSmithyBuild(services: List): String { * -.json * Example: renameShapes-MarketplaceCommerceAnalyticsException.json */ -fun transformsForService(service: AwsService): String { +fun transformsForService(service: AwsService): List? { val transformsDir = File(service.transformsDir) return transformsDir.listFiles()?.map { transformFile -> transformFile.readText() - }?.toString() ?: "[]" + } } val discoveredServices: List by lazy { discoverServices() } @@ -185,6 +173,7 @@ val sdkPackageNamePrefix = "aws.sdk.kotlin.services." * membership tests */ fun discoverServices(applyFilters: Boolean = true): List { + println("discover services called") val modelsDir: String by project val serviceMembership = parseMembership(getProperty("aws.services")) val protocolMembership = parseMembership(getProperty("aws.protocols")) @@ -294,43 +283,6 @@ fun java.util.Optional.orNull(): T? = this.orElse(null) fun String.kotlinNamespace(): String = split(".") .joinToString(separator = ".") { segment -> segment.filter { it.isLetterOrDigit() } } -// Generate smithy-build.json as first step in build task -task("generateSmithyBuild") { - group = "codegen" - description = "generate smithy-build.json" - doFirst { - projectDir - .resolve("smithy-build.json") - .writeText(generateSmithyBuild(discoveredServices)) - } -} - -tasks.create("generateSdk") { - group = "codegen" - // ensure the generated clients use the same version of the runtime as the aws aws-runtime - val smithyKotlinVersion: String by project - doFirst { - System.setProperty("smithy.kotlin.codegen.clientRuntimeVersion", smithyKotlinVersion) - } - - addRuntimeClasspath = true - dependsOn(tasks["generateSmithyBuild"]) - inputs.file(projectDir.resolve("smithy-build.json")) - // ensure smithy-aws-kotlin-codegen is up to date - inputs.files(configurations.compileClasspath) -} - -// Remove generated model file for clean -tasks["clean"].doFirst { - delete("smithy-build.json") -} - -/** - * The directory code is generated to - */ -val AwsService.projectionOutputDir: String - get() = project.file("${project.buildDir}/smithyprojections/${project.name}/${projectionName}/kotlin-codegen").absolutePath - /** * The project directory under `aws-sdk-kotlin/services` */ @@ -352,31 +304,33 @@ val AwsService.modelExtrasDir: String val AwsService.transformsDir: String get() = rootProject.file("${destinationDir}/transforms").absolutePath -task("stageSdks") { +val stageSdks = tasks.register("stageSdks") { group = "codegen" description = "relocate generated SDK(s) from build directory to services/ dir" - dependsOn("generateSdk") + dependsOn(tasks.named("generateSmithyProjections")) doLast { + println("discoveredServices = ${discoveredServices.joinToString { it.sdkId }}") discoveredServices.forEach { - logger.info("copying ${it.projectionOutputDir} to ${it.destinationDir}") + val projectionOutputDir = codegen.projections.getByName(it.projectionName).projectionRootDir + logger.info("copying $projectionOutputDir to ${it.destinationDir}") copy { - from("${it.projectionOutputDir}/src") + from("$projectionOutputDir/src") into("${it.destinationDir}/generated-src") } copy { - from("${it.projectionOutputDir}/build.gradle.kts") - into("${it.destinationDir}") + from("$projectionOutputDir/build.gradle.kts") + into(it.destinationDir) } } } } -tasks.create("bootstrap") { +tasks.register("bootstrap") { group = "codegen" description = "Generate AWS SDK's and register them with the build" - dependsOn(tasks["generateSdk"]) - finalizedBy(tasks["stageSdks"]) + dependsOn(tasks.named("generateSmithyProjections")) + finalizedBy(stageSdks) } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt index 6eb1491a1e0..addabad4e8c 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt @@ -19,7 +19,7 @@ import java.util.* /** * Get the root directory of the generated kotlin code for a projection */ -internal fun Project.projectionRootDir(projectionName: String): java.io.File = +fun Project.projectionRootDir(projectionName: String): java.io.File = file("${project.buildDir}/smithyprojections/${project.name}/$projectionName/kotlin-codegen") /** From d578b656abe1035f163fe1c26cbd5784a086da2d Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 1 Dec 2021 13:20:58 -0500 Subject: [PATCH 27/31] cleanup --- .gitignore | 2 -- .../aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt | 1 - .../kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt | 1 - .../main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt | 1 + 4 files changed, 1 insertion(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 022f474900b..2d0d9a18765 100644 --- a/.gitignore +++ b/.gitignore @@ -12,8 +12,6 @@ build/ .idea/ __pycache__/ local.properties -!gradle/sdk-plugins/src/** -!buildSrc/src/** # ignore generated files services/*/generated-src services/*/build.gradle.kts diff --git a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt index 5381aa694ad..f77c32d0639 100644 --- a/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt +++ b/codegen/smithy-aws-kotlin-codegen/src/main/kotlin/aws/sdk/kotlin/codegen/transforms/IncludeOperations.kt @@ -17,7 +17,6 @@ import software.amazon.smithy.model.shapes.OperationShape class IncludeOperations : ConfigurableProjectionTransformer() { class Config { -// var service: String = "" var operations: Set = emptySet() } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt index 93a30cf5bee..36065b8479c 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenExtension.kt @@ -14,7 +14,6 @@ import org.gradle.api.Project */ open class CodegenExtension(private val project: Project) { - // TODO - allow setting default build settings that apply to every projection (or every projection starts with)? val projections = project.objects.domainObjectContainer(SmithyProjection::class.java) { name -> SmithyProjection(name, project.projectionRootDir(name)) } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt index 2aba118cbb4..600bfb8bc84 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt @@ -27,6 +27,7 @@ class CodegenPlugin : Plugin { private fun Project.configurePlugins() { createSmithyCliConfiguration() // unfortunately all of the tasks provided by smithy rely on the plugin extension, so it also needs applied + // see https://github.com/awslabs/smithy-gradle-plugin/issues/45 plugins.apply("software.amazon.smithy") tasks.getByName("smithyBuildJar").enabled = false } From 1779d8e194a9bbd8bc6d9d0202af82250c6a7dcb Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 1 Dec 2021 14:56:58 -0500 Subject: [PATCH 28/31] remove rollup CodegenTask --- aws-runtime/aws-config/build.gradle.kts | 8 +++---- codegen/protocol-tests/build.gradle.kts | 6 ++--- .../kotlin/gradle/codegen/CodegenPlugin.kt | 4 ++-- .../gradle/codegen/tasks/CodegenTask.kt | 24 ------------------- .../gradle/codegen/tasks/SmithyTasks.kt | 5 ---- 5 files changed, 9 insertions(+), 38 deletions(-) delete mode 100644 gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index e1470f18698..d5d23c5fb11 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -155,17 +155,17 @@ NOTE: We need the following tasks to depend on codegen for gradle caching/up-to- * `compileKotlinMetadata` (Type=KotlinCompileCommon) * `sourcesJar` and `jvmSourcesJar` (Type=org.gradle.jvm.tasks.Jar) */ -val codegenTasks = tasks.withType() +val codegenTask = tasks.named("generateSmithyProjections") tasks.withType { - dependsOn(codegenTasks) + dependsOn(codegenTask) } tasks.withType { - dependsOn(codegenTasks) + dependsOn(codegenTask) } tasks.withType { - dependsOn(codegenTasks) + dependsOn(codegenTask) } diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index 0e4cf893631..2b2aa60049b 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -110,12 +110,12 @@ open class ProtocolTestTask : DefaultTask() { } } -val kotlinCodegenTask = tasks.getByName("kotlinCodegen") +val codegenTask = tasks.getByName("generateSmithyProjections") codegen.projections.forEach { val protocolName = it.name tasks.register("testProtocol-$protocolName") { - dependsOn(kotlinCodegenTask) + dependsOn(codegenTask) group = "Verification" projection = it } @@ -128,7 +128,7 @@ codegen.projections.forEach { into(it.projectionRootDir.resolve("src/main/kotlin/")) } - kotlinCodegenTask.finalizedBy(copyStaticFiles) + codegenTask.finalizedBy(copyStaticFiles) } tasks.register("testAllProtocols") { diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt index 600bfb8bc84..b5f77e5cdcf 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/CodegenPlugin.kt @@ -14,8 +14,8 @@ const val CODEGEN_EXTENSION_NAME = "codegen" /** * This plugin handles: - * - applying smithy plugins to the project to generate code - * - providing a [CodegenTask] to generate Kotlin sources from their respective smithy models. + * - applying smithy-gradle-plugin to the project to generate code + * - providing a tasks to generate Kotlin sources from their respective smithy models. */ class CodegenPlugin : Plugin { override fun apply(target: Project): Unit = target.run { diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt deleted file mode 100644 index d14521fb883..00000000000 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/CodegenTask.kt +++ /dev/null @@ -1,24 +0,0 @@ -/* - * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. - * SPDX-License-Identifier: Apache-2.0. - */ - -package aws.sdk.kotlin.gradle.codegen.tasks - -import org.gradle.api.DefaultTask -import org.gradle.api.tasks.TaskAction - -abstract class CodegenTask : DefaultTask() { - init { - group = "codegen" - description = "Generate code using smithy-kotlin" - } - - @TaskAction - fun generateCode() { - logger.info("generating kotlin code for projections") - // NOTE: this task has dependencies on a smithy build task for the projection - // it doesn't actually do any work (yet) but it does give us our own task to extend _AFTER_ code - // has been generated - } -} diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt index 3aedd698fac..ffcef8dcd5a 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt @@ -74,11 +74,6 @@ internal fun Project.registerCodegenTasks() { // ensure smithy-aws-kotlin-codegen is up to date inputs.files(codegenConfig) } - - project.tasks.register("kotlinCodegen") { - dependsOn(buildTask) - description = "generate code for projections" - } } /** From f665b309c26f3954dd79f06c9277c0b223e7b3de Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Wed, 1 Dec 2021 15:08:56 -0500 Subject: [PATCH 29/31] restore aws-config build for now --- aws-runtime/aws-config/build.gradle.kts | 124 ------------------------ 1 file changed, 124 deletions(-) diff --git a/aws-runtime/aws-config/build.gradle.kts b/aws-runtime/aws-config/build.gradle.kts index d5d23c5fb11..3ec3e91c229 100644 --- a/aws-runtime/aws-config/build.gradle.kts +++ b/aws-runtime/aws-config/build.gradle.kts @@ -1,12 +1,7 @@ -import aws.sdk.kotlin.gradle.codegen.dsl.smithyKotlinPlugin - /* * Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. * SPDX-License-Identifier: Apache-2.0. */ -plugins { - id("aws.sdk.kotlin.codegen") -} description = "Support for AWS configuration" extra["moduleName"] = "aws.sdk.kotlin.runtime.config" @@ -36,17 +31,6 @@ kotlin { implementation("aws.sdk.kotlin.crt:aws-crt-kotlin:$crtKotlinVersion") implementation(project(":aws-runtime:crt-util")) - - - // additional dependencies required by generated sts provider - implementation("aws.smithy.kotlin:serde-form-url:$smithyKotlinVersion") - implementation("aws.smithy.kotlin:serde-xml:$smithyKotlinVersion") - implementation(project(":aws-runtime:protocols:aws-xml-protocols")) - implementation(project(":aws-runtime:aws-endpoint")) - implementation(project(":aws-runtime:aws-signing")) - - // additional dependencies required by generated sso provider - implementation(project(":aws-runtime:protocols:aws-json-protocols")) } } commonTest { @@ -71,111 +55,3 @@ kotlin { } } } - -fun awsModelFile(name: String): String = - rootProject.file("codegen/sdk/aws-models/$name").absolutePath - -codegen { - val basePackage = "aws.sdk.kotlin.runtime.auth.credentials.internal" - - projections { - // generate an sts client - create("sts-credentials-provider") { - imports = listOf( - awsModelFile("sts.2011-06-15.json") - ) - - smithyKotlinPlugin { - serviceShapeId = "com.amazonaws.sts#AWSSecurityTokenServiceV20110615" - packageName = "${basePackage}.sts" - packageVersion = project.version.toString() - packageDescription = "Internal STS credentials provider" - sdkId = "STS" - buildSettings { - generateDefaultBuildFiles = false - generateFullProject = false - } - } - - // TODO - could we add a trait such that we change visibility to `internal` or a build setting...? - transforms = listOf( - """ - { - "name": "awsSdkKotlinIncludeOperations", - "args": { - "operations": [ - "com.amazonaws.sts#AssumeRoleWithWebIdentity", - "com.amazonaws.sts#AssumeRole" - ] - } - } - """ - ) - } - - // generate an sso client - create("sso-credentials-provider") { - imports = listOf( - awsModelFile("sso.2019-06-10.json") - ) - - val serviceShape = "com.amazonaws.sso#SWBPortalService" - smithyKotlinPlugin { - serviceShapeId = serviceShape - packageName = "${basePackage}.sso" - packageVersion = project.version.toString() - packageDescription = "Internal SSO credentials provider" - sdkId = "SSO" - buildSettings { - generateDefaultBuildFiles = false - generateFullProject = false - } - } - - transforms = listOf( - """ - { - "name": "awsSdkKotlinIncludeOperations", - "args": { - "operations": [ - "com.amazonaws.sso#GetRoleCredentials" - ] - } - } - """ - ) - } - } -} - -/* -NOTE: We need the following tasks to depend on codegen for gradle caching/up-to-date checks to work correctly: - -* `compileKotlinJvm` (Type=KotlinCompile) -* `compileKotlinMetadata` (Type=KotlinCompileCommon) -* `sourcesJar` and `jvmSourcesJar` (Type=org.gradle.jvm.tasks.Jar) -*/ -val codegenTask = tasks.named("generateSmithyProjections") -tasks.withType { - dependsOn(codegenTask) -} - -tasks.withType { - dependsOn(codegenTask) -} - -tasks.withType { - dependsOn(codegenTask) -} - - -codegen.projections.all { - // add this projected source dir to the common sourceSet - // NOTE - build.gradle.kts is still being generated, it's NOT used though - // TODO - we should probably either have a postProcessing spec or a plugin setting to not generate it to avoid confusion - val projectedSrcDir = projectionRootDir.resolve("src/main/kotlin") - kotlin.sourceSets.commonMain { - println("added $projectedSrcDir to common sourceSet") - kotlin.srcDir(projectedSrcDir) - } -} From de98c55acfab2d41f760184fa77d47d1f781b45e Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Fri, 3 Dec 2021 12:13:08 -0500 Subject: [PATCH 30/31] fix package description and version --- .../sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt index ceaa1d00082..9ff39c1e929 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt @@ -75,7 +75,7 @@ class SmithyKotlinPluginSettings : SmithyBuildPlugin { .withObjectMember("package") { withMember("name", packageName!!) withOptionalMember("version", packageVersion) - withOptionalMember("version", packageDescription) + withOptionalMember("description", packageDescription) } .withOptionalMember("sdkId", sdkId) .withOptionalMember("build", buildSettings) From 2368e0d0738c379c45b16fc215946d23b317c1c4 Mon Sep 17 00:00:00 2001 From: Aaron J Todd Date: Tue, 7 Dec 2021 10:01:24 -0500 Subject: [PATCH 31/31] pr feedback --- codegen/protocol-tests/build.gradle.kts | 3 +-- .../gradle/codegen/dsl/SmithyKotlinPluginSettings.kt | 12 ++++++------ .../aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt | 6 +++--- .../sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt | 2 +- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/codegen/protocol-tests/build.gradle.kts b/codegen/protocol-tests/build.gradle.kts index 2b2aa60049b..ee930801c6d 100644 --- a/codegen/protocol-tests/build.gradle.kts +++ b/codegen/protocol-tests/build.gradle.kts @@ -17,8 +17,7 @@ dependencies { } data class ProtocolTest(val projectionName: String, val serviceShapeId: String, val sdkId: String? = null) { - val packageName: String - get() = projectionName.toLowerCase().filter { it.isLetterOrDigit() } + val packageName: String = projectionName.toLowerCase().filter { it.isLetterOrDigit() } } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt index 9ff39c1e929..2f320dbd5eb 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/SmithyKotlinPluginSettings.kt @@ -19,8 +19,8 @@ class SmithyKotlinBuildSettings : ToNode { override fun toNode(): Node { val builder = ObjectNode.objectNodeBuilder() - builder.withOptionalMember("rootProject", generateFullProject) - builder.withOptionalMember("generateDefaultBuildFiles", generateDefaultBuildFiles) + builder.withNullableMember("rootProject", generateFullProject) + builder.withNullableMember("generateDefaultBuildFiles", generateDefaultBuildFiles) val optInArrNode = optInAnnotations?.map { Node.from(it) }?.let { ArrayNode.fromNodes(it) } builder.withOptionalMember("optInAnnotations", Optional.ofNullable(optInArrNode)) @@ -74,11 +74,11 @@ class SmithyKotlinPluginSettings : SmithyBuildPlugin { .withMember("service", serviceShapeId!!) .withObjectMember("package") { withMember("name", packageName!!) - withOptionalMember("version", packageVersion) - withOptionalMember("description", packageDescription) + withNullableMember("version", packageVersion) + withNullableMember("description", packageDescription) } - .withOptionalMember("sdkId", sdkId) - .withOptionalMember("build", buildSettings) + .withNullableMember("sdkId", sdkId) + .withNullableMember("build", buildSettings) return obj.build() } diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt index addabad4e8c..95f12d8a86e 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/dsl/Utils.kt @@ -33,17 +33,17 @@ internal fun ObjectNode.Builder.withObjectMember(key: String, block: ObjectNode. builder.apply(block) return withMember(key, builder.build()) } -internal fun ObjectNode.Builder.withOptionalMember(key: String, member: String?): ObjectNode.Builder = apply { +internal fun ObjectNode.Builder.withNullableMember(key: String, member: String?): ObjectNode.Builder = apply { if (member == null) return this return withMember(key, member) } -internal fun ObjectNode.Builder.withOptionalMember(key: String, member: Boolean?): ObjectNode.Builder = apply { +internal fun ObjectNode.Builder.withNullableMember(key: String, member: Boolean?): ObjectNode.Builder = apply { if (member == null) return this return withMember(key, member) } -internal fun ObjectNode.Builder.withOptionalMember(key: String, member: T?): ObjectNode.Builder = +internal fun ObjectNode.Builder.withNullableMember(key: String, member: T?): ObjectNode.Builder = withOptionalMember(key, Optional.ofNullable(member)) internal fun ObjectNode.Builder.withArrayMember(key: String, member: List): ObjectNode.Builder = apply { diff --git a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt index ffcef8dcd5a..774884f15e5 100644 --- a/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt +++ b/gradle/sdk-plugins/src/main/kotlin/aws/sdk/kotlin/gradle/codegen/tasks/SmithyTasks.kt @@ -49,7 +49,7 @@ internal fun Project.registerCodegenTasks() { } val codegenConfig = createCodegenConfiguration() - val buildTask = project.tasks.register(GENERATE_SMITHY_PROJECTIONS_TASK_NAME) { + project.tasks.register(GENERATE_SMITHY_PROJECTIONS_TASK_NAME) { dependsOn(generateSmithyBuild) description = "generate projections (code) using Smithy" group = "codegen"