Add manifest files generated from ksp into final jar (#1094)

* Repro for KSP issue related to files generated under META-INF

* Merge META-INF/services generated by KSP into output JAR

* Add manifest files generated from ksp into final jar

* Add EOL

* Add missing build file changes

* Add tests to check exact entry matches

* Fix build

* Fix more lint errors

* Revert pre-commit changes in module.bazel

* Rename test file

* Move tests to assertion test

---------

Co-authored-by: James Barr <jbarr@uber.com>

MODULE.bazel

@@ -55,6 +55,11 @@ maven.install(
         "org.pantsbuild:jarjar:1.7.2",
         "org.jetbrains.kotlinx:atomicfu-js:0.15.2",
         "org.jetbrains.kotlinx:kotlinx-serialization-runtime:1.0-M1-1.4.0-rc",
+        "dev.zacsweers.autoservice:auto-service-ksp:jar:1.1.0",
+        "com.squareup.moshi:moshi:1.15.0",
+        "com.squareup.moshi:moshi-kotlin:1.15.0",
+        "com.squareup.moshi:moshi-kotlin-codegen:1.15.0",
+        "com.google.auto.service:auto-service-annotations:jar:1.1.1",
     ],
     fetch_sources = True,
     repositories = [

examples/ksp/BUILD

@@ -38,6 +38,15 @@ kt_ksp_plugin(
     ],
 )
 
+kt_ksp_plugin(
+    name = "autoservice",
+    processor_class = "dev.zacsweers.autoservice.ksp.AutoServiceSymbolProcessor$Provider",
+    deps = [
+        "@maven//:com_google_auto_service_auto_service_annotations",
+        "@maven//:dev_zacsweers_autoservice_auto_service_ksp",
+    ],
+)
+
 kt_jvm_library(
     name = "coffee_lib",
     srcs = glob([
@@ -47,8 +56,10 @@ kt_jvm_library(
     plugins = [
         "//:moshi-kotlin-codegen",
         "//:autovalue",
+        "//:autoservice",
     ],
     deps = [
+        "@maven//:com_google_auto_service_auto_service_annotations",
         "@maven//:com_google_auto_value_auto_value_annotations",
         "@maven//:com_squareup_moshi_moshi",
         "@maven//:com_squareup_moshi_moshi_kotlin",

examples/ksp/CoffeeAppJavaModel.java

@@ -18,4 +18,4 @@ abstract static class Builder {
 
         abstract CoffeeAppJavaModel build();
     }
-}
+}

examples/ksp/CoffeeAppService.java

@@ -0,0 +1,7 @@
+package coffee;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(Object.class)
+public class CoffeeAppService {
+}

examples/ksp/WORKSPACE

@@ -45,8 +45,10 @@ maven_install(
         "com.squareup.moshi:moshi:1.14.0",
         "com.squareup.moshi:moshi-kotlin:1.14.0",
         "com.squareup.moshi:moshi-kotlin-codegen:1.14.0",
+        "com.google.auto.service:auto-service-annotations:jar:1.1.1",
         "com.google.auto.value:auto-value:1.10.1",
         "com.google.auto.value:auto-value-annotations:1.10.1",
+        "dev.zacsweers.autoservice:auto-service-ksp:jar:1.1.0",
     ],
     repositories = [
         "https://maven.google.com",

src/main/kotlin/io/bazel/kotlin/builder/tasks/jvm/CompilationTask.kt

@@ -26,6 +26,7 @@ import io.bazel.kotlin.builder.toolchain.KotlinToolchain
 import io.bazel.kotlin.builder.utils.IS_JVM_SOURCE_FILE
 import io.bazel.kotlin.builder.utils.bazelRuleKind
 import io.bazel.kotlin.builder.utils.jars.JarCreator
+import io.bazel.kotlin.builder.utils.jars.JarHelper.Companion.MANIFEST_DIR
 import io.bazel.kotlin.builder.utils.jars.SourceJarExtractor
 import io.bazel.kotlin.builder.utils.partitionJvmSources
 import io.bazel.kotlin.model.JvmCompilationTask
@@ -39,9 +40,13 @@ import java.nio.file.Files.isDirectory
 import java.nio.file.Files.walk
 import java.nio.file.Path
 import java.nio.file.Paths
+import java.nio.file.StandardCopyOption
 import java.util.Base64
 import java.util.stream.Collectors.toList
 import java.util.stream.Stream
+import kotlin.io.path.exists
+
+private const val SOURCE_JARS_DIR = "_srcjars"
 
 fun JvmCompilationTask.codeGenArgs(): CompilationArgs = CompilationArgs()
   .absolutePaths(info.friendPathsList) {
@@ -438,16 +443,16 @@ fun JvmCompilationTask.compileKotlin(
 
 /**
  * If any srcjars were provided expand the jars sources and create a new [JvmCompilationTask] with the
- * Java and Kotlin sources merged in.
+ * Java, Kotlin sources and META folder merged in.
  */
 internal fun JvmCompilationTask.expandWithSourceJarSources(): JvmCompilationTask =
   if (inputs.sourceJarsList.isEmpty()) {
     this
   } else {
     expandWithSources(
       SourceJarExtractor(
-        destDir = Paths.get(directories.temp).resolve("_srcjars"),
-        fileMatcher = IS_JVM_SOURCE_FILE,
+        destDir = Paths.get(directories.temp).resolve(SOURCE_JARS_DIR),
+        fileMatcher = { str: String -> IS_JVM_SOURCE_FILE.test(str) || "/$MANIFEST_DIR" in str },
       ).also {
         it.jarFiles.addAll(inputs.sourceJarsList.map { p -> Paths.get(p) })
         it.execute()
@@ -485,10 +490,12 @@ fun JvmCompilationTask.expandWithGeneratedSources(): JvmCompilationTask =
 
 private fun JvmCompilationTask.expandWithSources(sources: Iterator<String>): JvmCompilationTask =
   updateBuilder { builder ->
-    sources.filterOutNonCompilableSources().partitionJvmSources(
-      { builder.inputsBuilder.addKotlinSources(it) },
-      { builder.inputsBuilder.addJavaSources(it) },
-    )
+    sources.copyManifestFilesToGeneratedClasses(directories)
+      .filterOutNonCompilableSources()
+      .partitionJvmSources(
+        { builder.inputsBuilder.addKotlinSources(it) },
+        { builder.inputsBuilder.addJavaSources(it) },
+      )
   }
 
 private fun JvmCompilationTask.updateBuilder(
@@ -499,6 +506,29 @@ private fun JvmCompilationTask.updateBuilder(
     it.build()
   }
 
+/**
+ * Copy generated manifest files from KSP task into generated folder
+ */
+internal fun Iterator<String>.copyManifestFilesToGeneratedClasses(
+  directories: Directories,
+): Iterator<String> {
+  val result = mutableSetOf<String>()
+  this.forEach {
+    if ("/$MANIFEST_DIR" in it) {
+      val path = Paths.get(it)
+      val srcJarsPath = Paths.get(directories.temp, SOURCE_JARS_DIR)
+      if (srcJarsPath.exists()) {
+        val relativePath = srcJarsPath.relativize(path)
+        val destPath = Paths.get(directories.generatedClasses).resolve(relativePath)
+        destPath.parent.toFile().mkdirs()
+        Files.copy(path, destPath, StandardCopyOption.REPLACE_EXISTING)
+      }
+    }
+    result.add(it)
+  }
+  return result.iterator()
+}
+
 /**
  * Only keep java and kotlin files for the iterator. Filter our all other non-compilable files.
  */

src/main/kotlin/io/bazel/kotlin/builder/utils/jars/JarExtractor.kt

@@ -17,6 +17,7 @@ package io.bazel.kotlin.builder.utils.jars
 
 import java.nio.file.Files
 import java.nio.file.Path
+import java.nio.file.StandardCopyOption
 import java.util.jar.JarFile
 
 open class JarExtractor protected constructor(
@@ -42,7 +43,7 @@ open class JarExtractor protected constructor(
                     Files.createDirectories(target)
                   else -> jar.getInputStream(entry).use {
                     Files.createDirectories(target.parent)
-                    Files.copy(it, target)
+                    Files.copy(it, target, StandardCopyOption.REPLACE_EXISTING)
                   }
                 }
               }

src/main/starlark/core/repositories/setup.bzl

@@ -43,6 +43,11 @@ def kt_configure():
             "org.pantsbuild:jarjar:1.7.2",
             "org.jetbrains.kotlinx:atomicfu-js:0.15.2",
             "org.jetbrains.kotlinx:kotlinx-serialization-runtime:1.0-M1-1.4.0-rc",
+            "dev.zacsweers.autoservice:auto-service-ksp:jar:1.1.0",
+            "com.squareup.moshi:moshi:1.15.0",
+            "com.squareup.moshi:moshi-kotlin:1.15.0",
+            "com.squareup.moshi:moshi-kotlin-codegen:1.15.0",
+            "com.google.auto.service:auto-service-annotations:jar:1.1.1",
         ],
         repositories = [
             "https://maven-central.storage.googleapis.com/repos/central/data/",

src/test/data/jvm/ksp/BUILD

@@ -0,0 +1,64 @@
+load("//kotlin:core.bzl", "kt_ksp_plugin")
+load("//kotlin:jvm.bzl", "kt_jvm_library")
+
+# Copyright 2018 The Bazel Authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+package(default_visibility = ["//visibility:private"])
+
+kt_ksp_plugin(
+    name = "autoservice",
+    processor_class = "dev.zacsweers.autoservice.ksp.AutoServiceSymbolProcessor$Provider",
+    deps = [
+        "@kotlin_rules_maven//:dev_zacsweers_autoservice_auto_service_ksp",
+    ],
+)
+
+kt_ksp_plugin(
+    name = "moshi",
+    processor_class = "com.squareup.moshi.kotlin.codegen.ksp.JsonClassSymbolProcessorProvider",
+    deps = [
+        "@kotlin_rules_maven//:com_squareup_moshi_moshi",
+        "@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin",
+        "@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin_codegen",
+    ],
+)
+
+kt_jvm_library(
+    name = "moshi_lib",
+    srcs = ["CoffeeAppModel.kt"],
+    plugins = [":moshi"],
+    deps = [
+        "@kotlin_rules_maven//:com_squareup_moshi_moshi",
+        "@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin",
+        "@kotlin_rules_maven//:com_squareup_moshi_moshi_kotlin_codegen",
+    ],
+)
+
+kt_jvm_library(
+    name = "coffee_lib",
+    srcs = ["CoffeeAppService.java"],
+    plugins = [":autoservice"],
+    deps = [
+        "@kotlin_rules_maven//:dev_zacsweers_autoservice_auto_service_ksp",
+    ],
+)
+
+filegroup(
+    name = "ksp",
+    srcs = [
+        ":coffee_lib.jar",
+        ":moshi_lib.jar",
+    ],
+    visibility = ["//visibility:public"],
+)

src/test/data/jvm/ksp/CoffeeApp.kt

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package coffee
+
+import com.squareup.moshi.Moshi
+
+class CoffeeApp {
+
+  companion object {
+
+    private val adapter = CoffeeAppModelJsonAdapter(Moshi.Builder().build())
+    private val d = AutoValue_CoffeeAppJavaModel.Builder()
+      .setCoffeeAppModel(CoffeeAppModel("1"))
+      .build()
+  }
+}

src/test/data/jvm/ksp/CoffeeAppModel.kt

@@ -0,0 +1,21 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package src.test.data.jvm.ksp
+
+import com.squareup.moshi.JsonClass
+
+@JsonClass(generateAdapter = true)
+data class CoffeeAppModel(val id: Int)

src/test/data/jvm/ksp/CoffeeAppService.java

@@ -0,0 +1,22 @@
+/*
+ * Copyright 2018 The Bazel Authors. All rights reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package src.test.data.jvm.ksp;
+
+import com.google.auto.service.AutoService;
+
+@AutoService(Object.class)
+public class CoffeeAppService {
+}

src/test/kotlin/io/bazel/kotlin/BUILD

@@ -62,13 +62,20 @@ kt_rules_e2e_test(
     srcs = ["KotlinJvm13Test.kt"],
 )
 
+kt_rules_e2e_test(
+    name = "KotlinJvmKspAssertionTest",
+    srcs = ["KotlinJvmKspAssertionTest.kt"],
+    data = ["//src/test/data/jvm/ksp"],
+)
+
 test_suite(
     name = "assertion_tests",
     tests = [
         "KotlinJvm13Test",
         "KotlinJvmAssociatesBasicVisibilityTest",
         "KotlinJvmBasicAssertionTest",
         "KotlinJvmKaptAssertionTest",
+        "KotlinJvmKspAssertionTest",
     ],
 )
 

src/test/kotlin/io/bazel/kotlin/KotlinAssertionTestCase.kt

@@ -78,6 +78,15 @@ abstract class KotlinAssertionTestCase(root: String) : BasicAssertionTestCase()
     }
   }
 
+  protected fun JarFile.assertContainsExactEntries(vararg want: String) {
+    val got = this.entries().asSequence().map { it.name }.toSet()
+    val missing = want.toSet() - got
+    val extra = got - want.toSet()
+    check(missing.isEmpty() && extra.isEmpty()) {
+      "Entries Missing: $missing \nFrom: $got \nEntries Extra: $extra \nFrom: $got"
+    }
+  }
+
   /**
    * Validated the entry is compressed and has the DOS epoch for it's timestamp.
    */