[feature] generating a custom file with extension imports based on pr…

…oject's external rules | #BAZEL-556 Done just to rerun the build edit changelog, add some aspect documentation Merge remote-tracking branch 'origin/master' into kasia/language-extension-file triple quote strings cr fixes Merge remote-tracking branch 'origin/master' into kasia/language-extension-file # Conflicts: # aspects/core.bzl # aspects/utils/utils.bzl Revert "asdf" This reverts commit fed1d73. asdf moved language enum to top level add a comment to the generated file rename of the file that invokes the query fetching external rules and creating an extension file based on the result rename LanguageExtensionManager to BazelBspEnvironmentManager test reformatting add test cleanup moved extensions to extension file that is replaced on each project reload concurrent builder map grouping by target id and choosing one info per target id + comment pr fixes fixed target info matching better matching of proto files by file name some logging adding '-general' to TargetInfo file name, TargetInfoReader with bugs generating files in extensions slightly formatted code yay tests work!! still need to prettify the code unified interface for extensions Merge-request: BAZEL-MR-442 Merged-by: Katarzyna Mielnik <katarzyna.anna.mielnik@jetbrains.com>
JetBrains · Aug 24, 2023 · a29c1da · a29c1da
1 parent eb5df0a
commit a29c1da
Show file tree

Hide file tree

Showing 12 changed files with 350 additions and 21 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,9 @@
 
 ## [Unreleased]
 
+### Features 🎉
+- The server generates `extensions.bzl` based on languages (external rules) relevant to the project. 
+
 ## [3.0.0] - 09.08.2023
 
 ### BREAKING CHANGES 🚨

diff --git a/aspects/core.bzl b/aspects/core.bzl
@@ -1,20 +1,6 @@
-load("//aspects:rules/python/python_info.bzl", "extract_python_info")
-load("//aspects:rules/kt/kt_info.bzl", "extract_kotlin_info")
-load("//aspects:rules/cpp/cpp_info.bzl", "extract_cpp_info")
-load("//aspects:rules/scala/scala_info.bzl", "extract_scala_info", "extract_scala_toolchain_info")
-load("//aspects:rules/java/java_info.bzl", "JAVA_RUNTIME_TOOLCHAIN_TYPE", "extract_java_info", "extract_java_runtime", "extract_java_toolchain")
 load("//aspects:utils/utils.bzl", "abs", "create_struct", "file_location", "get_aspect_ids", "update_sync_output_groups")
-
-EXTENSIONS = [
-    extract_java_info,
-    extract_kotlin_info,
-    extract_java_toolchain,
-    extract_java_runtime,
-    extract_scala_info,
-    extract_scala_toolchain_info,
-    extract_python_info,
-    extract_cpp_info,
-]
+load("//aspects:rules/java/java_info.bzl", "JAVA_RUNTIME_TOOLCHAIN_TYPE")
+load("//aspects:extensions.bzl", "EXTENSIONS")
 
 def create_all_extension_info(target, ctx, output_groups, dep_targets):
     info = [create_extension_info(target = target, ctx = ctx, output_groups = output_groups, dep_targets = dep_targets) for create_extension_info in EXTENSIONS]

diff --git a/aspects/extensions.bzl b/aspects/extensions.bzl
@@ -0,0 +1,2 @@
+# This file is supposed to be overwritten with extensions relevant to the project.
+EXTENSIONS = []
diff --git a/docs/dev/ASPECTS.md b/docs/dev/ASPECTS.md
@@ -0,0 +1,9 @@
+# Aspects
+
+## Extensions
+
+### The `extensions.bzl` file
+The server generates the `extensions.bzl` file on each sync according to a list of external rules relevant to the project.
+See `server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspEnvironmentManager.kt` for the implementation.
+
+The file itself contains a declaration of functions invoked in `core.bzl`. Each function is supposed to provide language-specific data.
diff --git a/server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BUILD b/server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BUILD
@@ -1,8 +1,11 @@
-load("@rules_java//java:defs.bzl", "java_library")
+load("@io_bazel_rules_kotlin//kotlin:jvm.bzl", "kt_jvm_library")
 
-java_library(
+kt_jvm_library(
     name = "managers",
-    srcs = glob(["*.java"]),
+    srcs = glob([
+        "*.java",
+        "*.kt",
+    ]),
     visibility = ["//server:__subpackages__"],
     deps = [
         "//bazelrunner",

diff --git a/server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspAspectsManager.java b/server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspAspectsManager.java
@@ -13,23 +13,31 @@
 import org.jetbrains.bsp.bazel.server.bsp.utils.InternalAspectsResolver;
 import org.jetbrains.bsp.bazel.workspacecontext.TargetsSpec;
 
+import java.nio.file.Paths;
+import java.util.List;
+
 public class BazelBspAspectsManager {
 
   private final BazelBspCompilationManager bazelBspCompilationManager;
   private final InternalAspectsResolver aspectsResolver;
+  private final BazelBspEnvironmentManager bazelBspEnvironmentManager;
 
   public BazelBspAspectsManager(
       BazelBspCompilationManager bazelBspCompilationManager,
-      InternalAspectsResolver aspectResolver) {
+      InternalAspectsResolver aspectResolver,
+      BazelBspEnvironmentManager bazelBspEnvironmentManager) {
     this.bazelBspCompilationManager = bazelBspCompilationManager;
     this.aspectsResolver = aspectResolver;
+    this.bazelBspEnvironmentManager = bazelBspEnvironmentManager;
   }
 
   public BepOutput fetchFilesFromOutputGroups(
       CancelChecker cancelChecker,
       TargetsSpec targetSpecs,
       String aspect,
       List<String> outputGroups) {
+    bazelBspEnvironmentManager.generateLanguageExtensions(cancelChecker);
+
     if (targetSpecs.getValues().isEmpty()) return new BepOutput(new HashMap<>(), new HashMap<>(), new HashSet<>());
     return
         bazelBspCompilationManager

diff --git a/...r/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspEnvironmentManager.kt b/...r/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelBspEnvironmentManager.kt
@@ -0,0 +1,87 @@
+package org.jetbrains.bsp.bazel.server.bsp.managers
+
+import org.eclipse.lsp4j.jsonrpc.CancelChecker
+import org.jetbrains.bsp.bazel.commons.Constants
+import org.jetbrains.bsp.bazel.server.bsp.utils.InternalAspectsResolver
+import java.nio.file.Paths
+import kotlin.io.path.writeText
+
+enum class Language(private val fileName: String, val functions: List<String>) {
+    Java(
+        "//aspects:rules/java/java_info.bzl",
+        listOf("extract_java_info", "extract_java_toolchain", "extract_java_runtime")
+    ),
+    Scala("//aspects:rules/scala/scala_info.bzl", listOf("extract_scala_info", "extract_scala_toolchain_info")),
+    Cpp("//aspects:rules/cpp/cpp_info.bzl", listOf("extract_cpp_info")),
+    Kotlin("//aspects:rules/kt/kt_info.bzl", listOf("extract_kotlin_info")),
+    Python("//aspects:rules/python/python_info.bzl", listOf("extract_python_info"));
+
+    fun toLoadStatement(): String =
+        this.functions.joinToString(
+            prefix = """load("${this.fileName}", """,
+            separator = ", ",
+            postfix = ")"
+        ) { "\"$it\"" }
+}
+
+class BazelBspEnvironmentManager(
+    private val internalAspectsResolver: InternalAspectsResolver,
+    private val bazelExternalRulesQuery: BazelExternalRulesQuery,
+) {
+
+    fun generateLanguageExtensions(cancelChecker: CancelChecker) {
+        val ruleNames = bazelExternalRulesQuery.fetchExternalRuleNames(cancelChecker)
+        val languages = getProjectLanguages(ruleNames)
+        val fileContent = prepareFileContent(languages)
+
+        createNewExtensionsFile(fileContent)
+    }
+
+    private fun getProjectLanguages(allRuleNames: List<String>): List<Language> {
+        fun checkForLanguage(languageRuleNames: List<String>, language: Language) =
+            checkForLanguageRules(allRuleNames, languageRuleNames, language)
+
+        return listOfNotNull(
+            checkForLanguage(listOf("rules_python"), Language.Python),
+            checkForLanguage(listOf("rules_cc"), Language.Cpp),
+            checkForLanguage(listOf("io_bazel_rules_kotlin"), Language.Kotlin),
+            checkForLanguage(listOf("io_bazel_rules_scala"), Language.Scala),
+            checkForLanguage(listOf("rules_java"), Language.Java)
+        )
+    }
+
+    private fun checkForLanguageRules(
+        allRuleNames: List<String>,
+        languageRuleNames: List<String>,
+        language: Language
+    ): Language? =
+        if (languageRuleNames.any { allRuleNames.contains(it) })
+            language
+        else null
+
+    private fun prepareFileContent(languages: List<Language>) =
+        listOf(
+            "# This is a generated file, do not edit it",
+            createLoadStatementsString(languages),
+            createExtensionListString(languages)
+        ).joinToString(
+            separator = "\n",
+            postfix = "\n"
+        )
+
+    private fun createLoadStatementsString(languages: List<Language>): String {
+        val loadStatements = languages.map { it.toLoadStatement() }
+        return loadStatements.joinToString(postfix = "\n", separator = "\n")
+    }
+
+    private fun createExtensionListString(languages: List<Language>): String {
+        val functionNames = languages.flatMap { it.functions }
+        return functionNames.joinToString(prefix = "EXTENSIONS = [\n", postfix = "\n]", separator = ",\n ") { "\t$it" }
+    }
+
+    private fun createNewExtensionsFile(fileContent: String) {
+        val aspectsPath = Paths.get(internalAspectsResolver.bazelBspRoot, Constants.ASPECTS_ROOT)
+        val file = aspectsPath.resolve("extensions.bzl")
+        file.writeText(fileContent)
+    }
+}
diff --git a/server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelExternalRulesQuery.kt b/server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers/BazelExternalRulesQuery.kt
@@ -0,0 +1,17 @@
+package org.jetbrains.bsp.bazel.server.bsp.managers
+
+import org.eclipse.lsp4j.jsonrpc.CancelChecker
+import org.jetbrains.bsp.bazel.bazelrunner.BazelRunner
+
+interface BazelExternalRulesQuery {
+    fun fetchExternalRuleNames(cancelChecker: CancelChecker): List<String>
+}
+
+class BazelExternalRulesQueryImpl(private val bazelRunner: BazelRunner) : BazelExternalRulesQuery {
+
+    override fun fetchExternalRuleNames(cancelChecker: CancelChecker): List<String> =
+        bazelRunner.commandBuilder().query().withArgument("//external:*").executeBazelCommand()
+            .waitAndGetResult(cancelChecker, ensureAllOutputRead = true)
+            .stdoutLines
+            .mapNotNull { it.split(':').getOrNull(1) }
+}
diff --git a/server/src/main/java/org/jetbrains/bsp/bazel/server/common/ServerContainer.kt b/server/src/main/java/org/jetbrains/bsp/bazel/server/common/ServerContainer.kt
@@ -10,6 +10,8 @@ import org.jetbrains.bsp.bazel.server.sync.MetricsLogger
 import org.jetbrains.bsp.bazel.server.bsp.info.BspInfo
 import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspAspectsManager
 import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspCompilationManager
+import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspEnvironmentManager
+import org.jetbrains.bsp.bazel.server.bsp.managers.BazelExternalRulesQueryImpl
 import org.jetbrains.bsp.bazel.server.bsp.utils.InternalAspectsResolver
 import org.jetbrains.bsp.bazel.server.sync.*
 import org.jetbrains.bsp.bazel.server.sync.languages.LanguagePluginsService
@@ -53,7 +55,8 @@ class ServerContainer internal constructor(
             val bazelInfo = bazelDataResolver.resolveBazelInfo { }
 
             val aspectsResolver = InternalAspectsResolver(bspInfo)
-            val bazelBspAspectsManager = BazelBspAspectsManager(compilationManager, aspectsResolver)
+            val bazelBspEnvironmentManager = BazelBspEnvironmentManager(aspectsResolver, BazelExternalRulesQueryImpl(bazelRunner))
+            val bazelBspAspectsManager = BazelBspAspectsManager(compilationManager, aspectsResolver, bazelBspEnvironmentManager)
             val bazelPathsResolver = BazelPathsResolver(bazelInfo)
             val jdkResolver = JdkResolver(bazelPathsResolver, JdkVersionResolver())
             val javaLanguagePlugin = JavaLanguagePlugin(bazelPathsResolver, jdkResolver, bazelInfo)

diff --git a/server/src/main/java/org/jetbrains/bsp/bazel/server/sync/ProjectResolver.kt b/server/src/main/java/org/jetbrains/bsp/bazel/server/sync/ProjectResolver.kt
@@ -6,6 +6,7 @@ import org.jetbrains.bsp.bazel.commons.Stopwatch
 import org.jetbrains.bsp.bazel.logger.BspClientLogger
 import org.jetbrains.bsp.bazel.server.bep.BepOutput
 import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspAspectsManager
+import org.jetbrains.bsp.bazel.server.bsp.managers.BazelBspEnvironmentManager
 import org.jetbrains.bsp.bazel.server.sync.model.Project
 import org.jetbrains.bsp.bazel.workspacecontext.WorkspaceContext
 import org.jetbrains.bsp.bazel.workspacecontext.WorkspaceContextProvider

diff --git a/server/src/test/java/org/jetbrains/bsp/bazel/server/bsp/managers/BUILD b/server/src/test/java/org/jetbrains/bsp/bazel/server/bsp/managers/BUILD
@@ -0,0 +1,12 @@
+load("@//rules/kotlin:junit5.bzl", "kt_test")
+
+kt_test(
+    name = "BazelBspEnvironmentManagerTest",
+    size = "small",
+    src = "BazelBspEnvironmentManagerTest.kt",
+    deps = [
+        "//commons",
+        "//server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/managers",
+        "@maven//:org_eclipse_lsp4j_org_eclipse_lsp4j_jsonrpc",
+    ],
+)