[feature] python support (#323)

* e2e tests

* extracting python info

* python extension points

* language mapping

* aspects and proto mapping update

* fix variable type in aspects

* sha256 fix

* hardcoded interpreter wrapper path

* fix imports

* update base python project to expect workspace target

* python-library-project related test failing check why

* minor update added deps doesnt work still

* updated to only have one test but bigger, missing dataKind in lib

* add new fields to TargetInfo

* made BazelBspPythonProjectTest pass

* remove second python test, rely on one but extended

* revert addition of `main` and `imports` fields

* extended python test to check py_test rule

* changed interpreter path to URI, renamed library in tests

* Revert "Merge remote-tracking branch 'origin/python-lib-test' into python-lib-test"

This reverts commit c7d9377, reversing
changes made to 9da9d66.

* Revert "Merge remote-tracking branch 'origin/python-lib-test' into python-lib-test"

This reverts commit c7d9377, reversing
changes made to 8090415.

* changed interpreter path to URI, renamed library in tests - this time without messing previous commit

* changed interpreter path to URI, renamed library in tests - this time without messing previous commit

* fix missing test files that got somehow removed

* fix interpreter path in test and move workspaceroot build target up

* fix formatter errors

* add python plugin to service init in test

* Apply suggestions from code review

Co-authored-by: Marcin Abramowicz <44381959+abrams27@users.noreply.github.com>

* remove unnecessary comments

* remove bazel version specification in python project

* update python test

* add external dependency in python project

* set python interpreter and external dep in python example project

* implemented dependencySources ep for python

* style fixes

* style fix in python test

* bump testkit version, test pythonOptions

* remove resources expected build target

* apply pr review comments

* minor hotfix

* set default python sdk && add test of resources

* set dependency folder to site-packages for python

* remove accidentally added .iml files

* pr comment fixes

* small style fixes

* lil style fix

Co-authored-by: Marcin Abramowicz <44381959+abrams27@users.noreply.github.com>

* remove '}'

* buildifier run

* changelog update

---------

Co-authored-by: Wiktor Grzankowski <w.grzankowski@gmail.com>
Co-authored-by: Filip Głębocki <fg429202@students.mimuw.edu.pl>
Co-authored-by: WiktorGrzankowski <85792655+WiktorGrzankowski@users.noreply.github.com>
Co-authored-by: Marcin Abramowicz <44381959+abrams27@users.noreply.github.com>

CHANGELOG.md

@@ -6,6 +6,7 @@
 ## [Unreleased]
 ### Features 🎉
 - Experimental `workspace/libraries` endpoint that returns list of external libraries
+- Support for Python targets, including `buildTarget/pythonOptions` endpoint
 
 ### BREAKING CHANGES 🚨
 

WORKSPACE

@@ -159,7 +159,7 @@ sonatype_dependencies()
 
 git_repository(
     name = "testkit",
-    commit = "36bb3467d3df351780d0092dadfd9f3313787e8d",
+    commit = "9743061ce45653bdc46e9a67303a76c1d4455bbb",
     patch_args = ["-p1"],
     patches = ["//e2e:testkit.patch"],
     remote = "https://github.com/agluszak/bsp-testkit2.git",

e2e/BUILD

@@ -55,6 +55,15 @@ sh_binary(
     ],
 )
 
+sh_binary(
+    name = "BazelBspPythonProjectTest",
+    srcs = ["runTest.sh"],
+    args = [
+        "//e2e/src/main/java/org/jetbrains/bsp/bazel:BazelBspPythonProjectTest",
+        "e2e/test-resources/python-project",
+    ],
+)
+
 sh_binary(
     name = "BazelBspEntireRepositoryImportTest",
     srcs = ["runTest.sh"],

e2e/runAll.sh

@@ -17,6 +17,7 @@ runTest //e2e:BazelBspSampleRepoTest
 runTest //e2e:BazelBspLocalJdkTest
 runTest //e2e:BazelBspRemoteJdkTest
 #runTest //e2e:BazelBspCppProjectTest
+runTest //e2e:BazelBspPythonProjectTest
 
 echo -e "${GREEN}==================================="
 echo -e "==================================="

e2e/src/main/java/org/jetbrains/bsp/bazel/BUILD

@@ -51,3 +51,16 @@ kt_jvm_binary(
         "@maven//:ch_epfl_scala_bsp4j",
     ],
 )
+
+kt_jvm_binary(
+    name = "BazelBspPythonProjectTest",
+    srcs = ["BazelBspPythonProjectTest.kt"],
+    main_class = "org.jetbrains.bsp.bazel.BazelBspPythonProjectTest",
+    resources = ["//e2e/src/main/resources:bsp-e2e-resources"],
+    visibility = ["//e2e:__subpackages__"],
+    deps = [
+        "//commons",
+        "//e2e/src/main/java/org/jetbrains/bsp/bazel/base",
+        "@maven//:ch_epfl_scala_bsp4j",
+    ],
+)

e2e/src/main/java/org/jetbrains/bsp/bazel/BazelBspPythonProjectTest.kt

@@ -0,0 +1,166 @@
+package org.jetbrains.bsp.bazel
+
+import ch.epfl.scala.bsp4j.BuildTarget
+import ch.epfl.scala.bsp4j.BuildTargetCapabilities
+import ch.epfl.scala.bsp4j.BuildTargetIdentifier
+import ch.epfl.scala.bsp4j.WorkspaceBuildTargetsResult
+import ch.epfl.scala.bsp4j.PythonBuildTarget
+import ch.epfl.scala.bsp4j.BuildTargetDataKind
+import ch.epfl.scala.bsp4j.PythonOptionsItem
+import ch.epfl.scala.bsp4j.PythonOptionsParams
+import ch.epfl.scala.bsp4j.PythonOptionsResult
+import ch.epfl.scala.bsp4j.DependencySourcesItem
+import ch.epfl.scala.bsp4j.DependencySourcesParams
+import ch.epfl.scala.bsp4j.DependencySourcesResult
+import ch.epfl.scala.bsp4j.ResourcesItem
+import ch.epfl.scala.bsp4j.ResourcesParams
+import ch.epfl.scala.bsp4j.ResourcesResult
+import org.jetbrains.bsp.bazel.base.BazelBspTestBaseScenario
+import org.jetbrains.bsp.bazel.base.BazelBspTestScenarioStep
+
+import java.time.Duration
+
+object BazelBspPythonProjectTest : BazelBspTestBaseScenario() {
+
+    @JvmStatic
+    fun main(args: Array<String>) = executeScenario()
+
+    override fun repoName(): String = "python-project"
+
+    override fun scenarioSteps(): List<BazelBspTestScenarioStep> = listOf(
+        workspaceBuildTargets(),
+        dependencySourcesResults(),
+        pythonOptionsResults(),
+        resourcesResults()
+    )
+
+    private fun expectedWorkspaceBuildTargetsResult(): WorkspaceBuildTargetsResult {
+        val bspWorkspaceRootExampleBuildTarget =
+            BuildTarget(
+                BuildTargetIdentifier("bsp-workspace-root"),
+                listOf(),
+                listOf(),
+                listOf(),
+                BuildTargetCapabilities(false, false, false, false)
+            )
+        bspWorkspaceRootExampleBuildTarget.baseDirectory = "file://\$WORKSPACE/"
+        bspWorkspaceRootExampleBuildTarget.displayName = "bsp-workspace-root"
+
+        val examplePythonBuildTarget =
+            PythonBuildTarget(
+                "PY3",
+                "file://\$BAZEL_CACHE/external/python3_9_x86_64-unknown-linux-gnu/bin/python3",
+            )
+
+        val exampleExampleBuildTarget = BuildTarget(
+            BuildTargetIdentifier("$targetPrefix//example:example"),
+            listOf("application"),
+            listOf("python"),
+            listOf(
+                BuildTargetIdentifier("$targetPrefix//lib:example_library"),
+                BuildTargetIdentifier("@requests//:srcs")
+            ),
+            BuildTargetCapabilities(true, false, true, false)
+        )
+        exampleExampleBuildTarget.displayName = "$targetPrefix//example:example"
+        exampleExampleBuildTarget.baseDirectory = "file://\$WORKSPACE/example/"
+        exampleExampleBuildTarget.data = examplePythonBuildTarget
+        exampleExampleBuildTarget.dataKind = BuildTargetDataKind.PYTHON
+
+        val exampleExampleLibBuildTarget = BuildTarget(
+            BuildTargetIdentifier("$targetPrefix//lib:example_library"),
+            listOf("library"),
+            listOf("python"),
+            listOf(BuildTargetIdentifier("@pip_deps_numpy//:pkg")),
+            BuildTargetCapabilities(true, false, false, false)
+        )
+        exampleExampleLibBuildTarget.displayName = "$targetPrefix//lib:example_library"
+        exampleExampleLibBuildTarget.baseDirectory = "file://\$WORKSPACE/lib/"
+        exampleExampleLibBuildTarget.data = examplePythonBuildTarget
+        exampleExampleLibBuildTarget.dataKind = BuildTargetDataKind.PYTHON
+
+        val exampleExampleTestBuildTarget = BuildTarget(
+            BuildTargetIdentifier("$targetPrefix//test:test"),
+            listOf("test"),
+            listOf("python"),
+            listOf(),
+            BuildTargetCapabilities(true, true, false, false)
+        )
+        exampleExampleTestBuildTarget.displayName = "$targetPrefix//test:test"
+        exampleExampleTestBuildTarget.baseDirectory = "file://\$WORKSPACE/test/"
+        exampleExampleTestBuildTarget.data = examplePythonBuildTarget
+        exampleExampleTestBuildTarget.dataKind = BuildTargetDataKind.PYTHON
+
+        return WorkspaceBuildTargetsResult(
+            listOf(
+                bspWorkspaceRootExampleBuildTarget,
+                exampleExampleBuildTarget,
+                exampleExampleLibBuildTarget,
+                exampleExampleTestBuildTarget,
+            )
+        )
+    }
+
+    private fun workspaceBuildTargets(): BazelBspTestScenarioStep {
+        val workspaceBuildTargetsResult = expectedWorkspaceBuildTargetsResult()
+
+        return BazelBspTestScenarioStep("workspace build targets") {
+            testClient.testWorkspaceTargets(
+                Duration.ofSeconds(60),
+                workspaceBuildTargetsResult
+            )
+        }
+    }
+
+    private fun dependencySourcesResults(): BazelBspTestScenarioStep {
+        val expectedPythonDependencySourcesItems = expectedWorkspaceBuildTargetsResult().targets.map {
+            if (it.id == BuildTargetIdentifier("$targetPrefix//lib:example_library"))
+                DependencySourcesItem(it.id, listOf("file://\$BAZEL_CACHE/external/pip_deps_numpy/site-packages/"))
+            else
+                DependencySourcesItem(it.id, emptyList())
+        }
+
+        val expectedTargetIdentifiers = expectedTargetIdentifiers()
+        val expectedDependencies = DependencySourcesResult(expectedPythonDependencySourcesItems)
+        val dependencySourcesParams = DependencySourcesParams(expectedTargetIdentifiers)
+
+        return BazelBspTestScenarioStep(
+            "dependency sources results"
+        ) {
+            testClient.testDependencySources(
+                Duration.ofSeconds(30), dependencySourcesParams, expectedDependencies
+            )
+        }
+    }
+
+    private fun pythonOptionsResults(): BazelBspTestScenarioStep {
+        val expectedTargetIdentifiers = expectedTargetIdentifiers().filter { it.uri != "bsp-workspace-root" }
+        val expectedPythonOptionsItems = expectedTargetIdentifiers.map { PythonOptionsItem(it, emptyList()) }
+        val expectedPythonOptionsResult = PythonOptionsResult(expectedPythonOptionsItems)
+        val pythonOptionsParams = PythonOptionsParams(expectedTargetIdentifiers)
+
+        return BazelBspTestScenarioStep(
+            "pythonOptions results"
+        ) {
+            testClient.testPythonOptions(Duration.ofSeconds(30), pythonOptionsParams, expectedPythonOptionsResult)
+        }
+    }
+
+    private fun resourcesResults(): BazelBspTestScenarioStep {
+        val expectedTargetIdentifiers = expectedTargetIdentifiers().filter { it.uri != "bsp-workspace-root" }
+        val expectedResourcesItems = expectedTargetIdentifiers.map { ResourcesItem(it, emptyList()) }
+        val expectedResourcesResult = ResourcesResult(expectedResourcesItems)
+        val resourcesParams = ResourcesParams(expectedTargetIdentifiers)
+
+        return BazelBspTestScenarioStep(
+            "resources results"
+        ) {
+            testClient.testResources(Duration.ofSeconds(30), resourcesParams, expectedResourcesResult)
+        }
+    }
+
+    private fun expectedTargetIdentifiers(): List<BuildTargetIdentifier> =
+        expectedWorkspaceBuildTargetsResult()
+            .targets
+            .map { it.id }
+}

e2e/test-resources/python-project/.gitignore

@@ -0,0 +1,5 @@
+.bazelbsp/
+.bsp/
+.idea/
+.ijwb/
+bazel-*

e2e/test-resources/python-project/WORKSPACE

@@ -0,0 +1,41 @@
+workspace(name = "python_test")
+
+load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive")
+
+http_archive(
+    name = "rules_python",
+    sha256 = "8c8fe44ef0a9afc256d1e75ad5f448bb59b81aba149b8958f02f7b3a98f5d9b4",
+    strip_prefix = "rules_python-0.13.0",
+    url = "https://github.com/bazelbuild/rules_python/archive/refs/tags/0.13.0.tar.gz",
+)
+
+load("@rules_python//python:repositories.bzl", "python_register_toolchains")
+
+python_register_toolchains(
+    name = "python3_9",
+    python_version = "3.9",
+)
+
+load("@python3_9//:defs.bzl", "interpreter")
+load("@rules_python//python:pip.bzl", "pip_parse")
+
+pip_parse(
+    name = "pip_deps",
+    python_interpreter_target = interpreter,
+    requirements_lock = "//lib:requirements.txt",
+)
+
+load("@pip_deps//:requirements.bzl", "install_deps")
+
+install_deps()
+
+http_archive(
+    name = "requests",
+    build_file_content = """
+py_library(
+    name = "srcs",
+    srcs = glob(["requests/*.py"]),
+    visibility = ["//visibility:public"]
+)""",
+    urls = ["https://github.com/requests/requests/tarball/master/psf-requests-7f694b7.tar.gz"],
+)

e2e/test-resources/python-project/example/BUILD

@@ -0,0 +1,11 @@
+load("@rules_python//python:defs.bzl", "py_binary")
+
+py_binary(
+    name = "example",
+    srcs = ["example.py"],
+    python_version = "PY3",
+    deps = [
+        "//lib:example_library",
+        "@requests//:srcs",
+    ],
+)

e2e/test-resources/python-project/example/example.py

@@ -0,0 +1,3 @@
+from lib.example_lib import *
+import requests
+print("Hello, this is a test!")

e2e/test-resources/python-project/lib/BUILD

@@ -0,0 +1,11 @@
+load("@rules_python//python:defs.bzl", "py_library")
+load("@pip_deps//:requirements.bzl", "requirement")
+
+py_library(
+    name = "example_library",
+    srcs = ["example_lib.py"],
+    visibility = ["//visibility:public"],
+    deps = [
+        requirement("numpy"),
+    ],
+)

e2e/test-resources/python-project/lib/example_lib.py

@@ -0,0 +1,5 @@
+import numpy as np
+
+def add(a, b):
+    a = np.array([2, 3, 4])
+    return np.sum(a)

e2e/test-resources/python-project/lib/requirements.txt

@@ -0,0 +1 @@
+numpy==1.23.3

e2e/test-resources/python-project/test/BUILD

@@ -0,0 +1,7 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+py_test(
+    name = "test",
+    srcs = ["test.py"],
+    visibility = ["//visibility:public"],
+)

e2e/test-resources/python-project/test/test.py

@@ -0,0 +1 @@
+print('I am a test file')

install/src/main/resources/aspects.bzl

@@ -374,6 +374,20 @@ def extract_cpp_target_info(target, ctx):
         link_shared = getattr(ctx.rule.attr, "linkshared", False),
     )
 
+def extract_python_target_info(target, ctx):
+    if PyInfo not in target:
+        return None
+
+    if PyRuntimeInfo in target:
+        provider = target[PyRuntimeInfo]
+    else:
+        provider = None
+
+    return create_struct(
+        interpreter = file_location(getattr(provider, "interpreter", None)),
+        version = getattr(provider, "python_version", None),
+    )
+
 def get_aspect_ids(ctx, target):
     """Returns the all aspect ids, filtering out self."""
     aspect_ids = None
@@ -533,6 +547,7 @@ def _bsp_target_info_aspect_impl(target, ctx):
     java_runtime_info, java_runtime_info_exported = extract_java_runtime(target, ctx, dep_targets)
     cpp_target_info = extract_cpp_target_info(target, ctx)
     kotlin_target_info = extract_kotlin_info(target, ctx)
+    python_target_info = extract_python_target_info(target, ctx)
 
     result = dict(
         id = str(target.label),
@@ -548,6 +563,7 @@ def _bsp_target_info_aspect_impl(target, ctx):
         java_runtime_info = java_runtime_info,
         cpp_target_info = cpp_target_info,
         kotlin_target_info = kotlin_target_info,
+        python_target_info = python_target_info,
         env = getattr(rule_attrs, "env", {}),
         env_inherit = getattr(rule_attrs, "env_inherit", []),
     )

server/src/main/java/org/jetbrains/bsp/bazel/server/bsp/BspServerApi.java

@@ -29,6 +29,9 @@
 import ch.epfl.scala.bsp4j.JvmTestEnvironmentResult;
 import ch.epfl.scala.bsp4j.OutputPathsParams;
 import ch.epfl.scala.bsp4j.OutputPathsResult;
+import ch.epfl.scala.bsp4j.PythonBuildServer;
+import ch.epfl.scala.bsp4j.PythonOptionsParams;
+import ch.epfl.scala.bsp4j.PythonOptionsResult;
 import ch.epfl.scala.bsp4j.ResourcesParams;
 import ch.epfl.scala.bsp4j.ResourcesResult;
 import ch.epfl.scala.bsp4j.RunParams;
@@ -59,7 +62,8 @@ public class BspServerApi
         ScalaBuildServer,
         JavaBuildServer,
         CppBuildServer,
-        BazelBuildServer {
+        BazelBuildServer,
+        PythonBuildServer {
 
   private final Supplier<BazelServices> bazelServicesBuilder;
   private BazelBspServerLifetime serverLifetime = null;
@@ -223,6 +227,13 @@ public CompletableFuture<CppOptionsResult> buildTargetCppOptions(CppOptionsParam
         "buildTargetCppOptions", projectSyncService::buildTargetCppOptions, params);
   }
 
+  @Override
+  public CompletableFuture<PythonOptionsResult> buildTargetPythonOptions(
+      PythonOptionsParams params) {
+    return runner.handleRequest(
+        "buildTargetPythonOptions", projectSyncService::buildTargetPythonOptions, params);
+  }
+
   @Override
   public CompletableFuture<JvmRunEnvironmentResult> jvmRunEnvironment(
       JvmRunEnvironmentParams params) {