Remove coursier bootstrap and respect COURSIER_JVM_INDEX (#631)

build.sbt

@@ -250,6 +250,7 @@ lazy val cli = project
     libraryDependencies ++=
       List(
         "io.get-coursier" %% "coursier" % V.coursier,
+        "io.get-coursier" %% "coursier-jvm" % V.coursier,
         "org.scalameta" % "mtags-interfaces" % V.metals,
         "org.scala-lang.modules" %% "scala-xml" % V.scalaXml,
         "com.lihaoyi" %% "requests" % V.requests,

scip-java/src/main/scala/com/sourcegraph/scip_java/Embedded.scala

@@ -18,11 +18,6 @@ object Embedded {
     copyFile(tmpDir, "gradle-plugin.jar")
 
   def agentJar(tmpDir: Path): Path = copyFile(tmpDir, "semanticdb-agent.jar")
-  def coursier(tmpDir: Path): Path = {
-    val result = copyFile(tmpDir, "scip-java/coursier")
-    result.toFile().setExecutable(true)
-    result
-  }
 
   private def javacErrorpath(tmp: Path) = tmp.resolve("errorpath.txt")
 

scip-java/src/main/scala/com/sourcegraph/scip_java/buildtools/ScipBuildTool.scala

@@ -22,6 +22,9 @@ import java.util.jar.JarFile
 
 import scala.collection.mutable.ArrayBuffer
 import scala.collection.mutable.ListBuffer
+import scala.concurrent.Await
+import scala.concurrent.ExecutionContext
+import scala.concurrent.duration.Duration
 import scala.jdk.CollectionConverters._
 import scala.language.postfixOps
 import scala.util.Failure
@@ -40,6 +43,7 @@ import com.sourcegraph.scip_java.Embedded
 import com.sourcegraph.scip_java.commands.IndexCommand
 import com.sourcegraph.semanticdb_javac.Semanticdb.TextDocument
 import com.sourcegraph.semanticdb_javac.Semanticdb.TextDocuments
+import coursier.jvm.JvmIndex
 import moped.json.DecodingContext
 import moped.json.ErrorResult
 import moped.json.JsonCodec
@@ -606,30 +610,62 @@ class ScipBuildTool(index: IndexCommand) extends BuildTool("SCIP", index) {
   }
 
   private def javacPathViaCoursier(jvmVersion: String, tmp: Path): Path = {
-    val coursier = Embedded.coursier(tmp)
-    Paths.get(
-      os.proc(
-          coursier.toString,
-          "java-home",
-          "--jvm",
-          jvmVersion,
-          "--architecture",
-          jvmArchitecture
+    import _root_.coursier.jvm._
+
+    val jvmChannel = index
+      .app
+      .env
+      .environmentVariables
+      .get("COURSIER_JVM_INDEX")
+      .map { idx =>
+        JvmChannel
+          .parse(idx)
+          .fold(
+            msg =>
+              throw new RuntimeException(
+                s"Malformed COURSIER_JVM_INDEX environment variable variable: $msg"
+              ),
+            identity
+          )
+      }
+
+    val home = JavaHome().withCache(
+      JvmCache()
+        .withIndexChannel(
+          repositories = Seq.empty,
+          indexChannel = jvmChannel
+            .getOrElse(JvmChannel.url(JvmIndex.defaultIndexUrl))
         )
-        .call()
-        .out
-        .trim(),
-      "bin",
-      "javac"
+        .withArchitecture(jvmArchitecture(jvmVersion))
     )
 
+    val javaExecutable = Await.result(
+      home.javaBin(jvmVersion).value(ExecutionContext.global),
+      Duration.Inf
+    )
+
+    javaExecutable
+      .getParent()
+      .resolve {
+        if (scala.util.Properties.isWin)
+          "javac.exe"
+        else
+          "javac"
+      }
+
   }
 
-  private def jvmArchitecture: String =
-    if (scala.util.Properties.isMac && sys.props("os.arch") == "aarch64")
+  private def jvmArchitecture(jvm: String): String =
+    if (
+      // Hack only for local development on ARM64 Mac OS X
+      // won't affect any other system
+      scala.util.Properties.isMac && sys.props("os.arch") == "aarch64" &&
+      (jvm.startsWith("8") || jvm.startsWith("1.8"))
+    )
       "amd64"
     else
-      defaultCoursierJVMArchitecture
+      JvmIndex.defaultArchitecture()
+
   def defaultCoursierJVMArchitecture: String =
     sys.props("os.arch") match {
       case "x86_64" =>

tests/buildTools/src/test/scala/tests/ScipBuildToolSuite.scala

@@ -1,45 +1,76 @@
 package tests
 
 import com.sourcegraph.scip_java.{BuildInfo => V}
+import moped.testkit.FileLayout
 
 class ScipBuildToolSuite extends BaseBuildToolSuite {
   override def tags = List(SkipWindows)
 
-  test("COURSIER_CREDENTIALS and COURSIER_REPOSITORIES are respected") {
+  test("respect-coursier-jvm-index") {
 
-    val cli = sys.env.getOrElse("SCIP_JAVA_CLI", fail("wwaaaa"))
+    val (requests, _) = TracingServer.run { run =>
+      val env = Map(
+        "COURSIER_JVM_INDEX" -> s"${run.address.toString}/index.json"
+      )
+
+      val tmp = FileLayout
+        .fromString("""
+                      |/lsif-java.json
+                      |{"dependencies": ["junit:junit:4.13.1"],"jvm": "17"}
+                      |/src/main/java/Example.java
+                      |package foo;\npublic class Example{}
+        """.stripMargin)
+
+      val result = os
+        .proc(scipJavaCli(), "index", "--build-tool=scip")
+        .call(cwd = os.Path(tmp), env = env, check = false)
+
+      assertNotEquals(result.exitCode, 0)
+    }
+
+    assert(
+      requests.nonEmpty,
+      "No requests were sent to the local server - suggesting that COURSIER_JVM_INDEX is not respected by ScipBuildTool"
+    )
+
+    assert(
+      clue(requests)
+        .collectFirst {
+          case req if req.url.getPath() == "/index.json" =>
+            req
+        }
+        .nonEmpty,
+      "No requests were sent to the local server - suggesting that COURSIER_JVM_INDEX is not respected by ScipBuildTool"
+    )
+  }
+
+  test("respect-coursier-credentials-and-repositories") {
 
     val Username = "hello"
     val Password = "world"
 
-    val (requests, _) = PasswordProtectedServer(Username, Password).runWith {
-      run =>
+    val (requests, _) =
+      TracingServer.runWithAuth(Username, Password) { run =>
         val env = Map(
           "COURSIER_REPOSITORIES" -> run.address.toString(),
           "COURSIER_CREDENTIALS" -> s"localhost $Username:$Password"
         )
 
-        val tmp = os.temp.dir(prefix = "scip-java")
-        os.write(
-          tmp / "lsif-java.json",
-          // We use non-existent library to make sure caches are never used
-          s"""   {"dependencies": ["bla.bla.nonexistent-library:junit:4.13.1"]}  """
-            .trim
-        )
-        os.write(
-          tmp / "foo" / "Example.java",
-          "package foo;\npublic class Example{}",
-          createFolders = true
+        val tmp = FileLayout.fromString(
+          """
+            |/lsif-java.json
+            |{"dependencies": ["bla.bla.nonexistent-library:junit:4.13.1"]}
+            |/src/main/java/Example.java
+            |package foo;\npublic class Example{}
+        """.stripMargin
         )
 
         val result = os
-          .proc(cli, "index", "--build-tool=scip")
-          .call(cwd = tmp, env = env, check = false)
-
-        os.remove.all(tmp)
+          .proc(scipJavaCli(), "index", "--build-tool=scip")
+          .call(cwd = os.Path(tmp), env = env, check = false)
 
         assertNotEquals(result.exitCode, 0)
-    }
+      }
 
     assert(
       requests.nonEmpty,
@@ -69,6 +100,16 @@ class ScipBuildToolSuite extends BaseBuildToolSuite {
   private def base64(str: String) =
     new String(java.util.Base64.getEncoder().encode(str.getBytes))
 
+  private def scipJavaCli() =
+    sys
+      .env
+      .getOrElse(
+        "SCIP_JAVA_CLI",
+        fail(
+          "SCIP_JAVA_CLI env variable is not set, perhaps the build is misconfigured"
+        )
+      )
+
   checkBuild(
     "basic",
     """|/lsif-java.json

tests/buildTools/src/test/scala/tests/PasswordProtectedServer.scala → tests/buildTools/src/test/scala/tests/TracingServer.scala

@@ -7,50 +7,17 @@ import scala.jdk.CollectionConverters._
 
 import com.sun.net.httpserver._
 
-case class PasswordProtectedServer(user: String, pwd: String) {
-  import PasswordProtectedServer._
-  private class MyHandler(storage: ListBuffer[SimpleHttpRequest])
-      extends HttpHandler {
-    val auth =
-      new BasicAuthenticator("") {
-        override def checkCredentials(
-            username: String,
-            password: String
-        ): Boolean = username == user && password == pwd
-      }
-    override def handle(t: HttpExchange): Unit = {
-      val headers = t
-        .getRequestHeaders()
-        .asScala
-        .toMap
-        .flatMap { case (k, v) =>
-          v.asScala.headOption.map(k -> _)
-        }
-
-      val url = t.getRequestURI()
-      storage.synchronized {
-        storage.addOne(SimpleHttpRequest(url, headers))
-      }
+object TracingServer {
+  case class SimpleHttpRequest(url: URI, simpleHeaders: Map[String, String])
 
-      auth.authenticate(t) match {
-        case f: Authenticator.Failure =>
-          t.sendResponseHeaders(f.getResponseCode(), 0L)
-        case r: Authenticator.Retry =>
-          t.sendResponseHeaders(r.getResponseCode(), 0L)
-        case r: Authenticator.Success =>
-          t.sendResponseHeaders(404, 0L)
-      }
+  case class RunningServer(address: URI, shutdown: () => Unit)
 
-      t.close()
-    }
-  }
-  def runWith[A](f: RunningServer => A): (List[SimpleHttpRequest], A) = {
-    val storage = ListBuffer.empty[SimpleHttpRequest]
+  private def runImpl[A](handler: MyHandler)(f: RunningServer => A) = {
     val server = HttpServer
       .create(new java.net.InetSocketAddress("localhost", 0), 0);
     val result =
       try {
-        server.createContext("/", new MyHandler(storage))
+        server.createContext("/", handler)
         server.setExecutor(null) // creates a default executor
         server.start()
         f(
@@ -65,12 +32,60 @@ case class PasswordProtectedServer(user: String, pwd: String) {
         server.stop(0)
       }
 
-    storage.toList -> result
+    handler.tracedRequests -> result
   }
-}
 
-object PasswordProtectedServer {
-  case class SimpleHttpRequest(url: URI, simpleHeaders: Map[String, String])
+  def run[A](f: RunningServer => A): (List[SimpleHttpRequest], A) = {
+    runImpl(new MyHandler(auth = None))(f)
+  }
+  def runWithAuth[A](username: String, password: String)(
+      f: RunningServer => A
+  ): (List[SimpleHttpRequest], A) = {
+    val authenticator =
+      new BasicAuthenticator("") {
+        override def checkCredentials(
+            _username: String,
+            _password: String
+        ): Boolean = username == _username && _password == password
+      }
 
-  case class RunningServer(address: URI, shutdown: () => Unit)
+    runImpl(new MyHandler(auth = Some(authenticator)))(f)
+  }
+
+  private class MyHandler(auth: Option[Authenticator] = None)
+      extends HttpHandler {
+
+    val storage: ListBuffer[SimpleHttpRequest] = ListBuffer.empty
+
+    override def handle(t: HttpExchange): Unit = {
+      val headers = t
+        .getRequestHeaders()
+        .asScala
+        .toMap
+        .flatMap { case (k, v) =>
+          v.asScala.headOption.map(k -> _)
+        }
+
+      val url = t.getRequestURI()
+      storage.synchronized {
+        storage.addOne(SimpleHttpRequest(url, headers))
+      }
+
+      auth.foreach {
+        _.authenticate(t) match {
+          case f: Authenticator.Failure =>
+            t.sendResponseHeaders(f.getResponseCode(), 0L)
+          case r: Authenticator.Retry =>
+            t.sendResponseHeaders(r.getResponseCode(), 0L)
+          case r: Authenticator.Success =>
+            t.sendResponseHeaders(404, 0L)
+        }
+      }
+
+      t.close()
+    }
+
+    def tracedRequests = storage.result
+
+  }
 }