From 6547b731d61a7f9311461ae312a744f2170f125a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C3=93lafur=20P=C3=A1ll=20Geirsson?= Date: Mon, 22 Nov 2021 11:44:56 +0100 Subject: [PATCH] Automate Docker container build and fix Java version detection Previously, the Docker release process was manual. Now, the Docker container is built by sbt on CI release and pushed to Docker Hub. Also, the Docker container previously used the default Java version to launch `lsif-java`. This caused problems in cases where the sources of the project would be compiled with a different Java version from the version that's specified in `dump.lsif`. Now, the LSIF version should always match the version that was used to compile the sources. --- .devcontainer/Dockerfile | 11 --- .devcontainer/devcontainer.json | 27 ------- .github/workflows/docker.yml | 19 +++++ auto-indexing/Dockerfile | 7 -- bin/lsif-java-docker-script.sh | 25 ++++++ build.sbt | 76 ++++++++++++++----- .../lsif_java/buildtools/LsifBuildTool.scala | 16 +++- project/plugins.sbt | 3 +- 8 files changed, 118 insertions(+), 66 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/devcontainer.json create mode 100644 .github/workflows/docker.yml delete mode 100644 auto-indexing/Dockerfile create mode 100755 bin/lsif-java-docker-script.sh diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index 5e40a43e..00000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,11 +0,0 @@ -# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.158.0/containers/ubuntu/.devcontainer/base.Dockerfile - -# [Choice] Ubuntu version: bionic, focal -ARG VARIANT="focal" -FROM mcr.microsoft.com/vscode/devcontainers/base:0-${VARIANT} - -# [Optional] Uncomment this section to install additional OS packages. -RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \ - && apt-get -y install --no-install-recommends default-jdk gradle maven - - diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index bd7fb169..00000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,27 +0,0 @@ -// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.158.0/containers/ubuntu -{ - "name": "Ubuntu", - "build": { - "dockerfile": "Dockerfile", - // Update 'VARIANT' to pick an Ubuntu version: focal, bionic - "args": { "VARIANT": "focal" } - }, - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": "/bin/bash" - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - // "postCreateCommand": "uname -a", - - // Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root. - "remoteUser": "vscode" -} \ No newline at end of file diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml new file mode 100644 index 00000000..6e272e96 --- /dev/null +++ b/.github/workflows/docker.yml @@ -0,0 +1,19 @@ +name: Docker +on: + push: + tags: ["*"] +jobs: + publish: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + fetch-depth: 0 + - uses: olafurpg/setup-scala@v13 + - uses: docker/setup-buildx-action@v1 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: ${{ secrets.DOCKER_USERNAME }} + password: ${{ secrets.DOCKER_PASSWORD }} + - run: sbt cli/dockerBuildAndPush diff --git a/auto-indexing/Dockerfile b/auto-indexing/Dockerfile deleted file mode 100644 index 048ceec9..00000000 --- a/auto-indexing/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM gradle:7.2.0-jdk8@sha256:51a695072259173d04e0d6cb590d2590b1c58834c436b6b66e3a6269cdaf1aa4 -RUN apt-get update -RUN apt-get install --yes maven npm -RUN curl -fLo /coursier https://git.io/coursier-cli -RUN chmod +x /coursier -RUN npm install --global yarn -RUN /coursier bootstrap --standalone com.sourcegraph:lsif-java_2.13:0.6.11 -o /usr/local/bin/lsif-java diff --git a/bin/lsif-java-docker-script.sh b/bin/lsif-java-docker-script.sh new file mode 100755 index 00000000..f4da393a --- /dev/null +++ b/bin/lsif-java-docker-script.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Wrapper script for `lsif-java`, which automatically picks up the correct JVM +# version. It assumes that `coursier` is available on the `$PATH` and that the +# `lsif-java` binary is already installed at `/app/lsif-java/bin/lsif-java`. +set -eux +JVM_VERSION="8" +FILE="$PWD/lsif-java.json" +if test -f "$FILE"; then + FROM_CONFIG=$(jq -r '.jvm' "$FILE") + if [ "$FROM_CONFIG" != "null" ]; then + JVM_VERSION="$FROM_CONFIG" + fi +fi +if [ "$JVM_VERSION" = "17" ]; then + JVM_VERSION="temurin:17" +fi + +echo "Using JVM version '$JVM_VERSION'" +eval "$(coursier java --jvm "$JVM_VERSION" --env --jvm-index https://github.com/coursier/jvm-index/blob/master/index.json)" + +if [ "$JVM_VERSION" = "17" ]; then + export JAVA_OPTS="--add-exports jdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED --add-exports jdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED" +fi + +/app/lsif-java/bin/lsif-java "$@" diff --git a/build.sbt b/build.sbt index dfc0e0ba..8c052beb 100644 --- a/build.sbt +++ b/build.sbt @@ -149,6 +149,7 @@ lazy val cli = project version, sbtVersion, scalaVersion, + "javacModuleOptions" -> javacModuleOptions, "semanticdbScalacVersions" -> com .sourcegraph @@ -210,14 +211,50 @@ lazy val cli = project propsFile :: copiedJars.toList } .taskValue, - nativeImageOptions ++= + docker / imageNames := List( - "-H:IncludeResources=^semanticdb-.*jar$", - s"-H:ReflectionConfigurationFiles=${target.value / "native-image-configs" / "reflect-config.json"}" + ImageName("sourcegraph/lsif-java:latest"), + ImageName(s"sourcegraph/lsif-java:${version.value}") ), - nativeImageOutput := (NativeImage / target).value / "lsif-java" + docker / dockerfile := { + val binaryDistribution = pack.value + val script = (ThisBuild / baseDirectory).value / "bin" / + "lsif-java-docker-script.sh" + new Dockerfile { + from("gradle:7.2.0-jdk8") + + // Setup system dependencies. + run("apt-get", "update") + run("apt-get", "install", "--yes", "maven", "jq") + + // Install Coursier. + run( + "curl", + "-fLo", + "/usr/local/bin/coursier", + "https://git.io/coursier-cli" + ) + run("chmod", "+x", "/usr/local/bin/coursier") + + // Pre-download Java 8, 11 and 17. + run("coursier", "java-home", "--jvm", "8") + run("coursier", "java-home", "--jvm", "11") + run( + "coursier", + "java-home", + "--jvm", + "temurin:17", + "--jvm-index", + "https://github.com/coursier/jvm-index/blob/master/index.json" + ) + + // Install `lsif-java` binary. + add(script, "/usr/local/bin/lsif-java") + add(binaryDistribution, "/app/lsif-java") + } + } ) - .enablePlugins(NativeImagePlugin, BuildInfoPlugin) + .enablePlugins(PackPlugin, DockerPlugin, BuildInfoPlugin) .dependsOn(lsif) def commitAll(): Unit = { @@ -276,6 +313,20 @@ lazy val minimized8 = project .dependsOn(agent, plugin) .disablePlugins(JavaFormatterPlugin) +def javacModuleOptions = + List( + "-J--add-exports", + "-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", + "-J--add-exports", + "-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", + "-J--add-exports", + "-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", + "-J--add-exports", + "-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", + "-J--add-exports", + "-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED" + ) + lazy val minimized17 = project .in(file("tests/minimized/.j17")) .settings( @@ -283,19 +334,7 @@ lazy val minimized17 = project javaToolchainJvmIndex := Some("https://github.com/coursier/jvm-index/blob/master/index.json"), javaToolchainVersion := "temurin:17", - javacOptions ++= - List( - "-J--add-exports", - "-Jjdk.compiler/com.sun.tools.javac.api=ALL-UNNAMED", - "-J--add-exports", - "-Jjdk.compiler/com.sun.tools.javac.code=ALL-UNNAMED", - "-J--add-exports", - "-Jjdk.compiler/com.sun.tools.javac.model=ALL-UNNAMED", - "-J--add-exports", - "-Jjdk.compiler/com.sun.tools.javac.tree=ALL-UNNAMED", - "-J--add-exports", - "-Jjdk.compiler/com.sun.tools.javac.util=ALL-UNNAMED" - ) + javacOptions ++= javacModuleOptions ) .dependsOn(agent, plugin) .disablePlugins(JavaFormatterPlugin) @@ -373,6 +412,7 @@ lazy val bench = project lazy val docs = project .in(file("lsif-java-docs")) .settings( + moduleName := "lsif-java-docs", mdocOut := (ThisBuild / baseDirectory).value / "website" / "target" / "docs", fork := false, diff --git a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala index f6b628ce..91d79596 100644 --- a/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala +++ b/lsif-java/src/main/scala/com/sourcegraph/lsif_java/buildtools/LsifBuildTool.scala @@ -494,7 +494,14 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) { index.app.reporter.info(line) }) val javac = Paths.get( - os.proc(coursier.toString, "java-home", "--jvm", config.jvm) + os.proc( + coursier.toString, + "java-home", + "--jvm", + config.jvm, + "--jvm-index", + "https://github.com/coursier/jvm-index/blob/master/index.json" + ) .call() .out .trim(), @@ -502,8 +509,13 @@ class LsifBuildTool(index: IndexCommand) extends BuildTool("LSIF", index) { "javac" ) index.app.reporter.info(s"$$ $javac @$argsfile") + val javacModuleOptions: Seq[String] = + if (config.jvm != "8") + BuildInfo.javacModuleOptions + else + Nil val result = os - .proc(javac.toString, s"@$argsfile") + .proc(javac.toString, s"@$argsfile", javacModuleOptions) .call( stdout = pipe, stderr = pipe, diff --git a/project/plugins.sbt b/project/plugins.sbt index db9f4374..ef4c53ba 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,4 +1,5 @@ -addSbtPlugin("org.scalameta" % "sbt-native-image" % "0.3.2") +addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.14") +addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.8.2") addSbtPlugin("com.geirsson" % "sbt-ci-release" % "1.5.7") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.10.0") addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.4.3")