From 28995e1bd7cc69657778ed34c86a3b384a158a78 Mon Sep 17 00:00:00 2001 From: Jamie Thompson Date: Wed, 15 May 2024 19:43:06 +0200 Subject: [PATCH] download native launchers and overwrite scripts TODO: add overrides for windows --- build.sbt | 5 + dist/bin-native-overrides/cli-common-platform | 3 + dist/bin/cli-common | 56 +------- dist/bin/cli-common-platform | 5 + dist/bin/common | 39 +----- dist/bin/common-java | 40 ++++++ dist/bin/scala | 5 +- project/Build.scala | 61 +++++++-- project/RepublishPlugin.scala | 122 ++++++++++++++++-- 9 files changed, 217 insertions(+), 119 deletions(-) create mode 100644 dist/bin-native-overrides/cli-common-platform create mode 100644 dist/bin/cli-common-platform create mode 100644 dist/bin/common-java diff --git a/build.sbt b/build.sbt index 1bc74e5e23fb..705ef0ba0eb7 100644 --- a/build.sbt +++ b/build.sbt @@ -28,6 +28,11 @@ val `scaladoc-js-main` = Build.`scaladoc-js-main` val `scaladoc-js-contributors` = Build.`scaladoc-js-contributors` val `scala3-bench-run` = Build.`scala3-bench-run` val dist = Build.dist +val `dist-mac-x64` = Build.`dist-mac-x64` +val `dist-mac-aarch64` = Build.`dist-mac-aarch64` +val `dist-win-x64` = Build.`dist-win-x64` +val `dist-linux-x64` = Build.`dist-linux-x64` +val `dist-linux-aarch64` = Build.`dist-linux-aarch64` val `community-build` = Build.`community-build` val `sbt-community-build` = Build.`sbt-community-build` val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler` diff --git a/dist/bin-native-overrides/cli-common-platform b/dist/bin-native-overrides/cli-common-platform new file mode 100644 index 000000000000..49803d6282c5 --- /dev/null +++ b/dist/bin-native-overrides/cli-common-platform @@ -0,0 +1,3 @@ +#!/usr/bin/env bash + +SCALA_CLI_CMD_BASH=("\"$PROG_HOME/bin/scala-cli\"") diff --git a/dist/bin/cli-common b/dist/bin/cli-common index d295d58916da..eb77c9924d06 100644 --- a/dist/bin/cli-common +++ b/dist/bin/cli-common @@ -71,44 +71,7 @@ if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then esac fi -# Resolve JAVA_HOME from javac command path -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - javaExecutable="`readlink -f \"$javaExecutable\"`" - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "${JAVACMD-}" ] ; then - if [ -n "${JAVA_HOME-}" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." - echo " We cannot execute $JAVACMD" - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi +source "$PROG_HOME/bin/cli-common-platform" CLASSPATH_SUFFIX="" # Path separator used in EXTRA_CLASSPATH @@ -136,23 +99,6 @@ fi # * The code below is for Dotty # *-------------------------------------------------*/ -find_lib () { - for lib in "$PROG_HOME"/lib/$1 ; do - if [[ -f "$lib" ]]; then - if [ -n "$CYGPATHCMD" ]; then - "$CYGPATHCMD" -am "$lib" - elif [[ $mingw || $msys ]]; then - echo "$lib" | sed 's|/|\\\\|g' - else - echo "$lib" - fi - return - fi - done -} - -SCALA_CLI_JAR="$PROG_HOME/etc/scala-cli.jar" - declare -a scala_args addScala () { diff --git a/dist/bin/cli-common-platform b/dist/bin/cli-common-platform new file mode 100644 index 000000000000..ee1b4768dd51 --- /dev/null +++ b/dist/bin/cli-common-platform @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +source "$PROG_HOME/bin/common-java" + +SCALA_CLI_CMD_BASH=("\"$JAVACMD\"" "-jar \"$PROG_HOME/bin/scala-cli.jar\"") diff --git a/dist/bin/common b/dist/bin/common index e3e4253938fb..f9dd594c566b 100755 --- a/dist/bin/common +++ b/dist/bin/common @@ -67,44 +67,7 @@ if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then esac fi -# Resolve JAVA_HOME from javac command path -if [ -z "$JAVA_HOME" ]; then - javaExecutable="`which javac`" - if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then - # readlink(1) is not available as standard on Solaris 10. - readLink=`which readlink` - if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then - javaExecutable="`readlink -f \"$javaExecutable\"`" - javaHome="`dirname \"$javaExecutable\"`" - javaHome=`expr "$javaHome" : '\(.*\)/bin'` - JAVA_HOME="$javaHome" - export JAVA_HOME - fi - fi -fi - -if [ -z "${JAVACMD-}" ] ; then - if [ -n "${JAVA_HOME-}" ] ; then - if [ -x "$JAVA_HOME/jre/sh/java" ] ; then - # IBM's JDK on AIX uses strange locations for the executables - JAVACMD="$JAVA_HOME/jre/sh/java" - else - JAVACMD="$JAVA_HOME/bin/java" - fi - else - JAVACMD="`which java`" - fi -fi - -if [ ! -x "$JAVACMD" ] ; then - echo "Error: JAVA_HOME is not defined correctly." - echo " We cannot execute $JAVACMD" - exit 1 -fi - -if [ -z "$JAVA_HOME" ] ; then - echo "Warning: JAVA_HOME environment variable is not set." -fi +source "$PROG_HOME/bin/common-java" CLASSPATH_SUFFIX="" # Path separator used in EXTRA_CLASSPATH diff --git a/dist/bin/common-java b/dist/bin/common-java new file mode 100644 index 000000000000..fbc8f309543c --- /dev/null +++ b/dist/bin/common-java @@ -0,0 +1,40 @@ +#!/usr/bin/env bash + +# Resolve JAVA_HOME from javac command path +if [ -z "$JAVA_HOME" ]; then + javaExecutable="`which javac`" + if [ -n "$javaExecutable" -a -f "$javaExecutable" -a ! "`expr \"$javaExecutable\" : '\([^ ]*\)'`" = "no" ]; then + # readlink(1) is not available as standard on Solaris 10. + readLink=`which readlink` + if [ ! `expr "$readLink" : '\([^ ]*\)'` = "no" ]; then + javaExecutable="`readlink -f \"$javaExecutable\"`" + javaHome="`dirname \"$javaExecutable\"`" + javaHome=`expr "$javaHome" : '\(.*\)/bin'` + JAVA_HOME="$javaHome" + export JAVA_HOME + fi + fi +fi + +if [ -z "${JAVACMD-}" ] ; then + if [ -n "${JAVA_HOME-}" ] ; then + if [ -x "$JAVA_HOME/jre/sh/java" ] ; then + # IBM's JDK on AIX uses strange locations for the executables + JAVACMD="$JAVA_HOME/jre/sh/java" + else + JAVACMD="$JAVA_HOME/bin/java" + fi + else + JAVACMD="`which java`" + fi +fi + +if [ ! -x "$JAVACMD" ] ; then + echo "Error: JAVA_HOME is not defined correctly." + echo " We cannot execute $JAVACMD" + exit 1 +fi + +if [ -z "$JAVA_HOME" ] ; then + echo "Warning: JAVA_HOME environment variable is not set." +fi diff --git a/dist/bin/scala b/dist/bin/scala index 3040c5a9a0f3..6b5930e3c215 100755 --- a/dist/bin/scala +++ b/dist/bin/scala @@ -54,8 +54,9 @@ done # exec here would prevent onExit from being called, leaving terminal in unusable state [ -z "${ConEmuPID-}" -o -n "${cygwin-}" ] && export MSYSTEM= PWD= # workaround for #12405 -eval "\"$JAVACMD\"" \ - "-jar \"$SCALA_CLI_JAR\"" \ + +# SCALA_CLI_CMD_BASH is an array, set by cli-common-platform +eval "${SCALA_CLI_CMD_BASH[@]}" \ "--prog-name scala" \ "--cli-default-scala-version \"$SCALA_VERSION\"" \ "-r \"$MVN_REPOSITORY\"" \ diff --git a/project/Build.scala b/project/Build.scala index 3f5f7088098c..db5d59689283 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -2092,22 +2092,63 @@ object Build { packMain := Map(), publishArtifact := false, packGenerateMakefile := false, - packArchiveName := "scala3-" + dottyVersion, republishRepo := target.value / "republish", - republishLaunchers := { - val cliV = scalaCliLauncherVersion - Seq( - ("scala-cli.jar", cliV, url(s"https://github.com/VirtusLab/scala-cli/releases/download/v$cliV/scala-cli.jar")) - ) - }, + packResourceDir += (republishRepo.value / "bin" -> "bin"), + packResourceDir += (republishRepo.value / "maven2" -> "maven2"), Compile / pack := (Compile / pack).dependsOn(republish).value, ) lazy val dist = project.asDist(Bootstrapped) .settings( - packResourceDir += (baseDirectory.value / "bin" -> "bin"), - packResourceDir += (republishRepo.value / "maven2" -> "maven2"), - packResourceDir += (republishRepo.value / "etc" -> "etc"), + packArchiveName := "scala3-" + dottyVersion, + republishBinDir := baseDirectory.value / "bin", + republishLaunchers += + ("scala-cli.jar" -> s"https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli.jar") + ) + + lazy val `dist-mac-x64` = project.in(file("dist/mac-x64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-x86_64-apple-darwin", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-apple-darwin.gz") + ) + + lazy val `dist-mac-aarch64` = project.in(file("dist/mac-aarch64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-aarch64-apple-darwin", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-apple-darwin.gz") + ) + + lazy val `dist-win-x64` = project.in(file("dist/win-x64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-win32", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli.exe" -> s"zip+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-win32.zip!/scala-cli.exe") + ) + + lazy val `dist-linux-x64` = project.in(file("dist/linux-x64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-x86_64-pc-linux", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-x86_64-pc-linux.gz") + ) + + lazy val `dist-linux-aarch64` = project.in(file("dist/linux-aarch64")).asDist(Bootstrapped) + .settings( + republishBinDir := (dist / republishBinDir).value, + packArchiveName := (dist / packArchiveName).value + "-aarch64-pc-linux", + republishBinOverrides += (dist / baseDirectory).value / "bin-native-overrides", + republishLaunchers += + ("scala-cli" -> s"gz+https://github.com/VirtusLab/scala-cli/releases/download/v$scalaCliLauncherVersion/scala-cli-aarch64-pc-linux.gz") ) private def customMimaReportBinaryIssues(issueFilterLocation: String) = mimaReportBinaryIssues := { diff --git a/project/RepublishPlugin.scala b/project/RepublishPlugin.scala index 72b5400a321e..13e4adace1c0 100644 --- a/project/RepublishPlugin.scala +++ b/project/RepublishPlugin.scala @@ -13,10 +13,47 @@ import scala.collection.mutable import java.nio.file.Files import versionhelpers.DottyVersion._ +import java.nio.file.attribute.PosixFilePermission +import java.nio.file.{Files, Path} + +import scala.jdk.CollectionConverters._ +import org.checkerframework.checker.units.qual.s +import org.checkerframework.checker.units.qual.A +import com.google.protobuf.Message + /** This local plugin provides ways of publishing a project classpath and library dependencies to * .a local repository */ object RepublishPlugin extends AutoPlugin { + private object FileUtil { + + def tryMakeExecutable(path: Path): Boolean = + try { + val perms = Files.getPosixFilePermissions(path).asScala.toSet + + var newPerms = perms + if (perms(PosixFilePermission.OWNER_READ)) + newPerms += PosixFilePermission.OWNER_EXECUTE + if (perms(PosixFilePermission.GROUP_READ)) + newPerms += PosixFilePermission.GROUP_EXECUTE + if (perms(PosixFilePermission.OTHERS_READ)) + newPerms += PosixFilePermission.OTHERS_EXECUTE + + if (newPerms != perms) + Files.setPosixFilePermissions( + path, + newPerms.asJava + ) + + true + } + catch { + case _: UnsupportedOperationException => + false + } + + } + override def trigger = allRequirements override def requires = super.requires && PublishBinPlugin && PackPlugin @@ -26,9 +63,12 @@ object RepublishPlugin extends AutoPlugin { val republishAllResolved = taskKey[Seq[ResolvedArtifacts]]("Resolve the dependencies for the distribution") val republishClasspath = taskKey[Set[File]]("cache the dependencies for the distribution") val republishFetchLaunchers = taskKey[Set[File]]("cache the launcher deps for the distribution") + val republishPrepareBin = taskKey[File]("prepare the bin directory, including launchers and scripts.") + val republishBinDir = settingKey[File]("where to find static files for the bin dir.") + val republishBinOverrides = settingKey[Seq[File]]("files to override those in bin-dir.") val republish = taskKey[File]("cache the dependencies and download launchers for the distribution") val republishRepo = settingKey[File]("the location to store the republished artifacts.") - val republishLaunchers = settingKey[Seq[(String, String, URL)]]("launchers to download. Sequence of (name, version, URL).") + val republishLaunchers = settingKey[Seq[(String, String)]]("launchers to download. Sequence of (name, version, URL).") } import autoImport._ @@ -39,6 +79,8 @@ object RepublishPlugin extends AutoPlugin { case class ResolvedArtifacts(id: SimpleModuleId, jar: File, pom: File) override val projectSettings: Seq[Def.Setting[_]] = Def.settings( + republishLaunchers := Seq.empty, + republishBinOverrides := Seq.empty, republishLocalResolved / republishProjectRefs := { val proj = thisProjectRef.value val deps = buildDependencies.value @@ -151,42 +193,94 @@ object RepublishPlugin extends AutoPlugin { val log = s.log val repoDir = republishRepo.value val launcherVersions = republishLaunchers.value + val libexec = republishPrepareBin.value - val etc = repoDir / "etc" + val dlCache = repoDir / "cache" val store = s.cacheStoreFactory / "versions" - def work(dest: File, launcher: URL) = { - IO.delete(dest) - Using.urlInputStream(launcher) { in => - IO.createDirectory(etc) - log.info(s"[republish] Downloading $launcher to $dest...") - IO.transfer(in, dest) - log.info(s"[republish] Downloaded $launcher to $dest...") + def work(name: String, dest: File, launcher: String): File = { + val (launcherURL, workFile, prefix, subPart) = { + if (launcher.startsWith("gz+")) { + IO.createDirectory(dlCache) + val launcherURL = url(launcher.stripPrefix("gz+")) + (launcherURL, dlCache / s"$name.gz", "gz", "") + } else if (launcher.startsWith("zip+")) { + IO.createDirectory(dlCache) + val (urlPart, subPath) = launcher.split("!/") match { + case Array(urlPart, subPath) => (urlPart, subPath) + case _ => + throw new MessageOnlyException(s"[republish] Invalid zip+ URL, expected ! to mark subpath: $launcher") + } + val launcherURL = url(urlPart.stripPrefix("zip+")) + (launcherURL, dlCache / s"$name.zip", "zip", subPath) + } else { + IO.createDirectory(libexec) + (url(launcher), dest, "", "") + } + } + IO.delete(workFile) + Using.urlInputStream(launcherURL) { in => + log.info(s"[republish] Downloading $launcherURL to $workFile...") + IO.transfer(in, workFile) + log.info(s"[republish] Downloaded $launcherURL to $workFile...") + } + if (prefix == "gz") { + IO.delete(dest) + Using.fileInputStream(workFile) { in => + Using.gzipInputStream(in) { gzIn => + IO.transfer(gzIn, dest) + } + } + log.info(s"[republish] uncompressed gz file $workFile to $dest...") + FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable + IO.delete(workFile) + } else if (prefix == "zip") { + IO.delete(dest) + val files = IO.unzip(workFile, dlCache, new ExactFilter(subPart)) + val extracted = files.headOption.getOrElse(throw new MessageOnlyException(s"[republish] No files extracted from $workFile matching $subPart")) + log.info(s"[republish] unzipped $workFile to $extracted...") + IO.move(extracted, dest) + log.info(s"[republish] moved $extracted to $dest...") + FileUtil.tryMakeExecutable(dest.toPath) // TODO: we also need to copy to the bin directory so the archive makes it executable + IO.delete(workFile) } dest } val allLaunchers = { - for ((name, version, launcher) <- launcherVersions) yield { - val dest = etc / name + for ((name, launcher) <- launcherVersions) yield { + val dest = libexec / name val id = name.replaceAll("[^a-zA-Z0-9]", "_") - val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, version) => + val fetchAction = Tracked.inputChanged[String, File](store.make(id)) { (inChanged, launcher) => if (inChanged || !Files.exists(dest.toPath)) { - work(dest, launcher) + work(name, dest, launcher) } else { log.info(s"[republish] Using cached $launcher at $dest...") dest } } - fetchAction(version) + fetchAction(launcher) } } allLaunchers.toSet }, + republishPrepareBin := { + val baseDir = baseDirectory.value + val srcBin = republishBinDir.value + val overrides = republishBinOverrides.value + val repoDir = republishRepo.value + + val targetBin = repoDir / "bin" + IO.copyDirectory(srcBin, targetBin) + overrides.foreach { dir => + IO.copyDirectory(dir, targetBin, overwrite = true) + } + targetBin + }, republish := { val cacheDir = republishRepo.value val artifacts = republishClasspath.value