Backport "Bundle scala cli in scala command" to 3.5.0 (#20706)

Backports #20351 to 3.5.0-RC2

.github/workflows/ci.yaml

@@ -141,7 +141,8 @@ jobs:
 
       - name: Cmd Tests
         run: |
-          ./project/scripts/sbt ";dist/pack; scala3-bootstrapped/compile; scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*; scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
+          ./project/scripts/buildScalaBinary
+          ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/* ;scala3-compiler-bootstrapped/scala3CompilerCoursierTest:test"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 
@@ -221,7 +222,7 @@ jobs:
         shell: cmd
 
       - name: build binary
-        run: sbt "dist/pack" & bash -version
+        run: sbt "dist-win-x86_64/pack" & bash -version
         shell: cmd
 
       - name: cygwin tests
@@ -254,8 +255,12 @@ jobs:
       - name: Git Checkout
         uses: actions/checkout@v4
 
+      - name: build binary
+        run: sbt "dist-win-x86_64/pack"
+        shell: cmd
+
       - name: Test
-        run: sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test"
+        run: sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test"
         shell: cmd
 
       - name: Scala.js Test
@@ -581,7 +586,8 @@ jobs:
 
       - name: Test
         run: |
-          ./project/scripts/sbt ";dist/pack ;scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*"
+          ./project/scripts/buildScalaBinary
+          ./project/scripts/sbt ";scala3-bootstrapped/compile ;scala3-bootstrapped/test ;sbt-test/scripted scala2-compat/*"
           ./project/scripts/cmdTests
           ./project/scripts/bootstrappedOnlyCmdTests
 

.github/workflows/launchers.yml

@@ -0,0 +1,96 @@
+name: Test CLI Launchers on all the platforms
+on:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  linux-x86_64:
+    name: Deploy and Test on Linux x64 architecture
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-linux-x86_64"
+
+  linux-aarch64:
+    name: Deploy and Test on Linux ARM64 architecture
+    runs-on: macos-latest
+    if: ${{ false }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      # https://github.com/actions/runner-images/issues/9369
+      - name: Install sbt
+        run: brew install sbt
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-linux-aarch64"
+
+  mac-x86_64:
+    name: Deploy and Test on Mac x64 architecture
+    runs-on: macos-13
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      # https://github.com/actions/runner-images/issues/9369
+      - name: Install sbt
+        run: brew install sbt
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-mac-x86_64"
+
+  mac-aarch64:
+    name: Deploy and Test on Mac ARM64 architecture
+    runs-on: macos-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      # https://github.com/actions/runner-images/issues/9369
+      - name: Install sbt
+        run: brew install sbt
+      - name: Build and test launcher command
+        run: ./project/scripts/native-integration/bashTests
+        env:
+          LAUNCHER_EXPECTED_PROJECT: "dist-mac-aarch64"
+
+  win-x86_64:
+    name: Deploy and Test on Windows x64 architecture
+    runs-on: windows-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Set up JDK 17
+        uses: actions/setup-java@v4
+        with:
+          java-version: '17'
+          distribution: 'temurin'
+          cache: 'sbt'
+      - name: Build the launcher command
+        run: sbt "dist-win-x86_64/pack"
+      - name: Run the launcher command tests
+        run: './project/scripts/native-integration/winTests.bat'
+        shell: cmd

bin/common

@@ -9,15 +9,18 @@ target="$1"
 
 shift # Mutates $@ by deleting the first element ($1)
 
+# set the $DIST_PROJECT and $DIST_DIR variables
+source "$ROOT/bin/common-platform"
+
 # Marker file used to obtain the date of latest call to sbt-back
-version="$ROOT/dist/target/pack/VERSION"
+version="$ROOT/$DIST_DIR/target/pack/VERSION"
 
 # Create the target if absent or if file changed in ROOT/compiler
 new_files="$(find "$ROOT/compiler" \( -iname "*.scala" -o -iname "*.java" \) -newer "$version" 2> /dev/null)"
 
 if [ ! -f "$version" ] || [ ! -z "$new_files" ]; then
   echo "Building Dotty..."
-  (cd $ROOT && sbt "dist/pack")
+  (cd $ROOT && sbt "$DIST_PROJECT/pack")
 fi
 
-"$target" "$@"
+"$ROOT/$DIST_DIR/target/pack/bin/$target" "$@"

bin/common-platform

@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+
+unset cygwin mingw msys darwin
+
+# COLUMNS is used together with command line option '-pageWidth'.
+if command -v tput >/dev/null 2>&1; then
+  export COLUMNS="$(tput -Tdumb cols)"
+fi
+
+case "`uname`" in
+  CYGWIN*) cygwin=true
+          ;;
+  MINGW*) mingw=true
+          ;;
+  MSYS*) msys=true
+          ;;
+  Darwin*) darwin=true
+           ;;
+esac
+
+unset DIST_PROJECT DIST_DIR
+
+if [[ ${cygwin-} || ${mingw-} || ${msys-} ]]; then
+  DIST_PROJECT="dist-win-x86_64"
+  DIST_DIR="dist/win-x86_64"
+else
+  # OS and arch logic taken from https://github.com/VirtusLab/scala-cli/blob/main/scala-cli.sh
+  unset arch ARCH_NORM
+  arch=$(uname -m)
+  if [[ "$arch" == "aarch64" ]] || [[ "$arch" == "x86_64" ]]; then
+    ARCH_NORM="$arch"
+  elif [[ "$arch" == "amd64" ]]; then
+    ARCH_NORM="x86_64"
+  elif [[ "$arch" == "arm64" ]]; then
+    ARCH_NORM="aarch64"
+  else
+    ARCH_NORM="unknown"
+  fi
+
+  if [ "$(expr substr $(uname -s) 1 5 2>/dev/null)" == "Linux" ]; then
+    if [[ "$ARCH_NORM" == "unknown" ]]; then
+      echo >&2 "unknown Linux CPU architecture, defaulting to JVM launcher"
+      DIST_PROJECT="dist"
+      DIST_DIR="dist"
+    else
+      DIST_PROJECT="dist-linux-$ARCH_NORM"
+      DIST_DIR="dist/linux-$ARCH_NORM"
+    fi
+  elif [ "$(uname)" == "Darwin" ]; then
+    if [[ "$ARCH_NORM" == "unknown" ]]; then
+      echo >&2 "unknown Darwin CPU architecture, defaulting to JVM launcher"
+      DIST_PROJECT="dist"
+      DIST_DIR="dist"
+    else
+      DIST_PROJECT="dist-mac-$ARCH_NORM"
+      DIST_DIR="dist/mac-$ARCH_NORM"
+    fi
+  else
+    echo >&2 "unknown OS, defaulting to JVM launcher"
+    DIST_PROJECT="dist"
+    DIST_DIR="dist"
+  fi
+fi

bin/scala

@@ -2,4 +2,37 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scala" "--power" "$@" "--offline" "--server=false"
+scala_args() {
+
+  declare -a CLI_ARGS
+  declare -a SCRIPT_ARGS
+  declare DISABLE_BLOOP=1
+
+  while (( "$#" )); do
+    case "$1" in
+      "--")
+        shift
+        SCRIPT_ARGS+=("--")
+        SCRIPT_ARGS+=("$@")
+        break
+        ;;
+      "clean" | "version" | "--version" | "-version" | "help" | "--help" | "-help")
+        CLI_ARGS+=("$1")
+        DISABLE_BLOOP=0 # clean command should not add --offline --server=false
+        shift
+        ;;
+      *)
+        CLI_ARGS+=("$1")
+        shift
+        ;;
+    esac
+  done
+
+  if [ $DISABLE_BLOOP -eq 1 ]; then
+    CLI_ARGS+=("--offline" "--server=false")
+  fi
+
+  echo "--power ${CLI_ARGS[@]} ${SCRIPT_ARGS[@]}"
+}
+
+"$ROOT/bin/common" "scala" $(scala_args "$@")

bin/scalac

@@ -2,4 +2,4 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scalac" "$@"
+"$ROOT/bin/common" "scalac" "$@"

bin/scaladoc

@@ -2,4 +2,4 @@
 
 ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" >& /dev/null && pwd)/.."
 
-"$ROOT/bin/common" "$ROOT/dist/target/pack/bin/scaladoc" "$@"
+"$ROOT/bin/common" "scaladoc" "$@"

build.sbt

@@ -28,6 +28,11 @@ val `scaladoc-js-main` = Build.`scaladoc-js-main`
 val `scaladoc-js-contributors` = Build.`scaladoc-js-contributors`
 val `scala3-bench-run` = Build.`scala3-bench-run`
 val dist = Build.dist
+val `dist-mac-x86_64` = Build.`dist-mac-x86_64`
+val `dist-mac-aarch64` = Build.`dist-mac-aarch64`
+val `dist-win-x86_64` = Build.`dist-win-x86_64`
+val `dist-linux-x86_64` = Build.`dist-linux-x86_64`
+val `dist-linux-aarch64` = Build.`dist-linux-aarch64`
 val `community-build` = Build.`community-build`
 val `sbt-community-build` = Build.`sbt-community-build`
 val `scala3-presentation-compiler` = Build.`scala3-presentation-compiler`

compiler/src/dotty/tools/MainGenericRunner.scala

@@ -270,7 +270,7 @@ object MainGenericRunner {
 
     val ranByCoursierBootstrap =
       sys.props.isDefinedAt("coursier.mainJar")
-      || sys.props.get("bootstrap.mainClass").filter(_ == "dotty.tools.MainGenericRunner").isDefined
+      || sys.props.get("bootstrap.mainClass").contains("dotty.tools.MainGenericRunner")
 
     val silenced = sys.props.get("scala.use_legacy_launcher") == Some("true")
 

compiler/test-resources/scripting/classpathReport.sc → compiler/test-resources/scripting/classpathReport_scalacli.sc

@@ -1,5 +1,5 @@
 #!/usr/bin/env bin/scala
-
+// This file is a Scala CLI script.
 import java.nio.file.Paths
 
 // def main(args: Array[String]): Unit = // MIGRATION: Scala CLI expects `*.sc` files to be straight-line code

compiler/test-resources/scripting/envtest.sc

@@ -1,2 +1,4 @@
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
+
 def main(args: Array[String]): Unit =
   println("Hello " + util.Properties.propOrNull("key"))

compiler/test-resources/scripting/envtest_scalacli.sc

@@ -0,0 +1,3 @@
+// This file is a Scala CLI script.
+
+println("Hello " + util.Properties.propOrNull("key"))

compiler/test-resources/scripting/hashBang.sc

@@ -1,4 +1,4 @@
-#!/usr/bin/env scala
+#!/usr/bin/env fake-program-to-test-hashbang-removal
 # comment
 STUFF=nada
 !#

compiler/test-resources/scripting/hashBang.scala

@@ -1,8 +1,8 @@
-#!/usr/bin/env scala
+#!/usr/bin/env fake-program-to-test-hashbang-removal
 # comment
 STUFF=nada
 !#
-
+// everything above this point should be ignored by the compiler
 def main(args: Array[String]): Unit =
   System.err.printf("mainClassFromStack: %s\n",mainFromStack)
   assert(mainFromStack.contains("hashBang"),s"fromStack[$mainFromStack]")

compiler/test-resources/scripting/scriptName.scala

@@ -1,4 +1,4 @@
-#!/usr/bin/env scala
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
 
   def main(args: Array[String]): Unit =
     val name = Option(sys.props("script.name")) match {

compiler/test-resources/scripting/scriptPath.sc

@@ -1,4 +1,4 @@
-#!dist/target/pack/bin/scala
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
 
   def main(args: Array[String]): Unit =
     args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }

compiler/test-resources/scripting/scriptPath_scalacli.sc

@@ -0,0 +1,13 @@
+#!/usr/bin/env bin/scala
+
+// THIS FILE IS RAN WITH SCALA CLI, which wraps scripts exposing scriptPath and args variables
+
+args.zipWithIndex.foreach { case (arg,i) => printf("arg %d: [%s]\n",i,arg) }
+
+if !scriptPath.endsWith("scriptPath_scalacli.sc") then
+  printf( s"incorrect script.path defined as [$scriptPath]")
+else
+  printf("scriptPath: %s\n", scriptPath) // report the value
+
+extension(s: String)
+  def norm: String = s.replace('\\', '/')

compiler/test-resources/scripting/showArgs.sc

@@ -1,4 +1,4 @@
-#!/usr/bin/env bin/scala
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
 
 // precise output format expected by BashScriptsTests.scala
 def main(args: Array[String]): Unit =

compiler/test-resources/scripting/showArgs_scalacli.sc

@@ -0,0 +1,7 @@
+#!/usr/bin/env bin/scala
+
+// This file is a Scala CLI script.
+
+// precise output format expected by BashScriptsTests.scala
+for (a,i) <- args.zipWithIndex do
+  printf(s"arg %2d:[%s]\n",i,a)

compiler/test-resources/scripting/sqlDateError.sc

@@ -1,4 +1,4 @@
-#!/usr/bin/env bin/scala
+// this file is intended to be ran as an argument to the dotty.tools.scripting.ScriptingDriver class
 
 def main(args: Array[String]): Unit = {
   println(new java.sql.Date(100L))

compiler/test-resources/scripting/sqlDateError_scalacli.sc

@@ -0,0 +1,6 @@
+#!/usr/bin/env bin/scala
+
+// This file is a Scala CLI script.
+
+println(new java.sql.Date(100L))
+System.err.println("SCALA_OPTS="+Option(System.getenv("SCALA_OPTS")).getOrElse(""))