Load virtual data as byte arrays without using UTF-8 encoding

modules/build/src/main/scala/scala/build/CrossSources.scala

@@ -291,7 +291,7 @@ object CrossSources {
           val baseReqs0 = baseReqs(m.scopePath)
           WithBuildRequirements(
             m.requirements.fold(baseReqs0)(_ orElse baseReqs0),
-            Sources.InMemory(m.originalPath, m.relPath, m.code, m.wrapperParamsOpt)
+            Sources.InMemory(m.originalPath, m.relPath, m.content, m.wrapperParamsOpt)
           ) -> m.directivesPositions
       }
     val unwrappedScriptsWithDirectivePositions

modules/build/src/main/scala/scala/build/ScopedSources.scala

@@ -1,5 +1,7 @@
 package scala.build
 
+import java.nio.charset.StandardCharsets
+
 import scala.build.info.{BuildInfo, ScopedBuildInfo}
 import scala.build.options.{BuildOptions, HasScope, Scope}
 import scala.build.preprocessing.ScriptPreprocessor
@@ -63,7 +65,7 @@ final case class ScopedSources(
       Seq(Sources.InMemory(
         Left("build-info"),
         os.rel / "BuildInfo.scala",
-        buildInfo(combinedOptions).generateContents(),
+        buildInfo(combinedOptions).generateContents().getBytes(StandardCharsets.UTF_8),
         None
       ))
     else Nil

modules/build/src/main/scala/scala/build/Sources.scala

@@ -3,6 +3,8 @@ package scala.build
 import coursier.cache.ArchiveCache
 import coursier.util.Task
 
+import java.nio.charset.StandardCharsets
+
 import scala.build.info.BuildInfo
 import scala.build.input.Inputs
 import scala.build.internal.{CodeWrapper, WrapperParams}
@@ -39,7 +41,7 @@ final case class Sources(
       for (inMemSource <- inMemory) yield {
         os.write.over(
           generatedSrcRoot / inMemSource.generatedRelPath,
-          inMemSource.generatedContent.getBytes("UTF-8"),
+          inMemSource.content,
           createFolders = true
         )
         (
@@ -75,7 +77,7 @@ object Sources {
   final case class InMemory(
     originalPath: Either[String, (os.SubPath, os.Path)],
     generatedRelPath: os.RelPath,
-    generatedContent: String,
+    content: Array[Byte],
     wrapperParamsOpt: Option[WrapperParams]
   )
 
@@ -86,7 +88,12 @@ object Sources {
   ) {
     def wrap(wrapper: CodeWrapper): InMemory = {
       val (content, wrapperParams) = wrapScriptFun(wrapper)
-      InMemory(originalPath, generatedRelPath, content, Some(wrapperParams))
+      InMemory(
+        originalPath,
+        generatedRelPath,
+        content.getBytes(StandardCharsets.UTF_8),
+        Some(wrapperParams)
+      )
     }
   }
 

modules/build/src/main/scala/scala/build/preprocessing/DataPreprocessor.scala

@@ -25,31 +25,19 @@ case object DataPreprocessor extends Preprocessor {
     input match {
       case file: VirtualData =>
         val res = either {
-          val content = new String(file.content, StandardCharsets.UTF_8)
-          val preprocessedDirectives: PreprocessedDirectives = value {
-            DirectivesPreprocessor.preprocess(
-              content,
-              Left(file.subPath.toString),
-              file.scopePath,
-              logger,
-              allowRestrictedFeatures,
-              suppressWarningOptions,
-              maybeRecoverOnError
-            )
-          }
           val inMemory = Seq(
             PreprocessedSource.InMemory(
               originalPath = Left(file.source),
               relPath = file.subPath,
-              code = content,
+              content = file.content,
               wrapperParamsOpt = None,
-              options = Some(preprocessedDirectives.globalUsings),
-              optionsWithTargetRequirements = preprocessedDirectives.usingsWithReqs,
+              options = None,
+              optionsWithTargetRequirements = Nil,
               requirements = Some(BuildRequirements()),
               scopedRequirements = Nil,
               mainClassOpt = None,
               scopePath = file.scopePath,
-              preprocessedDirectives.directivesPositions
+              directivesPositions = None
             )
           )
           inMemory

modules/build/src/main/scala/scala/build/preprocessing/JavaPreprocessor.scala

@@ -102,7 +102,7 @@ final case class JavaPreprocessor(
           val s = PreprocessedSource.InMemory(
             originalPath = Left(v.source),
             relPath = relPath,
-            code = content,
+            content = v.content,
             wrapperParamsOpt = None,
             options = Some(preprocessedDirectives.globalUsings),
             optionsWithTargetRequirements = preprocessedDirectives.usingsWithReqs,

modules/build/src/main/scala/scala/build/preprocessing/MarkdownPreprocessor.scala

@@ -100,7 +100,7 @@ case object MarkdownPreprocessor extends Preprocessor {
             PreprocessedSource.InMemory(
               originalPath = reportingPath.map(subPath -> _),
               relPath = os.rel / (subPath / os.up) / s"${subPath.last}$generatedSourceNameSuffix",
-              processedCode,
+              processedCode.getBytes(StandardCharsets.UTF_8),
               wrapperParamsOpt = None,
               options = Some(processingOutput.opts),
               optionsWithTargetRequirements = processingOutput.optsWithReqs,

modules/build/src/main/scala/scala/build/preprocessing/PreprocessedSource.scala

@@ -37,7 +37,7 @@ object PreprocessedSource {
   final case class InMemory(
     originalPath: Either[String, (os.SubPath, os.Path)],
     relPath: os.RelPath,
-    code: String,
+    content: Array[Byte],
     wrapperParamsOpt: Option[WrapperParams],
     options: Option[BuildOptions],
     optionsWithTargetRequirements: List[WithBuildRequirements[BuildOptions]],

modules/build/src/main/scala/scala/build/preprocessing/ScalaPreprocessor.scala

@@ -83,7 +83,7 @@ case object ScalaPreprocessor extends Preprocessor {
                 PreprocessedSource.InMemory(
                   originalPath = Right((f.subPath, f.path)),
                   relPath = f.subPath,
-                  code = updatedCode,
+                  content = updatedCode.getBytes(StandardCharsets.UTF_8),
                   wrapperParamsOpt = None,
                   options = Some(options),
                   optionsWithTargetRequirements = optionsWithReqs,
@@ -161,7 +161,7 @@ case object ScalaPreprocessor extends Preprocessor {
           val s = PreprocessedSource.InMemory(
             originalPath = Left(v.source),
             relPath = relPath,
-            updatedContentOpt.getOrElse(content),
+            updatedContentOpt.map(_.getBytes(StandardCharsets.UTF_8)).getOrElse(v.content),
             wrapperParamsOpt = None,
             options = Some(options),
             optionsWithTargetRequirements = optionsWithTargetRequirements,

modules/build/src/test/scala/scala/build/tests/SourcesTests.scala

@@ -5,6 +5,7 @@ import coursier.cache.{ArchiveCache, Cache}
 import coursier.util.{Artifact, Task}
 import dependency.*
 
+import java.nio.charset.StandardCharsets
 import scala.build.Ops.*
 import scala.build.{CrossSources, Position, Sources}
 import scala.build.internal.ObjectCodeWrapper
@@ -310,7 +311,8 @@ class SourcesTests extends munit.FunSuite {
       val sources =
         scopedSources.sources(Scope.Main, crossSources.sharedOptions(BuildOptions()))
 
-      val parsedCodes: Seq[String] = sources.inMemory.map(_.generatedContent)
+      val parsedCodes: Seq[String] =
+        sources.inMemory.map(_.content).map(s => new String(s, StandardCharsets.UTF_8))
 
       parsedCodes.zip(expectedParsedCodes).foreach { case (parsedCode, expectedCode) =>
         expect(parsedCode.contains(expectedCode))

modules/cli/src/main/scala/scala/cli/commands/package0/Package.scala

@@ -593,7 +593,7 @@ object Package extends ScalaCommand[PackageOptions] with BuildCommandHelpers {
       val prefix = if (originalOpt.isEmpty) os.rel else generatedSourcesPrefix
       val generated = (
         prefix / inMemSource.generatedRelPath,
-        inMemSource.generatedContent.getBytes(StandardCharsets.UTF_8),
+        inMemSource.content,
         lastModified
       )
       Iterator(generated) ++ originalOpt.iterator

modules/cli/src/main/scala/scala/cli/exportCmd/JsonProjectDescriptor.scala

@@ -23,7 +23,6 @@ final case class JsonProjectDescriptor(
   projectName: Option[String] = None,
   logger: Logger
 ) extends ProjectDescriptor {
-  private val charSet = StandardCharsets.UTF_8
 
   def `export`(
     optionsMain: BuildOptions,

modules/cli/src/main/scala/scala/cli/exportCmd/MillProjectDescriptor.scala

@@ -21,11 +21,9 @@ final case class MillProjectDescriptor(
   logger: Logger
 ) extends ProjectDescriptor {
 
-  private val charSet = StandardCharsets.UTF_8
-
   private def sourcesSettings(mainSources: Sources, testSources: Sources): MillProject = {
-    val mainSources0 = ProjectDescriptor.sources(mainSources, charSet)
-    val testSources0 = ProjectDescriptor.sources(testSources, charSet)
+    val mainSources0 = ProjectDescriptor.sources(mainSources)
+    val testSources0 = ProjectDescriptor.sources(testSources)
     MillProject(mainSources = mainSources0, testSources = testSources0)
   }
 

modules/cli/src/main/scala/scala/cli/exportCmd/ProjectDescriptor.scala

@@ -15,7 +15,7 @@ abstract class ProjectDescriptor extends Product with Serializable {
 }
 
 object ProjectDescriptor {
-  def sources(sources: Sources, charSet: Charset): Seq[(os.SubPath, String, Array[Byte])] = {
+  def sources(sources: Sources): Seq[(os.SubPath, String, Array[Byte])] = {
 
     val mainSources = sources.paths.map {
       case (path, relPath) =>
@@ -33,7 +33,7 @@ object ProjectDescriptor {
       (
         inMemSource.generatedRelPath.asSubPath,
         language,
-        inMemSource.generatedContent.getBytes(charSet)
+        inMemSource.content
       )
     }
 

modules/cli/src/main/scala/scala/cli/exportCmd/SbtProjectDescriptor.scala

@@ -26,13 +26,12 @@ final case class SbtProjectDescriptor(
   extraSettings: Seq[String],
   logger: Logger
 ) extends ProjectDescriptor {
-  private val charSet = StandardCharsets.UTF_8
-  private val q       = "\""
-  private val nl      = System.lineSeparator()
+  private val q  = "\""
+  private val nl = System.lineSeparator()
 
   private def sources(sourcesMain: Sources, sourcesTest: Sources): SbtProject = {
-    val mainSources = ProjectDescriptor.sources(sourcesMain, charSet)
-    val testSources = ProjectDescriptor.sources(sourcesTest, charSet)
+    val mainSources = ProjectDescriptor.sources(sourcesMain)
+    val testSources = ProjectDescriptor.sources(sourcesTest)
     SbtProject(
       mainSources = mainSources,
       testSources = testSources

modules/integration/src/main/scala/scala/cli/integration/TestInputs.scala

@@ -26,7 +26,7 @@ final case class TestInputs(maybeCharset: Option[Charset], files: (os.RelPath, S
   def asZip[T](f: (os.Path, os.Path) => T): T =
     TestInputs.withTmpDir { tmpDir =>
       val zipArchivePath = tmpDir / s"${tmpDir.last}.zip"
-      compress(zipArchivePath, files)
+      compress(zipArchivePath, files.map { case (relPath, content) => (relPath, content, charset) })
       f(tmpDir, zipArchivePath)
     }
   def fromRoot[T](f: os.Path => T): T =
@@ -46,11 +46,11 @@ object TestInputs {
 
   def empty: TestInputs = TestInputs()
 
-  def compress(zipFilepath: os.Path, files: Seq[(os.RelPath, String)]) = {
+  def compress(zipFilepath: os.Path, files: Seq[(os.RelPath, String, Charset)]) = {
     val zip = new ZipOutputStream(new FileOutputStream(zipFilepath.toString()))
-    try for ((relPath, content) <- files) {
+    try for ((relPath, content, charset) <- files) {
         zip.putNextEntry(new ZipEntry(relPath.toString()))
-        val in: Array[Byte] = content.getBytes
+        val in: Array[Byte] = content.getBytes(charset)
         zip.write(in)
         zip.closeEntry()
       }

modules/integration/src/test/scala/scala/cli/integration/RunZipTestDefinitions.scala

@@ -2,6 +2,10 @@ package scala.cli.integration
 
 import com.eed3si9n.expecty.Expecty.expect
 
+import java.nio.charset.{Charset, StandardCharsets}
+
+import scala.cli.integration.TestInputs.compress
+
 trait RunZipTestDefinitions { _: RunTestDefinitions =>
   test("Zip with multiple Scala files") {
     val inputs = TestInputs(
@@ -24,6 +28,47 @@ trait RunZipTestDefinitions { _: RunTestDefinitions =>
       expect(output == message)
     }
   }
+  test("load virtual data with UTF_16 encoding") {
+    val zipInputs: Seq[(os.RelPath, String, Charset)] = Seq(
+      (
+        os.rel / "Hello.scala",
+        s"""//> using resourceDir "./"
+           |import scala.io.Source
+           |import java.nio.charset.StandardCharsets
+           |import java.io.{BufferedReader, InputStreamReader}
+           |import java.util.stream.Collectors
+           |
+           |object Hello extends App {
+           |    val inputStream = getClass().getResourceAsStream("input")
+           |    val nativeResourceText = new BufferedReader(
+           |      new InputStreamReader(inputStream, StandardCharsets.UTF_16)
+           |    ).lines().collect(Collectors.joining("\\n"));
+           |    println(nativeResourceText)
+           |}
+           |""".stripMargin,
+        StandardCharsets.UTF_8
+      ),
+      (
+        os.rel / "input",
+        s"""1
+           |2
+           |""".stripMargin,
+        StandardCharsets.UTF_16
+      )
+    )
+    TestInputs().fromRoot { root =>
+      val zipArchivePath = root / "hello.zip"
+      compress(zipArchivePath, zipInputs)
+
+      val output = os.proc(TestUtil.cli, extraOptions, zipArchivePath.toString)
+        .call(cwd = root)
+        .out.trim()
+
+      val expectedOutput = "1\n2"
+
+      expect(output == expectedOutput)
+    }
+  }
 
   test("Zip with Scala containing resource directive") {
     val inputs = TestInputs(