diff --git a/.gitattributes b/.gitattributes index d9d8b125c878..a4689a686da3 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,15 +1,16 @@ # These files are text and should be normalized (convert crlf => lf) -*.c text -*.check text -*.css text -*.html text -*.java text -*.js text -*.sbt text -*.scala text -*.sh text -*.txt text -*.xml text + +*.c text eol=lf +*.check text eol=lf +*.css text eol=lf +*.html text eol=lf +*.java text eol=lf +*.js text eol=lf +*.sbt text eol=lf +*.scala text eol=lf +*.sh text eol=lf +*.txt text eol=lf +*.xml text eol=lf # Windows-specific files get windows endings *.bat eol=crlf diff --git a/dottydoc/test/BaseTest.scala b/dottydoc/test/BaseTest.scala index 2233d03c8b53..808387a44aa5 100644 --- a/dottydoc/test/BaseTest.scala +++ b/dottydoc/test/BaseTest.scala @@ -12,7 +12,7 @@ import model.Package trait DottyTest { dotty.tools.dotc.parsing.Scanners // initialize keywords - implicit var ctx: FreshContext = { + implicit val ctx: FreshContext = { val base = new ContextBase import base.settings._ val ctx = base.initialCtx.fresh diff --git a/project/Build.scala b/project/Build.scala index 1412556a9732..88ee7dca749b 100644 --- a/project/Build.scala +++ b/project/Build.scala @@ -40,7 +40,7 @@ object DottyBuild extends Build { homepage in Global := Some(url("https://github.com/lampepfl/dotty")), // scalac options - scalacOptions in Global ++= Seq("-feature", "-deprecation", "-language:existentials,higherKinds,implicitConversions"), + scalacOptions in Global ++= Seq("-feature", "-deprecation", "-encoding", "UTF8", "-language:existentials,higherKinds,implicitConversions"), javacOptions in Global ++= Seq("-Xlint:unchecked", "-Xlint:deprecation") ) diff --git a/src/dotty/tools/dotc/Run.scala b/src/dotty/tools/dotc/Run.scala index 1a26748bff8f..fa69530b50aa 100644 --- a/src/dotty/tools/dotc/Run.scala +++ b/src/dotty/tools/dotc/Run.scala @@ -9,6 +9,7 @@ import Phases._ import Decorators._ import dotty.tools.dotc.transform.TreeTransforms.TreeTransformer import io.PlainFile +import scala.io.Codec import util._ import reporting.Reporter import transform.TreeChecker @@ -28,8 +29,9 @@ class Run(comp: Compiler)(implicit ctx: Context) { var units: List[CompilationUnit] = _ def getSource(fileName: String): SourceFile = { + val encoding = ctx.settings.encoding.value val f = new PlainFile(fileName) - if (f.exists) new SourceFile(f) + if (f.exists) new SourceFile(f, Codec(encoding)) else { ctx.error(s"not found: $fileName") NoSource @@ -113,7 +115,7 @@ class Run(comp: Compiler)(implicit ctx: Context) { val writer = new BufferedWriter(new OutputStreamWriter(virtualFile.output, "UTF-8")) // buffering is still advised by javadoc writer.write(sourceCode) writer.close() - compileSources(List(new SourceFile(virtualFile))) + compileSources(List(new SourceFile(virtualFile, Codec.UTF8))) } /** The context created for this run */ diff --git a/src/dotty/tools/dotc/config/Settings.scala b/src/dotty/tools/dotc/config/Settings.scala index 73df4e1ecafa..cffa047fee16 100644 --- a/src/dotty/tools/dotc/config/Settings.scala +++ b/src/dotty/tools/dotc/config/Settings.scala @@ -25,6 +25,8 @@ object Settings { private var values = ArrayBuffer(initialValues: _*) private var _wasRead: Boolean = false + override def toString = s"SettingsState(values: ${values.toList})" + def value(idx: Int): Any = { _wasRead = true values(idx) diff --git a/src/dotty/tools/dotc/util/SourceFile.scala b/src/dotty/tools/dotc/util/SourceFile.scala index 6b547203e1f9..344bc253a231 100644 --- a/src/dotty/tools/dotc/util/SourceFile.scala +++ b/src/dotty/tools/dotc/util/SourceFile.scala @@ -10,6 +10,7 @@ import java.io.IOException import Chars._ import ScriptSourceFile._ import Positions._ +import scala.io.Codec import java.util.Optional @@ -36,9 +37,9 @@ object ScriptSourceFile { case class SourceFile(file: AbstractFile, content: Array[Char]) extends interfaces.SourceFile { - def this(_file: AbstractFile) = this(_file, _file.toCharArray) - def this(sourceName: String, cs: Seq[Char]) = this(new VirtualFile(sourceName), cs.toArray) - def this(file: AbstractFile, cs: Seq[Char]) = this(file, cs.toArray) + def this(_file: AbstractFile, codec: Codec) = this(_file, new String(_file.toByteArray, codec.charSet).toCharArray) + def this(sourceName: String, cs: Seq[Char]) = this(new VirtualFile(sourceName), cs.toArray) + def this(file: AbstractFile, cs: Seq[Char]) = this(file, cs.toArray) /** Tab increment; can be overridden */ def tabInc = 8 diff --git a/test/dotc/tests.scala b/test/dotc/tests.scala index 9f95a30c17c3..c4d8085f5ff3 100644 --- a/test/dotc/tests.scala +++ b/test/dotc/tests.scala @@ -23,8 +23,7 @@ class tests extends CompilerTest { val defaultOutputDir = "./out/" implicit val defaultOptions = noCheckOptions ++ List( - "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases", - "-d", defaultOutputDir) ++ { + "-Yno-deep-subtypes", "-Yno-double-bindings", "-Yforce-sbt-phases", "-d", defaultOutputDir) ++ { if (isRunByJenkins) List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") // should be Ycheck:all, but #725 else List("-Ycheck:tailrec,resolveSuper,mixin,restoreScopes,labelDef") } @@ -38,6 +37,9 @@ class tests extends CompilerTest { val allowDoubleBindings = defaultOptions diff List("-Yno-double-bindings") val scala2mode = List("-language:Scala2") + val explicitUTF8 = List("-encoding", "UTF8") + val explicitUTF16 = List("-encoding", "UTF16") + val testsDir = "./tests/" val posDir = testsDir + "pos/" val posSpecialDir = testsDir + "pos-special/" @@ -95,7 +97,7 @@ class tests extends CompilerTest { @Test def pos_overloadedAccess = compileFile(posDir, "overloadedAccess", twice) @Test def pos_approximateUnion = compileFile(posDir, "approximateUnion", twice) @Test def pos_tailcall = compileDir(posDir, "tailcall", twice) - @Test def pos_valueclasses = compileFiles(posDir + "valueclasses/", twice) + @Test def pos_valueclasses = compileFiles(posDir + "pos_valueclasses/", twice) @Test def pos_nullarify = compileFile(posDir, "nullarify", args = "-Ycheck:nullarify" :: Nil) @Test def pos_subtyping = compileFile(posDir, "subtyping", twice) @Test def pos_packageObj = compileFile(posDir, "i0239", twice) @@ -118,6 +120,8 @@ class tests extends CompilerTest { compileFile(posSpecialDir, "spec-t5545/S_1") compileFile(posSpecialDir, "spec-t5545/S_2") } + @Test def pos_utf8 = compileFile(posSpecialDir, "utf8encoded", explicitUTF8) + @Test def pos_utf16 = compileFile(posSpecialDir, "utf16encoded", explicitUTF16) @Test def new_all = compileFiles(newDir, twice) @Test def repl_all = replFiles(replDir) diff --git a/test/test/CompilerTest.scala b/test/test/CompilerTest.scala index 56b9e1099e3f..dea6a30b1432 100644 --- a/test/test/CompilerTest.scala +++ b/test/test/CompilerTest.scala @@ -228,7 +228,7 @@ abstract class CompilerTest { private def expectedErrors(filePath: String): List[ErrorsInFile] = expectedErrors(List(filePath)) - private def isNegTest(testPath: String) = testPath.contains(JFile.separator + "neg" + JFile.separator) + private def isNegTest(testPath: String) = testPath.contains("/neg/") private def compileArgs(args: Array[String], expectedErrorsPerFile: List[ErrorsInFile]) (implicit defaultOptions: List[String]): Unit = { @@ -413,7 +413,8 @@ abstract class CompilerTest { val flags = oldFlags.map(f => if (f == oldOutput) partestOutput else f) ++ List(s"-classpath $partestOutput") // Required for separate compilation tests - getExisting(dest).isDifferent(source, flags, nerr) match { + val difference = getExisting(dest).isDifferent(source, flags, nerr) + difference match { case NotExists => copyFiles(source, dest, partestOutput, flags, nerr, kind) case ExistsSame => // nothing else to do case ExistsDifferent => @@ -449,11 +450,37 @@ abstract class CompilerTest { /** Recursively copy over source files and directories, excluding extensions * that aren't in extensionsToCopy. */ private def recCopyFiles(sourceFile: Path, dest: Path): Unit = { - processFileDir(sourceFile, { sf => + + def copyfile(file: SFile, bytewise: Boolean): Unit = { + if (bytewise) { + val in = file.inputStream() + val out = SFile(dest).outputStream() + val buffer = new Array[Byte](1024) + def loop(available: Int):Unit = { + if (available < 0) {()} + else { + out.write(buffer, 0, available) + val read = in.read(buffer) + loop(read) + } + } + loop(0) + in.close() + out.close() + } else { + try { + SFile(dest)(scala.io.Codec.UTF8).writeAll((s"/* !!!!! WARNING: DO NOT MODIFY. Original is at: $file !!!!! */").replace("\\", "/"), file.slurp("UTF-8")) + } catch { + case unmappable: java.nio.charset.MalformedInputException => + copyfile(file, true) //there are bytes that can't be mapped with UTF-8. Bail and just do a straight byte-wise copy without the warning header. + } + } + } + + processFileDir(sourceFile, { sf => if (extensionsToCopy.contains(sf.extension)) { dest.parent.jfile.mkdirs - dest.toFile.writeAll("/* !!!!! WARNING: DO NOT MODIFY. Original is at: $sf !!!!! */", - sf.slurp()) + copyfile(sf, false) } else { log(s"WARNING: ignoring $sf") } @@ -465,7 +492,7 @@ abstract class CompilerTest { /** Reads the existing files for the given test source if any. */ private def getExisting(dest: Path): ExistingFiles = { - val content: Option[Option[String]] = processFileDir(dest, f => f.safeSlurp, d => Some("")) + val content: Option[Option[String]] = processFileDir(dest, f => try Some(f.slurp("UTF8")) catch {case io: java.io.IOException => Some(io.toString())}, d => Some("")) if (content.isDefined && content.get.isDefined) { val flags = (dest changeExtension "flags").toFile.safeSlurp val nerr = (dest changeExtension "nerr").toFile.safeSlurp @@ -479,7 +506,7 @@ abstract class CompilerTest { if (!genSrc.isDefined) { NotExists } else { - val source = processFileDir(sourceFile, { f => f.safeSlurp }, { d => Some("") }, + val source = processFileDir(sourceFile, { f => try Some(f.slurp("UTF8")) catch {case _: java.io.IOException => None} }, { d => Some("") }, Some("DPCompilerTest sourceFile doesn't exist: " + sourceFile)).get if (source == genSrc) { nerr match { diff --git a/test/test/DottyTest.scala b/test/test/DottyTest.scala index 15d82c208b79..4b767b318bdd 100644 --- a/test/test/DottyTest.scala +++ b/test/test/DottyTest.scala @@ -23,6 +23,7 @@ class DottyTest /*extends ContextEscapeDetection*/ { import base.settings._ val ctx = base.initialCtx.fresh base.initialize()(ctx) + ctx.setSetting(ctx.settings.encoding, "UTF8") ctx } /* diff --git a/test/test/InterfaceEntryPointTest.scala b/test/test/InterfaceEntryPointTest.scala index b193b5f647b5..438a9fa4771a 100644 --- a/test/test/InterfaceEntryPointTest.scala +++ b/test/test/InterfaceEntryPointTest.scala @@ -18,7 +18,7 @@ import scala.collection.mutable.ListBuffer */ class InterfaceEntryPointTest { @Test def runCompilerFromInterface = { - val sources = List("./tests/pos/HelloWorld.scala") + val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath()) val args = sources ++ List("-d", "./out/") val mainClass = Class.forName("dotty.tools.dotc.Main") diff --git a/test/test/OtherEntryPointsTest.scala b/test/test/OtherEntryPointsTest.scala index 0186b357b431..5f8681d3864c 100644 --- a/test/test/OtherEntryPointsTest.scala +++ b/test/test/OtherEntryPointsTest.scala @@ -17,7 +17,7 @@ import scala.collection.mutable.ListBuffer */ class OtherEntryPointsTest { @Test def runCompiler = { - val sources = List("./tests/pos/HelloWorld.scala") + val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath()) val args = sources ++ List("-d", "./out/") val reporter = new CustomReporter @@ -31,7 +31,7 @@ class OtherEntryPointsTest { } @Test def runCompilerWithContext = { - val sources = List("./tests/pos/HelloWorld.scala") + val sources = List("./tests/pos/HelloWorld.scala").map(p => new java.io.File(p).getPath()) val args = sources ++ List("-d", "./out/") val reporter = new CustomReporter diff --git a/test/test/ParserTest.scala b/test/test/ParserTest.scala index 524be272d6ba..f66dbf55d21a 100644 --- a/test/test/ParserTest.scala +++ b/test/test/ParserTest.scala @@ -4,6 +4,7 @@ import scala.reflect.io._ import dotty.tools.dotc.util._ import dotty.tools.dotc.core._ import dotty.tools.dotc.parsing._ +import scala.io.Codec import Tokens._, Parsers._ import dotty.tools.dotc.ast.untpd._ import org.junit.Test @@ -23,7 +24,7 @@ class ParserTest extends DottyTest { def parse(file: PlainFile): Tree = { //println("***** parsing " + file) - val source = new SourceFile(file) + val source = new SourceFile(file, Codec.UTF8) val parser = new Parser(source) val tree = parser.parse() parsed += 1 diff --git a/test/test/ScannerTest.scala b/test/test/ScannerTest.scala index f8f09ff6fb32..5ff9bba0cc38 100644 --- a/test/test/ScannerTest.scala +++ b/test/test/ScannerTest.scala @@ -1,6 +1,7 @@ package test import scala.reflect.io._ +import scala.io.Codec import dotty.tools.dotc.util._ import dotty.tools.dotc.parsing._ import Tokens._, Scanners._ @@ -16,8 +17,8 @@ class ScannerTest extends DottyTest { def scan(name: String): Unit = scan(new PlainFile(name)) def scan(file: PlainFile): Unit = { - println("***** scanning " + file) - val source = new SourceFile(file) + //println("***** scanning " + file) + val source = new SourceFile(file, Codec.UTF8) val scanner = new Scanner(source) var i = 0 while (scanner.token != EOF) { diff --git a/test/test/TestREPL.scala b/test/test/TestREPL.scala index 1ba456ce2f47..9867cb4ecf20 100644 --- a/test/test/TestREPL.scala +++ b/test/test/TestREPL.scala @@ -34,7 +34,7 @@ class TestREPL(script: String) extends REPL { while (lines.hasNext && lines.head.startsWith(continuationPrompt)) { val continued = lines.next output.println(continued) - buf append "\n" + buf append System.lineSeparator() buf append continued.drop(continuationPrompt.length) } buf.toString @@ -49,7 +49,7 @@ class TestREPL(script: String) extends REPL { out.close() val printed = out.toString val transcript = printed.drop(printed.indexOf(config.prompt)) - if (transcript.toString != script) { + if (transcript.toString.lines.toList != script.lines.toList) { println("input differs from transcript:") println(transcript) assert(false) diff --git a/tests/pos-special/utf16encoded.scala b/tests/pos-special/utf16encoded.scala new file mode 100644 index 000000000000..d77529f19f22 Binary files /dev/null and b/tests/pos-special/utf16encoded.scala differ diff --git a/tests/pos-special/utf8encoded.scala b/tests/pos-special/utf8encoded.scala new file mode 100644 index 000000000000..6eac24acd7df --- /dev/null +++ b/tests/pos-special/utf8encoded.scala @@ -0,0 +1,8 @@ +//this file is saved as UTF-8 +object Test { + def main(args: Array[String]): Unit = { + val testchar = '⇒' + println(testchar == '\u21D2') + } + +} \ No newline at end of file diff --git a/tests/pos/valueclasses/nullAsInstanceOfVC.scala b/tests/pos/pos_valueclasses/nullAsInstanceOfVC.scala similarity index 100% rename from tests/pos/valueclasses/nullAsInstanceOfVC.scala rename to tests/pos/pos_valueclasses/nullAsInstanceOfVC.scala diff --git a/tests/pos/valueclasses/optmatch.scala b/tests/pos/pos_valueclasses/optmatch.scala similarity index 100% rename from tests/pos/valueclasses/optmatch.scala rename to tests/pos/pos_valueclasses/optmatch.scala diff --git a/tests/pos/valueclasses/paramlists.scala b/tests/pos/pos_valueclasses/paramlists.scala similarity index 100% rename from tests/pos/valueclasses/paramlists.scala rename to tests/pos/pos_valueclasses/paramlists.scala diff --git a/tests/pos/valueclasses/privatethisparam.scala b/tests/pos/pos_valueclasses/privatethisparam.scala similarity index 100% rename from tests/pos/valueclasses/privatethisparam.scala rename to tests/pos/pos_valueclasses/privatethisparam.scala diff --git a/tests/pos/valueclasses/t5667.scala b/tests/pos/pos_valueclasses/t5667.scala similarity index 100% rename from tests/pos/valueclasses/t5667.scala rename to tests/pos/pos_valueclasses/t5667.scala diff --git a/tests/pos/valueclasses/t5853.scala b/tests/pos/pos_valueclasses/t5853.scala similarity index 100% rename from tests/pos/valueclasses/t5853.scala rename to tests/pos/pos_valueclasses/t5853.scala diff --git a/tests/pos/valueclasses/t5953.scala b/tests/pos/pos_valueclasses/t5953.scala similarity index 100% rename from tests/pos/valueclasses/t5953.scala rename to tests/pos/pos_valueclasses/t5953.scala diff --git a/tests/pos/valueclasses/t6029.scala b/tests/pos/pos_valueclasses/t6029.scala similarity index 100% rename from tests/pos/valueclasses/t6029.scala rename to tests/pos/pos_valueclasses/t6029.scala diff --git a/tests/pos/valueclasses/t6034.scala b/tests/pos/pos_valueclasses/t6034.scala similarity index 100% rename from tests/pos/valueclasses/t6034.scala rename to tests/pos/pos_valueclasses/t6034.scala diff --git a/tests/pos/valueclasses/t6215.scala b/tests/pos/pos_valueclasses/t6215.scala similarity index 100% rename from tests/pos/valueclasses/t6215.scala rename to tests/pos/pos_valueclasses/t6215.scala diff --git a/tests/pos/valueclasses/t6260.scala b/tests/pos/pos_valueclasses/t6260.scala similarity index 100% rename from tests/pos/valueclasses/t6260.scala rename to tests/pos/pos_valueclasses/t6260.scala diff --git a/tests/pos/valueclasses/t6260a.scala b/tests/pos/pos_valueclasses/t6260a.scala similarity index 100% rename from tests/pos/valueclasses/t6260a.scala rename to tests/pos/pos_valueclasses/t6260a.scala diff --git a/tests/pos/valueclasses/t6260b.scala b/tests/pos/pos_valueclasses/t6260b.scala similarity index 100% rename from tests/pos/valueclasses/t6260b.scala rename to tests/pos/pos_valueclasses/t6260b.scala diff --git a/tests/pos/valueclasses/t6358.scala b/tests/pos/pos_valueclasses/t6358.scala similarity index 100% rename from tests/pos/valueclasses/t6358.scala rename to tests/pos/pos_valueclasses/t6358.scala diff --git a/tests/pos/valueclasses/t6358_2.scala b/tests/pos/pos_valueclasses/t6358_2.scala similarity index 100% rename from tests/pos/valueclasses/t6358_2.scala rename to tests/pos/pos_valueclasses/t6358_2.scala diff --git a/tests/pos/valueclasses/t6601/PrivateValueClass_1.scala b/tests/pos/pos_valueclasses/t6601/PrivateValueClass_1.scala similarity index 100% rename from tests/pos/valueclasses/t6601/PrivateValueClass_1.scala rename to tests/pos/pos_valueclasses/t6601/PrivateValueClass_1.scala diff --git a/tests/pos/valueclasses/t6601/UsePrivateValueClass_2.scala b/tests/pos/pos_valueclasses/t6601/UsePrivateValueClass_2.scala similarity index 100% rename from tests/pos/valueclasses/t6601/UsePrivateValueClass_2.scala rename to tests/pos/pos_valueclasses/t6601/UsePrivateValueClass_2.scala diff --git a/tests/pos/valueclasses/t6651.scala b/tests/pos/pos_valueclasses/t6651.scala similarity index 100% rename from tests/pos/valueclasses/t6651.scala rename to tests/pos/pos_valueclasses/t6651.scala diff --git a/tests/pos/valueclasses/t7818.scala b/tests/pos/pos_valueclasses/t7818.scala similarity index 100% rename from tests/pos/valueclasses/t7818.scala rename to tests/pos/pos_valueclasses/t7818.scala diff --git a/tests/pos/valueclasses/t8011.scala b/tests/pos/pos_valueclasses/t8011.scala similarity index 100% rename from tests/pos/valueclasses/t8011.scala rename to tests/pos/pos_valueclasses/t8011.scala diff --git a/tests/pos/valueclasses/t9298/JUse.java b/tests/pos/pos_valueclasses/t9298/JUse.java similarity index 100% rename from tests/pos/valueclasses/t9298/JUse.java rename to tests/pos/pos_valueclasses/t9298/JUse.java diff --git a/tests/pos/valueclasses/t9298/Meter.scala b/tests/pos/pos_valueclasses/t9298/Meter.scala similarity index 100% rename from tests/pos/valueclasses/t9298/Meter.scala rename to tests/pos/pos_valueclasses/t9298/Meter.scala diff --git a/tests/pos/valueclasses/t9298/Use.scala b/tests/pos/pos_valueclasses/t9298/Use.scala similarity index 100% rename from tests/pos/valueclasses/t9298/Use.scala rename to tests/pos/pos_valueclasses/t9298/Use.scala diff --git a/tests/pos/valueclasses/value-class-override-no-spec.flags b/tests/pos/pos_valueclasses/value-class-override-no-spec.flags similarity index 100% rename from tests/pos/valueclasses/value-class-override-no-spec.flags rename to tests/pos/pos_valueclasses/value-class-override-no-spec.flags diff --git a/tests/pos/valueclasses/value-class-override-no-spec.scala b/tests/pos/pos_valueclasses/value-class-override-no-spec.scala similarity index 100% rename from tests/pos/valueclasses/value-class-override-no-spec.scala rename to tests/pos/pos_valueclasses/value-class-override-no-spec.scala diff --git a/tests/pos/valueclasses/value-class-override-spec.scala b/tests/pos/pos_valueclasses/value-class-override-spec.scala similarity index 100% rename from tests/pos/valueclasses/value-class-override-spec.scala rename to tests/pos/pos_valueclasses/value-class-override-spec.scala diff --git a/tests/pos/valueclasses/xlint1.flags b/tests/pos/pos_valueclasses/xlint1.flags similarity index 100% rename from tests/pos/valueclasses/xlint1.flags rename to tests/pos/pos_valueclasses/xlint1.flags diff --git a/tests/pos/valueclasses/xlint1.scala b/tests/pos/pos_valueclasses/xlint1.scala similarity index 100% rename from tests/pos/valueclasses/xlint1.scala rename to tests/pos/pos_valueclasses/xlint1.scala