Use Rust Parser from Java instead of AST.scala

CHANGELOG.md

@@ -400,6 +400,7 @@
 - [Explicit `self`][3569]
 - [Added benchmarking tool for the language server][3578]
 - [Support module imports using a qualified name][3608]
+- [Using parser written in Rust.][3611]
 - [Enable caching in visualisation functions][3618]
 - [Update Scala compiler and libraries][3631]
 - [Support importing module methods][3633]
@@ -460,6 +461,7 @@
 [3538]: https://github.com/enso-org/enso/pull/3538
 [3569]: https://github.com/enso-org/enso/pull/3569
 [3578]: https://github.com/enso-org/enso/pull/3578
+[3611]: https://github.com/enso-org/enso/pull/3611
 [3618]: https://github.com/enso-org/enso/pull/3618
 [3608]: https://github.com/enso-org/enso/pull/3608
 [3608]: https://github.com/enso-org/enso/pull/3608

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeServerTest.scala

@@ -1269,7 +1269,7 @@ class RuntimeServerTest
     val contextId  = UUID.randomUUID()
     val requestId  = UUID.randomUUID()
     val moduleName = "Enso_Test.Test.Main"
-    val idMain     = context.Main.metadata.addItem(54, 47, "aaaaa")
+    val idMain     = context.Main.metadata.addItem(54, 46, "aaaaa")
     val contents   = context.Main.code
     val mainFile   = context.writeMain(contents)
 
@@ -1442,7 +1442,7 @@ class RuntimeServerTest
     val moduleName = "Enso_Test.Test.Main"
 
     val metadata = new Metadata
-    val idMain   = metadata.addItem(77, 35, "aaaa")
+    val idMain   = metadata.addItem(77, 34, "aaaa")
     val idMainA  = metadata.addItem(86, 8, "aabb")
     val idMainP  = metadata.addItem(99, 12, "aacc")
     // pie id
@@ -1726,7 +1726,7 @@ class RuntimeServerTest
     val moduleName = "Enso_Test.Test.Main"
 
     val metadata = new Metadata
-    val idMain   = metadata.addItem(122, 88, "aaaa")
+    val idMain   = metadata.addItem(122, 87, "aaaa")
     val id1      = metadata.addItem(131, 15, "aad1")
     val id2      = metadata.addItem(151, 18, "aad2")
     val id3      = metadata.addItem(174, 15, "aad3")
@@ -2147,7 +2147,7 @@ class RuntimeServerTest
     val contextId  = UUID.randomUUID()
     val requestId  = UUID.randomUUID()
     val moduleName = "Enso_Test.Test.Main"
-    val idMain     = context.Main.metadata.addItem(54, 47, "aaaa")
+    val idMain     = context.Main.metadata.addItem(54, 46, "aaaa")
 
     val mainFile = context.writeMain(context.Main.code)
 

engine/runtime-with-instruments/src/test/scala/org/enso/interpreter/test/instrument/RuntimeSuggestionUpdatesTest.scala

@@ -807,7 +807,7 @@ class RuntimeSuggestionUpdatesTest
                         ConstantsGen.ANY,
                         Suggestion.Scope(
                           Suggestion.Position(4, 6),
-                          Suggestion.Position(9, 0)
+                          Suggestion.Position(8, 11)
                         )
                       ),
                       Api.SuggestionAction.Add()

engine/runtime/src/bench/scala/org/enso/interpreter/bench/fixtures/semantic/AtomFixtures.scala

@@ -39,7 +39,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
   val reverseListCode =
     """from Standard.Base.Data.List import all
       |
-      |main = list ->
+      |main = self -> list ->
       |    reverser = acc -> list -> case list of
       |        Cons h t -> @Tail_Call reverser (Cons h acc) t
       |        Nil -> acc
@@ -56,7 +56,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
       |    Cons h t -> @Tail_Call t.rev (Cons h acc)
       |    _ -> acc
       |
-      |main = list ->
+      |main = self -> list ->
       |    res = list.rev Nil
       |    res
       |""".stripMargin
@@ -65,7 +65,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
   val sumListCode =
     """from Standard.Base.Data.List import all
       |
-      |main = list ->
+      |main = self -> list ->
       |    summator = acc -> list -> case list of
       |        Cons h t -> @Tail_Call summator acc+h t
       |        Nil -> acc
@@ -78,7 +78,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
   val sumListLeftFoldCode =
     """from Standard.Base.Data.List import all
       |
-      |main = list ->
+      |main = self -> list ->
       |    fold = f -> acc -> list -> case list of
       |        Cons h t -> @Tail_Call fold f (f acc h) t
       |        _ -> acc
@@ -91,7 +91,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
   val sumListFallbackCode =
     """from Standard.Base.Data.List import all
       |
-      |main = list ->
+      |main = self -> list ->
       |    summator = acc -> list -> case list of
       |        Cons h t -> @Tail_Call summator acc+h t
       |        _ -> acc
@@ -108,7 +108,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
       |    Cons h t -> @Tail_Call t.sum h+acc
       |    _ -> acc
       |
-      |main = list ->
+      |main = self -> list ->
       |    res = list.sum 0
       |    res
       |""".stripMargin
@@ -121,7 +121,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
       |    Cons h t -> @Tail_Call t.mapReverse f (Cons (f h) acc)
       |    _ -> acc
       |
-      |main = list ->
+      |main = self -> list ->
       |    res = list.mapReverse (x -> x + 1) Nil
       |    res
       |""".stripMargin
@@ -134,7 +134,7 @@ class AtomFixtures extends DefaultInterpreterRunner {
       |    Cons h t -> @Tail_Call t.mapReverse f (Cons (f h) acc)
       |    _ -> acc
       |
-      |main = list ->
+      |main = self -> list ->
       |    adder = x -> y -> x + y
       |    res = list.mapReverse (adder 1) Nil
       |    res

engine/runtime/src/main/java/org/enso/compiler/TreeToIr.java

@@ -678,7 +678,7 @@ yield switch (op.codeRepr()) {
           expressions.remove(expressions.size()-1);
         }
         var list = CollectionConverters.asScala(expressions.iterator()).toList();
-        var locationWithANewLine = getIdentifiedLocation(body, 0, 1, null);
+        var locationWithANewLine = getIdentifiedLocation(body, 0, 0, null);
         if (last != null && last.location().isDefined() && last.location().get().end() != locationWithANewLine.get().end()) {
             var patched = new Location(last.location().get().start(), locationWithANewLine.get().end() - 1);
             var id = new IdentifiedLocation(patched, last.location().get().id());
@@ -713,11 +713,10 @@ yield switch (op.codeRepr()) {
           List<IR.CallArgument> args = nil();
           for (var line : body.getArguments()) {
             var expr = line.getExpression();
-            if (expr != null) {
-              continue;
+            if (expr instanceof Tree.Ident) {
+                var call = translateCallArgument(expr);
+                args = cons(call, args);
             }
-            var call = translateCallArgument(expr);
-            args = cons(call, args);
           }
           yield switch (fn) {
             case IR$Application$Prefix pref -> patchPrefixWithBlock(pref, block, args);
@@ -958,18 +957,34 @@ IR.Literal translateLiteral(Tree.TextLiteral txt) {
     return new IR$Literal$Text(value, getIdentifiedLocation(txt), meta(), diag());
   }
   String buildTextConstant(Iterable elements, boolean stripComments) {
-    StringBuilder sb = new StringBuilder();
+    var sb = new StringBuilder();
     for (var t : elements) {
       switch (t) {
         case TextElement.Section s -> {
           var text = s.getText().codeRepr();
           if (stripComments) {
             // Reproduce an AstToIr bug for testing.
-            text = text.replaceAll("#[^\n\r]*", "");
+            var quotedSegments = text.split("\"", -1);
+            for (int i = 0; i < quotedSegments.length; i++) {
+              var seg = quotedSegments[i];
+              if (i % 2 == 0) {
+                sb.append(seg.replaceAll("#[^\n\r]*", ""));
+              } else {
+                sb.append('"');
+                sb.append(seg);
+                if (i + 1 < quotedSegments.length) {
+                  sb.append('"');
+                }
+              }
+            }
+          } else {
+            sb.append(text);
           }
-          sb.append(text);
         }
-        case TextElement.Escape e -> sb.appendCodePoint(e.getToken().getValue());
+        case TextElement.Escape e -> {
+          var val = e.getToken().getValue();
+          sb.appendCodePoint(val);
+        }
         default -> throw new UnhandledEntity(t, "buildTextConstant");
       }
     }

engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala

@@ -441,17 +441,19 @@ class Compiler(
 
     val src = module.getSource
     def oldParser() = {
+      System.err.println("Using old parser to process " + src.getURI())
       val tree = parse(src)
       generateIR(tree)
     }
     def newParser() = {
-      System.err.println("Using new parser to process " + src.getURI())
       val tree = ensoCompiler.parse(src)
       ensoCompiler.generateIR(tree)
     }
-    val size = src.getCharacters().length()
-    // change the condition to use old or new parser
-    val expr = if (size >= 0) oldParser() else newParser()
+    val expr = if ("scala".equals(System.getenv("ENSO_PARSER"))) {
+      oldParser()
+    } else {
+      newParser()
+    }
 
     val exprWithModuleExports =
       if (module.isSynthetic)