Espresso support with most recent Enso@GraalVM

.vscode/settings.json

@@ -5,5 +5,8 @@
   "vue.complete.casing.tags": "pascal",
   "auto-snippets.snippets": [
     { "language": "vue", "snippet": "Vue single-file component" }
-  ]
+  ],
+  "files.watcherExclude": {
+    "**/target": true
+  }
 }

CHANGELOG.md

@@ -923,6 +923,7 @@
 - [Warning.get_all returns only unique warnings][6372]
 - [Reimplement `enso_project` as a proper builtin][6352]
 - [Limit number of reported warnings per value][6577]
+- [Experimental support for Espresso Java interpreter][6966]
 - [Suggestions are updated only when the type of the expression changes][6755]
 - [Add project creation time to project metadata][6780]
 - [Upgrade GraalVM to 22.3.1 JDK17][6750]
@@ -1064,6 +1065,7 @@
 [6372]: https://github.com/enso-org/enso/pull/6372
 [6352]: https://github.com/enso-org/enso/pull/6352
 [6577]: https://github.com/enso-org/enso/pull/6577
+[6966]: https://github.com/enso-org/enso/pull/6966
 [6750]: https://github.com/enso-org/enso/pull/6750
 [6755]: https://github.com/enso-org/enso/pull/6755
 [6780]: https://github.com/enso-org/enso/pull/6780

build.sbt

@@ -1696,10 +1696,12 @@ lazy val `engine-runner` = project
         staticOnLinux = false,
         additionalOptions = Seq(
           "-Dorg.apache.commons.logging.Log=org.apache.commons.logging.impl.NoOpLog",
+          "-Dorg.graalvm.launcher.home=" + System.getProperty("java.home"),
           "-H:IncludeResources=.*Main.enso$",
           "--macro:truffle",
           "--language:js",
-          //          "-g",
+          "--language:java",
+          "-g",
           //          "-H:+DashboardAll",
           //          "-H:DashboardDump=runner.bgv"
           "-Dnic=nic"
@@ -1711,6 +1713,7 @@ lazy val `engine-runner` = project
           "io.methvin.watchservice.jna.CarbonAPI",
           "org.enso.syntax2.Parser",
           "zio.internal.ZScheduler$$anon$4",
+          "org.enso.runner.Main$",
           "sun.awt",
           "sun.java2d",
           "sun.font",

docs/infrastructure/native-image.md

@@ -201,9 +201,7 @@ safely.
 ### Engine runner Configuration
 
 The Native Image generation for the Engine Runner is currently in a preview
-state. Limitations are currently mostly due to
-[Java interop](https://www.pivotaltracker.com/story/show/183260380) and loading
-of stdlib components. To generate the Native Image for runner simply execute
+state. To generate the Native Image for runner simply execute
 
 ```
 sbt> engine-runner/buildNativeImage
@@ -217,4 +215,27 @@ and execute the binary on a sample factorial test program
 
 The task that generates the Native Image, along with all the necessary
 configuration, reside in a separate project due to a bug in the currently used
-GraalVM version.
+GraalVM version. As September 2023 it can execute all Enso code, but cannot invoke `IO.println`
+or other library functions that require [polyglot java import](../../docs/polyglot/java.md),
+but read on...
+
+### Engine with Espresso
+
+Since [PR-6966](https://github.com/enso-org/enso/pull/6966) there is an experimental
+support for including [Espresso Java interpreter](https://www.graalvm.org/jdk17/reference-manual/java-on-truffle/)
+to allow use of some library functions (like `IO.println`) in the _Native Image_ built
+runner.
+
+The support can be enabled by setting environment variable `ENSO_JAVA=espresso` and
+making sure Espresso is installed in GraalVM executing the Enso engine - e.g. by
+running `graalvm/bin/gu install espresso`. Then execute:
+
+```bash
+$ cat >hello.enso
+import Standard.Base.IO
+
+main = IO.println <| "Hello World!"
+$ ENSO_JAVA=espresso ./enso-x.y.z-dev/bin/enso --run hello.enso
+```
+Unless you see a warning containing _"No language for id java found."_ your code
+has just successfully been executed by [Espresso](https://www.graalvm.org/jdk17/reference-manual/java-on-truffle/)!

engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala

@@ -291,6 +291,9 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
     .allowAllAccess(true)
     .allowHostAccess(new HostAccessFactory().allWithTypeMapping())
     .allowExperimentalOptions(true)
+    .option("java.ExposeNativeJavaVM", "true")
+    .option("java.Polyglot", "true")
+    .option("java.UseBindingsLoader", "true")
     .option(RuntimeServerInfo.ENABLE_OPTION, "true")
     .option(RuntimeOptions.INTERACTIVE_MODE, "true")
     .option(RuntimeOptions.PROJECT_ROOT, serverConfig.contentRootPath)

engine/runner/src/main/scala/org/enso/runner/ContextFactory.scala

@@ -7,6 +7,7 @@ import org.enso.polyglot.debugger.{
   DebuggerSessionManagerEndpoint
 }
 import org.enso.polyglot.{HostAccessFactory, PolyglotContext, RuntimeOptions}
+import org.graalvm.polyglot.Engine
 import org.graalvm.polyglot.Context
 import org.slf4j.event.Level
 
@@ -51,12 +52,20 @@ class ContextFactory {
     executionEnvironment.foreach { name =>
       options.put("enso.ExecutionEnvironment", name)
     }
+    var javaHome = System.getenv("JAVA_HOME");
+    if (javaHome == null) {
+      javaHome = System.getProperty("java.home");
+    }
+    if (javaHome == null) {
+      throw new IllegalStateException("Specify JAVA_HOME environment property");
+    }
     val logLevelName = Converter.toJavaLevel(logLevel).getName
     val builder = Context
       .newBuilder()
       .allowExperimentalOptions(true)
       .allowAllAccess(true)
-      .allowHostAccess(new HostAccessFactory().allWithTypeMapping())
+      .allowHostAccess(new HostAccessFactory()
+      .allWithTypeMapping())
       .option(RuntimeOptions.PROJECT_ROOT, projectRoot)
       .option(RuntimeOptions.STRICT_ERRORS, strictErrors.toString)
       .option(RuntimeOptions.WAIT_FOR_PENDING_SERIALIZATION_JOBS, "true")
@@ -95,10 +104,17 @@ class ContextFactory {
         "bin"
       ),
       "graalpy"
-    );
+    )
     if (graalpy.exists()) {
       builder.option("python.Executable", graalpy.getAbsolutePath());
     }
+    if (Engine.create().getLanguages().containsKey("java")) {
+      builder
+        .option("java.ExposeNativeJavaVM", "true")
+        .option("java.Polyglot", "true")
+        .option("java.UseBindingsLoader", "true")
+        .option("java.JavaHome", javaHome)
+    }
     new PolyglotContext(builder.build)
   }
 }

engine/runtime/src/main/java/org/enso/interpreter/node/ExpressionNode.java

@@ -79,10 +79,19 @@ public void setSourceLocation(int sourceStartIndex, int sourceLength) {
    */
   @Override
   public SourceSection getSourceSection() {
+    var bounds = getSourceSectionBounds();
+    return bounds == null ? null : EnsoRootNode.findSourceSection(getRootNode(), bounds[0], bounds[1]);
+  }
+
+  public int[] getSourceSectionBounds() {
     if (this instanceof ExpressionNodeWrapper wrapper) {
-      return wrapper.getDelegateNode().getSourceSection();
+      return wrapper.getDelegateNode().getSourceSectionBounds();
     } else {
-      return EnsoRootNode.findSourceSection(getRootNode(), sourceStartIndex, sourceLength);
+      if (sourceStartIndex == EnsoRootNode.NO_SOURCE && sourceLength == EnsoRootNode.NO_SOURCE) {
+        return null;
+      } else {
+        return new int[] { sourceStartIndex, sourceLength };
+      }
     }
   }
 

engine/runtime/src/main/java/org/enso/interpreter/node/callable/function/StatementNode.java

@@ -3,7 +3,6 @@
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.instrumentation.StandardTags;
 import com.oracle.truffle.api.instrumentation.Tag;
-import com.oracle.truffle.api.source.SourceSection;
 import org.enso.interpreter.node.ClosureRootNode;
 import org.enso.interpreter.node.ExpressionNode;
 import org.enso.interpreter.runtime.tag.AvoidIdInstrumentationTag;
@@ -28,13 +27,13 @@ static StatementNode wrap(ExpressionNode node) {
   }
 
   @Override
-  public SourceSection getSourceSection() {
-    return node.getSourceSection();
+  public int[] getSourceSectionBounds() {
+    return node.getSourceSectionBounds();
   }
 
   @Override
   public boolean isInstrumentable() {
-    return getSourceSection() != null && node.isInstrumentable();
+    return getSourceSectionBounds() != null && node.isInstrumentable();
   }
 
   @Override

engine/runtime/src/main/java/org/enso/interpreter/node/expression/builtin/interop/java/LookupClassNode.java

@@ -1,5 +1,6 @@
 package org.enso.interpreter.node.expression.builtin.interop.java;
 
+import com.oracle.truffle.api.CompilerDirectives;
 import com.oracle.truffle.api.dsl.Cached;
 import com.oracle.truffle.api.dsl.Specialization;
 import com.oracle.truffle.api.nodes.Node;
@@ -18,6 +19,7 @@ static LookupClassNode build() {
   }
 
   @Specialization
+  @CompilerDirectives.TruffleBoundary
   Object doExecute(Object name, @Cached("build()") ExpectStringNode expectStringNode) {
     return EnsoContext.get(this).lookupJavaClass(expectStringNode.execute(name));
   }

engine/runtime/src/main/java/org/enso/interpreter/node/expression/literal/PatchableLiteralNode.java

@@ -3,7 +3,6 @@
 import com.oracle.truffle.api.frame.VirtualFrame;
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.nodes.NodeInfo;
-import com.oracle.truffle.api.source.SourceSection;
 import java.util.function.Predicate;
 import org.enso.compiler.core.IR;
 import org.enso.compiler.core.ir.Expression;
@@ -51,8 +50,8 @@ public boolean test(Expression ir) {
   }
 
   @Override
-  public SourceSection getSourceSection() {
-    return node.getSourceSection();
+  public int[] getSourceSectionBounds() {
+    return node.getSourceSectionBounds();
   }
 
   @Override

engine/runtime/src/main/java/org/enso/interpreter/runtime/EnsoContext.java

@@ -13,6 +13,7 @@
 import java.util.UUID;
 import java.util.concurrent.ExecutorService;
 import java.util.concurrent.atomic.AtomicLong;
+import java.util.logging.Level;
 
 import org.enso.compiler.Compiler;
 import org.enso.compiler.PackageRepository;
@@ -53,6 +54,7 @@
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
 import com.oracle.truffle.api.nodes.Node;
 import com.oracle.truffle.api.object.Shape;
+import com.oracle.truffle.api.source.Source;
 
 import scala.jdk.javaapi.OptionConverters;
 
@@ -399,18 +401,52 @@ public Object lookupJavaClass(String className) {
       List<String> nestedClassPart =
           i < items.size() - 1 ? items.subList(i + 1, items.size()) : List.of();
       try {
-        Object hostSymbol = environment.lookupHostSymbol(pkgName + "." + curClassName);
+        Object hostSymbol;
+        if (findGuestJava() == null) {
+          hostSymbol = environment.lookupHostSymbol(pkgName + "." + curClassName);
+        } else {
+          hostSymbol = InteropLibrary.getUncached().readMember(findGuestJava(), pkgName + "." + curClassName);
+        }
         if (nestedClassPart.isEmpty()) {
           return hostSymbol;
         } else {
           return getNestedClass(hostSymbol, nestedClassPart);
         }
-      } catch (RuntimeException ignored) {
+      } catch (RuntimeException | UnsupportedMessageException | UnknownIdentifierException ex) {
+        logger.log(Level.WARNING, null, ex);
       }
     }
     return null;
   }
 
+  private Object guestJava = this;
+
+  @TruffleBoundary
+  private Object findGuestJava() throws IllegalStateException {
+    if (guestJava != this) {
+      return guestJava;
+    }
+    guestJava = null;
+    var envJava = System.getenv("ENSO_JAVA");
+    if ("espresso".equals(envJava)) {
+      var src = Source.newBuilder("java", "<Bindings>", "getbindings.java").build();
+      try {
+        guestJava = environment.parsePublic(src).call();
+      } catch (Exception ex) {
+        if (ex.getMessage().contains("No language for id java found.")) {
+          logger.log(Level.SEVERE, "Environment variable ENSO_JAVA=" + envJava + ", but " + ex.getMessage());
+          logger.log(Level.SEVERE, "Use " + System.getProperty("java.home") + "/bin/gu install espresso");
+          logger.log(Level.SEVERE, "Continuing in regular Java mode");
+        } else {
+          var ise = new IllegalStateException(ex.getMessage());
+          ise.setStackTrace(ex.getStackTrace());
+          throw ise;
+        }
+      }
+    }
+    return guestJava;
+  }
+
   /**
    * Finds the package the provided module belongs to.
    *

engine/runtime/src/main/java/org/enso/interpreter/runtime/scope/DebugLocalScope.java

@@ -6,6 +6,7 @@
 import com.oracle.truffle.api.interop.InteropLibrary;
 import com.oracle.truffle.api.interop.UnknownIdentifierException;
 import com.oracle.truffle.api.interop.UnsupportedMessageException;
+import com.oracle.truffle.api.library.CachedLibrary;
 import com.oracle.truffle.api.library.ExportLibrary;
 import com.oracle.truffle.api.library.ExportMessage;
 import com.oracle.truffle.api.source.SourceSection;
@@ -77,11 +78,13 @@ private DebugLocalScope(
             && this.bindingsByLevelsIdx < this.bindingsByLevels.size());
   }
 
+  @TruffleBoundary
   public static DebugLocalScope createFromFrame(EnsoRootNode rootNode, MaterializedFrame frame) {
     return new DebugLocalScope(
         rootNode, frame, gatherBindingsByLevels(rootNode.getLocalScope().flattenBindings()), 0);
   }
 
+  @TruffleBoundary
   private static DebugLocalScope createParent(DebugLocalScope childScope) {
     return new DebugLocalScope(
         childScope.rootNode,
@@ -137,13 +140,15 @@ boolean hasMembers() {
 
   /** Returns the members from the current local scope and all the parent scopes. */
   @ExportMessage
+  @TruffleBoundary
   ScopeMembers getMembers(boolean includeInternal) {
     List<String> members = new ArrayList<>();
     bindingsByLevels.stream().skip(bindingsByLevelsIdx).forEach(members::addAll);
     return new ScopeMembers(members);
   }
 
   @ExportMessage
+  @TruffleBoundary
   boolean isMemberModifiable(String memberName) {
     return allBindings.containsKey(memberName);
   }
@@ -170,6 +175,7 @@ boolean hasMemberWriteSideEffects(String member) {
   }
 
   @ExportMessage
+  @TruffleBoundary
   boolean isMemberReadable(String memberName) {
     // When a value in a frame is null, it means that the corresponding
     // AssignmentNode was not run yet, and the slot kind of the
@@ -179,13 +185,15 @@ boolean isMemberReadable(String memberName) {
   }
 
   @ExportMessage
-  Object readMember(String member) {
+  @TruffleBoundary
+  Object readMember(String member, @CachedLibrary("this") InteropLibrary interop) {
     FramePointer framePtr = allBindings.get(member);
     var value = getValue(frame, framePtr);
     return value != null ? value : DataflowError.UNINITIALIZED;
   }
 
   @ExportMessage
+  @TruffleBoundary
   void writeMember(String member, Object value) throws UnknownIdentifierException {
     if (!allBindings.containsKey(member)) {
       throw UnknownIdentifierException.create(member);
@@ -225,6 +233,7 @@ boolean hasSourceLocation() {
   }
 
   @ExportMessage
+  @TruffleBoundary
   SourceSection getSourceLocation() {
     return rootNode.getSourceSection();
   }
@@ -236,6 +245,7 @@ String toDisplayString(boolean allowSideEffects) {
   }
 
   @Override
+  @TruffleBoundary
   public String toString() {
     return String.format(
         "DebugLocalScope{rootNode = '%s', bindingsByLevels = %s, idx = %d}",