Arrow language

CHANGELOG.md

@@ -1014,6 +1014,7 @@
 - [Export of non-existing symbols results in error][7960]
 - [Upgrade GraalVM to 23.1.0 JDK21][7991]
 - [Added opt-in type checks of return type][8502]
+- [Introduce Arrow language][8512]
 - [DataflowError.withoutTrace doesn't store stacktrace][8608]
 
 [3227]: https://github.com/enso-org/enso/pull/3227
@@ -1166,6 +1167,7 @@
 [7960]: https://github.com/enso-org/enso/pull/7960
 [7991]: https://github.com/enso-org/enso/pull/7991
 [8502]: https://github.com/enso-org/enso/pull/8502
+[8512]: https://github.com/enso-org/enso/pull/8512
 [8608]: https://github.com/enso-org/enso/pull/8608
 
 # Enso 2.0.0-alpha.18 (2021-10-12)

build.sbt

@@ -293,6 +293,7 @@ lazy val enso = (project in file("."))
     `runtime-parser`,
     `runtime-compiler`,
     `runtime-language-epb`,
+    `runtime-language-arrow`,
     `runtime-instrument-common`,
     `runtime-instrument-id-execution`,
     `runtime-instrument-repl-debugger`,
@@ -496,6 +497,7 @@ val hamcrestVersion         = "1.3"
 val netbeansApiVersion      = "RELEASE180"
 val fansiVersion            = "0.4.0"
 val httpComponentsVersion   = "4.4.1"
+val apacheArrowVersion      = "14.0.1"
 
 // ============================================================================
 // === Utility methods =====================================================
@@ -1487,6 +1489,49 @@ lazy val `runtime-language-epb` =
       )
     )
 
+lazy val `runtime-language-arrow` =
+  (project in file("engine/runtime-language-arrow"))
+    .enablePlugins(JPMSPlugin)
+    .settings(
+      crossPaths := false,
+      autoScalaLibrary := false,
+      inConfig(Compile)(truffleRunOptionsSettings),
+      instrumentationSettings,
+      libraryDependencies ++= GraalVM.modules ++ Seq(
+        "junit"            % "junit"              % junitVersion       % Test,
+        "com.github.sbt"   % "junit-interface"    % junitIfVersion     % Test,
+        "org.slf4j"        % "slf4j-nop"          % slf4jVersion       % Test,
+        "org.slf4j"        % "slf4j-api"          % slf4jVersion       % Test,
+        "org.apache.arrow" % "arrow-vector"       % apacheArrowVersion % Test,
+        "org.apache.arrow" % "arrow-memory-netty" % apacheArrowVersion % Test
+      ),
+      modulePath := {
+        val updateReport = (Test / update).value
+        JPMSUtils.filterModulesFromUpdate(
+          updateReport,
+          GraalVM.modules,
+          streams.value.log,
+          shouldContainAll = true
+        ) ++ Seq(
+          (LocalProject(
+            "runtime-language-arrow"
+          ) / Compile / productDirectories).value.head
+        )
+      },
+      Test / patchModules := {
+        val testClassesDir = (Test / productDirectories).value.head
+        Map("org.enso.interpreter.arrow" -> Seq(testClassesDir))
+      },
+      Test / addModules := Seq("org.enso.interpreter.arrow"),
+      Test / javaOptions ++= Seq(
+        "--add-opens=java.base/java.nio=org.enso.interpreter.arrow", // DirectByteBuffer in MemoryUtil init is in-accessible
+        "--add-opens=java.base/java.nio=ALL-UNNAMED"                 // Tests use Apache Arrow
+      ),
+      Test / addReads := {
+        Map("org.enso.interpreter.arrow" -> Seq("ALL-UNNAMED"))
+      }
+    )
+
 /** `runtime-test-instruments` project contains Truffle instruments that are used solely for testing.
   * It is compiled into an explicit Java module. Note that this project cannot have compile-time dependency on `runtime`
   * project, so if you need access to classes from `runtime`, you need to use reflection.
@@ -2075,7 +2120,8 @@ lazy val `engine-runner` = project
             "com.sun.imageio",
             "com.sun.jna.internal.Cleaner",
             "com.sun.jna.Structure$FFIType",
-            "akka.http"
+            "akka.http",
+            "org.enso.interpreter.arrow.util.MemoryUtil"
           )
         )
         .dependsOn(assembly)

engine/runtime-language-arrow/src/main/java/module-info.java

@@ -0,0 +1,6 @@
+open module org.enso.interpreter.arrow {
+  requires org.graalvm.truffle;
+
+  provides com.oracle.truffle.api.provider.TruffleLanguageProvider with
+      org.enso.interpreter.arrow.ArrowLanguageProvider;
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/ArrowContext.java

@@ -0,0 +1,15 @@
+package org.enso.interpreter.arrow;
+
+import com.oracle.truffle.api.TruffleLanguage;
+
+final class ArrowContext {
+  private final TruffleLanguage.Env env;
+
+  public ArrowContext(TruffleLanguage.Env env) {
+    this.env = env;
+  }
+
+  public void initialize() {
+    //
+  }
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/ArrowLanguage.java

@@ -0,0 +1,48 @@
+package org.enso.interpreter.arrow;
+
+import com.oracle.truffle.api.CallTarget;
+import com.oracle.truffle.api.TruffleLanguage;
+import org.enso.interpreter.arrow.node.ArrowEvalNode;
+
+/** An internal language that implements Arrow specification. */
+@TruffleLanguage.Registration(
+    id = ArrowLanguage.ID,
+    name = "Truffle implementation of Arrow",
+    characterMimeTypes = {ArrowLanguage.MIME},
+    defaultMimeType = ArrowLanguage.MIME,
+    contextPolicy = TruffleLanguage.ContextPolicy.SHARED)
+public class ArrowLanguage extends TruffleLanguage<ArrowContext> {
+
+  public static final String ID = "arrow";
+  public static final String MIME = "application/vnd.apache.arrow.file";
+
+  public ArrowLanguage() {}
+
+  @Override
+  protected ArrowContext createContext(TruffleLanguage.Env env) {
+    var ctx = new ArrowContext(env);
+    return ctx;
+  }
+
+  @Override
+  protected void initializeContext(ArrowContext context) {
+    context.initialize();
+  }
+
+  @Override
+  protected CallTarget parse(ParsingRequest request) {
+    ArrowParser.Result code = ArrowParser.parse(request.getSource());
+    if (code != null) {
+      ArrowEvalNode node = ArrowEvalNode.create(this, code);
+      return node.getCallTarget();
+    } else {
+      throw new IllegalArgumentException(
+          "unable to parse the code: " + request.getSource().getCharacters().toString());
+    }
+  }
+
+  @Override
+  protected boolean isThreadAccessAllowed(Thread thread, boolean singleThreaded) {
+    return true;
+  }
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/ArrowParser.java

@@ -0,0 +1,69 @@
+package org.enso.interpreter.arrow;
+
+import com.oracle.truffle.api.source.Source;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+public final class ArrowParser {
+
+  private ArrowParser() {}
+
+  public static class Result {
+
+    private final PhysicalLayout physicalLayout;
+    private final LogicalLayout logicalLayout;
+    private final Mode mode;
+
+    private Result(PhysicalLayout physicalLayout, LogicalLayout logicalLayout, Mode mode) {
+      this.physicalLayout = physicalLayout;
+      this.logicalLayout = logicalLayout;
+      this.mode = mode;
+    }
+
+    public PhysicalLayout getPhysicalLayout() {
+      return physicalLayout;
+    }
+
+    public LogicalLayout getLogicalLayout() {
+      return logicalLayout;
+    }
+
+    public Mode getMode() {
+      return mode;
+    }
+  }
+
+  public static Result parse(Source source) {
+    String src = source.getCharacters().toString();
+    Matcher m = ARRAY_PATTERN.matcher(src);
+    if (m.find()) {
+      try {
+        var layout = LogicalLayout.valueOf(m.group(1));
+        return new Result(PhysicalLayout.Primitive, layout, Mode.Allocate);
+      } catch (IllegalArgumentException iae) {
+        // propagate warning
+        return null;
+      }
+    }
+
+    m = CAST_PATTERN.matcher(src);
+    if (m.find()) {
+      try {
+        var layout = LogicalLayout.valueOf(m.group(1));
+        return new Result(PhysicalLayout.Primitive, layout, Mode.Cast);
+      } catch (IllegalArgumentException iae) {
+        // propagate warning
+        return null;
+      }
+    }
+    return null;
+  }
+
+  private static final Pattern ARRAY_PATTERN = Pattern.compile("new\\[(.+)\\]");
+  private static final Pattern CAST_PATTERN = Pattern.compile("cast\\[(.+)\\]");
+
+  public enum Mode {
+    Allocate,
+    Cast
+  }
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/LogicalLayout.java

@@ -0,0 +1,23 @@
+package org.enso.interpreter.arrow;
+
+import org.enso.interpreter.arrow.runtime.SizeInBytes;
+
+public enum LogicalLayout implements SizeInBytes {
+  Date32(32),
+  Date64(64),
+  Int8(8),
+  Int16(16),
+  Int32(32),
+  Int64(64);
+
+  private final int bits;
+
+  LogicalLayout(int bits) {
+    this.bits = bits;
+  }
+
+  @Override
+  public int sizeInBytes() {
+    return bits / 8;
+  }
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/PhysicalLayout.java

@@ -0,0 +1,6 @@
+package org.enso.interpreter.arrow;
+
+public enum PhysicalLayout {
+  Primitive,
+  VariableSizeBinary
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/node/ArrowCastFixedSizeNode.java

@@ -0,0 +1,16 @@
+package org.enso.interpreter.arrow.node;
+
+import com.oracle.truffle.api.nodes.Node;
+import org.enso.interpreter.arrow.LogicalLayout;
+import org.enso.interpreter.arrow.runtime.ArrowCastToFixedSizeArrayFactory;
+
+public class ArrowCastFixedSizeNode extends Node {
+
+  static ArrowCastFixedSizeNode create() {
+    return new ArrowCastFixedSizeNode();
+  }
+
+  public Object execute(LogicalLayout layoutType) {
+    return new ArrowCastToFixedSizeArrayFactory(layoutType);
+  }
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/node/ArrowEvalNode.java

@@ -0,0 +1,34 @@
+package org.enso.interpreter.arrow.node;
+
+import com.oracle.truffle.api.CompilerDirectives;
+import com.oracle.truffle.api.frame.FrameDescriptor;
+import com.oracle.truffle.api.frame.VirtualFrame;
+import com.oracle.truffle.api.nodes.RootNode;
+import org.enso.interpreter.arrow.ArrowLanguage;
+import org.enso.interpreter.arrow.ArrowParser;
+
+public class ArrowEvalNode extends RootNode {
+  private final ArrowParser.Result code;
+
+  @Child private ArrowFixedSizeNode fixedPhysicalLayout = ArrowFixedSizeNode.create();
+  @Child private ArrowCastFixedSizeNode castToFixedPhysicalLayout = ArrowCastFixedSizeNode.create();
+
+  public static ArrowEvalNode create(ArrowLanguage language, ArrowParser.Result code) {
+    return new ArrowEvalNode(language, code);
+  }
+
+  private ArrowEvalNode(ArrowLanguage language, ArrowParser.Result code) {
+    super(language, new FrameDescriptor());
+    this.code = code;
+  }
+
+  public Object execute(VirtualFrame frame) {
+    return switch (code.getPhysicalLayout()) {
+      case Primitive -> switch (code.getMode()) {
+        case Allocate -> fixedPhysicalLayout.execute(code.getLogicalLayout());
+        case Cast -> castToFixedPhysicalLayout.execute(code.getLogicalLayout());
+      };
+      default -> throw CompilerDirectives.shouldNotReachHere("unsupported physical layout");
+    };
+  }
+}

engine/runtime-language-arrow/src/main/java/org/enso/interpreter/arrow/node/ArrowFixedSizeNode.java

@@ -0,0 +1,16 @@
+package org.enso.interpreter.arrow.node;
+
+import com.oracle.truffle.api.nodes.Node;
+import org.enso.interpreter.arrow.LogicalLayout;
+import org.enso.interpreter.arrow.runtime.ArrowFixedSizeArrayFactory;
+
+public class ArrowFixedSizeNode extends Node {
+
+  static ArrowFixedSizeNode create() {
+    return new ArrowFixedSizeNode();
+  }
+
+  public Object execute(LogicalLayout layoutType) {
+    return new ArrowFixedSizeArrayFactory(layoutType);
+  }
+}