ydoc-server is a separate module (#10156)

- Remove remnants of deprecated Scala parser
- The following projects are now JPMS modules provided on system module-path (in components directory):
  - `ydoc-server`
  - `profiling-utils`
  - `syntax-rust-definition`
- The contents of the aforementioned modules are excluded from both `runner.jar` and `runtime.jar` fat jars.
- Suggestions are serialized and deserialized with our Persistance framework, rather than via the default Java OutputObjectWriter.

docs/sbt-cheatsheet.md

@@ -0,0 +1,70 @@
+# sbt cheatsheet
+
+This document contains various references and notes about sbt build definition.
+One of the motivation for this document is that the
+[official sbt documentation](https://www.scala-sbt.org/1.x/docs/index.html) is
+not very informative.
+
+### Resources
+
+- [Official sbt documentation](https://www.scala-sbt.org/1.x/docs/index.html)
+- [sbt.Keys](https://github.com/sbt/sbt/blob/1.10.x/main/src/main/scala/sbt/Keys.scala)
+  defines all the tasks and settings keys, including their documentation.
+
+### Logging
+
+- Get reference to the logger via `streams.value.log`.
+- Log and error via `streams.value.log.error("My error")`
+  - Note that this will not cause the task to fail, this will only log an
+    `[error]` message. IF you want to fail a task, refer to
+    [Fail a task](#fail-a-task).
+
+### Fail a task
+
+- Simply throw a `java.lang.RuntimeException`.
+  - This will always work - it will crash the whole sbt process with the
+    RuntimeException. But it is not the official recommended way.
+
+### Inspect tasks and settings
+
+- `inspect tree project/assembly`
+  - This prints all the tasks and settings that will be executed when
+    `project / assembly` is run.
+  - It may be helpful to increase the width of the ASCI tree with
+    `set asciiGraphWidth := 150`.
+- See `help inspect`.
+
+## See all the transitive dependencies
+
+- `print dependencyTree`
+  - This shows all the transitive dependencies for the default `Compile` scope.
+- `print Test/dependencyTree`
+  - This show all the transitive dependencies for the `Test` scope.
+
+### Debugging sbt tasks
+
+- There is `--jvm-debug` cmd line option to sbt which allows to attach a
+  debugger to the sbt process.
+  - This option is not very handy, as you will still not be able to add a
+    breakpoint to various task definitions inside `build.sbt`, only to some
+    classes in the `project` directory. Moreover, once you run `reload`, the
+    debugging will not work for sure.
+    - This is because sbt internally compiles everything from `build.sbt` a
+      various anonymous classes and the debugger does not see their sources.
+- It is better to either use `println` directly, or to use
+  `streams.value.log.info` to log messages.
+
+### Exclude internal sbt project from a fat jar
+
+- Fat jar for `fat-project` is assembled via the `fat-project / assembly` task.
+- There is `fat-project / assembly / assemblyExcludedJars` setting which is of
+  type `Def.ClassPath`.
+- To exclude internal `project` one must:
+  - Set `project / Compile / exportJars := true`. This ensures that
+    `project / Compile / exportedProducts` is a path to the jar.
+  - In `fat-project / assembly / assemblyExcludedJars` get the path to the
+    `project` jar via `project / Compile / exportedProducts`.
+  - Declare dependency on `project / Compile / packageBin` from
+    `fat-project / assembly`.
+- Note that this is complicated because `assemblyExcludedJars` setting only
+  works if it points to an already existing jar file.

engine/interpreter-dsl-test/src/test/java/org/enso/interpreter/dsl/test/ThrowBuiltinNode.java

@@ -9,11 +9,6 @@
 public class ThrowBuiltinNode extends Node {
   public Object execute(Text type, long exceptionIdx) {
     switch (type.toString()) {
-      case "exception" -> {
-        Supplier<RuntimeException> exceptionSupplier =
-            ThrowableCatchTest.exceptionSuppliers.get((int) exceptionIdx);
-        throw exceptionSupplier.get();
-      }
       case "error" -> {
         Supplier<Error> errorSupplier = ThrowableCatchTest.errorSuppliers.get((int) exceptionIdx);
         throw errorSupplier.get();

engine/interpreter-dsl-test/src/test/java/org/enso/interpreter/dsl/test/ThrowableCatchTest.java

@@ -4,7 +4,6 @@
 import static org.junit.Assert.fail;
 
 import com.oracle.truffle.api.dsl.UnsupportedSpecializationException;
-import com.oracle.truffle.api.nodes.Node;
 import java.nio.file.Paths;
 import java.util.List;
 import java.util.function.Supplier;
@@ -33,13 +32,6 @@
  * <p>These tests checks this contract.
  */
 public class ThrowableCatchTest {
-  static List<Supplier<RuntimeException>> exceptionSuppliers =
-      List.of(
-          () -> new RuntimeException("First exception"),
-          () -> new IllegalStateException("Illegal state"),
-          () -> new CustomException(new PanicException("Panic", null)),
-          () -> new UnsupportedSpecializationException(null, new Node[] {null}, 42));
-
   private static List<Class<?>> shouldBeHandledExceptionTypes =
       List.of(UnsupportedSpecializationException.class);
 
@@ -53,12 +45,6 @@ public class ThrowableCatchTest {
   private Context ctx;
   private EnsoContext ensoCtx;
 
-  private static class CustomException extends RuntimeException {
-    CustomException(PanicException panic) {
-      super(panic);
-    }
-  }
-
   private static class CustomError extends Error {}
 
   @Before
@@ -86,37 +72,6 @@ public void tearDown() {
     ctx.close();
   }
 
-  @Test
-  public void testMostRuntimeExceptionsCanPropagateFromBuiltinMethods() {
-    var func = ThrowBuiltinMethodGen.makeFunction(EnsoLanguage.get(null));
-    var funcCallTarget = func.getCallTarget();
-    var emptyState = ensoCtx.emptyState();
-    for (long exceptionSupplierIdx = 0;
-        exceptionSupplierIdx < exceptionSuppliers.size();
-        exceptionSupplierIdx++) {
-      Object self = null;
-      Object[] args =
-          Function.ArgumentsHelper.buildArguments(
-              func,
-              null,
-              emptyState,
-              new Object[] {self, Text.create("exception"), exceptionSupplierIdx});
-      try {
-        funcCallTarget.call(args);
-      } catch (Throwable t) {
-        var expectedException = exceptionSuppliers.get((int) exceptionSupplierIdx).get();
-        if (shouldExceptionBeHandled(t)) {
-          expectPanicOrDataflowErrorWithMessage(t, expectedException.getMessage());
-        } else {
-          assertSameExceptions(
-              "Thrown RuntimeException should not be modified in the builtin method",
-              expectedException,
-              t);
-        }
-      }
-    }
-  }
-
   @Test
   public void testMostErrorsCanPropagateFromBuiltinMethods() {
     var func = ThrowBuiltinMethodGen.makeFunction(EnsoLanguage.get(null));

engine/language-server/src/main/scala/org/enso/languageserver/boot/MainModule.scala

@@ -50,10 +50,12 @@ import org.enso.logger.masking.Masking
 import org.enso.logger.JulHandler
 import org.enso.logger.akka.AkkaConverter
 import org.enso.common.HostAccessFactory
+import org.enso.languageserver.boot.config.ApplicationConfig
 import org.enso.polyglot.{RuntimeOptions, RuntimeServerInfo}
 import org.enso.profiling.events.NoopEventsMonitor
 import org.enso.searcher.memory.InMemorySuggestionsRepo
 import org.enso.text.{ContentBasedVersioning, Sha3_224VersionCalculator}
+import org.enso.ydoc.Ydoc
 import org.graalvm.polyglot.Engine
 import org.graalvm.polyglot.Context
 import org.graalvm.polyglot.io.MessageEndpoint
@@ -82,6 +84,8 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
     logLevel
   )
 
+  private val applicationConfig = ApplicationConfig.load()
+
   private val utcClock = Clock.systemUTC()
 
   val directoriesConfig = ProjectDirectoriesConfig(serverConfig.contentRootPath)
@@ -506,6 +510,14 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
     )
   log.trace("Created Binary WebSocket Server [{}].", binaryServer)
 
+  private val ydoc = Ydoc
+    .builder()
+    .hostname(applicationConfig.ydoc.hostname)
+    .port(applicationConfig.ydoc.port)
+    .build()
+  ydoc.start()
+  log.debug("Started Ydoc server.")
+
   log.info(
     "Main module of the Language Server initialized with config [{}].",
     languageServerConfig
@@ -516,6 +528,7 @@ class MainModule(serverConfig: LanguageServerConfig, logLevel: Level) {
     suggestionsRepo.close()
     context.close()
     runtimeEventsMonitor.close()
+    ydoc.close()
     log.info("Closed Language Server main module.")
   }
 

engine/polyglot-api/src/main/scala/org/enso/polyglot/Suggestion.scala

@@ -45,7 +45,6 @@ import scala.collection.immutable.ListSet
     )
   )
 )
-@SerialVersionUID(9650L)
 sealed trait Suggestion extends ToLogString {
 
   def externalId:    Option[Suggestion.ExternalID]

engine/runtime-compiler/src/main/scala/org/enso/compiler/pass/analyse/alias/Graph.scala

@@ -1,7 +1,7 @@
 package org.enso.compiler.pass.analyse.alias
 
 import org.enso.compiler.core.{CompilerError, ExternalID, Identifier}
-import org.enso.syntax.text.Debug
+import org.enso.compiler.debug.Debug
 import org.enso.compiler.pass.analyse.alias.Graph.{Occurrence, Scope}
 
 import java.util.UUID

engine/runtime-fat-jar/src/main/java/module-info.java

@@ -3,6 +3,12 @@
   requires java.net.http;
   // Because of akka.util.Unsafe
   requires jdk.unsupported;
+  requires org.enso.syntax;
+  // org.enso.profiling and org.enso.doc are not needed for runtime. It is needed for runner.jar,
+  // but we have to include it here so that the forwarding in IsolatedClassLoader
+  // works.
+  requires org.enso.profiling;
+  requires org.enso.ydoc;
   requires org.graalvm.polyglot;
   requires org.graalvm.truffle;
   requires static org.slf4j;

engine/runtime-parser/src/main/java/org/enso/compiler/core/IR.java

@@ -6,7 +6,7 @@
 import org.enso.compiler.core.ir.Expression;
 import org.enso.compiler.core.ir.IdentifiedLocation;
 import org.enso.compiler.core.ir.MetadataStorage;
-import org.enso.syntax.text.Debug;
+import org.enso.compiler.debug.Debug;
 import scala.Option;
 import scala.collection.immutable.$colon$colon$;
 import scala.collection.immutable.List;

lib/scala/syntax/definition/src/main/scala/org/enso/syntax/text/Debug.scala → engine/runtime-parser/src/main/scala/org/enso/compiler/debug/Debug.scala

@@ -1,4 +1,4 @@
-package org.enso.syntax.text
+package org.enso.compiler.debug
 
 import scala.annotation.tailrec
 

engine/runtime/src/main/java/org/enso/ClassLoaderConstants.java

@@ -13,10 +13,15 @@ public class ClassLoaderConstants {
    * consistent.
    */
   public static final List<String> CLASS_DELEGATION_PATTERNS =
-      List.of("org.graalvm", "java", "org.slf4j", "ch.qos", "io.helidon");
-
-  public static final List<String> RESOURCE_DELEGATION_PATTERNS =
-      List.of("org.slf4j", "ch.qos", "io.helidon");
+      List.of(
+          "org.graalvm",
+          "java",
+          "org.slf4j",
+          "ch.qos",
+          "io.helidon",
+          "org.enso.profiling",
+          "org.enso.syntax2",
+          "org.enso.ydoc");
 
   /**
    * Path to the {@code runner.jar} fat jar. This must not be on the system's module-path, because

engine/runtime/src/main/java/org/enso/EngineRunnerBootLoader.java

@@ -76,7 +76,7 @@ public Class<?> loadClass(String name) throws ClassNotFoundException {
 
     @Override
     public URL findResource(String name) {
-      if (ClassLoaderConstants.RESOURCE_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) {
+      if (ClassLoaderConstants.CLASS_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) {
         return systemClassLoader.getResource(name);
       } else {
         return super.findResource(name);
@@ -85,7 +85,7 @@ public URL findResource(String name) {
 
     @Override
     public Enumeration<URL> findResources(String name) throws IOException {
-      if (ClassLoaderConstants.RESOURCE_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) {
+      if (ClassLoaderConstants.CLASS_DELEGATION_PATTERNS.stream().anyMatch(name::startsWith)) {
         return systemClassLoader.getResources(name);
       } else {
         return super.findResources(name);

engine/runtime/src/main/java/org/enso/interpreter/caches/Cache.java

@@ -287,10 +287,7 @@ private Optional<T> loadCacheFrom(
             return Optional.empty();
           }
         } catch (ClassNotFoundException | IOException ex) {
-          logger.log(
-              Level.WARNING,
-              "`" + logName + "` in " + dataPath + " failed to load: " + ex.getMessage());
-          logger.log(logLevel, "`" + logName + "` failed to load.", ex);
+          logger.log(Level.WARNING, "`" + logName + "` in " + dataPath + " failed to load: ", ex);
           invalidateCache(cacheRoot, logger);
           return Optional.empty();
         }

engine/runtime/src/main/java/org/enso/interpreter/caches/CacheUtils.java

@@ -11,7 +11,9 @@
 import java.util.function.Function;
 import org.enso.compiler.context.CompilerContext;
 import org.enso.compiler.core.ir.ProcessingPass;
+import org.enso.editions.LibraryName;
 import org.enso.pkg.SourceFile;
+import org.enso.polyglot.Suggestion;
 import org.enso.text.Hex;
 
 final class CacheUtils {
@@ -88,6 +90,22 @@ static final String computeDigestOfLibrarySources(List<SourceFile<TruffleFile>>
     }
   }
 
+  public static String computeDigestFromSuggestions(List<Suggestion> suggestions) {
+    var digest = messageDigest();
+    for (var suggestion : suggestions) {
+      digest.update(suggestion.module().getBytes());
+      digest.update(suggestion.name().getBytes());
+      digest.update(suggestion.returnType().getBytes());
+    }
+    return Hex.toHexString(digest.digest());
+  }
+
+  public static String computeDigestFromLibName(LibraryName libName) {
+    var digest = messageDigest();
+    digest.update(libName.qualifiedName().getBytes());
+    return Hex.toHexString(digest.digest());
+  }
+
   @SuppressWarnings("unchecked")
   static <T extends Exception> T raise(Class<T> cls, Exception e) throws T {
     throw (T) e;

engine/runtime/src/main/java/org/enso/interpreter/caches/ImportExportCache.java

@@ -72,7 +72,7 @@ public byte[] serialize(EnsoContext context, CachedBindings entry) throws IOExce
   @Override
   public CachedBindings deserialize(
       EnsoContext context, ByteBuffer data, Metadata meta, TruffleLogger logger)
-      throws ClassNotFoundException, IOException, ClassNotFoundException {
+      throws IOException {
     var ref = Persistance.read(data, CacheUtils.readResolve(context.getCompiler().context()));
     var bindings = ref.get(MapToBindings.class);
     return new CachedBindings(libraryName, bindings, Optional.empty());

engine/runtime/src/main/java/org/enso/interpreter/caches/ModuleCache.java

@@ -67,7 +67,7 @@ public byte[] serialize(EnsoContext context, CachedModule entry) throws IOExcept
   @Override
   public CachedModule deserialize(
       EnsoContext context, ByteBuffer data, Metadata meta, TruffleLogger logger)
-      throws ClassNotFoundException, IOException, ClassNotFoundException {
+      throws IOException {
     var ref = Persistance.read(data, CacheUtils.readResolve(context.getCompiler().context()));
     var mod = ref.get(Module.class);
     return new CachedModule(

engine/runtime/src/main/java/org/enso/interpreter/caches/PersistUtils.java

@@ -0,0 +1,34 @@
+package org.enso.interpreter.caches;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import org.enso.persist.Persistance;
+import org.openide.util.lookup.ServiceProvider;
+
+public class PersistUtils {
+  @ServiceProvider(service = Persistance.class)
+  public static final class PersistArrayList extends Persistance<ArrayList> {
+    public PersistArrayList() {
+      super(ArrayList.class, true, 30360);
+    }
+
+    @Override
+    protected void writeObject(ArrayList obj, Output out) throws IOException {
+      out.writeInt(obj.size());
+      for (Object o : obj) {
+        out.writeObject(o);
+      }
+    }
+
+    @Override
+    protected ArrayList readObject(Input in) throws IOException, ClassNotFoundException {
+      int size = in.readInt();
+      var lst = new ArrayList<>(size);
+      for (int i = 0; i < size; i++) {
+        var obj = in.readObject();
+        lst.add(obj);
+      }
+      return lst;
+    }
+  }
+}