Refactoring and Analysis Fixes from WIP Automatic Parallelism

RELEASES.md

@@ -6,6 +6,10 @@
   [GraalVM 21.1.0](https://github.com/graalvm/graalvm-ce-builds/releases/tag/vm-21.1.0)
   ([#1738](https://github.com/enso-org/enso/pull/1738)). This brings a raft of
   bug-fixes and improvements to how quickly Enso can reach its peak performance.
+- Added support for bidirectional dataflow tracking to the `DataflowAnalysis`
+  pass ([#1748](https://github.com/enso-org/enso/pull/1748)). This will allow
+  the interpreter to perform more detailed analyses in the future to enable
+  optimisations and new features.
 
 ## Tooling
 
@@ -17,6 +21,9 @@
 - Implemented log masking ([#1732](https://github.com/enso-org/enso/pull/1732)).
   This feature masks personally identifiable information in the logs, such as
   code literals, computed values, and user environment variables.
+- Added support for evaluating one-shot expressions on the result values of
+  arbitrary expressions ([#1749](https://github.com/enso-org/enso/pull/1749)).
+  This is very useful for enabling more advanced introspection in the IDE.
 
 ## Libraries
 

engine/polyglot-api/src/main/java/org/enso/polyglot/RuntimeOptions.java

@@ -1,12 +1,11 @@
 package org.enso.polyglot;
 
+import java.util.Arrays;
+import java.util.logging.Level;
 import org.graalvm.options.OptionDescriptor;
 import org.graalvm.options.OptionDescriptors;
 import org.graalvm.options.OptionKey;
 
-import java.util.Arrays;
-import java.util.logging.Level;
-
 /** Class representing runtime options supported by the Enso engine. */
 public class RuntimeOptions {
   public static final String PACKAGES_PATH = optionName("packagesPath");
@@ -56,9 +55,9 @@ public class RuntimeOptions {
               STRICT_ERRORS_DESCRIPTOR,
               LOG_LEVEL_DESCRIPTOR,
               DISABLE_INLINE_CACHES_DESCRIPTOR,
-              INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION_DESCRIPTOR,
               ENABLE_PROJECT_SUGGESTIONS_DESCRIPTOR,
-              ENABLE_GLOBAL_SUGGESTIONS_DESCRIPTOR));
+              ENABLE_GLOBAL_SUGGESTIONS_DESCRIPTOR,
+              INTERPRETER_SEQUENTIAL_COMMAND_EXECUTION_DESCRIPTOR));
 
   /**
    * Canonicalizes the option name by prefixing it with the language name.

engine/runtime/src/main/java/org/enso/interpreter/node/expression/debug/EvalNode.java

@@ -61,14 +61,13 @@ public static EvalNode buildWithResultScopeCapture() {
   public abstract Stateful execute(CallerInfo callerInfo, Object state, Text expression);
 
   RootCallTarget parseExpression(LocalScope scope, ModuleScope moduleScope, String expression) {
+    Context context = lookupContextReference(Language.class).get();
     LocalScope localScope = scope.createChild();
-    InlineContext inlineContext = InlineContext.fromJava(localScope, moduleScope, getTailStatus());
+    InlineContext inlineContext =
+        InlineContext.fromJava(
+            localScope, moduleScope, getTailStatus(), context.getCompilerConfig());
     ExpressionNode expr =
-        lookupContextReference(Language.class)
-            .get()
-            .getCompiler()
-            .runInline(expression, inlineContext)
-            .getOrElse(null);
+        context.getCompiler().runInline(expression, inlineContext).getOrElse(null);
     if (expr == null) {
       throw new RuntimeException("Invalid code passed to `eval`: " + expression);
     }

engine/runtime/src/main/java/org/enso/interpreter/runtime/Context.java

@@ -18,6 +18,7 @@
 import java.util.function.Function;
 import java.util.stream.Collectors;
 import org.enso.compiler.Compiler;
+import org.enso.compiler.data.CompilerConfig;
 import org.enso.home.HomeManager;
 import org.enso.interpreter.Language;
 import org.enso.interpreter.OptionsHelper;
@@ -53,6 +54,7 @@ public class Context {
   private final Builtins builtins;
   private final String home;
   private final List<ShadowedPackage> shadowedPackages;
+  private final CompilerConfig compilerConfig;
 
   /**
    * Creates a new Enso context.
@@ -70,12 +72,13 @@ public Context(Language language, String home, Env environment) {
     this.threadManager = new ThreadManager();
     this.resourceManager = new ResourceManager(this);
     this.isCachingDisabled = environment.getOptions().get(RuntimeOptions.DISABLE_INLINE_CACHES_KEY);
+    this.compilerConfig = new CompilerConfig(false, true);
     this.home = home;
     this.shadowedPackages = new ArrayList<>();
 
     builtins = new Builtins(this);
 
-    this.compiler = new Compiler(this, builtins);
+    this.compiler = new Compiler(this, builtins, compilerConfig);
   }
 
   /** Perform expensive initialization logic for the context. */
@@ -384,4 +387,9 @@ public boolean isCachingDisabled() {
   public List<ShadowedPackage> getShadowedPackages() {
     return shadowedPackages;
   }
+
+  /** @return the compiler configuration for this language */
+  public CompilerConfig getCompilerConfig() {
+    return compilerConfig;
+  }
 }

engine/runtime/src/main/scala/org/enso/compiler/Compiler.scala

@@ -5,6 +5,7 @@ import org.enso.compiler.codegen.{AstToIr, IrToTruffle, RuntimeStubsGenerator}
 import org.enso.compiler.context.{FreshNameSupply, InlineContext, ModuleContext}
 import org.enso.compiler.core.IR
 import org.enso.compiler.core.IR.Expression
+import org.enso.compiler.data.CompilerConfig
 import org.enso.compiler.exception.{CompilationAbortedException, CompilerError}
 import org.enso.compiler.pass.PassManager
 import org.enso.compiler.pass.analyse._
@@ -31,9 +32,13 @@ import scala.jdk.OptionConverters._
   *
   * @param context the language context
   */
-class Compiler(val context: Context, private val builtins: Builtins) {
+class Compiler(
+  val context: Context,
+  val builtins: Builtins,
+  config: CompilerConfig
+) {
   private val freshNameSupply: FreshNameSupply = new FreshNameSupply
-  private val passes: Passes                   = new Passes
+  private val passes: Passes                   = new Passes(config)
   private val passManager: PassManager         = passes.passManager
   private val importResolver: ImportResolver   = new ImportResolver(this)
   private val stubsGenerator: RuntimeStubsGenerator =
@@ -80,7 +85,8 @@ class Compiler(val context: Context, private val builtins: Builtins) {
 
         val moduleContext = ModuleContext(
           module          = module,
-          freshNameSupply = Some(freshNameSupply)
+          freshNameSupply = Some(freshNameSupply),
+          compilerConfig  = config
         )
         val compilerOutput = runCompilerPhases(module.getIr, moduleContext)
         module.unsafeSetIr(compilerOutput)
@@ -121,7 +127,8 @@ class Compiler(val context: Context, private val builtins: Builtins) {
     module.getScope.reset()
     val moduleContext = ModuleContext(
       module          = module,
-      freshNameSupply = Some(freshNameSupply)
+      freshNameSupply = Some(freshNameSupply),
+      compilerConfig  = config
     )
     val parsedAST        = parse(module.getSource)
     val expr             = generateIR(parsedAST)
@@ -331,7 +338,10 @@ class Compiler(val context: Context, private val builtins: Builtins) {
     if (context.isStrictErrors) {
       val diagnostics = modules.map { module =>
         val errors = GatherDiagnostics
-          .runModule(module.getIr, new ModuleContext(module))
+          .runModule(
+            module.getIr,
+            ModuleContext(module, compilerConfig = config)
+          )
           .unsafeGetMetadata(
             GatherDiagnostics,
             "No diagnostics metadata right after the gathering pass."
@@ -467,7 +477,7 @@ class Compiler(val context: Context, private val builtins: Builtins) {
     source: Source,
     scope: ModuleScope
   ): Unit = {
-    new IrToTruffle(context, source, scope).run(ir)
+    new IrToTruffle(context, source, scope, config).run(ir)
   }
 
   /** Generates code for the truffle interpreter in an inline context.
@@ -486,7 +496,8 @@ class Compiler(val context: Context, private val builtins: Builtins) {
     new IrToTruffle(
       context,
       source,
-      inlineContext.module.getScope
+      inlineContext.module.getScope,
+      config
     ).runInline(
       ir,
       inlineContext.localScope.getOrElse(LocalScope.root),

engine/runtime/src/main/scala/org/enso/compiler/Passes.scala

@@ -1,5 +1,6 @@
 package org.enso.compiler
 
+import org.enso.compiler.data.CompilerConfig
 import org.enso.compiler.pass.PassConfiguration._
 import org.enso.compiler.pass.analyse._
 import org.enso.compiler.pass.desugar._
@@ -17,7 +18,10 @@ import org.enso.compiler.pass.{
   PassManager
 }
 
-class Passes(passes: Option[List[PassGroup]] = None) {
+class Passes(
+  config: CompilerConfig,
+  passes: Option[List[PassGroup]] = None
+) {
 
   val moduleDiscoveryPasses = new PassGroup(
     List(
@@ -60,7 +64,16 @@ class Passes(passes: Option[List[PassGroup]] = None) {
       Patterns,
       AliasAnalysis,
       UndefinedVariables,
-      DataflowAnalysis,
+      DataflowAnalysis
+    ) ++
+    (if (config.autoParallelismEnabled) {
+       List(
+         AutomaticParallelism,
+         AliasAnalysis,
+         DataflowAnalysis
+       )
+     } else List()) ++
+    List(
       CachePreferenceAnalysis,
       UnusedBindings
     )
@@ -79,7 +92,10 @@ class Passes(passes: Option[List[PassGroup]] = None) {
   /** Configuration for the passes. */
   private val passConfig: PassConfiguration = PassConfiguration(
     ApplicationSaturation -->> ApplicationSaturation.Configuration(),
-    AliasAnalysis         -->> AliasAnalysis.Configuration()
+    AliasAnalysis         -->> AliasAnalysis.Configuration(),
+    AutomaticParallelism -->> AutomaticParallelism.Configuration(
+      config.autoParallelismEnabled
+    )
   )
 
   /** The pass manager for running compiler passes. */

engine/runtime/src/main/scala/org/enso/compiler/codegen/IrToTruffle.scala

@@ -6,7 +6,7 @@ import com.oracle.truffle.api.source.{Source, SourceSection}
 import org.enso.compiler.core.IR
 import org.enso.compiler.core.IR.Module.Scope.Import
 import org.enso.compiler.core.IR.{Error, IdentifiedLocation, Pattern}
-import org.enso.compiler.data.BindingsMap
+import org.enso.compiler.data.{BindingsMap, CompilerConfig}
 import org.enso.compiler.exception.{BadPatternMatch, CompilerError}
 import org.enso.compiler.pass.analyse.AliasAnalysis.Graph.{Scope => AliasScope}
 import org.enso.compiler.pass.analyse.AliasAnalysis.{Graph => AliasGraph}
@@ -85,11 +85,13 @@ import scala.collection.mutable.ArrayBuffer
   * @param source the source code that corresponds to the text for which code
   *               is being generated
   * @param moduleScope the scope of the module for which code is being generated
+  * @param compilerConfig the configuration for the compiler
   */
 class IrToTruffle(
   val context: Context,
   val source: Source,
-  val moduleScope: ModuleScope
+  val moduleScope: ModuleScope,
+  val compilerConfig: CompilerConfig
 ) {
 
   val language: Language = context.getLanguage

engine/runtime/src/main/scala/org/enso/compiler/context/ChangesetBuilder.scala

@@ -68,7 +68,7 @@ final class ChangesetBuilder[A: TextEditor: IndexedSource](
       if (queue.isEmpty) visited.flatMap(_.externalId).toSet
       else {
         val elem       = queue.dequeue()
-        val transitive = metadata.get(elem).getOrElse(Set())
+        val transitive = metadata.dependents.get(elem).getOrElse(Set())
         val dynamic = transitive
           .flatMap {
             case DataflowAnalysis.DependencyInfo.Type.Static(int, _) =>
@@ -80,7 +80,7 @@ final class ChangesetBuilder[A: TextEditor: IndexedSource](
             case _ =>
               None
           }
-          .flatMap(metadata.get)
+          .flatMap(metadata.dependents.get)
           .flatten
         val combined = transitive.union(dynamic)
 

engine/runtime/src/main/scala/org/enso/compiler/context/InlineContext.scala

@@ -1,9 +1,10 @@
 package org.enso.compiler.context
 
+import org.enso.compiler.data.CompilerConfig
 import org.enso.compiler.pass.PassConfiguration
 import org.enso.interpreter.node.BaseNode.TailStatus
-import org.enso.interpreter.runtime.scope.{LocalScope, ModuleScope}
 import org.enso.interpreter.runtime.Module
+import org.enso.interpreter.runtime.scope.{LocalScope, ModuleScope}
 
 /** A type containing the information about the execution context for an inline
   * expression.
@@ -14,14 +15,15 @@ import org.enso.interpreter.runtime.Module
   *                         position ([[None]] indicates no information)
   * @param freshNameSupply the compiler's supply of fresh names
   * @param passConfiguration the pass configuration
+  * @param compilerConfig the compiler configuration
   */
 case class InlineContext(
   module: Module,
   localScope: Option[LocalScope]               = None,
   isInTailPosition: Option[Boolean]            = None,
   freshNameSupply: Option[FreshNameSupply]     = None,
   passConfiguration: Option[PassConfiguration] = None,
-  noWarnings: Boolean                          = false
+  compilerConfig: CompilerConfig
 )
 object InlineContext {
 
@@ -37,12 +39,31 @@ object InlineContext {
   def fromJava(
     localScope: LocalScope,
     moduleScope: ModuleScope,
-    isInTailPosition: TailStatus
+    isInTailPosition: TailStatus,
+    compilerConfig: CompilerConfig
   ): InlineContext = {
     InlineContext(
       localScope       = Option(localScope),
       module           = moduleScope.getModule,
-      isInTailPosition = Option(isInTailPosition != TailStatus.NOT_TAIL)
+      isInTailPosition = Option(isInTailPosition != TailStatus.NOT_TAIL),
+      compilerConfig   = compilerConfig
+    )
+  }
+
+  /** Transform a module context into an inline context, retaining the useful
+    * information.
+    *
+    * @param moduleContext the module context
+    * @return an inline context wrapping the same data as `moduleContext`
+    */
+  def fromModuleContext(moduleContext: ModuleContext): InlineContext = {
+    InlineContext(
+      localScope        = None,
+      module            = moduleContext.module,
+      isInTailPosition  = None,
+      freshNameSupply   = moduleContext.freshNameSupply,
+      passConfiguration = moduleContext.passConfiguration,
+      compilerConfig    = moduleContext.compilerConfig
     )
   }
 }

engine/runtime/src/main/scala/org/enso/compiler/context/ModuleContext.scala

@@ -1,5 +1,6 @@
 package org.enso.compiler.context
 
+import org.enso.compiler.data.CompilerConfig
 import org.enso.compiler.pass.PassConfiguration
 import org.enso.interpreter.runtime.Module
 
@@ -8,10 +9,11 @@ import org.enso.interpreter.runtime.Module
   * @param module the current module scope
   * @param freshNameSupply the compiler's supply of fresh names
   * @param passConfiguration the pass configuration
+  * @param compilerConfig the compiler configuration
   */
 case class ModuleContext(
   module: Module,
   freshNameSupply: Option[FreshNameSupply]     = None,
   passConfiguration: Option[PassConfiguration] = None,
-  noWarnings: Boolean                          = false
+  compilerConfig: CompilerConfig
 )

engine/runtime/src/main/scala/org/enso/compiler/core/IR.scala

@@ -5331,6 +5331,21 @@ object IR {
           s"The pattern field $shadowedName is shadowed by $shadower."
       }
     }
+
+    /** A warning raised when a call is annotated with `@Auto_Parallel`, but the
+      * annotation cannot be obeyed.
+      *
+      * @param ir the annotated application
+      * @param reason the reason why the annotation cannot be obeyed
+      */
+    case class FailedParallelism(
+      ir: IR,
+      reason: String
+    ) extends Warning {
+      override val location: Option[IdentifiedLocation] = ir.location
+      override def message: String =
+        s"The expression ${ir.showCode()} could not be parallelised: $reason."
+    }
   }
 
   // === Errors ===============================================================

engine/runtime/src/main/scala/org/enso/compiler/data/CompilerConfig.scala

@@ -0,0 +1,12 @@
+package org.enso.compiler.data
+
+/** Configuration for the compiler.
+  *
+  * @param autoParallelismEnabled whether or not automatic parallelism detection
+  *                               is enabled.
+  * @param warningsEnabled whether or not warnings are enabled
+  */
+case class CompilerConfig(
+  autoParallelismEnabled: Boolean = false,
+  warningsEnabled: Boolean        = true
+)