diff --git a/build.sbt b/build.sbt index c40ef652fe3a..ef48f291925f 100644 --- a/build.sbt +++ b/build.sbt @@ -1770,6 +1770,119 @@ lazy val `distribution-manager` = project .dependsOn(pkg) .dependsOn(`logging-utils`) +lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) + .settings( + frgaalJavaCompilerSetting, + libraryDependencies ++= Seq( + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, + "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, + "org.openjdk.jmh" % "jmh-core" % jmhVersion % "provided", + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % "provided", + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Test + ), + Compile / javacOptions := ((Compile / javacOptions).value ++ + // Only run ServiceProvider processor and ignore those defined in META-INF, thus + // fixing incremental compilation setup + Seq( + "-processor", + "org.netbeans.modules.openide.util.ServiceProviderProcessor" + )), + commands += WithDebugCommand.withDebug, + (Test / fork) := true, + (Test / parallelExecution) := false, + (Test / javaOptions) ++= { + val runtimeJars = + (LocalProject("runtime") / Compile / fullClasspath).value + val jarsStr = runtimeJars.map(_.data).mkString(File.pathSeparator) + Seq( + s"-Dtruffle.class.path.append=${jarsStr}" + ) + } + ) + .dependsOn(`polyglot-api`) + .dependsOn(runtime) + +lazy val `std-benchmarks` = (project in file("std-bits/benchmarks")) + .settings( + frgaalJavaCompilerSetting, + libraryDependencies ++= jmh ++ Seq( + "org.openjdk.jmh" % "jmh-core" % jmhVersion % Benchmark, + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % Benchmark, + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark + ), + commands += WithDebugCommand.withDebug, + (Compile / logManager) := + sbt.internal.util.CustomLogManager.excludeMsg( + "Could not determine source for class ", + Level.Warn + ) + ) + .configs(Benchmark) + .settings( + inConfig(Benchmark)(Defaults.testSettings) + ) + .settings( + (Benchmark / parallelExecution) := false, + (Benchmark / run / fork) := true, + (Benchmark / run / connectInput) := true, + // Pass -Dtruffle.class.path.append to javac + (Benchmark / compile / javacOptions) ++= { + val runtimeClasspath = + (LocalProject("runtime") / Compile / fullClasspath).value + val runtimeInstrumentsClasspath = + (LocalProject( + "runtime-with-instruments" + ) / Compile / fullClasspath).value + val appendClasspath = + (runtimeClasspath ++ runtimeInstrumentsClasspath) + .map(_.data) + .mkString(File.pathSeparator) + Seq( + s"-J-Dtruffle.class.path.append=$appendClasspath" + ) + }, + (Benchmark / compile / javacOptions) ++= Seq( + "-s", + (Benchmark / sourceManaged).value.getAbsolutePath, + "-Xlint:unchecked" + ), + (Benchmark / run / javaOptions) ++= { + val runtimeClasspath = + (LocalProject("runtime") / Compile / fullClasspath).value + val runtimeInstrumentsClasspath = + (LocalProject( + "runtime-with-instruments" + ) / Compile / fullClasspath).value + val appendClasspath = + (runtimeClasspath ++ runtimeInstrumentsClasspath) + .map(_.data) + .mkString(File.pathSeparator) + Seq( + s"-Dtruffle.class.path.append=$appendClasspath" + ) + } + ) + .settings( + bench := (Benchmark / run).toTask("").tag(Exclusive).value, + benchOnly := Def.inputTaskDyn { + import complete.Parsers.spaceDelimited + val name = spaceDelimited("").parsed match { + case List(name) => name + case _ => throw new IllegalArgumentException("Expected one argument.") + } + Def.task { + (Benchmark / run).toTask(" " + name).value + } + }.evaluated + ) + .dependsOn(`bench-processor` % Benchmark) + .dependsOn(runtime % Benchmark) + lazy val editions = project .in(file("lib/scala/editions")) .configs(Test) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 57459aef2738..0e8079437d15 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -387,6 +387,7 @@ the following flags: allow for manual analysis and discovery of optimisation failures. - `--showCompilations`: Prints the truffle compilation trace information. - `--printAssembly`: Prints the assembly output from the HotSpot JIT tier. +- `--debugger`: Launches the JVM with the remote debugger enabled. For more information on this sbt command, please see [WithDebugCommand.scala](../project/WithDebugCommand.scala). @@ -756,7 +757,8 @@ interface of the runner prints all server options when you execute it with Below are options uses by the Language Server: - `--server`: Runs the Language Server -- `--root-id `: Content root id. +- `--root-id `: Content root id. The Language Server chooses one randomly, + so any valid UUID can be passed. - `--path `: Path to the content root. - `--interface `: Interface for processing all incoming connections. Default value is 127.0.0.1 diff --git a/docs/debugger/README.md b/docs/debugger/README.md index 7012cbe39153..b45d073bc4f7 100644 --- a/docs/debugger/README.md +++ b/docs/debugger/README.md @@ -8,96 +8,14 @@ order: 0 # Enso Debugger -The Enso Debugger allows amongst other things, to execute arbitrary expressions -in a given execution context - this is used to implement a debugging REPL. The -REPL can be launched when triggering a breakpoint in the code. - -This folder contains all documentation pertaining to the REPL and the debugger, -which is broken up as follows: - -- [**The Enso Debugger Protocol:**](./protocol.md) The protocol for the Debugger - -# Chrome Developer Tools Debugger - -As a well written citizen of the [GraalVM](http://graalvm.org) project the Enso -language can be used with existing tools available for the overall platform. One -of them is -[Chrome Debugger](https://www.graalvm.org/22.1/tools/chrome-debugger/) and Enso -language is fully integrated with it. Launch the `bin/enso` executable with -additional `--inspect` option and debug your Enso programs in _Chrome Developer -Tools_. - -```bash -enso$ ./built-distribution/enso-engine-*/enso-*/bin/enso --inspect --run ./test/Tests/src/Data/Numbers_Spec.enso -Debugger listening on ws://127.0.0.1:9229/Wugyrg9 -For help, see: https://www.graalvm.org/tools/chrome-debugger -E.g. in Chrome open: devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/Wugyrg9 -``` - -copy the printed URL into chrome browser and you should see: - -![Chrome Debugger](https://user-images.githubusercontent.com/26887752/209614265-684f530e-cf7e-45d5-9450-7ea1e4f65986.png) - -Step in, step over, set breakpoints, watch values of the variables as well as -evaluate arbitrary expressions in the console. Note that as of December 2022, -with GraalVM 22.3.0, there is a well-known -[bug in Truffle](https://github.com/oracle/graal/issues/5513) that causes -`NullPointerException` when a host object gets into the chrome inspector. There -is a workaround for that, but it may not work in certain situations. Therefore, -if you encounter `NullPointerException` thrown from - -``` -at org.graalvm.truffle/com.oracle.truffle.polyglot.PolyglotContextImpl.getContext(PolyglotContextImpl.java:685) -``` - -simply ignore it. It will be handled within the debugger and should not affect -the rest of the environment. - -# Debugging Enso and Java Code at Once - -Enso libraries are written in a mixture of Enso code and Java libraries. -Debugging both sides (the Java as well as Enso code) is possible with a decent -IDE. - -Get [NetBeans](http://netbeans.apache.org) version 13 or newer or -[VS Code with Apache Language Server extension](https://cwiki.apache.org/confluence/display/NETBEANS/Apache+NetBeans+Extension+for+Visual+Studio+Code) -and _start listening on port 5005_ with _Debug/Attach Debugger_ or by specifying -following debug configuration in VSCode: - -```json -{ - "name": "Listen to 5005", - "type": "java+", - "request": "attach", - "listen": "true", - "hostName": "localhost", - "port": "5005" -} -``` - -Then it is just about executing following Sbt command which builds CLI version -of the engine, launches it in debug mode and passes all other arguments to the -started process: - -```bash -sbt:enso> runEngineDistribution --debug --run ./test/Tests/src/Data/Numbers_Spec.enso -``` - -Alternatively you can pass in special JVM arguments when launching the -`bin/enso` launcher: - -```bash -enso$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=n,address=5005 ./built-distribution/enso-engine-*/enso-*/bin/enso --run ./test/Tests/src/Data/Numbers_Spec.enso -``` - -As soon as the debuggee connects and the Enso language starts - choose the -_Toggle Pause in GraalVM Script_ button in the toolbar: - -![NetBeans Debugger](https://user-images.githubusercontent.com/26887752/209614191-b0513635-819b-4c64-a6f9-9823b90a1513.png) - -and your execution shall stop on the next `.enso` line of code. This mode allows -to debug both - the Enso code as well as Java code. The stack traces shows a -mixture of Java and Enso stack frames by default. Right-clicking on the thread -allows one to switch to plain Java view (with a way more stack frames) and back. -Analyzing low level details as well as Enso developer point of view shall be -simple with such tool. +This folder contains all documentation pertaining to the debugging facilities +used by Enso, broken up as follows: + +- [**The Enso Debugger Protocol:**](./protocol.md) The protocol for the REPL + Debugger. +- [**Chrome devtools debugger:**](./chrome-devtools.md) A guide how to debug + Enso code using Chrome devtools. +- [**Debugging Enso and Java code at once:**](./mixed-debugging.md) A + step-by-step guide how to debug both Enso and Java code in a single debugger. +- [**Debugging Engine (Runtime) only Java code:**](./runtime-debugging.md) A + guide how to debug the internal Engine Java code. diff --git a/docs/debugger/chrome-devtools.md b/docs/debugger/chrome-devtools.md new file mode 100644 index 000000000000..7b704457619c --- /dev/null +++ b/docs/debugger/chrome-devtools.md @@ -0,0 +1,41 @@ +# Chrome Developer Tools Debugger + +As a well written citizen of the [GraalVM](http://graalvm.org) project the Enso +language can be used with existing tools available for the overall platform. One +of them is +[Chrome Debugger](https://www.graalvm.org/22.1/tools/chrome-debugger/) and Enso +language is fully integrated with it. Launch the `bin/enso` executable with +additional `--inspect` option and debug your Enso programs in _Chrome Developer +Tools_. + +```bash +enso$ ./built-distribution/enso-engine-*/enso-*/bin/enso --inspect --run ./test/Tests/src/Data/Numbers_Spec.enso +Debugger listening on ws://127.0.0.1:9229/Wugyrg9 +For help, see: https://www.graalvm.org/tools/chrome-debugger +E.g. in Chrome open: devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/Wugyrg9 +``` + +copy the printed URL into chrome browser and you should see: + +![Chrome Debugger](https://user-images.githubusercontent.com/26887752/209614265-684f530e-cf7e-45d5-9450-7ea1e4f65986.png) + +Step in, step over, set breakpoints, watch values of the variables as well as +evaluate arbitrary expressions in the console. Note that as of December 2022, +with GraalVM 22.3.0, there is a well-known +[bug in Truffle](https://github.com/oracle/graal/issues/5513) that causes +`NullPointerException` when a host object gets into the chrome inspector. There +is a workaround for that, but it may not work in certain situations. Therefore, +if you encounter `NullPointerException` thrown from + +``` +at org.graalvm.truffle/com.oracle.truffle.polyglot.PolyglotContextImpl.getContext(PolyglotContextImpl.java:685) +``` + +simply ignore it. It will be handled within the debugger and should not affect +the rest of the environment. + +## Tips and tricks + +- Use `env JAVA_OPTS=-Dpolyglot.inspect.Path=enso_debug` to set the chrome to + use a fixed URL. In this case the URL is + `devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/enso_debug` diff --git a/docs/debugger/mixed-debugging.md b/docs/debugger/mixed-debugging.md new file mode 100644 index 000000000000..48f0f4107f79 --- /dev/null +++ b/docs/debugger/mixed-debugging.md @@ -0,0 +1,48 @@ +# Debugging Enso and Java Code at Once + +Enso libraries are written in a mixture of Enso code and Java libraries. +Debugging both sides (the Java as well as Enso code) is possible with a decent +IDE. + +Get [NetBeans](http://netbeans.apache.org) version 13 or newer or +[VS Code with Apache Language Server extension](https://cwiki.apache.org/confluence/display/NETBEANS/Apache+NetBeans+Extension+for+Visual+Studio+Code) +and _start listening on port 5005_ with _Debug/Attach Debugger_ or by specifying +following debug configuration in VSCode: + +```json +{ + "name": "Listen to 5005", + "type": "java+", + "request": "attach", + "listen": "true", + "hostName": "localhost", + "port": "5005" +} +``` + +Then it is just about executing following Sbt command which builds CLI version +of the engine, launches it in debug mode and passes all other arguments to the +started process: + +```bash +sbt:enso> runEngineDistribution --debug --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +Alternatively you can pass in special JVM arguments when launching the +`bin/enso` launcher: + +```bash +enso$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=n,address=5005 ./built-distribution/enso-engine-*/enso-*/bin/enso --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +As soon as the debuggee connects and the Enso language starts - choose the +_Toggle Pause in GraalVM Script_ button in the toolbar: + +![NetBeans Debugger](https://user-images.githubusercontent.com/26887752/209614191-b0513635-819b-4c64-a6f9-9823b90a1513.png) + +and your execution shall stop on the next `.enso` line of code. This mode allows +to debug both - the Enso code as well as Java code. The stack traces shows a +mixture of Java and Enso stack frames by default. Right-clicking on the thread +allows one to switch to plain Java view (with a way more stack frames) and back. +Analyzing low level details as well as Enso developer point of view shall be +simple with such tool. diff --git a/docs/debugger/runtime-debugging.md b/docs/debugger/runtime-debugging.md new file mode 100644 index 000000000000..6ab9ea242737 --- /dev/null +++ b/docs/debugger/runtime-debugging.md @@ -0,0 +1,69 @@ +# Runtime (Engine) debugging + +This section explains how to debug various parts of the Engine. By Engine, we +mean all the Java code located in the `runtime` SBT project, in `engine` +directory. + +## Debugging source file evaluation + +This subsection provides a guide how to debug a single Enso source file +evaluation. To evaluate a single source file, we use the _Engine distribution_ +built with `buildEngineDistribution` command. Both of the following two options +starts the JVM in a debug mode. After the JVM is started in a debug mode, simply +attach the debugger to the JVM process at the specified port. + +The first option is to invoke the engine distribution from SBT shell with: + +```sh +sbt:enso> runEngineDistribution --debug --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +The second options is to pass in special JVM arguments when launching the +`bin/enso` from the engine distribution: + +```bash +enso$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=n,address=5005 ./built-distribution/enso-engine-*/enso-*/bin/enso --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +### Tips and tricks + +There is no simple mapping of the Enso source code to the engine's Java code, so +if you try to debug a specific expression, it might be a bit tricky. However, +the following steps should help you to skip all the irrelevant code and get to +the code you are interested in: + +- To debug a method called `foo`, put a breakpoint in + `org.enso.interpreter.node.ClosureRootNode#execute` with a condition on + `this.name.contains("foo")` +- To debug a specific expression, put some _unique_ expression, like + `Debug.eval "1+1"`, in front of it and put a breakpoint in a Truffle node + corresponding to that unique expression, in this case that is + `org.enso.interpreter.node.expression.builtin.debug.DebugEvalNode`. + +## Debugging annotation processors + +The Engine uses annotation processors to generate some of the Java code, e.g., +the builtin methods with `org.enso.interpreter.dsl.MethodProcessor`, or JMH +benchmark sources with `org.enso.benchmarks.processor.BenchProcessor`. +Annotation processors are invoked by the Java compiler (`javac`), therefore, we +need special instructions to attach the debugger to them. + +Let's debug `org.enso.interpreter.dsl.MethodProcessor` as an example. The +following are the commands invoked in the `sbt` shell: + +- `project runtime` +- `clean` + - Delete all the compiled class files along with all the generated sources by + the annotation processor. This ensures that the annotation processor will be + invoked. +- `set javacOptions += FrgaalJavaCompiler.debugArg` + - This sets a special flag that is passed to the frgaal Java compiler, which + in turn waits for the debugger to attach. Note that this setting is not + persisted and will be reset once the project is reloaded. +- `compile` + - Launches the Java compiler, which will wait for the debugger to attach. Put + a breakpoint in some class of `org.enso.interpreter.dsl` package. Wait for + the message in the console instructing to attach the debugger. +- To reset the `javacOptions` setting, either run + `set javacOptions -= FrgaalJavaCompiler.debugArg`, or reload the project with + `reload`. diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index 4fd4a4a49d92..623ceb1b3fe2 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -22,5 +22,7 @@ up as follows: - [**Upgrading GraalVM:**](upgrading-graalvm.md) Description of steps that have to be performed by each developer when the project is upgraded to a new version of GraalVM. +- [**Benchmarks:**](benchmarks.md) Description of the benchmarking + infrastructure used for measuring performance of the runtime. - [**Logging**:](logging.md) Description of an unified and centralized logging infrastructure that should be used by all components. diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md new file mode 100644 index 000000000000..a5b5a95eeaa4 --- /dev/null +++ b/docs/infrastructure/benchmarks.md @@ -0,0 +1,102 @@ +# Benchmarks + +In this document, we describe the benchmark types used for the runtime - Engine +micro benchmarks in the section +[Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and standard library +benchmarks in the section +[Standard library benchmarks](#standard-library-benchmarks), and how and where +are the results stored and visualized in the section +[Visualization](#visualization). + +To track the performance of the engine, we use +[JMH](https://openjdk.org/projects/code-tools/jmh/). There are two types of +benchmarks: + +- [micro benchmarks](#engine-jmh-microbenchmarks) located directly in the + `runtime` SBT project. These benchmarks are written in Java, and are used to + measure the performance of specific parts of the engine. +- [standard library benchmarks](#standard-library-benchmarks) located in the + `test/Benchmarks` Enso project. These benchmarks are entirelly written in + Enso, along with the harness code. + +## Engine JMH microbenchmarks + +These benchmarks are written in Java and are used to measure the performance of +specific parts of the engine. The sources are located in the `runtime` SBT +project, under `src/bench` source directory. + +### Running the benchmarks + +To run the benchmarks, use `bench` or `benchOnly` command - `bench` runs all the +benchmarks and `benchOnly` runs only one benchmark specified with the fully +qualified name. The parameters for these benchmarks are hard-coded inside the +JMH annotations in the source files. In order to change, e.g., the number of +measurement iterations, you need to modify the parameter to the `@Measurement` +annotation. + +### Debugging the benchmarks + +Currently, the best way to debug the benchmark is to set the `@Fork` annotation +to 0, and to run `withDebug` command like this: + +``` +withDebug --debugger benchOnly -- +``` + +## Standard library benchmarks + +Unlike the Engine micro benchmarks, these benchmarks are written entirely in +Enso and located in the `test/Benchmarks` Enso project. There are two ways to +run these benchmarks: + +- [Running standalone](#running-standalone) +- [Running via JMH launcher](#running-via-jmh-launcher) + +### Running standalone + +A single source file in the project may contain multiple benchmark definitions. +If the source file defines `main` method, we can evaluate it the same way as any +other Enso source file, for example via +`runEngineDistribution --in-project test/Benchmarks --run `. The +harness within the project is not meant for any sophisticated benchmarking, but +rather for quick local evaluation. See the `Bench.measure` method documentation +for more details. For more sophisticated approach, run the benchmarks via the +JMH launcher. + +### Running via JMH launcher + +The JMH launcher is located in `std-bits/benchmarks` directory, as +`std-benchmarks` SBT project. It is a single Java class with a `main` method +that just delegates to the +[standard JMH launcher](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/Main.java), +therefore, supports all the command line options as the standard launcher. For +the full options summary, either see the +[JMH source code](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java), +or run the launcher with `-h` option. + +The `std-benchmarks` SBT project supports `bench` and `benchOnly` commands, that +work the same as in the `runtime` project, with the exception that the benchmark +name does not have to be specified as a fully qualified name, but as a regular +expression. To access the full flexibility of the JMH launcher, run it via +`Bench/run` - for example, to see the help message: `Bench/run -h`. For example, +you can run all the benchmarks that have "New_Vector" in their name with just 3 +seconds for warmup iterations and 2 measurement iterations with +`Bench/run -w 3 -i 2 New_Vector`. + +Whenever you add or delete any benchmarks from `test/Benchmarks` project, the +generated JMH sources need to be recompiled with `Bench/clean; Bench/compile`. +You do not need to recompile the `std-benchmarks` project if you only modify the +benchmark sources. + +## Visualization + +The benchmarks are invoked as a daily +[GitHub Action](https://github.com/enso-org/enso/actions/workflows/benchmark.yml), +that can be invoked manually on a specific branch as well. The results are kept +in the artifacts produced from the actions. In +`tools/performance/engine-benchmarks` directory, there is a simple Python script +for collecting and processing the results. See the +[README in that directory](../../tools/performance/engine-benchmarks/README.md) +for more information about how to run that script. This script is invoked +regularly on a private machine and the results are published in +[https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java new file mode 100644 index 000000000000..db83ee9da2b1 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java @@ -0,0 +1,10 @@ +package org.enso.benchmarks; + +/** + * A configuration for a {@link BenchGroup benchmark group}. + * Corresponds to {@code Bench_Options} in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso} + */ +public interface BenchConfig { + int size(); + int iter(); +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java new file mode 100644 index 000000000000..fea8a3a68a55 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java @@ -0,0 +1,13 @@ +package org.enso.benchmarks; + +import java.util.List; + +/** + * A group of benchmarks with its own name and configuration. + * Corresponds to {@code Bench.Group} defined in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso}. + */ +public interface BenchGroup { + String name(); + BenchConfig configuration(); + List specs(); +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java new file mode 100644 index 000000000000..f95decbdf58e --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java @@ -0,0 +1,12 @@ +package org.enso.benchmarks; + +import org.graalvm.polyglot.Value; + +/** + * Specification of a single benchmark. + * Corresponds to {@code Bench.Spec} defined in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso}. + */ +public interface BenchSpec { + String name(); + Value code(); +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java new file mode 100644 index 000000000000..ded838b19a02 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java @@ -0,0 +1,11 @@ +package org.enso.benchmarks; + +import java.util.List; + +/** + * Wraps all the groups of benchmarks specified in a single module. + * Corresponds to {@code Bench.All} defined in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso}. + */ +public interface BenchSuite { + List groups(); +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java new file mode 100644 index 000000000000..200820b695c0 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java @@ -0,0 +1,50 @@ +package org.enso.benchmarks; + +import java.util.List; +import java.util.Optional; + +/** + * A {@link BenchSuite} with a qualified name of the module it is defined in. + */ +public final class ModuleBenchSuite { + private final BenchSuite suite; + private final String moduleQualifiedName; + + public ModuleBenchSuite(BenchSuite suite, String moduleQualifiedName) { + this.suite = suite; + this.moduleQualifiedName = moduleQualifiedName; + } + + public List getGroups() { + return suite.groups(); + } + + public String getModuleQualifiedName() { + return moduleQualifiedName; + } + + public BenchGroup findGroupByName(String groupName) { + return suite + .groups() + .stream() + .filter(grp -> grp.name().equals(groupName)) + .findFirst() + .orElse(null); + } + + public BenchSpec findSpecByName(String groupName, String specName) { + Optional group = suite + .groups() + .stream() + .filter(grp -> grp.name().equals(groupName)) + .findFirst(); + if (group.isPresent()) { + return group.get().specs() + .stream() + .filter(spec -> spec.name().equals(specName)) + .findFirst() + .orElseGet(() -> null); + } + return null; + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java new file mode 100644 index 000000000000..cfcc579fce5c --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java @@ -0,0 +1,54 @@ +package org.enso.benchmarks; + +import java.io.File; +import java.net.URISyntaxException; + +/** Utility methods used by the benchmark classes from the generated code */ +public class Utils { + + /** + * Returns the path to the {@link org.enso.polyglot.RuntimeOptions#LANGUAGE_HOME_OVERRIDE language + * home override directory}. + * + *

Note that the returned file may not exist. + * + * @return Non-null file pointing to the language home override directory. + */ + public static File findLanguageHomeOverride() { + File ensoDir = findRepoRootDir(); + // Note that ensoHomeOverride does not have to exist, only its parent directory + return ensoDir.toPath().resolve("distribution").resolve("component").toFile(); + } + + /** + * Returns the root directory of the Enso repository. + * + * @return Non-null file pointing to the root directory of the Enso repository. + */ + public static File findRepoRootDir() { + File ensoDir; + try { + ensoDir = new File(Utils.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unrecheable: ensoDir not found", e); + } + for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { + if (ensoDir.getName().equals("enso")) { + break; + } + } + if (ensoDir == null || !ensoDir.exists() || !ensoDir.isDirectory() || !ensoDir.canRead()) { + throw new IllegalStateException("Unrecheable: ensoDir does not exist or is not readable"); + } + return ensoDir; + } + + public static BenchSpec findSpecByName(BenchGroup group, String specName) { + for (BenchSpec spec : group.specs()) { + if (spec.name().equals(specName)) { + return spec; + } + } + return null; + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java new file mode 100644 index 000000000000..71b942d5dea1 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -0,0 +1,319 @@ +package org.enso.benchmarks.processor; + +import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.net.URISyntaxException; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.FilerException; +import javax.annotation.processing.Processor; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.TypeElement; +import javax.tools.Diagnostic.Kind; +import org.enso.benchmarks.BenchGroup; +import org.enso.benchmarks.BenchSpec; +import org.enso.benchmarks.ModuleBenchSuite; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames.TopScope; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; +import org.openide.util.lookup.ServiceProvider; + +@SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") +@ServiceProvider(service = Processor.class) +public class BenchProcessor extends AbstractProcessor { + + private final File ensoHomeOverride; + private final File ensoDir; + private File projectRootDir; + private static final String generatedSourcesPackagePrefix = "org.enso.benchmarks.generated"; + private static final List imports = + List.of( + "import java.nio.file.Paths;", + "import java.io.ByteArrayOutputStream;", + "import java.io.File;", + "import java.util.List;", + "import java.util.Objects;", + "import org.openjdk.jmh.annotations.Benchmark;", + "import org.openjdk.jmh.annotations.BenchmarkMode;", + "import org.openjdk.jmh.annotations.Mode;", + "import org.openjdk.jmh.annotations.Fork;", + "import org.openjdk.jmh.annotations.Measurement;", + "import org.openjdk.jmh.annotations.OutputTimeUnit;", + "import org.openjdk.jmh.annotations.Setup;", + "import org.openjdk.jmh.annotations.State;", + "import org.openjdk.jmh.annotations.Scope;", + "import org.openjdk.jmh.infra.BenchmarkParams;", + "import org.openjdk.jmh.infra.Blackhole;", + "import org.graalvm.polyglot.Context;", + "import org.graalvm.polyglot.Value;", + "import org.graalvm.polyglot.io.IOAccess;", + "import org.enso.polyglot.LanguageInfo;", + "import org.enso.polyglot.MethodNames;", + "import org.enso.polyglot.RuntimeOptions;", + "import org.enso.benchmarks.processor.SpecCollector;", + "import org.enso.benchmarks.ModuleBenchSuite;", + "import org.enso.benchmarks.BenchSpec;", + "import org.enso.benchmarks.BenchGroup;", + "import org.enso.benchmarks.Utils;"); + + public BenchProcessor() { + File currentDir = null; + try { + currentDir = + new File( + BenchProcessor.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + } catch (URISyntaxException e) { + failWithMessage("ensoDir not found: " + e.getMessage()); + } + for (; currentDir != null; currentDir = currentDir.getParentFile()) { + if (currentDir.getName().equals("enso")) { + break; + } + } + if (currentDir == null) { + failWithMessage("Unreachable: Could not find Enso root directory"); + } + ensoDir = currentDir; + + // Note that ensoHomeOverride does not have to exist, only its parent directory + ensoHomeOverride = ensoDir.toPath().resolve("distribution").resolve("component").toFile(); + } + + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latest(); + } + + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) { + var elements = roundEnv.getElementsAnnotatedWith(GenerateBenchSources.class); + for (var element : elements) { + GenerateBenchSources annotation = element.getAnnotation(GenerateBenchSources.class); + projectRootDir = new File(annotation.projectRootPath()); + if (!projectRootDir.exists() || !projectRootDir.isDirectory() || !projectRootDir.canRead()) { + failWithMessage( + "Project root dir '" + + projectRootDir.getAbsolutePath() + + "' specified in the annotation does not exist or is not readable"); + } + try (var ctx = + Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(IOAccess.ALL) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option(RuntimeOptions.PROJECT_ROOT, projectRootDir.getAbsolutePath()) + .option(RuntimeOptions.LANGUAGE_HOME_OVERRIDE, ensoHomeOverride.getAbsolutePath()) + .build()) { + Value module = getModule(ctx, annotation.moduleName()); + assert module != null; + List benchSuites = + SpecCollector.collectBenchSpecsFromModule(module, annotation.variableName()); + for (ModuleBenchSuite benchSuite : benchSuites) { + for (BenchGroup group : benchSuite.getGroups()) { + generateClassForGroup( + group, benchSuite.getModuleQualifiedName(), annotation.variableName()); + } + } + return true; + } catch (Exception e) { + failWithMessage("Uncaught exception in " + getClass().getName() + ": " + e.getMessage()); + return false; + } + } + return true; + } + + private Value getModule(Context ctx, String moduleName) { + try { + return ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); + } catch (PolyglotException e) { + failWithMessage("Cannot get module '" + moduleName + "': " + e.getMessage()); + return null; + } + } + + private void generateClassForGroup(BenchGroup group, String moduleQualifiedName, String varName) { + String fullClassName = createGroupClassName(group); + try (Writer srcFileWriter = + processingEnv.getFiler().createSourceFile(fullClassName).openWriter()) { + generateClassForGroup(srcFileWriter, moduleQualifiedName, varName, group); + } catch (IOException e) { + if (!isResourceAlreadyExistsException(e)) { + failWithMessage( + "Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); + } + } + } + + /** + * Returns true iff the given exception is thrown because a file already exists exception. There + * is no better way to check this. + * + * @param e Exception to check. + * @return true iff the given exception is thrown because a file already exists exception. + */ + private static boolean isResourceAlreadyExistsException(IOException e) { + List messages = + List.of( + "Source file already created", + "Resource already created", + "Attempt to recreate a file"); + return e instanceof FilerException + && messages.stream().anyMatch(msg -> e.getMessage().contains(msg)); + } + + private void generateClassForGroup( + Writer javaSrcFileWriter, String moduleQualifiedName, String varName, BenchGroup group) + throws IOException { + String groupFullClassName = createGroupClassName(group); + String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); + List specs = group.specs(); + List specJavaNames = + specs.stream().map(spec -> normalize(spec.name())).collect(Collectors.toUnmodifiableList()); + + javaSrcFileWriter.append("package " + generatedSourcesPackagePrefix + ";\n"); + javaSrcFileWriter.append("\n"); + javaSrcFileWriter.append(String.join("\n", imports)); + javaSrcFileWriter.append("\n"); + javaSrcFileWriter.append("\n"); + javaSrcFileWriter.append("/**\n"); + javaSrcFileWriter.append(" * Generated from:\n"); + javaSrcFileWriter.append(" * - Module: " + moduleQualifiedName + "\n"); + javaSrcFileWriter.append(" * - Group: \"" + group.name() + "\"\n"); + javaSrcFileWriter.append(" * Generated by {@link " + getClass().getName() + "}.\n"); + javaSrcFileWriter.append(" */\n"); + javaSrcFileWriter.append("@BenchmarkMode(Mode.AverageTime)\n"); + javaSrcFileWriter.append("@Fork(1)\n"); + javaSrcFileWriter.append("@State(Scope.Benchmark)\n"); + javaSrcFileWriter.append("public class " + className + " {\n"); + javaSrcFileWriter.append(" private Value groupInputArg;\n"); + for (var specJavaName : specJavaNames) { + javaSrcFileWriter.append(" private Value benchFunc_" + specJavaName + ";\n"); + } + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" @Setup\n"); + javaSrcFileWriter.append(" public void setup(BenchmarkParams params) throws Exception {\n"); + javaSrcFileWriter + .append(" File projectRootDir = Utils.findRepoRootDir().toPath().resolve(\"") + .append(projectRootDir.toString()) + .append("\").toFile();\n"); + javaSrcFileWriter.append( + " if (projectRootDir == null || !projectRootDir.exists() || !projectRootDir.canRead()) {\n"); + javaSrcFileWriter.append( + " throw new IllegalStateException(\"Project root directory does not exist or cannot be read: \" + Objects.toString(projectRootDir));\n"); + javaSrcFileWriter.append(" }\n"); + javaSrcFileWriter.append(" File languageHomeOverride = Utils.findLanguageHomeOverride();\n"); + javaSrcFileWriter.append(" var ctx = Context.newBuilder()\n"); + javaSrcFileWriter.append(" .allowExperimentalOptions(true)\n"); + javaSrcFileWriter.append(" .allowIO(IOAccess.ALL)\n"); + javaSrcFileWriter.append(" .allowAllAccess(true)\n"); + javaSrcFileWriter.append(" .logHandler(new ByteArrayOutputStream())\n"); + javaSrcFileWriter.append(" .option(\n"); + javaSrcFileWriter.append(" RuntimeOptions.LANGUAGE_HOME_OVERRIDE,\n"); + javaSrcFileWriter.append(" languageHomeOverride.getAbsolutePath()\n"); + javaSrcFileWriter.append(" )\n"); + javaSrcFileWriter.append(" .option(\n"); + javaSrcFileWriter.append(" RuntimeOptions.PROJECT_ROOT,\n"); + javaSrcFileWriter.append(" projectRootDir.getAbsolutePath()\n"); + javaSrcFileWriter.append(" )\n"); + javaSrcFileWriter.append(" .build();\n"); + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" Value bindings = ctx.getBindings(LanguageInfo.ID);\n"); + javaSrcFileWriter.append( + " Value module = bindings.invokeMember(MethodNames.TopScope.GET_MODULE, \"" + + moduleQualifiedName + + "\");\n"); + javaSrcFileWriter.append( + " BenchGroup group = SpecCollector.collectBenchGroupFromModule(module, \"" + + group.name() + + "\", \"" + + varName + + "\");\n"); + javaSrcFileWriter.append(" \n"); + for (int i = 0; i < specs.size(); i++) { + var specJavaName = specJavaNames.get(i); + var specName = specs.get(i).name(); + javaSrcFileWriter.append( + " BenchSpec benchSpec_" + + specJavaName + + " = Utils.findSpecByName(group, \"" + + specName + + "\");\n"); + javaSrcFileWriter.append( + " this.benchFunc_" + specJavaName + " = benchSpec_" + specJavaName + ".code();\n"); + } + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" this.groupInputArg = Value.asValue(null);\n"); + javaSrcFileWriter.append(" } \n"); // end of setup method + javaSrcFileWriter.append(" \n"); + + // Benchmark methods + for (var specJavaName : specJavaNames) { + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" @Benchmark\n"); + javaSrcFileWriter.append(" public void " + specJavaName + "(Blackhole blackhole) {\n"); + javaSrcFileWriter.append( + " Value result = this.benchFunc_" + specJavaName + ".execute(this.groupInputArg);\n"); + javaSrcFileWriter.append(" blackhole.consume(result);\n"); + javaSrcFileWriter.append(" }\n"); // end of benchmark method + } + + javaSrcFileWriter.append("}\n"); // end of class className + } + + /** + * Returns Java FQN for a benchmark spec. + * + * @param group Group name will be converted to Java package name. + * @return + */ + private static String createGroupClassName(BenchGroup group) { + var groupPkgName = normalize(group.name()); + return generatedSourcesPackagePrefix + "." + groupPkgName; + } + + private static boolean isValidChar(char c) { + return Character.isAlphabetic(c) || Character.isDigit(c) || c == '_'; + } + + /** + * Converts Text to valid Java identifier. + * + * @param name Text to convert. + * @return Valid Java identifier, non null. + */ + private static String normalize(String name) { + var normalizedNameSb = new StringBuilder(); + for (char c : name.toCharArray()) { + if (isValidChar(c)) { + normalizedNameSb.append(c); + } else if (Character.isWhitespace(c) && (peekLastChar(normalizedNameSb) != '_')) { + normalizedNameSb.append('_'); + } + } + return normalizedNameSb.toString(); + } + + private static char peekLastChar(StringBuilder sb) { + if (!sb.isEmpty()) { + return sb.charAt(sb.length() - 1); + } else { + return 0; + } + } + + private void failWithMessage(String msg) { + processingEnv.getMessager().printMessage(Kind.ERROR, msg); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java new file mode 100644 index 000000000000..ca75cedcd2b4 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java @@ -0,0 +1,32 @@ +package org.enso.benchmarks.processor; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Use this annotation to force the {@link BenchProcessor} to generate JMH sources + * for all the collected Enso benchmarks. The location of the benchmarks is encoded + * by the {@code projectRootPath}, {@code moduleName} and {@code variableName} parameters. + */ +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface GenerateBenchSources { + + /** + * Path to the project root directory. Relative to the Enso repository root. + */ + String projectRootPath(); + + /** + * Fully qualified name of the module within the project that defines all the benchmark {@link org.enso.benchmarks.BenchSuite suites}. + * For example {@code local.Benchmarks.Main}. + */ + String moduleName(); + + /** + * Name of the variable that holds a list of all the benchmark {@link org.enso.benchmarks.BenchSuite suites}. + */ + String variableName() default "all_benchmarks"; +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java new file mode 100644 index 000000000000..2d51f76c845b --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java @@ -0,0 +1,60 @@ +package org.enso.benchmarks.processor; + +import java.util.ArrayList; +import java.util.List; +import org.enso.benchmarks.BenchGroup; +import org.enso.benchmarks.BenchSuite; +import org.enso.benchmarks.ModuleBenchSuite; +import org.enso.polyglot.MethodNames.Module; +import org.graalvm.polyglot.Value; + +/** + * Collect benchmark specifications from Enso source files. + */ +public class SpecCollector { + private SpecCollector() {} + + /** + * Collects all the bench specifications from the given module in a variable with the given name. + * @param varName Name of the variable that holds all the collected bench suites. + * @return Empty list if no such variable exists, or if it is not a vector. + */ + public static List collectBenchSpecsFromModule(Value module, String varName) { + Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); + Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, varName); + String moduleQualifiedName = module.invokeMember(Module.GET_NAME).asString(); + if (!allSuitesVar.isNull()) { + Value suitesValue = module.invokeMember(Module.EVAL_EXPRESSION, varName); + if (!suitesValue.hasArrayElements()) { + return List.of(); + } + List suites = new ArrayList<>(); + for (long i = 0; i < suitesValue.getArraySize(); i++) { + Value suite = suitesValue.getArrayElement(i); + BenchSuite benchSuite = suite.as(BenchSuite.class); + suites.add( + new ModuleBenchSuite(benchSuite, moduleQualifiedName) + ); + } + return suites; + } + return List.of(); + } + + /** + * Collects all the bench specifications from the given module in a variable with the given name. + * @param groupName Name of the benchmark group + * @param varName Name of the variable that holds all the collected bench suites. + * @return null if no such group exists. + */ + public static BenchGroup collectBenchGroupFromModule(Value module, String groupName, String varName) { + var specs = collectBenchSpecsFromModule(module, varName); + for (ModuleBenchSuite suite : specs) { + BenchGroup group = suite.findGroupByName(groupName); + if (group != null) { + return group; + } + } + return null; + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java new file mode 100644 index 000000000000..25276f6675a3 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java @@ -0,0 +1,116 @@ +package org.enso.benchmarks.runner; + +import jakarta.xml.bind.JAXBException; +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.BenchmarkList; +import org.openjdk.jmh.runner.BenchmarkListEntry; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +public class BenchRunner { + public static final File REPORT_FILE = new File("./bench-report.xml"); + + /** @return A list of qualified names of all benchmarks visible to JMH. */ + public List getAvailable() { + return BenchmarkList.defaultList().getAll(null, new ArrayList<>()).stream() + .map(BenchmarkListEntry::getUsername) + .collect(Collectors.toList()); + } + + public static void run(String[] args) throws RunnerException { + CommandLineOptions cmdOpts = null; + try { + cmdOpts = new CommandLineOptions(args); + } catch (CommandLineOptionException e) { + System.err.println("Error parsing command line args:"); + System.err.println(" " + e.getMessage()); + System.exit(1); + } + + if (Boolean.getBoolean("bench.compileOnly")) { + // Do not report results from `compileOnly` mode + runCompileOnly(cmdOpts.getIncludes()); + } else { + Runner jmhRunner = new Runner(cmdOpts); + + if (cmdOpts.shouldHelp()) { + System.err.println("Enso benchmark runner: A modified JMH runner for Enso benchmarks."); + try { + cmdOpts.showHelp(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + System.exit(0); + } + + if (cmdOpts.shouldList()) { + jmhRunner.list(); + System.exit(0); + } + + Collection results; + results = jmhRunner.run(); + + for (RunResult result : results) { + try { + reportResult(result.getParams().getBenchmark(), result); + } catch (JAXBException e) { + throw new IllegalStateException("Benchmark result report writing failed", e); + } + } + System.out.println("Benchmark results reported into " + REPORT_FILE.getAbsolutePath()); + } + } + + private static Collection runCompileOnly(List includes) throws RunnerException { + System.out.println("Running benchmarks " + includes + " in compileOnly mode"); + var optsBuilder = new OptionsBuilder() + .measurementIterations(1) + .warmupIterations(0) + .forks(0); + includes.forEach(optsBuilder::include); + var opts = optsBuilder.build(); + var runner = new Runner(opts); + return runner.run(); + } + + public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { + String includeRegex = "^" + label + "$"; + if (Boolean.getBoolean("bench.compileOnly")) { + var results = runCompileOnly(List.of(includeRegex)); + var firstResult = results.iterator().next(); + return reportResult(label, firstResult); + } else { + var opts = new OptionsBuilder() + .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") + .include(includeRegex) + .build(); + RunResult benchmarksResult = new Runner(opts).runSingle(); + return reportResult(label, benchmarksResult); + } + } + + private static BenchmarkItem reportResult(String label, RunResult result) throws JAXBException { + Report report; + if (REPORT_FILE.exists()) { + report = Report.readFromFile(REPORT_FILE); + } else { + report = new Report(); + } + + BenchmarkItem benchItem = + new BenchmarkResultProcessor().processResult(label, report, result); + + Report.writeToFile(report, REPORT_FILE); + return benchItem; + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java new file mode 100644 index 000000000000..eaa022a624ed --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java @@ -0,0 +1,35 @@ +package org.enso.benchmarks.runner; + +import org.openjdk.jmh.results.Result; + +/** Convenience class for clients to compare historic results with the last JMH run. */ +public class BenchmarkItem { + private final Result result; + private final ReportItem previousResults; + + public BenchmarkItem(Result result, ReportItem previousResults) { + this.result = result; + this.previousResults = previousResults; + } + + public Result getResult() { + return result; + } + + public ReportItem getPreviousResults() { + return previousResults; + } + + /** @return Best historic score for the given benchmark (including current run). */ + public double getBestScore() { + return previousResults.getBestScore().orElse(result.getScore()); + } + + public double getScore() { + return result.getScore(); + } + + public String getLabel() { + return result.getLabel(); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java new file mode 100644 index 000000000000..5e3613643683 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java @@ -0,0 +1,21 @@ +package org.enso.benchmarks.runner; + +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.RunResult; + +public class BenchmarkResultProcessor { + /** + * Matches the new result with historic results from the report and updates the report. + * + * @param label The name by which this result should be referred to as in the result. + * @param report Historic runs report. + * @param result Fresh JMH benchmark result. + * @return + */ + public BenchmarkItem processResult(String label, Report report, RunResult result) { + Result primary = result.getPrimaryResult(); + ReportItem item = report.findOrCreateByLabel(label); + item.addScore(primary.getScore()); + return new BenchmarkItem(primary, item); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java new file mode 100644 index 000000000000..aa4e39c5369c --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java @@ -0,0 +1,98 @@ +package org.enso.benchmarks.runner; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; +import jakarta.xml.bind.Unmarshaller; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; + +/** Historic runs report. Supports XML serialization. */ +@XmlRootElement +public class Report { + + private List tests; + + public Report() { + tests = new ArrayList<>(); + } + + public Report(List tests) { + this.tests = tests; + } + + @XmlElementWrapper(name = "cases") + @XmlElement(name = "case") + public List getTests() { + return tests; + } + + public void setTests(List tests) { + this.tests = tests; + } + + /** + * Finds a historic result by label. + * + * @param label name of the result to find. + * @return Result for the given label, if found. + */ + public Optional findByLabel(String label) { + return getTests().stream().filter(item -> item.getLabel().equals(label)).findFirst(); + } + + /** + * Inserts a new report item for a given label. + * + * @param label name for which to allocate a report slot. + * @return Item allocated for the label. + */ + public ReportItem createByLabel(String label) { + ReportItem newItem = new ReportItem(label, new ArrayList<>()); + getTests().add(newItem); + return newItem; + } + + /** + * Finds or creates a new report item for a given label. + * + * @param label name for which an item is needed. + * @return The report item for the label, guaranteed to be present in this Report. + */ + public ReportItem findOrCreateByLabel(String label) { + return findByLabel(label).orElseGet(() -> createByLabel(label)); + } + + /** + * Reads a Report from XML file. + * + * @param file the file from which to read the report. + * @return the Report read from given file. + * @throws JAXBException when the file cannot be read or does not conform to the Report XML + * format. + */ + public static Report readFromFile(File file) throws JAXBException { + JAXBContext jaxbContext = JAXBContext.newInstance(Report.class); + Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + return (Report) unmarshaller.unmarshal(file); + } + + /** + * Serializes a report to an XML file. + * + * @param report Report to serialize. + * @param file File to which the serialized report should be written. + * @throws JAXBException when the file cannot be written to. + */ + public static void writeToFile(Report report, File file) throws JAXBException { + JAXBContext jaxbContext = JAXBContext.newInstance(Report.class); + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + marshaller.marshal(report, file); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java new file mode 100644 index 000000000000..4de292505332 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java @@ -0,0 +1,62 @@ +package org.enso.benchmarks.runner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalDouble; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlTransient; + +/** Contains historic results for a single benchmark identified by label. */ +@XmlRootElement +public class ReportItem { + + private String label; + + private List scores; + + public ReportItem() {} + + public ReportItem(String label, List scores) { + this.label = label; + this.scores = scores; + } + + @XmlElement + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + @XmlElementWrapper(name = "scores") + @XmlElement(name = "score") + public List getScores() { + return scores; + } + + public void setScores(List scores) { + if (scores == null) scores = new ArrayList<>(); + this.scores = scores; + } + + /** + * Registers a new score for this item. + * + * @param score Score to register. + */ + public void addScore(double score) { + getScores().add(score); + } + + /** @return The best (lowest) historic result for this benchmark. */ + @XmlTransient + public Optional getBestScore() { + OptionalDouble min = getScores().stream().mapToDouble(s -> s).min(); + return min.isPresent() ? Optional.of(min.getAsDouble()) : Optional.empty(); + } +} diff --git a/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java b/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java new file mode 100644 index 000000000000..bb2a2c9daf0f --- /dev/null +++ b/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java @@ -0,0 +1,97 @@ +package org.enso.benchmarks; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.net.URISyntaxException; +import java.util.List; +import org.enso.benchmarks.processor.SpecCollector; +import org.enso.pkg.PackageManager; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames.TopScope; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestSpecCollector { + + private Context ctx; + private File ensoDir; + private File ensoHomeOverride; + private Value testCollectorMainModule; + + @Before + public void setup() { + try { + ensoDir = + new File( + TestSpecCollector.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + } catch (URISyntaxException e) { + fail("ensoDir not found: " + e.getMessage()); + } + for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { + if (ensoDir.getName().equals("enso")) { + break; + } + } + assertNotNull("Could not find Enso root directory", ensoDir); + assertTrue(ensoDir.exists()); + assertTrue(ensoDir.isDirectory()); + assertTrue(ensoDir.canRead()); + + // Note that ensoHomeOverride does not have to exist, only its parent directory + ensoHomeOverride = ensoDir.toPath().resolve("distribution").resolve("component").toFile(); + File testCollectorPath = + new File(getClass().getClassLoader().getResource("Test_Collector").getPath()); + var pkg = PackageManager.Default().fromDirectory(testCollectorPath).get(); + var mainModuleName = pkg.moduleNameForFile(pkg.mainFile()); + ctx = + Context.newBuilder(LanguageInfo.ID) + .allowAllAccess(true) + .allowIO(IOAccess.ALL) + .option(RuntimeOptions.LOG_LEVEL, "WARNING") + .option(RuntimeOptions.PROJECT_ROOT, testCollectorPath.getAbsolutePath()) + .option(RuntimeOptions.LANGUAGE_HOME_OVERRIDE, ensoHomeOverride.getAbsolutePath()) + .build(); + testCollectorMainModule = + ctx.getBindings(LanguageInfo.ID) + .invokeMember(TopScope.GET_MODULE, mainModuleName.toString()); + } + + @After + public void tearDown() { + ctx.close(); + } + + @Test + public void testCollectAllSuitesFromMainModule() { + List moduleBenchSuites = + SpecCollector.collectBenchSpecsFromModule(testCollectorMainModule, "group_1"); + for (ModuleBenchSuite moduleBenchSuite : moduleBenchSuites) { + assertEquals(1, moduleBenchSuite.getGroups().size()); + assertEquals("Test Group", moduleBenchSuite.getGroups().get(0).name()); + List specs = moduleBenchSuite.getGroups().get(0).specs(); + assertEquals(1, specs.size()); + assertEquals("Test Spec", specs.get(0).name()); + Value code = specs.get(0).code(); + Value res = code.execute(Value.asValue(null)); + assertEquals(2, res.asInt()); + } + assertFalse(moduleBenchSuites.isEmpty()); + } + + @Test + public void testCollectSuitesNonExistendVarName() { + List moduleBenchSuites = + SpecCollector.collectBenchSpecsFromModule(testCollectorMainModule, "NOOOON_existent_FOO"); + assertTrue(moduleBenchSuites.isEmpty()); + } +} diff --git a/lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml b/lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml new file mode 100644 index 000000000000..5ac425716bcd --- /dev/null +++ b/lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml @@ -0,0 +1,6 @@ +name: Test_Collector +license: APLv2 +enso-version: default +version: "0.0.1" +author: "Enso Team " +maintainer: "Enso Team " diff --git a/lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso b/lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso new file mode 100644 index 000000000000..1f0c3f322c32 --- /dev/null +++ b/lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso @@ -0,0 +1,11 @@ +from Standard.Base import all +from Standard.Test import Bench + +options = Bench.options.size 10 . iter 10 + +collect_benches = Bench.build builder-> + builder.group "Test Group" options group_builder-> + group_builder.specify "Test Spec" (1 + 1) + +group_1 = [collect_benches] + diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala index ea09e1f4c210..10edaed6ee84 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala @@ -6,7 +6,8 @@ import java.nio.file.Path * engine distribution from which the runtime is being run. * * @param languageHome the path to the directory containing the runner.jar and - * runtime.jar of the currently running language runtime + * runtime.jar of the currently running language runtime. + * The path does not have to exist. */ case class LanguageHome(languageHome: Path) { private val rootPath = languageHome.getParent.toAbsolutePath.normalize diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index fae1f27fe381..a8677afe54a4 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -29,6 +29,9 @@ object FrgaalJavaCompiler { val frgaal = "org.frgaal" % "compiler" % "19.0.1" % "provided" val sourceLevel = "19" + val debugArg = + "-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000" + def compilers( classpath: sbt.Keys.Classpath, sbtCompilers: xsbti.compile.Compilers, @@ -65,7 +68,8 @@ object FrgaalJavaCompiler { xsbti.compile.Compilers.of(sbtCompilers.scalac, javaTools) } - /** Helper method to launch programs. */ + /** Helper method to launch programs. + */ def launch( javaHome: Option[Path], compilerJar: Path, @@ -77,8 +81,11 @@ object FrgaalJavaCompiler { source: Option[String], target: String ): Boolean = { - val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) - val outputOption = CompilerArguments.outputOption(output) + val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) + val debugAnotProcessorOpt = jArgs.contains(debugArg) + val strippedJArgs = jArgs + .map(_.stripPrefix("-J")) + val outputOption = CompilerArguments.outputOption(output) val sources = sources0 map { case x: PathBasedFile => x.toPath.toAbsolutePath.toString } @@ -211,11 +218,12 @@ object FrgaalJavaCompiler { val allArguments = outputOption ++ frgaalOptions ++ nonJArgs ++ sources withArgumentFile(allArguments) { argsFile => - // Need to disable standard compiler tools that come with used jdk and replace them - // with the ones provided with Frgaal. - val forkArgs = (jArgs ++ Seq( + val limitModulesArgs = Seq( "--limit-modules", - "java.base,jdk.zipfs,jdk.internal.vm.compiler.management", + "java.base,jdk.zipfs,jdk.internal.vm.compiler.management,org.graalvm.locator,java.desktop,java.net.http" + ) + // strippedJArgs needs to be passed via cmd line, and not via the argument file + val forkArgs = (strippedJArgs ++ limitModulesArgs ++ Seq( "-jar", compilerJar.toString )) :+ @@ -224,6 +232,14 @@ object FrgaalJavaCompiler { val cwd = new File(new File(".").getAbsolutePath).getCanonicalFile val javacLogger = new JavacLogger(log, reporter, cwd) var exitCode = -1 + if (debugAnotProcessorOpt) { + log.info( + s"Frgaal compiler is about to be launched with $debugArg, which means that" + + " it will wait for a debugger to attach. The output from the compiler is by default" + + " redirected, therefore \"Listening to the debugger\" message will not be displayed." + + " You should attach the debugger now." + ) + } try { exitCode = Process(exe +: forkArgs, cwd) ! javacLogger } finally { diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java new file mode 100644 index 000000000000..3714dcc5d4c3 --- /dev/null +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -0,0 +1,21 @@ +package org.enso.benchmarks.libs; + +import org.enso.benchmarks.processor.GenerateBenchSources; +import org.enso.benchmarks.runner.BenchRunner; +import org.openjdk.jmh.runner.RunnerException; + +/** + * The only purpose for this class is to enable the {@link org.enso.benchmarks.processor.BenchProcessor} + * to generate JMH sources for the benchmarks in {@code test/Benchmarks} project. + * For more information see {@code docs/infrastructure/benchmarks.md#Standard-library-benchmarks}. + */ +@GenerateBenchSources( + projectRootPath = "test/Benchmarks", + moduleName = "local.Benchmarks.Main" +) +public class LibBenchRunner { + + public static void main(String[] args) throws RunnerException { + BenchRunner.run(args); + } +} diff --git a/test/Benchmarks/src/Main.enso b/test/Benchmarks/src/Main.enso index daf1b98308a4..d3cc7418f6ba 100644 --- a/test/Benchmarks/src/Main.enso +++ b/test/Benchmarks/src/Main.enso @@ -1,32 +1,11 @@ -import project.Collections -import project.Column_Numeric -import project.Equality -import project.Json_Bench -import project.Natural_Order_Sort -import project.Number_Parse -import project.Numeric -import project.Sum +import project.Vector.Distinct +import project.Vector.Operations -import project.Statistics -import project.Table -import project.Text -import project.Time -import project.Vector +all_benchmarks = + vec_ops = Operations.all + vec_distinct = Distinct.collect_benches + [vec_ops, vec_distinct] -## A common entry point for all benchmarks. Usually just one suite is run, but - sometimes we want to run all benchmarks to ensure our language changes are - not breaking them. main = - Collections.bench - Column_Numeric.bench - Equality.bench - Json_Bench.bench - Natural_Order_Sort.bench - Number_Parse.bench - Numeric.bench - Sum.bench - Statistics.Main.bench - Table.Main.bench - Text.Main.bench - Time.Main.bench - Vector.Main.bench + all_benchmarks.each suite-> + suite.run_main diff --git a/test/Benchmarks/src/Vector/Distinct.enso b/test/Benchmarks/src/Vector/Distinct.enso index 0fc5a4e91c87..37bc755c3f25 100644 --- a/test/Benchmarks/src/Vector/Distinct.enso +++ b/test/Benchmarks/src/Vector/Distinct.enso @@ -1,31 +1,27 @@ from Standard.Base import all -import Standard.Base from Standard.Test import Bench import project.Vector.Utils -polyglot java import org.enso.base.Time_Utils +options = Bench.options . size 100 . iter 20 -## Bench Utilities ============================================================ +random_vec = Utils.make_random_vec 100000 +uniform_vec = Vector.fill 100000 1 +random_text_vec = random_vec.map .to_text +uniform_text_vec = random_vec.map .to_text -iter_size = 100 -num_iterations = 20 +collect_benches = Bench.build builder-> + builder.group "Vector Distinct" options group_builder-> + group_builder.specify "Random Integer Vector Distinct" <| + random_vec.distinct + group_builder.specify "Uniform Integer Vector Distinct" <| + uniform_vec.distinct + group_builder.specify "Random Text Vector Distinct" <| + random_text_vec.distinct + group_builder.specify "Uniform Text Vector Distinct" <| + uniform_text_vec.distinct -# The Benchmarks ============================================================== -bench = - random_vec = Utils.make_random_vec 100000 - uniform_vec = Base.Vector.fill 100000 1 - - random_text_vec = random_vec.map .to_text - uniform_text_vec = random_vec.map .to_text - - Bench.measure (random_vec.distinct) "Random Integer Vector Distinct" iter_size num_iterations - Bench.measure (uniform_vec.distinct) "Uniform Integer Vector Distinct" iter_size num_iterations - - Bench.measure (random_text_vec.distinct) "Random Text Vector Distinct" iter_size num_iterations - Bench.measure (uniform_text_vec.distinct) "Uniform Text Vector Distinct" iter_size num_iterations - -main = bench +main = collect_benches . run_main diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index 678059aaad15..8d0ea4744c1c 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -49,13 +49,10 @@ collect_benches group_builder = State.put s+x bench_measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" -bench = - options = Bench.options . size iter_size . iter num_iterations +options = Bench.options . size iter_size . iter num_iterations - all = Bench.build builder-> - builder.group "Vector Operations" options group_builder-> - collect_benches group_builder +all = Bench.build builder-> + builder.group "Vector Operations" options group_builder-> + collect_benches group_builder - all . run_main - -main = bench +main = all . run_main diff --git a/test/Benchmarks/src/Vector/Utils.enso b/test/Benchmarks/src/Vector/Utils.enso index 9e13a53bac0b..5580c1074b0e 100644 --- a/test/Benchmarks/src/Vector/Utils.enso +++ b/test/Benchmarks/src/Vector/Utils.enso @@ -5,4 +5,4 @@ polyglot java import java.util.Random as Java_Random make_random_vec : Integer -> Vector make_random_vec n = random_gen = Java_Random.new n - Vector.fill n random_gen.nextLong + Vector.fill n random_gen.nextDouble