From 188d465c5f329423eb072e4c1d9683bd34bb4b95 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 22 Jun 2023 11:24:19 +0200 Subject: [PATCH 01/71] [WIP] Invoke Enso bench via JMH --- .../benchmarks/meso/VectorOperations.java | 58 +++++++++++++++++++ .../Vector_Operations.enso | 46 +++++++++++++++ .../org/enso/interpreter/bench/IRUtils.scala | 46 +++++++++++++++ 3 files changed, 150 insertions(+) create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java create mode 100644 engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso create mode 100644 engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java new file mode 100644 index 000000000000..6cd8cea27cfb --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java @@ -0,0 +1,58 @@ +package org.enso.interpreter.bench.benchmarks.meso; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Paths; +import java.util.concurrent.TimeUnit; +import org.enso.compiler.core.IR; +import org.enso.compiler.core.IR$; +import org.graalvm.polyglot.Context; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.BenchmarkMode; +import org.openjdk.jmh.annotations.Fork; +import org.openjdk.jmh.annotations.Measurement; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.annotations.OutputTimeUnit; +import org.openjdk.jmh.annotations.Scope; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.annotations.State; +import org.openjdk.jmh.annotations.Warmup; +import org.openjdk.jmh.infra.BenchmarkParams; +import org.openjdk.jmh.infra.Blackhole; + +@BenchmarkMode(Mode.AverageTime) +@Fork(1) +@Warmup(iterations = 3) +@Measurement(iterations = 5) +@OutputTimeUnit(TimeUnit.MILLISECONDS) +@State(Scope.Benchmark) +public class VectorOperations { + + @Setup + public void initializeBenchmark(BenchmarkParams params) throws Exception { + var ctx = Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(true) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option( + "enso.languageHomeOverride", + Paths.get("../../distribution/component").toFile().getAbsolutePath() + ).build(); + + String benchMeasureRedef = """ + type Bench + measure : Any -> Text -> Integer -> Integer -> Nothing + measure ~act label iter_size num_iters = + ... + """; + } + + private void collectBenchMeasureMethodCalls() { + new IR.Error.ImportExport.ModuleDoesNotExist("bla"); + } + + @Benchmark + public void benchmark(Blackhole blackhole) { + + } +} diff --git a/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso b/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso new file mode 100644 index 000000000000..bfe31103f89c --- /dev/null +++ b/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso @@ -0,0 +1,46 @@ +from Standard.Base import all +import Standard.Base.Runtime.State +import Standard.Base + +from Standard.Test import Bench + +import project.Vector.Utils + +polyglot java import java.util.Random as Java_Random + +## Bench Utilities ============================================================ + +vector_size = 1000000 +iter_size = 100 +num_iterations = 10 + +# The Benchmarks ============================================================== + +bench = + random_vec = Utils.make_random_vec vector_size + random_vec_2 = Utils.make_random_vec 100000 + random_gen = Java_Random.new 123456 + + Bench.measure (Base.Vector.new vector_size i->i) "New Vector" iter_size num_iterations + Bench.measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations + Bench.measure (Base.Vector.new vector_size _->random_gen.nextLong) "New Random" iter_size num_iterations + Bench.measure (Base.Vector.fill vector_size 42) "Fill Constant" iter_size num_iterations + Bench.measure (Base.Vector.fill vector_size random_gen.nextLong) "Fill Random (constant)" iter_size num_iterations + Bench.measure (random_vec + [1]) "Append Single" iter_size num_iterations + Bench.measure (random_vec + random_vec_2) "Append Large" iter_size num_iterations + Bench.measure (random_vec.reduce (+)) "Sum" iter_size num_iterations + Bench.measure ((random_vec.drop (First 20)).reduce (+)) "Drop First 20 and Sum" iter_size num_iterations + Bench.measure ((random_vec.drop (Last 20)).reduce (+)) "Drop Last 20 and Sum" iter_size num_iterations + Bench.measure (random_vec.filter (x -> x % 3 == 1)) "Filter" iter_size num_iterations + Bench.measure (random_vec.filter_with_index (i-> x-> (i+x) % 3 == 1)) "Filter With Index" iter_size num_iterations + + Bench.measure (random_vec . map (x -> x + random_gen.nextLong) . filter (x -> x % 3 == 1)) "Map & Filter" iter_size num_iterations + Bench.measure (random_vec.partition (x -> x % 3 == 1)) "Partition" iter_size num_iterations + Bench.measure (random_vec.partition_with_index (i-> x-> (i+x) % 3 == 1)) "Partition With Index" iter_size num_iterations + + stateful_fun x = + s = State.get Number + State.put s+x + Bench.measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations + +main = bench diff --git a/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala b/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala new file mode 100644 index 000000000000..94d92381fae6 --- /dev/null +++ b/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala @@ -0,0 +1,46 @@ +package org.enso.interpreter.bench + +import org.enso.compiler.core.IR + +object IRUtils { + private def collectBenchCalls(ir: IR.Module): List[BenchCall] = { + val methods = ir.bindings.collect { case m : IR.Module.Scope.Definition.Method => m } + + for (method <- methods) { + method.body.preorder.flatMap(ir => { + ir match { + case application: IR.Application.Prefix => + application.function match { + case IR.Name.Literal("measure", true, _, _, _) => + application.arguments.head match { + case IR.CallArgument.Specified(_, value, _, _, _) => + value match { + case IR.Name.Literal("Bench", _, _, _, _) => + Some(new BenchCall(application)) + case _ => None + } + } + case _ => None + } + } + }) + } + + } + + private class BenchCall( + val call: IR.Application.Prefix + ) { + val args: List[IR.CallArgument.Specified] = call.arguments.collect { case s : IR.CallArgument.Specified => s } + val location: IR.IdentifiedLocation = call.location.get + assert(args.length == 5) + assert(args.head.value.asInstanceOf[IR.Name.Literal].name == "Bench") + val action: IR.Expression = args(1).value + val title: String = { + args(2).value match { + case lit: IR.Literal.Text => lit.text + case unexpected => throw new IllegalStateException(s"Unexpected bench title IR: ${unexpected}") + } + } + } +} From 8836629b4bff7e067caa77665092eaf7b0e637da Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Sat, 1 Jul 2023 08:57:20 +0200 Subject: [PATCH 02/71] Abstract from direct call to Bench.measure --- test/Benchmarks/src/Vector/Operations.enso | 36 ++++++++++++---------- 1 file changed, 19 insertions(+), 17 deletions(-) diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index bfe31103f89c..67c795212221 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -21,26 +21,28 @@ bench = random_vec_2 = Utils.make_random_vec 100000 random_gen = Java_Random.new 123456 - Bench.measure (Base.Vector.new vector_size i->i) "New Vector" iter_size num_iterations - Bench.measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations - Bench.measure (Base.Vector.new vector_size _->random_gen.nextLong) "New Random" iter_size num_iterations - Bench.measure (Base.Vector.fill vector_size 42) "Fill Constant" iter_size num_iterations - Bench.measure (Base.Vector.fill vector_size random_gen.nextLong) "Fill Random (constant)" iter_size num_iterations - Bench.measure (random_vec + [1]) "Append Single" iter_size num_iterations - Bench.measure (random_vec + random_vec_2) "Append Large" iter_size num_iterations - Bench.measure (random_vec.reduce (+)) "Sum" iter_size num_iterations - Bench.measure ((random_vec.drop (First 20)).reduce (+)) "Drop First 20 and Sum" iter_size num_iterations - Bench.measure ((random_vec.drop (Last 20)).reduce (+)) "Drop Last 20 and Sum" iter_size num_iterations - Bench.measure (random_vec.filter (x -> x % 3 == 1)) "Filter" iter_size num_iterations - Bench.measure (random_vec.filter_with_index (i-> x-> (i+x) % 3 == 1)) "Filter With Index" iter_size num_iterations - - Bench.measure (random_vec . map (x -> x + random_gen.nextLong) . filter (x -> x % 3 == 1)) "Map & Filter" iter_size num_iterations - Bench.measure (random_vec.partition (x -> x % 3 == 1)) "Partition" iter_size num_iterations - Bench.measure (random_vec.partition_with_index (i-> x-> (i+x) % 3 == 1)) "Partition With Index" iter_size num_iterations + bench_measure ~act name s n = Bench.measure act name s n + + bench_measure (Base.Vector.new vector_size i->i) "New Vector" iter_size num_iterations + bench_measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations + bench_measure (Base.Vector.new vector_size _->random_gen.nextLong) "New Random" iter_size num_iterations + bench_measure (Base.Vector.fill vector_size 42) "Fill Constant" iter_size num_iterations + bench_measure (Base.Vector.fill vector_size random_gen.nextLong) "Fill Random (constant)" iter_size num_iterations + bench_measure (random_vec + [1]) "Append Single" iter_size num_iterations + bench_measure (random_vec + random_vec_2) "Append Large" iter_size num_iterations + bench_measure (random_vec.reduce (+)) "Sum" iter_size num_iterations + bench_measure ((random_vec.drop (First 20)).reduce (+)) "Drop First 20 and Sum" iter_size num_iterations + bench_measure ((random_vec.drop (Last 20)).reduce (+)) "Drop Last 20 and Sum" iter_size num_iterations + bench_measure (random_vec.filter (x -> x % 3 == 1)) "Filter" iter_size num_iterations + bench_measure (random_vec.filter_with_index (i-> x-> (i+x) % 3 == 1)) "Filter With Index" iter_size num_iterations + + bench_measure (random_vec . map (x -> x + random_gen.nextLong) . filter (x -> x % 3 == 1)) "Map & Filter" iter_size num_iterations + bench_measure (random_vec.partition (x -> x % 3 == 1)) "Partition" iter_size num_iterations + bench_measure (random_vec.partition_with_index (i-> x-> (i+x) % 3 == 1)) "Partition With Index" iter_size num_iterations stateful_fun x = s = State.get Number State.put s+x - Bench.measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations + bench_measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations main = bench From 88fd6fb9888d939c2d9745a7397f6e87dec4fa4d Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Sat, 1 Jul 2023 09:03:04 +0200 Subject: [PATCH 03/71] Builder pattern to collect all benchmarks first and only then execute them --- .../Standard/Test/0.0.0-dev/src/Bench.enso | 64 +++++++++++++++++++ test/Benchmarks/src/Vector/Operations.enso | 13 +++- 2 files changed, 74 insertions(+), 3 deletions(-) diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso index 63b078898e56..115d9e39e42c 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso @@ -1,7 +1,71 @@ from Standard.Base import all import Standard.Base.Runtime.Ref.Ref + +type Bench_Options + ## PRIVATE + Impl iter_size num_iters + + size : Integer -> Bench_Options + size self v = Bench_Options.Impl v self.num_iters + + iter : Integer -> Bench_Options + iter self v = Bench_Options.Impl self.iter_size v + + to_text self = "[iter_size=" + self.iter_size.to_text + ", num_iters=" + self.num_iters.to_text + "]" + +type Bench_Builder + ## PRIVATE + Impl builder + + group : Text -> Bench_Options -> (Group_Builder -> Any) -> Any + group self (name:Text) (configuration:Bench_Options) fn = + b = Vector.new_builder + fn (Group_Builder.Impl b) + self.builder.append <| Bench.Group name configuration b.to_vector + +type Group_Builder + ## PRIVATE + Impl builder + + specify : Text -> Any -> Bench + specify self (name:Text) ~benchmark = + self.builder.append <| Bench.Spec name (_ -> benchmark) + + type Bench + All (groups : Vector Bench) + Group (name:Text) (configuration:Bench_Options) (specs : Vector Bench) + Spec (name:Text) (code : Any -> Any) + + build : (Bench_Builder -> Any) -> Bench + build fn = + b = Vector.new_builder + fn (Bench_Builder.Impl b) + Bench.All b.to_vector + + options : Bench_Options + options = Bench_Options.Impl -1 -1 + + fold : Any -> (Any -> Bench -> Bench -> Any) -> Any + fold self value fn = case self of + Bench.All groups -> groups.fold value (v-> g-> g.fold v fn) + Bench.Group _ _ specs -> specs.fold value (v-> s-> fn v self s) + Bench.Spec _ _ -> fn value self self + + run_main self = + count = self.fold 0 v-> g-> s-> + IO.println <| "Found group: " + g.name + " with config: " + g.configuration.to_text + " and spec: " + s.name + v+1 + + IO.println <| "Found " + count.to_text + " cases to execute" + + self.fold Nothing _-> g-> s-> + c = g.configuration + IO.println <| "Benchmarking " + s.name + " configuration: " + c.to_text + Bench.measure (s.code 0) s.name c.iter_size c.num_iters + IO.println <| "Done bench of " + s.name + ## Measure the amount of time it takes to execute a given computation. Arguments: diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index 67c795212221..a475866e7219 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -16,12 +16,12 @@ num_iterations = 10 # The Benchmarks ============================================================== -bench = +bench group_builder = random_vec = Utils.make_random_vec vector_size random_vec_2 = Utils.make_random_vec 100000 random_gen = Java_Random.new 123456 - bench_measure ~act name s n = Bench.measure act name s n + bench_measure ~act name _ _ = group_builder.specify name act bench_measure (Base.Vector.new vector_size i->i) "New Vector" iter_size num_iterations bench_measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations @@ -45,4 +45,11 @@ bench = State.put s+x bench_measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations -main = bench +main = + options = Bench.options . size iter_size . iter num_iterations + + all = Bench.build builder-> + builder.group "Vector Operations" options group_builder-> + bench group_builder + + all . run_main From 19a439e37092377a70871c64b3d213770e6542f5 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 7 Jul 2023 17:44:31 +0200 Subject: [PATCH 04/71] Improve type ascriptions of Bench type --- .../lib/Standard/Test/0.0.0-dev/src/Bench.enso | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso index 115d9e39e42c..d68e06578044 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso @@ -1,40 +1,43 @@ from Standard.Base import all import Standard.Base.Runtime.Ref.Ref +import Standard.Base.Data.Vector.Builder as Vector_Builder type Bench_Options ## PRIVATE - Impl iter_size num_iters + Impl (iter_size:Integer) (num_iters:Integer) size : Integer -> Bench_Options - size self v = Bench_Options.Impl v self.num_iters + size self s = Bench_Options.Impl s self.num_iters iter : Integer -> Bench_Options - iter self v = Bench_Options.Impl self.iter_size v + iter self i = Bench_Options.Impl self.iter_size i to_text self = "[iter_size=" + self.iter_size.to_text + ", num_iters=" + self.num_iters.to_text + "]" type Bench_Builder ## PRIVATE - Impl builder + Impl (builder : Vector_Builder) - group : Text -> Bench_Options -> (Group_Builder -> Any) -> Any + group : Text -> Bench_Options -> (Group_Builder -> Any) -> Bench_Builder group self (name:Text) (configuration:Bench_Options) fn = b = Vector.new_builder fn (Group_Builder.Impl b) self.builder.append <| Bench.Group name configuration b.to_vector + self type Group_Builder ## PRIVATE Impl builder - specify : Text -> Any -> Bench + specify : Text -> Any -> Group_Builder specify self (name:Text) ~benchmark = self.builder.append <| Bench.Spec name (_ -> benchmark) + self type Bench - All (groups : Vector Bench) + All (groups : Vector Group) Group (name:Text) (configuration:Bench_Options) (specs : Vector Bench) Spec (name:Text) (code : Any -> Any) From f6dd4237d5ab3ca3011c1c4ae60358ba8f4c5b4e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 7 Jul 2023 17:45:25 +0200 Subject: [PATCH 05/71] [WIP] Add LibBenchRunner --- .../bench/benchmarks/meso/LibBench.java | 17 +++++ .../bench/benchmarks/meso/LibBenchRunner.java | 66 +++++++++++++++++++ .../simple.enso | 19 ++++++ 3 files changed, 102 insertions(+) create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java create mode 100644 engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java new file mode 100644 index 000000000000..646372265ef4 --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java @@ -0,0 +1,17 @@ +package org.enso.interpreter.bench.benchmarks.meso; + +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Setup; +import org.openjdk.jmh.infra.BenchmarkParams; + +public class LibBench { + @Setup + public void setup(BenchmarkParams params) { + String benchName = params.getBenchmark(); + } + + @Benchmark + public void benchmark() { + + } +} diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java new file mode 100644 index 000000000000..4ec5446ae4d8 --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java @@ -0,0 +1,66 @@ +package org.enso.interpreter.bench.benchmarks.meso; + +import java.io.ByteArrayOutputStream; +import java.nio.file.Paths; +import java.util.Collection; +import java.util.Collections; +import org.graalvm.polyglot.Context; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.options.OptionsBuilder; + +// TODO: Move to different project? +public class LibBenchRunner { + public static void main(String[] args) { + CommandLineOptions cmdOpts = null; + try { + cmdOpts = new CommandLineOptions(args); + } catch (CommandLineOptionException e) { + System.err.println("Error parsing command line args:"); + System.err.println(" " + e.getMessage()); + System.exit(1); + } + + // Merge cmdOpts with chainedOptsBuilder + ChainedOptionsBuilder optsBuilder = new OptionsBuilder() + .include(cmdOpts.getIncludes().get(0)); + + Runner jmhRunner = new Runner(optsBuilder.build()); + Collection results = Collections.emptyList(); + try { + results = jmhRunner.run(); + } catch (RunnerException e) { + throw new RuntimeException(e); + } + + // TODO: Collect all the results + + } + + public void run(String label) { + ChainedOptionsBuilder builder = new OptionsBuilder() + .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") + .include("^" + label + "$"); + } + + private void collectBenchSpecs() { + String benchmarksProjectRootDir = "test/Benchmarks/src"; + String singleFile = "/home/pavel/dev/enso/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso"; + var ctx = Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(true) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option( + "enso.languageHomeOverride", + Paths.get("../../distribution/component").toFile().getAbsolutePath() + ).build(); + + ctx.eval()... + } +} diff --git a/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso b/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso new file mode 100644 index 000000000000..0241465d964e --- /dev/null +++ b/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso @@ -0,0 +1,19 @@ +from Standard.Base import all +import Standard.Test.Bench.Bench + +options = Bench.options . size 10 . iter 5 + +vec = [1,2,3,4,5] + +all = Bench.build builder-> + builder.group "group_1" options group_builder-> + group_builder.specify "bench_1" <| + vec + [10] + builder.group "group_2" options group_builder-> + group_builder.specify "bench_2" <| + vec + [20] + +main = + all.run_main + + From 58c1e0d0e208397a15e2381186d3754fb3d1d5cf Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Sun, 9 Jul 2023 17:39:38 +0200 Subject: [PATCH 06/71] Making the project compilable (again) --- .../org/enso/interpreter/bench/benchmarks/meso/LibBench.java | 2 +- .../interpreter/bench/benchmarks/meso/LibBenchRunner.java | 5 ++--- .../src/bench/scala/org/enso/interpreter/bench/IRUtils.scala | 5 ++--- 3 files changed, 5 insertions(+), 7 deletions(-) diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java index 646372265ef4..c4b1c00e7eab 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java @@ -5,7 +5,7 @@ import org.openjdk.jmh.infra.BenchmarkParams; public class LibBench { - @Setup + // @Setup public void setup(BenchmarkParams params) { String benchName = params.getBenchmark(); } diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java index 4ec5446ae4d8..f4b905c6f77b 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java @@ -5,13 +5,12 @@ import java.util.Collection; import java.util.Collections; import org.graalvm.polyglot.Context; +import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; import org.openjdk.jmh.runner.options.CommandLineOptionException; import org.openjdk.jmh.runner.options.CommandLineOptions; -import org.openjdk.jmh.runner.options.Options; -import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.options.OptionsBuilder; // TODO: Move to different project? @@ -61,6 +60,6 @@ private void collectBenchSpecs() { Paths.get("../../distribution/component").toFile().getAbsolutePath() ).build(); - ctx.eval()... + // ctx.eval()... } } diff --git a/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala b/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala index 94d92381fae6..3265b42f4b42 100644 --- a/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala +++ b/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala @@ -1,8 +1,7 @@ package org.enso.interpreter.bench -import org.enso.compiler.core.IR - object IRUtils { + /* private def collectBenchCalls(ir: IR.Module): List[BenchCall] = { val methods = ir.bindings.collect { case m : IR.Module.Scope.Definition.Method => m } @@ -25,7 +24,6 @@ object IRUtils { } }) } - } private class BenchCall( @@ -43,4 +41,5 @@ object IRUtils { } } } + */ } From f5a9aaea2c0c3d46fb789b12a7962e9a76f226f4 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Sun, 9 Jul 2023 17:43:23 +0200 Subject: [PATCH 07/71] Executed few suites from Vector/Operations.enso in JMH --- .../benchmarks/meso/VectorOperations.java | 118 ++++++++++++++---- test/Benchmarks/src/Vector/Operations.enso | 7 +- 2 files changed, 101 insertions(+), 24 deletions(-) diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java index 6cd8cea27cfb..a6e5a1d73db3 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java @@ -1,11 +1,17 @@ package org.enso.interpreter.bench.benchmarks.meso; import java.io.ByteArrayOutputStream; +import java.io.File; import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.TimeUnit; -import org.enso.compiler.core.IR; -import org.enso.compiler.core.IR$; +import java.util.function.Predicate; +import java.util.stream.Collectors; +import org.enso.polyglot.MethodNames; import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.BenchmarkMode; import org.openjdk.jmh.annotations.Fork; @@ -21,38 +27,108 @@ @BenchmarkMode(Mode.AverageTime) @Fork(1) -@Warmup(iterations = 3) -@Measurement(iterations = 5) +@Warmup(iterations = 3, time=3) +@Measurement(iterations = 3, time=1) @OutputTimeUnit(TimeUnit.MILLISECONDS) @State(Scope.Benchmark) public class VectorOperations { + private Value code; @Setup public void initializeBenchmark(BenchmarkParams params) throws Exception { + File sourceFile; + for (var folder = Paths.get("../../distribution/component").toFile();; folder = folder.getParentFile()) { + sourceFile = fileFrom(folder, "test", "Benchmarks", "src", "Vector", "Operations.enso"); + if (sourceFile.canRead()) { + break; + } + } + sourceFile = sourceFile.getCanonicalFile(); + var projectRoot = sourceFile.getParentFile().getParentFile().getParent(); + var ctx = Context.newBuilder() - .allowExperimentalOptions(true) - .allowIO(true) - .allowAllAccess(true) - .logHandler(new ByteArrayOutputStream()) - .option( - "enso.languageHomeOverride", - Paths.get("../../distribution/component").toFile().getAbsolutePath() - ).build(); + .allowExperimentalOptions(true) + .allowIO(true) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option("enso.projectRoot", projectRoot).option( + "enso.languageHomeOverride", + Paths.get("../../distribution/component").toFile().getAbsolutePath() + ).build(); + + var src = Source.newBuilder("enso", sourceFile).build(); + var module = ctx.eval(src); + var all = module.invokeMember(MethodNames.Module.EVAL_EXPRESSION, "bench_suites").as(All.class); + var allSuites = new ArrayList(); + for (var g : all.groups()) { + for (var s : g.specs()) { + allSuites.add(s); + } + } + + var shortName = params.getBenchmark().replaceAll(".*\\.", ""); + + Predicate matches = (s) -> { + var name = s.name().replaceAll(" ", ""); + return shortName.equalsIgnoreCase(name); + }; + + var firstOption = allSuites.stream().filter(matches).findFirst(); + if (firstOption.isEmpty()) { + var names = allSuites.stream().map(Suite::name).collect(Collectors.toList()); + throw new IllegalStateException("Cannot find suite: " + shortName + " among: " + names); + } + var found = firstOption.get(); + if (found.code() == null) { + throw new NullPointerException("no code member: " + found); + } + this.code = found.code(); + } + + public static interface All { + List groups(); + } + + public static interface Group { + String name(); + List specs(); + } - String benchMeasureRedef = """ - type Bench - measure : Any -> Text -> Integer -> Integer -> Nothing - measure ~act label iter_size num_iters = - ... - """; + public static interface Suite { + String name(); + Value code(); } - private void collectBenchMeasureMethodCalls() { - new IR.Error.ImportExport.ModuleDoesNotExist("bla"); + private static File fileFrom(File folder, String... children) { + File f = folder; + for (int i = 0; i < children.length; i++) { + f = new File(f, children[i]); + } + return f; } @Benchmark - public void benchmark(Blackhole blackhole) { + public void newVector(Blackhole blackhole) { + benchmark(blackhole); + } + + @Benchmark + public void newConstant(Blackhole blackhole) { + benchmark(blackhole); + } + + @Benchmark + public void newRandom(Blackhole blackhole) { + benchmark(blackhole); + } + + @Benchmark + public void sum(Blackhole blackhole) { + benchmark(blackhole); + } + protected final void benchmark(Blackhole blackhole) { + var result = code.execute(0); + blackhole.consume(result); } } diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index a475866e7219..6253ddadf984 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -45,11 +45,12 @@ bench group_builder = State.put s+x bench_measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations -main = + +bench_suites = options = Bench.options . size iter_size . iter num_iterations - all = Bench.build builder-> + Bench.build builder-> builder.group "Vector Operations" options group_builder-> bench group_builder - all . run_main +main = bench_suites . run_main From 7c33b3a4bfae20f8e7de5745cbada077c7d94044 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 10 Jul 2023 14:46:45 +0100 Subject: [PATCH 08/71] Bench is the type, not Group --- distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso index d68e06578044..04a1381630b0 100644 --- a/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso +++ b/distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso @@ -37,7 +37,7 @@ type Group_Builder type Bench - All (groups : Vector Group) + All (groups : Vector Bench) Group (name:Text) (configuration:Bench_Options) (specs : Vector Bench) Spec (name:Text) (code : Any -> Any) From f70a6da226b94d39f0b5e0cbf9b419ba74705ef3 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 10 Jul 2023 14:54:03 +0100 Subject: [PATCH 09/71] sumStatistic benchmark --- .../interpreter/bench/benchmarks/meso/VectorOperations.java | 5 +++++ test/Benchmarks/src/Vector/Operations.enso | 1 + 2 files changed, 6 insertions(+) diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java index a6e5a1d73db3..7a02205c76ba 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java @@ -127,6 +127,11 @@ public void sum(Blackhole blackhole) { benchmark(blackhole); } + @Benchmark + public void sumStatistic(Blackhole blackhole) { + benchmark(blackhole); + } + protected final void benchmark(Blackhole blackhole) { var result = code.execute(0); blackhole.consume(result); diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index 6253ddadf984..7c552915c32a 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -31,6 +31,7 @@ bench group_builder = bench_measure (random_vec + [1]) "Append Single" iter_size num_iterations bench_measure (random_vec + random_vec_2) "Append Large" iter_size num_iterations bench_measure (random_vec.reduce (+)) "Sum" iter_size num_iterations + bench_measure (random_vec.compute Statistic.Sum) "Sum Statistic" iter_size num_iterations bench_measure ((random_vec.drop (First 20)).reduce (+)) "Drop First 20 and Sum" iter_size num_iterations bench_measure ((random_vec.drop (Last 20)).reduce (+)) "Drop Last 20 and Sum" iter_size num_iterations bench_measure (random_vec.filter (x -> x % 3 == 1)) "Filter" iter_size num_iterations From 15843572be9d49c3b46652c48f60650f72b6d011 Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Mon, 10 Jul 2023 16:43:17 +0100 Subject: [PATCH 10/71] Generate double values --- test/Benchmarks/src/Vector/Utils.enso | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Benchmarks/src/Vector/Utils.enso b/test/Benchmarks/src/Vector/Utils.enso index 9e13a53bac0b..5580c1074b0e 100644 --- a/test/Benchmarks/src/Vector/Utils.enso +++ b/test/Benchmarks/src/Vector/Utils.enso @@ -5,4 +5,4 @@ polyglot java import java.util.Random as Java_Random make_random_vec : Integer -> Vector make_random_vec n = random_gen = Java_Random.new n - Vector.fill n random_gen.nextLong + Vector.fill n random_gen.nextDouble From 0f4177722bb67eb6010b5194780ee076f49cd09f Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 17 Jul 2023 17:45:13 +0200 Subject: [PATCH 11/71] Add prototype of bench suites discovery --- .../bench/benchmarks/meso/BenchConfig.java | 5 ++ .../bench/benchmarks/meso/BenchGroup.java | 9 ++++ .../bench/benchmarks/meso/BenchSpec.java | 11 +++++ .../bench/benchmarks/meso/BenchSuite.java | 7 +++ .../bench/benchmarks/meso/LibBenchRunner.java | 46 ++++++++++++++++--- 5 files changed, 72 insertions(+), 6 deletions(-) create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java create mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java new file mode 100644 index 000000000000..03ee10782f71 --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java @@ -0,0 +1,5 @@ +package org.enso.interpreter.bench.benchmarks.meso; +public interface BenchConfig { + int size(); + int iter(); +} diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java new file mode 100644 index 000000000000..76932aa14b0a --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java @@ -0,0 +1,9 @@ +package org.enso.interpreter.bench.benchmarks.meso; + +import java.util.List; + +public interface BenchGroup { + String name(); + BenchConfig configuration(); + List specs(); +} diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java new file mode 100644 index 000000000000..965b8441d404 --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java @@ -0,0 +1,11 @@ +package org.enso.interpreter.bench.benchmarks.meso; + +import org.graalvm.polyglot.Value; + +/** + * Specification of a single benchmark. + */ +public interface BenchSpec { + String name(); + Value code(); +} diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java new file mode 100644 index 000000000000..7670d9daf9d9 --- /dev/null +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java @@ -0,0 +1,7 @@ +package org.enso.interpreter.bench.benchmarks.meso; + +import java.util.List; + +public interface BenchSuite { + List groups(); +} diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java index f4b905c6f77b..5ff24ab8dfc8 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java @@ -1,10 +1,21 @@ package org.enso.interpreter.bench.benchmarks.meso; import java.io.ByteArrayOutputStream; +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.Set; +import org.enso.polyglot.MethodNames; +import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; @@ -25,6 +36,13 @@ public static void main(String[] args) { System.exit(1); } + Set benchSpecs = collectAllBenchSpecs(); + for (BenchSuite benchSpec : benchSpecs) { + for (BenchGroup group : benchSpec.groups()) { + System.out.println("Discovered group: " + group.name()); + } + } + // Merge cmdOpts with chainedOptsBuilder ChainedOptionsBuilder optsBuilder = new OptionsBuilder() .include(cmdOpts.getIncludes().get(0)); @@ -47,10 +65,16 @@ public void run(String label) { .include("^" + label + "$"); } - private void collectBenchSpecs() { + private static Set collectAllBenchSpecs() { String benchmarksProjectRootDir = "test/Benchmarks/src"; - String singleFile = "/home/pavel/dev/enso/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso"; - var ctx = Context.newBuilder() + File singleFile = new File("/home/pavel/dev/enso/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso"); + return Set.of(collectBenchSpecsFromSingleFile(singleFile)); + } + + private static BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { + assert benchFile.exists(); + assert benchFile.canRead(); + try (var ctx = Context.newBuilder() .allowExperimentalOptions(true) .allowIO(true) .allowAllAccess(true) @@ -58,8 +82,18 @@ private void collectBenchSpecs() { .option( "enso.languageHomeOverride", Paths.get("../../distribution/component").toFile().getAbsolutePath() - ).build(); - - // ctx.eval()... + ).build()) { + Source source; + try { + source = Source.newBuilder("enso", benchFile).build(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + Value module = ctx.eval(source); + var benchSuite = module + .invokeMember(Module.EVAL_EXPRESSION, "all") + .as(BenchSuite.class); + return benchSuite; + } } } From 9d9f328ec384e303d4cdf6a23e6deccd2cf8c755 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 17 Jul 2023 18:43:33 +0200 Subject: [PATCH 12/71] Move LibBenchRunner to a separate project `bench-libs` --- build.sbt | 32 +++++++++++ .../enso/benchmarks/libs}/BenchConfig.java | 2 +- .../org/enso/benchmarks/libs}/BenchGroup.java | 2 +- .../org/enso/benchmarks/libs}/BenchSpec.java | 2 +- .../org/enso/benchmarks/libs}/BenchSuite.java | 2 +- .../org/enso/benchmarks/libs}/LibBench.java | 2 +- .../enso/benchmarks/libs}/LibBenchRunner.java | 57 ++++++++++--------- 7 files changed, 67 insertions(+), 32 deletions(-) rename {engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso => tools/bench-libs/src/main/java/org/enso/benchmarks/libs}/BenchConfig.java (53%) rename {engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso => tools/bench-libs/src/main/java/org/enso/benchmarks/libs}/BenchGroup.java (71%) rename {engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso => tools/bench-libs/src/main/java/org/enso/benchmarks/libs}/BenchSpec.java (74%) rename {engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso => tools/bench-libs/src/main/java/org/enso/benchmarks/libs}/BenchSuite.java (62%) rename {engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso => tools/bench-libs/src/main/java/org/enso/benchmarks/libs}/LibBench.java (85%) rename {engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso => tools/bench-libs/src/main/java/org/enso/benchmarks/libs}/LibBenchRunner.java (80%) diff --git a/build.sbt b/build.sbt index feec6f5a2616..36bc6d96d008 100644 --- a/build.sbt +++ b/build.sbt @@ -1722,6 +1722,38 @@ lazy val `distribution-manager` = project .dependsOn(pkg) .dependsOn(`logging-utils`) +lazy val `bench-libs` = (project in file("tools/bench-libs")) + .configs(Benchmark) + .settings( + frgaalJavaCompilerSetting, + assembly / mainClass := (Compile / run / mainClass).value, + libraryDependencies ++= jmh ++ Seq( + "org.openjdk.jmh" % "jmh-core" % jmhVersion, + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, + "org.graalvm.sdk" % "graal-sdk" % graalVersion % "provided", + ), + commands += WithDebugCommand.withDebug, + (Compile / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), + (Compile / run / fork) := true, + (Compile / run / connectInput) := true, + (Compile / javaOptions) ++= { + val runtimeClasspath = + (LocalProject("runtime") / Compile / fullClasspath).value + val runtimeInstrumentsClasspath = + (LocalProject( + "runtime-with-instruments" + ) / Compile / fullClasspath).value + val appendClasspath = + (runtimeClasspath ++ runtimeInstrumentsClasspath) + .map(_.data) + .mkString(File.pathSeparator) + Seq( + s"-Dtruffle.class.path.append=$appendClasspath", + ) + }, + ) + .dependsOn(runtime) + lazy val editions = project .in(file("lib/scala/editions")) .configs(Test) diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchConfig.java similarity index 53% rename from engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java rename to tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchConfig.java index 03ee10782f71..6cb29c6b182d 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchConfig.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchConfig.java @@ -1,4 +1,4 @@ -package org.enso.interpreter.bench.benchmarks.meso; +package org.enso.benchmarks.libs; public interface BenchConfig { int size(); int iter(); diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchGroup.java similarity index 71% rename from engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java rename to tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchGroup.java index 76932aa14b0a..dd8278c81dc7 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchGroup.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchGroup.java @@ -1,4 +1,4 @@ -package org.enso.interpreter.bench.benchmarks.meso; +package org.enso.benchmarks.libs; import java.util.List; diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSpec.java similarity index 74% rename from engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java rename to tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSpec.java index 965b8441d404..41a76dd198c6 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSpec.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSpec.java @@ -1,4 +1,4 @@ -package org.enso.interpreter.bench.benchmarks.meso; +package org.enso.benchmarks.libs; import org.graalvm.polyglot.Value; diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSuite.java similarity index 62% rename from engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java rename to tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSuite.java index 7670d9daf9d9..6ef8db91cd84 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/BenchSuite.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSuite.java @@ -1,4 +1,4 @@ -package org.enso.interpreter.bench.benchmarks.meso; +package org.enso.benchmarks.libs; import java.util.List; diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java similarity index 85% rename from engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java rename to tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java index c4b1c00e7eab..307671755eb5 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBench.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java @@ -1,4 +1,4 @@ -package org.enso.interpreter.bench.benchmarks.meso; +package org.enso.benchmarks.libs; import org.openjdk.jmh.annotations.Benchmark; import org.openjdk.jmh.annotations.Setup; diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java similarity index 80% rename from engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java rename to tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java index 5ff24ab8dfc8..28db0ed9b54c 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/LibBenchRunner.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -1,17 +1,12 @@ -package org.enso.interpreter.bench.benchmarks.meso; +package org.enso.benchmarks.libs; import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; -import java.util.List; -import java.util.Objects; import java.util.Set; -import org.enso.polyglot.MethodNames; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.Source; @@ -24,8 +19,9 @@ import org.openjdk.jmh.runner.options.CommandLineOptions; import org.openjdk.jmh.runner.options.OptionsBuilder; -// TODO: Move to different project? public class LibBenchRunner { + private static Context ctx; + public static void main(String[] args) { CommandLineOptions cmdOpts = null; try { @@ -36,6 +32,7 @@ public static void main(String[] args) { System.exit(1); } + initCtx(); Set benchSpecs = collectAllBenchSpecs(); for (BenchSuite benchSpec : benchSpecs) { for (BenchGroup group : benchSpec.groups()) { @@ -56,7 +53,23 @@ public static void main(String[] args) { } // TODO: Collect all the results + closeCtx(); + } + + private static void initCtx() { + ctx = Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(true) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option( + "enso.languageHomeOverride", + Paths.get("../../distribution/component").toFile().getAbsolutePath() + ).build(); + } + private static void closeCtx() { + ctx.close(); } public void run(String label) { @@ -74,26 +87,16 @@ private static Set collectAllBenchSpecs() { private static BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { assert benchFile.exists(); assert benchFile.canRead(); - try (var ctx = Context.newBuilder() - .allowExperimentalOptions(true) - .allowIO(true) - .allowAllAccess(true) - .logHandler(new ByteArrayOutputStream()) - .option( - "enso.languageHomeOverride", - Paths.get("../../distribution/component").toFile().getAbsolutePath() - ).build()) { - Source source; - try { - source = Source.newBuilder("enso", benchFile).build(); - } catch (IOException e) { - throw new IllegalStateException("Unreachable", e); - } - Value module = ctx.eval(source); - var benchSuite = module - .invokeMember(Module.EVAL_EXPRESSION, "all") - .as(BenchSuite.class); - return benchSuite; + Source source; + try { + source = Source.newBuilder("enso", benchFile).build(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); } + Value module = ctx.eval(source); + var benchSuite = module + .invokeMember(Module.EVAL_EXPRESSION, "all") + .as(BenchSuite.class); + return benchSuite; } } From 0ee38dfb8444eae3e841a578916b6148f1d611e9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 18 Jul 2023 18:41:05 +0200 Subject: [PATCH 13/71] Get rid of IRUtils.scala --- .../org/enso/interpreter/bench/IRUtils.scala | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala diff --git a/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala b/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala deleted file mode 100644 index 3265b42f4b42..000000000000 --- a/engine/runtime/src/bench/scala/org/enso/interpreter/bench/IRUtils.scala +++ /dev/null @@ -1,45 +0,0 @@ -package org.enso.interpreter.bench - -object IRUtils { - /* - private def collectBenchCalls(ir: IR.Module): List[BenchCall] = { - val methods = ir.bindings.collect { case m : IR.Module.Scope.Definition.Method => m } - - for (method <- methods) { - method.body.preorder.flatMap(ir => { - ir match { - case application: IR.Application.Prefix => - application.function match { - case IR.Name.Literal("measure", true, _, _, _) => - application.arguments.head match { - case IR.CallArgument.Specified(_, value, _, _, _) => - value match { - case IR.Name.Literal("Bench", _, _, _, _) => - Some(new BenchCall(application)) - case _ => None - } - } - case _ => None - } - } - }) - } - } - - private class BenchCall( - val call: IR.Application.Prefix - ) { - val args: List[IR.CallArgument.Specified] = call.arguments.collect { case s : IR.CallArgument.Specified => s } - val location: IR.IdentifiedLocation = call.location.get - assert(args.length == 5) - assert(args.head.value.asInstanceOf[IR.Name.Literal].name == "Bench") - val action: IR.Expression = args(1).value - val title: String = { - args(2).value match { - case lit: IR.Literal.Text => lit.text - case unexpected => throw new IllegalStateException(s"Unexpected bench title IR: ${unexpected}") - } - } - } - */ -} From 733338daa035f6f3536618dc3cc45978acccba2d Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 20 Jul 2023 11:45:54 +0200 Subject: [PATCH 14/71] Add SpecCollector --- .../enso/benchmarks/libs/SpecCollector.java | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java new file mode 100644 index 000000000000..ce6a0d1907f5 --- /dev/null +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java @@ -0,0 +1,56 @@ +package org.enso.benchmarks.libs; + +import java.io.File; +import java.io.IOException; +import java.util.Collection; +import java.util.Objects; +import org.enso.polyglot.MethodNames.Module; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Source; +import org.graalvm.polyglot.Value; + +/** + * Collect benchmark specifications from source files. + */ +public class SpecCollector { + private final Context ctx; + + public SpecCollector(Context ctx) { + this.ctx = ctx; + } + + public Collection collectBenchSpecsFromDir(File rootDir) { + throw new UnsupportedOperationException("unimplemented"); + } + + /** + * + * @param benchFile + * @return null if there are no benchmark specs in the file. + */ + public BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { + Objects.requireNonNull(benchFile); + assert benchFile.exists(); + assert benchFile.canRead(); + Source source; + try { + source = Source.newBuilder("enso", benchFile).build(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + Value module = ctx.eval(source); + BenchSuite benchSuite = null; + Value associatedType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); + // TODO: Try to + try { + benchSuite = module + .invokeMember(Module.EVAL_EXPRESSION, "all") + .as(BenchSuite.class); + } catch (PolyglotException e) { + // TODO: Handle this properly + throw new IllegalStateException("Unreachable", e); + } + return benchSuite; + } +} From 5b543ef9f1caaba567770eb413ef78dd8608aa23 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 20 Jul 2023 11:46:10 +0200 Subject: [PATCH 15/71] LibBenchRunner prints help --- .../enso/benchmarks/libs/LibBenchRunner.java | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java index 28db0ed9b54c..255533fad877 100644 --- a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -6,6 +6,7 @@ import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; +import java.util.List; import java.util.Set; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Context; @@ -31,8 +32,38 @@ public static void main(String[] args) { System.err.println(" " + e.getMessage()); System.exit(1); } + if (cmdOpts.shouldHelp()) { + // TODO: Print help + System.err.println("Enso libs benchmark runner: A modified JMH runner for Enso benchmarks."); + System.err.println(); + System.err.println("Usage: runner [options] [benchmark-regex].."); + System.err.println(" [benchmark-regex].. - regexes of benchmarks to run"); + System.err.println(" [options] - options passed to JMH runner."); + System.err.println(); + System.err.println("Benchmark regex format: "); + System.err.println(" - note the slash between regexes"); + System.err.println(" - `group-regex` or `label-regex` can be omitted, in such case, it is treated as `.*`"); + System.err.println(); + System.err.println("Options from JMH Runner:"); + try { + cmdOpts.showHelp(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + System.exit(0); + } + if (cmdOpts.shouldList()) { + // TODO: List benchmarks + System.exit(0); + } + List includes = cmdOpts.getIncludes(); + + + File benchRootDir = Paths.get("../../distribution/component/test/Benchmarks").toFile(); initCtx(); + var specCollector = new SpecCollector(ctx, rootDir); + specCollector.collectBenchSpecsFromSingleFile() Set benchSpecs = collectAllBenchSpecs(); for (BenchSuite benchSpec : benchSpecs) { for (BenchGroup group : benchSpec.groups()) { From 40a2ad78e080d3feae2592ebeca0d13a9e4adb04 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 20 Jul 2023 19:54:06 +0200 Subject: [PATCH 16/71] LibBenchRunner can list benchmarks --- build.sbt | 3 +- .../org/enso/compiler/PackageRepository.scala | 2 +- .../org/enso/distribution/LanguageHome.scala | 3 +- .../enso/benchmarks/libs/LibBenchRunner.java | 101 ++++++------------ .../enso/benchmarks/libs/SpecCollector.java | 76 +++++++++---- 5 files changed, 95 insertions(+), 90 deletions(-) diff --git a/build.sbt b/build.sbt index 21cdbb8b6225..99b62f418b58 100644 --- a/build.sbt +++ b/build.sbt @@ -1772,7 +1772,7 @@ lazy val `bench-libs` = (project in file("tools/bench-libs")) libraryDependencies ++= jmh ++ Seq( "org.openjdk.jmh" % "jmh-core" % jmhVersion, "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, - "org.graalvm.sdk" % "graal-sdk" % graalVersion % "provided", + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", ), commands += WithDebugCommand.withDebug, (Compile / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), @@ -1790,6 +1790,7 @@ lazy val `bench-libs` = (project in file("tools/bench-libs")) .map(_.data) .mkString(File.pathSeparator) Seq( + "-ea", s"-Dtruffle.class.path.append=$appendClasspath", ) }, diff --git a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala index e39a2c05c061..ca9de36c5d91 100644 --- a/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala +++ b/engine/runtime/src/main/scala/org/enso/compiler/PackageRepository.scala @@ -736,7 +736,7 @@ object PackageRepository { * language home (if it is provided). * * @param projectPackage the package of the current project (if ran inside of a project) - * @param languageHome the language home (if set) + * @param languageHome the language home (if set). The path does not have to exist. * @param distributionManager the distribution manager * @param resourceManager the resource manager instance * @param context the context reference, needed to add polyglot libraries to diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala index ea09e1f4c210..09355fa7ba1f 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala @@ -6,7 +6,8 @@ import java.nio.file.Path * engine distribution from which the runtime is being run. * * @param languageHome the path to the directory containing the runner.jar and - * runtime.jar of the currently running language runtime + * runtime.jar of the currently running language runtime. + * The path does not have to exist. */ case class LanguageHome(languageHome: Path) { private val rootPath = languageHome.getParent.toAbsolutePath.normalize diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java index 255533fad877..df3923df7a9a 100644 --- a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -1,17 +1,11 @@ package org.enso.benchmarks.libs; -import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; import java.util.List; -import java.util.Set; -import org.enso.polyglot.MethodNames.Module; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.Source; -import org.graalvm.polyglot.Value; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; @@ -21,7 +15,6 @@ import org.openjdk.jmh.runner.options.OptionsBuilder; public class LibBenchRunner { - private static Context ctx; public static void main(String[] args) { CommandLineOptions cmdOpts = null; @@ -32,8 +25,8 @@ public static void main(String[] args) { System.err.println(" " + e.getMessage()); System.exit(1); } + if (cmdOpts.shouldHelp()) { - // TODO: Print help System.err.println("Enso libs benchmark runner: A modified JMH runner for Enso benchmarks."); System.err.println(); System.err.println("Usage: runner [options] [benchmark-regex].."); @@ -52,30 +45,45 @@ public static void main(String[] args) { } System.exit(0); } - if (cmdOpts.shouldList()) { - // TODO: List benchmarks - System.exit(0); - } - List includes = cmdOpts.getIncludes(); - - File benchRootDir = Paths.get("../../distribution/component/test/Benchmarks").toFile(); - - initCtx(); - var specCollector = new SpecCollector(ctx, rootDir); - specCollector.collectBenchSpecsFromSingleFile() - Set benchSpecs = collectAllBenchSpecs(); - for (BenchSuite benchSpec : benchSpecs) { - for (BenchGroup group : benchSpec.groups()) { - System.out.println("Discovered group: " + group.name()); + File ensoRootDir = Paths.get(System.getProperty("user.dir")).toFile(); + for (; ensoRootDir != null; ensoRootDir = ensoRootDir.getParentFile()) { + if (ensoRootDir.getName().equals("enso")) { + break; } } + if (ensoRootDir == null) { + throw new IllegalStateException("Unreachable: Could not find Enso root directory"); + } + File benchRootDir = ensoRootDir.toPath() + .resolve("test") + .resolve("Benchmarks") + .toFile(); + if (!benchRootDir.isDirectory() || !benchRootDir.canRead()) { + throw new IllegalStateException("Unreachable: Could not find Enso benchmarks directory"); + } + // Note that ensoHomeOverride does not have to exist, only its parent directory + File ensoHomeOverride = ensoRootDir.toPath() + .resolve("distribution") + .resolve("component") + .toFile(); + var specCollector = new SpecCollector(benchRootDir, ensoHomeOverride); + Collection benchSuites = specCollector.collectAllBenchSpecs(); - // Merge cmdOpts with chainedOptsBuilder - ChainedOptionsBuilder optsBuilder = new OptionsBuilder() - .include(cmdOpts.getIncludes().get(0)); + if (cmdOpts.shouldList()) { + for (BenchSuite benchSuite : benchSuites) { + for (BenchGroup group : benchSuite.groups()) { + System.out.println("Group \"" + group.name() + "\": "); + group.specs().forEach( + spec -> System.out.println(" - " + spec.name()) + ); + } + } + System.exit(0); + } - Runner jmhRunner = new Runner(optsBuilder.build()); + List includes = cmdOpts.getIncludes(); + Runner jmhRunner = new Runner(cmdOpts); Collection results = Collections.emptyList(); try { results = jmhRunner.run(); @@ -84,23 +92,6 @@ public static void main(String[] args) { } // TODO: Collect all the results - closeCtx(); - } - - private static void initCtx() { - ctx = Context.newBuilder() - .allowExperimentalOptions(true) - .allowIO(true) - .allowAllAccess(true) - .logHandler(new ByteArrayOutputStream()) - .option( - "enso.languageHomeOverride", - Paths.get("../../distribution/component").toFile().getAbsolutePath() - ).build(); - } - - private static void closeCtx() { - ctx.close(); } public void run(String label) { @@ -108,26 +99,4 @@ public void run(String label) { .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") .include("^" + label + "$"); } - - private static Set collectAllBenchSpecs() { - String benchmarksProjectRootDir = "test/Benchmarks/src"; - File singleFile = new File("/home/pavel/dev/enso/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso"); - return Set.of(collectBenchSpecsFromSingleFile(singleFile)); - } - - private static BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { - assert benchFile.exists(); - assert benchFile.canRead(); - Source source; - try { - source = Source.newBuilder("enso", benchFile).build(); - } catch (IOException e) { - throw new IllegalStateException("Unreachable", e); - } - Value module = ctx.eval(source); - var benchSuite = module - .invokeMember(Module.EVAL_EXPRESSION, "all") - .as(BenchSuite.class); - return benchSuite; - } } diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java index ce6a0d1907f5..9c2e7f5422ec 100644 --- a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java @@ -1,38 +1,71 @@ package org.enso.benchmarks.libs; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; import java.util.Collection; import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.PolyglotException; import org.graalvm.polyglot.Source; import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; /** * Collect benchmark specifications from source files. */ public class SpecCollector { + private final File rootDir; private final Context ctx; + private static final String allSuitesVarName = "all"; - public SpecCollector(Context ctx) { - this.ctx = ctx; + public SpecCollector(File benchProjectDir, File languageHomeOverride) { + if (!benchProjectDir.exists() || !benchProjectDir.isDirectory()) { + throw new IllegalArgumentException(benchProjectDir + " is not a directory."); + } + this.rootDir = benchProjectDir; + this.ctx = Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(IOAccess.ALL) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option( + "enso.projectRoot", + benchProjectDir.getAbsolutePath() + ) + .option( + "enso.languageHomeOverride", + languageHomeOverride.getAbsolutePath() + ) + .build(); } - public Collection collectBenchSpecsFromDir(File rootDir) { - throw new UnsupportedOperationException("unimplemented"); + public Collection collectAllBenchSpecs() { + try (Stream stream = Files.walk(rootDir.toPath())) { + return stream + .map(Path::toFile) + .filter(file -> file.isFile() && file.canRead() && file.getName().endsWith(".enso")) + .map(this::collectBenchSpecsFromSingleFile) + .filter(Objects::nonNull) + .collect(Collectors.toList()); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + } /** - * - * @param benchFile + * Collects benchmark specifications from a single file. Returns null if no specification + * is found in the file. + * @param benchFile Path to the source file. * @return null if there are no benchmark specs in the file. */ - public BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { - Objects.requireNonNull(benchFile); - assert benchFile.exists(); - assert benchFile.canRead(); + private BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { Source source; try { source = Source.newBuilder("enso", benchFile).build(); @@ -40,17 +73,18 @@ public BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { throw new IllegalStateException("Unreachable", e); } Value module = ctx.eval(source); - BenchSuite benchSuite = null; - Value associatedType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); - // TODO: Try to - try { - benchSuite = module - .invokeMember(Module.EVAL_EXPRESSION, "all") - .as(BenchSuite.class); - } catch (PolyglotException e) { - // TODO: Handle this properly - throw new IllegalStateException("Unreachable", e); + Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); + Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, allSuitesVarName); + if (!allSuitesVar.isNull()) { + try { + return module + .invokeMember(Module.EVAL_EXPRESSION, "all") + .as(BenchSuite.class); + } catch (PolyglotException e) { + // TODO: Replace with proper logging + System.err.println("WARN: An exception occured while evaluating benchmark suite var: " + e.getMessage()); + } } - return benchSuite; + return null; } } From 65051a4e2541c90275a773dd24ec8b7ae1cff3ab Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 20 Jul 2023 19:54:23 +0200 Subject: [PATCH 17/71] Fix Operations.enso to provide specs and not to run them --- test/Benchmarks/src/Vector/Operations.enso | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/test/Benchmarks/src/Vector/Operations.enso b/test/Benchmarks/src/Vector/Operations.enso index 27cd4bb90296..dfd08a5d0492 100644 --- a/test/Benchmarks/src/Vector/Operations.enso +++ b/test/Benchmarks/src/Vector/Operations.enso @@ -45,13 +45,10 @@ collect_benches group_builder = State.put s+x bench_measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" -bench = - options = Bench.options . size iter_size . iter num_iterations +options = Bench.options . size iter_size . iter num_iterations - all = Bench.build builder-> - builder.group "Vector Operations" options group_builder-> - collect_benches group_builder +all = Bench.build builder-> + builder.group "Vector Operations" options group_builder-> + collect_benches group_builder - all . run_main - -main = bench +main = all . run_main From d5b41f36b5f3061ae82184ae87b5c5a9dac2079c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 24 Jul 2023 12:17:16 +0200 Subject: [PATCH 18/71] Add simple annotation processor --- build.sbt | 15 ++++++-- .../org/enso/benchmarks/libs/LibBench.java | 17 --------- .../enso/benchmarks/libs/LibBenchRunner.java | 37 ++++++++++++++++++- .../libs/processor/BenchProcessor.java | 31 ++++++++++++++++ .../enso/benchmarks/libs/processor/Dummy.java | 7 ++++ 5 files changed, 86 insertions(+), 21 deletions(-) delete mode 100644 tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java create mode 100644 tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java create mode 100644 tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java diff --git a/build.sbt b/build.sbt index 99b62f418b58..087a009450e0 100644 --- a/build.sbt +++ b/build.sbt @@ -1770,14 +1770,23 @@ lazy val `bench-libs` = (project in file("tools/bench-libs")) frgaalJavaCompilerSetting, assembly / mainClass := (Compile / run / mainClass).value, libraryDependencies ++= jmh ++ Seq( - "org.openjdk.jmh" % "jmh-core" % jmhVersion, - "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, - "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, + "org.openjdk.jmh" % "jmh-core" % jmhVersion, + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", ), commands += WithDebugCommand.withDebug, (Compile / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), (Compile / run / fork) := true, (Compile / run / connectInput) := true, + Compile / javacOptions := ((Compile / javacOptions).value ++ + // Only run ServiceProvider processor and ignore those defined in META-INF, thus + // fixing incremental compilation setup + Seq( + "-processor", + "org.netbeans.modules.openide.util.ServiceProviderProcessor" + ) + ), (Compile / javaOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java deleted file mode 100644 index 307671755eb5..000000000000 --- a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBench.java +++ /dev/null @@ -1,17 +0,0 @@ -package org.enso.benchmarks.libs; - -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.infra.BenchmarkParams; - -public class LibBench { - // @Setup - public void setup(BenchmarkParams params) { - String benchName = params.getBenchmark(); - } - - @Benchmark - public void benchmark() { - - } -} diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java index df3923df7a9a..765843f17540 100644 --- a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -5,7 +5,11 @@ import java.nio.file.Paths; import java.util.Collection; import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Set; +import java.util.regex.Pattern; +import org.enso.benchmarks.libs.processor.Dummy; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; @@ -14,8 +18,11 @@ import org.openjdk.jmh.runner.options.CommandLineOptions; import org.openjdk.jmh.runner.options.OptionsBuilder; +@Dummy public class LibBenchRunner { + private static final String benchNameDelimiter = "/"; + public static void main(String[] args) { CommandLineOptions cmdOpts = null; try { @@ -82,7 +89,35 @@ public static void main(String[] args) { System.exit(0); } - List includes = cmdOpts.getIncludes(); + Set benchNames = new HashSet<>(); + for (BenchSuite benchSuite: benchSuites) { + for (BenchGroup group : benchSuite.groups()) { + String groupName = group.name(); + for (BenchSpec spec : group.specs()) { + String specName = spec.name(); + benchNames.add(groupName + benchNameDelimiter + specName); + } + } + } + + List includePatterns = cmdOpts.getIncludes() + .stream() + .map(Pattern::compile) + .toList(); + // Filter benchNames that match includePatterns + benchNames.removeIf(benchName -> { + for (Pattern includePattern : includePatterns) { + if (includePattern.matcher(benchName).matches()) { + return false; + } + } + return true; + }); + if (benchNames.isEmpty()) { + System.err.println("No benchmarks to run"); + System.exit(1); + } + Runner jmhRunner = new Runner(cmdOpts); Collection results = Collections.emptyList(); try { diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java new file mode 100644 index 000000000000..27b6e6bea6ae --- /dev/null +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java @@ -0,0 +1,31 @@ +package org.enso.benchmarks.libs.processor; + +import java.util.Set; +import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.Completion; +import javax.annotation.processing.ProcessingEnvironment; +import javax.annotation.processing.Processor; +import javax.annotation.processing.RoundEnvironment; +import javax.annotation.processing.SupportedAnnotationTypes; +import javax.annotation.processing.SupportedSourceVersion; +import javax.lang.model.SourceVersion; +import javax.lang.model.element.AnnotationMirror; +import javax.lang.model.element.Element; +import javax.lang.model.element.ExecutableElement; +import javax.lang.model.element.TypeElement; +import org.openide.util.lookup.ServiceProvider; + +//@SupportedAnnotationTypes("org.enso.benchmarks.libs.processor.Dummy") +//@SupportedSourceVersion(SourceVersion.RELEASE_17) +@ServiceProvider(service = Processor.class) +public class BenchProcessor extends AbstractProcessor { + public BenchProcessor() { + super(); + } + + @Override + public boolean process(Set annotations, RoundEnvironment roundEnv) { + System.out.println("[mylog] Running BenchProcessor"); + return false; + } +} diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java new file mode 100644 index 000000000000..a3aa86c08c95 --- /dev/null +++ b/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java @@ -0,0 +1,7 @@ +package org.enso.benchmarks.libs.processor; + +/** + * Just a dummy annotation to force the annotation processor in this package to run. + */ +public @interface Dummy { +} From 1adad1a411cfc2c6ca9076de9fad8887673efbad Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 24 Jul 2023 13:24:37 +0200 Subject: [PATCH 19/71] Move bench-libs to std-bits/benchmarks --- build.sbt | 2 +- .../src/main/java/org/enso/benchmarks/libs/BenchConfig.java | 0 .../src/main/java/org/enso/benchmarks/libs/BenchGroup.java | 0 .../src/main/java/org/enso/benchmarks/libs/BenchSpec.java | 0 .../src/main/java/org/enso/benchmarks/libs/BenchSuite.java | 0 .../src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java | 0 .../src/main/java/org/enso/benchmarks/libs/SpecCollector.java | 0 .../java/org/enso/benchmarks/libs/processor/BenchProcessor.java | 0 .../src/main/java/org/enso/benchmarks/libs/processor/Dummy.java | 0 9 files changed, 1 insertion(+), 1 deletion(-) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/BenchConfig.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/BenchGroup.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/BenchSpec.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/BenchSuite.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/SpecCollector.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java (100%) rename {tools/bench-libs => std-bits/benchmarks}/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java (100%) diff --git a/build.sbt b/build.sbt index 087a009450e0..4160ee6e655d 100644 --- a/build.sbt +++ b/build.sbt @@ -1764,7 +1764,7 @@ lazy val `distribution-manager` = project .dependsOn(pkg) .dependsOn(`logging-utils`) -lazy val `bench-libs` = (project in file("tools/bench-libs")) +lazy val `bench-libs` = (project in file("std-bits/benchmarks")) .configs(Benchmark) .settings( frgaalJavaCompilerSetting, diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchConfig.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchConfig.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchConfig.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchConfig.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchGroup.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchGroup.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchGroup.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchGroup.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSpec.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSpec.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSpec.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSpec.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSuite.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSuite.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/BenchSuite.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSuite.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/SpecCollector.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java diff --git a/tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java similarity index 100% rename from tools/bench-libs/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java rename to std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java From 85c3b96c49b21b09c29dfd6b5b8b85d9dcd941fd Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 24 Jul 2023 18:11:45 +0200 Subject: [PATCH 20/71] Integrate annotation processor into bench-libs --- build.sbt | 27 ++++++++++++------- .../org/enso/benchmarks}/BenchConfig.java | 2 +- .../java/org/enso/benchmarks}/BenchGroup.java | 2 +- .../java/org/enso/benchmarks}/BenchSpec.java | 2 +- .../java/org/enso/benchmarks}/BenchSuite.java | 2 +- .../benchmarks}/processor/BenchProcessor.java | 8 +++--- .../org/enso/benchmarks}/processor/Dummy.java | 2 +- .../enso/benchmarks/libs/LibBenchRunner.java | 5 +++- .../enso/benchmarks/libs/SpecCollector.java | 1 + 9 files changed, 32 insertions(+), 19 deletions(-) rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks}/BenchConfig.java (64%) rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks}/BenchGroup.java (79%) rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks}/BenchSpec.java (81%) rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks}/BenchSuite.java (71%) rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks}/processor/BenchProcessor.java (81%) rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks}/processor/Dummy.java (73%) diff --git a/build.sbt b/build.sbt index 4160ee6e655d..c0e6b21d05f2 100644 --- a/build.sbt +++ b/build.sbt @@ -1764,13 +1764,29 @@ lazy val `distribution-manager` = project .dependsOn(pkg) .dependsOn(`logging-utils`) +lazy val `bench-processor` = (project in file("std-bits/bench-processor")) + .settings( + frgaalJavaCompilerSetting, + libraryDependencies ++= Seq( + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + ), + Compile / javacOptions := ((Compile / javacOptions).value ++ + // Only run ServiceProvider processor and ignore those defined in META-INF, thus + // fixing incremental compilation setup + Seq( + "-processor", + "org.netbeans.modules.openide.util.ServiceProviderProcessor" + ) + ), + ) + lazy val `bench-libs` = (project in file("std-bits/benchmarks")) .configs(Benchmark) .settings( frgaalJavaCompilerSetting, assembly / mainClass := (Compile / run / mainClass).value, libraryDependencies ++= jmh ++ Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, "org.openjdk.jmh" % "jmh-core" % jmhVersion, "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", @@ -1779,14 +1795,6 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) (Compile / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), (Compile / run / fork) := true, (Compile / run / connectInput) := true, - Compile / javacOptions := ((Compile / javacOptions).value ++ - // Only run ServiceProvider processor and ignore those defined in META-INF, thus - // fixing incremental compilation setup - Seq( - "-processor", - "org.netbeans.modules.openide.util.ServiceProviderProcessor" - ) - ), (Compile / javaOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value @@ -1804,6 +1812,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) ) }, ) + .dependsOn(`bench-processor`) .dependsOn(runtime) lazy val editions = project diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchConfig.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java similarity index 64% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchConfig.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java index 6cb29c6b182d..dae6b1b22bf1 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchConfig.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs; +package org.enso.benchmarks; public interface BenchConfig { int size(); int iter(); diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchGroup.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java similarity index 79% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchGroup.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java index dd8278c81dc7..c7cc890e2046 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchGroup.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs; +package org.enso.benchmarks; import java.util.List; diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSpec.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java similarity index 81% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSpec.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java index 41a76dd198c6..fcb71c1d8443 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSpec.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs; +package org.enso.benchmarks; import org.graalvm.polyglot.Value; diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSuite.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java similarity index 71% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSuite.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java index 6ef8db91cd84..b9ae4d12d76e 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/BenchSuite.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs; +package org.enso.benchmarks; import java.util.List; diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java similarity index 81% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 27b6e6bea6ae..dc88882cf43c 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs.processor; +package org.enso.benchmarks.processor; import java.util.Set; import javax.annotation.processing.AbstractProcessor; @@ -15,8 +15,8 @@ import javax.lang.model.element.TypeElement; import org.openide.util.lookup.ServiceProvider; -//@SupportedAnnotationTypes("org.enso.benchmarks.libs.processor.Dummy") -//@SupportedSourceVersion(SourceVersion.RELEASE_17) +@SupportedAnnotationTypes("org.enso.benchmarks.processor.Dummy") +@SupportedSourceVersion(SourceVersion.RELEASE_17) @ServiceProvider(service = Processor.class) public class BenchProcessor extends AbstractProcessor { public BenchProcessor() { @@ -26,6 +26,6 @@ public BenchProcessor() { @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { System.out.println("[mylog] Running BenchProcessor"); - return false; + throw new IllegalStateException("[mylog] Running bench proc - unimpl"); } } diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java similarity index 73% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java index a3aa86c08c95..3537e908c250 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/processor/Dummy.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs.processor; +package org.enso.benchmarks.processor; /** * Just a dummy annotation to force the annotation processor in this package to run. diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java index 765843f17540..18eb0329f016 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -9,7 +9,10 @@ import java.util.List; import java.util.Set; import java.util.regex.Pattern; -import org.enso.benchmarks.libs.processor.Dummy; +import org.enso.benchmarks.BenchGroup; +import org.enso.benchmarks.BenchSpec; +import org.enso.benchmarks.BenchSuite; +import org.enso.benchmarks.processor.Dummy; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java index 9c2e7f5422ec..88143e0af7a3 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java +++ b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java @@ -9,6 +9,7 @@ import java.util.Objects; import java.util.stream.Collectors; import java.util.stream.Stream; +import org.enso.benchmarks.BenchSuite; import org.enso.polyglot.MethodNames.Module; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.PolyglotException; From 533923357b1b28367e3bc0e228263fd6a11f91ff Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 25 Jul 2023 18:56:54 +0200 Subject: [PATCH 21/71] Add truffle and runtime to annotation processor's class path --- build.sbt | 10 +++++--- project/FrgaalJavaCompiler.scala | 39 +++++++++++++++++++++++++------- 2 files changed, 38 insertions(+), 11 deletions(-) diff --git a/build.sbt b/build.sbt index c0e6b21d05f2..202f126a9a7d 100644 --- a/build.sbt +++ b/build.sbt @@ -1780,6 +1780,8 @@ lazy val `bench-processor` = (project in file("std-bits/bench-processor")) ) ), ) + .dependsOn(`polyglot-api`) + .dependsOn(runtime) lazy val `bench-libs` = (project in file("std-bits/benchmarks")) .configs(Benchmark) @@ -1795,7 +1797,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) (Compile / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), (Compile / run / fork) := true, (Compile / run / connectInput) := true, - (Compile / javaOptions) ++= { + (Compile / javacOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value val runtimeInstrumentsClasspath = @@ -1806,9 +1808,11 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) (runtimeClasspath ++ runtimeInstrumentsClasspath) .map(_.data) .mkString(File.pathSeparator) + // Only run ServiceProvider processor and ignore those defined in META-INF, thus + // fixing incremental compilation setup Seq( - "-ea", - s"-Dtruffle.class.path.append=$appendClasspath", + "-J--no-limit-modules", + s"-J-Dtruffle.class.path.append=$appendClasspath", ) }, ) diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index fae1f27fe381..4572311f4e15 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -9,19 +9,20 @@ * (http://www.apache.org/licenses/LICENSE-2.0). */ -import sbt._ +import sbt.* import sbt.internal.inc.CompilerArguments import sbt.internal.inc.javac.JavacLogger import sbt.io.IO import sbt.util.Logger -import xsbti.{PathBasedFile, Reporter, VirtualFile, Logger => XLogger} -import xsbti.compile.{IncToolOptions, Output, JavaCompiler => XJavaCompiler} +import xsbti.{PathBasedFile, Reporter, VirtualFile, Logger as XLogger} +import xsbti.compile.{IncToolOptions, Output, JavaCompiler as XJavaCompiler} import java.io.File -import java.nio.file.{Path, Paths} +import java.nio.file.{Files, Path, Paths, StandardCopyOption} import scala.sys.process.Process import scala.util.Using import java.io.FileWriter +import scala.collection.mutable.ArrayBuffer object FrgaalJavaCompiler { private val ENSO_SOURCES = ".enso-sources" @@ -65,7 +66,14 @@ object FrgaalJavaCompiler { xsbti.compile.Compilers.of(sbtCompilers.scalac, javaTools) } - /** Helper method to launch programs. */ + /** + * Helper method to launch programs. + * + * If -J--no-limit-modules is passed to javac, no --limit-modules option is passed to frgaal. + * With -J--no-limit-modules, one can use any class path in annotation processors. + * All the options with "-J" prefix are prepended to the frgaal command line, instead of passing + * them via an argument file. Moreover, the "-J" prefix is stripped. + * */ def launch( javaHome: Option[Path], compilerJar: Path, @@ -78,6 +86,15 @@ object FrgaalJavaCompiler { target: String ): Boolean = { val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) + var noLimitModules = false + val strippedJArgs: ArrayBuffer[String] = ArrayBuffer() + for (strippedJArg <- jArgs.map(_.stripPrefix("-J"))) { + if (strippedJArg == "--no-limit-modules") { + noLimitModules = true + } else { + strippedJArgs += strippedJArg + } + } val outputOption = CompilerArguments.outputOption(output) val sources = sources0 map { case x: PathBasedFile => x.toPath.toAbsolutePath.toString @@ -213,9 +230,15 @@ object FrgaalJavaCompiler { withArgumentFile(allArguments) { argsFile => // Need to disable standard compiler tools that come with used jdk and replace them // with the ones provided with Frgaal. - val forkArgs = (jArgs ++ Seq( - "--limit-modules", - "java.base,jdk.zipfs,jdk.internal.vm.compiler.management", + val limitModulesArgs = if (noLimitModules) { + Seq() + } else { + Seq( + "--limit-modules", + "java.base,jdk.zipfs,jdk.internal.vm.compiler.management", + ) + } + val forkArgs = (strippedJArgs ++ limitModulesArgs ++ Seq( "-jar", compilerJar.toString )) :+ From a9cdd19dba250613402cfa036b44748dd36eded6 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 26 Jul 2023 16:09:29 +0200 Subject: [PATCH 22/71] BenchProcessor creates JMH sources for all benchmarks --- .../enso/benchmarks/BenchSuiteWrapper.java | 52 +++++ .../benchmarks/processor/BenchProcessor.java | 218 +++++++++++++++++- .../org/enso/benchmarks/processor/Dummy.java | 7 - .../processor/GenerateBenchSources.java | 15 ++ .../benchmarks/processor}/SpecCollector.java | 30 ++- 5 files changed, 303 insertions(+), 19 deletions(-) create mode 100644 std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java delete mode 100644 std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java create mode 100644 std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java rename std-bits/{benchmarks/src/main/java/org/enso/benchmarks/libs => bench-processor/src/main/java/org/enso/benchmarks/processor}/SpecCollector.java (70%) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java new file mode 100644 index 000000000000..a39de263f648 --- /dev/null +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java @@ -0,0 +1,52 @@ +package org.enso.benchmarks; + +import java.util.List; +import java.util.Optional; +import org.graalvm.polyglot.Value; + +public class BenchSuiteWrapper { + private final BenchSuite suite; + private final String moduleQualifiedName; + + public BenchSuiteWrapper(BenchSuite suite, String moduleQualifiedName) { + this.suite = suite; + this.moduleQualifiedName = moduleQualifiedName; + } + + public List getGroups() { + return suite.groups(); + } + + public String getModuleQualifiedName() { + return moduleQualifiedName; + } + + public Value getDefaultInputArgument() { + return Value.asValue(null); + } + + public BenchGroup findGroupByName(String groupName) { + return suite + .groups() + .stream() + .filter(grp -> grp.name().equals(groupName)) + .findFirst() + .orElse(null); + } + + public BenchSpec findSpecByName(String groupName, String specName) { + Optional group = suite + .groups() + .stream() + .filter(grp -> grp.name().equals(groupName)) + .findFirst(); + if (group.isPresent()) { + return group.get().specs() + .stream() + .filter(spec -> spec.name().equals(specName)) + .findFirst() + .orElseGet(() -> null); + } + return null; + } +} diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index dc88882cf43c..aed2a34403c8 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -1,31 +1,233 @@ package org.enso.benchmarks.processor; +import java.io.File; +import java.io.IOException; +import java.io.Writer; +import java.net.URISyntaxException; +import java.util.Collection; +import java.util.List; import java.util.Set; +import java.util.stream.Collectors; import javax.annotation.processing.AbstractProcessor; -import javax.annotation.processing.Completion; -import javax.annotation.processing.ProcessingEnvironment; import javax.annotation.processing.Processor; import javax.annotation.processing.RoundEnvironment; import javax.annotation.processing.SupportedAnnotationTypes; import javax.annotation.processing.SupportedSourceVersion; import javax.lang.model.SourceVersion; -import javax.lang.model.element.AnnotationMirror; -import javax.lang.model.element.Element; -import javax.lang.model.element.ExecutableElement; import javax.lang.model.element.TypeElement; +import org.enso.benchmarks.BenchGroup; +import org.enso.benchmarks.BenchSpec; +import org.enso.benchmarks.BenchSuiteWrapper; +import org.graalvm.polyglot.Engine; import org.openide.util.lookup.ServiceProvider; -@SupportedAnnotationTypes("org.enso.benchmarks.processor.Dummy") +@SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") @SupportedSourceVersion(SourceVersion.RELEASE_17) @ServiceProvider(service = Processor.class) public class BenchProcessor extends AbstractProcessor { + + private final File ensoHomeOverride; + private File ensoDir; + private final File benchRootDir; + private final SpecCollector specCollector; + private static final String generatedSourcesPackagePrefix = "org.enso.benchmarks.libs.generated"; + private static final List imports = List.of( + "import java.nio.file.Paths;", + "import java.io.ByteArrayOutputStream;", + "import java.io.File;", + "import org.openjdk.jmh.annotations.Benchmark;", + "import org.openjdk.jmh.annotations.BenchmarkMode;", + "import org.openjdk.jmh.annotations.Mode;", + "import org.openjdk.jmh.annotations.Fork;", + "import org.openjdk.jmh.annotations.Measurement;", + "import org.openjdk.jmh.annotations.OutputTimeUnit;", + "import org.openjdk.jmh.annotations.Setup;", + "import org.openjdk.jmh.annotations.State;", + "import org.openjdk.jmh.annotations.Scope;", + "import org.openjdk.jmh.infra.BenchmarkParams;", + "import org.openjdk.jmh.infra.Blackhole;", + "import org.graalvm.polyglot.Context;", + "import org.graalvm.polyglot.Value;", + "import org.graalvm.polyglot.io.IOAccess;", + "import org.enso.polyglot.LanguageInfo;", + "import org.enso.polyglot.MethodNames;", + "import org.enso.benchmarks.processor.SpecCollector;", + "import org.enso.benchmarks.BenchSuiteWrapper;", + "import org.enso.benchmarks.BenchSpec;", + "import org.enso.benchmarks.BenchGroup;" + ); + public BenchProcessor() { - super(); + System.out.println("Initializing BenchProcessor"); + var langs = Engine.create().getLanguages(); + System.out.println("Languages: "); + for (var lang : langs.keySet()) { + System.out.println(" " + lang); + } + try { + ensoDir = new File( + BenchProcessor.class + .getProtectionDomain() + .getCodeSource() + .getLocation() + .toURI() + ); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unreachable", e); + } + for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { + if (ensoDir.getName().equals("enso")) { + break; + } + } + if (ensoDir == null) { + throw new IllegalStateException("Unreachable: Could not find Enso root directory"); + } + + benchRootDir = ensoDir.toPath() + .resolve("test") + .resolve("Benchmarks") + .toFile(); + if (!benchRootDir.isDirectory() || !benchRootDir.canRead()) { + throw new IllegalStateException("Unreachable: Could not find Enso benchmarks directory"); + } + + // Note that ensoHomeOverride does not have to exist, only its parent directory + ensoHomeOverride = ensoDir.toPath() + .resolve("distribution") + .resolve("component") + .toFile(); + specCollector = + new SpecCollector(benchRootDir, ensoHomeOverride); } @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { System.out.println("[mylog] Running BenchProcessor"); - throw new IllegalStateException("[mylog] Running bench proc - unimpl"); + System.out.println("[mylog] ensoRootDir: " + ensoDir); + Collection benchSuites = specCollector.collectAllBenchSpecs(); + for (BenchSuiteWrapper benchSuite : benchSuites) { + for (BenchGroup group : benchSuite.getGroups()) { + generateClassForGroup(group, benchSuite.getModuleQualifiedName()); + } + } + return true; + } + + private void generateClassForGroup(BenchGroup group, String moduleName) { + String fullClassName = createGroupClassName(group); + try (Writer srcFileWriter = processingEnv.getFiler().createSourceFile(fullClassName).openWriter()) { + generateClassForGroup(srcFileWriter, moduleName, group); + } catch (IOException e) { + System.err.println("Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); + } + } + + private void generateClassForGroup(Writer javaSrcFileWriter, String moduleName, BenchGroup group) throws IOException { + String groupFullClassName = createGroupClassName(group); + System.out.println("[mylog] Generating spec code for group '" + groupFullClassName + "'"); + String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); + List specs = group.specs(); + List specJavaNames = specs + .stream() + .map(spec -> normalize(spec.name())) + .collect(Collectors.toUnmodifiableList()); + + javaSrcFileWriter.append("package " + generatedSourcesPackagePrefix + ";\n"); + javaSrcFileWriter.append("\n"); + javaSrcFileWriter.append(String.join("\n", imports)); + javaSrcFileWriter.append("\n"); + javaSrcFileWriter.append("\n"); + javaSrcFileWriter.append("/**\n"); + javaSrcFileWriter.append(" * Generated from:\n"); + javaSrcFileWriter.append(" * - Module: " + moduleName + "\n"); + javaSrcFileWriter.append(" * - Group: \"" + group.name() + "\"\n"); + javaSrcFileWriter.append(" * Generated by {@link " + getClass().getName() + "}.\n"); + javaSrcFileWriter.append(" */\n"); + javaSrcFileWriter.append("@BenchmarkMode(Mode.AverageTime)\n"); + javaSrcFileWriter.append("@Fork(1)\n"); + javaSrcFileWriter.append("@State(Scope.Benchmark)\n"); + javaSrcFileWriter.append("public class " + className + " {\n"); + javaSrcFileWriter.append(" private Value groupInputArg;\n"); + for (var specJavaName : specJavaNames) { + javaSrcFileWriter.append(" private Value benchFunc_" + specJavaName + ";\n"); + } + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" @Setup\n"); + javaSrcFileWriter.append(" public void setup(BenchmarkParams params) throws Exception {\n"); + javaSrcFileWriter.append(" var ctx = Context.newBuilder()\n"); + javaSrcFileWriter.append(" .allowExperimentalOptions(true)\n"); + javaSrcFileWriter.append(" .allowIO(IOAccess.ALL)\n"); + javaSrcFileWriter.append(" .allowAllAccess(true)\n"); + javaSrcFileWriter.append(" .logHandler(new ByteArrayOutputStream())\n"); + javaSrcFileWriter.append(" .option(\n"); + javaSrcFileWriter.append(" \"enso.languageHomeOverride\",\n"); + javaSrcFileWriter.append(" Paths.get(\"../../distribution/component\").toFile().getAbsolutePath()\n"); + javaSrcFileWriter.append(" ).build();\n"); + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(MethodNames.TopScope.GET_MODULE, \"" + moduleName + "\");\n"); + javaSrcFileWriter.append(" File benchProjectDir = new File(\"" + benchRootDir.getAbsolutePath() + "\");\n"); + javaSrcFileWriter.append(" File languageHomeOverride = new File(\"" + ensoHomeOverride.getAbsolutePath() + "\");\n"); + javaSrcFileWriter.append(" var specCollector = new SpecCollector(benchProjectDir, languageHomeOverride);\n"); + javaSrcFileWriter.append(" BenchSuiteWrapper benchSuite = specCollector.collectBenchSpecFromModuleName(\"" + moduleName + "\");\n"); + javaSrcFileWriter.append(" \n"); + for (int i = 0; i < specs.size(); i++) { + var specJavaName = specJavaNames.get(i); + var specName = specs.get(i).name(); + javaSrcFileWriter.append(" BenchSpec benchSpec_" + specJavaName + " = benchSuite.findSpecByName(\"" + group.name() + "\", \"" + specName + "\");\n"); + javaSrcFileWriter.append(" this.benchFunc_" + specJavaName + " = benchSpec_" + specJavaName + ".code();\n"); + } + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" this.groupInputArg = benchSuite.getDefaultInputArgument();\n"); + javaSrcFileWriter.append(" } \n"); // end of setup method + javaSrcFileWriter.append(" \n"); + for (var specJavaName : specJavaNames) { + javaSrcFileWriter.append(" \n"); + javaSrcFileWriter.append(" @Benchmark\n"); + javaSrcFileWriter.append(" public void " + specJavaName + "(Blackhole blackhole) {\n"); + javaSrcFileWriter.append(" Value result = this.benchFunc_" + specJavaName + ".execute(this.groupInputArg);\n"); + javaSrcFileWriter.append(" blackhole.consume(result);\n"); + javaSrcFileWriter.append(" }\n"); // end of benchmark method + } + javaSrcFileWriter.append("}\n"); // end of class className + } + + /** + * Returns Java FQN for a benchmark spec. + * @param group Group name will be converted to Java package name. + * @return + */ + private static String createGroupClassName(BenchGroup group) { + var groupPkgName = normalize(group.name()); + return generatedSourcesPackagePrefix + "." + groupPkgName; + } + + private static boolean isValidChar(char c) { + return Character.isAlphabetic(c) || Character.isDigit(c) || c == '_'; + } + + /** + * Converts Text to valid Java identifier. + * @param name Text to convert. + * @return Valid Java identifier, non null. + */ + private static String normalize(String name) { + var normalizedNameSb = new StringBuilder(); + for (char c : name.toCharArray()) { + if (isValidChar(c)) { + normalizedNameSb.append(c); + } else if (Character.isWhitespace(c) && (peekLastChar(normalizedNameSb) != '_')) { + normalizedNameSb.append('_'); + } + } + return normalizedNameSb.toString(); + } + + private static char peekLastChar(StringBuilder sb) { + if (sb.length() > 0) { + return sb.charAt(sb.length() - 1); + } else { + return 0; + } } } diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java deleted file mode 100644 index 3537e908c250..000000000000 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/Dummy.java +++ /dev/null @@ -1,7 +0,0 @@ -package org.enso.benchmarks.processor; - -/** - * Just a dummy annotation to force the annotation processor in this package to run. - */ -public @interface Dummy { -} diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java new file mode 100644 index 000000000000..b086b3ab529f --- /dev/null +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java @@ -0,0 +1,15 @@ +package org.enso.benchmarks.processor; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +/** + * Just a dummy annotation to force the {@link BenchProcessor} to generate JMH sources + * for all the discovered Enso benchmarks. + */ +@Retention(RetentionPolicy.CLASS) +@Target(ElementType.TYPE) +public @interface GenerateBenchSources { +} diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java similarity index 70% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java index 88143e0af7a3..a4de3ead5630 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/SpecCollector.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java @@ -1,4 +1,4 @@ -package org.enso.benchmarks.libs; +package org.enso.benchmarks.processor; import java.io.ByteArrayOutputStream; import java.io.File; @@ -10,7 +10,10 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.enso.benchmarks.BenchSuite; +import org.enso.benchmarks.BenchSuiteWrapper; +import org.enso.polyglot.LanguageInfo; import org.enso.polyglot.MethodNames.Module; +import org.enso.polyglot.MethodNames.TopScope; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.PolyglotException; import org.graalvm.polyglot.Source; @@ -21,9 +24,11 @@ * Collect benchmark specifications from source files. */ public class SpecCollector { + private final File rootDir; private final Context ctx; private static final String allSuitesVarName = "all"; + private final String benchPackagePrefix = "local.Benchmarks"; public SpecCollector(File benchProjectDir, File languageHomeOverride) { if (!benchProjectDir.exists() || !benchProjectDir.isDirectory()) { @@ -46,7 +51,7 @@ public SpecCollector(File benchProjectDir, File languageHomeOverride) { .build(); } - public Collection collectAllBenchSpecs() { + public Collection collectAllBenchSpecs() { try (Stream stream = Files.walk(rootDir.toPath())) { return stream .map(Path::toFile) @@ -57,7 +62,15 @@ public Collection collectAllBenchSpecs() { } catch (IOException e) { throw new IllegalStateException("Unreachable", e); } + } + public BenchSuiteWrapper collectBenchSpecFromModuleName(String moduleName) { + Objects.requireNonNull(moduleName); + if (!moduleName.startsWith(benchPackagePrefix)) { + throw new IllegalArgumentException("Module name must start with " + benchPackagePrefix); + } + Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); + return collectBenchSpecFromModule(module); } /** @@ -66,7 +79,7 @@ public Collection collectAllBenchSpecs() { * @param benchFile Path to the source file. * @return null if there are no benchmark specs in the file. */ - private BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { + private BenchSuiteWrapper collectBenchSpecsFromSingleFile(File benchFile) { Source source; try { source = Source.newBuilder("enso", benchFile).build(); @@ -74,17 +87,26 @@ private BenchSuite collectBenchSpecsFromSingleFile(File benchFile) { throw new IllegalStateException("Unreachable", e); } Value module = ctx.eval(source); + return collectBenchSpecFromModule(module); + } + + private BenchSuiteWrapper collectBenchSpecFromModule(Value module) { Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, allSuitesVarName); if (!allSuitesVar.isNull()) { + BenchSuite suite; try { - return module + suite = module .invokeMember(Module.EVAL_EXPRESSION, "all") .as(BenchSuite.class); } catch (PolyglotException e) { // TODO: Replace with proper logging System.err.println("WARN: An exception occured while evaluating benchmark suite var: " + e.getMessage()); + return null; } + String moduleName = module.invokeMember(Module.GET_NAME).asString(); + String moduleQualifiedName = benchPackagePrefix + "." + moduleName; + return new BenchSuiteWrapper(suite, moduleQualifiedName); } return null; } From 906cc4d6ee0315732501a90c5e6cf3489599dcd8 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 26 Jul 2023 16:23:19 +0200 Subject: [PATCH 23/71] Put generated code into src_managed directory --- build.sbt | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/build.sbt b/build.sbt index 202f126a9a7d..d6b59011d6b6 100644 --- a/build.sbt +++ b/build.sbt @@ -1816,6 +1816,20 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) ) }, ) + .settings( + (Compile / javacOptions) ++= Seq( + "-s", + (Compile / sourceManaged).value.getAbsolutePath, + "-Xlint:unchecked" + ) + ) + .settings( + (Compile / compile) := (Compile / compile) + .dependsOn(Def.task { + (Compile / sourceManaged).value.mkdirs + }) + .value + ) .dependsOn(`bench-processor`) .dependsOn(runtime) From 8f3d761e697af6ddb5e43e63f48aa899ab426a66 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 26 Jul 2023 17:18:21 +0200 Subject: [PATCH 24/71] Remove benchmark discovery code from LibBenchRunner --- .../enso/benchmarks/libs/LibBenchRunner.java | 113 +++--------------- 1 file changed, 14 insertions(+), 99 deletions(-) diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java index 18eb0329f016..9f852cd058ab 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -1,32 +1,25 @@ package org.enso.benchmarks.libs; -import java.io.File; import java.io.IOException; -import java.nio.file.Paths; import java.util.Collection; -import java.util.Collections; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.regex.Pattern; -import org.enso.benchmarks.BenchGroup; -import org.enso.benchmarks.BenchSpec; -import org.enso.benchmarks.BenchSuite; -import org.enso.benchmarks.processor.Dummy; +import org.enso.benchmarks.processor.GenerateBenchSources; +import org.graalvm.polyglot.Engine; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; import org.openjdk.jmh.runner.options.CommandLineOptionException; import org.openjdk.jmh.runner.options.CommandLineOptions; -import org.openjdk.jmh.runner.options.OptionsBuilder; -@Dummy +@GenerateBenchSources public class LibBenchRunner { - private static final String benchNameDelimiter = "/"; - public static void main(String[] args) { + System.out.println("Running LibBenchRunner.Main:"); + var langs = Engine.create().getLanguages(); + System.out.println("Languages: "); + for (var lang : langs.keySet()) { + System.out.println(" " + lang); + } CommandLineOptions cmdOpts = null; try { cmdOpts = new CommandLineOptions(args); @@ -38,16 +31,6 @@ public static void main(String[] args) { if (cmdOpts.shouldHelp()) { System.err.println("Enso libs benchmark runner: A modified JMH runner for Enso benchmarks."); - System.err.println(); - System.err.println("Usage: runner [options] [benchmark-regex].."); - System.err.println(" [benchmark-regex].. - regexes of benchmarks to run"); - System.err.println(" [options] - options passed to JMH runner."); - System.err.println(); - System.err.println("Benchmark regex format: "); - System.err.println(" - note the slash between regexes"); - System.err.println(" - `group-regex` or `label-regex` can be omitted, in such case, it is treated as `.*`"); - System.err.println(); - System.err.println("Options from JMH Runner:"); try { cmdOpts.showHelp(); } catch (IOException e) { @@ -56,85 +39,17 @@ public static void main(String[] args) { System.exit(0); } - File ensoRootDir = Paths.get(System.getProperty("user.dir")).toFile(); - for (; ensoRootDir != null; ensoRootDir = ensoRootDir.getParentFile()) { - if (ensoRootDir.getName().equals("enso")) { - break; - } - } - if (ensoRootDir == null) { - throw new IllegalStateException("Unreachable: Could not find Enso root directory"); - } - File benchRootDir = ensoRootDir.toPath() - .resolve("test") - .resolve("Benchmarks") - .toFile(); - if (!benchRootDir.isDirectory() || !benchRootDir.canRead()) { - throw new IllegalStateException("Unreachable: Could not find Enso benchmarks directory"); - } - // Note that ensoHomeOverride does not have to exist, only its parent directory - File ensoHomeOverride = ensoRootDir.toPath() - .resolve("distribution") - .resolve("component") - .toFile(); - var specCollector = new SpecCollector(benchRootDir, ensoHomeOverride); - Collection benchSuites = specCollector.collectAllBenchSpecs(); - - if (cmdOpts.shouldList()) { - for (BenchSuite benchSuite : benchSuites) { - for (BenchGroup group : benchSuite.groups()) { - System.out.println("Group \"" + group.name() + "\": "); - group.specs().forEach( - spec -> System.out.println(" - " + spec.name()) - ); - } - } - System.exit(0); - } - - Set benchNames = new HashSet<>(); - for (BenchSuite benchSuite: benchSuites) { - for (BenchGroup group : benchSuite.groups()) { - String groupName = group.name(); - for (BenchSpec spec : group.specs()) { - String specName = spec.name(); - benchNames.add(groupName + benchNameDelimiter + specName); - } - } - } - - List includePatterns = cmdOpts.getIncludes() - .stream() - .map(Pattern::compile) - .toList(); - // Filter benchNames that match includePatterns - benchNames.removeIf(benchName -> { - for (Pattern includePattern : includePatterns) { - if (includePattern.matcher(benchName).matches()) { - return false; - } - } - return true; - }); - if (benchNames.isEmpty()) { - System.err.println("No benchmarks to run"); - System.exit(1); - } - Runner jmhRunner = new Runner(cmdOpts); - Collection results = Collections.emptyList(); + Collection results; try { results = jmhRunner.run(); } catch (RunnerException e) { throw new RuntimeException(e); } - // TODO: Collect all the results - } - - public void run(String label) { - ChainedOptionsBuilder builder = new OptionsBuilder() - .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") - .include("^" + label + "$"); + System.out.println("Results:"); + for (RunResult result : results) { + System.out.println(result.toString()); + } } } From 179949ebce3a3edad6b7de5562b63d8f03d4fd4d Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 11:59:25 +0200 Subject: [PATCH 25/71] Do not use generate package name for generated sources --- .../main/java/org/enso/benchmarks/processor/BenchProcessor.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index aed2a34403c8..72d1545331cb 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -30,7 +30,7 @@ public class BenchProcessor extends AbstractProcessor { private File ensoDir; private final File benchRootDir; private final SpecCollector specCollector; - private static final String generatedSourcesPackagePrefix = "org.enso.benchmarks.libs.generated"; + private static final String generatedSourcesPackagePrefix = "org.enso.benchmarks.libs"; private static final List imports = List.of( "import java.nio.file.Paths;", "import java.io.ByteArrayOutputStream;", From df82640ac41a6c7d493ba064226efd3bcb3b2bbb Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 12:00:47 +0200 Subject: [PATCH 26/71] Incorporate bench and benchOnly commands in bench-libs project --- build.sbt | 50 +++++++++++++++++++++++++++++++++----------------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/build.sbt b/build.sbt index d6b59011d6b6..c2cdaf969147 100644 --- a/build.sbt +++ b/build.sbt @@ -1784,7 +1784,6 @@ lazy val `bench-processor` = (project in file("std-bits/bench-processor")) .dependsOn(runtime) lazy val `bench-libs` = (project in file("std-bits/benchmarks")) - .configs(Benchmark) .settings( frgaalJavaCompilerSetting, assembly / mainClass := (Compile / run / mainClass).value, @@ -1794,10 +1793,11 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", ), commands += WithDebugCommand.withDebug, - (Compile / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), - (Compile / run / fork) := true, - (Compile / run / connectInput) := true, - (Compile / javacOptions) ++= { + (Benchmark / run / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), + (Benchmark / run / fork) := true, + (Benchmark / run / connectInput) := true, + // Pass -Dtruffle.class.path.append to javac + (Benchmark / compile / javacOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value val runtimeInstrumentsClasspath = @@ -1808,27 +1808,43 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) (runtimeClasspath ++ runtimeInstrumentsClasspath) .map(_.data) .mkString(File.pathSeparator) - // Only run ServiceProvider processor and ignore those defined in META-INF, thus - // fixing incremental compilation setup Seq( "-J--no-limit-modules", s"-J-Dtruffle.class.path.append=$appendClasspath", ) }, + (Benchmark / run / javaOptions) ++= { + val runtimeClasspath = + (LocalProject("runtime") / Compile / products).value + val runtimeInstrumentsClasspath = + (LocalProject( + "runtime-with-instruments" + ) / Compile / products).value + val appendClasspath = + (runtimeClasspath ++ runtimeInstrumentsClasspath) + .mkString(File.pathSeparator) + Seq( + s"-Dtruffle.class.path.append=$appendClasspath", + ) + }, ) + .configs(Benchmark) .settings( - (Compile / javacOptions) ++= Seq( - "-s", - (Compile / sourceManaged).value.getAbsolutePath, - "-Xlint:unchecked" - ) + inConfig(Benchmark)(Defaults.testSettings), ) .settings( - (Compile / compile) := (Compile / compile) - .dependsOn(Def.task { - (Compile / sourceManaged).value.mkdirs - }) - .value + bench := (Benchmark / run).toTask("").tag(Exclusive).value, + benchOnly := Def.inputTaskDyn { + import complete.Parsers.spaceDelimited + val name = spaceDelimited("").parsed match { + case List(name) => name + case _ => throw new IllegalArgumentException("Expected one argument.") + } + Def.task { + (Benchmark / run).toTask(" " + name).value + } + }.evaluated, + Benchmark / parallelExecution := false ) .dependsOn(`bench-processor`) .dependsOn(runtime) From 6ec55f964905cb427ef793d7fd542e1ad88c4b7e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 14:40:31 +0200 Subject: [PATCH 27/71] Move LibBenchRunner under src/bench from src/main --- .../java/org/enso/benchmarks/libs/LibBenchRunner.java | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) rename std-bits/benchmarks/src/{main => bench}/java/org/enso/benchmarks/libs/LibBenchRunner.java (95%) diff --git a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java similarity index 95% rename from std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java rename to std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index 9f852cd058ab..c325a925ea84 100644 --- a/std-bits/benchmarks/src/main/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -28,6 +28,7 @@ public static void main(String[] args) { System.err.println(" " + e.getMessage()); System.exit(1); } + Runner jmhRunner = new Runner(cmdOpts); if (cmdOpts.shouldHelp()) { System.err.println("Enso libs benchmark runner: A modified JMH runner for Enso benchmarks."); @@ -39,7 +40,11 @@ public static void main(String[] args) { System.exit(0); } - Runner jmhRunner = new Runner(cmdOpts); + if (cmdOpts.shouldList()) { + jmhRunner.list(); + System.exit(0); + } + Collection results; try { results = jmhRunner.run(); From 954e96f2207bc384986483b36027312ba9dccc87 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 14:45:44 +0200 Subject: [PATCH 28/71] Benchmark/run/javaOptions gets fullClasspath --- build.sbt | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index c2cdaf969147..55fed6ca5d4b 100644 --- a/build.sbt +++ b/build.sbt @@ -1815,13 +1815,14 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) }, (Benchmark / run / javaOptions) ++= { val runtimeClasspath = - (LocalProject("runtime") / Compile / products).value + (LocalProject("runtime") / Compile / fullClasspath).value val runtimeInstrumentsClasspath = (LocalProject( "runtime-with-instruments" - ) / Compile / products).value + ) / Compile / fullClasspath).value val appendClasspath = (runtimeClasspath ++ runtimeInstrumentsClasspath) + .map(_.data) .mkString(File.pathSeparator) Seq( s"-Dtruffle.class.path.append=$appendClasspath", From 18e313ac1471db00f5a779f1ab56cabfa470ce03 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 14:54:00 +0200 Subject: [PATCH 29/71] Generate source into Compile/sourceManaged. So that they can be tracked by an IDE. --- build.sbt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.sbt b/build.sbt index 55fed6ca5d4b..c253114d15d0 100644 --- a/build.sbt +++ b/build.sbt @@ -1813,6 +1813,11 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) s"-J-Dtruffle.class.path.append=$appendClasspath", ) }, + (Compile / javacOptions) ++= Seq( + "-s", + (Compile / sourceManaged).value.getAbsolutePath, + "-Xlint:unchecked" + ), (Benchmark / run / javaOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value From 4e9afc2a7de766725f815cf8a9bfdd0e0aef2f87 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 17:48:30 +0200 Subject: [PATCH 30/71] Use fully qualified module names in BenchProcessor --- .../benchmarks/processor/BenchProcessor.java | 24 ++++++++++------- .../benchmarks/processor/SpecCollector.java | 27 +++++++++---------- 2 files changed, 27 insertions(+), 24 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 72d1545331cb..4b23350b6a24 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -114,16 +114,16 @@ public boolean process(Set annotations, RoundEnvironment return true; } - private void generateClassForGroup(BenchGroup group, String moduleName) { + private void generateClassForGroup(BenchGroup group, String moduleQualifiedName) { String fullClassName = createGroupClassName(group); try (Writer srcFileWriter = processingEnv.getFiler().createSourceFile(fullClassName).openWriter()) { - generateClassForGroup(srcFileWriter, moduleName, group); + generateClassForGroup(srcFileWriter, moduleQualifiedName, group); } catch (IOException e) { System.err.println("Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); } } - private void generateClassForGroup(Writer javaSrcFileWriter, String moduleName, BenchGroup group) throws IOException { + private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualifiedName, BenchGroup group) throws IOException { String groupFullClassName = createGroupClassName(group); System.out.println("[mylog] Generating spec code for group '" + groupFullClassName + "'"); String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); @@ -140,7 +140,7 @@ private void generateClassForGroup(Writer javaSrcFileWriter, String moduleName, javaSrcFileWriter.append("\n"); javaSrcFileWriter.append("/**\n"); javaSrcFileWriter.append(" * Generated from:\n"); - javaSrcFileWriter.append(" * - Module: " + moduleName + "\n"); + javaSrcFileWriter.append(" * - Module: " + moduleQualifiedName + "\n"); javaSrcFileWriter.append(" * - Group: \"" + group.name() + "\"\n"); javaSrcFileWriter.append(" * Generated by {@link " + getClass().getName() + "}.\n"); javaSrcFileWriter.append(" */\n"); @@ -155,6 +155,8 @@ private void generateClassForGroup(Writer javaSrcFileWriter, String moduleName, javaSrcFileWriter.append(" \n"); javaSrcFileWriter.append(" @Setup\n"); javaSrcFileWriter.append(" public void setup(BenchmarkParams params) throws Exception {\n"); + javaSrcFileWriter.append(" File benchProjectDir = new File(\"" + benchRootDir.getAbsolutePath() + "\");\n"); + javaSrcFileWriter.append(" File languageHomeOverride = new File(\"" + ensoHomeOverride.getAbsolutePath() + "\");\n"); javaSrcFileWriter.append(" var ctx = Context.newBuilder()\n"); javaSrcFileWriter.append(" .allowExperimentalOptions(true)\n"); javaSrcFileWriter.append(" .allowIO(IOAccess.ALL)\n"); @@ -163,13 +165,17 @@ private void generateClassForGroup(Writer javaSrcFileWriter, String moduleName, javaSrcFileWriter.append(" .option(\n"); javaSrcFileWriter.append(" \"enso.languageHomeOverride\",\n"); javaSrcFileWriter.append(" Paths.get(\"../../distribution/component\").toFile().getAbsolutePath()\n"); - javaSrcFileWriter.append(" ).build();\n"); + javaSrcFileWriter.append(" )\n"); + javaSrcFileWriter.append(" .option(\n"); + javaSrcFileWriter.append(" \"enso.projectRoot\",\n"); + javaSrcFileWriter.append(" benchProjectDir.getAbsolutePath()\n"); + javaSrcFileWriter.append(" )\n"); + javaSrcFileWriter.append(" .build();\n"); javaSrcFileWriter.append(" \n"); - javaSrcFileWriter.append(" Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(MethodNames.TopScope.GET_MODULE, \"" + moduleName + "\");\n"); - javaSrcFileWriter.append(" File benchProjectDir = new File(\"" + benchRootDir.getAbsolutePath() + "\");\n"); - javaSrcFileWriter.append(" File languageHomeOverride = new File(\"" + ensoHomeOverride.getAbsolutePath() + "\");\n"); + javaSrcFileWriter.append(" Value bindings = ctx.getBindings(LanguageInfo.ID);\n"); + javaSrcFileWriter.append(" Value module = bindings.invokeMember(MethodNames.TopScope.GET_MODULE, \"" + moduleQualifiedName + "\");\n"); javaSrcFileWriter.append(" var specCollector = new SpecCollector(benchProjectDir, languageHomeOverride);\n"); - javaSrcFileWriter.append(" BenchSuiteWrapper benchSuite = specCollector.collectBenchSpecFromModuleName(\"" + moduleName + "\");\n"); + javaSrcFileWriter.append(" BenchSuiteWrapper benchSuite = specCollector.collectBenchSpecFromModuleName(\"" + moduleQualifiedName + "\");\n"); javaSrcFileWriter.append(" \n"); for (int i = 0; i < specs.size(); i++) { var specJavaName = specJavaNames.get(i); diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java index a4de3ead5630..877837cf5a98 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java @@ -16,7 +16,6 @@ import org.enso.polyglot.MethodNames.TopScope; import org.graalvm.polyglot.Context; import org.graalvm.polyglot.PolyglotException; -import org.graalvm.polyglot.Source; import org.graalvm.polyglot.Value; import org.graalvm.polyglot.io.IOAccess; @@ -29,6 +28,7 @@ public class SpecCollector { private final Context ctx; private static final String allSuitesVarName = "all"; private final String benchPackagePrefix = "local.Benchmarks"; + private final String ensoSuffix = ".enso"; public SpecCollector(File benchProjectDir, File languageHomeOverride) { if (!benchProjectDir.exists() || !benchProjectDir.isDirectory()) { @@ -55,7 +55,7 @@ public Collection collectAllBenchSpecs() { try (Stream stream = Files.walk(rootDir.toPath())) { return stream .map(Path::toFile) - .filter(file -> file.isFile() && file.canRead() && file.getName().endsWith(".enso")) + .filter(file -> file.isFile() && file.canRead() && file.getName().endsWith(ensoSuffix)) .map(this::collectBenchSpecsFromSingleFile) .filter(Objects::nonNull) .collect(Collectors.toList()); @@ -70,7 +70,7 @@ public BenchSuiteWrapper collectBenchSpecFromModuleName(String moduleName) { throw new IllegalArgumentException("Module name must start with " + benchPackagePrefix); } Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); - return collectBenchSpecFromModule(module); + return collectBenchSpecFromModule(module, moduleName); } /** @@ -80,17 +80,16 @@ public BenchSuiteWrapper collectBenchSpecFromModuleName(String moduleName) { * @return null if there are no benchmark specs in the file. */ private BenchSuiteWrapper collectBenchSpecsFromSingleFile(File benchFile) { - Source source; - try { - source = Source.newBuilder("enso", benchFile).build(); - } catch (IOException e) { - throw new IllegalStateException("Unreachable", e); - } - Value module = ctx.eval(source); - return collectBenchSpecFromModule(module); + Path relativePath = rootDir.toPath().relativize(benchFile.toPath()); + // Strip the "src" directory + Path modulePath = relativePath.subpath(1, relativePath.getNameCount()); + String moduleFullName = benchPackagePrefix + "." + modulePath.toString().replace(File.separatorChar, '.'); + moduleFullName = moduleFullName.substring(0, moduleFullName.length() - ensoSuffix.length()); + Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleFullName); + return collectBenchSpecFromModule(module, moduleFullName); } - private BenchSuiteWrapper collectBenchSpecFromModule(Value module) { + private BenchSuiteWrapper collectBenchSpecFromModule(Value module, String moduleQualifiedName) { Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, allSuitesVarName); if (!allSuitesVar.isNull()) { @@ -99,14 +98,12 @@ private BenchSuiteWrapper collectBenchSpecFromModule(Value module) { suite = module .invokeMember(Module.EVAL_EXPRESSION, "all") .as(BenchSuite.class); + return new BenchSuiteWrapper(suite, moduleQualifiedName); } catch (PolyglotException e) { // TODO: Replace with proper logging System.err.println("WARN: An exception occured while evaluating benchmark suite var: " + e.getMessage()); return null; } - String moduleName = module.invokeMember(Module.GET_NAME).asString(); - String moduleQualifiedName = benchPackagePrefix + "." + moduleName; - return new BenchSuiteWrapper(suite, moduleQualifiedName); } return null; } From 45d1143b07910c509db880f13f6082ffcd5dd717 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:10:54 +0200 Subject: [PATCH 31/71] Add ability to attach debugger to the annotation processors. --- docs/CONTRIBUTING.md | 27 +++++++++++++++++++++++++++ project/FrgaalJavaCompiler.scala | 8 ++++++++ 2 files changed, 35 insertions(+) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 57459aef2738..93bc56c1e26f 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -387,6 +387,7 @@ the following flags: allow for manual analysis and discovery of optimisation failures. - `--showCompilations`: Prints the truffle compilation trace information. - `--printAssembly`: Prints the assembly output from the HotSpot JIT tier. +- `--debugger`: Launches the JVM with the remote debugger enabled. For more information on this sbt command, please see [WithDebugCommand.scala](../project/WithDebugCommand.scala). @@ -415,6 +416,32 @@ sbt:project-manager> withDebug run --debugger read more about [debugging Java & Enso code](debugger/README.md). +#### Debugging annotation processors +The Engine uses annotation processors to generate some of the Java code, e.g., the +builtin methods with `org.enso.interpreter.dsl.MethodProcessor`, or JMH benchmark +sources with `org.enso.benchmarks.processor.BenchProcessor`. Annotation +processors are invoked by the Java compiler (`javac`), therefore, we need special +instructions to attach the debugger to them. + +Let's debug `org.enso.interpreter.dsl.MethodProcessor` as an example. The following +are the commands invoked in the `sbt` shell: +- `project runtime` +- `clean` + - Delete all the compiled class files along with all the generated sources by + the annotation processor. This ensures that the annotation processor will + be invoked. +- `set javacOptions += FrgaalJavaCompiler.debugArg` + - This sets a special flag that is passed to the frgaal Java compiler, which + in turn waits for the debugger to attach. Note that this setting is not + persisted and will be reset once the project is reloaded. +- `compile` + - Launches the Java compiler, which will wait for the debugger to attach. + Put a breakpoint in some class of `org.enso.interpreter.dsl` package. + Wait for the message in the console instructing to attach the debugger. +- To reset the `javacOptions` setting, either run `set javacOptions -= FrgaalJavaCompiler.debugArg`, + or reload the project with `reload`. + + #### Working with Assembly In order to examine the assembly generated by GraalVM and HotSpot you need to diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index 4572311f4e15..ee7c85441101 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -29,6 +29,7 @@ object FrgaalJavaCompiler { val frgaal = "org.frgaal" % "compiler" % "19.0.1" % "provided" val sourceLevel = "19" + val debugArg = "-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000" def compilers( classpath: sbt.Keys.Classpath, @@ -87,6 +88,7 @@ object FrgaalJavaCompiler { ): Boolean = { val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) var noLimitModules = false + val debugAnotProcessorOpt = jArgs.contains(debugArg) val strippedJArgs: ArrayBuffer[String] = ArrayBuffer() for (strippedJArg <- jArgs.map(_.stripPrefix("-J"))) { if (strippedJArg == "--no-limit-modules") { @@ -247,6 +249,12 @@ object FrgaalJavaCompiler { val cwd = new File(new File(".").getAbsolutePath).getCanonicalFile val javacLogger = new JavacLogger(log, reporter, cwd) var exitCode = -1 + if (debugAnotProcessorOpt) { + log.info(s"Frgaal compiler is about to be launched with $debugArg, which means that" + + " it will wait for a debugger to attach. The output from the compiler is by default" + + " redirected, therefore \"Listening to the debugger\" message will not be displayed." + + " You should attach the debugger now.") + } try { exitCode = Process(exe +: forkArgs, cwd) ! javacLogger } finally { From a124a0b9f6f6a333cc573f3f59ab2629b0d6d922 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:11:28 +0200 Subject: [PATCH 32/71] Add the benchmarks documentation --- docs/infrastructure/README.md | 2 + docs/infrastructure/benchmarks.md | 78 +++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 docs/infrastructure/benchmarks.md diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index 4fd4a4a49d92..ec6421b1ae6a 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -22,5 +22,7 @@ up as follows: - [**Upgrading GraalVM:**](upgrading-graalvm.md) Description of steps that have to be performed by each developer when the project is upgraded to a new version of GraalVM. +- [**Benchmarks:**](benchmarks.md) Description of the benchmarking infrastructure + used for measuring performance of the runtime. - [**Logging**:](logging.md) Description of an unified and centralized logging infrastructure that should be used by all components. diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md new file mode 100644 index 000000000000..327288d89cf9 --- /dev/null +++ b/docs/infrastructure/benchmarks.md @@ -0,0 +1,78 @@ +# Benchmarks +In this document, we describe the benchmark types used for the runtime - micro benchmarks in +the section [Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and meso benchmarks +in the section [Standard library benchmarks](#standard-library-benchmarks), and how and where +are the results stored and visualized in the section [Visualization](#visualization). + +To track the performance of the engine, we use [JMH](https://openjdk.org/projects/code-tools/jmh/). +There are two types of benchmarks: +- [micro benchmarks](#engine-jmh-microbenchmarks) located directly in the `runtime` SBT project. + These benchmarks are written in Java, and are used to measure the performance of + specific parts of the engine. +- [standard library benchmarks](#standard-library-benchmarks) (meso benchmarks) located in the `test/Benchmarks` Enso project. + These benchmarks are entirelly written in Enso, along with the harness code. + +## Engine JMH microbenchmarks +These benchmarks are written in Java and are used to measure the performance of +specific parts of the engine. +The sources are located in the `runtime` SBT project, under `src/bench` source directory. + +### Running the benchmarks +To run the benchmarks, use `bench` or `benchOnly` command - `bench` runs all the benchmarks and +`benchOnly` runs only one benchmark specified with the fully qualified name. +The parameters for these benchmarks are hard-coded inside the JMH annotations in the +source files. In order to change, e.g., the number of measurement iterations, you need to +modify the parameter to the `@Measurement` annotation. + +### Debugging the benchmarks +Currently, the best way to debug the benchmark is to set the `@Fork` annotation to 0, and to +run `withDebug` command like this: +``` +withDebug --debugger benchOnly -- +``` + +## Standard library benchmarks +Unlike the micro benchmarks, these benchmarks are written entirelly in Enso and located in the +`test/Benchmarks` Enso project. Sometimes, we call those *meso* benchmarks. +There are two ways to run these benchmarks: +- [Running standalone](#running-standalone) +- [Running via JMH launcher](#running-via-jmh-launcher) + +### Running standalone +A single source file in the project may contain multiple benchmark definitions. +If the source file defines `main` method, we can evaluate it the same way as any other +Enso source file, for example via `runEngineDistribution --in-project test/Benchmarks --run `. +The harness within the project is not meant for any sophisticated benchmarking, but rather +for quick local evaluation. See the `Bench.measure` method documentation for more details. +For more sophisticated approach, run the benchmarks via JMH launcher. + +### Running via JMH launcher +The JMH launcher is located in `std-bits/benchmarks` directory, as `bench-libs` SBT project. +It is a single Java class with a `main` method that just delegates to the +[standard JMH launcher](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/Main.java), +therefore, supports all the command line options as the standard launcher. +For the full options summary, either see the [JMH source code](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java), +or run the launcher with `-h` option. + +The `bench-libs` SBT project supports `bench` and `benchOnly` commands, that work the same +as in the `runtime` project, with the exception that the benchmark name does not have to be +specified as a fully qualified name, but as a regular expression. +To access the full flexibility of the JMH launcher, run it via `Bench/run` - for example, +to see the help message: `Bench/run -h`. + +To debug a single benchmark, set the `javaOptions` with something like `set javaOptions += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000"`, +and launch a single benchmark without forking with `Bench/run -f 0 `. +Then, attach to the debugger with your IDE. + + +## Visualization +The benchmarks are invoked as a daily [GitHub Action](https://github.com/enso-org/enso/actions/workflows/benchmark.yml), +that can be invoked manually on a specific branch as well. +The results are kept in the artifacts produced from the actions. +In `tools/performance/engine-benchmarks` directory, there is a simple Python +script for collecting and processing the results. See the README in that directory +for more information about how to run that script. +This script is invoked regularly on a private machine and the results are +published in [https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). + +This is not meant as a permanent solution. From 8b6b96dcc4ce60a2250db78dc4f44fc1a94286f5 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:15:45 +0200 Subject: [PATCH 33/71] Discard spurious warnings from javac for generated sources --- build.sbt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/build.sbt b/build.sbt index c253114d15d0..e1bb27c81a38 100644 --- a/build.sbt +++ b/build.sbt @@ -1791,6 +1791,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) "org.openjdk.jmh" % "jmh-core" % jmhVersion, "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark, ), commands += WithDebugCommand.withDebug, (Benchmark / run / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), @@ -1818,6 +1819,11 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) (Compile / sourceManaged).value.getAbsolutePath, "-Xlint:unchecked" ), + (Compile / logManager) := + sbt.internal.util.CustomLogManager.excludeMsg( + "Could not determine source for class ", + Level.Warn + ), (Benchmark / run / javaOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value From 16d2f560f222e53ae05c96b99320c6d939a3e05e Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:17:14 +0200 Subject: [PATCH 34/71] Remove some temporary logging --- .../org/enso/benchmarks/processor/BenchProcessor.java | 9 --------- .../java/org/enso/benchmarks/libs/LibBenchRunner.java | 6 ------ 2 files changed, 15 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 4b23350b6a24..05e95052dc08 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -58,12 +58,6 @@ public class BenchProcessor extends AbstractProcessor { ); public BenchProcessor() { - System.out.println("Initializing BenchProcessor"); - var langs = Engine.create().getLanguages(); - System.out.println("Languages: "); - for (var lang : langs.keySet()) { - System.out.println(" " + lang); - } try { ensoDir = new File( BenchProcessor.class @@ -103,8 +97,6 @@ public BenchProcessor() { @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { - System.out.println("[mylog] Running BenchProcessor"); - System.out.println("[mylog] ensoRootDir: " + ensoDir); Collection benchSuites = specCollector.collectAllBenchSpecs(); for (BenchSuiteWrapper benchSuite : benchSuites) { for (BenchGroup group : benchSuite.getGroups()) { @@ -125,7 +117,6 @@ private void generateClassForGroup(BenchGroup group, String moduleQualifiedName) private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualifiedName, BenchGroup group) throws IOException { String groupFullClassName = createGroupClassName(group); - System.out.println("[mylog] Generating spec code for group '" + groupFullClassName + "'"); String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); List specs = group.specs(); List specJavaNames = specs diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index c325a925ea84..31bc47064e2f 100644 --- a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -14,12 +14,6 @@ public class LibBenchRunner { public static void main(String[] args) { - System.out.println("Running LibBenchRunner.Main:"); - var langs = Engine.create().getLanguages(); - System.out.println("Languages: "); - for (var lang : langs.keySet()) { - System.out.println(" " + lang); - } CommandLineOptions cmdOpts = null; try { cmdOpts = new CommandLineOptions(args); From ad538e80347fc0c35e6bfa76ef98b518d460b692 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:57:23 +0200 Subject: [PATCH 35/71] Fail annotation processing properly instead of throwing exceptions --- .../benchmarks/processor/BenchProcessor.java | 52 +++++++++++++++---- 1 file changed, 42 insertions(+), 10 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 05e95052dc08..eba6601a316e 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -9,16 +9,17 @@ import java.util.Set; import java.util.stream.Collectors; import javax.annotation.processing.AbstractProcessor; +import javax.annotation.processing.FilerException; import javax.annotation.processing.Processor; import javax.annotation.processing.RoundEnvironment; import javax.annotation.processing.SupportedAnnotationTypes; import javax.annotation.processing.SupportedSourceVersion; import javax.lang.model.SourceVersion; import javax.lang.model.element.TypeElement; +import javax.tools.Diagnostic.Kind; import org.enso.benchmarks.BenchGroup; import org.enso.benchmarks.BenchSpec; import org.enso.benchmarks.BenchSuiteWrapper; -import org.graalvm.polyglot.Engine; import org.openide.util.lookup.ServiceProvider; @SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") @@ -67,7 +68,7 @@ public BenchProcessor() { .toURI() ); } catch (URISyntaxException e) { - throw new IllegalStateException("Unreachable", e); + failWithMessage("ensoDir not found: " + e.getMessage()); } for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { if (ensoDir.getName().equals("enso")) { @@ -75,7 +76,7 @@ public BenchProcessor() { } } if (ensoDir == null) { - throw new IllegalStateException("Unreachable: Could not find Enso root directory"); + failWithMessage("Unreachable: Could not find Enso root directory"); } benchRootDir = ensoDir.toPath() @@ -83,7 +84,7 @@ public BenchProcessor() { .resolve("Benchmarks") .toFile(); if (!benchRootDir.isDirectory() || !benchRootDir.canRead()) { - throw new IllegalStateException("Unreachable: Could not find Enso benchmarks directory"); + failWithMessage("Unreachable: Could not find Enso benchmarks directory"); } // Note that ensoHomeOverride does not have to exist, only its parent directory @@ -95,15 +96,25 @@ public BenchProcessor() { new SpecCollector(benchRootDir, ensoHomeOverride); } + @Override + public SourceVersion getSupportedSourceVersion() { + return SourceVersion.latest(); + } + @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { - Collection benchSuites = specCollector.collectAllBenchSpecs(); - for (BenchSuiteWrapper benchSuite : benchSuites) { - for (BenchGroup group : benchSuite.getGroups()) { - generateClassForGroup(group, benchSuite.getModuleQualifiedName()); + try { + Collection benchSuites = specCollector.collectAllBenchSpecs(); + for (BenchSuiteWrapper benchSuite : benchSuites) { + for (BenchGroup group : benchSuite.getGroups()) { + generateClassForGroup(group, benchSuite.getModuleQualifiedName()); + } } + return true; + } catch (Throwable throwable) { + failWithMessage("Uncaught exception in " + getClass().getName() + ": " + throwable.getMessage()); + return false; } - return true; } private void generateClassForGroup(BenchGroup group, String moduleQualifiedName) { @@ -111,10 +122,27 @@ private void generateClassForGroup(BenchGroup group, String moduleQualifiedName) try (Writer srcFileWriter = processingEnv.getFiler().createSourceFile(fullClassName).openWriter()) { generateClassForGroup(srcFileWriter, moduleQualifiedName, group); } catch (IOException e) { - System.err.println("Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); + if (!isResourceAlreadyExistsException(e)) { + failWithMessage("Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); + } } } + /** + * Returns true iff the given exception is thrown because a file already exists exception. + * There is no better way to check this. + * @param e Exception to check. + * @return true iff the given exception is thrown because a file already exists exception. + */ + private static boolean isResourceAlreadyExistsException(IOException e) { + List messages = List.of( + "Source file already created", + "Resource already created", + "Attempt to recreate a file" + ); + return e instanceof FilerException && messages.stream().anyMatch(msg -> e.getMessage().contains(msg)); + } + private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualifiedName, BenchGroup group) throws IOException { String groupFullClassName = createGroupClassName(group); String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); @@ -227,4 +255,8 @@ private static char peekLastChar(StringBuilder sb) { return 0; } } + + private void failWithMessage(String msg) { + processingEnv.getMessager().printMessage(Kind.ERROR, msg); + } } From dc09d02993433df412e3449df86ec584cffd2fe8 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:57:40 +0200 Subject: [PATCH 36/71] Fix SourceVersion warning --- .../main/java/org/enso/benchmarks/processor/BenchProcessor.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index eba6601a316e..c2d39416c8e0 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -13,7 +13,6 @@ import javax.annotation.processing.Processor; import javax.annotation.processing.RoundEnvironment; import javax.annotation.processing.SupportedAnnotationTypes; -import javax.annotation.processing.SupportedSourceVersion; import javax.lang.model.SourceVersion; import javax.lang.model.element.TypeElement; import javax.tools.Diagnostic.Kind; @@ -23,7 +22,6 @@ import org.openide.util.lookup.ServiceProvider; @SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") -@SupportedSourceVersion(SourceVersion.RELEASE_17) @ServiceProvider(service = Processor.class) public class BenchProcessor extends AbstractProcessor { From 60f23721ead8798373511116161e8f08ced2c70b Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 19:58:51 +0200 Subject: [PATCH 37/71] Catch exception, not throwable --- .../java/org/enso/benchmarks/processor/BenchProcessor.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index c2d39416c8e0..b4f96a9e5272 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -109,8 +109,8 @@ public boolean process(Set annotations, RoundEnvironment } } return true; - } catch (Throwable throwable) { - failWithMessage("Uncaught exception in " + getClass().getName() + ": " + throwable.getMessage()); + } catch (Exception e) { + failWithMessage("Uncaught exception in " + getClass().getName() + ": " + e.getMessage()); return false; } } From 4888e45a2da738c3b910fc1b9b0897d1510db921 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 27 Jul 2023 20:05:48 +0200 Subject: [PATCH 38/71] cosmetics --- .../main/java/org/enso/benchmarks/processor/BenchProcessor.java | 2 +- .../src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index b4f96a9e5272..6dc50f5adfa8 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -247,7 +247,7 @@ private static String normalize(String name) { } private static char peekLastChar(StringBuilder sb) { - if (sb.length() > 0) { + if (!sb.isEmpty()) { return sb.charAt(sb.length() - 1); } else { return 0; diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index 31bc47064e2f..9fb2e9cbea7d 100644 --- a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -3,7 +3,6 @@ import java.io.IOException; import java.util.Collection; import org.enso.benchmarks.processor.GenerateBenchSources; -import org.graalvm.polyglot.Engine; import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; @@ -46,6 +45,7 @@ public static void main(String[] args) { throw new RuntimeException(e); } + // TODO[pm]: Process the results in the same way as we do in `runtime/bench`. System.out.println("Results:"); for (RunResult result : results) { System.out.println(result.toString()); From 95e6deaeba270cfc0ad70df89b5423d871dd0dba Mon Sep 17 00:00:00 2001 From: Jaroslav Tulach Date: Fri, 28 Jul 2023 06:01:57 +0200 Subject: [PATCH 39/71] Easier way to debug the runtime/benchOnly command --- docs/infrastructure/benchmarks.md | 5 +++-- .../org/enso/interpreter/bench/BenchmarksRunner.java | 7 +++++-- project/WithDebugCommand.scala | 9 +++++++-- 3 files changed, 15 insertions(+), 6 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index 327288d89cf9..60d0be28438f 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -25,8 +25,9 @@ source files. In order to change, e.g., the number of measurement iterations, yo modify the parameter to the `@Measurement` annotation. ### Debugging the benchmarks -Currently, the best way to debug the benchmark is to set the `@Fork` annotation to 0, and to -run `withDebug` command like this: + +Make sure your IDE listens for JDWP connection at port 5005. +Debug the benchmark by running `withDebug` command like this: ``` withDebug --debugger benchOnly -- ``` diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java index 16954b0282f5..ffcee2cdf819 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -1,10 +1,10 @@ package org.enso.interpreter.bench; -import jakarta.xml.bind.JAXBException; import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; + import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.BenchmarkList; import org.openjdk.jmh.runner.BenchmarkListEntry; @@ -14,6 +14,8 @@ import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; +import jakarta.xml.bind.JAXBException; + /** Runner class for the benchmarks. Discovers, runs and reports benchmark results. */ public class BenchmarksRunner { public static final File REPORT_FILE = new File("./bench-report.xml"); @@ -39,7 +41,8 @@ public BenchmarkItem run(String label) throws RunnerException, JAXBException { if (Boolean.getBoolean("bench.compileOnly")) { builder .measurementIterations(1) - .warmupIterations(0); + .warmupIterations(0) + .forks(0); } Options benchmarkOptions = builder.build(); diff --git a/project/WithDebugCommand.scala b/project/WithDebugCommand.scala index 824160dc8491..aa450f9bb3e7 100644 --- a/project/WithDebugCommand.scala +++ b/project/WithDebugCommand.scala @@ -16,7 +16,12 @@ import sbt._ */ object WithDebugCommand { val DEBUG_OPTION = - "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y"; + "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y" + + val debugOptions = Seq( + DEBUG_OPTION, + "-Dbench.compileOnly=true" + ) val truffleNoBackgroundCompilationOptions = Seq( "-Dpolyglot.engine.BackgroundCompilation=false" @@ -80,7 +85,7 @@ object WithDebugCommand { else Seq() val debuggerOpts = if (debugFlags.contains(debuggerOption)) - Seq(DEBUG_OPTION) + debugOptions else Seq() val javaOpts: Seq[String] = Seq( truffleNoBackgroundCompilationOptions, From df993839d0142255b8c90f8658769120c1f550af Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 12:09:52 +0200 Subject: [PATCH 40/71] Restrict dependencies scopes --- build.sbt | 39 ++++++++++++++++--------------- project/FrgaalJavaCompiler.scala | 40 +++++++++++++++++--------------- 2 files changed, 41 insertions(+), 38 deletions(-) diff --git a/build.sbt b/build.sbt index 594772758fea..684982cd1c9e 100644 --- a/build.sbt +++ b/build.sbt @@ -1773,17 +1773,16 @@ lazy val `bench-processor` = (project in file("std-bits/bench-processor")) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion, - "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided" ), Compile / javacOptions := ((Compile / javacOptions).value ++ - // Only run ServiceProvider processor and ignore those defined in META-INF, thus - // fixing incremental compilation setup - Seq( - "-processor", - "org.netbeans.modules.openide.util.ServiceProviderProcessor" - ) - ), + // Only run ServiceProvider processor and ignore those defined in META-INF, thus + // fixing incremental compilation setup + Seq( + "-processor", + "org.netbeans.modules.openide.util.ServiceProviderProcessor" + )) ) .dependsOn(`polyglot-api`) .dependsOn(runtime) @@ -1793,13 +1792,15 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) frgaalJavaCompilerSetting, assembly / mainClass := (Compile / run / mainClass).value, libraryDependencies ++= jmh ++ Seq( - "org.openjdk.jmh" % "jmh-core" % jmhVersion, - "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion, - "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", - "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark, + "org.openjdk.jmh" % "jmh-core" % jmhVersion % Benchmark, + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % Benchmark, + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark ), commands += WithDebugCommand.withDebug, - (Benchmark / run / mainClass) := Some("org.enso.benchmarks.libs.LibBenchRunner"), + (Benchmark / run / mainClass) := Some( + "org.enso.benchmarks.libs.LibBenchRunner" + ), (Benchmark / run / fork) := true, (Benchmark / run / connectInput) := true, // Pass -Dtruffle.class.path.append to javac @@ -1816,7 +1817,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) .mkString(File.pathSeparator) Seq( "-J--no-limit-modules", - s"-J-Dtruffle.class.path.append=$appendClasspath", + s"-J-Dtruffle.class.path.append=$appendClasspath" ) }, (Compile / javacOptions) ++= Seq( @@ -1841,13 +1842,13 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) .map(_.data) .mkString(File.pathSeparator) Seq( - s"-Dtruffle.class.path.append=$appendClasspath", + s"-Dtruffle.class.path.append=$appendClasspath" ) - }, + } ) .configs(Benchmark) .settings( - inConfig(Benchmark)(Defaults.testSettings), + inConfig(Benchmark)(Defaults.testSettings) ) .settings( bench := (Benchmark / run).toTask("").tag(Exclusive).value, @@ -1855,7 +1856,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) import complete.Parsers.spaceDelimited val name = spaceDelimited("").parsed match { case List(name) => name - case _ => throw new IllegalArgumentException("Expected one argument.") + case _ => throw new IllegalArgumentException("Expected one argument.") } Def.task { (Benchmark / run).toTask(" " + name).value diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index ee7c85441101..aa97e6c58c09 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -14,11 +14,11 @@ import sbt.internal.inc.CompilerArguments import sbt.internal.inc.javac.JavacLogger import sbt.io.IO import sbt.util.Logger -import xsbti.{PathBasedFile, Reporter, VirtualFile, Logger as XLogger} -import xsbti.compile.{IncToolOptions, Output, JavaCompiler as XJavaCompiler} +import xsbti.{PathBasedFile, Reporter, VirtualFile, Logger => XLogger} +import xsbti.compile.{IncToolOptions, Output, JavaCompiler => XJavaCompiler} import java.io.File -import java.nio.file.{Files, Path, Paths, StandardCopyOption} +import java.nio.file.{Path, Paths} import scala.sys.process.Process import scala.util.Using import java.io.FileWriter @@ -29,7 +29,8 @@ object FrgaalJavaCompiler { val frgaal = "org.frgaal" % "compiler" % "19.0.1" % "provided" val sourceLevel = "19" - val debugArg = "-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000" + val debugArg = + "-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000" def compilers( classpath: sbt.Keys.Classpath, @@ -67,14 +68,13 @@ object FrgaalJavaCompiler { xsbti.compile.Compilers.of(sbtCompilers.scalac, javaTools) } - /** - * Helper method to launch programs. - * - * If -J--no-limit-modules is passed to javac, no --limit-modules option is passed to frgaal. - * With -J--no-limit-modules, one can use any class path in annotation processors. - * All the options with "-J" prefix are prepended to the frgaal command line, instead of passing - * them via an argument file. Moreover, the "-J" prefix is stripped. - * */ + /** Helper method to launch programs. + * + * If -J--no-limit-modules is passed to javac, no --limit-modules option is passed to frgaal. + * With -J--no-limit-modules, one can use any class path in annotation processors. + * All the options with "-J" prefix are prepended to the frgaal command line, instead of passing + * them via an argument file. Moreover, the "-J" prefix is stripped. + */ def launch( javaHome: Option[Path], compilerJar: Path, @@ -86,9 +86,9 @@ object FrgaalJavaCompiler { source: Option[String], target: String ): Boolean = { - val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) - var noLimitModules = false - val debugAnotProcessorOpt = jArgs.contains(debugArg) + val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) + var noLimitModules = false + val debugAnotProcessorOpt = jArgs.contains(debugArg) val strippedJArgs: ArrayBuffer[String] = ArrayBuffer() for (strippedJArg <- jArgs.map(_.stripPrefix("-J"))) { if (strippedJArg == "--no-limit-modules") { @@ -97,7 +97,7 @@ object FrgaalJavaCompiler { strippedJArgs += strippedJArg } } - val outputOption = CompilerArguments.outputOption(output) + val outputOption = CompilerArguments.outputOption(output) val sources = sources0 map { case x: PathBasedFile => x.toPath.toAbsolutePath.toString } @@ -237,7 +237,7 @@ object FrgaalJavaCompiler { } else { Seq( "--limit-modules", - "java.base,jdk.zipfs,jdk.internal.vm.compiler.management", + "java.base,jdk.zipfs,jdk.internal.vm.compiler.management" ) } val forkArgs = (strippedJArgs ++ limitModulesArgs ++ Seq( @@ -250,10 +250,12 @@ object FrgaalJavaCompiler { val javacLogger = new JavacLogger(log, reporter, cwd) var exitCode = -1 if (debugAnotProcessorOpt) { - log.info(s"Frgaal compiler is about to be launched with $debugArg, which means that" + + log.info( + s"Frgaal compiler is about to be launched with $debugArg, which means that" + " it will wait for a debugger to attach. The output from the compiler is by default" + " redirected, therefore \"Listening to the debugger\" message will not be displayed." + - " You should attach the debugger now.") + " You should attach the debugger now." + ) } try { exitCode = Process(exe +: forkArgs, cwd) ! javacLogger From 00e6cf39804f8a0e9a00fbb102ef65f3b7524701 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 12:58:59 +0200 Subject: [PATCH 41/71] Restructure debugging docs, reformat. --- docs/CONTRIBUTING.md | 29 +------ docs/debugger/README.md | 104 +++---------------------- docs/debugger/chrome-devtools.md | 41 ++++++++++ docs/debugger/mixed-debugging.md | 48 ++++++++++++ docs/debugger/runtime-debugging.md | 69 +++++++++++++++++ docs/infrastructure/README.md | 4 +- docs/infrastructure/benchmarks.md | 120 +++++++++++++++++------------ 7 files changed, 243 insertions(+), 172 deletions(-) create mode 100644 docs/debugger/chrome-devtools.md create mode 100644 docs/debugger/mixed-debugging.md create mode 100644 docs/debugger/runtime-debugging.md diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index 93bc56c1e26f..f13bbeddad73 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -416,32 +416,6 @@ sbt:project-manager> withDebug run --debugger read more about [debugging Java & Enso code](debugger/README.md). -#### Debugging annotation processors -The Engine uses annotation processors to generate some of the Java code, e.g., the -builtin methods with `org.enso.interpreter.dsl.MethodProcessor`, or JMH benchmark -sources with `org.enso.benchmarks.processor.BenchProcessor`. Annotation -processors are invoked by the Java compiler (`javac`), therefore, we need special -instructions to attach the debugger to them. - -Let's debug `org.enso.interpreter.dsl.MethodProcessor` as an example. The following -are the commands invoked in the `sbt` shell: -- `project runtime` -- `clean` - - Delete all the compiled class files along with all the generated sources by - the annotation processor. This ensures that the annotation processor will - be invoked. -- `set javacOptions += FrgaalJavaCompiler.debugArg` - - This sets a special flag that is passed to the frgaal Java compiler, which - in turn waits for the debugger to attach. Note that this setting is not - persisted and will be reset once the project is reloaded. -- `compile` - - Launches the Java compiler, which will wait for the debugger to attach. - Put a breakpoint in some class of `org.enso.interpreter.dsl` package. - Wait for the message in the console instructing to attach the debugger. -- To reset the `javacOptions` setting, either run `set javacOptions -= FrgaalJavaCompiler.debugArg`, - or reload the project with `reload`. - - #### Working with Assembly In order to examine the assembly generated by GraalVM and HotSpot you need to @@ -783,7 +757,8 @@ interface of the runner prints all server options when you execute it with Below are options uses by the Language Server: - `--server`: Runs the Language Server -- `--root-id `: Content root id. +- `--root-id `: Content root id. The Language Server chooses one randomly, + so can pass any valid UUID. - `--path `: Path to the content root. - `--interface `: Interface for processing all incoming connections. Default value is 127.0.0.1 diff --git a/docs/debugger/README.md b/docs/debugger/README.md index 7012cbe39153..b45d073bc4f7 100644 --- a/docs/debugger/README.md +++ b/docs/debugger/README.md @@ -8,96 +8,14 @@ order: 0 # Enso Debugger -The Enso Debugger allows amongst other things, to execute arbitrary expressions -in a given execution context - this is used to implement a debugging REPL. The -REPL can be launched when triggering a breakpoint in the code. - -This folder contains all documentation pertaining to the REPL and the debugger, -which is broken up as follows: - -- [**The Enso Debugger Protocol:**](./protocol.md) The protocol for the Debugger - -# Chrome Developer Tools Debugger - -As a well written citizen of the [GraalVM](http://graalvm.org) project the Enso -language can be used with existing tools available for the overall platform. One -of them is -[Chrome Debugger](https://www.graalvm.org/22.1/tools/chrome-debugger/) and Enso -language is fully integrated with it. Launch the `bin/enso` executable with -additional `--inspect` option and debug your Enso programs in _Chrome Developer -Tools_. - -```bash -enso$ ./built-distribution/enso-engine-*/enso-*/bin/enso --inspect --run ./test/Tests/src/Data/Numbers_Spec.enso -Debugger listening on ws://127.0.0.1:9229/Wugyrg9 -For help, see: https://www.graalvm.org/tools/chrome-debugger -E.g. in Chrome open: devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/Wugyrg9 -``` - -copy the printed URL into chrome browser and you should see: - -![Chrome Debugger](https://user-images.githubusercontent.com/26887752/209614265-684f530e-cf7e-45d5-9450-7ea1e4f65986.png) - -Step in, step over, set breakpoints, watch values of the variables as well as -evaluate arbitrary expressions in the console. Note that as of December 2022, -with GraalVM 22.3.0, there is a well-known -[bug in Truffle](https://github.com/oracle/graal/issues/5513) that causes -`NullPointerException` when a host object gets into the chrome inspector. There -is a workaround for that, but it may not work in certain situations. Therefore, -if you encounter `NullPointerException` thrown from - -``` -at org.graalvm.truffle/com.oracle.truffle.polyglot.PolyglotContextImpl.getContext(PolyglotContextImpl.java:685) -``` - -simply ignore it. It will be handled within the debugger and should not affect -the rest of the environment. - -# Debugging Enso and Java Code at Once - -Enso libraries are written in a mixture of Enso code and Java libraries. -Debugging both sides (the Java as well as Enso code) is possible with a decent -IDE. - -Get [NetBeans](http://netbeans.apache.org) version 13 or newer or -[VS Code with Apache Language Server extension](https://cwiki.apache.org/confluence/display/NETBEANS/Apache+NetBeans+Extension+for+Visual+Studio+Code) -and _start listening on port 5005_ with _Debug/Attach Debugger_ or by specifying -following debug configuration in VSCode: - -```json -{ - "name": "Listen to 5005", - "type": "java+", - "request": "attach", - "listen": "true", - "hostName": "localhost", - "port": "5005" -} -``` - -Then it is just about executing following Sbt command which builds CLI version -of the engine, launches it in debug mode and passes all other arguments to the -started process: - -```bash -sbt:enso> runEngineDistribution --debug --run ./test/Tests/src/Data/Numbers_Spec.enso -``` - -Alternatively you can pass in special JVM arguments when launching the -`bin/enso` launcher: - -```bash -enso$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=n,address=5005 ./built-distribution/enso-engine-*/enso-*/bin/enso --run ./test/Tests/src/Data/Numbers_Spec.enso -``` - -As soon as the debuggee connects and the Enso language starts - choose the -_Toggle Pause in GraalVM Script_ button in the toolbar: - -![NetBeans Debugger](https://user-images.githubusercontent.com/26887752/209614191-b0513635-819b-4c64-a6f9-9823b90a1513.png) - -and your execution shall stop on the next `.enso` line of code. This mode allows -to debug both - the Enso code as well as Java code. The stack traces shows a -mixture of Java and Enso stack frames by default. Right-clicking on the thread -allows one to switch to plain Java view (with a way more stack frames) and back. -Analyzing low level details as well as Enso developer point of view shall be -simple with such tool. +This folder contains all documentation pertaining to the debugging facilities +used by Enso, broken up as follows: + +- [**The Enso Debugger Protocol:**](./protocol.md) The protocol for the REPL + Debugger. +- [**Chrome devtools debugger:**](./chrome-devtools.md) A guide how to debug + Enso code using Chrome devtools. +- [**Debugging Enso and Java code at once:**](./mixed-debugging.md) A + step-by-step guide how to debug both Enso and Java code in a single debugger. +- [**Debugging Engine (Runtime) only Java code:**](./runtime-debugging.md) A + guide how to debug the internal Engine Java code. diff --git a/docs/debugger/chrome-devtools.md b/docs/debugger/chrome-devtools.md new file mode 100644 index 000000000000..7b704457619c --- /dev/null +++ b/docs/debugger/chrome-devtools.md @@ -0,0 +1,41 @@ +# Chrome Developer Tools Debugger + +As a well written citizen of the [GraalVM](http://graalvm.org) project the Enso +language can be used with existing tools available for the overall platform. One +of them is +[Chrome Debugger](https://www.graalvm.org/22.1/tools/chrome-debugger/) and Enso +language is fully integrated with it. Launch the `bin/enso` executable with +additional `--inspect` option and debug your Enso programs in _Chrome Developer +Tools_. + +```bash +enso$ ./built-distribution/enso-engine-*/enso-*/bin/enso --inspect --run ./test/Tests/src/Data/Numbers_Spec.enso +Debugger listening on ws://127.0.0.1:9229/Wugyrg9 +For help, see: https://www.graalvm.org/tools/chrome-debugger +E.g. in Chrome open: devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/Wugyrg9 +``` + +copy the printed URL into chrome browser and you should see: + +![Chrome Debugger](https://user-images.githubusercontent.com/26887752/209614265-684f530e-cf7e-45d5-9450-7ea1e4f65986.png) + +Step in, step over, set breakpoints, watch values of the variables as well as +evaluate arbitrary expressions in the console. Note that as of December 2022, +with GraalVM 22.3.0, there is a well-known +[bug in Truffle](https://github.com/oracle/graal/issues/5513) that causes +`NullPointerException` when a host object gets into the chrome inspector. There +is a workaround for that, but it may not work in certain situations. Therefore, +if you encounter `NullPointerException` thrown from + +``` +at org.graalvm.truffle/com.oracle.truffle.polyglot.PolyglotContextImpl.getContext(PolyglotContextImpl.java:685) +``` + +simply ignore it. It will be handled within the debugger and should not affect +the rest of the environment. + +## Tips and tricks + +- Use `env JAVA_OPTS=-Dpolyglot.inspect.Path=enso_debug` to set the chrome to + use a fixed URL. In this case the URL is + `devtools://devtools/bundled/js_app.html?ws=127.0.0.1:9229/enso_debug` diff --git a/docs/debugger/mixed-debugging.md b/docs/debugger/mixed-debugging.md new file mode 100644 index 000000000000..48f0f4107f79 --- /dev/null +++ b/docs/debugger/mixed-debugging.md @@ -0,0 +1,48 @@ +# Debugging Enso and Java Code at Once + +Enso libraries are written in a mixture of Enso code and Java libraries. +Debugging both sides (the Java as well as Enso code) is possible with a decent +IDE. + +Get [NetBeans](http://netbeans.apache.org) version 13 or newer or +[VS Code with Apache Language Server extension](https://cwiki.apache.org/confluence/display/NETBEANS/Apache+NetBeans+Extension+for+Visual+Studio+Code) +and _start listening on port 5005_ with _Debug/Attach Debugger_ or by specifying +following debug configuration in VSCode: + +```json +{ + "name": "Listen to 5005", + "type": "java+", + "request": "attach", + "listen": "true", + "hostName": "localhost", + "port": "5005" +} +``` + +Then it is just about executing following Sbt command which builds CLI version +of the engine, launches it in debug mode and passes all other arguments to the +started process: + +```bash +sbt:enso> runEngineDistribution --debug --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +Alternatively you can pass in special JVM arguments when launching the +`bin/enso` launcher: + +```bash +enso$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=n,address=5005 ./built-distribution/enso-engine-*/enso-*/bin/enso --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +As soon as the debuggee connects and the Enso language starts - choose the +_Toggle Pause in GraalVM Script_ button in the toolbar: + +![NetBeans Debugger](https://user-images.githubusercontent.com/26887752/209614191-b0513635-819b-4c64-a6f9-9823b90a1513.png) + +and your execution shall stop on the next `.enso` line of code. This mode allows +to debug both - the Enso code as well as Java code. The stack traces shows a +mixture of Java and Enso stack frames by default. Right-clicking on the thread +allows one to switch to plain Java view (with a way more stack frames) and back. +Analyzing low level details as well as Enso developer point of view shall be +simple with such tool. diff --git a/docs/debugger/runtime-debugging.md b/docs/debugger/runtime-debugging.md new file mode 100644 index 000000000000..6ab9ea242737 --- /dev/null +++ b/docs/debugger/runtime-debugging.md @@ -0,0 +1,69 @@ +# Runtime (Engine) debugging + +This section explains how to debug various parts of the Engine. By Engine, we +mean all the Java code located in the `runtime` SBT project, in `engine` +directory. + +## Debugging source file evaluation + +This subsection provides a guide how to debug a single Enso source file +evaluation. To evaluate a single source file, we use the _Engine distribution_ +built with `buildEngineDistribution` command. Both of the following two options +starts the JVM in a debug mode. After the JVM is started in a debug mode, simply +attach the debugger to the JVM process at the specified port. + +The first option is to invoke the engine distribution from SBT shell with: + +```sh +sbt:enso> runEngineDistribution --debug --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +The second options is to pass in special JVM arguments when launching the +`bin/enso` from the engine distribution: + +```bash +enso$ JAVA_OPTS=-agentlib:jdwp=transport=dt_socket,server=n,address=5005 ./built-distribution/enso-engine-*/enso-*/bin/enso --run ./test/Tests/src/Data/Numbers_Spec.enso +``` + +### Tips and tricks + +There is no simple mapping of the Enso source code to the engine's Java code, so +if you try to debug a specific expression, it might be a bit tricky. However, +the following steps should help you to skip all the irrelevant code and get to +the code you are interested in: + +- To debug a method called `foo`, put a breakpoint in + `org.enso.interpreter.node.ClosureRootNode#execute` with a condition on + `this.name.contains("foo")` +- To debug a specific expression, put some _unique_ expression, like + `Debug.eval "1+1"`, in front of it and put a breakpoint in a Truffle node + corresponding to that unique expression, in this case that is + `org.enso.interpreter.node.expression.builtin.debug.DebugEvalNode`. + +## Debugging annotation processors + +The Engine uses annotation processors to generate some of the Java code, e.g., +the builtin methods with `org.enso.interpreter.dsl.MethodProcessor`, or JMH +benchmark sources with `org.enso.benchmarks.processor.BenchProcessor`. +Annotation processors are invoked by the Java compiler (`javac`), therefore, we +need special instructions to attach the debugger to them. + +Let's debug `org.enso.interpreter.dsl.MethodProcessor` as an example. The +following are the commands invoked in the `sbt` shell: + +- `project runtime` +- `clean` + - Delete all the compiled class files along with all the generated sources by + the annotation processor. This ensures that the annotation processor will be + invoked. +- `set javacOptions += FrgaalJavaCompiler.debugArg` + - This sets a special flag that is passed to the frgaal Java compiler, which + in turn waits for the debugger to attach. Note that this setting is not + persisted and will be reset once the project is reloaded. +- `compile` + - Launches the Java compiler, which will wait for the debugger to attach. Put + a breakpoint in some class of `org.enso.interpreter.dsl` package. Wait for + the message in the console instructing to attach the debugger. +- To reset the `javacOptions` setting, either run + `set javacOptions -= FrgaalJavaCompiler.debugArg`, or reload the project with + `reload`. diff --git a/docs/infrastructure/README.md b/docs/infrastructure/README.md index ec6421b1ae6a..623ceb1b3fe2 100644 --- a/docs/infrastructure/README.md +++ b/docs/infrastructure/README.md @@ -22,7 +22,7 @@ up as follows: - [**Upgrading GraalVM:**](upgrading-graalvm.md) Description of steps that have to be performed by each developer when the project is upgraded to a new version of GraalVM. -- [**Benchmarks:**](benchmarks.md) Description of the benchmarking infrastructure - used for measuring performance of the runtime. +- [**Benchmarks:**](benchmarks.md) Description of the benchmarking + infrastructure used for measuring performance of the runtime. - [**Logging**:](logging.md) Description of an unified and centralized logging infrastructure that should be used by all components. diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index 60d0be28438f..bd4cc4763856 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -1,79 +1,99 @@ # Benchmarks -In this document, we describe the benchmark types used for the runtime - micro benchmarks in -the section [Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and meso benchmarks -in the section [Standard library benchmarks](#standard-library-benchmarks), and how and where -are the results stored and visualized in the section [Visualization](#visualization). - -To track the performance of the engine, we use [JMH](https://openjdk.org/projects/code-tools/jmh/). -There are two types of benchmarks: -- [micro benchmarks](#engine-jmh-microbenchmarks) located directly in the `runtime` SBT project. - These benchmarks are written in Java, and are used to measure the performance of - specific parts of the engine. -- [standard library benchmarks](#standard-library-benchmarks) (meso benchmarks) located in the `test/Benchmarks` Enso project. - These benchmarks are entirelly written in Enso, along with the harness code. + +In this document, we describe the benchmark types used for the runtime - micro +benchmarks in the section +[Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and meso benchmarks in +the section [Standard library benchmarks](#standard-library-benchmarks), and how +and where are the results stored and visualized in the section +[Visualization](#visualization). + +To track the performance of the engine, we use +[JMH](https://openjdk.org/projects/code-tools/jmh/). There are two types of +benchmarks: + +- [micro benchmarks](#engine-jmh-microbenchmarks) located directly in the + `runtime` SBT project. These benchmarks are written in Java, and are used to + measure the performance of specific parts of the engine. +- [standard library benchmarks](#standard-library-benchmarks) (meso benchmarks) + located in the `test/Benchmarks` Enso project. These benchmarks are entirelly + written in Enso, along with the harness code. ## Engine JMH microbenchmarks + These benchmarks are written in Java and are used to measure the performance of -specific parts of the engine. -The sources are located in the `runtime` SBT project, under `src/bench` source directory. +specific parts of the engine. The sources are located in the `runtime` SBT +project, under `src/bench` source directory. ### Running the benchmarks -To run the benchmarks, use `bench` or `benchOnly` command - `bench` runs all the benchmarks and -`benchOnly` runs only one benchmark specified with the fully qualified name. -The parameters for these benchmarks are hard-coded inside the JMH annotations in the -source files. In order to change, e.g., the number of measurement iterations, you need to -modify the parameter to the `@Measurement` annotation. + +To run the benchmarks, use `bench` or `benchOnly` command - `bench` runs all the +benchmarks and `benchOnly` runs only one benchmark specified with the fully +qualified name. The parameters for these benchmarks are hard-coded inside the +JMH annotations in the source files. In order to change, e.g., the number of +measurement iterations, you need to modify the parameter to the `@Measurement` +annotation. ### Debugging the benchmarks -Make sure your IDE listens for JDWP connection at port 5005. -Debug the benchmark by running `withDebug` command like this: +Make sure your IDE listens for JDWP connection at port 5005. Debug the benchmark +by running `withDebug` command like this: + ``` withDebug --debugger benchOnly -- ``` ## Standard library benchmarks -Unlike the micro benchmarks, these benchmarks are written entirelly in Enso and located in the -`test/Benchmarks` Enso project. Sometimes, we call those *meso* benchmarks. -There are two ways to run these benchmarks: + +Unlike the micro benchmarks, these benchmarks are written entirelly in Enso and +located in the `test/Benchmarks` Enso project. Sometimes, we call those _meso_ +benchmarks. There are two ways to run these benchmarks: + - [Running standalone](#running-standalone) - [Running via JMH launcher](#running-via-jmh-launcher) ### Running standalone + A single source file in the project may contain multiple benchmark definitions. -If the source file defines `main` method, we can evaluate it the same way as any other -Enso source file, for example via `runEngineDistribution --in-project test/Benchmarks --run `. -The harness within the project is not meant for any sophisticated benchmarking, but rather -for quick local evaluation. See the `Bench.measure` method documentation for more details. -For more sophisticated approach, run the benchmarks via JMH launcher. +If the source file defines `main` method, we can evaluate it the same way as any +other Enso source file, for example via +`runEngineDistribution --in-project test/Benchmarks --run `. The +harness within the project is not meant for any sophisticated benchmarking, but +rather for quick local evaluation. See the `Bench.measure` method documentation +for more details. For more sophisticated approach, run the benchmarks via JMH +launcher. ### Running via JMH launcher -The JMH launcher is located in `std-bits/benchmarks` directory, as `bench-libs` SBT project. -It is a single Java class with a `main` method that just delegates to the + +The JMH launcher is located in `std-bits/benchmarks` directory, as `bench-libs` +SBT project. It is a single Java class with a `main` method that just delegates +to the [standard JMH launcher](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/Main.java), -therefore, supports all the command line options as the standard launcher. -For the full options summary, either see the [JMH source code](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java), +therefore, supports all the command line options as the standard launcher. For +the full options summary, either see the +[JMH source code](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java), or run the launcher with `-h` option. -The `bench-libs` SBT project supports `bench` and `benchOnly` commands, that work the same -as in the `runtime` project, with the exception that the benchmark name does not have to be -specified as a fully qualified name, but as a regular expression. -To access the full flexibility of the JMH launcher, run it via `Bench/run` - for example, -to see the help message: `Bench/run -h`. - -To debug a single benchmark, set the `javaOptions` with something like `set javaOptions += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000"`, -and launch a single benchmark without forking with `Bench/run -f 0 `. -Then, attach to the debugger with your IDE. +The `bench-libs` SBT project supports `bench` and `benchOnly` commands, that +work the same as in the `runtime` project, with the exception that the benchmark +name does not have to be specified as a fully qualified name, but as a regular +expression. To access the full flexibility of the JMH launcher, run it via +`Bench/run` - for example, to see the help message: `Bench/run -h`. +To debug a single benchmark, set the `javaOptions` with something like +`set javaOptions += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000"`, +and launch a single benchmark without forking with +`Bench/run -f 0 `. Then, attach to the debugger with your IDE. ## Visualization -The benchmarks are invoked as a daily [GitHub Action](https://github.com/enso-org/enso/actions/workflows/benchmark.yml), -that can be invoked manually on a specific branch as well. -The results are kept in the artifacts produced from the actions. -In `tools/performance/engine-benchmarks` directory, there is a simple Python -script for collecting and processing the results. See the README in that directory -for more information about how to run that script. -This script is invoked regularly on a private machine and the results are -published in [https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). + +The benchmarks are invoked as a daily +[GitHub Action](https://github.com/enso-org/enso/actions/workflows/benchmark.yml), +that can be invoked manually on a specific branch as well. The results are kept +in the artifacts produced from the actions. In +`tools/performance/engine-benchmarks` directory, there is a simple Python script +for collecting and processing the results. See the README in that directory for +more information about how to run that script. This script is invoked regularly +on a private machine and the results are published in +[https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). This is not meant as a permanent solution. From 62f34801a925290558a5593816bb130d1ace0743 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 13:55:54 +0200 Subject: [PATCH 42/71] Remove meso benchmark terminology --- docs/infrastructure/benchmarks.md | 11 +++++------ .../java/org/enso/benchmarks/BenchSuiteWrapper.java | 2 +- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index bd4cc4763856..c7c512467114 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -1,8 +1,8 @@ # Benchmarks -In this document, we describe the benchmark types used for the runtime - micro +In this document, we describe the benchmark types used for the runtime - Engine micro benchmarks in the section -[Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and meso benchmarks in +[Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and standard library benchmarks in the section [Standard library benchmarks](#standard-library-benchmarks), and how and where are the results stored and visualized in the section [Visualization](#visualization). @@ -14,7 +14,7 @@ benchmarks: - [micro benchmarks](#engine-jmh-microbenchmarks) located directly in the `runtime` SBT project. These benchmarks are written in Java, and are used to measure the performance of specific parts of the engine. -- [standard library benchmarks](#standard-library-benchmarks) (meso benchmarks) +- [standard library benchmarks](#standard-library-benchmarks) located in the `test/Benchmarks` Enso project. These benchmarks are entirelly written in Enso, along with the harness code. @@ -44,9 +44,8 @@ withDebug --debugger benchOnly -- ## Standard library benchmarks -Unlike the micro benchmarks, these benchmarks are written entirelly in Enso and -located in the `test/Benchmarks` Enso project. Sometimes, we call those _meso_ -benchmarks. There are two ways to run these benchmarks: +Unlike the Engine micro benchmarks, these benchmarks are written entirelly in Enso and +located in the `test/Benchmarks` Enso project. There are two ways to run these benchmarks: - [Running standalone](#running-standalone) - [Running via JMH launcher](#running-via-jmh-launcher) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java index a39de263f648..86b49b38c4ce 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java @@ -4,7 +4,7 @@ import java.util.Optional; import org.graalvm.polyglot.Value; -public class BenchSuiteWrapper { +public final class BenchSuiteWrapper { private final BenchSuite suite; private final String moduleQualifiedName; From 80564b00fae2d7e046bb17713033dbd8c83133f9 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 14:05:50 +0200 Subject: [PATCH 43/71] Remove unecessary method from BenchSUiteWrapper --- .../src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java | 4 ---- .../java/org/enso/benchmarks/processor/BenchProcessor.java | 2 +- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java index 86b49b38c4ce..358cd8e86454 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java @@ -21,10 +21,6 @@ public String getModuleQualifiedName() { return moduleQualifiedName; } - public Value getDefaultInputArgument() { - return Value.asValue(null); - } - public BenchGroup findGroupByName(String groupName) { return suite .groups() diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 6dc50f5adfa8..9bca716e7d24 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -201,7 +201,7 @@ private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualif javaSrcFileWriter.append(" this.benchFunc_" + specJavaName + " = benchSpec_" + specJavaName + ".code();\n"); } javaSrcFileWriter.append(" \n"); - javaSrcFileWriter.append(" this.groupInputArg = benchSuite.getDefaultInputArgument();\n"); + javaSrcFileWriter.append(" this.groupInputArg = Value.asValue(null);\n"); javaSrcFileWriter.append(" } \n"); // end of setup method javaSrcFileWriter.append(" \n"); for (var specJavaName : specJavaNames) { From 2e57598fba196f5f6f27a9d77055e25c8f0ffd67 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 14:37:13 +0200 Subject: [PATCH 44/71] Some refactoring. --- .../java/org/enso/benchmarks/BenchGroup.java | 4 +++ .../java/org/enso/benchmarks/BenchSpec.java | 1 + .../java/org/enso/benchmarks/BenchSuite.java | 4 +++ ...uiteWrapper.java => ModuleBenchSuite.java} | 8 ++++-- .../benchmarks/processor/BenchProcessor.java | 10 +++---- .../benchmarks/processor/SpecCollector.java | 28 +++++++++---------- 6 files changed, 33 insertions(+), 22 deletions(-) rename std-bits/bench-processor/src/main/java/org/enso/benchmarks/{BenchSuiteWrapper.java => ModuleBenchSuite.java} (84%) diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java index c7cc890e2046..fea8a3a68a55 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java @@ -2,6 +2,10 @@ import java.util.List; +/** + * A group of benchmarks with its own name and configuration. + * Corresponds to {@code Bench.Group} defined in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso}. + */ public interface BenchGroup { String name(); BenchConfig configuration(); diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java index fcb71c1d8443..f95decbdf58e 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java @@ -4,6 +4,7 @@ /** * Specification of a single benchmark. + * Corresponds to {@code Bench.Spec} defined in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso}. */ public interface BenchSpec { String name(); diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java index b9ae4d12d76e..ded838b19a02 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java @@ -2,6 +2,10 @@ import java.util.List; +/** + * Wraps all the groups of benchmarks specified in a single module. + * Corresponds to {@code Bench.All} defined in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso}. + */ public interface BenchSuite { List groups(); } diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java similarity index 84% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java rename to std-bits/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java index 358cd8e86454..200820b695c0 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuiteWrapper.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java @@ -2,13 +2,15 @@ import java.util.List; import java.util.Optional; -import org.graalvm.polyglot.Value; -public final class BenchSuiteWrapper { +/** + * A {@link BenchSuite} with a qualified name of the module it is defined in. + */ +public final class ModuleBenchSuite { private final BenchSuite suite; private final String moduleQualifiedName; - public BenchSuiteWrapper(BenchSuite suite, String moduleQualifiedName) { + public ModuleBenchSuite(BenchSuite suite, String moduleQualifiedName) { this.suite = suite; this.moduleQualifiedName = moduleQualifiedName; } diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 9bca716e7d24..d313ac34215a 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -18,7 +18,7 @@ import javax.tools.Diagnostic.Kind; import org.enso.benchmarks.BenchGroup; import org.enso.benchmarks.BenchSpec; -import org.enso.benchmarks.BenchSuiteWrapper; +import org.enso.benchmarks.ModuleBenchSuite; import org.openide.util.lookup.ServiceProvider; @SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") @@ -51,7 +51,7 @@ public class BenchProcessor extends AbstractProcessor { "import org.enso.polyglot.LanguageInfo;", "import org.enso.polyglot.MethodNames;", "import org.enso.benchmarks.processor.SpecCollector;", - "import org.enso.benchmarks.BenchSuiteWrapper;", + "import org.enso.benchmarks.ModuleBenchSuite;", "import org.enso.benchmarks.BenchSpec;", "import org.enso.benchmarks.BenchGroup;" ); @@ -102,8 +102,8 @@ public SourceVersion getSupportedSourceVersion() { @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { try { - Collection benchSuites = specCollector.collectAllBenchSpecs(); - for (BenchSuiteWrapper benchSuite : benchSuites) { + Collection benchSuites = specCollector.collectAllBenchSpecs(); + for (ModuleBenchSuite benchSuite : benchSuites) { for (BenchGroup group : benchSuite.getGroups()) { generateClassForGroup(group, benchSuite.getModuleQualifiedName()); } @@ -192,7 +192,7 @@ private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualif javaSrcFileWriter.append(" Value bindings = ctx.getBindings(LanguageInfo.ID);\n"); javaSrcFileWriter.append(" Value module = bindings.invokeMember(MethodNames.TopScope.GET_MODULE, \"" + moduleQualifiedName + "\");\n"); javaSrcFileWriter.append(" var specCollector = new SpecCollector(benchProjectDir, languageHomeOverride);\n"); - javaSrcFileWriter.append(" BenchSuiteWrapper benchSuite = specCollector.collectBenchSpecFromModuleName(\"" + moduleQualifiedName + "\");\n"); + javaSrcFileWriter.append(" ModuleBenchSuite benchSuite = specCollector.collectBenchSpecFromModuleName(\"" + moduleQualifiedName + "\");\n"); javaSrcFileWriter.append(" \n"); for (int i = 0; i < specs.size(); i++) { var specJavaName = specJavaNames.get(i); diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java index 877837cf5a98..b32584732826 100644 --- a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java +++ b/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java @@ -10,7 +10,7 @@ import java.util.stream.Collectors; import java.util.stream.Stream; import org.enso.benchmarks.BenchSuite; -import org.enso.benchmarks.BenchSuiteWrapper; +import org.enso.benchmarks.ModuleBenchSuite; import org.enso.polyglot.LanguageInfo; import org.enso.polyglot.MethodNames.Module; import org.enso.polyglot.MethodNames.TopScope; @@ -51,7 +51,16 @@ public SpecCollector(File benchProjectDir, File languageHomeOverride) { .build(); } - public Collection collectAllBenchSpecs() { + public ModuleBenchSuite collectBenchSpecFromModuleName(String moduleName) { + Objects.requireNonNull(moduleName); + if (!moduleName.startsWith(benchPackagePrefix)) { + throw new IllegalArgumentException("Module name must start with " + benchPackagePrefix); + } + Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); + return collectBenchSpecFromModule(module, moduleName); + } + + Collection collectAllBenchSpecs() { try (Stream stream = Files.walk(rootDir.toPath())) { return stream .map(Path::toFile) @@ -64,22 +73,13 @@ public Collection collectAllBenchSpecs() { } } - public BenchSuiteWrapper collectBenchSpecFromModuleName(String moduleName) { - Objects.requireNonNull(moduleName); - if (!moduleName.startsWith(benchPackagePrefix)) { - throw new IllegalArgumentException("Module name must start with " + benchPackagePrefix); - } - Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); - return collectBenchSpecFromModule(module, moduleName); - } - /** * Collects benchmark specifications from a single file. Returns null if no specification * is found in the file. * @param benchFile Path to the source file. * @return null if there are no benchmark specs in the file. */ - private BenchSuiteWrapper collectBenchSpecsFromSingleFile(File benchFile) { + private ModuleBenchSuite collectBenchSpecsFromSingleFile(File benchFile) { Path relativePath = rootDir.toPath().relativize(benchFile.toPath()); // Strip the "src" directory Path modulePath = relativePath.subpath(1, relativePath.getNameCount()); @@ -89,7 +89,7 @@ private BenchSuiteWrapper collectBenchSpecsFromSingleFile(File benchFile) { return collectBenchSpecFromModule(module, moduleFullName); } - private BenchSuiteWrapper collectBenchSpecFromModule(Value module, String moduleQualifiedName) { + private ModuleBenchSuite collectBenchSpecFromModule(Value module, String moduleQualifiedName) { Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, allSuitesVarName); if (!allSuitesVar.isNull()) { @@ -98,7 +98,7 @@ private BenchSuiteWrapper collectBenchSpecFromModule(Value module, String module suite = module .invokeMember(Module.EVAL_EXPRESSION, "all") .as(BenchSuite.class); - return new BenchSuiteWrapper(suite, moduleQualifiedName); + return new ModuleBenchSuite(suite, moduleQualifiedName); } catch (PolyglotException e) { // TODO: Replace with proper logging System.err.println("WARN: An exception occured while evaluating benchmark suite var: " + e.getMessage()); From f6bdc676c1035167a408c7c1420bc5947132ca53 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 14:41:09 +0200 Subject: [PATCH 45/71] Move bench-processor from std-bits to lib/scala/bench-processor --- build.sbt | 2 +- .../src/main/java/org/enso/benchmarks/BenchConfig.java | 0 .../src/main/java/org/enso/benchmarks/BenchGroup.java | 0 .../src/main/java/org/enso/benchmarks/BenchSpec.java | 0 .../src/main/java/org/enso/benchmarks/BenchSuite.java | 0 .../src/main/java/org/enso/benchmarks/ModuleBenchSuite.java | 0 .../main/java/org/enso/benchmarks/processor/BenchProcessor.java | 0 .../org/enso/benchmarks/processor/GenerateBenchSources.java | 0 .../main/java/org/enso/benchmarks/processor/SpecCollector.java | 0 9 files changed, 1 insertion(+), 1 deletion(-) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java (100%) rename {std-bits => lib/scala}/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java (100%) diff --git a/build.sbt b/build.sbt index 684982cd1c9e..2c1588c7217c 100644 --- a/build.sbt +++ b/build.sbt @@ -1769,7 +1769,7 @@ lazy val `distribution-manager` = project .dependsOn(pkg) .dependsOn(`logging-utils`) -lazy val `bench-processor` = (project in file("std-bits/bench-processor")) +lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= Seq( diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchGroup.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSpec.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchSuite.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/ModuleBenchSuite.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java diff --git a/std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java similarity index 100% rename from std-bits/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java rename to lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java From 5001012a7e43b2307a94682c9b5490781d49b189 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 15:00:47 +0200 Subject: [PATCH 46/71] Add Benchmark runner code to bench-processor --- build.sbt | 8 +- .../enso/benchmarks/runner/BenchRunner.java | 100 ++++++++++++++++++ .../enso/benchmarks/runner/BenchmarkItem.java | 35 ++++++ .../runner/BenchmarkResultProcessor.java | 21 ++++ .../org/enso/benchmarks/runner/Report.java | 98 +++++++++++++++++ .../enso/benchmarks/runner/ReportItem.java | 62 +++++++++++ .../org/enso/distribution/LanguageHome.scala | 2 +- .../enso/benchmarks/libs/LibBenchRunner.java | 46 +------- 8 files changed, 325 insertions(+), 47 deletions(-) create mode 100644 lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java create mode 100644 lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java create mode 100644 lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java create mode 100644 lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java create mode 100644 lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java diff --git a/build.sbt b/build.sbt index 2c1588c7217c..e360b8a6386c 100644 --- a/build.sbt +++ b/build.sbt @@ -1773,8 +1773,12 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= Seq( - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided" + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion % "provided", + "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion % "provided", + "org.openjdk.jmh" % "jmh-core" % jmhVersion % "provided", + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % "provided", + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided" ), Compile / javacOptions := ((Compile / javacOptions).value ++ // Only run ServiceProvider processor and ignore those defined in META-INF, thus diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java new file mode 100644 index 000000000000..25fbc3634c85 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java @@ -0,0 +1,100 @@ +package org.enso.benchmarks.runner; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; +import org.openjdk.jmh.results.RunResult; +import org.openjdk.jmh.runner.BenchmarkList; +import org.openjdk.jmh.runner.BenchmarkListEntry; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.RunnerException; +import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; +import org.openjdk.jmh.runner.options.CommandLineOptionException; +import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import jakarta.xml.bind.JAXBException; + +public class BenchRunner { + public static final File REPORT_FILE = new File("./bench-report.xml"); + + /** @return A list of qualified names of all benchmarks visible to JMH. */ + public List getAvailable() { + return BenchmarkList.defaultList().getAll(null, new ArrayList<>()).stream() + .map(BenchmarkListEntry::getUsername) + .collect(Collectors.toList()); + } + + public static void run(String[] args) { + CommandLineOptions cmdOpts = null; + try { + cmdOpts = new CommandLineOptions(args); + } catch (CommandLineOptionException e) { + System.err.println("Error parsing command line args:"); + System.err.println(" " + e.getMessage()); + System.exit(1); + } + Runner jmhRunner = new Runner(cmdOpts); + + if (cmdOpts.shouldHelp()) { + System.err.println("Enso benchmark runner: A modified JMH runner for Enso benchmarks."); + try { + cmdOpts.showHelp(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + System.exit(0); + } + + if (cmdOpts.shouldList()) { + jmhRunner.list(); + System.exit(0); + } + + Collection results; + try { + results = jmhRunner.run(); + } catch (RunnerException e) { + throw new RuntimeException(e); + } + + // TODO[pm]: Process the results in the same way as we do in `runtime/bench`. + System.out.println("Results:"); + for (RunResult result : results) { + System.out.println(result.toString()); + } + } + + // TODO: Accept additional arguments + public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { + ChainedOptionsBuilder builder = new OptionsBuilder() + .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") + .include("^" + label + "$"); + + if (Boolean.getBoolean("bench.compileOnly")) { + builder + .measurementIterations(1) + .warmupIterations(0) + .forks(0); + } + + Options benchmarkOptions = builder.build(); + RunResult benchmarksResult = new Runner(benchmarkOptions).runSingle(); + + Report report; + if (REPORT_FILE.exists()) { + report = Report.readFromFile(REPORT_FILE); + } else { + report = new Report(); + } + + BenchmarkItem benchItem = + new BenchmarkResultProcessor().processResult(label, report, benchmarksResult); + + Report.writeToFile(report, REPORT_FILE); + return benchItem; + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java new file mode 100644 index 000000000000..eaa022a624ed --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkItem.java @@ -0,0 +1,35 @@ +package org.enso.benchmarks.runner; + +import org.openjdk.jmh.results.Result; + +/** Convenience class for clients to compare historic results with the last JMH run. */ +public class BenchmarkItem { + private final Result result; + private final ReportItem previousResults; + + public BenchmarkItem(Result result, ReportItem previousResults) { + this.result = result; + this.previousResults = previousResults; + } + + public Result getResult() { + return result; + } + + public ReportItem getPreviousResults() { + return previousResults; + } + + /** @return Best historic score for the given benchmark (including current run). */ + public double getBestScore() { + return previousResults.getBestScore().orElse(result.getScore()); + } + + public double getScore() { + return result.getScore(); + } + + public String getLabel() { + return result.getLabel(); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java new file mode 100644 index 000000000000..5e3613643683 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchmarkResultProcessor.java @@ -0,0 +1,21 @@ +package org.enso.benchmarks.runner; + +import org.openjdk.jmh.results.Result; +import org.openjdk.jmh.results.RunResult; + +public class BenchmarkResultProcessor { + /** + * Matches the new result with historic results from the report and updates the report. + * + * @param label The name by which this result should be referred to as in the result. + * @param report Historic runs report. + * @param result Fresh JMH benchmark result. + * @return + */ + public BenchmarkItem processResult(String label, Report report, RunResult result) { + Result primary = result.getPrimaryResult(); + ReportItem item = report.findOrCreateByLabel(label); + item.addScore(primary.getScore()); + return new BenchmarkItem(primary, item); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java new file mode 100644 index 000000000000..aa4e39c5369c --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/Report.java @@ -0,0 +1,98 @@ +package org.enso.benchmarks.runner; + +import java.io.File; +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import jakarta.xml.bind.JAXBContext; +import jakarta.xml.bind.JAXBException; +import jakarta.xml.bind.Marshaller; +import jakarta.xml.bind.Unmarshaller; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; + +/** Historic runs report. Supports XML serialization. */ +@XmlRootElement +public class Report { + + private List tests; + + public Report() { + tests = new ArrayList<>(); + } + + public Report(List tests) { + this.tests = tests; + } + + @XmlElementWrapper(name = "cases") + @XmlElement(name = "case") + public List getTests() { + return tests; + } + + public void setTests(List tests) { + this.tests = tests; + } + + /** + * Finds a historic result by label. + * + * @param label name of the result to find. + * @return Result for the given label, if found. + */ + public Optional findByLabel(String label) { + return getTests().stream().filter(item -> item.getLabel().equals(label)).findFirst(); + } + + /** + * Inserts a new report item for a given label. + * + * @param label name for which to allocate a report slot. + * @return Item allocated for the label. + */ + public ReportItem createByLabel(String label) { + ReportItem newItem = new ReportItem(label, new ArrayList<>()); + getTests().add(newItem); + return newItem; + } + + /** + * Finds or creates a new report item for a given label. + * + * @param label name for which an item is needed. + * @return The report item for the label, guaranteed to be present in this Report. + */ + public ReportItem findOrCreateByLabel(String label) { + return findByLabel(label).orElseGet(() -> createByLabel(label)); + } + + /** + * Reads a Report from XML file. + * + * @param file the file from which to read the report. + * @return the Report read from given file. + * @throws JAXBException when the file cannot be read or does not conform to the Report XML + * format. + */ + public static Report readFromFile(File file) throws JAXBException { + JAXBContext jaxbContext = JAXBContext.newInstance(Report.class); + Unmarshaller unmarshaller = jaxbContext.createUnmarshaller(); + return (Report) unmarshaller.unmarshal(file); + } + + /** + * Serializes a report to an XML file. + * + * @param report Report to serialize. + * @param file File to which the serialized report should be written. + * @throws JAXBException when the file cannot be written to. + */ + public static void writeToFile(Report report, File file) throws JAXBException { + JAXBContext jaxbContext = JAXBContext.newInstance(Report.class); + Marshaller marshaller = jaxbContext.createMarshaller(); + marshaller.setProperty(Marshaller.JAXB_FORMATTED_OUTPUT, true); + marshaller.marshal(report, file); + } +} diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java new file mode 100644 index 000000000000..4de292505332 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/ReportItem.java @@ -0,0 +1,62 @@ +package org.enso.benchmarks.runner; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; +import java.util.OptionalDouble; +import jakarta.xml.bind.annotation.XmlElement; +import jakarta.xml.bind.annotation.XmlElementWrapper; +import jakarta.xml.bind.annotation.XmlRootElement; +import jakarta.xml.bind.annotation.XmlTransient; + +/** Contains historic results for a single benchmark identified by label. */ +@XmlRootElement +public class ReportItem { + + private String label; + + private List scores; + + public ReportItem() {} + + public ReportItem(String label, List scores) { + this.label = label; + this.scores = scores; + } + + @XmlElement + public String getLabel() { + return label; + } + + public void setLabel(String label) { + this.label = label; + } + + @XmlElementWrapper(name = "scores") + @XmlElement(name = "score") + public List getScores() { + return scores; + } + + public void setScores(List scores) { + if (scores == null) scores = new ArrayList<>(); + this.scores = scores; + } + + /** + * Registers a new score for this item. + * + * @param score Score to register. + */ + public void addScore(double score) { + getScores().add(score); + } + + /** @return The best (lowest) historic result for this benchmark. */ + @XmlTransient + public Optional getBestScore() { + OptionalDouble min = getScores().stream().mapToDouble(s -> s).min(); + return min.isPresent() ? Optional.of(min.getAsDouble()) : Optional.empty(); + } +} diff --git a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala index 09355fa7ba1f..10edaed6ee84 100644 --- a/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala +++ b/lib/scala/distribution-manager/src/main/scala/org/enso/distribution/LanguageHome.scala @@ -7,7 +7,7 @@ import java.nio.file.Path * * @param languageHome the path to the directory containing the runner.jar and * runtime.jar of the currently running language runtime. - * The path does not have to exist. + * The path does not have to exist. */ case class LanguageHome(languageHome: Path) { private val rootPath = languageHome.getParent.toAbsolutePath.normalize diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index 9fb2e9cbea7d..1f4cea8eb258 100644 --- a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -1,54 +1,12 @@ package org.enso.benchmarks.libs; -import java.io.IOException; -import java.util.Collection; import org.enso.benchmarks.processor.GenerateBenchSources; -import org.openjdk.jmh.results.RunResult; -import org.openjdk.jmh.runner.Runner; -import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.CommandLineOptionException; -import org.openjdk.jmh.runner.options.CommandLineOptions; +import org.enso.benchmarks.runner.BenchRunner; @GenerateBenchSources public class LibBenchRunner { public static void main(String[] args) { - CommandLineOptions cmdOpts = null; - try { - cmdOpts = new CommandLineOptions(args); - } catch (CommandLineOptionException e) { - System.err.println("Error parsing command line args:"); - System.err.println(" " + e.getMessage()); - System.exit(1); - } - Runner jmhRunner = new Runner(cmdOpts); - - if (cmdOpts.shouldHelp()) { - System.err.println("Enso libs benchmark runner: A modified JMH runner for Enso benchmarks."); - try { - cmdOpts.showHelp(); - } catch (IOException e) { - throw new IllegalStateException("Unreachable", e); - } - System.exit(0); - } - - if (cmdOpts.shouldList()) { - jmhRunner.list(); - System.exit(0); - } - - Collection results; - try { - results = jmhRunner.run(); - } catch (RunnerException e) { - throw new RuntimeException(e); - } - - // TODO[pm]: Process the results in the same way as we do in `runtime/bench`. - System.out.println("Results:"); - for (RunResult result : results) { - System.out.println(result.toString()); - } + BenchRunner.run(args); } } From 882ae8ab3df0f421ffc19157ead60c13b7609ffa Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 15:06:40 +0200 Subject: [PATCH 47/71] Delete unused benchmark code --- .../benchmarks/meso/VectorOperations.java | 139 ------------------ .../Vector_Operations.enso | 46 ------ .../simple.enso | 19 --- 3 files changed, 204 deletions(-) delete mode 100644 engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java delete mode 100644 engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso delete mode 100644 engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java deleted file mode 100644 index 7a02205c76ba..000000000000 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/benchmarks/meso/VectorOperations.java +++ /dev/null @@ -1,139 +0,0 @@ -package org.enso.interpreter.bench.benchmarks.meso; - -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.List; -import java.util.concurrent.TimeUnit; -import java.util.function.Predicate; -import java.util.stream.Collectors; -import org.enso.polyglot.MethodNames; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.Source; -import org.graalvm.polyglot.Value; -import org.openjdk.jmh.annotations.Benchmark; -import org.openjdk.jmh.annotations.BenchmarkMode; -import org.openjdk.jmh.annotations.Fork; -import org.openjdk.jmh.annotations.Measurement; -import org.openjdk.jmh.annotations.Mode; -import org.openjdk.jmh.annotations.OutputTimeUnit; -import org.openjdk.jmh.annotations.Scope; -import org.openjdk.jmh.annotations.Setup; -import org.openjdk.jmh.annotations.State; -import org.openjdk.jmh.annotations.Warmup; -import org.openjdk.jmh.infra.BenchmarkParams; -import org.openjdk.jmh.infra.Blackhole; - -@BenchmarkMode(Mode.AverageTime) -@Fork(1) -@Warmup(iterations = 3, time=3) -@Measurement(iterations = 3, time=1) -@OutputTimeUnit(TimeUnit.MILLISECONDS) -@State(Scope.Benchmark) -public class VectorOperations { - private Value code; - - @Setup - public void initializeBenchmark(BenchmarkParams params) throws Exception { - File sourceFile; - for (var folder = Paths.get("../../distribution/component").toFile();; folder = folder.getParentFile()) { - sourceFile = fileFrom(folder, "test", "Benchmarks", "src", "Vector", "Operations.enso"); - if (sourceFile.canRead()) { - break; - } - } - sourceFile = sourceFile.getCanonicalFile(); - var projectRoot = sourceFile.getParentFile().getParentFile().getParent(); - - var ctx = Context.newBuilder() - .allowExperimentalOptions(true) - .allowIO(true) - .allowAllAccess(true) - .logHandler(new ByteArrayOutputStream()) - .option("enso.projectRoot", projectRoot).option( - "enso.languageHomeOverride", - Paths.get("../../distribution/component").toFile().getAbsolutePath() - ).build(); - - var src = Source.newBuilder("enso", sourceFile).build(); - var module = ctx.eval(src); - var all = module.invokeMember(MethodNames.Module.EVAL_EXPRESSION, "bench_suites").as(All.class); - var allSuites = new ArrayList(); - for (var g : all.groups()) { - for (var s : g.specs()) { - allSuites.add(s); - } - } - - var shortName = params.getBenchmark().replaceAll(".*\\.", ""); - - Predicate matches = (s) -> { - var name = s.name().replaceAll(" ", ""); - return shortName.equalsIgnoreCase(name); - }; - - var firstOption = allSuites.stream().filter(matches).findFirst(); - if (firstOption.isEmpty()) { - var names = allSuites.stream().map(Suite::name).collect(Collectors.toList()); - throw new IllegalStateException("Cannot find suite: " + shortName + " among: " + names); - } - var found = firstOption.get(); - if (found.code() == null) { - throw new NullPointerException("no code member: " + found); - } - this.code = found.code(); - } - - public static interface All { - List groups(); - } - - public static interface Group { - String name(); - List specs(); - } - - public static interface Suite { - String name(); - Value code(); - } - - private static File fileFrom(File folder, String... children) { - File f = folder; - for (int i = 0; i < children.length; i++) { - f = new File(f, children[i]); - } - return f; - } - - @Benchmark - public void newVector(Blackhole blackhole) { - benchmark(blackhole); - } - - @Benchmark - public void newConstant(Blackhole blackhole) { - benchmark(blackhole); - } - - @Benchmark - public void newRandom(Blackhole blackhole) { - benchmark(blackhole); - } - - @Benchmark - public void sum(Blackhole blackhole) { - benchmark(blackhole); - } - - @Benchmark - public void sumStatistic(Blackhole blackhole) { - benchmark(blackhole); - } - - protected final void benchmark(Blackhole blackhole) { - var result = code.execute(0); - blackhole.consume(result); - } -} diff --git a/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso b/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso deleted file mode 100644 index bfe31103f89c..000000000000 --- a/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/Vector_Operations.enso +++ /dev/null @@ -1,46 +0,0 @@ -from Standard.Base import all -import Standard.Base.Runtime.State -import Standard.Base - -from Standard.Test import Bench - -import project.Vector.Utils - -polyglot java import java.util.Random as Java_Random - -## Bench Utilities ============================================================ - -vector_size = 1000000 -iter_size = 100 -num_iterations = 10 - -# The Benchmarks ============================================================== - -bench = - random_vec = Utils.make_random_vec vector_size - random_vec_2 = Utils.make_random_vec 100000 - random_gen = Java_Random.new 123456 - - Bench.measure (Base.Vector.new vector_size i->i) "New Vector" iter_size num_iterations - Bench.measure (Base.Vector.new vector_size _->42) "New Constant" iter_size num_iterations - Bench.measure (Base.Vector.new vector_size _->random_gen.nextLong) "New Random" iter_size num_iterations - Bench.measure (Base.Vector.fill vector_size 42) "Fill Constant" iter_size num_iterations - Bench.measure (Base.Vector.fill vector_size random_gen.nextLong) "Fill Random (constant)" iter_size num_iterations - Bench.measure (random_vec + [1]) "Append Single" iter_size num_iterations - Bench.measure (random_vec + random_vec_2) "Append Large" iter_size num_iterations - Bench.measure (random_vec.reduce (+)) "Sum" iter_size num_iterations - Bench.measure ((random_vec.drop (First 20)).reduce (+)) "Drop First 20 and Sum" iter_size num_iterations - Bench.measure ((random_vec.drop (Last 20)).reduce (+)) "Drop Last 20 and Sum" iter_size num_iterations - Bench.measure (random_vec.filter (x -> x % 3 == 1)) "Filter" iter_size num_iterations - Bench.measure (random_vec.filter_with_index (i-> x-> (i+x) % 3 == 1)) "Filter With Index" iter_size num_iterations - - Bench.measure (random_vec . map (x -> x + random_gen.nextLong) . filter (x -> x % 3 == 1)) "Map & Filter" iter_size num_iterations - Bench.measure (random_vec.partition (x -> x % 3 == 1)) "Partition" iter_size num_iterations - Bench.measure (random_vec.partition_with_index (i-> x-> (i+x) % 3 == 1)) "Partition With Index" iter_size num_iterations - - stateful_fun x = - s = State.get Number - State.put s+x - Bench.measure (State.run Number 0 <| random_vec.each stateful_fun) "Each" iter_size num_iterations - -main = bench diff --git a/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso b/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso deleted file mode 100644 index 0241465d964e..000000000000 --- a/engine/runtime/src/bench/resources/org.enso.interpreter.bench.benchmarks.meso/simple.enso +++ /dev/null @@ -1,19 +0,0 @@ -from Standard.Base import all -import Standard.Test.Bench.Bench - -options = Bench.options . size 10 . iter 5 - -vec = [1,2,3,4,5] - -all = Bench.build builder-> - builder.group "group_1" options group_builder-> - group_builder.specify "bench_1" <| - vec + [10] - builder.group "group_2" options group_builder-> - group_builder.specify "bench_2" <| - vec + [20] - -main = - all.run_main - - From 53373b7182a0846614cb4e2716036e49947d3563 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 15:21:55 +0200 Subject: [PATCH 48/71] Distinct.enso benchmark uses the new bench builder API --- test/Benchmarks/src/Vector/Distinct.enso | 36 +++++++++++------------- 1 file changed, 16 insertions(+), 20 deletions(-) diff --git a/test/Benchmarks/src/Vector/Distinct.enso b/test/Benchmarks/src/Vector/Distinct.enso index 0fc5a4e91c87..37bc755c3f25 100644 --- a/test/Benchmarks/src/Vector/Distinct.enso +++ b/test/Benchmarks/src/Vector/Distinct.enso @@ -1,31 +1,27 @@ from Standard.Base import all -import Standard.Base from Standard.Test import Bench import project.Vector.Utils -polyglot java import org.enso.base.Time_Utils +options = Bench.options . size 100 . iter 20 -## Bench Utilities ============================================================ +random_vec = Utils.make_random_vec 100000 +uniform_vec = Vector.fill 100000 1 +random_text_vec = random_vec.map .to_text +uniform_text_vec = random_vec.map .to_text -iter_size = 100 -num_iterations = 20 +collect_benches = Bench.build builder-> + builder.group "Vector Distinct" options group_builder-> + group_builder.specify "Random Integer Vector Distinct" <| + random_vec.distinct + group_builder.specify "Uniform Integer Vector Distinct" <| + uniform_vec.distinct + group_builder.specify "Random Text Vector Distinct" <| + random_text_vec.distinct + group_builder.specify "Uniform Text Vector Distinct" <| + uniform_text_vec.distinct -# The Benchmarks ============================================================== -bench = - random_vec = Utils.make_random_vec 100000 - uniform_vec = Base.Vector.fill 100000 1 - - random_text_vec = random_vec.map .to_text - uniform_text_vec = random_vec.map .to_text - - Bench.measure (random_vec.distinct) "Random Integer Vector Distinct" iter_size num_iterations - Bench.measure (uniform_vec.distinct) "Uniform Integer Vector Distinct" iter_size num_iterations - - Bench.measure (random_text_vec.distinct) "Random Text Vector Distinct" iter_size num_iterations - Bench.measure (uniform_text_vec.distinct) "Uniform Text Vector Distinct" iter_size num_iterations - -main = bench +main = collect_benches . run_main From 81353d24f0b7258187a6d67a5592c3a0d3ee5b32 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 15:27:39 +0200 Subject: [PATCH 49/71] Main module of test/Benchmarks collects all bench specs --- test/Benchmarks/src/Main.enso | 37 ++++++++--------------------------- 1 file changed, 8 insertions(+), 29 deletions(-) diff --git a/test/Benchmarks/src/Main.enso b/test/Benchmarks/src/Main.enso index daf1b98308a4..d6b93b834834 100644 --- a/test/Benchmarks/src/Main.enso +++ b/test/Benchmarks/src/Main.enso @@ -1,32 +1,11 @@ -import project.Collections -import project.Column_Numeric -import project.Equality -import project.Json_Bench -import project.Natural_Order_Sort -import project.Number_Parse -import project.Numeric -import project.Sum +import project.Vector.Distinct +import project.Vector.Operations -import project.Statistics -import project.Table -import project.Text -import project.Time -import project.Vector +collect_all = + vec_ops = Operations.all + vec_distinct = Distinct.collect_benches + [vec_ops, vec_distinct] -## A common entry point for all benchmarks. Usually just one suite is run, but - sometimes we want to run all benchmarks to ensure our language changes are - not breaking them. main = - Collections.bench - Column_Numeric.bench - Equality.bench - Json_Bench.bench - Natural_Order_Sort.bench - Number_Parse.bench - Numeric.bench - Sum.bench - Statistics.Main.bench - Table.Main.bench - Text.Main.bench - Time.Main.bench - Vector.Main.bench + collect_all.each suite-> + suite.run_main From 1d389d915e7a114376c2e48e57a28b0c28fa8172 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 18:53:46 +0200 Subject: [PATCH 50/71] Fix jakarta.xml dependency --- build.sbt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/build.sbt b/build.sbt index e360b8a6386c..b8288d22e02c 100644 --- a/build.sbt +++ b/build.sbt @@ -1773,8 +1773,8 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= Seq( - "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion % "provided", - "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion % "provided", + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, + "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, "org.openjdk.jmh" % "jmh-core" % jmhVersion % "provided", "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % "provided", "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", From c19b53cf6c5b2a1de23fcd89e2c63212debfb374 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Fri, 28 Jul 2023 19:07:20 +0200 Subject: [PATCH 51/71] Add module-discovery parameters to GenerateBenchSources annotation --- .../processor/GenerateBenchSources.java | 20 +++++++++++++++++-- .../enso/benchmarks/libs/LibBenchRunner.java | 5 ++++- test/Benchmarks/src/Main.enso | 4 ++-- 3 files changed, 24 insertions(+), 5 deletions(-) diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java index b086b3ab529f..10a1667f354f 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java @@ -6,10 +6,26 @@ import java.lang.annotation.Target; /** - * Just a dummy annotation to force the {@link BenchProcessor} to generate JMH sources - * for all the discovered Enso benchmarks. + * Use this annotation to force the {@link BenchProcessor} to generate JMH sources + * for all the collected Enso benchmarks. The location of the benchmarks is encoded + * by the {@code projectRootPath}, {@code moduleName} and {@code variableName} parameters. */ @Retention(RetentionPolicy.CLASS) @Target(ElementType.TYPE) public @interface GenerateBenchSources { + + /** + * Path to the project root directory. + */ + String projectRootPath(); + + /** + * Fully qualified name of the module within the project that defines all the benchmark {@link org.enso.benchmarks.BenchSuite suites}. + */ + String moduleName(); + + /** + * Name of the variable that holds a list of all the benchmark {@link org.enso.benchmarks.BenchSuite suites}. + */ + String variableName() default "all_benchmarks"; } diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index 1f4cea8eb258..a7caa746ed1d 100644 --- a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -3,7 +3,10 @@ import org.enso.benchmarks.processor.GenerateBenchSources; import org.enso.benchmarks.runner.BenchRunner; -@GenerateBenchSources +@GenerateBenchSources( + projectRootPath = "test/Benchmarks", + moduleName = "local.Benchmarks.Main" +) public class LibBenchRunner { public static void main(String[] args) { diff --git a/test/Benchmarks/src/Main.enso b/test/Benchmarks/src/Main.enso index d6b93b834834..d3cc7418f6ba 100644 --- a/test/Benchmarks/src/Main.enso +++ b/test/Benchmarks/src/Main.enso @@ -1,11 +1,11 @@ import project.Vector.Distinct import project.Vector.Operations -collect_all = +all_benchmarks = vec_ops = Operations.all vec_distinct = Distinct.collect_benches [vec_ops, vec_distinct] main = - collect_all.each suite-> + all_benchmarks.each suite-> suite.run_main From c5ec01e25edcc49be2d9e382c7220396ec1098b1 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 1 Aug 2023 11:02:05 +0200 Subject: [PATCH 52/71] Add tests for SpecCollector --- build.sbt | 28 ++++-- .../enso/benchmarks/TestSpecCollector.java | 90 +++++++++++++++++++ .../resources/Test_Collector/package.yaml | 6 ++ .../resources/Test_Collector/src/Main.enso | 11 +++ 4 files changed, 128 insertions(+), 7 deletions(-) create mode 100644 lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java create mode 100644 lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml create mode 100644 lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso diff --git a/build.sbt b/build.sbt index b8288d22e02c..aac4aa96d9cc 100644 --- a/build.sbt +++ b/build.sbt @@ -1773,12 +1773,15 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= Seq( - "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, - "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, - "org.openjdk.jmh" % "jmh-core" % jmhVersion % "provided", - "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % "provided", - "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", - "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided" + "jakarta.xml.bind" % "jakarta.xml.bind-api" % jaxbVersion, + "com.sun.xml.bind" % "jaxb-impl" % jaxbVersion, + "org.openjdk.jmh" % "jmh-core" % jmhVersion % "provided", + "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % "provided", + "org.netbeans.api" % "org-openide-util-lookup" % netbeansApiVersion % "provided", + "org.graalvm.sdk" % "graal-sdk" % graalMavenPackagesVersion % "provided", + "junit" % "junit" % junitVersion % Test, + "com.github.sbt" % "junit-interface" % junitIfVersion % Test, + "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Test ), Compile / javacOptions := ((Compile / javacOptions).value ++ // Only run ServiceProvider processor and ignore those defined in META-INF, thus @@ -1786,7 +1789,18 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) Seq( "-processor", "org.netbeans.modules.openide.util.ServiceProviderProcessor" - )) + )), + commands += WithDebugCommand.withDebug, + (Test / fork) := true, + (Test / parallelExecution) := false, + (Test / javaOptions) ++= { + val runtimeJars = + (LocalProject("runtime") / Compile / fullClasspath).value + val jarsStr = runtimeJars.map(_.data).mkString(File.pathSeparator) + Seq( + s"-Dtruffle.class.path.append=${jarsStr}" + ) + } ) .dependsOn(`polyglot-api`) .dependsOn(runtime) diff --git a/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java b/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java new file mode 100644 index 000000000000..59ac0c02250f --- /dev/null +++ b/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java @@ -0,0 +1,90 @@ +package org.enso.benchmarks; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +import java.io.File; +import java.net.URISyntaxException; +import java.util.List; +import org.enso.benchmarks.processor.SpecCollector; +import org.enso.pkg.PackageManager; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames.TopScope; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +public class TestSpecCollector { + + private Context ctx; + private File ensoDir; + private File ensoHomeOverride; + private Value testCollectorMainModule; + + @Before + public void setup() { + try { + ensoDir = + new File( + TestSpecCollector.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + } catch (URISyntaxException e) { + fail("ensoDir not found: " + e.getMessage()); + } + for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { + if (ensoDir.getName().equals("enso")) { + break; + } + } + assertNotNull("Could not find Enso root directory", ensoDir); + assertTrue(ensoDir.exists()); + assertTrue(ensoDir.isDirectory()); + assertTrue(ensoDir.canRead()); + + // Note that ensoHomeOverride does not have to exist, only its parent directory + ensoHomeOverride = ensoDir.toPath().resolve("distribution").resolve("component").toFile(); + File testCollectorPath = + new File(getClass().getClassLoader().getResource("Test_Collector").getPath()); + var pkg = PackageManager.Default().fromDirectory(testCollectorPath).get(); + var mainModuleName = pkg.moduleNameForFile(pkg.mainFile()); + ctx = + Context.newBuilder(LanguageInfo.ID) + .allowAllAccess(true) + .allowIO(IOAccess.ALL) + .option(RuntimeOptions.LOG_LEVEL, "WARNING") + .option(RuntimeOptions.PROJECT_ROOT, testCollectorPath.getAbsolutePath()) + .option(RuntimeOptions.LANGUAGE_HOME_OVERRIDE, ensoHomeOverride.getAbsolutePath()) + .build(); + testCollectorMainModule = + ctx.getBindings(LanguageInfo.ID) + .invokeMember(TopScope.GET_MODULE, mainModuleName.toString()); + } + + @After + public void tearDown() { + ctx.close(); + } + + @Test + public void testCollectAllSuitesFromMainModule() { + List moduleBenchSuites = + SpecCollector.collectBenchSpecsFromModule(testCollectorMainModule, "group_1"); + for (ModuleBenchSuite moduleBenchSuite : moduleBenchSuites) { + assertEquals(1, moduleBenchSuite.getGroups().size()); + assertEquals("Test Group", moduleBenchSuite.getGroups().get(0).name()); + List specs = moduleBenchSuite.getGroups().get(0).specs(); + assertEquals(1, specs.size()); + assertEquals("Test Spec", specs.get(0).name()); + Value code = specs.get(0).code(); + Value res = code.execute(Value.asValue(null)); + assertEquals(2, res.asInt()); + } + assertFalse(moduleBenchSuites.isEmpty()); + } +} diff --git a/lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml b/lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml new file mode 100644 index 000000000000..5ac425716bcd --- /dev/null +++ b/lib/scala/bench-processor/src/test/resources/Test_Collector/package.yaml @@ -0,0 +1,6 @@ +name: Test_Collector +license: APLv2 +enso-version: default +version: "0.0.1" +author: "Enso Team " +maintainer: "Enso Team " diff --git a/lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso b/lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso new file mode 100644 index 000000000000..1f0c3f322c32 --- /dev/null +++ b/lib/scala/bench-processor/src/test/resources/Test_Collector/src/Main.enso @@ -0,0 +1,11 @@ +from Standard.Base import all +from Standard.Test import Bench + +options = Bench.options.size 10 . iter 10 + +collect_benches = Bench.build builder-> + builder.group "Test Group" options group_builder-> + group_builder.specify "Test Spec" (1 + 1) + +group_1 = [collect_benches] + From 9b34546bc7664d34a26751431c414f4cf7e65fa3 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 1 Aug 2023 18:21:13 +0200 Subject: [PATCH 53/71] BenchProcessor supports new module-discovery parameters --- .../benchmarks/processor/BenchProcessor.java | 234 +++++++++++------- .../benchmarks/processor/SpecCollector.java | 126 +++------- 2 files changed, 179 insertions(+), 181 deletions(-) diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index d313ac34215a..9eb912e6aa64 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -1,10 +1,10 @@ package org.enso.benchmarks.processor; +import java.io.ByteArrayOutputStream; import java.io.File; import java.io.IOException; import java.io.Writer; import java.net.URISyntaxException; -import java.util.Collection; import java.util.List; import java.util.Set; import java.util.stream.Collectors; @@ -19,6 +19,13 @@ import org.enso.benchmarks.BenchGroup; import org.enso.benchmarks.BenchSpec; import org.enso.benchmarks.ModuleBenchSuite; +import org.enso.polyglot.LanguageInfo; +import org.enso.polyglot.MethodNames.TopScope; +import org.enso.polyglot.RuntimeOptions; +import org.graalvm.polyglot.Context; +import org.graalvm.polyglot.PolyglotException; +import org.graalvm.polyglot.Value; +import org.graalvm.polyglot.io.IOAccess; import org.openide.util.lookup.ServiceProvider; @SupportedAnnotationTypes("org.enso.benchmarks.processor.GenerateBenchSources") @@ -26,72 +33,59 @@ public class BenchProcessor extends AbstractProcessor { private final File ensoHomeOverride; - private File ensoDir; - private final File benchRootDir; - private final SpecCollector specCollector; - private static final String generatedSourcesPackagePrefix = "org.enso.benchmarks.libs"; - private static final List imports = List.of( - "import java.nio.file.Paths;", - "import java.io.ByteArrayOutputStream;", - "import java.io.File;", - "import org.openjdk.jmh.annotations.Benchmark;", - "import org.openjdk.jmh.annotations.BenchmarkMode;", - "import org.openjdk.jmh.annotations.Mode;", - "import org.openjdk.jmh.annotations.Fork;", - "import org.openjdk.jmh.annotations.Measurement;", - "import org.openjdk.jmh.annotations.OutputTimeUnit;", - "import org.openjdk.jmh.annotations.Setup;", - "import org.openjdk.jmh.annotations.State;", - "import org.openjdk.jmh.annotations.Scope;", - "import org.openjdk.jmh.infra.BenchmarkParams;", - "import org.openjdk.jmh.infra.Blackhole;", - "import org.graalvm.polyglot.Context;", - "import org.graalvm.polyglot.Value;", - "import org.graalvm.polyglot.io.IOAccess;", - "import org.enso.polyglot.LanguageInfo;", - "import org.enso.polyglot.MethodNames;", - "import org.enso.benchmarks.processor.SpecCollector;", - "import org.enso.benchmarks.ModuleBenchSuite;", - "import org.enso.benchmarks.BenchSpec;", - "import org.enso.benchmarks.BenchGroup;" - ); + private final File ensoDir; + private File projectRootDir; + private static final String generatedSourcesPackagePrefix = "org.enso.benchmarks.generated"; + private static final List imports = + List.of( + "import java.nio.file.Paths;", + "import java.io.ByteArrayOutputStream;", + "import java.io.File;", + "import java.util.List;", + "import org.openjdk.jmh.annotations.Benchmark;", + "import org.openjdk.jmh.annotations.BenchmarkMode;", + "import org.openjdk.jmh.annotations.Mode;", + "import org.openjdk.jmh.annotations.Fork;", + "import org.openjdk.jmh.annotations.Measurement;", + "import org.openjdk.jmh.annotations.OutputTimeUnit;", + "import org.openjdk.jmh.annotations.Setup;", + "import org.openjdk.jmh.annotations.State;", + "import org.openjdk.jmh.annotations.Scope;", + "import org.openjdk.jmh.infra.BenchmarkParams;", + "import org.openjdk.jmh.infra.Blackhole;", + "import org.graalvm.polyglot.Context;", + "import org.graalvm.polyglot.Value;", + "import org.graalvm.polyglot.io.IOAccess;", + "import org.enso.polyglot.LanguageInfo;", + "import org.enso.polyglot.MethodNames;", + "import org.enso.polyglot.RuntimeOptions;", + "import org.enso.benchmarks.processor.SpecCollector;", + "import org.enso.benchmarks.ModuleBenchSuite;", + "import org.enso.benchmarks.BenchSpec;", + "import org.enso.benchmarks.BenchGroup;", + "import org.enso.benchmarks.Utils;"); public BenchProcessor() { + File currentDir = null; try { - ensoDir = new File( - BenchProcessor.class - .getProtectionDomain() - .getCodeSource() - .getLocation() - .toURI() - ); + currentDir = + new File( + BenchProcessor.class.getProtectionDomain().getCodeSource().getLocation().toURI()); } catch (URISyntaxException e) { failWithMessage("ensoDir not found: " + e.getMessage()); } - for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { - if (ensoDir.getName().equals("enso")) { + for (; currentDir != null; currentDir = currentDir.getParentFile()) { + if (currentDir.getName().equals("enso")) { break; } } - if (ensoDir == null) { + if (currentDir == null) { failWithMessage("Unreachable: Could not find Enso root directory"); } - - benchRootDir = ensoDir.toPath() - .resolve("test") - .resolve("Benchmarks") - .toFile(); - if (!benchRootDir.isDirectory() || !benchRootDir.canRead()) { - failWithMessage("Unreachable: Could not find Enso benchmarks directory"); - } + ensoDir = currentDir; // Note that ensoHomeOverride does not have to exist, only its parent directory - ensoHomeOverride = ensoDir.toPath() - .resolve("distribution") - .resolve("component") - .toFile(); - specCollector = - new SpecCollector(benchRootDir, ensoHomeOverride); + ensoHomeOverride = ensoDir.toPath().resolve("distribution").resolve("component").toFile(); } @Override @@ -101,54 +95,89 @@ public SourceVersion getSupportedSourceVersion() { @Override public boolean process(Set annotations, RoundEnvironment roundEnv) { - try { - Collection benchSuites = specCollector.collectAllBenchSpecs(); - for (ModuleBenchSuite benchSuite : benchSuites) { - for (BenchGroup group : benchSuite.getGroups()) { - generateClassForGroup(group, benchSuite.getModuleQualifiedName()); + var elements = roundEnv.getElementsAnnotatedWith(GenerateBenchSources.class); + for (var element : elements) { + GenerateBenchSources annotation = element.getAnnotation(GenerateBenchSources.class); + projectRootDir = new File(annotation.projectRootPath()); + if (!projectRootDir.exists() || !projectRootDir.isDirectory() || !projectRootDir.canRead()) { + failWithMessage( + "Project root dir '" + + projectRootDir.getAbsolutePath() + + "' specified in the annotation does not exist or is not readable"); + } + try (var ctx = + Context.newBuilder() + .allowExperimentalOptions(true) + .allowIO(IOAccess.ALL) + .allowAllAccess(true) + .logHandler(new ByteArrayOutputStream()) + .option(RuntimeOptions.PROJECT_ROOT, projectRootDir.getAbsolutePath()) + .option(RuntimeOptions.LANGUAGE_HOME_OVERRIDE, ensoHomeOverride.getAbsolutePath()) + .build()) { + Value module = getModule(ctx, annotation.moduleName()); + assert module != null; + List benchSuites = + SpecCollector.collectBenchSpecsFromModule(module, annotation.variableName()); + for (ModuleBenchSuite benchSuite : benchSuites) { + for (BenchGroup group : benchSuite.getGroups()) { + generateClassForGroup(group, benchSuite.getModuleQualifiedName(), annotation.variableName()); + } } + return true; + } catch (Exception e) { + failWithMessage("Uncaught exception in " + getClass().getName() + ": " + e.getMessage()); + return false; } - return true; - } catch (Exception e) { - failWithMessage("Uncaught exception in " + getClass().getName() + ": " + e.getMessage()); - return false; + } + return true; + } + + private Value getModule(Context ctx, String moduleName) { + try { + return ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); + } catch (PolyglotException e) { + failWithMessage("Cannot get module '" + moduleName + "': " + e.getMessage()); + return null; } } - private void generateClassForGroup(BenchGroup group, String moduleQualifiedName) { + private void generateClassForGroup(BenchGroup group, String moduleQualifiedName, String varName) { String fullClassName = createGroupClassName(group); - try (Writer srcFileWriter = processingEnv.getFiler().createSourceFile(fullClassName).openWriter()) { - generateClassForGroup(srcFileWriter, moduleQualifiedName, group); + try (Writer srcFileWriter = + processingEnv.getFiler().createSourceFile(fullClassName).openWriter()) { + generateClassForGroup(srcFileWriter, moduleQualifiedName, varName, group); } catch (IOException e) { if (!isResourceAlreadyExistsException(e)) { - failWithMessage("Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); + failWithMessage( + "Failed to generate source file for group '" + group.name() + "': " + e.getMessage()); } } } /** - * Returns true iff the given exception is thrown because a file already exists exception. - * There is no better way to check this. + * Returns true iff the given exception is thrown because a file already exists exception. There + * is no better way to check this. + * * @param e Exception to check. * @return true iff the given exception is thrown because a file already exists exception. */ private static boolean isResourceAlreadyExistsException(IOException e) { - List messages = List.of( - "Source file already created", - "Resource already created", - "Attempt to recreate a file" - ); - return e instanceof FilerException && messages.stream().anyMatch(msg -> e.getMessage().contains(msg)); + List messages = + List.of( + "Source file already created", + "Resource already created", + "Attempt to recreate a file"); + return e instanceof FilerException + && messages.stream().anyMatch(msg -> e.getMessage().contains(msg)); } - private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualifiedName, BenchGroup group) throws IOException { + private void generateClassForGroup( + Writer javaSrcFileWriter, String moduleQualifiedName, String varName, BenchGroup group) throws IOException { String groupFullClassName = createGroupClassName(group); String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); List specs = group.specs(); - List specJavaNames = specs - .stream() - .map(spec -> normalize(spec.name())) - .collect(Collectors.toUnmodifiableList()); + List specJavaNames = + specs.stream().map(spec -> normalize(spec.name())).collect(Collectors.toUnmodifiableList()); javaSrcFileWriter.append("package " + generatedSourcesPackagePrefix + ";\n"); javaSrcFileWriter.append("\n"); @@ -172,51 +201,69 @@ private void generateClassForGroup(Writer javaSrcFileWriter, String moduleQualif javaSrcFileWriter.append(" \n"); javaSrcFileWriter.append(" @Setup\n"); javaSrcFileWriter.append(" public void setup(BenchmarkParams params) throws Exception {\n"); - javaSrcFileWriter.append(" File benchProjectDir = new File(\"" + benchRootDir.getAbsolutePath() + "\");\n"); - javaSrcFileWriter.append(" File languageHomeOverride = new File(\"" + ensoHomeOverride.getAbsolutePath() + "\");\n"); + javaSrcFileWriter.append( + " File projectRootDir = new File(\"" + projectRootDir.getAbsolutePath() + "\");\n"); + javaSrcFileWriter.append( + " File languageHomeOverride = new File(\"" + + ensoHomeOverride.getAbsolutePath() + + "\");\n"); javaSrcFileWriter.append(" var ctx = Context.newBuilder()\n"); javaSrcFileWriter.append(" .allowExperimentalOptions(true)\n"); javaSrcFileWriter.append(" .allowIO(IOAccess.ALL)\n"); javaSrcFileWriter.append(" .allowAllAccess(true)\n"); javaSrcFileWriter.append(" .logHandler(new ByteArrayOutputStream())\n"); javaSrcFileWriter.append(" .option(\n"); - javaSrcFileWriter.append(" \"enso.languageHomeOverride\",\n"); - javaSrcFileWriter.append(" Paths.get(\"../../distribution/component\").toFile().getAbsolutePath()\n"); + javaSrcFileWriter.append(" RuntimeOptions.LANGUAGE_HOME_OVERRIDE,\n"); + javaSrcFileWriter.append(" languageHomeOverride.getAbsolutePath()\n"); javaSrcFileWriter.append(" )\n"); javaSrcFileWriter.append(" .option(\n"); - javaSrcFileWriter.append(" \"enso.projectRoot\",\n"); - javaSrcFileWriter.append(" benchProjectDir.getAbsolutePath()\n"); + javaSrcFileWriter.append(" RuntimeOptions.PROJECT_ROOT,\n"); + javaSrcFileWriter.append(" projectRootDir.getAbsolutePath()\n"); javaSrcFileWriter.append(" )\n"); javaSrcFileWriter.append(" .build();\n"); javaSrcFileWriter.append(" \n"); javaSrcFileWriter.append(" Value bindings = ctx.getBindings(LanguageInfo.ID);\n"); - javaSrcFileWriter.append(" Value module = bindings.invokeMember(MethodNames.TopScope.GET_MODULE, \"" + moduleQualifiedName + "\");\n"); - javaSrcFileWriter.append(" var specCollector = new SpecCollector(benchProjectDir, languageHomeOverride);\n"); - javaSrcFileWriter.append(" ModuleBenchSuite benchSuite = specCollector.collectBenchSpecFromModuleName(\"" + moduleQualifiedName + "\");\n"); + javaSrcFileWriter.append( + " Value module = bindings.invokeMember(MethodNames.TopScope.GET_MODULE, \"" + + moduleQualifiedName + + "\");\n"); + javaSrcFileWriter.append( + " BenchGroup group = SpecCollector.collectBenchGroupFromModule(module, \"" + + group.name() + + "\", \"" + varName + "\");\n"); javaSrcFileWriter.append(" \n"); for (int i = 0; i < specs.size(); i++) { var specJavaName = specJavaNames.get(i); var specName = specs.get(i).name(); - javaSrcFileWriter.append(" BenchSpec benchSpec_" + specJavaName + " = benchSuite.findSpecByName(\"" + group.name() + "\", \"" + specName + "\");\n"); - javaSrcFileWriter.append(" this.benchFunc_" + specJavaName + " = benchSpec_" + specJavaName + ".code();\n"); + javaSrcFileWriter.append( + " BenchSpec benchSpec_" + + specJavaName + + " = Utils.findSpecByName(group, \"" + specName + "\");\n"); + javaSrcFileWriter.append( + " this.benchFunc_" + specJavaName + " = benchSpec_" + specJavaName + ".code();\n"); } javaSrcFileWriter.append(" \n"); javaSrcFileWriter.append(" this.groupInputArg = Value.asValue(null);\n"); javaSrcFileWriter.append(" } \n"); // end of setup method javaSrcFileWriter.append(" \n"); + + // Benchmark methods for (var specJavaName : specJavaNames) { javaSrcFileWriter.append(" \n"); javaSrcFileWriter.append(" @Benchmark\n"); javaSrcFileWriter.append(" public void " + specJavaName + "(Blackhole blackhole) {\n"); - javaSrcFileWriter.append(" Value result = this.benchFunc_" + specJavaName + ".execute(this.groupInputArg);\n"); + javaSrcFileWriter.append( + " Value result = this.benchFunc_" + specJavaName + ".execute(this.groupInputArg);\n"); javaSrcFileWriter.append(" blackhole.consume(result);\n"); javaSrcFileWriter.append(" }\n"); // end of benchmark method } + javaSrcFileWriter.append("}\n"); // end of class className } /** * Returns Java FQN for a benchmark spec. + * * @param group Group name will be converted to Java package name. * @return */ @@ -231,6 +278,7 @@ private static boolean isValidChar(char c) { /** * Converts Text to valid Java identifier. + * * @param name Text to convert. * @return Valid Java identifier, non null. */ @@ -239,7 +287,7 @@ private static String normalize(String name) { for (char c : name.toCharArray()) { if (isValidChar(c)) { normalizedNameSb.append(c); - } else if (Character.isWhitespace(c) && (peekLastChar(normalizedNameSb) != '_')) { + } else if (Character.isWhitespace(c) && (peekLastChar(normalizedNameSb) != '_')) { normalizedNameSb.append('_'); } } diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java index b32584732826..7a029850c88a 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java @@ -1,108 +1,58 @@ package org.enso.benchmarks.processor; -import java.io.ByteArrayOutputStream; -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Collection; -import java.util.Objects; -import java.util.stream.Collectors; -import java.util.stream.Stream; +import java.util.ArrayList; +import java.util.List; +import org.enso.benchmarks.BenchGroup; import org.enso.benchmarks.BenchSuite; import org.enso.benchmarks.ModuleBenchSuite; -import org.enso.polyglot.LanguageInfo; import org.enso.polyglot.MethodNames.Module; -import org.enso.polyglot.MethodNames.TopScope; -import org.graalvm.polyglot.Context; -import org.graalvm.polyglot.PolyglotException; import org.graalvm.polyglot.Value; -import org.graalvm.polyglot.io.IOAccess; /** * Collect benchmark specifications from source files. */ public class SpecCollector { + private SpecCollector() {} - private final File rootDir; - private final Context ctx; - private static final String allSuitesVarName = "all"; - private final String benchPackagePrefix = "local.Benchmarks"; - private final String ensoSuffix = ".enso"; - - public SpecCollector(File benchProjectDir, File languageHomeOverride) { - if (!benchProjectDir.exists() || !benchProjectDir.isDirectory()) { - throw new IllegalArgumentException(benchProjectDir + " is not a directory."); - } - this.rootDir = benchProjectDir; - this.ctx = Context.newBuilder() - .allowExperimentalOptions(true) - .allowIO(IOAccess.ALL) - .allowAllAccess(true) - .logHandler(new ByteArrayOutputStream()) - .option( - "enso.projectRoot", - benchProjectDir.getAbsolutePath() - ) - .option( - "enso.languageHomeOverride", - languageHomeOverride.getAbsolutePath() - ) - .build(); - } - - public ModuleBenchSuite collectBenchSpecFromModuleName(String moduleName) { - Objects.requireNonNull(moduleName); - if (!moduleName.startsWith(benchPackagePrefix)) { - throw new IllegalArgumentException("Module name must start with " + benchPackagePrefix); - } - Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleName); - return collectBenchSpecFromModule(module, moduleName); - } - - Collection collectAllBenchSpecs() { - try (Stream stream = Files.walk(rootDir.toPath())) { - return stream - .map(Path::toFile) - .filter(file -> file.isFile() && file.canRead() && file.getName().endsWith(ensoSuffix)) - .map(this::collectBenchSpecsFromSingleFile) - .filter(Objects::nonNull) - .collect(Collectors.toList()); - } catch (IOException e) { - throw new IllegalStateException("Unreachable", e); + /** + * Collects all the bench specifications from the given module in a variable with the given name. + * @param varName Name of the variable that holds all the collected bench suites. + * @return Empty list if no such variable exists, or if it is not a vector. + */ + public static List collectBenchSpecsFromModule(Value module, String varName) { + Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); + Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, varName); + String moduleQualifiedName = module.invokeMember(Module.GET_NAME).asString(); + if (!allSuitesVar.isNull()) { + Value suitesValue = module.invokeMember(Module.EVAL_EXPRESSION, varName); + if (!suitesValue.hasArrayElements()) { + return List.of(); + } + List suites = new ArrayList<>(); + for (long i = 0; i < suitesValue.getArraySize(); i++) { + Value suite = suitesValue.getArrayElement(i); + BenchSuite benchSuite = suite.as(BenchSuite.class); + suites.add( + new ModuleBenchSuite(benchSuite, moduleQualifiedName) + ); + } + return suites; } + return List.of(); } /** - * Collects benchmark specifications from a single file. Returns null if no specification - * is found in the file. - * @param benchFile Path to the source file. - * @return null if there are no benchmark specs in the file. + * Collects all the bench specifications from the given module in a variable with the given name. + * @param groupName Name of the benchmark group + * @param varName Name of the variable that holds all the collected bench suites. + * @return null if no such group exists. */ - private ModuleBenchSuite collectBenchSpecsFromSingleFile(File benchFile) { - Path relativePath = rootDir.toPath().relativize(benchFile.toPath()); - // Strip the "src" directory - Path modulePath = relativePath.subpath(1, relativePath.getNameCount()); - String moduleFullName = benchPackagePrefix + "." + modulePath.toString().replace(File.separatorChar, '.'); - moduleFullName = moduleFullName.substring(0, moduleFullName.length() - ensoSuffix.length()); - Value module = ctx.getBindings(LanguageInfo.ID).invokeMember(TopScope.GET_MODULE, moduleFullName); - return collectBenchSpecFromModule(module, moduleFullName); - } - - private ModuleBenchSuite collectBenchSpecFromModule(Value module, String moduleQualifiedName) { - Value moduleType = module.invokeMember(Module.GET_ASSOCIATED_TYPE); - Value allSuitesVar = module.invokeMember(Module.GET_METHOD, moduleType, allSuitesVarName); - if (!allSuitesVar.isNull()) { - BenchSuite suite; - try { - suite = module - .invokeMember(Module.EVAL_EXPRESSION, "all") - .as(BenchSuite.class); - return new ModuleBenchSuite(suite, moduleQualifiedName); - } catch (PolyglotException e) { - // TODO: Replace with proper logging - System.err.println("WARN: An exception occured while evaluating benchmark suite var: " + e.getMessage()); - return null; + public static BenchGroup collectBenchGroupFromModule(Value module, String groupName, String varName) { + var specs = collectBenchSpecsFromModule(module, varName); + for (ModuleBenchSuite suite : specs) { + BenchGroup group = suite.findGroupByName(groupName); + if (group != null) { + return group; } } return null; From 8bad812ca433f7c173814e78839280ef692c9244 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 1 Aug 2023 18:22:07 +0200 Subject: [PATCH 54/71] Some improvements of bench-libs SBT project. - No explicit mainClass setting needed. - Remove specified excludeMsg log manager --- build.sbt | 19 +++++-------------- 1 file changed, 5 insertions(+), 14 deletions(-) diff --git a/build.sbt b/build.sbt index aac4aa96d9cc..1b120a344fd2 100644 --- a/build.sbt +++ b/build.sbt @@ -1806,9 +1806,9 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .dependsOn(runtime) lazy val `bench-libs` = (project in file("std-bits/benchmarks")) + .configs(Benchmark) .settings( frgaalJavaCompilerSetting, - assembly / mainClass := (Compile / run / mainClass).value, libraryDependencies ++= jmh ++ Seq( "org.openjdk.jmh" % "jmh-core" % jmhVersion % Benchmark, "org.openjdk.jmh" % "jmh-generator-annprocess" % jmhVersion % Benchmark, @@ -1816,9 +1816,6 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark ), commands += WithDebugCommand.withDebug, - (Benchmark / run / mainClass) := Some( - "org.enso.benchmarks.libs.LibBenchRunner" - ), (Benchmark / run / fork) := true, (Benchmark / run / connectInput) := true, // Pass -Dtruffle.class.path.append to javac @@ -1838,16 +1835,11 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) s"-J-Dtruffle.class.path.append=$appendClasspath" ) }, - (Compile / javacOptions) ++= Seq( + (Benchmark / compile / javacOptions) ++= Seq( "-s", - (Compile / sourceManaged).value.getAbsolutePath, + (Benchmark / sourceManaged).value.getAbsolutePath, "-Xlint:unchecked" ), - (Compile / logManager) := - sbt.internal.util.CustomLogManager.excludeMsg( - "Could not determine source for class ", - Level.Warn - ), (Benchmark / run / javaOptions) ++= { val runtimeClasspath = (LocalProject("runtime") / Compile / fullClasspath).value @@ -1864,7 +1856,6 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) ) } ) - .configs(Benchmark) .settings( inConfig(Benchmark)(Defaults.testSettings) ) @@ -1882,8 +1873,8 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) }.evaluated, Benchmark / parallelExecution := false ) - .dependsOn(`bench-processor`) - .dependsOn(runtime) + .dependsOn(`bench-processor` % Benchmark) + .dependsOn(runtime % Benchmark) lazy val editions = project .in(file("lib/scala/editions")) From 2f409fba088a3f4a182a65d070a0765236ecce52 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 1 Aug 2023 18:22:35 +0200 Subject: [PATCH 55/71] Unify report collecting --- .../org/enso/benchmarks/runner/BenchRunner.java | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java index 25fbc3634c85..3f57eecf2a00 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java @@ -61,14 +61,16 @@ public static void run(String[] args) { throw new RuntimeException(e); } - // TODO[pm]: Process the results in the same way as we do in `runtime/bench`. - System.out.println("Results:"); for (RunResult result : results) { - System.out.println(result.toString()); + try { + reportResult(result.getParams().getBenchmark(), result); + } catch (JAXBException e) { + throw new IllegalStateException("Unreachable", e); + } } + System.out.println("Benchmark results reported into " + REPORT_FILE.getAbsolutePath()); } - // TODO: Accept additional arguments public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { ChainedOptionsBuilder builder = new OptionsBuilder() .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") @@ -84,6 +86,9 @@ public static BenchmarkItem runSingle(String label) throws RunnerException, JAXB Options benchmarkOptions = builder.build(); RunResult benchmarksResult = new Runner(benchmarkOptions).runSingle(); + return reportResult(label, benchmarksResult); + } + private static BenchmarkItem reportResult(String label, RunResult result) throws JAXBException { Report report; if (REPORT_FILE.exists()) { report = Report.readFromFile(REPORT_FILE); @@ -92,7 +97,7 @@ public static BenchmarkItem runSingle(String label) throws RunnerException, JAXB } BenchmarkItem benchItem = - new BenchmarkResultProcessor().processResult(label, report, benchmarksResult); + new BenchmarkResultProcessor().processResult(label, report, result); Report.writeToFile(report, REPORT_FILE); return benchItem; From 30bb9f00eb0f2fe7229323dbc0710407ed9be089 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 1 Aug 2023 18:22:42 +0200 Subject: [PATCH 56/71] Add Utils class --- .../main/java/org/enso/benchmarks/Utils.java | 42 +++++++++++++++++++ 1 file changed, 42 insertions(+) create mode 100644 lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java new file mode 100644 index 000000000000..6ffc2c2a87e8 --- /dev/null +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java @@ -0,0 +1,42 @@ +package org.enso.benchmarks; + +import java.io.File; +import java.net.URISyntaxException; + +/** + * Used by the benchmark classes from the generated code + */ +public class Utils { + public static String findLanguageHomeOverride() { + File ensoDir; + try { + ensoDir = + new File( + Utils.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + } catch (URISyntaxException e) { + throw new IllegalStateException("Unrecheable: ensoDir not found", e); + } + for (; ensoDir != null; ensoDir = ensoDir.getParentFile()) { + if (ensoDir.getName().equals("enso")) { + break; + } + } + assert ensoDir != null; + assert ensoDir.exists(); + assert ensoDir.isDirectory(); + assert ensoDir.canRead(); + + // Note that ensoHomeOverride does not have to exist, only its parent directory + File ensoHomeOverride = ensoDir.toPath().resolve("distribution").resolve("component").toFile(); + return ensoHomeOverride.getAbsolutePath(); + } + + public static BenchSpec findSpecByName(BenchGroup group, String specName) { + for (BenchSpec spec : group.specs()) { + if (spec.name().equals(specName)) { + return spec; + } + } + return null; + } +} From d3e6a87ce092e917204b2449cbaee5c82e77b116 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Tue, 1 Aug 2023 18:27:09 +0200 Subject: [PATCH 57/71] Add some tests --- .../test/java/org/enso/benchmarks/TestSpecCollector.java | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java b/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java index 59ac0c02250f..bb2a2c9daf0f 100644 --- a/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java +++ b/lib/scala/bench-processor/src/test/java/org/enso/benchmarks/TestSpecCollector.java @@ -87,4 +87,11 @@ public void testCollectAllSuitesFromMainModule() { } assertFalse(moduleBenchSuites.isEmpty()); } + + @Test + public void testCollectSuitesNonExistendVarName() { + List moduleBenchSuites = + SpecCollector.collectBenchSpecsFromModule(testCollectorMainModule, "NOOOON_existent_FOO"); + assertTrue(moduleBenchSuites.isEmpty()); + } } From 7df60c5346f11f7baaeb55727243bd1290519065 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 10:34:25 +0200 Subject: [PATCH 58/71] No star import in Scala sources in project --- project/FrgaalJavaCompiler.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index aa97e6c58c09..d1645315d5a8 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -9,7 +9,7 @@ * (http://www.apache.org/licenses/LICENSE-2.0). */ -import sbt.* +import sbt._ import sbt.internal.inc.CompilerArguments import sbt.internal.inc.javac.JavacLogger import sbt.io.IO From 6676b8cde22e31eb569c5bd0bdffa63eff6165cf Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 11:04:53 +0200 Subject: [PATCH 59/71] --no-limit-modules is not a hardcoded, but a documentted constant in `FrgaalJavaCompiler`. --- build.sbt | 2 +- project/FrgaalJavaCompiler.scala | 36 +++++++++++++++++++------------- 2 files changed, 22 insertions(+), 16 deletions(-) diff --git a/build.sbt b/build.sbt index 1b120a344fd2..bb91b50ea721 100644 --- a/build.sbt +++ b/build.sbt @@ -1831,7 +1831,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) .map(_.data) .mkString(File.pathSeparator) Seq( - "-J--no-limit-modules", + FrgaalJavaCompiler.noLimitModulesArg, s"-J-Dtruffle.class.path.append=$appendClasspath" ) }, diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index d1645315d5a8..5a4c3a59df88 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -22,13 +22,23 @@ import java.nio.file.{Path, Paths} import scala.sys.process.Process import scala.util.Using import java.io.FileWriter -import scala.collection.mutable.ArrayBuffer object FrgaalJavaCompiler { private val ENSO_SOURCES = ".enso-sources" val frgaal = "org.frgaal" % "compiler" % "19.0.1" % "provided" val sourceLevel = "19" + + /** By default, the Frgaal compiler is invoked with `--limit-modules` argument that limits + * the set of modules that are accessible at compile time to just a small list of internal + * JDK modules. Pass this flag to the compiler in order to disable this behavior. + * This is usable in cases when an annotation processor invoked by Frgaal needs to access + * classes from other modules. + * + * Note that to pass this flag to the compiler, set `javacOptions` in the sbt shell with + * `set javacOptions += FrgaalJavaCompiler.noLimitModulesArg`. + */ + val noLimitModulesArg = "-J--no-limit-modules" val debugArg = "-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000" @@ -86,17 +96,12 @@ object FrgaalJavaCompiler { source: Option[String], target: String ): Boolean = { - val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) - var noLimitModules = false - val debugAnotProcessorOpt = jArgs.contains(debugArg) - val strippedJArgs: ArrayBuffer[String] = ArrayBuffer() - for (strippedJArg <- jArgs.map(_.stripPrefix("-J"))) { - if (strippedJArg == "--no-limit-modules") { - noLimitModules = true - } else { - strippedJArgs += strippedJArg - } - } + val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) + val debugAnotProcessorOpt = jArgs.contains(debugArg) + val noLimitModules = jArgs.contains(noLimitModulesArg) + val strippedJArgs = jArgs + .filterNot(_ == noLimitModulesArg) + .map(_.stripPrefix("-J")) val outputOption = CompilerArguments.outputOption(output) val sources = sources0 map { case x: PathBasedFile => x.toPath.toAbsolutePath.toString @@ -230,16 +235,17 @@ object FrgaalJavaCompiler { val allArguments = outputOption ++ frgaalOptions ++ nonJArgs ++ sources withArgumentFile(allArguments) { argsFile => - // Need to disable standard compiler tools that come with used jdk and replace them - // with the ones provided with Frgaal. val limitModulesArgs = if (noLimitModules) { Seq() } else { + // Need to disable standard compiler tools that come with used jdk and replace them + // with the ones provided with Frgaal. Seq( "--limit-modules", - "java.base,jdk.zipfs,jdk.internal.vm.compiler.management" + "java.base,jdk.zipfs,jdk.internal.vm.compiler.management,org.graalvm.locator" ) } + // strippedJArgs needs to be passed via cmd line, and not via the argument file val forkArgs = (strippedJArgs ++ limitModulesArgs ++ Seq( "-jar", compilerJar.toString From 60899de25549c6ef571d665180c249e6803c7bd8 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 11:07:27 +0200 Subject: [PATCH 60/71] Filter out spurious warning messages from javac --- build.sbt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/build.sbt b/build.sbt index bb91b50ea721..db423edde70d 100644 --- a/build.sbt +++ b/build.sbt @@ -1835,6 +1835,11 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) s"-J-Dtruffle.class.path.append=$appendClasspath" ) }, + (Compile / logManager) := + sbt.internal.util.CustomLogManager.excludeMsg( + "Could not determine source for class ", + Level.Warn + ), (Benchmark / compile / javacOptions) ++= Seq( "-s", (Benchmark / sourceManaged).value.getAbsolutePath, From 97a0036bf2d80d2cfc3ff6bf121566ca473b854a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 11:10:09 +0200 Subject: [PATCH 61/71] Reformat bench-libs project in sbt - group settings by the configuration. --- build.sbt | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/build.sbt b/build.sbt index db423edde70d..d2da2831cb13 100644 --- a/build.sbt +++ b/build.sbt @@ -1806,7 +1806,6 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .dependsOn(runtime) lazy val `bench-libs` = (project in file("std-bits/benchmarks")) - .configs(Benchmark) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= jmh ++ Seq( @@ -1816,6 +1815,18 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) "org.graalvm.truffle" % "truffle-api" % graalMavenPackagesVersion % Benchmark ), commands += WithDebugCommand.withDebug, + (Compile / logManager) := + sbt.internal.util.CustomLogManager.excludeMsg( + "Could not determine source for class ", + Level.Warn + ) + ) + .configs(Benchmark) + .settings( + inConfig(Benchmark)(Defaults.testSettings) + ) + .settings( + (Benchmark / parallelExecution) := false, (Benchmark / run / fork) := true, (Benchmark / run / connectInput) := true, // Pass -Dtruffle.class.path.append to javac @@ -1835,11 +1846,6 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) s"-J-Dtruffle.class.path.append=$appendClasspath" ) }, - (Compile / logManager) := - sbt.internal.util.CustomLogManager.excludeMsg( - "Could not determine source for class ", - Level.Warn - ), (Benchmark / compile / javacOptions) ++= Seq( "-s", (Benchmark / sourceManaged).value.getAbsolutePath, @@ -1861,9 +1867,6 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) ) } ) - .settings( - inConfig(Benchmark)(Defaults.testSettings) - ) .settings( bench := (Benchmark / run).toTask("").tag(Exclusive).value, benchOnly := Def.inputTaskDyn { @@ -1875,8 +1878,7 @@ lazy val `bench-libs` = (project in file("std-bits/benchmarks")) Def.task { (Benchmark / run).toTask(" " + name).value } - }.evaluated, - Benchmark / parallelExecution := false + }.evaluated ) .dependsOn(`bench-processor` % Benchmark) .dependsOn(runtime % Benchmark) From f5622840c9ec7e3a4f978a57898f21b238f88945 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 11:56:58 +0200 Subject: [PATCH 62/71] Generated source does not contain absolute paths --- .../main/java/org/enso/benchmarks/Utils.java | 42 ++++++++++++------- .../benchmarks/processor/BenchProcessor.java | 27 ++++++++---- .../processor/GenerateBenchSources.java | 2 +- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java index 6ffc2c2a87e8..cfcc579fce5c 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/Utils.java @@ -3,16 +3,32 @@ import java.io.File; import java.net.URISyntaxException; -/** - * Used by the benchmark classes from the generated code - */ +/** Utility methods used by the benchmark classes from the generated code */ public class Utils { - public static String findLanguageHomeOverride() { + + /** + * Returns the path to the {@link org.enso.polyglot.RuntimeOptions#LANGUAGE_HOME_OVERRIDE language + * home override directory}. + * + *

Note that the returned file may not exist. + * + * @return Non-null file pointing to the language home override directory. + */ + public static File findLanguageHomeOverride() { + File ensoDir = findRepoRootDir(); + // Note that ensoHomeOverride does not have to exist, only its parent directory + return ensoDir.toPath().resolve("distribution").resolve("component").toFile(); + } + + /** + * Returns the root directory of the Enso repository. + * + * @return Non-null file pointing to the root directory of the Enso repository. + */ + public static File findRepoRootDir() { File ensoDir; try { - ensoDir = - new File( - Utils.class.getProtectionDomain().getCodeSource().getLocation().toURI()); + ensoDir = new File(Utils.class.getProtectionDomain().getCodeSource().getLocation().toURI()); } catch (URISyntaxException e) { throw new IllegalStateException("Unrecheable: ensoDir not found", e); } @@ -21,14 +37,10 @@ public static String findLanguageHomeOverride() { break; } } - assert ensoDir != null; - assert ensoDir.exists(); - assert ensoDir.isDirectory(); - assert ensoDir.canRead(); - - // Note that ensoHomeOverride does not have to exist, only its parent directory - File ensoHomeOverride = ensoDir.toPath().resolve("distribution").resolve("component").toFile(); - return ensoHomeOverride.getAbsolutePath(); + if (ensoDir == null || !ensoDir.exists() || !ensoDir.isDirectory() || !ensoDir.canRead()) { + throw new IllegalStateException("Unrecheable: ensoDir does not exist or is not readable"); + } + return ensoDir; } public static BenchSpec findSpecByName(BenchGroup group, String specName) { diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java index 9eb912e6aa64..71b942d5dea1 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/BenchProcessor.java @@ -42,6 +42,7 @@ public class BenchProcessor extends AbstractProcessor { "import java.io.ByteArrayOutputStream;", "import java.io.File;", "import java.util.List;", + "import java.util.Objects;", "import org.openjdk.jmh.annotations.Benchmark;", "import org.openjdk.jmh.annotations.BenchmarkMode;", "import org.openjdk.jmh.annotations.Mode;", @@ -120,7 +121,8 @@ public boolean process(Set annotations, RoundEnvironment SpecCollector.collectBenchSpecsFromModule(module, annotation.variableName()); for (ModuleBenchSuite benchSuite : benchSuites) { for (BenchGroup group : benchSuite.getGroups()) { - generateClassForGroup(group, benchSuite.getModuleQualifiedName(), annotation.variableName()); + generateClassForGroup( + group, benchSuite.getModuleQualifiedName(), annotation.variableName()); } } return true; @@ -172,7 +174,8 @@ private static boolean isResourceAlreadyExistsException(IOException e) { } private void generateClassForGroup( - Writer javaSrcFileWriter, String moduleQualifiedName, String varName, BenchGroup group) throws IOException { + Writer javaSrcFileWriter, String moduleQualifiedName, String varName, BenchGroup group) + throws IOException { String groupFullClassName = createGroupClassName(group); String className = groupFullClassName.substring(groupFullClassName.lastIndexOf('.') + 1); List specs = group.specs(); @@ -201,12 +204,16 @@ private void generateClassForGroup( javaSrcFileWriter.append(" \n"); javaSrcFileWriter.append(" @Setup\n"); javaSrcFileWriter.append(" public void setup(BenchmarkParams params) throws Exception {\n"); + javaSrcFileWriter + .append(" File projectRootDir = Utils.findRepoRootDir().toPath().resolve(\"") + .append(projectRootDir.toString()) + .append("\").toFile();\n"); javaSrcFileWriter.append( - " File projectRootDir = new File(\"" + projectRootDir.getAbsolutePath() + "\");\n"); + " if (projectRootDir == null || !projectRootDir.exists() || !projectRootDir.canRead()) {\n"); javaSrcFileWriter.append( - " File languageHomeOverride = new File(\"" - + ensoHomeOverride.getAbsolutePath() - + "\");\n"); + " throw new IllegalStateException(\"Project root directory does not exist or cannot be read: \" + Objects.toString(projectRootDir));\n"); + javaSrcFileWriter.append(" }\n"); + javaSrcFileWriter.append(" File languageHomeOverride = Utils.findLanguageHomeOverride();\n"); javaSrcFileWriter.append(" var ctx = Context.newBuilder()\n"); javaSrcFileWriter.append(" .allowExperimentalOptions(true)\n"); javaSrcFileWriter.append(" .allowIO(IOAccess.ALL)\n"); @@ -230,7 +237,9 @@ private void generateClassForGroup( javaSrcFileWriter.append( " BenchGroup group = SpecCollector.collectBenchGroupFromModule(module, \"" + group.name() - + "\", \"" + varName + "\");\n"); + + "\", \"" + + varName + + "\");\n"); javaSrcFileWriter.append(" \n"); for (int i = 0; i < specs.size(); i++) { var specJavaName = specJavaNames.get(i); @@ -238,7 +247,9 @@ private void generateClassForGroup( javaSrcFileWriter.append( " BenchSpec benchSpec_" + specJavaName - + " = Utils.findSpecByName(group, \"" + specName + "\");\n"); + + " = Utils.findSpecByName(group, \"" + + specName + + "\");\n"); javaSrcFileWriter.append( " this.benchFunc_" + specJavaName + " = benchSpec_" + specJavaName + ".code();\n"); } diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java index 10a1667f354f..d66c456ed399 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java @@ -15,7 +15,7 @@ public @interface GenerateBenchSources { /** - * Path to the project root directory. + * Path to the project root directory. Relative to the Enso repository root. */ String projectRootPath(); From ea5315f2abc1fa87dcbbe07b2ae6885459e40708 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 13:27:26 +0200 Subject: [PATCH 63/71] Rename `bench-libs` sbt project to `std-benchmarks`. --- build.sbt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build.sbt b/build.sbt index d2da2831cb13..53d7cb01d4bb 100644 --- a/build.sbt +++ b/build.sbt @@ -1805,7 +1805,7 @@ lazy val `bench-processor` = (project in file("lib/scala/bench-processor")) .dependsOn(`polyglot-api`) .dependsOn(runtime) -lazy val `bench-libs` = (project in file("std-bits/benchmarks")) +lazy val `std-benchmarks` = (project in file("std-bits/benchmarks")) .settings( frgaalJavaCompilerSetting, libraryDependencies ++= jmh ++ Seq( From ed1909dc9d975611e199bdaa99cbc4283edd33c5 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Wed, 2 Aug 2023 18:57:50 +0200 Subject: [PATCH 64/71] In std-benchmarks project, withDebug command works the same as in runtime benchmarks --- docs/infrastructure/benchmarks.md | 7 -- .../enso/benchmarks/runner/BenchRunner.java | 93 +++++++++++-------- .../enso/benchmarks/libs/LibBenchRunner.java | 3 +- 3 files changed, 54 insertions(+), 49 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index c7c512467114..0e8d7b654a86 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -78,11 +78,6 @@ name does not have to be specified as a fully qualified name, but as a regular expression. To access the full flexibility of the JMH launcher, run it via `Bench/run` - for example, to see the help message: `Bench/run -h`. -To debug a single benchmark, set the `javaOptions` with something like -`set javaOptions += "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000"`, -and launch a single benchmark without forking with -`Bench/run -f 0 `. Then, attach to the debugger with your IDE. - ## Visualization The benchmarks are invoked as a daily @@ -94,5 +89,3 @@ for collecting and processing the results. See the README in that directory for more information about how to run that script. This script is invoked regularly on a private machine and the results are published in [https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). - -This is not meant as a permanent solution. diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java index 3f57eecf2a00..25276f6675a3 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/runner/BenchRunner.java @@ -1,5 +1,6 @@ package org.enso.benchmarks.runner; +import jakarta.xml.bind.JAXBException; import java.io.File; import java.io.IOException; import java.util.ArrayList; @@ -11,12 +12,9 @@ import org.openjdk.jmh.runner.BenchmarkListEntry; import org.openjdk.jmh.runner.Runner; import org.openjdk.jmh.runner.RunnerException; -import org.openjdk.jmh.runner.options.ChainedOptionsBuilder; import org.openjdk.jmh.runner.options.CommandLineOptionException; import org.openjdk.jmh.runner.options.CommandLineOptions; -import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; -import jakarta.xml.bind.JAXBException; public class BenchRunner { public static final File REPORT_FILE = new File("./bench-report.xml"); @@ -28,7 +26,7 @@ public List getAvailable() { .collect(Collectors.toList()); } - public static void run(String[] args) { + public static void run(String[] args) throws RunnerException { CommandLineOptions cmdOpts = null; try { cmdOpts = new CommandLineOptions(args); @@ -37,57 +35,70 @@ public static void run(String[] args) { System.err.println(" " + e.getMessage()); System.exit(1); } - Runner jmhRunner = new Runner(cmdOpts); - if (cmdOpts.shouldHelp()) { - System.err.println("Enso benchmark runner: A modified JMH runner for Enso benchmarks."); - try { - cmdOpts.showHelp(); - } catch (IOException e) { - throw new IllegalStateException("Unreachable", e); + if (Boolean.getBoolean("bench.compileOnly")) { + // Do not report results from `compileOnly` mode + runCompileOnly(cmdOpts.getIncludes()); + } else { + Runner jmhRunner = new Runner(cmdOpts); + + if (cmdOpts.shouldHelp()) { + System.err.println("Enso benchmark runner: A modified JMH runner for Enso benchmarks."); + try { + cmdOpts.showHelp(); + } catch (IOException e) { + throw new IllegalStateException("Unreachable", e); + } + System.exit(0); } - System.exit(0); - } - if (cmdOpts.shouldList()) { - jmhRunner.list(); - System.exit(0); - } + if (cmdOpts.shouldList()) { + jmhRunner.list(); + System.exit(0); + } - Collection results; - try { + Collection results; results = jmhRunner.run(); - } catch (RunnerException e) { - throw new RuntimeException(e); - } - for (RunResult result : results) { - try { - reportResult(result.getParams().getBenchmark(), result); - } catch (JAXBException e) { - throw new IllegalStateException("Unreachable", e); + for (RunResult result : results) { + try { + reportResult(result.getParams().getBenchmark(), result); + } catch (JAXBException e) { + throw new IllegalStateException("Benchmark result report writing failed", e); + } } + System.out.println("Benchmark results reported into " + REPORT_FILE.getAbsolutePath()); } - System.out.println("Benchmark results reported into " + REPORT_FILE.getAbsolutePath()); } - public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { - ChainedOptionsBuilder builder = new OptionsBuilder() - .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") - .include("^" + label + "$"); + private static Collection runCompileOnly(List includes) throws RunnerException { + System.out.println("Running benchmarks " + includes + " in compileOnly mode"); + var optsBuilder = new OptionsBuilder() + .measurementIterations(1) + .warmupIterations(0) + .forks(0); + includes.forEach(optsBuilder::include); + var opts = optsBuilder.build(); + var runner = new Runner(opts); + return runner.run(); + } + public static BenchmarkItem runSingle(String label) throws RunnerException, JAXBException { + String includeRegex = "^" + label + "$"; if (Boolean.getBoolean("bench.compileOnly")) { - builder - .measurementIterations(1) - .warmupIterations(0) - .forks(0); + var results = runCompileOnly(List.of(includeRegex)); + var firstResult = results.iterator().next(); + return reportResult(label, firstResult); + } else { + var opts = new OptionsBuilder() + .jvmArgsAppend("-Xss16M", "-Dpolyglot.engine.MultiTier=false") + .include(includeRegex) + .build(); + RunResult benchmarksResult = new Runner(opts).runSingle(); + return reportResult(label, benchmarksResult); } - - Options benchmarkOptions = builder.build(); - RunResult benchmarksResult = new Runner(benchmarkOptions).runSingle(); - - return reportResult(label, benchmarksResult); } + private static BenchmarkItem reportResult(String label, RunResult result) throws JAXBException { Report report; if (REPORT_FILE.exists()) { diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index a7caa746ed1d..e33b2fdfb51e 100644 --- a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -2,6 +2,7 @@ import org.enso.benchmarks.processor.GenerateBenchSources; import org.enso.benchmarks.runner.BenchRunner; +import org.openjdk.jmh.runner.RunnerException; @GenerateBenchSources( projectRootPath = "test/Benchmarks", @@ -9,7 +10,7 @@ ) public class LibBenchRunner { - public static void main(String[] args) { + public static void main(String[] args) throws RunnerException { BenchRunner.run(args); } } From f5e9715721b8b9ed1ebb413a1d57f662ea356519 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 3 Aug 2023 12:14:39 +0200 Subject: [PATCH 65/71] Add some docs --- docs/infrastructure/benchmarks.md | 18 +++++++++++++----- .../processor/GenerateBenchSources.java | 1 + .../benchmarks/processor/SpecCollector.java | 2 +- .../enso/benchmarks/libs/LibBenchRunner.java | 5 +++++ 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index 0e8d7b654a86..a860b83cd077 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -58,12 +58,12 @@ other Enso source file, for example via `runEngineDistribution --in-project test/Benchmarks --run `. The harness within the project is not meant for any sophisticated benchmarking, but rather for quick local evaluation. See the `Bench.measure` method documentation -for more details. For more sophisticated approach, run the benchmarks via JMH +for more details. For more sophisticated approach, run the benchmarks via the JMH launcher. ### Running via JMH launcher -The JMH launcher is located in `std-bits/benchmarks` directory, as `bench-libs` +The JMH launcher is located in `std-bits/benchmarks` directory, as `std-benchmarks` SBT project. It is a single Java class with a `main` method that just delegates to the [standard JMH launcher](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/Main.java), @@ -72,11 +72,19 @@ the full options summary, either see the [JMH source code](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/runner/options/CommandLineOptions.java), or run the launcher with `-h` option. -The `bench-libs` SBT project supports `bench` and `benchOnly` commands, that +The `std-benchmarks` SBT project supports `bench` and `benchOnly` commands, that work the same as in the `runtime` project, with the exception that the benchmark name does not have to be specified as a fully qualified name, but as a regular expression. To access the full flexibility of the JMH launcher, run it via `Bench/run` - for example, to see the help message: `Bench/run -h`. +For example, you can run all the benchmarks that have "New_Vector" in their name +with just 3 seconds for warmup iterations and 2 measurement iterations with +`Bench/run -w 3 -i 2 New_Vector`. + +Whenever you add or delete any benchmarks from `test/Benchmarks` project, the +generated JMH sources need to be recompiled with `Bench/clean; Bench/compile`. +You do not need to recompile the `std-benchmarks` project if you only modify the +benchmark sources. ## Visualization @@ -85,7 +93,7 @@ The benchmarks are invoked as a daily that can be invoked manually on a specific branch as well. The results are kept in the artifacts produced from the actions. In `tools/performance/engine-benchmarks` directory, there is a simple Python script -for collecting and processing the results. See the README in that directory for -more information about how to run that script. This script is invoked regularly +for collecting and processing the results. See the [README in that directory](../../tools/performance/engine-benchmarks/README.md) +for more information about how to run that script. This script is invoked regularly on a private machine and the results are published in [https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java index d66c456ed399..ca75cedcd2b4 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/GenerateBenchSources.java @@ -21,6 +21,7 @@ /** * Fully qualified name of the module within the project that defines all the benchmark {@link org.enso.benchmarks.BenchSuite suites}. + * For example {@code local.Benchmarks.Main}. */ String moduleName(); diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java index 7a029850c88a..2d51f76c845b 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/processor/SpecCollector.java @@ -9,7 +9,7 @@ import org.graalvm.polyglot.Value; /** - * Collect benchmark specifications from source files. + * Collect benchmark specifications from Enso source files. */ public class SpecCollector { private SpecCollector() {} diff --git a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java index e33b2fdfb51e..3714dcc5d4c3 100644 --- a/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java +++ b/std-bits/benchmarks/src/bench/java/org/enso/benchmarks/libs/LibBenchRunner.java @@ -4,6 +4,11 @@ import org.enso.benchmarks.runner.BenchRunner; import org.openjdk.jmh.runner.RunnerException; +/** + * The only purpose for this class is to enable the {@link org.enso.benchmarks.processor.BenchProcessor} + * to generate JMH sources for the benchmarks in {@code test/Benchmarks} project. + * For more information see {@code docs/infrastructure/benchmarks.md#Standard-library-benchmarks}. + */ @GenerateBenchSources( projectRootPath = "test/Benchmarks", moduleName = "local.Benchmarks.Main" From 2664ded4c0a56ad8ecca43a342b0c58c3b3e3a60 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 3 Aug 2023 15:37:59 +0200 Subject: [PATCH 66/71] Remove --no-limit-modules --- build.sbt | 1 - project/FrgaalJavaCompiler.scala | 31 ++++--------------------------- 2 files changed, 4 insertions(+), 28 deletions(-) diff --git a/build.sbt b/build.sbt index 53d7cb01d4bb..191b058df9ce 100644 --- a/build.sbt +++ b/build.sbt @@ -1842,7 +1842,6 @@ lazy val `std-benchmarks` = (project in file("std-bits/benchmarks")) .map(_.data) .mkString(File.pathSeparator) Seq( - FrgaalJavaCompiler.noLimitModulesArg, s"-J-Dtruffle.class.path.append=$appendClasspath" ) }, diff --git a/project/FrgaalJavaCompiler.scala b/project/FrgaalJavaCompiler.scala index 5a4c3a59df88..a8677afe54a4 100644 --- a/project/FrgaalJavaCompiler.scala +++ b/project/FrgaalJavaCompiler.scala @@ -29,16 +29,6 @@ object FrgaalJavaCompiler { val frgaal = "org.frgaal" % "compiler" % "19.0.1" % "provided" val sourceLevel = "19" - /** By default, the Frgaal compiler is invoked with `--limit-modules` argument that limits - * the set of modules that are accessible at compile time to just a small list of internal - * JDK modules. Pass this flag to the compiler in order to disable this behavior. - * This is usable in cases when an annotation processor invoked by Frgaal needs to access - * classes from other modules. - * - * Note that to pass this flag to the compiler, set `javacOptions` in the sbt shell with - * `set javacOptions += FrgaalJavaCompiler.noLimitModulesArg`. - */ - val noLimitModulesArg = "-J--no-limit-modules" val debugArg = "-J-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:8000" @@ -79,11 +69,6 @@ object FrgaalJavaCompiler { } /** Helper method to launch programs. - * - * If -J--no-limit-modules is passed to javac, no --limit-modules option is passed to frgaal. - * With -J--no-limit-modules, one can use any class path in annotation processors. - * All the options with "-J" prefix are prepended to the frgaal command line, instead of passing - * them via an argument file. Moreover, the "-J" prefix is stripped. */ def launch( javaHome: Option[Path], @@ -98,9 +83,7 @@ object FrgaalJavaCompiler { ): Boolean = { val (jArgs, nonJArgs) = options.partition(_.startsWith("-J")) val debugAnotProcessorOpt = jArgs.contains(debugArg) - val noLimitModules = jArgs.contains(noLimitModulesArg) val strippedJArgs = jArgs - .filterNot(_ == noLimitModulesArg) .map(_.stripPrefix("-J")) val outputOption = CompilerArguments.outputOption(output) val sources = sources0 map { case x: PathBasedFile => @@ -235,16 +218,10 @@ object FrgaalJavaCompiler { val allArguments = outputOption ++ frgaalOptions ++ nonJArgs ++ sources withArgumentFile(allArguments) { argsFile => - val limitModulesArgs = if (noLimitModules) { - Seq() - } else { - // Need to disable standard compiler tools that come with used jdk and replace them - // with the ones provided with Frgaal. - Seq( - "--limit-modules", - "java.base,jdk.zipfs,jdk.internal.vm.compiler.management,org.graalvm.locator" - ) - } + val limitModulesArgs = Seq( + "--limit-modules", + "java.base,jdk.zipfs,jdk.internal.vm.compiler.management,org.graalvm.locator,java.desktop,java.net.http" + ) // strippedJArgs needs to be passed via cmd line, and not via the argument file val forkArgs = (strippedJArgs ++ limitModulesArgs ++ Seq( "-jar", From 08fa60b04e11fe6792323f2bf934a89c812716bc Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 3 Aug 2023 16:00:20 +0200 Subject: [PATCH 67/71] fmt docs --- docs/infrastructure/benchmarks.md | 43 +++++++++++++++++-------------- 1 file changed, 23 insertions(+), 20 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index a860b83cd077..1c142f13a418 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -1,10 +1,11 @@ # Benchmarks -In this document, we describe the benchmark types used for the runtime - Engine micro +In this document, we describe the benchmark types used for the runtime - Engine +micro benchmarks in the section +[Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and standard library benchmarks in the section -[Engine JMH microbenchmarks](#engine-jmh-microbenchmarks) and standard library benchmarks in -the section [Standard library benchmarks](#standard-library-benchmarks), and how -and where are the results stored and visualized in the section +[Standard library benchmarks](#standard-library-benchmarks), and how and where +are the results stored and visualized in the section [Visualization](#visualization). To track the performance of the engine, we use @@ -14,9 +15,9 @@ benchmarks: - [micro benchmarks](#engine-jmh-microbenchmarks) located directly in the `runtime` SBT project. These benchmarks are written in Java, and are used to measure the performance of specific parts of the engine. -- [standard library benchmarks](#standard-library-benchmarks) - located in the `test/Benchmarks` Enso project. These benchmarks are entirelly - written in Enso, along with the harness code. +- [standard library benchmarks](#standard-library-benchmarks) located in the + `test/Benchmarks` Enso project. These benchmarks are entirelly written in + Enso, along with the harness code. ## Engine JMH microbenchmarks @@ -44,8 +45,9 @@ withDebug --debugger benchOnly -- ## Standard library benchmarks -Unlike the Engine micro benchmarks, these benchmarks are written entirelly in Enso and -located in the `test/Benchmarks` Enso project. There are two ways to run these benchmarks: +Unlike the Engine micro benchmarks, these benchmarks are written entirelly in +Enso and located in the `test/Benchmarks` Enso project. There are two ways to +run these benchmarks: - [Running standalone](#running-standalone) - [Running via JMH launcher](#running-via-jmh-launcher) @@ -58,14 +60,14 @@ other Enso source file, for example via `runEngineDistribution --in-project test/Benchmarks --run `. The harness within the project is not meant for any sophisticated benchmarking, but rather for quick local evaluation. See the `Bench.measure` method documentation -for more details. For more sophisticated approach, run the benchmarks via the JMH -launcher. +for more details. For more sophisticated approach, run the benchmarks via the +JMH launcher. ### Running via JMH launcher -The JMH launcher is located in `std-bits/benchmarks` directory, as `std-benchmarks` -SBT project. It is a single Java class with a `main` method that just delegates -to the +The JMH launcher is located in `std-bits/benchmarks` directory, as +`std-benchmarks` SBT project. It is a single Java class with a `main` method +that just delegates to the [standard JMH launcher](https://github.com/openjdk/jmh/blob/master/jmh-core/src/main/java/org/openjdk/jmh/Main.java), therefore, supports all the command line options as the standard launcher. For the full options summary, either see the @@ -76,9 +78,9 @@ The `std-benchmarks` SBT project supports `bench` and `benchOnly` commands, that work the same as in the `runtime` project, with the exception that the benchmark name does not have to be specified as a fully qualified name, but as a regular expression. To access the full flexibility of the JMH launcher, run it via -`Bench/run` - for example, to see the help message: `Bench/run -h`. -For example, you can run all the benchmarks that have "New_Vector" in their name -with just 3 seconds for warmup iterations and 2 measurement iterations with +`Bench/run` - for example, to see the help message: `Bench/run -h`. For example, +you can run all the benchmarks that have "New_Vector" in their name with just 3 +seconds for warmup iterations and 2 measurement iterations with `Bench/run -w 3 -i 2 New_Vector`. Whenever you add or delete any benchmarks from `test/Benchmarks` project, the @@ -93,7 +95,8 @@ The benchmarks are invoked as a daily that can be invoked manually on a specific branch as well. The results are kept in the artifacts produced from the actions. In `tools/performance/engine-benchmarks` directory, there is a simple Python script -for collecting and processing the results. See the [README in that directory](../../tools/performance/engine-benchmarks/README.md) -for more information about how to run that script. This script is invoked regularly -on a private machine and the results are published in +for collecting and processing the results. See the +[README in that directory](../../tools/performance/engine-benchmarks/README.md) +for more information about how to run that script. This script is invoked +regularly on a private machine and the results are published in [https://enso-org.github.io/engine-benchmark-results/](https://enso-org.github.io/engine-benchmark-results/). From 495e6e66c7d1aceb066dd58d64133f46dc938e7f Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 3 Aug 2023 18:12:05 +0200 Subject: [PATCH 68/71] Add some docs --- .../src/main/java/org/enso/benchmarks/BenchConfig.java | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java index dae6b1b22bf1..db83ee9da2b1 100644 --- a/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java +++ b/lib/scala/bench-processor/src/main/java/org/enso/benchmarks/BenchConfig.java @@ -1,4 +1,9 @@ package org.enso.benchmarks; + +/** + * A configuration for a {@link BenchGroup benchmark group}. + * Corresponds to {@code Bench_Options} in {@code distribution/lib/Standard/Test/0.0.0-dev/src/Bench.enso} + */ public interface BenchConfig { int size(); int iter(); From 939e8fb6917a40fe89b2b0331952f4232f0576f6 Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Thu, 3 Aug 2023 18:16:44 +0200 Subject: [PATCH 69/71] Minor fixes in the docs --- docs/CONTRIBUTING.md | 2 +- docs/infrastructure/benchmarks.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/CONTRIBUTING.md b/docs/CONTRIBUTING.md index f13bbeddad73..0e8079437d15 100644 --- a/docs/CONTRIBUTING.md +++ b/docs/CONTRIBUTING.md @@ -758,7 +758,7 @@ Below are options uses by the Language Server: - `--server`: Runs the Language Server - `--root-id `: Content root id. The Language Server chooses one randomly, - so can pass any valid UUID. + so any valid UUID can be passed. - `--path `: Path to the content root. - `--interface `: Interface for processing all incoming connections. Default value is 127.0.0.1 diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index 1c142f13a418..269d06d20222 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -45,7 +45,7 @@ withDebug --debugger benchOnly -- ## Standard library benchmarks -Unlike the Engine micro benchmarks, these benchmarks are written entirelly in +Unlike the Engine micro benchmarks, these benchmarks are written entirely in Enso and located in the `test/Benchmarks` Enso project. There are two ways to run these benchmarks: From 11e0a12c6fc449afede4a325659e071daef0104c Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Aug 2023 12:20:16 +0200 Subject: [PATCH 70/71] Revert "Easier way to debug the runtime/benchOnly command" This reverts commit 95e6deaeba270cfc0ad70df89b5423d871dd0dba. --- docs/infrastructure/benchmarks.md | 6 ++---- .../org/enso/interpreter/bench/BenchmarksRunner.java | 7 ++----- project/WithDebugCommand.scala | 9 ++------- 3 files changed, 6 insertions(+), 16 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index 269d06d20222..474af56ec61f 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -35,10 +35,8 @@ measurement iterations, you need to modify the parameter to the `@Measurement` annotation. ### Debugging the benchmarks - -Make sure your IDE listens for JDWP connection at port 5005. Debug the benchmark -by running `withDebug` command like this: - +Currently, the best way to debug the benchmark is to set the `@Fork` annotation to 0, and to +run `withDebug` command like this: ``` withDebug --debugger benchOnly -- ``` diff --git a/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java b/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java index ffcee2cdf819..16954b0282f5 100644 --- a/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java +++ b/engine/runtime/src/bench/java/org/enso/interpreter/bench/BenchmarksRunner.java @@ -1,10 +1,10 @@ package org.enso.interpreter.bench; +import jakarta.xml.bind.JAXBException; import java.io.File; import java.util.ArrayList; import java.util.List; import java.util.stream.Collectors; - import org.openjdk.jmh.results.RunResult; import org.openjdk.jmh.runner.BenchmarkList; import org.openjdk.jmh.runner.BenchmarkListEntry; @@ -14,8 +14,6 @@ import org.openjdk.jmh.runner.options.Options; import org.openjdk.jmh.runner.options.OptionsBuilder; -import jakarta.xml.bind.JAXBException; - /** Runner class for the benchmarks. Discovers, runs and reports benchmark results. */ public class BenchmarksRunner { public static final File REPORT_FILE = new File("./bench-report.xml"); @@ -41,8 +39,7 @@ public BenchmarkItem run(String label) throws RunnerException, JAXBException { if (Boolean.getBoolean("bench.compileOnly")) { builder .measurementIterations(1) - .warmupIterations(0) - .forks(0); + .warmupIterations(0); } Options benchmarkOptions = builder.build(); diff --git a/project/WithDebugCommand.scala b/project/WithDebugCommand.scala index aa450f9bb3e7..824160dc8491 100644 --- a/project/WithDebugCommand.scala +++ b/project/WithDebugCommand.scala @@ -16,12 +16,7 @@ import sbt._ */ object WithDebugCommand { val DEBUG_OPTION = - "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y" - - val debugOptions = Seq( - DEBUG_OPTION, - "-Dbench.compileOnly=true" - ) + "-agentlib:jdwp=transport=dt_socket,server=n,address=localhost:5005,suspend=y"; val truffleNoBackgroundCompilationOptions = Seq( "-Dpolyglot.engine.BackgroundCompilation=false" @@ -85,7 +80,7 @@ object WithDebugCommand { else Seq() val debuggerOpts = if (debugFlags.contains(debuggerOption)) - debugOptions + Seq(DEBUG_OPTION) else Seq() val javaOpts: Seq[String] = Seq( truffleNoBackgroundCompilationOptions, From 7141b0833b500238ebe08ba20106c82397d0c16a Mon Sep 17 00:00:00 2001 From: Pavel Marek Date: Mon, 7 Aug 2023 13:34:25 +0200 Subject: [PATCH 71/71] fmt docs --- docs/infrastructure/benchmarks.md | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/docs/infrastructure/benchmarks.md b/docs/infrastructure/benchmarks.md index 474af56ec61f..a5b5a95eeaa4 100644 --- a/docs/infrastructure/benchmarks.md +++ b/docs/infrastructure/benchmarks.md @@ -35,8 +35,10 @@ measurement iterations, you need to modify the parameter to the `@Measurement` annotation. ### Debugging the benchmarks -Currently, the best way to debug the benchmark is to set the `@Fork` annotation to 0, and to -run `withDebug` command like this: + +Currently, the best way to debug the benchmark is to set the `@Fork` annotation +to 0, and to run `withDebug` command like this: + ``` withDebug --debugger benchOnly -- ```