Skip to content

Commit

Permalink
Restructure benchmarks (#1283)
Browse files Browse the repository at this point in the history
  • Loading branch information
lahma committed Sep 4, 2022
1 parent 704a016 commit f0198a9
Show file tree
Hide file tree
Showing 23 changed files with 300 additions and 143 deletions.
6 changes: 2 additions & 4 deletions Jint.Benchmark/ArrayStressBenchmark.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,5 @@ namespace Jint.Benchmark;
[MemoryDiagnoser]
public class ArrayStressBenchmark : SingleScriptBenchmark
{
protected override string Script => "var ret=[],tmp,num=100,i=256;for(var j1=0;j1<i*15;j1++){ret=[];ret.length=i}for(var j2=0;j2<i*10;j2++){ret=new Array(i)}ret=[];for(var j3=0;j3<i;j3++){ret.unshift(j3)}ret=[];for(var j4=0;j4<i;j4++){ret.splice(0,0,j4)}var a=ret.slice();for(var j5=0;j5<i;j5++){tmp=a.shift()}var b=ret.slice();for(var j6=0;j6<i;j6++){tmp=b.splice(0,1)}ret=[];for(var j7=0;j7<i*25;j7++){ret.push(j7)}var c=ret.slice();for(var j8=0;j8<i*25;j8++){tmp=c.pop()}var done = true;";

public override int N => 10;
}
protected override string FileName => "array-stress.js";
}
4 changes: 2 additions & 2 deletions Jint.Benchmark/DromaeoBenchmark.cs
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ public void Setup()
{
foreach (var fileName in files.Keys.ToList())
{
files[fileName] = File.ReadAllText($"Scripts/dromaeo/{fileName}.js");
files[fileName] = File.ReadAllText($"Scripts/{fileName}.js");
}

engine = new Engine()
Expand Down Expand Up @@ -53,4 +53,4 @@ public void Run()
{
engine.Execute(files[FileName]);
}
}
}
96 changes: 96 additions & 0 deletions Jint.Benchmark/EngineComparisonBenchmark.cs
Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Configs;
using Esprima;
using Esprima.Ast;

namespace Jint.Benchmark;

[RankColumn]
[MemoryDiagnoser]
[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByParams)]
[HideColumns("Error", "Gen0", "Gen1", "Gen2")]
[BenchmarkCategory("EngineComparison")]
public class EngineComparisonBenchmark
{
private static readonly Dictionary<string, Script> _parsedScripts = new();

private static readonly Dictionary<string, string> _files = new()
{
{ "array-stress", null },
{ "evaluation", null },
{ "linq-js", null },
{ "minimal", null },
{ "stopwatch", null },
{ "dromaeo-3d-cube", null },
{ "dromaeo-core-eval", null },
{ "dromaeo-object-array", null },
{ "dromaeo-object-regexp", null },
{ "dromaeo-object-string", null },
{ "dromaeo-string-base64", null },
};

private static readonly string _dromaeoHelpers = @"
var startTest = function () { };
var test = function (name, fn) { fn(); };
var endTest = function () { };
var prep = function (fn) { fn(); };";

[GlobalSetup]
public void Setup()
{
var javaScriptParser = new JavaScriptParser();
foreach (var fileName in _files.Keys.ToList())
{
var script = File.ReadAllText($"Scripts/{fileName}.js");
if (fileName.Contains("dromaeo"))
{
script = _dromaeoHelpers + Environment.NewLine + Environment.NewLine + script;
}
_files[fileName] = script;
_parsedScripts[fileName] = javaScriptParser.ParseScript(script);
}
}

[ParamsSource(nameof(FileNames))]
public string FileName { get; set; }

public IEnumerable<string> FileNames()
{
return _files.Select(entry => entry.Key);
}

[Benchmark]
public void Jint()
{
var engine = new Engine(static options => options.Strict());
engine.Execute(_files[FileName]);
}

[Benchmark]
public void Jint_ParsedScript()
{
var engine = new Engine(static options => options.Strict());
engine.Execute(_parsedScripts[FileName]);
}

[Benchmark]
public void Jurassic()
{
var engine = new Jurassic.ScriptEngine { ForceStrictMode = true };
engine.Execute(_files[FileName]);
}

[Benchmark]
public void NilJS()
{
var engine = new NiL.JS.Core.Context(strict: true);
engine.Eval(_files[FileName]);
}

[Benchmark]
public void YantraJS()
{
var engine = new YantraJS.Core.JSContext();
engine.Eval(_files[FileName]);
}
}
25 changes: 2 additions & 23 deletions Jint.Benchmark/EvaluationBenchmark.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,26 +5,5 @@ namespace Jint.Benchmark;
[MemoryDiagnoser]
public class EvaluationBenchmark : SingleScriptBenchmark
{
protected override string Script => @"
var o = {};
o.Foo = 'bar';
o.Baz = 42.0001;
o.Blah = o.Foo + o.Baz;
if(o.Blah != 'bar42.0001') throw TypeError;
function fib(n){
if(n<2) {
return n;
}
return fib(n-1) + fib(n-2);
}
if(fib(3) != 2) throw TypeError;
var done = true;
";

public override int N => 20;
}
protected override string FileName => "evaluation.js";
}
4 changes: 2 additions & 2 deletions Jint.Benchmark/Jint.Benchmark.csproj
Original file line number Diff line number Diff line change
Expand Up @@ -18,8 +18,8 @@
<ImplicitUsings>enable</ImplicitUsings>
</PropertyGroup>
<ItemGroup>
<None Include=".\Scripts\**" CopyToOutputDirectory="PreserveNewest" />
<None Include="..\Jint.Tests.CommonScripts\Scripts\**" CopyToOutputDirectory="PreserveNewest" LinkBase="SunSpider" />
<None Include=".\Scripts\*.*" CopyToOutputDirectory="PreserveNewest" />
<None Include="..\Jint.Tests.CommonScripts\Scripts\**" CopyToOutputDirectory="PreserveNewest" LinkBase="Scripts" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Jint\Jint.csproj" />
Expand Down
6 changes: 2 additions & 4 deletions Jint.Benchmark/LinqJsBenchmark.cs

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion Jint.Benchmark/MinimalScriptBenchmark.cs
Original file line number Diff line number Diff line change
Expand Up @@ -5,5 +5,5 @@ namespace Jint.Benchmark;
[MemoryDiagnoser]
public class MinimalScriptBenchmark : SingleScriptBenchmark
{
protected override string Script => "var done = 1 + 1 === 2;";
protected override string FileName => "minimal.js";
}
96 changes: 95 additions & 1 deletion Jint.Benchmark/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,98 @@ To run tests comparing engines, use:

```
dotnet run -c Release --allCategories EngineComparison
```
```

## Engine comparison results

* tests are run in global engine strict mode, as YantraJS always uses strict mode which improves performance
* `Jint` and `Jint_ParsedScript` shows the difference between always parsing the script source file and reusing parsed `Script` instance.

Last updated 2022-09-03

* Jint main
* Jurassic 3.2.6
* NiL.JS.NetCore 2.5.1419
* YantraJS.Core 1.2.47


``` ini

BenchmarkDotNet=v0.13.2, OS=Windows 11 (10.0.22621.457)
AMD Ryzen 9 5950X, 1 CPU, 32 logical and 16 physical cores
.NET SDK=6.0.400
[Host] : .NET 6.0.8 (6.0.822.36306), X64 RyuJIT AVX2
DefaultJob : .NET 6.0.8 (6.0.822.36306), X64 RyuJIT AVX2


```
| Method | FileName | Mean | StdDev | Rank | Allocated |
|------------------ |--------------------- |-----------------:|---------------:|-----:|--------------:|
| **Jint** | **array-stress** | **13,507.095 μs** | **23.0399 μs** | **5** | **8745627 B** |
| Jint_ParsedScript | array-stress | 13,102.050 μs | 15.5290 μs | 4 | 8723795 B |
| Jurassic | array-stress | 11,213.554 μs | 14.3872 μs | 3 | 11926461 B |
| NilJS | array-stress | 5,334.318 μs | 23.5378 μs | 1 | 4553815 B |
| YantraJS | array-stress | 7,286.025 μs | 32.1957 μs | 2 | 6318022 B |
| | | | | | |
| **Jint** | **dromaeo-3d-cube** | **23,587.542 μs** | **23.6725 μs** | **4** | **7247466 B** |
| Jint_ParsedScript | dromaeo-3d-cube | 21,772.708 μs | 40.7036 μs | 3 | 7005434 B |
| Jurassic | dromaeo-3d-cube | 39,741.721 μs | 64.2544 μs | 5 | 10925914 B |
| NilJS | dromaeo-3d-cube | 8,708.896 μs | 12.4054 μs | 2 | 4125451 B |
| YantraJS | dromaeo-3d-cube | 5,017.989 μs | 19.0233 μs | 1 | 8786605 B |
| | | | | | |
| **Jint** | **dromaeo-core-eval** | **5,705.326 μs** | **21.1657 μs** | **2** | **350102 B** |
| Jint_ParsedScript | dromaeo-core-eval | 5,668.950 μs | 12.4161 μs | 2 | 336990 B |
| Jurassic | dromaeo-core-eval | 13,447.357 μs | 30.6629 μs | 4 | 2971062 B |
| NilJS | dromaeo-core-eval | 2,173.203 μs | 46.5428 μs | 1 | 1595219 B |
| YantraJS | dromaeo-core-eval | 8,265.873 μs | 56.7684 μs | 3 | 37131603 B |
| | | | | | |
| **Jint** | **dromaeo-object-array** | **108,627.168 μs** | **511.8916 μs** | **4** | **103655627 B** |
| Jint_ParsedScript | dromaeo-object-array | 108,528.990 μs | 288.8372 μs | 4 | 103625558 B |
| Jurassic | dromaeo-object-array | 41,209.118 μs | 44.6309 μs | 1 | 26433962 B |
| NilJS | dromaeo-object-array | 53,391.875 μs | 732.5365 μs | 2 | 18027054 B |
| YantraJS | dromaeo-object-array | 74,332.930 μs | 528.6855 μs | 3 | 24735409 B |
| | | | | | |
| **Jint** | **droma(...)egexp [21]** | **299,361.023 μs** | **7,904.1811 μs** | **1** | **241632484 B** |
| Jint_ParsedScript | droma(...)egexp [21] | 295,042.982 μs | 8,424.5753 μs | 1 | 247151352 B |
| Jurassic | droma(...)egexp [21] | 683,250.476 μs | 13,213.8098 μs | 2 | 842918784 B |
| NilJS | droma(...)egexp [21] | 801,835.247 μs | 13,416.6210 μs | 3 | 901970880 B |
| YantraJS | droma(...)egexp [21] | 1,017,060.643 μs | 7,046.5834 μs | 4 | 964707752 B |
| | | | | | |
| **Jint** | **droma(...)tring [21]** | **628,645.310 μs** | **21,985.7843 μs** | **4** | **1363791136 B** |
| Jint_ParsedScript | droma(...)tring [21] | 524,500.457 μs | 22,516.2241 μs | 2 | 1363541192 B |
| Jurassic | droma(...)tring [21] | 557,218.173 μs | 23,822.7984 μs | 3 | 1493044472 B |
| NilJS | droma(...)tring [21] | 400,573.819 μs | 12,223.2081 μs | 1 | 1446930864 B |
| YantraJS | droma(...)tring [21] | 2,258,766.944 μs | 60,256.3876 μs | 5 | 16092618624 B |
| | | | | | |
| **Jint** | **droma(...)ase64 [21]** | **66,060.024 μs** | **79.4200 μs** | **4** | **8049804 B** |
| Jint_ParsedScript | droma(...)ase64 [21] | 64,215.565 μs | 62.6330 μs | 3 | 7968098 B |
| Jurassic | droma(...)ase64 [21] | 69,690.458 μs | 199.4819 μs | 5 | 76103531 B |
| NilJS | droma(...)ase64 [21] | 45,237.687 μs | 534.6955 μs | 2 | 51047211 B |
| YantraJS | droma(...)ase64 [21] | 42,177.560 μs | 246.7996 μs | 1 | 778591540 B |
| | | | | | |
| **Jint** | **evaluation** | **28.520 μs** | **0.0348 μs** | **2** | **34784 B** |
| Jint_ParsedScript | evaluation | 12.906 μs | 0.0298 μs | 1 | 26752 B |
| Jurassic | evaluation | 1,286.204 μs | 3.0882 μs | 5 | 430506 B |
| NilJS | evaluation | 43.866 μs | 1.2033 μs | 3 | 22456 B |
| YantraJS | evaluation | 178.979 μs | 0.6314 μs | 4 | 178517 B |
| | | | | | |
| **Jint** | **linq-js** | **1,876.845 μs** | **3.2033 μs** | **3** | **1301929 B** |
| Jint_ParsedScript | linq-js | 123.396 μs | 0.1138 μs | 1 | 230841 B |
| Jurassic | linq-js | 36,984.974 μs | 395.7409 μs | 4 | 9526082 B |
| NilJS | linq-js | NA | NA | ? | - |
| YantraJS | linq-js | 391.523 μs | 1.8922 μs | 2 | 490377 B |
| | | | | | |
| **Jint** | **minimal** | **5.334 μs** | **0.0065 μs** | **3** | **14120 B** |
| Jint_ParsedScript | minimal | 3.535 μs | 0.0143 μs | 1 | 12680 B |
| Jurassic | minimal | 234.184 μs | 0.6536 μs | 5 | 395505 B |
| NilJS | minimal | 4.516 μs | 0.0933 μs | 2 | 4272 B |
| YantraJS | minimal | 173.909 μs | 0.1640 μs | 4 | 174668 B |
| | | | | | |
| **Jint** | **stopwatch** | **357,288.654 μs** | **1,123.5726 μs** | **4** | **38906408 B** |
| Jint_ParsedScript | stopwatch | 362,800.054 μs | 962.7038 μs | 5 | 38890808 B |
| Jurassic | stopwatch | 209,929.047 μs | 346.3075 μs | 2 | 160703632 B |
| NilJS | stopwatch | 247,228.571 μs | 527.9922 μs | 3 | 85866368 B |
| YantraJS | stopwatch | 76,725.368 μs | 194.3363 μs | 1 | 259048377 B |

Benchmarks with issues:
EngineComparisonBenchmark.NilJS: DefaultJob [FileName=linq-js]
1 change: 1 addition & 0 deletions Jint.Benchmark/Scripts/array-stress.js

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

File renamed without changes.
File renamed without changes.
16 changes: 16 additions & 0 deletions Jint.Benchmark/Scripts/evaluation.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
var o = {};
o.Foo = 'bar';
o.Baz = 42.0001;
o.Blah = o.Foo + o.Baz;

if(o.Blah != 'bar42.0001') throw TypeError;

function fib(n){
if(n<2) {
return n;
}

return fib(n-1) + fib(n-2);
}

if(fib(3) != 2) throw TypeError;
1 change: 1 addition & 0 deletions Jint.Benchmark/Scripts/linq-js.js

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions Jint.Benchmark/Scripts/minimal.js
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
var x = 1 + 1 === 2;
56 changes: 56 additions & 0 deletions Jint.Benchmark/Scripts/stopwatch.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
function Stopwatch() {
var sw = this;
var start = null;
var stop = null;
var isRunning = false;

sw.Start = function () {
if (isRunning)
return;

start = new Date();
stop = null;
isRunning = true;
}

sw.Stop = function () {
if (!isRunning)
return;

stop = new Date();
isRunning = false;
}

sw.Reset = function () {
start = isRunning ? new Date() : null;
stop = null;
}

sw.Restart = function () {
isRunning = true;
sw.Reset();
}

sw.ElapsedMilliseconds = function () { return (isRunning ? new Date() : stop) - start; }
sw.IsRunning = function() { return isRunning; }

}

var sw = new Stopwatch();
sw.Start();
for (var x = 0; x < 1021; x++) {
for (var y = 0; y < 383; y++) {
var z = x ^ y;
if (z % 2 == 0)
sw.Start();
else if(z % 3 == 0)
sw.Stop();
else if (z % 5 == 0)
sw.Reset();
else if (z % 7 == 0)
sw.Restart();
var ms = sw.ElapsedMilliseconds;
var rn = sw.IsRunning;
}
}
sw.Stop();

0 comments on commit f0198a9

Please sign in to comment.