diff --git a/examples/compare-benchmark-runs.ipynb b/examples/compare-benchmark-runs.ipynb new file mode 100644 index 00000000..24d675ac --- /dev/null +++ b/examples/compare-benchmark-runs.ipynb @@ -0,0 +1,3222 @@ +{ + "cells": [ + { + "metadata": { + "collapsed": true + }, + "cell_type": "markdown", + "source": [ + "# Compare Benchmark Runs\n", + "This notebook demonstrates how you can analyze the differences between two benchmark runs of the same benchmark and find the tests that differ the most, which probably means that they require further analysis to figure out why they changed.\n", + "\n", + "Several projects exist in the `examples` folder, but this notebook assumes we are working on the\n", + "JVM part of the `kotlin-multiplatform` project. But the same approach can be used for the other projects.\n", + "\n", + "First, you need to run the benchmark twice. This can be done by running these commands from the root of the project:\n", + "\n", + "```shell\n", + "> ./gradlew :examples:kotlin-multiplatform:jvmBenchmark\n", + "> ./gradlew :examples:kotlin-multiplatform:jvmBenchmark\n", + "```\n", + "\n", + "Once it is completed, run this notebook, and it will automatically find the latest result." + ] + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:36.539161Z", + "start_time": "2025-10-15T06:52:32.150691Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_2_jupyter", + "Line_3_jupyter", + "Line_4_jupyter", + "Line_5_jupyter", + "Line_6_jupyter", + "Line_7_jupyter", + "Line_8_jupyter", + "Line_9_jupyter", + "Line_10_jupyter", + "Line_11_jupyter", + "Line_12_jupyter", + "Line_13_jupyter", + "Line_14_jupyter", + "Line_15_jupyter" + ] + } + }, + "cell_type": "code", + "source": "%use serialization, dataframe, kandy", + "outputs": [], + "execution_count": 1 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:36.792711Z", + "start_time": "2025-10-15T06:52:36.540356Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_16_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "import java.nio.file.Files\n", + "import java.nio.file.attribute.BasicFileAttributes\n", + "import kotlin.io.path.exists\n", + "import kotlin.io.path.forEachDirectoryEntry\n", + "import kotlin.io.path.isDirectory\n", + "import kotlin.io.path.listDirectoryEntries\n", + "import kotlin.io.path.readText\n", + "\n", + "// Find latest result file, based on the their timestamp.\n", + "val runsDir = notebook.workingDir.resolve(\"kotlin-multiplatform/build/reports/benchmarks/main\")\n", + "val outputFiles = runsDir.listDirectoryEntries()\n", + " .filter { it.isDirectory() }\n", + " .sortedByDescending { dir -> Files.readAttributes(dir, BasicFileAttributes::class.java).creationTime() }\n", + " .subList(0, 2)\n", + " .map { it.resolve(\"jvm.json\") }" + ], + "outputs": [], + "execution_count": 2 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:37.905784Z", + "start_time": "2025-10-15T06:52:36.794274Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_17_jupyter", + "Line_18_jupyter", + "Line_19_jupyter", + "Line_20_jupyter", + "Line_21_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Convert to \"typed\" JSON\n", + "val newRun = outputFiles.first().readText().deserializeJson()\n", + "val oldRun = outputFiles.last().readText().deserializeJson()" + ], + "outputs": [], + "execution_count": 3 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:38.536966Z", + "start_time": "2025-10-15T06:52:37.906464Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_22_jupyter", + "Line_23_jupyter", + "Line_24_jupyter", + "Line_25_jupyter", + "Line_26_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Convert to DataFrames for easier processing. As there is not \"id\" keys for the benchmark, we invent one by just\n", + "// assigning the test row index as their \"primary key\". We could attempt to use the benchmark name and param values,\n", + "// but that is complicated by how paramers are represented in the JSON file. So, since we assume that the two files\n", + "// are equal using row index should be safe.\n", + "val oldDf = oldRun.toDataFrame().addId(\"rowIndex\")\n", + "val newDf = newRun.toDataFrame().addId(\"rowIndex\")" + ], + "outputs": [], + "execution_count": 4 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:38.914275Z", + "start_time": "2025-10-15T06:52:38.543089Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_27_jupyter", + "Line_28_jupyter", + "Line_29_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "val combinedData = oldDf.innerJoin(newDf) { rowIndex }\n", + "// Un-commont this to see the intermediate dataframe:\n", + "// combinedData" + ], + "outputs": [], + "execution_count": 5 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:39.257941Z", + "start_time": "2025-10-15T06:52:38.915574Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_30_jupyter", + "Line_31_jupyter", + "Line_32_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Reduce the combined data into the exact format we need\n", + "val resultData = combinedData.mapToFrame {\n", + " \"name\" from { it.benchmark }\n", + " \"params\" from { it.params?.let { params -> \"data=${params.data},text=\\\"${params.text}\\\",value=${params.value}\"} ?: \"\" }\n", + " \"mode\" from { it.mode } // \"avgt\" or \"thrpt\"\n", + " \"unit\" from { it.primaryMetric.scoreUnit }\n", + " \"runOld\" {\n", + " \"score\" from { it.primaryMetric.score }\n", + " \"range\" from { it.primaryMetric.scoreConfidence[0]..it.primaryMetric.scoreConfidence[1] }\n", + " }\n", + " \"runNew\" {\n", + " \"score\" from { it.primaryMetric1.score }\n", + " \"range\" from { it.primaryMetric1.scoreConfidence[0]..it.primaryMetric1.scoreConfidence[1] }\n", + " }\n", + "}\n", + "// Un-commont this to see the intermediate dataframe:\n", + "// resultData" + ], + "outputs": [], + "execution_count": 6 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:39.400795Z", + "start_time": "2025-10-15T06:52:39.259607Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_33_jupyter", + "Line_34_jupyter", + "Line_35_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Flatten the data so it is easier to plot\n", + "val mergedData = resultData.unfold { runOld and runNew }.flatten()\n", + "// Un-commont this to see the intermediate dataframe:\n", + "// mergedData" + ], + "outputs": [], + "execution_count": 7 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:39.899417Z", + "start_time": "2025-10-15T06:52:39.403744Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_36_jupyter", + "Line_37_jupyter", + "Line_38_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Before plotting the data, we calculate the change between the two runs. This is saved\n", + "// in \"scoreDiff\". This is done slightly different depending on the test mode:\n", + "//\n", + "// - \"avgt\": For the average time we use \"oldScore - newScore\", so improvements in the\n", + "// benchmark result in positive numbers.\n", + "// - \"thrpt\": For throughput, we use \"newScore - oldScore\", so improvements here also\n", + "// result in positive numbers.\n", + "//\n", + "// We also normalize this value as a percentage change from `scoreOld`. This is saved in\n", + "// \"scoreDiffPercentage\".\n", + "val plotData = mergedData\n", + " .add(\"diffScore\") {\n", + " when (mode) {\n", + " \"avgt\" -> score - score1\n", + " \"thrpt\" -> score1 - score\n", + " else -> error(\"Unknown mode: $mode\")\n", + " }\n", + " }\n", + " .add(\"diffScorePercentage\") {\n", + " (get(\"diffScore\") as Double) * 100.0 / score\n", + " }\n", + " .add(\"testLabel\") {\n", + " if (params.isNullOrBlank()) {\n", + " name\n", + " } else {\n", + " \"$name\\n[$params]\"\n", + " }\n", + " }\n", + " .add(\"barColor\") {\n", + " val value = get(\"diffScorePercentage\") as Double\n", + " if (value < 0.0) \"neg\" else \"pos\"\n", + " }\n", + "plotData" + ], + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nameparamsmodeunitscorerangescore1range1diffScorediffScorePercentagetestLabelbarColor
test.InheritedBenchmark.baseBenchmarkthrptops/s1124169.3020371048963.3505892197..1199375.25348484451104972.6706891055663.3534587221..1154281.987920151-19196.631348-1.707628test.InheritedBenchmark.baseBenchmarkneg
test.InheritedBenchmark.inheritedBenc...thrptops/s145630568.9018341.4257469243792653E8..1.4868644536574...146349897.4560971.437328117109918E8..1.489669832012015E8719328.5542630.493941test.InheritedBenchmark.inheritedBenc...pos
test.ParamBenchmark.mathBenchmarkdata=1,text="a "string" with quotes",...thrptops/ms215040.426488209159.44934415395..220921.40363117142213019.578414204650.0532153221..221389.10361238578-2020.848074-0.939753test.ParamBenchmark.mathBenchmark\n", + "[da...neg
test.ParamBenchmark.mathBenchmarkdata=1,text="a "string" with quotes",...thrptops/ms216167.949476209816.26197479176..222519.63697705854214917.724126208616.32754332913..221219.12070789465-1250.225350-0.578358test.ParamBenchmark.mathBenchmark\n", + "[da...neg
test.ParamBenchmark.mathBenchmarkdata=2,text="a "string" with quotes",...thrptops/ms102937.094929102198.8667788953..103675.32307942082102424.759291101113.46667154715..103736.05191104818-512.335638-0.497717test.ParamBenchmark.mathBenchmark\n", + "[da...neg
test.ParamBenchmark.mathBenchmarkdata=2,text="a "string" with quotes",...thrptops/ms103082.844856102746.60675391225..103419.08295814389101173.75249399908.94648785368..102438.55849807907-1909.092363-1.851998test.ParamBenchmark.mathBenchmark\n", + "[da...neg
test.ParamBenchmark.otherBenchmarkdata=1,text="a "string" with quotes",...thrptops/ms2672623.9977172581766.221460874..2763481.77397249272680506.7861612646872.2297869264..2714141.34253436937882.7884440.294946test.ParamBenchmark.otherBenchmark\n", + "[d...pos
test.ParamBenchmark.otherBenchmarkdata=1,text="a "string" with quotes",...thrptops/ms2654913.2949252530927.3446583785..2778899.2451910742585022.6644682454777.568459309..2715267.760476164-69890.630457-2.632501test.ParamBenchmark.otherBenchmark\n", + "[d...neg
test.ParamBenchmark.otherBenchmarkdata=2,text="a "string" with quotes",...thrptops/ms2676852.3892642585653.063020375..2768051.7155078992614856.4118722580741.3402635106..2648971.483481024-61995.977392-2.316003test.ParamBenchmark.otherBenchmark\n", + "[d...neg
test.ParamBenchmark.otherBenchmarkdata=2,text="a "string" with quotes",...thrptops/ms2676496.8351542585056.261771287..2767937.40853723322585237.2445242552358.4905912033..2618115.998457625-91259.590630-3.409666test.ParamBenchmark.otherBenchmark\n", + "[d...neg
test.ParamBenchmark.textContentCheckdata=1,text="a "string" with quotes",...thrptops/ms155929.815236155267.27942953032..156592.35104171414150449.209123147414.465281937..153483.95296329833-5480.606113-3.514790test.ParamBenchmark.textContentCheck\n", + "...neg
test.ParamBenchmark.textContentCheckdata=1,text="a "string" with quotes",...thrptops/ms156259.919330155967.2972273164..156552.54143332926147058.565627140116.58313969668..154000.5481141382-9201.353703-5.888493test.ParamBenchmark.textContentCheck\n", + "...neg
test.ParamBenchmark.textContentCheckdata=2,text="a "string" with quotes",...thrptops/ms155811.883306154012.99823010628..157610.76838288622146125.997244143504.1570471993..148747.83743989808-9685.886063-6.216398test.ParamBenchmark.textContentCheck\n", + "...neg
test.ParamBenchmark.textContentCheckdata=2,text="a "string" with quotes",...thrptops/ms154835.234029152299.566575624..157370.90148168168150019.932114145881.2852993276..154158.57892833705-4815.301915-3.109952test.ParamBenchmark.textContentCheck\n", + "...neg
test.nested.CommonBenchmark.mathBench...thrptops/ms149758.678427149465.14095246932..150052.21590221935145509.901147143242.89296995336..147776.90932420595-4248.777280-2.837083test.nested.CommonBenchmark.mathBench...neg
test.CommonBenchmark.longBenchmarkavgtms/op0.0008448.372522775198425E-4..8.5116426929504...0.0008978.737790976669418E-4..9.2064844530355...-0.000053-6.278723test.CommonBenchmark.longBenchmarkneg
test.CommonBenchmark.longBlackholeBen...avgtms/op0.0000222.1394152325114E-5..2.190102450243732...0.0000242.3161777788037864E-5..2.422199277814...-0.000002-9.443532test.CommonBenchmark.longBlackholeBen...neg
test.CommonBenchmark.mathBenchmarkavgtms/op0.0000076.650531071713415E-6..6.7362130570941...0.0000076.710269330854009E-6..6.8730920645926...-0.000000-1.468746test.CommonBenchmark.mathBenchmarkneg
test.JvmTestBenchmark.cosBenchmarkavgtns/op3.4724853.4596934638924464..3.4852772244583043.5445403.5170213910539556..3.572058960798929-0.072055-2.075022test.JvmTestBenchmark.cosBenchmarkneg
test.JvmTestBenchmark.sqrtBenchmarkavgtns/op0.5348080.5300538318352048..0.53956265532672830.5428280.5388189143575554..0.5468378157356791-0.008020-1.499626test.JvmTestBenchmark.sqrtBenchmarkneg
\n", + " \n", + " \n", + " " + ], + "application/kotlindataframe+json": "{\"$version\":\"2.2.0\",\"metadata\":{\"columns\":[\"name\",\"params\",\"mode\",\"unit\",\"score\",\"range\",\"score1\",\"range1\",\"diffScore\",\"diffScorePercentage\",\"testLabel\",\"barColor\"],\"types\":[{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.ranges.ClosedFloatingPointRange\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.ranges.ClosedFloatingPointRange\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"}],\"nrow\":20,\"ncol\":12,\"is_formatted\":false},\"kotlin_dataframe\":[{\"name\":\"test.InheritedBenchmark.baseBenchmark\",\"params\":\"\",\"mode\":\"thrpt\",\"unit\":\"ops/s\",\"score\":1124169.302037032,\"range\":\"1048963.3505892197..1199375.2534848445\",\"score1\":1104972.6706894366,\"range1\":\"1055663.3534587221..1154281.987920151\",\"diffScore\":-19196.63134759548,\"diffScorePercentage\":-1.7076281404242712,\"testLabel\":\"test.InheritedBenchmark.baseBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.InheritedBenchmark.inheritedBenchmark\",\"params\":\"\",\"mode\":\"thrpt\",\"unit\":\"ops/s\",\"score\":1.4563056890183368E8,\"range\":\"1.4257469243792653E8..1.4868644536574084E8\",\"score1\":1.4634989745609665E8,\"range1\":\"1.437328117109918E8..1.489669832012015E8\",\"diffScore\":719328.5542629659,\"diffScorePercentage\":0.4939406332662542,\"testLabel\":\"test.InheritedBenchmark.inheritedBenchmark\",\"barColor\":\"pos\"},{\"name\":\"test.ParamBenchmark.mathBenchmark\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":215040.42648766268,\"range\":\"209159.44934415395..220921.40363117142\",\"score1\":213019.57841385395,\"range1\":\"204650.0532153221..221389.10361238578\",\"diffScore\":-2020.8480738087383,\"diffScorePercentage\":-0.9397526348026837,\"testLabel\":\"test.ParamBenchmark.mathBenchmark\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.mathBenchmark\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":216167.94947592515,\"range\":\"209816.26197479176..222519.63697705854\",\"score1\":214917.7241256119,\"range1\":\"208616.32754332913..221219.12070789465\",\"diffScore\":-1250.225350313267,\"diffScorePercentage\":-0.5783583335754896,\"testLabel\":\"test.ParamBenchmark.mathBenchmark\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.mathBenchmark\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":102937.09492915806,\"range\":\"102198.8667788953..103675.32307942082\",\"score1\":102424.75929129767,\"range1\":\"101113.46667154715..103736.05191104818\",\"diffScore\":-512.335637860393,\"diffScorePercentage\":-0.4977172108975735,\"testLabel\":\"test.ParamBenchmark.mathBenchmark\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.mathBenchmark\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":103082.84485602807,\"range\":\"102746.60675391225..103419.08295814389\",\"score1\":101173.75249296638,\"range1\":\"99908.94648785368..102438.55849807907\",\"diffScore\":-1909.092363061689,\"diffScorePercentage\":-1.8519981338583025,\"testLabel\":\"test.ParamBenchmark.mathBenchmark\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.otherBenchmark\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":2672623.9977166834,\"range\":\"2581766.221460874..2763481.7739724927\",\"score1\":2680506.786160648,\"range1\":\"2646872.2297869264..2714141.3425343693\",\"diffScore\":7882.78844396444,\"diffScorePercentage\":0.2949456583005684,\"testLabel\":\"test.ParamBenchmark.otherBenchmark\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"pos\"},{\"name\":\"test.ParamBenchmark.otherBenchmark\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":2654913.2949247262,\"range\":\"2530927.3446583785..2778899.245191074\",\"score1\":2585022.6644677366,\"range1\":\"2454777.568459309..2715267.760476164\",\"diffScore\":-69890.63045698963,\"diffScorePercentage\":-2.6325014301068244,\"testLabel\":\"test.ParamBenchmark.otherBenchmark\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.otherBenchmark\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":2676852.389264137,\"range\":\"2585653.063020375..2768051.715507899\",\"score1\":2614856.4118722673,\"range1\":\"2580741.3402635106..2648971.483481024\",\"diffScore\":-61995.97739186976,\"diffScorePercentage\":-2.3160028412665805,\"testLabel\":\"test.ParamBenchmark.otherBenchmark\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.otherBenchmark\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":2676496.83515426,\"range\":\"2585056.261771287..2767937.4085372332\",\"score1\":2585237.244524414,\"range1\":\"2552358.4905912033..2618115.998457625\",\"diffScore\":-91259.59062984586,\"diffScorePercentage\":-3.409665553540104,\"testLabel\":\"test.ParamBenchmark.otherBenchmark\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":155929.81523562223,\"range\":\"155267.27942953032..156592.35104171414\",\"score1\":150449.20912261767,\"range1\":\"147414.465281937..153483.95296329833\",\"diffScore\":-5480.606113004556,\"diffScorePercentage\":-3.5147903591900813,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":156259.91933032282,\"range\":\"155967.2972273164..156552.54143332926\",\"score1\":147058.56562691744,\"range1\":\"140116.58313969668..154000.5481141382\",\"diffScore\":-9201.353703405388,\"diffScorePercentage\":-5.888492546802327,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":155811.88330649625,\"range\":\"154012.99823010628..157610.76838288622\",\"score1\":146125.99724354869,\"range1\":\"143504.1570471993..148747.83743989808\",\"diffScore\":-9685.886062947568,\"diffScorePercentage\":-6.216397528482819,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":154835.23402865283,\"range\":\"152299.566575624..157370.90148168168\",\"score1\":150019.93211383233,\"range1\":\"145881.2852993276..154158.57892833705\",\"diffScore\":-4815.301914820506,\"diffScorePercentage\":-3.109952295437753,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.nested.CommonBenchmark.mathBenchmark\",\"params\":\"\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":149758.67842734433,\"range\":\"149465.14095246932..150052.21590221935\",\"score1\":145509.90114707965,\"range1\":\"143242.89296995336..147776.90932420595\",\"diffScore\":-4248.77728026468,\"diffScorePercentage\":-2.8370825149381784,\"testLabel\":\"test.nested.CommonBenchmark.mathBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.CommonBenchmark.longBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ms/op\",\"score\":8.442082734074419E-4,\"range\":\"8.372522775198425E-4..8.511642692950413E-4\",\"score1\":8.972137714852465E-4,\"range1\":\"8.737790976669418E-4..9.206484453035511E-4\",\"diffScore\":-5.3005498077804544E-5,\"diffScorePercentage\":-6.278722887168673,\"testLabel\":\"test.CommonBenchmark.longBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.CommonBenchmark.longBlackholeBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ms/op\",\"score\":2.1647588413775662E-5,\"range\":\"2.1394152325114E-5..2.1901024502437323E-5\",\"score1\":2.369188528309145E-5,\"range1\":\"2.3161777788037864E-5..2.4221992778145036E-5\",\"diffScore\":-2.044296869315788E-6,\"diffScorePercentage\":-9.443531677712787,\"testLabel\":\"test.CommonBenchmark.longBlackholeBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.CommonBenchmark.mathBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ms/op\",\"score\":6.6933720644037785E-6,\"range\":\"6.650531071713415E-6..6.736213057094142E-6\",\"score1\":6.7916806977233165E-6,\"range1\":\"6.710269330854009E-6..6.8730920645926235E-6\",\"diffScore\":-9.830863331953794E-8,\"diffScorePercentage\":-1.4687459829456666,\"testLabel\":\"test.CommonBenchmark.mathBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.JvmTestBenchmark.cosBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ns/op\",\"score\":3.472485344175375,\"range\":\"3.4596934638924464..3.485277224458304\",\"score1\":3.5445401759264423,\"range1\":\"3.5170213910539556..3.572058960798929\",\"diffScore\":-0.0720548317510672,\"diffScorePercentage\":-2.075021911091128,\"testLabel\":\"test.JvmTestBenchmark.cosBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.JvmTestBenchmark.sqrtBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ns/op\",\"score\":0.5348082435809666,\"range\":\"0.5300538318352048..0.5395626553267283\",\"score1\":0.5428283650466172,\"range1\":\"0.5388189143575554..0.5468378157356791\",\"diffScore\":-0.008020121465650676,\"diffScorePercentage\":-1.4996256250557365,\"testLabel\":\"test.JvmTestBenchmark.sqrtBenchmark\",\"barColor\":\"neg\"}]}" + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 8 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:40.654419Z", + "start_time": "2025-10-15T06:52:39.985683Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_40_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "import org.jetbrains.kotlinx.kandy.util.color.Color\n", + "import org.jetbrains.letsPlot.core.spec.plotson.fill\n", + "import org.jetbrains.letsPlot.core.spec.plotson.format\n", + "import org.jetbrains.letsPlot.core.spec.plotson.title\n", + "import org.jetbrains.letsPlot.label.ggtitle\n", + "import org.jetbrains.letsPlot.scale.guideLegend\n", + "import org.jetbrains.letsPlot.scale.guides\n", + "\n", + "// Now we can plot this data. First we create a basic plot just showing the difference in percent between all scores.\n", + "plotData.sortBy { diffScorePercentage }.plot {\n", + " barsH {\n", + " x(diffScorePercentage) {\n", + " axis.name = \"Diff %\"\n", + " }\n", + " y(testLabel) {\n", + " axis.name = \"\"\n", + " }\n", + " fillColor(barColor) {\n", + " scale = categorical(\"neg\" to Color.RED, \"pos\" to Color.GREEN)\n", + " legend.type = LegendType.None\n", + " }\n", + " tooltips {\n", + " line(diffScorePercentage, format = \".2f\")\n", + " }\n", + " }\n", + " layout {\n", + " size = 800 to ((40 * plotData.size().nrow) + 100)\n", + " style {\n", + " global {\n", + " title {\n", + " margin(10.0, 0.0)\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}" + ], + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -8\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -6\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -4\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.CommonBenchmark.longBlackholeBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.CommonBenchmark.longBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.otherBenchmark\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.nested.CommonBenchmark.mathBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.otherBenchmark\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.otherBenchmark\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.JvmTestBenchmark.cosBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.mathBenchmark\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.InheritedBenchmark.baseBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.JvmTestBenchmark.sqrtBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.CommonBenchmark.mathBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.mathBenchmark\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.mathBenchmark\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.mathBenchmark\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.otherBenchmark\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.InheritedBenchmark.inheritedBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " Diff %\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "data": { + "testLabel": [ + "test.CommonBenchmark.longBlackholeBenchmark", + "test.CommonBenchmark.longBenchmark", + "test.ParamBenchmark.textContentCheck\n[data=2,text=\"a \"string\" with quotes\",value=1]", + "test.ParamBenchmark.textContentCheck\n[data=1,text=\"a \"string\" with quotes\",value=2]", + "test.ParamBenchmark.textContentCheck\n[data=1,text=\"a \"string\" with quotes\",value=1]", + "test.ParamBenchmark.otherBenchmark\n[data=2,text=\"a \"string\" with quotes\",value=2]", + "test.ParamBenchmark.textContentCheck\n[data=2,text=\"a \"string\" with quotes\",value=2]", + "test.nested.CommonBenchmark.mathBenchmark", + "test.ParamBenchmark.otherBenchmark\n[data=1,text=\"a \"string\" with quotes\",value=2]", + "test.ParamBenchmark.otherBenchmark\n[data=2,text=\"a \"string\" with quotes\",value=1]", + "test.JvmTestBenchmark.cosBenchmark", + "test.ParamBenchmark.mathBenchmark\n[data=2,text=\"a \"string\" with quotes\",value=2]", + "test.InheritedBenchmark.baseBenchmark", + "test.JvmTestBenchmark.sqrtBenchmark", + "test.CommonBenchmark.mathBenchmark", + "test.ParamBenchmark.mathBenchmark\n[data=1,text=\"a \"string\" with quotes\",value=1]", + "test.ParamBenchmark.mathBenchmark\n[data=1,text=\"a \"string\" with quotes\",value=2]", + "test.ParamBenchmark.mathBenchmark\n[data=2,text=\"a \"string\" with quotes\",value=1]", + "test.ParamBenchmark.otherBenchmark\n[data=1,text=\"a \"string\" with quotes\",value=1]", + "test.InheritedBenchmark.inheritedBenchmark" + ], + "diffScorePercentage": [ + -9.443531677712787, + -6.278722887168673, + -6.216397528482819, + -5.888492546802327, + -3.5147903591900813, + -3.409665553540104, + -3.109952295437753, + -2.8370825149381784, + -2.6325014301068244, + -2.3160028412665805, + -2.075021911091128, + -1.8519981338583025, + -1.7076281404242712, + -1.4996256250557365, + -1.4687459829456666, + -0.9397526348026837, + -0.5783583335754896, + -0.4977172108975735, + 0.2949456583005684, + 0.4939406332662542 + ], + "barColor": [ + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "pos", + "pos" + ] + }, + "ggsize": { + "width": 800.0, + "height": 900.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "x", + "name": "Diff %", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "y", + "discrete": true, + "name": "" + }, + { + "aesthetic": "fill", + "values": [ + "#ee6666", + "#3ba272" + ], + "limits": [ + "neg", + "pos" + ], + "guide": "none" + } + ], + "layers": [ + { + "mapping": { + "x": "diffScorePercentage", + "y": "testLabel", + "fill": "barColor" + }, + "stat": "identity", + "orientation": "y", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar", + "tooltips": { + "lines": [ + "@|@{diffScorePercentage}" + ], + "formats": [ + { + "field": "diffScorePercentage", + "format": ".2f" + } + ], + "disable_splitting": true + } + } + ], + "theme": { + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "float", + "column": "diffScorePercentage" + }, + { + "type": "str", + "column": "testLabel" + }, + { + "type": "str", + "column": "barColor" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "execution_count": 9, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 9 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:40.787999Z", + "start_time": "2025-10-15T06:52:40.667895Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_41_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Just comparing the score values is a bit simplistic as the benchmark results are actually a range: score +/- error.\n", + "// So, instead of plotting all tests, we want to focus only on the benchmarks that looks \"interesting\". This is\n", + "// defined as any benchmark that differ so much that the benchmark ranges do not overlap, i.e., we no longer just\n", + "// look at only the score but consider the full error range.\n", + "//\n", + "// We still use the \"score\" to calculate the change in percent, but now on a filtered list\n", + "fun kotlin.ranges.ClosedFloatingPointRange.overlaps(other: ClosedFloatingPointRange): Boolean =\n", + " this.start <= other.endInclusive && other.start <= this.endInclusive\n", + "\n", + "val interestingBenchmarks = plotData.filter {\n", + " !it.range.overlaps(it.range1)\n", + "}\n", + "interestingBenchmarks" + ], + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
nameparamsmodeunitscorerangescore1range1diffScorediffScorePercentagetestLabelbarColor
test.ParamBenchmark.mathBenchmarkdata=2,text="a "string" with quotes",...thrptops/ms103082.844856102746.60675391225..103419.08295814389101173.75249399908.94648785368..102438.55849807907-1909.092363-1.851998test.ParamBenchmark.mathBenchmark\n", + "[da...neg
test.ParamBenchmark.textContentCheckdata=1,text="a "string" with quotes",...thrptops/ms155929.815236155267.27942953032..156592.35104171414150449.209123147414.465281937..153483.95296329833-5480.606113-3.514790test.ParamBenchmark.textContentCheck\n", + "...neg
test.ParamBenchmark.textContentCheckdata=1,text="a "string" with quotes",...thrptops/ms156259.919330155967.2972273164..156552.54143332926147058.565627140116.58313969668..154000.5481141382-9201.353703-5.888493test.ParamBenchmark.textContentCheck\n", + "...neg
test.ParamBenchmark.textContentCheckdata=2,text="a "string" with quotes",...thrptops/ms155811.883306154012.99823010628..157610.76838288622146125.997244143504.1570471993..148747.83743989808-9685.886063-6.216398test.ParamBenchmark.textContentCheck\n", + "...neg
test.nested.CommonBenchmark.mathBench...thrptops/ms149758.678427149465.14095246932..150052.21590221935145509.901147143242.89296995336..147776.90932420595-4248.777280-2.837083test.nested.CommonBenchmark.mathBench...neg
test.CommonBenchmark.longBenchmarkavgtms/op0.0008448.372522775198425E-4..8.5116426929504...0.0008978.737790976669418E-4..9.2064844530355...-0.000053-6.278723test.CommonBenchmark.longBenchmarkneg
test.CommonBenchmark.longBlackholeBen...avgtms/op0.0000222.1394152325114E-5..2.190102450243732...0.0000242.3161777788037864E-5..2.422199277814...-0.000002-9.443532test.CommonBenchmark.longBlackholeBen...neg
test.JvmTestBenchmark.cosBenchmarkavgtns/op3.4724853.4596934638924464..3.4852772244583043.5445403.5170213910539556..3.572058960798929-0.072055-2.075022test.JvmTestBenchmark.cosBenchmarkneg
\n", + " \n", + " \n", + " " + ], + "application/kotlindataframe+json": "{\"$version\":\"2.2.0\",\"metadata\":{\"columns\":[\"name\",\"params\",\"mode\",\"unit\",\"score\",\"range\",\"score1\",\"range1\",\"diffScore\",\"diffScorePercentage\",\"testLabel\",\"barColor\"],\"types\":[{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.ranges.ClosedFloatingPointRange\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.ranges.ClosedFloatingPointRange\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.Double\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"},{\"kind\":\"ValueColumn\",\"type\":\"kotlin.String\"}],\"nrow\":8,\"ncol\":12,\"is_formatted\":false},\"kotlin_dataframe\":[{\"name\":\"test.ParamBenchmark.mathBenchmark\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":103082.84485602807,\"range\":\"102746.60675391225..103419.08295814389\",\"score1\":101173.75249296638,\"range1\":\"99908.94648785368..102438.55849807907\",\"diffScore\":-1909.092363061689,\"diffScorePercentage\":-1.8519981338583025,\"testLabel\":\"test.ParamBenchmark.mathBenchmark\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":155929.81523562223,\"range\":\"155267.27942953032..156592.35104171414\",\"score1\":150449.20912261767,\"range1\":\"147414.465281937..153483.95296329833\",\"diffScore\":-5480.606113004556,\"diffScorePercentage\":-3.5147903591900813,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":156259.91933032282,\"range\":\"155967.2972273164..156552.54143332926\",\"score1\":147058.56562691744,\"range1\":\"140116.58313969668..154000.5481141382\",\"diffScore\":-9201.353703405388,\"diffScorePercentage\":-5.888492546802327,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=1,text=\\\"a \\\"string\\\" with quotes\\\",value=2]\",\"barColor\":\"neg\"},{\"name\":\"test.ParamBenchmark.textContentCheck\",\"params\":\"data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":155811.88330649625,\"range\":\"154012.99823010628..157610.76838288622\",\"score1\":146125.99724354869,\"range1\":\"143504.1570471993..148747.83743989808\",\"diffScore\":-9685.886062947568,\"diffScorePercentage\":-6.216397528482819,\"testLabel\":\"test.ParamBenchmark.textContentCheck\\n[data=2,text=\\\"a \\\"string\\\" with quotes\\\",value=1]\",\"barColor\":\"neg\"},{\"name\":\"test.nested.CommonBenchmark.mathBenchmark\",\"params\":\"\",\"mode\":\"thrpt\",\"unit\":\"ops/ms\",\"score\":149758.67842734433,\"range\":\"149465.14095246932..150052.21590221935\",\"score1\":145509.90114707965,\"range1\":\"143242.89296995336..147776.90932420595\",\"diffScore\":-4248.77728026468,\"diffScorePercentage\":-2.8370825149381784,\"testLabel\":\"test.nested.CommonBenchmark.mathBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.CommonBenchmark.longBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ms/op\",\"score\":8.442082734074419E-4,\"range\":\"8.372522775198425E-4..8.511642692950413E-4\",\"score1\":8.972137714852465E-4,\"range1\":\"8.737790976669418E-4..9.206484453035511E-4\",\"diffScore\":-5.3005498077804544E-5,\"diffScorePercentage\":-6.278722887168673,\"testLabel\":\"test.CommonBenchmark.longBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.CommonBenchmark.longBlackholeBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ms/op\",\"score\":2.1647588413775662E-5,\"range\":\"2.1394152325114E-5..2.1901024502437323E-5\",\"score1\":2.369188528309145E-5,\"range1\":\"2.3161777788037864E-5..2.4221992778145036E-5\",\"diffScore\":-2.044296869315788E-6,\"diffScorePercentage\":-9.443531677712787,\"testLabel\":\"test.CommonBenchmark.longBlackholeBenchmark\",\"barColor\":\"neg\"},{\"name\":\"test.JvmTestBenchmark.cosBenchmark\",\"params\":\"\",\"mode\":\"avgt\",\"unit\":\"ns/op\",\"score\":3.472485344175375,\"range\":\"3.4596934638924464..3.485277224458304\",\"score1\":3.5445401759264423,\"range1\":\"3.5170213910539556..3.572058960798929\",\"diffScore\":-0.0720548317510672,\"diffScorePercentage\":-2.075021911091128,\"testLabel\":\"test.JvmTestBenchmark.cosBenchmark\",\"barColor\":\"neg\"}]}" + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 10 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T06:52:41.006960Z", + "start_time": "2025-10-15T06:52:40.830521Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_43_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Now lets plot the interesting benchmarks, similar to before.\n", + "interestingBenchmarks.sortBy { diffScorePercentage }.plot {\n", + " barsH {\n", + " x(diffScorePercentage) {\n", + " axis.name = \"Diff %\"\n", + " }\n", + " y(testLabel) {\n", + " axis.name = \"\"\n", + " }\n", + " fillColor(barColor) {\n", + " scale = categorical(\"neg\" to Color.RED, \"pos\" to Color.GREEN)\n", + " legend.type = LegendType.None\n", + " }\n", + " tooltips {\n", + " line(diffScorePercentage, format = \".2f\")\n", + " }\n", + " }\n", + " layout {\n", + " size = 800 to ((40 * interestingBenchmarks.size().nrow) + 100)\n", + " style {\n", + " global {\n", + " title {\n", + " margin(10.0, 0.0)\n", + " }\n", + " }\n", + " }\n", + " }\n", + "}" + ], + "outputs": [ + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -8\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -6\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -4\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " -2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.CommonBenchmark.longBlackholeBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.CommonBenchmark.longBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.textContentCheck\n", + " \n", + " \n", + " [data=1,text="a "string" with quotes",value=1]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.nested.CommonBenchmark.mathBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.JvmTestBenchmark.cosBenchmark\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " test.ParamBenchmark.mathBenchmark\n", + " \n", + " \n", + " [data=2,text="a "string" with quotes",value=2]\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " Diff %\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "data": { + "testLabel": [ + "test.CommonBenchmark.longBlackholeBenchmark", + "test.CommonBenchmark.longBenchmark", + "test.ParamBenchmark.textContentCheck\n[data=2,text=\"a \"string\" with quotes\",value=1]", + "test.ParamBenchmark.textContentCheck\n[data=1,text=\"a \"string\" with quotes\",value=2]", + "test.ParamBenchmark.textContentCheck\n[data=1,text=\"a \"string\" with quotes\",value=1]", + "test.nested.CommonBenchmark.mathBenchmark", + "test.JvmTestBenchmark.cosBenchmark", + "test.ParamBenchmark.mathBenchmark\n[data=2,text=\"a \"string\" with quotes\",value=2]" + ], + "diffScorePercentage": [ + -9.443531677712787, + -6.278722887168673, + -6.216397528482819, + -5.888492546802327, + -3.5147903591900813, + -2.8370825149381784, + -2.075021911091128, + -1.8519981338583025 + ], + "barColor": [ + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg", + "neg" + ] + }, + "ggsize": { + "width": 800.0, + "height": 420.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "x", + "name": "Diff %", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "y", + "discrete": true, + "name": "" + }, + { + "aesthetic": "fill", + "values": [ + "#ee6666", + "#3ba272" + ], + "limits": [ + "neg", + "pos" + ], + "guide": "none" + } + ], + "layers": [ + { + "mapping": { + "x": "diffScorePercentage", + "y": "testLabel", + "fill": "barColor" + }, + "stat": "identity", + "orientation": "y", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar", + "tooltips": { + "lines": [ + "@|@{diffScorePercentage}" + ], + "formats": [ + { + "field": "diffScorePercentage", + "format": ".2f" + } + ], + "disable_splitting": true + } + } + ], + "theme": { + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "float", + "column": "diffScorePercentage" + }, + { + "type": "str", + "column": "testLabel" + }, + { + "type": "str", + "column": "barColor" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "execution_count": 11 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Kotlin", + "language": "kotlin", + "name": "kotlin" + }, + "language_info": { + "name": "kotlin", + "version": "2.2.20", + "mimetype": "text/x-kotlin", + "file_extension": ".kt", + "pygments_lexer": "kotlin", + "codemirror_mode": "text/x-kotlin", + "nbconvert_exporter": "" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/examples/simple-benchmark-analysis.ipynb b/examples/simple-benchmark-analysis.ipynb new file mode 100644 index 00000000..b67bf5b4 --- /dev/null +++ b/examples/simple-benchmark-analysis.ipynb @@ -0,0 +1,4868 @@ +{ + "cells": [ + { + "metadata": { + "collapsed": true + }, + "cell_type": "markdown", + "source": [ + "# Simple Benchmark Analysis\n", + "This notebook demonstrates how you can analyze and plot benchmark results from a single benchmark run.\n", + "Several projects exist in the `examples` folder, but this notebook assumes we are working on the\n", + "JVM part of the `kotlin-multiplatform` project. But the same approach can be used for the other projects.\n", + "\n", + "First, you need to run the benchmark. This can be done by running this command from the root of the project:\n", + "\n", + "```shell\n", + "./gradlew :examples:kotlin-multiplatform:jvmBenchmark\n", + "```\n", + "\n", + "Once it is completed, run this notebook, and it will automatically find the latest result." + ] + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T07:10:42.159154Z", + "start_time": "2025-10-15T07:10:38.291838Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_2_jupyter", + "Line_3_jupyter", + "Line_4_jupyter", + "Line_5_jupyter", + "Line_6_jupyter", + "Line_7_jupyter", + "Line_8_jupyter", + "Line_9_jupyter", + "Line_10_jupyter", + "Line_11_jupyter", + "Line_12_jupyter", + "Line_13_jupyter", + "Line_14_jupyter", + "Line_15_jupyter" + ] + } + }, + "cell_type": "code", + "source": "%use serialization, dataframe, kandy", + "outputs": [], + "execution_count": 1 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T07:10:43.097995Z", + "start_time": "2025-10-15T07:10:42.160589Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_16_jupyter", + "Line_17_jupyter", + "Line_18_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "import java.nio.file.Files\n", + "import java.nio.file.attribute.BasicFileAttributes\n", + "import kotlin.io.path.exists\n", + "import kotlin.io.path.forEachDirectoryEntry\n", + "import kotlin.io.path.isDirectory\n", + "import kotlin.io.path.listDirectoryEntries\n", + "import kotlin.io.path.readText\n", + "\n", + "// Find latest result file, based on the their timestamp.\n", + "val runsDir = notebook.workingDir.resolve(\"kotlin-multiplatform/build/reports/benchmarks/main\")\n", + "val lastRunDir = runsDir.listDirectoryEntries()\n", + " .filter { it.isDirectory() }\n", + " .sortedByDescending { dir -> Files.readAttributes(dir, BasicFileAttributes::class.java).creationTime() }\n", + " .first()\n", + "val outputFile = lastRunDir.resolve(\"jvm.json\")\n", + "val benchmarkData = outputFile.readText().deserializeJson()" + ], + "outputs": [], + "execution_count": 2 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T07:10:43.289848Z", + "start_time": "2025-10-15T07:10:43.099489Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_19_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Helper class for tracking the information we need to use.\n", + "data class Benchmark(val name: String, val params: String, val score: Double, val error: Double, val unit: String)\n", + "\n", + "// Split benchmark results into groups. Generally, each group consist of all tests from one test file,\n", + "// except when it is an parameterized test. In this case, each test (with all its variants) are put\n", + "// in its own group.\n", + "val benchmarkGroups = benchmarkData\n", + " .groupBy {\n", + " if (it.params != null) {\n", + " it.benchmark\n", + " } else {\n", + " it.benchmark.substringBeforeLast(\".\")\n", + " }\n", + " }\n", + " .mapValues { group ->\n", + " val benchmarks = group.value.map { benchmark ->\n", + " // Parameters are specific to each test. `deserializeJson()` will generate the appropriate data classes,\n", + " // but for generic handling of parameters we would need to fallback to reading the JSON. In this case\n", + " // we just handle them through the typed API.\n", + " val paramInfo = benchmark.params?.let { params -> \"data=${params.data},text=\\\"${params.text}\\\",value=${params.value}\"} ?: \"\"\n", + " val name = benchmark.benchmark\n", + " Benchmark(\n", + " name,\n", + " paramInfo,\n", + " benchmark.primaryMetric.score,\n", + " benchmark.primaryMetric.scoreError,\n", + " benchmark.primaryMetric.scoreUnit\n", + " )\n", + " }\n", + " benchmarks.toDataFrame()\n", + " }\n", + "\n", + "// Un-commont this to see the benchmark data as DataFrames\n", + "// benchmarkGroups.forEach {\n", + "// DISPLAY(it.value)\n", + "// }" + ], + "outputs": [], + "execution_count": 3 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T07:10:43.474182Z", + "start_time": "2025-10-15T07:10:43.290338Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_20_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "// Prepare the data frames for plotting by:\n", + "// - Add calculated columns for errorMin / errorMax\n", + "// - Tests with parameters use the parameter values as the label\n", + "// - Tests without paramaters use the test name as the label\n", + "val plotData = benchmarkGroups.mapValues {\n", + " it.value\n", + " .add(\"errorMin\") { it.getValue(\"score\") - it.getValue(\"error\") }\n", + " .add(\"errorMax\") { it.getValue(\"score\") + it.getValue(\"error\") }\n", + " .insert(\"label\") {\n", + " // Re-format the benchmark labels to make them look \"nicer\"\n", + " if (!it.getValue(\"params\").isBlank()) {\n", + " it.getValue(\"params\").replace(\",\", \"\\n\")\n", + " } else {\n", + " it.getValue(\"name\").substringAfterLast(\".\").removeSuffix(\"Benchmark\")\n", + " }\n", + " }.at(0)\n", + " .remove(\"name\", \"params\")\n", + "}" + ], + "outputs": [], + "execution_count": 4 + }, + { + "metadata": { + "ExecuteTime": { + "end_time": "2025-10-15T07:10:44.614236Z", + "start_time": "2025-10-15T07:10:43.477739Z" + }, + "executionRelatedData": { + "compiledClasses": [ + "Line_21_jupyter" + ] + } + }, + "cell_type": "code", + "source": [ + "import org.jetbrains.letsPlot.Geom\n", + "import org.jetbrains.letsPlot.core.spec.plotson.coord\n", + "import org.jetbrains.letsPlot.themes.margin\n", + "\n", + "// Plot each group as a bar plot with the error displayed as error bars.\n", + "// This approach assumes that each group has tests roughly within the same \"scale\".\n", + "// If this is not the case, some plots might look very squished. If this happens,\n", + "// you can play around with using a LOG10 scale or modifying the limits to focus\n", + "// on the changes.\n", + "plotData.forEach { (fileName, dataframe) ->\n", + " val plot = dataframe.plot {\n", + " bars {\n", + " x(\"label\") {\n", + " axis.name = \"\"\n", + " }\n", + " y(\"score\")\n", + " }\n", + " errorBars {\n", + " x(\"label\")\n", + " y(\"score\")\n", + " yMin(\"errorMin\")\n", + " yMax(\"errorMax\")\n", + " }\n", + " coordinatesTransformation = CoordinatesTransformation.cartesianFlipped()\n", + " // y.axis.limits = dataframe.min(\"errorMin\")..dataframe.max(\"errorMax\")\n", + " layout {\n", + " this.yAxisLabel = dataframe.first().getValue(\"unit\")\n", + " style {\n", + " global {\n", + " title {\n", + " margin(10.0, 0.0)\n", + " }\n", + " text {\n", + " fontFamily = FontFamily.MONO\n", + " }\n", + " }\n", + " }\n", + " // Adjust the height of the Kandy plot based on the number of tests.\n", + " size = 800 to ((50 * dataframe.size().nrow) + 100)\n", + " }\n", + " }\n", + " DISPLAY(HTML(\"

$fileName

\"))\n", + " DISPLAY(plot)\n", + "}" + ], + "outputs": [ + { + "data": { + "text/html": [ + "

test.InheritedBenchmark

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 2e+7\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 4e+7\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 6e+7\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 8e+7\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1e+8\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1.2e+8\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1.4e+8\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " base\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " inherited\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ops/s\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ops/s" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 1104972.6706894366, + 1.4634989745609665E8 + ], + "errorMax": [ + 1154281.987920151, + 1.489669832012015E8 + ], + "label": [ + "base", + "inherited" + ], + "errorMin": [ + 1055663.3534587221, + 1.437328117109918E8 + ] + }, + "ggsize": { + "width": 800.0, + "height": 200.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + "

test.ParamBenchmark.mathBenchmark

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 50,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 100,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 150,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 200,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=1\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=1\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=2\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=2\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ops/ms\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ops/ms" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 213019.57841385395, + 214917.7241256119, + 102424.75929129767, + 101173.75249296638 + ], + "errorMax": [ + 221389.10361238578, + 221219.12070789465, + 103736.05191104818, + 102438.55849807907 + ], + "label": [ + "data=1\ntext=\"a \"string\" with quotes\"\nvalue=1", + "data=1\ntext=\"a \"string\" with quotes\"\nvalue=2", + "data=2\ntext=\"a \"string\" with quotes\"\nvalue=1", + "data=2\ntext=\"a \"string\" with quotes\"\nvalue=2" + ], + "errorMin": [ + 204650.0532153221, + 208616.32754332913, + 101113.46667154715, + 99908.94648785368 + ] + }, + "ggsize": { + "width": 800.0, + "height": 300.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + "

test.ParamBenchmark.otherBenchmark

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 500,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1e+6\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1.5e+6\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 2e+6\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 2.5e+6\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=1\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=1\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=2\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=2\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ops/ms\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ops/ms" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 2680506.786160648, + 2585022.6644677366, + 2614856.4118722673, + 2585237.244524414 + ], + "errorMax": [ + 2714141.3425343693, + 2715267.760476164, + 2648971.483481024, + 2618115.998457625 + ], + "label": [ + "data=1\ntext=\"a \"string\" with quotes\"\nvalue=1", + "data=1\ntext=\"a \"string\" with quotes\"\nvalue=2", + "data=2\ntext=\"a \"string\" with quotes\"\nvalue=1", + "data=2\ntext=\"a \"string\" with quotes\"\nvalue=2" + ], + "errorMin": [ + 2646872.2297869264, + 2454777.568459309, + 2580741.3402635106, + 2552358.4905912033 + ] + }, + "ggsize": { + "width": 800.0, + "height": 300.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + "

test.ParamBenchmark.textContentCheck

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 20,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 40,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 60,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 80,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 100,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 120,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 140,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 160,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=1\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=1\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=2\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " data=2\n", + " \n", + " \n", + " text="a "string" with quotes"\n", + " \n", + " \n", + " value=2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ops/ms\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ops/ms" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 150449.20912261767, + 147058.56562691744, + 146125.99724354869, + 150019.93211383233 + ], + "errorMax": [ + 153483.95296329833, + 154000.5481141382, + 148747.83743989808, + 154158.57892833705 + ], + "label": [ + "data=1\ntext=\"a \"string\" with quotes\"\nvalue=1", + "data=1\ntext=\"a \"string\" with quotes\"\nvalue=2", + "data=2\ntext=\"a \"string\" with quotes\"\nvalue=1", + "data=2\ntext=\"a \"string\" with quotes\"\nvalue=2" + ], + "errorMin": [ + 147414.465281937, + 140116.58313969668, + 143504.1570471993, + 145881.2852993276 + ] + }, + "ggsize": { + "width": 800.0, + "height": 300.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + "

test.nested.CommonBenchmark

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 20,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 40,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 60,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 80,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 100,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 120,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 140,000\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " math\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ops/ms\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ops/ms" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 145509.90114707965 + ], + "errorMax": [ + 147776.90932420595 + ], + "label": [ + "math" + ], + "errorMin": [ + 143242.89296995336 + ] + }, + "ggsize": { + "width": 800.0, + "height": 150.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + "

test.CommonBenchmark

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0001\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0002\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0003\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0004\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0005\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0006\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0007\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0008\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.0009\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " long\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " longBlackhole\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " math\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ms/op\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ms/op" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 8.972137714852465E-4, + 2.369188528309145E-5, + 6.7916806977233165E-6 + ], + "errorMax": [ + 9.206484453035511E-4, + 2.4221992778145036E-5, + 6.8730920645926235E-6 + ], + "label": [ + "long", + "longBlackhole", + "math" + ], + "errorMin": [ + 8.737790976669418E-4, + 2.3161777788037864E-5, + 6.710269330854009E-6 + ] + }, + "ggsize": { + "width": 800.0, + "height": 250.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + "

test.JvmTestBenchmark

" + ] + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + }, + { + "data": { + "text/html": [ + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 0.5\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 1.5\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 2\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 2.5\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 3\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " 3.5\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " cos\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " sqrt\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " ns/op\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "\n", + " " + ], + "application/plot+json": { + "output_type": "lets_plot_spec", + "output": { + "mapping": {}, + "guides": { + "y": { + "title": "ns/op" + } + }, + "coord": { + "name": "flip", + "flip": true + }, + "data": { + "score": [ + 3.5445401759264423, + 0.5428283650466172 + ], + "errorMax": [ + 3.572058960798929, + 0.5468378157356791 + ], + "label": [ + "cos", + "sqrt" + ], + "errorMin": [ + 3.5170213910539556, + 0.5388189143575554 + ] + }, + "ggsize": { + "width": 800.0, + "height": 200.0 + }, + "kind": "plot", + "scales": [ + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true, + "name": "" + }, + { + "aesthetic": "y", + "limits": [ + null, + null + ] + }, + { + "aesthetic": "x", + "discrete": true + } + ], + "layers": [ + { + "mapping": { + "y": "score", + "x": "label" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "bar" + }, + { + "mapping": { + "y": "score", + "x": "label", + "ymin": "errorMin", + "ymax": "errorMax" + }, + "stat": "identity", + "sampling": "none", + "inherit_aes": false, + "position": "dodge", + "geom": "errorbar" + } + ], + "theme": { + "text": { + "family": "mono", + "blank": false + }, + "title": { + "margin": [ + 10.0, + 0.0, + 10.0, + 0.0 + ], + "blank": false + }, + "axis_ontop": false, + "axis_ontop_y": false, + "axis_ontop_x": false + }, + "data_meta": { + "series_annotations": [ + { + "type": "str", + "column": "label" + }, + { + "type": "float", + "column": "score" + }, + { + "type": "float", + "column": "errorMin" + }, + { + "type": "float", + "column": "errorMax" + } + ] + } + }, + "apply_color_scheme": true, + "swing_enabled": true + } + }, + "metadata": {}, + "output_type": "display_data", + "jetTransient": { + "display_id": null + } + } + ], + "execution_count": 5 + } + ], + "metadata": { + "kernelspec": { + "display_name": "Kotlin", + "language": "kotlin", + "name": "kotlin" + }, + "language_info": { + "name": "kotlin", + "version": "2.2.20", + "mimetype": "text/x-kotlin", + "file_extension": ".kt", + "pygments_lexer": "kotlin", + "codemirror_mode": "text/x-kotlin", + "nbconvert_exporter": "" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +}