Skip to content
Permalink
Browse files

Introduce benchmarks-runner project to ease benchmark running

  • Loading branch information...
qurbonzoda committed Jul 16, 2019
1 parent 18b7c32 commit 83f67fa92fa1e55c78493fa52bde8a6b03d7bd3a
@@ -5,3 +5,5 @@
target
build
/kotlinx-collections-immutable/dependency-reduced-pom.xml
/benchmarks-runner/benchmarkResults
/benchmarks-runner/localReferenceBenchmarkResults
@@ -111,7 +111,7 @@ collection.mutate { some_actions_on(it) }
The library is published to [kotlinx](https://bintray.com/kotlin/kotlinx/kotlinx.collections.immutable) bintray repository.
The library depends on the Kotlin Standard Library of the version at least `1.3.30`.
The library depends on the Kotlin Standard Library of the version at least `1.3.40`.
### Maven
@@ -1,11 +1,13 @@
plugins {
id 'org.jetbrains.kotlin.multiplatform' version '1.3.31'
id 'org.jetbrains.gradle.benchmarks.plugin' version '0.1.7-dev-21'
id 'org.jetbrains.kotlin.multiplatform' version '1.3.40'
id 'kotlinx.benchmark' version '0.2.0-dev-2'
}

repositories {
mavenCentral()
maven { url 'https://dl.bintray.com/orangy/maven' }
maven { url 'https://dl.bintray.com/kotlin/kotlinx' }
maven { url 'https://dl.bintray.com/kotlin/kotlin-eap' }
}

apply plugin: 'maven-publish'
@@ -22,7 +24,7 @@ kotlin {
dependencies {
implementation kotlin('stdlib-jdk8')
implementation project(path: ':kotlinx-collections-immutable')
implementation 'org.jetbrains.gradle.benchmarks:runtime-jvm:0.1.7-dev-21'
implementation 'org.jetbrains.kotlinx:kotlinx.benchmark.runtime-jvm:0.2.0-dev-2'
}
}
jvmTest {
@@ -31,9 +33,19 @@ kotlin {
}

benchmark {
configurations {
targets {
register("jvm") {
jmhVersion = "1.21"
}
}
}

configurations {
benchmarksJar
}

afterEvaluate {
artifacts {
benchmarksJar jvmBenchmarkJar
}
}
@@ -0,0 +1,115 @@
plugins {
id 'org.jetbrains.kotlin.jvm' version '1.3.40'
}

repositories {
mavenCentral()
jcenter()
maven { url 'https://dl.bintray.com/orangy/maven' }
}

dependencies {
implementation "org.jetbrains.kotlin:kotlin-stdlib"
implementation 'org.openjdk.jmh:jmh-core:1.21'

runtimeOnly project(path: ':benchmarks-mpp', configuration: 'benchmarksJar')
runtimeOnly project(path: ':kotlinx-collections-immutable')
}

// map
task benchmarkHashMap(type: JavaExec, group: "Benchmark") {
main = 'runners.HashMapRunnerKt'
}

task benchmarkHashMapBuilder(type: JavaExec, group: "Benchmark") {
main = 'runners.HashMapBuilderRunnerKt'
}

task benchmarkOrderedMap(type: JavaExec, group: "Benchmark") {
main = 'runners.OrderedMapRunnerKt'
}

task benchmarkOrderedMapBuilder(type: JavaExec, group: "Benchmark") {
main = 'runners.OrderedMapBuilderRunnerKt'
}

task benchmarkAllMaps(group: "Benchmark") {
dependsOn benchmarkHashMap
dependsOn benchmarkHashMapBuilder
dependsOn benchmarkOrderedMap
dependsOn benchmarkOrderedMapBuilder
}

// set
task benchmarkHashSet(type: JavaExec, group: "Benchmark") {
main = 'runners.HashSetRunnerKt'
}

task benchmarkHashSetBuilder(type: JavaExec, group: "Benchmark") {
main = 'runners.HashSetBuilderRunnerKt'
}

task benchmarkOrderedSet(type: JavaExec, group: "Benchmark") {
main = 'runners.OrderedSetRunnerKt'
}

task benchmarkOrderedSetBuilder(type: JavaExec, group: "Benchmark") {
main = 'runners.OrderedSetBuilderRunnerKt'
}

task benchmarkAllSets(group: "Benchmark") {
dependsOn benchmarkHashSet
dependsOn benchmarkHashSetBuilder
dependsOn benchmarkOrderedSet
dependsOn benchmarkOrderedSetBuilder
}

// list
task benchmarkList(type: JavaExec, group: "Benchmark") {
main = 'runners.ListRunnerKt'
}

task benchmarkListBuilder(type: JavaExec, group: "Benchmark") {
main = 'runners.ListBuilderRunnerKt'
}

task benchmarkAllLists(group: "Benchmark") {
dependsOn benchmarkList
dependsOn benchmarkListBuilder
}

// all
task benchmarkAll(group: "Benchmark") {
dependsOn benchmarkAllMaps
dependsOn benchmarkAllSets
dependsOn benchmarkAllLists
}


// configure runner tasks

def benchmarkParams = [
'remote',
'forks',
'measurementIterations',
'measurementTime',
'warmupIterations',
'warmupTime',
// 'exclude',
// 'include',
'size',
'hashCodeType',
'immutablePercentage'
]

tasks.withType(JavaExec) {
if (group == "Benchmark") {
classpath = sourceSets.main.runtimeClasspath

benchmarkParams.forEach { param ->
if (project.hasProperty(param)) {
systemProperty(param, project.property(param))
}
}
}
}
@@ -0,0 +1,63 @@
/*
* Copyright 2016-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import org.openjdk.jmh.runner.options.TimeValue


const val localReferenceBenchmarkResultsDirectory = "localReferenceBenchmarkResults"
const val remoteReferenceBenchmarkResultsDirectory = "remoteReferenceBenchmarkResults"
const val benchmarkResultsDirectory = "benchmarkResults"

const val hashMapOutputFileName = "hashMap"
const val hashMapBuilderOutputFileName = "hashMapBuilder"
const val orderedMapOutputFileName = "orderedMap"
const val orderedMapBuilderOutputFileName = "orderedMapBuilder"

const val hashSetOutputFileName = "hashSet"
const val hashSetBuilderOutputFileName = "hashSetBuilder"
const val orderedSetOutputFileName = "orderedSet"
const val orderedSetBuilderOutputFileName = "orderedSetBuilder"

const val listOutputFileName = "list"
const val listBuilderOutputFileName = "listBuilder"


const val benchmarkMethod = "Benchmark"
const val benchmarkScore = "Score(ns/op)"
const val benchmarkScoreError = "ScoreError(ns/op)"
const val benchmarkAllocRate = "AllocRate(B/op)"

const val benchmarkScoreRegressPercent = "Score(%)"
const val benchmarkAllocRateRegressPercent = "AllocRate(%)"


const val sizeParam = "size"
const val hashCodeTypeParam = "hashCodeType"
const val implementationParam = "implementation"
const val immutablePercentageParam = "immutablePercentage"


val jvmArgs = arrayOf("-Xms2048m", "-Xmx2048m")

const val forks = 1
const val warmupIterations = 10
const val measurementIterations = 20
val warmupTime = TimeValue.milliseconds(500)!!
val measurementTime = TimeValue.milliseconds(1000)!!

val sizeParamValues = arrayOf("1", "10", "100", "1000", "10000", "100000", "1000000")
val hashCodeTypeParamValues = arrayOf("ascending", "random", "collision", "nonExisting")
val immutablePercentageParamValues = arrayOf("0.0", "20.0", "50.0", "90.0")
@@ -0,0 +1,74 @@
/*
* Copyright 2016-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import java.io.File
import java.io.FileReader
import java.io.FileWriter


fun printCsvResults(benchmarkResults: BenchmarkResults, outputPath: String) {
val paramsNamesString = benchmarkResults.paramsNames.joinToString(",")
val csvHeader = "$benchmarkMethod,$paramsNamesString,$benchmarkScore,$benchmarkScoreError,$benchmarkAllocRate"

File(outputPath).parentFile?.mkdirs()
val fileWriter = FileWriter(outputPath)

fileWriter.appendln(csvHeader)
benchmarkResults.runResults.forEach { res ->
val paramsValuesString = benchmarkResults.paramsNames.joinToString(",") { res.paramValue(it) }
val csvRow = "${res.benchmark},$paramsValuesString,${res.score.formatted()},${res.scoreError.formatted()},${res.allocRate.formatted()}"
fileWriter.appendln(csvRow)
}

fileWriter.flush()
fileWriter.close()
}


fun readCsvResults(file: File): BenchmarkResults {
val fileReader = FileReader(file)
val lines = fileReader.readLines().map { it.split(',') }
fileReader.close()

check(lines.isNotEmpty())
check(lines.all { it.size == lines.first().size })

val header = lines.first()

val benchmarkColumn = header.indexOf(benchmarkMethod)
val scoreColumn = header.indexOf(benchmarkScore)
val scoreErrorColumn = header.indexOf(benchmarkScoreError)
val allocRateColumn = header.indexOf(benchmarkAllocRate)

val paramsColumns = header.indices.filter {
it !in listOf(benchmarkColumn, scoreColumn, scoreErrorColumn, allocRateColumn)
}

val runResults = lines.drop(1).map { line ->
BenchmarkRunResult(
benchmark = line[benchmarkColumn],
params = paramsColumns.associate { header[it] to line[it] },
score = line[scoreColumn].toDouble(),
scoreError = line[scoreErrorColumn].toDouble(),
allocRate = line[allocRateColumn].toDouble()
)
}

return BenchmarkResults(paramsColumns.map { header[it] }, runResults)
}


fun Double.formatted(): String = "%.3f".format(this)
@@ -0,0 +1,61 @@
/*
* Copyright 2016-2019 JetBrains s.r.o.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

import org.openjdk.jmh.results.RunResult


class BenchmarkResults(
val paramsNames: List<String>,
val runResults: List<BenchmarkRunResult>
)

fun Collection<RunResult>.toBenchmarkResults(): BenchmarkResults {
val paramsNames = first().params.paramsKeys

check(all { it.params.paramsKeys == paramsNames })

return BenchmarkResults(paramsNames.toList(), map(RunResult::toBenchmarkRunResult))
}


class BenchmarkRunResult(
val benchmark: String,
val params: Map<String, String>,
val score: Double,
val scoreError: Double,
val allocRate: Double
) {
fun paramValue(paramName: String): String = params.getValue(paramName)
}

private fun RunResult.toBenchmarkRunResult(): BenchmarkRunResult {
val allocRateLabel = "·gc.alloc.rate.norm"
val allocRate = secondaryResults[allocRateLabel]!!

check(primaryResult.getScoreUnit() == "us/op")
check(allocRate.getScoreUnit() == "B/op")

val nanosInMicros = 1000
val size = params.getParam(sizeParam).toInt()

return BenchmarkRunResult(
benchmark = params.benchmark,
params = params.paramsKeys.associateWith { params.getParam(it) },
score = primaryResult.getScore() * nanosInMicros / size,
scoreError = primaryResult.getScoreError() * nanosInMicros / size,
allocRate = allocRate.getScore() / size
)
}

0 comments on commit 83f67fa

Please sign in to comment.
You can’t perform that action at this time.