Benchmarks base (#29)

* improve grammar DSL (add initialization in constructor)
* replace intersection logic into other objects (intersection engine) to separate input and recovery
* replace Java7 sources in examples subproject
* update bench_run.sh script for independent generation and running

.github/workflows/build.yml

@@ -18,15 +18,17 @@ jobs:
           java-version: '11'
           distribution: 'temurin'
 
-      - name: Generate JFlex files
+      - name: Install JFlex
         run: |
           sudo apt-get install -y jflex
-          ./benchmarks/generate_lexers.sh
 
-      - name: Generate ANTLR4 files
+      - name: Install ANTLR4 files
         run: |
           sudo apt-get install antlr4
-          ./benchmarks/generate_antlr_classes.sh
+
+      - name: Generate files
+        run: |
+          ./scripts/generate_all.sh
 
       - name: Build with Gradle
         uses: gradle/gradle-build-action@bd5760595778326ba7f1441bcf7e88b49de61a25 # v2.6.0

.gitignore

@@ -52,8 +52,6 @@ out/
 hs_err_pid*
 
 .gradle
-build/
-out/
 !gradle/wrapper/gradle-wrapper.jar
 !**/src/main/**/build/
 !**/src/test/**/build/
@@ -88,3 +86,9 @@ bin/
 
 ### Generated files ###
 /gen/
+/benchmarks/src/main/java/org/antlr/*
+!/benchmarks/src/main/java/org/antlr/Java8.g4
+/benchmarks/src/main/kotlin/org/ucfs/*
+/benchmarks/src/main/java/org/ucfs/*
+/benchmarks/logs/
+/examples/src/main/java/java7/JavaLexer.java

benchmarks/README.md

@@ -0,0 +1,19 @@
+# UCFSBenchmarks
+
+## Prerequisites
+
+```text
+(1) Gradle (version >= 7.2)
+(2) Antlr V4
+(3) Jflex
+```
+## Generate files
+Run `/scripts/generate_all.sh`
+## Run benchmarks
+Set data-set folder and parser mode in script `/scripts/run_bench.sh` and run it
+
+## Logging
+
+Logs are stored in `logs`
+
+Results are stored in  `build/reports/benchmarks`

benchmarks/build.gradle.kts

@@ -1,7 +1,7 @@
 plugins {
     java
     kotlin("jvm") version "1.9.20"
-    id("me.champeau.jmh") version "0.7.2"
+    id("org.jetbrains.kotlinx.benchmark") version "0.4.10"
     kotlin("plugin.allopen") version "1.9.20"
 }
 
@@ -11,42 +11,64 @@ repositories {
 }
 
 dependencies {
+    //benchmarks tool
+    testImplementation("org.jetbrains.kotlin:kotlin-test")
+    implementation("org.jetbrains.kotlinx:kotlinx-benchmark-runtime:0.4.10")
+    //compared projects
+    // 1. for ucfs
     implementation(project(":solver"))
+    implementation(project(":generator"))
+    implementation(project(":examples"))
+    // 2. for java_cup (?)
     implementation("java_cup:java_cup:0.9e")
+    // 3. for antlr
     implementation("org.antlr:antlr4:4.13.1")
-    jmhImplementation("org.openjdk.jmh:jmh-core:1.36")
-    jmhImplementation("org.openjdk.jmh:jmh-generator-annprocess:1.36")
-    jmhImplementation("org.openjdk.jmh:jmh-generator-bytecode:1.36")
+    // 4. for iguana
+    implementation("io.usethesource:capsule:0.6.3")
+    implementation("info.picocli:picocli:4.7.0")
+    implementation("com.google.guava:guava-testlib:23.0")
+    implementation("com.fasterxml.jackson.core:jackson-core:2.14.0")
+    implementation("com.fasterxml.jackson.core:jackson-databind:2.14.0")
 }
-kotlin { jvmToolchain(11) }
 
-configure<SourceSetContainer> {
-    named("jmh") {
-        kotlin.srcDir("benchmarks/src/jmh/kotlin")
-        resources.srcDir("benchmarks/src/jmh/resources")
+fun getArgs(strFolder: String): Array<String> {
+    val resourcesDir = File(strFolder)
+    val files = resourcesDir.listFiles()!!
+    return files.map { it.toString() }.sorted().toTypedArray()
+}
+
+benchmark {
+    configurations {
+        named("main") {
+            val dataset = "dataset"
+            if (!hasProperty(dataset)) {
+                println("BENCHMARKS FAILED! Set dataset folder by property '$dataset'")
+            }
+            else{
+                param("fileName", *getArgs(property(dataset).toString()))
+            }
+            this.reportFormat = "csv"
+            iterations = 15
+            iterationTime = 1000
+            iterationTimeUnit = "ms"
+            warmups = 5
+            outputTimeUnit = "ms"
+            mode = "avgt"
+            val tools = "toolName"
+            if (hasProperty(tools)) {
+                println("Run benchmarks for: .*${property(tools)}.*")
+                include(".*${property(tools)}.*")
+            }
+
+        }
+    }
+    targets {
+        register("main")
     }
 }
 
-jmh {
-    duplicateClassesStrategy = DuplicatesStrategy.EXCLUDE
-    zip64 = true
-    warmupForks = 0
-    warmupBatchSize = 1
-    warmupIterations = 5
-    warmup = "0s"
-    timeOnIteration = "0s"
-    fork = 1
-    batchSize = 1
-    iterations = 15
-    verbosity = "EXTRA"
-    jmhTimeout = "300s"
-    benchmarkMode.addAll("ss")
-    failOnError = false
-    forceGC = true
-    resultFormat = "CSV"
-    jvmArgs.addAll("-Xmx4096m", "-Xss4m", "-XX:+UseG1GC")
+allOpen {
+    annotation("org.openjdk.jmh.annotations.State")
 }
 
-tasks.processJmhResources {
-    duplicatesStrategy = DuplicatesStrategy.EXCLUDE
-}
+kotlin { jvmToolchain(11) }

benchmarks/gradle.properties

@@ -0,0 +1,3 @@
+kotlin.code.style=official
+org.gradle.daemon=true
+org.gradle.jvmargs=-Xmx4096m -Xss4m -XX:+UseG1GC

benchmarks/gradle/wrapper/gradle-wrapper.properties

@@ -0,0 +1,5 @@
+distributionBase=GRADLE_USER_HOME
+distributionPath=wrapper/dists
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.2-bin.zip
+zipStoreBase=GRADLE_USER_HOME
+zipStorePath=wrapper/dists