From d2c4b8ebcf0925fefa61a1c97d4e36bbbf69c8fa Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Mon, 18 Aug 2025 10:58:07 -0400
Subject: [PATCH 01/12] v0

---
 benchmark/Dockerfile    | 58 +++++++++++++++++++++++++++++++++++++++++
 benchmark/README.md     | 29 +++++++++++++++++++++
 benchmark/benchmarks.sh | 41 +++++++++++++++++++++++++++++
 benchmark/run.sh        | 43 ++++++++++++++++++++++++++++++
 4 files changed, 171 insertions(+)
 create mode 100644 benchmark/Dockerfile
 create mode 100644 benchmark/README.md
 create mode 100755 benchmark/benchmarks.sh
 create mode 100755 benchmark/run.sh

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
new file mode 100644
index 00000000..0ab6ba8e
--- /dev/null
+++ b/benchmark/Dockerfile
@@ -0,0 +1,58 @@
+# Dacapo download
+FROM debian:bookworm-slim as dacapo
+RUN apt-get update \
+  && apt-get -y install wget unzip \
+  && apt-get -y clean \
+  && rm -rf /var/lib/apt/lists/*
+
+ARG DACAPO_VERSION=23.11-chopin
+# The data for the big benchmarks is removed too ensure the final docker image is not too big
+RUN wget -nv -O dacapo.zip https://download.dacapobench.org/chopin/dacapo-$DACAPO_VERSION.zip \
+	&& mkdir /dacapo \
+	&& unzip dacapo.zip -d /dacapo/ \
+	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/luindex \
+	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/lusearch \
+	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/graphchi \
+	&& rm dacapo.zip
+
+FROM debian:bookworm-slim
+
+RUN apt-get update \
+  && apt-get -y install git curl wget procps gettext-base \
+  && apt-get -y clean \
+  && rm -rf /var/lib/apt/lists/*
+
+COPY --from=eclipse-temurin:8-jammy /opt/java/openjdk /usr/lib/jvm/8
+COPY --from=eclipse-temurin:11-jammy /opt/java/openjdk /usr/lib/jvm/11
+COPY --from=eclipse-temurin:17-jammy /opt/java/openjdk /usr/lib/jvm/17
+
+RUN rm -rf \
+    /usr/lib/jvm/*/man \
+    /usr/lib/jvm/*/src.zip \
+    /usr/lib/jvm/*/lib/src.zip \
+    /usr/lib/jvm/*/demo \
+    /usr/lib/jvm/*/sample
+
+ENV JAVA_8_HOME=/usr/lib/jvm/8
+ENV JAVA_11_HOME=/usr/lib/jvm/11
+ENV JAVA_17_HOME=/usr/lib/jvm/17
+ENV JAVA_HOME=${JAVA_8_HOME}
+ENV PATH=${PATH}:${JAVA_HOME}/bin
+
+ARG SIRUN_VERSION=0.1.11
+RUN wget -O sirun.tar.gz https://github.com/DataDog/sirun/releases/download/v$SIRUN_VERSION/sirun-v$SIRUN_VERSION-x86_64-unknown-linux-musl.tar.gz \
+	&& tar -xzf sirun.tar.gz \
+	&& rm sirun.tar.gz \
+  && mv sirun /usr/bin/sirun
+
+ARG K6_VERSION=0.45.1
+RUN wget -O k6.tar.gz https://github.com/grafana/k6/releases/download/v$K6_VERSION/k6-v$K6_VERSION-linux-amd64.tar.gz \
+	&& tar --strip-components=1 -xzf k6.tar.gz \
+	&& rm k6.tar.gz \
+  && mv k6 /usr/bin/k6
+
+RUN mkdir -p /app
+
+COPY --from=dacapo /dacapo/ /app/
+ARG DACAPO_VERSION=23.11-chopin
+ENV DACAPO=/app/dacapo-$DACAPO_VERSION.jar
diff --git a/benchmark/README.md b/benchmark/README.md
new file mode 100644
index 00000000..30f3bbcf
--- /dev/null
+++ b/benchmark/README.md
@@ -0,0 +1,29 @@
+# Benchmarks
+
+This directory contains different types of benchmarks.
+
+## Running Benchmarks via Docker
+
+Docker allows the execution of benchmarks without needing to install and configure your development environment. For example, package installation and installation of sirun are performed automatically.
+
+In order to run benchmarks using Docker, issue the following command from the `benchmark/` folder of this project:
+
+```sh
+./run.sh
+```
+
+If you run into storage errors (e.g. running out of disk space), try removing all unused Docker containers, networks, and images with `docker system prune -af` before running the script again. Once finished, the reports will be available in the `benchmark/reports/` folder. Note that the script can take ~40 minutes to run.
+
+### Running specific benchmarks
+
+If you want to run only a specific category of benchmarks, you can do so via arguments:
+
+1. Run startup benchmarks
+```sh
+./run.sh startup [application]?
+```
+
+2. Run load benchmarks
+```sh
+./run.sh load [application]?
+```
diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh
new file mode 100755
index 00000000..3bc9d81c
--- /dev/null
+++ b/benchmark/benchmarks.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -eu
+
+readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+export PROFILER_DIR="${SCRIPT_DIR}/.."
+export REPORTS_DIR="${SCRIPT_DIR}/reports"
+export UTILS_DIR="${SCRIPT_DIR}/utils"
+export SHELL_UTILS_DIR="${UTILS_DIR}/shell"
+export K6_UTILS_DIR="${UTILS_DIR}/k6"
+export PROFILER="${SCRIPT_DIR}/profiler/libjavaProfiler.so"
+
+run_benchmarks() {
+  local type=$1
+  if [[ -d "${type}" ]] && [[ -f "${type}/run.sh" ]]; then
+    cd "${type}"
+    ./run.sh "$@"
+    cd "${SCRIPT_DIR}"
+  fi
+}
+
+# Find or rebuild tracer to be used in the benchmarks
+if [[ ! -f "${PROFILER}" ]]; then
+  mkdir -p "${SCRIPT_DIR}/profiler"
+  cd "${PROFILER_DIR}"
+  readonly PROFILER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
+  readonly PROFILER_COMPILED="${PROFILER_DIR}/ddprof-lib/build/native/release/META-INF/native-libs/linux-x64/libjavaProfiler.so"
+  if [[ ! -f "${PROFILER_COMPILED}" ]]; then
+    echo "Tracer not found, starting gradle compile ..."
+    ./gradlew assemble
+  fi
+  cp "${TRACER_COMPILED}" "${PROFILER}"
+  cd "${SCRIPT_DIR}"
+fi
+
+if [[ "$#" == '0' ]]; then
+  for type in 'dacapo'; do
+    run_benchmarks "$type"
+  done
+else
+  run_benchmarks "$@"
+fi
diff --git a/benchmark/run.sh b/benchmark/run.sh
new file mode 100755
index 00000000..bcd3649e
--- /dev/null
+++ b/benchmark/run.sh
@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+set -eu
+
+readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+readonly INITIAL_DIR="$(pwd)"
+readonly TRACER="${SCRIPT_DIR}/tracer/dd-java-agent.jar"
+
+cd "${SCRIPT_DIR}"
+
+# Build container image
+echo "Building base image ..."
+docker build \
+  -t dd-trace-java/benchmark \
+  .
+
+# Find or rebuild tracer to be used in the benchmarks
+if [[ ! -f "${TRACER}" ]]; then
+  mkdir -p "${SCRIPT_DIR}/tracer"
+  cd "${SCRIPT_DIR}/.."
+  readonly TRACER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
+  readonly TRACER_COMPILED="${SCRIPT_DIR}/../dd-java-agent/build/libs/dd-java-agent-${TRACER_VERSION}.jar"
+  if [ ! -f "${TRACER_COMPILED}" ]; then
+    echo "Tracer not found, starting gradle compile ..."
+    ./gradlew assemble
+  fi
+  cp "${TRACER_COMPILED}" "${TRACER}"
+  cd "${SCRIPT_DIR}"
+fi
+
+# Trigger benchmarks
+echo "Running benchmarks ..."
+docker run --rm \
+  -v "${HOME}/.gradle":/home/benchmark/.gradle:delegated \
+  -v "${PWD}/..":/tracer:delegated \
+  -w /tracer/benchmark \
+  -e GRADLE_OPTS="-Dorg.gradle.daemon=false" \
+  --entrypoint=./benchmarks.sh \
+  --name dd-trace-java-benchmark \
+  --cap-add SYS_ADMIN \
+  dd-trace-java/benchmark \
+  "$@"
+
+cd "${INITIAL_DIR}"

From d49ae19e8281fb82a2828cd8202a5149f54f5720 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Mon, 18 Aug 2025 15:16:54 -0400
Subject: [PATCH 02/12] v1

---
 benchmark/Dockerfile    | 2 +-
 benchmark/benchmarks.sh | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
index 0ab6ba8e..1e407539 100644
--- a/benchmark/Dockerfile
+++ b/benchmark/Dockerfile
@@ -1,7 +1,7 @@
 # Dacapo download
 FROM debian:bookworm-slim as dacapo
 RUN apt-get update \
-  && apt-get -y install wget unzip \
+  && apt-get -y install wget unzip libgmock-dev \
   && apt-get -y clean \
   && rm -rf /var/lib/apt/lists/*
 
diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh
index 3bc9d81c..40798344 100755
--- a/benchmark/benchmarks.sh
+++ b/benchmark/benchmarks.sh
@@ -18,17 +18,17 @@ run_benchmarks() {
   fi
 }
 
-# Find or rebuild tracer to be used in the benchmarks
+# Find or rebuild profiler to be used in the benchmarks
 if [[ ! -f "${PROFILER}" ]]; then
   mkdir -p "${SCRIPT_DIR}/profiler"
   cd "${PROFILER_DIR}"
   readonly PROFILER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
   readonly PROFILER_COMPILED="${PROFILER_DIR}/ddprof-lib/build/native/release/META-INF/native-libs/linux-x64/libjavaProfiler.so"
   if [[ ! -f "${PROFILER_COMPILED}" ]]; then
-    echo "Tracer not found, starting gradle compile ..."
-    ./gradlew assemble
+    echo "Profiler not found, starting gradle compile ..."
+    ./gradlew assembleAll
   fi
-  cp "${TRACER_COMPILED}" "${PROFILER}"
+  cp "${PROFILER_COMPILED}" "${PROFILER}"
   cd "${SCRIPT_DIR}"
 fi
 

From 010e55938324f66699c79fe6a04a373a9a286f86 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Mon, 18 Aug 2025 15:55:19 -0400
Subject: [PATCH 03/12] Fix gradle build task

---
 ddprof-lib/benchmarks/build.gradle | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/ddprof-lib/benchmarks/build.gradle b/ddprof-lib/benchmarks/build.gradle
index c6bd1db5..fd3ded88 100644
--- a/ddprof-lib/benchmarks/build.gradle
+++ b/ddprof-lib/benchmarks/build.gradle
@@ -17,7 +17,10 @@ application {
 
 // Include the main library headers
 tasks.withType(CppCompile).configureEach {
+  dependsOn ':ddprof-lib:copyUpstreamFiles'
+
   includes file('../src/main/cpp').toString()
+  includes file('../src/main/cpp-external').toString()
 }
 
 // Add a task to run the benchmark

From a3edb4f550fafbe7a78270fb7cc8e76eed062965 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Mon, 18 Aug 2025 20:53:15 -0400
Subject: [PATCH 04/12] v2

---
 benchmark/benchmarks.sh | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh
index 40798344..c8e7edb3 100755
--- a/benchmark/benchmarks.sh
+++ b/benchmark/benchmarks.sh
@@ -26,7 +26,7 @@ if [[ ! -f "${PROFILER}" ]]; then
   readonly PROFILER_COMPILED="${PROFILER_DIR}/ddprof-lib/build/native/release/META-INF/native-libs/linux-x64/libjavaProfiler.so"
   if [[ ! -f "${PROFILER_COMPILED}" ]]; then
     echo "Profiler not found, starting gradle compile ..."
-    ./gradlew assembleAll
+    ./gradlew assemble
   fi
   cp "${PROFILER_COMPILED}" "${PROFILER}"
   cd "${SCRIPT_DIR}"

From 87b44b955f2067a274387dd4aa7eafd9910fb458 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Mon, 18 Aug 2025 21:09:04 -0400
Subject: [PATCH 05/12] v3

---
 benchmark/run.sh | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/benchmark/run.sh b/benchmark/run.sh
index bcd3649e..3283dba3 100755
--- a/benchmark/run.sh
+++ b/benchmark/run.sh
@@ -3,27 +3,27 @@ set -eu
 
 readonly SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
 readonly INITIAL_DIR="$(pwd)"
-readonly TRACER="${SCRIPT_DIR}/tracer/dd-java-agent.jar"
+export PROFILER="${SCRIPT_DIR}/profiler/libjavaProfiler.so"
 
 cd "${SCRIPT_DIR}"
 
 # Build container image
 echo "Building base image ..."
 docker build \
-  -t dd-trace-java/benchmark \
+  -t profiler-java/benchmark \
   .
 
-# Find or rebuild tracer to be used in the benchmarks
-if [[ ! -f "${TRACER}" ]]; then
+# Find or rebuild profiler to be used in the benchmarks
+if [[ ! -f "${PROFILER}" ]]; then
   mkdir -p "${SCRIPT_DIR}/tracer"
   cd "${SCRIPT_DIR}/.."
-  readonly TRACER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
-  readonly TRACER_COMPILED="${SCRIPT_DIR}/../dd-java-agent/build/libs/dd-java-agent-${TRACER_VERSION}.jar"
+  readonly PROFILER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
+  readonly PROFILER_COMPILED="${SCRIPT_DIR}/../java-profiler//ddprof-lib/build/lib/main/release/linux/x64/libjavaProfiler.so"
   if [ ! -f "${TRACER_COMPILED}" ]; then
-    echo "Tracer not found, starting gradle compile ..."
+    echo "Profiler not found, starting gradle compile ..."
     ./gradlew assemble
   fi
-  cp "${TRACER_COMPILED}" "${TRACER}"
+  cp "${PROFILER_COMPILED}" "${PROFILER}"
   cd "${SCRIPT_DIR}"
 fi
 
@@ -32,12 +32,12 @@ echo "Running benchmarks ..."
 docker run --rm \
   -v "${HOME}/.gradle":/home/benchmark/.gradle:delegated \
   -v "${PWD}/..":/tracer:delegated \
-  -w /tracer/benchmark \
+  -w /profiler/benchmark \
   -e GRADLE_OPTS="-Dorg.gradle.daemon=false" \
   --entrypoint=./benchmarks.sh \
-  --name dd-trace-java-benchmark \
+  --name java-profiler-benchmark \
   --cap-add SYS_ADMIN \
-  dd-trace-java/benchmark \
+  java-profiler/benchmark \
   "$@"
 
 cd "${INITIAL_DIR}"

From 8d2bf683408a8c6aec9b7d82a1ddeeedc74f31e0 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Wed, 20 Aug 2025 10:57:33 -0400
Subject: [PATCH 06/12] v4

---
 benchmark/Dockerfile    |  2 +-
 benchmark/benchmarks.sh |  9 ++++++++-
 benchmark/run.sh        | 17 +++++++++++------
 3 files changed, 20 insertions(+), 8 deletions(-)

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
index 1e407539..b307a724 100644
--- a/benchmark/Dockerfile
+++ b/benchmark/Dockerfile
@@ -1,7 +1,7 @@
 # Dacapo download
 FROM debian:bookworm-slim as dacapo
 RUN apt-get update \
-  && apt-get -y install wget unzip libgmock-dev \
+  && apt-get -y install wget unzip libc6 \
   && apt-get -y clean \
   && rm -rf /var/lib/apt/lists/*
 
diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh
index c8e7edb3..e584731b 100755
--- a/benchmark/benchmarks.sh
+++ b/benchmark/benchmarks.sh
@@ -1,6 +1,8 @@
 #!/usr/bin/env bash
 set -eu
 
+echo "Running benchmarks ..."
+
 readonly SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
 export PROFILER_DIR="${SCRIPT_DIR}/.."
 export REPORTS_DIR="${SCRIPT_DIR}/reports"
@@ -22,8 +24,13 @@ run_benchmarks() {
 if [[ ! -f "${PROFILER}" ]]; then
   mkdir -p "${SCRIPT_DIR}/profiler"
   cd "${PROFILER_DIR}"
+  ARCH=$(uname -p)
+  if [ $ARCH eq "x86_64" ]; then
+    ARCH="x64"
+  fi
+
   readonly PROFILER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
-  readonly PROFILER_COMPILED="${PROFILER_DIR}/ddprof-lib/build/native/release/META-INF/native-libs/linux-x64/libjavaProfiler.so"
+  readonly PROFILER_COMPILED="${SCRIPT_DIR}/../ddprof-lib/build/lib/main/release/linux/${ARCH}/libjavaProfiler.so"
   if [[ ! -f "${PROFILER_COMPILED}" ]]; then
     echo "Profiler not found, starting gradle compile ..."
     ./gradlew assemble
diff --git a/benchmark/run.sh b/benchmark/run.sh
index 3283dba3..9d7853c4 100755
--- a/benchmark/run.sh
+++ b/benchmark/run.sh
@@ -10,16 +10,22 @@ cd "${SCRIPT_DIR}"
 # Build container image
 echo "Building base image ..."
 docker build \
-  -t profiler-java/benchmark \
+  -t java-profiler/benchmark \
   .
 
 # Find or rebuild profiler to be used in the benchmarks
 if [[ ! -f "${PROFILER}" ]]; then
-  mkdir -p "${SCRIPT_DIR}/tracer"
+  mkdir -p "${SCRIPT_DIR}/profiler"
   cd "${SCRIPT_DIR}/.."
+
+  ARCH=$(uname -p)
+  if [ $ARCH eq "x86_64" ]; then
+    ARCH="x64"
+  fi
+
   readonly PROFILER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
-  readonly PROFILER_COMPILED="${SCRIPT_DIR}/../java-profiler//ddprof-lib/build/lib/main/release/linux/x64/libjavaProfiler.so"
-  if [ ! -f "${TRACER_COMPILED}" ]; then
+  readonly PROFILER_COMPILED="${SCRIPT_DIR}/../ddprof-lib/build/lib/main/release/linux/${ARCH}/libjavaProfiler.so"
+  if [ ! -f "${PROFILER_COMPILED}" ]; then
     echo "Profiler not found, starting gradle compile ..."
     ./gradlew assemble
   fi
@@ -28,10 +34,9 @@ if [[ ! -f "${PROFILER}" ]]; then
 fi
 
 # Trigger benchmarks
-echo "Running benchmarks ..."
 docker run --rm \
   -v "${HOME}/.gradle":/home/benchmark/.gradle:delegated \
-  -v "${PWD}/..":/tracer:delegated \
+  -v "${PWD}/..":/profiler:delegated \
   -w /profiler/benchmark \
   -e GRADLE_OPTS="-Dorg.gradle.daemon=false" \
   --entrypoint=./benchmarks.sh \

From 8d9737f07e70eed9e41169940faf0af3d7235f89 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Wed, 20 Aug 2025 15:30:09 -0400
Subject: [PATCH 07/12] v5

---
 benchmark/dacapo/benchmark.json         | 21 +++++++++++++
 benchmark/dacapo/run.sh                 | 41 +++++++++++++++++++++++++
 benchmark/run.sh                        |  5 ++-
 benchmark/utils/k6.js                   | 21 +++++++++++++
 benchmark/utils/run-k6-load-test.sh     | 21 +++++++++++++
 benchmark/utils/run-on-server-ready.sh  | 11 +++++++
 benchmark/utils/run-sirun-benchmarks.sh | 41 +++++++++++++++++++++++++
 benchmark/utils/update-java-version.sh  |  5 +++
 8 files changed, 165 insertions(+), 1 deletion(-)
 create mode 100644 benchmark/dacapo/benchmark.json
 create mode 100755 benchmark/dacapo/run.sh
 create mode 100644 benchmark/utils/k6.js
 create mode 100755 benchmark/utils/run-k6-load-test.sh
 create mode 100755 benchmark/utils/run-on-server-ready.sh
 create mode 100755 benchmark/utils/run-sirun-benchmarks.sh
 create mode 100755 benchmark/utils/update-java-version.sh

diff --git a/benchmark/dacapo/benchmark.json b/benchmark/dacapo/benchmark.json
new file mode 100644
index 00000000..9033141c
--- /dev/null
+++ b/benchmark/dacapo/benchmark.json
@@ -0,0 +1,21 @@
+{
+  "name": "dacapo_${BENCHMARK}",
+  "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"",
+  "run": "bash -c \"java ${JAVA_OPTS} -jar ${DACAPO} --converge --scratch-directory=${OUTPUT_DIR}/${VARIANT}/scratch --latency-csv ${BENCHMARK} &> ${OUTPUT_DIR}/${VARIANT}/dacapo.log\"",
+  "timeout": 150,
+  "iterations": 1,
+  "variants": {
+    "baseline": {
+      "env": {
+        "VARIANT": "baseline",
+        "JAVA_OPTS": ""
+      }
+    },
+    "profiling": {
+      "env": {
+        "VARIANT": "profiling",
+        "JAVA_OPTS": "-javapath:${PROFILER} -Ddd.profiling.enabled=true"
+      }
+    }
+  }
+}
diff --git a/benchmark/dacapo/run.sh b/benchmark/dacapo/run.sh
new file mode 100755
index 00000000..ece44f9e
--- /dev/null
+++ b/benchmark/dacapo/run.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -eu
+
+source "${UTILS_DIR}/update-java-version.sh" 11
+
+function message() {
+  echo "$(date +"%T"): $1"
+}
+
+run_benchmark() {
+  local type=$1
+
+  message "dacapo benchmark: ${type} started"
+
+  # export the benchmark
+  export BENCHMARK="${type}"
+
+  # create output folder for the test
+  export OUTPUT_DIR="${REPORTS_DIR}/dacapo/${type}"
+  mkdir -p "${OUTPUT_DIR}"
+
+  # substitute environment variables in the json file
+  benchmark=$(mktemp)
+  # shellcheck disable=SC2046
+  # shellcheck disable=SC2016
+  envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"
+
+  # run the sirun test
+  sirun "${benchmark}" &>"${OUTPUT_DIR}/${type}.json"
+
+  message "dacapo benchmark: ${type} finished"
+}
+
+if [ "$#" == '2' ]; then
+  run_benchmark "$2"
+else
+  for benchmark in biojava tomcat ; do
+    run_benchmark "${benchmark}"
+  done
+fi
+
diff --git a/benchmark/run.sh b/benchmark/run.sh
index 9d7853c4..06ae3e95 100755
--- a/benchmark/run.sh
+++ b/benchmark/run.sh
@@ -19,8 +19,11 @@ if [[ ! -f "${PROFILER}" ]]; then
   cd "${SCRIPT_DIR}/.."
 
   ARCH=$(uname -p)
-  if [ $ARCH eq "x86_64" ]; then
+  if [[ "$ARCH" == "x86_64" ]];
+  then
     ARCH="x64"
+  elif [[ "$ARCH" == "aarch64" ]]
+    ARCH="arm64"
   fi
 
   readonly PROFILER_VERSION=$(./gradlew properties -q | grep "version:" | awk '{print $2}')
diff --git a/benchmark/utils/k6.js b/benchmark/utils/k6.js
new file mode 100644
index 00000000..aa5147ae
--- /dev/null
+++ b/benchmark/utils/k6.js
@@ -0,0 +1,21 @@
+import {check} from 'k6';
+
+export function checkResponse(response) {
+  const checks = Array.prototype.slice.call(arguments, 1);
+  const reduced = checks.reduce((result, current) => Object.assign(result, current), {});
+  check(response, reduced);
+}
+
+export const isOk = {
+  'is OK': r => r.status === 200
+};
+
+export const isRedirect = {
+  'is redirect': r => r.status >= 300 && r.status < 400
+};
+
+export function bodyContains(text) {
+  return {
+    'body contains': r => r.body.includes(text)
+  }
+}
diff --git a/benchmark/utils/run-k6-load-test.sh b/benchmark/utils/run-k6-load-test.sh
new file mode 100755
index 00000000..d3415f54
--- /dev/null
+++ b/benchmark/utils/run-k6-load-test.sh
@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+set -eu
+
+command=$1
+exit_code=0
+
+cleanup() {
+  # run the exit command
+  bash -c "${command}"
+  exit $exit_code
+}
+
+trap cleanup EXIT ERR INT TERM
+
+echo "Starting k6 load test, logs are recorded into ${LOGS_DIR}/k6.log..."
+
+# run the k6 benchmark and store the result as JSON
+k6 run k6.js --out "json=${OUTPUT_DIR}/k6_$(date +%s).json" > "${LOGS_DIR}/k6.log" 2>&1
+exit_code=$?
+
+echo "k6 load test done !!!"
diff --git a/benchmark/utils/run-on-server-ready.sh b/benchmark/utils/run-on-server-ready.sh
new file mode 100755
index 00000000..2aad5aa9
--- /dev/null
+++ b/benchmark/utils/run-on-server-ready.sh
@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+set -eu
+
+url=$1
+command=$2
+# wait for an HTTP server to come up and runs the selected command
+while true; do
+  if [[ $(curl -fso /dev/null -w "%{http_code}" "${url}") = 200 ]]; then
+    bash -c "${command}"
+  fi
+done
diff --git a/benchmark/utils/run-sirun-benchmarks.sh b/benchmark/utils/run-sirun-benchmarks.sh
new file mode 100755
index 00000000..c0bc732d
--- /dev/null
+++ b/benchmark/utils/run-sirun-benchmarks.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -eu
+
+function message() {
+  echo "$(date +"%T"): $1"
+}
+
+run_benchmark() {
+  local type=$1
+  local app=$2
+  if [[ -d "${app}" ]] && [[ -f "${app}/benchmark.json" ]]; then
+
+    message "${type} benchmark: ${app} started"
+    cd "${app}"
+
+    # create output folder for the test
+    export OUTPUT_DIR="${REPORTS_DIR}/${type}/${app}"
+    mkdir -p "${OUTPUT_DIR}"
+
+    # substitute environment variables in the json file
+    benchmark=$(mktemp)
+    # shellcheck disable=SC2046
+    # shellcheck disable=SC2016
+    envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"
+
+    # run the sirun test
+    sirun "${benchmark}" &>"${OUTPUT_DIR}/${app}.json"
+
+    message "${type} benchmark: ${app} finished"
+
+    cd ..
+  fi
+}
+
+if [ "$#" == '2' ]; then
+  run_benchmark "$@"
+else
+  for folder in *; do
+    run_benchmark "$1" "${folder}"
+  done
+fi
diff --git a/benchmark/utils/update-java-version.sh b/benchmark/utils/update-java-version.sh
new file mode 100755
index 00000000..3d76603e
--- /dev/null
+++ b/benchmark/utils/update-java-version.sh
@@ -0,0 +1,5 @@
+readonly target=$1
+readonly NEW_PATH=$(echo "${PATH}" | sed -e "s@/usr/lib/jvm/[[:digit:]]\+@/usr/lib/jvm/${target}@g")
+export PATH="${NEW_PATH}"
+
+java --version

From 036341e47b50a5c64380dbc0600fd429cb79a663 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Wed, 20 Aug 2025 16:56:45 -0400
Subject: [PATCH 08/12] v6

---
 benchmark/dacapo/benchmark.json | 2 +-
 benchmark/run.sh                | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/benchmark/dacapo/benchmark.json b/benchmark/dacapo/benchmark.json
index 9033141c..d7d9cdad 100644
--- a/benchmark/dacapo/benchmark.json
+++ b/benchmark/dacapo/benchmark.json
@@ -14,7 +14,7 @@
     "profiling": {
       "env": {
         "VARIANT": "profiling",
-        "JAVA_OPTS": "-javapath:${PROFILER} -Ddd.profiling.enabled=true"
+        "JAVA_OPTS": "-agentpath:${PROFILER}=start,wall=10ms,file=${OUTPUT_DIR}/${VARIANT}/profiler.jfr"
       }
     }
   }
diff --git a/benchmark/run.sh b/benchmark/run.sh
index 06ae3e95..c738b42d 100755
--- a/benchmark/run.sh
+++ b/benchmark/run.sh
@@ -22,7 +22,8 @@ if [[ ! -f "${PROFILER}" ]]; then
   if [[ "$ARCH" == "x86_64" ]];
   then
     ARCH="x64"
-  elif [[ "$ARCH" == "aarch64" ]]
+  elif [[ "$ARCH" == "aarch64" ]];
+  then
     ARCH="arm64"
   fi
 

From e7d6e36f4450c17a5685cbf7dd99253ff0dba643 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Fri, 22 Aug 2025 14:11:25 -0400
Subject: [PATCH 09/12] Download install renassance benchmark

---
 benchmark/Dockerfile | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
index b307a724..e8212a53 100644
--- a/benchmark/Dockerfile
+++ b/benchmark/Dockerfile
@@ -15,6 +15,11 @@ RUN wget -nv -O dacapo.zip https://download.dacapobench.org/chopin/dacapo-$DACAP
 	&& rm -rf /dacapo/dacapo-$DACAPO_VERSION/dat/graphchi \
 	&& rm dacapo.zip
 
+# Download and install renaissance benchmark
+ARG RENAISSANCE_VERSION=0.16.0
+RUN mkdir /renaissance \
+    && wget -nv -O  ./renaissance/renaissance-gpl.jar https://github.com/renaissance-benchmarks/renaissance/releases/download/v${RENAISSANCE_VERSION}/renaissance-gpl-${RENAISSANCE_VERSION}.jar \
+
 FROM debian:bookworm-slim
 
 RUN apt-get update \
@@ -56,3 +61,6 @@ RUN mkdir -p /app
 COPY --from=dacapo /dacapo/ /app/
 ARG DACAPO_VERSION=23.11-chopin
 ENV DACAPO=/app/dacapo-$DACAPO_VERSION.jar
+
+COPY --from=renaissance /renaissance/ /app/
+ENV RENAISSANCE=/app/renaissance.jar

From bcbf11f8ee6fe2e1b1cded2bc0fafcb397a7314b Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Fri, 22 Aug 2025 14:23:12 -0400
Subject: [PATCH 10/12] v7

---
 benchmark/renaissance/benchmark.json | 21 ++++++++++++++
 benchmark/renaissance/run.sh         | 41 ++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 benchmark/renaissance/benchmark.json
 create mode 100755 benchmark/renaissance/run.sh

diff --git a/benchmark/renaissance/benchmark.json b/benchmark/renaissance/benchmark.json
new file mode 100644
index 00000000..fe481d79
--- /dev/null
+++ b/benchmark/renaissance/benchmark.json
@@ -0,0 +1,21 @@
+{
+  "name": "renassance_${BENCHMARK}",
+  "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"",
+  "run": "bash -c \"java ${JAVA_OPTS} -jar ${RENASSANCE} --converge --scratch-base=${OUTPUT_DIR}/${VARIANT}/scratch --json ${OUTPUT_DIR}/${VARIANT}/${BENCHMARK}.json ${BENCHMARK} \"",
+  "timeout": 150,
+  "iterations": 1,
+  "variants": {
+    "baseline": {
+      "env": {
+        "VARIANT": "baseline",
+        "JAVA_OPTS": ""
+      }
+    },
+    "profiling": {
+      "env": {
+        "VARIANT": "profiling",
+        "JAVA_OPTS": "-agentpath:${PROFILER}=start,wall=10ms,file=${OUTPUT_DIR}/${VARIANT}/profiler.jfr"
+      }
+    }
+  }
+}
diff --git a/benchmark/renaissance/run.sh b/benchmark/renaissance/run.sh
new file mode 100755
index 00000000..20871a97
--- /dev/null
+++ b/benchmark/renaissance/run.sh
@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+set -eu
+
+source "${UTILS_DIR}/update-java-version.sh" 11
+
+function message() {
+  echo "$(date +"%T"): $1"
+}
+
+run_benchmark() {
+  local type=$1
+
+  message "renaissance benchmark: ${type} started"
+
+  # export the benchmark
+  export BENCHMARK="${type}"
+
+  # create output folder for the test
+  export OUTPUT_DIR="${REPORTS_DIR}/renaissance/${type}"
+  mkdir -p "${OUTPUT_DIR}"
+
+  # substitute environment variables in the json file
+  benchmark=$(mktemp)
+  # shellcheck disable=SC2046
+  # shellcheck disable=SC2016
+  envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"
+
+  # run the sirun test
+  sirun "${benchmark}" &>"${OUTPUT_DIR}/${type}.json"
+
+  message "renaissance benchmark: ${type} finished"
+}
+
+if [ "$#" == '2' ]; then
+  run_benchmark "$2"
+else
+  for benchmark in akka-uct neo4j-analytics ; do
+    run_benchmark "${benchmark}"
+  done
+fi
+

From 5444fd3309cf785c253cd97a5d552d1507f15dce Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Tue, 2 Sep 2025 09:06:30 -0400
Subject: [PATCH 11/12] v8

---
 benchmark/Dockerfile                 | 8 ++++----
 benchmark/renaissance/benchmark.json | 4 ++--
 2 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
index e8212a53..1c537dc6 100644
--- a/benchmark/Dockerfile
+++ b/benchmark/Dockerfile
@@ -1,5 +1,5 @@
 # Dacapo download
-FROM debian:bookworm-slim as dacapo
+FROM debian:bookworm-slim as benchmarks
 RUN apt-get update \
   && apt-get -y install wget unzip libc6 \
   && apt-get -y clean \
@@ -18,7 +18,7 @@ RUN wget -nv -O dacapo.zip https://download.dacapobench.org/chopin/dacapo-$DACAP
 # Download and install renaissance benchmark
 ARG RENAISSANCE_VERSION=0.16.0
 RUN mkdir /renaissance \
-    && wget -nv -O  ./renaissance/renaissance-gpl.jar https://github.com/renaissance-benchmarks/renaissance/releases/download/v${RENAISSANCE_VERSION}/renaissance-gpl-${RENAISSANCE_VERSION}.jar \
+    && wget -nv -O  ./renaissance/renaissance-gpl.jar https://github.com/renaissance-benchmarks/renaissance/releases/download/v${RENAISSANCE_VERSION}/renaissance-gpl-${RENAISSANCE_VERSION}.jar
 
 FROM debian:bookworm-slim
 
@@ -58,9 +58,9 @@ RUN wget -O k6.tar.gz https://github.com/grafana/k6/releases/download/v$K6_VERSI
 
 RUN mkdir -p /app
 
-COPY --from=dacapo /dacapo/ /app/
+COPY --from=benchmarks /dacapo/ /app/
 ARG DACAPO_VERSION=23.11-chopin
 ENV DACAPO=/app/dacapo-$DACAPO_VERSION.jar
 
-COPY --from=renaissance /renaissance/ /app/
+COPY --from=benchmarks /renaissance/ /app/
 ENV RENAISSANCE=/app/renaissance.jar
diff --git a/benchmark/renaissance/benchmark.json b/benchmark/renaissance/benchmark.json
index fe481d79..d51faa3d 100644
--- a/benchmark/renaissance/benchmark.json
+++ b/benchmark/renaissance/benchmark.json
@@ -1,7 +1,7 @@
 {
   "name": "renassance_${BENCHMARK}",
-  "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}\"",
-  "run": "bash -c \"java ${JAVA_OPTS} -jar ${RENASSANCE} --converge --scratch-base=${OUTPUT_DIR}/${VARIANT}/scratch --json ${OUTPUT_DIR}/${VARIANT}/${BENCHMARK}.json ${BENCHMARK} \"",
+  "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}/scratch\"",
+  "run": "bash -c \"java ${JAVA_OPTS} -jar ${RENAISSANCE} -scratch-base=${OUTPUT_DIR}/${VARIANT}/scratch --json ${OUTPUT_DIR}/${VARIANT}/${BENCHMARK}.json ${BENCHMARK} \"",
   "timeout": 150,
   "iterations": 1,
   "variants": {

From 6b26491d04bb177bbf25c2eb90e7d6c9965296b2 Mon Sep 17 00:00:00 2001
From: Zhengyu Gu <zhengyu.gu@datadoghq.com>
Date: Wed, 3 Sep 2025 15:52:30 -0400
Subject: [PATCH 12/12] v9

---
 benchmark/Dockerfile                 | 2 +-
 benchmark/benchmarks.sh              | 2 +-
 benchmark/renaissance/benchmark.json | 2 +-
 benchmark/renaissance/run.sh         | 3 +--
 4 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/benchmark/Dockerfile b/benchmark/Dockerfile
index 1c537dc6..52ab7a92 100644
--- a/benchmark/Dockerfile
+++ b/benchmark/Dockerfile
@@ -63,4 +63,4 @@ ARG DACAPO_VERSION=23.11-chopin
 ENV DACAPO=/app/dacapo-$DACAPO_VERSION.jar
 
 COPY --from=benchmarks /renaissance/ /app/
-ENV RENAISSANCE=/app/renaissance.jar
+ENV RENAISSANCE=/app/renaissance-gpl.jar
diff --git a/benchmark/benchmarks.sh b/benchmark/benchmarks.sh
index e584731b..09e00b6f 100755
--- a/benchmark/benchmarks.sh
+++ b/benchmark/benchmarks.sh
@@ -40,7 +40,7 @@ if [[ ! -f "${PROFILER}" ]]; then
 fi
 
 if [[ "$#" == '0' ]]; then
-  for type in 'dacapo'; do
+  for type in 'dacapo' 'renaissance'; do
     run_benchmarks "$type"
   done
 else
diff --git a/benchmark/renaissance/benchmark.json b/benchmark/renaissance/benchmark.json
index d51faa3d..77d92d60 100644
--- a/benchmark/renaissance/benchmark.json
+++ b/benchmark/renaissance/benchmark.json
@@ -1,7 +1,7 @@
 {
   "name": "renassance_${BENCHMARK}",
   "setup": "bash -c \"mkdir -p ${OUTPUT_DIR}/${VARIANT}/scratch\"",
-  "run": "bash -c \"java ${JAVA_OPTS} -jar ${RENAISSANCE} -scratch-base=${OUTPUT_DIR}/${VARIANT}/scratch --json ${OUTPUT_DIR}/${VARIANT}/${BENCHMARK}.json ${BENCHMARK} \"",
+  "run": "bash -c \"java ${JAVA_OPTS} -jar ${RENAISSANCE} --scratch-base=${OUTPUT_DIR}/${VARIANT}/scratch --json ${OUTPUT_DIR}/${VARIANT}/${BENCHMARK}.json ${BENCHMARK} \"",
   "timeout": 150,
   "iterations": 1,
   "variants": {
diff --git a/benchmark/renaissance/run.sh b/benchmark/renaissance/run.sh
index 20871a97..5f095d64 100755
--- a/benchmark/renaissance/run.sh
+++ b/benchmark/renaissance/run.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -eu
 
-source "${UTILS_DIR}/update-java-version.sh" 11
+source "${UTILS_DIR}/update-java-version.sh" 17
 
 function message() {
   echo "$(date +"%T"): $1"
@@ -21,7 +21,6 @@ run_benchmark() {
 
   # substitute environment variables in the json file
   benchmark=$(mktemp)
-  # shellcheck disable=SC2046
   # shellcheck disable=SC2016
   envsubst "$(printf '${%s} ' $(env | cut -d'=' -f1))" <benchmark.json >"${benchmark}"