Add profiler runs on regression

.gitlab/scripts/analyze-results.sh

@@ -1,11 +1,9 @@
 #!/usr/bin/env bash
 
-# Change threshold for detection of regression
-# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference
-export UNCONFIDENCE_THRESHOLD=2.0
+source ./.gitlab/scripts/config-benchmarks.sh
+INITIAL_DIR=$(pwd)
 
 CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME
-CANDIDATE_SRC="/app/candidate/"
 
 cd "$CANDIDATE_SRC"
 CANDIDATE_COMMIT_SHA=$(git rev-parse --short HEAD)
@@ -20,7 +18,6 @@ benchmark_analyzer convert \
   --outpath="${ARTIFACTS_DIR}/pr.converted.json" \
   "${ARTIFACTS_DIR}/pr_bench.txt"
 
-BASELINE_SRC="/app/baseline/"
 if [ -d $BASELINE_SRC ]; then
   BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :)
 
@@ -37,8 +34,24 @@ if [ -d $BASELINE_SRC ]; then
     --outpath="${ARTIFACTS_DIR}/main.converted.json" \
     "${ARTIFACTS_DIR}/main_bench.txt"
 
-  benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath "${ARTIFACTS_DIR}/report.md" --format md-nodejs "${ARTIFACTS_DIR}/main.converted.json" "${ARTIFACTS_DIR}/pr.converted.json"
-  benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath "${ARTIFACTS_DIR}/report_full.html" --format html "${ARTIFACTS_DIR}/main.converted.json" "${ARTIFACTS_DIR}/pr.converted.json"
+  benchmark_analyzer compare pairwise \
+    --baseline='{"config":"baseline"}' \
+    --candidate='{"config":"candidate"}' \
+    --outpath "${ARTIFACTS_DIR}/report_full.html" \
+    --format html \
+    "${ARTIFACTS_DIR}/main.converted.json" \
+    "${ARTIFACTS_DIR}/pr.converted.json"
+
+  if ! benchmark_analyzer compare pairwise \
+    --baseline='{"config":"baseline"}' \
+    --candidate='{"config":"candidate"}' \
+    --outpath "${ARTIFACTS_DIR}/report.md" \
+    --fail_on_regression=True \
+    --format md-nodejs \
+    "${ARTIFACTS_DIR}/main.converted.json" \
+    "${ARTIFACTS_DIR}/pr.converted.json"; then
+      "$INITIAL_DIR/.gitlab/scripts/run-benchmarks-with-profiler.sh"
+  fi
 else
   benchmark_analyzer analyze --outpath "${ARTIFACTS_DIR}/analysis.html" --format html "${ARTIFACTS_DIR}/pr.converted.json"
 fi

.gitlab/scripts/config-benchmarks.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+export CANDIDATE_SRC="/app/candidate/"
+export BASELINE_SRC="/app/baseline/"
+
+# Change threshold for detection of regression
+# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference
+export UNCONFIDENCE_THRESHOLD=2.0
+export FAIL_ON_REGRESSION_THRESHOLD=$UNCONFIDENCE_THRESHOLD
+
+export BENCHMARK_TARGETS="BenchmarkConcurrentTracing|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkInjectW3C"

.gitlab/scripts/run-benchmarks-with-profiler.sh

@@ -0,0 +1,78 @@
+#!/usr/bin/env bash
+
+set -e
+
+source ./.gitlab/scripts/config-benchmarks.sh
+
+DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-go.dd_api_key --with-decryption --query "Parameter.Value" --out text)
+INTAKE_API_HOST=intake.profile.datad0g.com
+
+PROFILER_RUNS_START_DATE=$(date -u --iso-8601=minutes | sed 's/\+.*/Z/')
+
+# Run candidate and baseline the same way like in run-benchmarks.sh, but with profiler enabled
+mkdir -p "${ARTIFACTS_DIR}/candidate-profile"
+cd "$CANDIDATE_SRC/ddtrace/tracer/"
+CANDIDATE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/')
+go test -run=XXX -bench $BENCHMARK_TARGETS \
+  -cpuprofile "${ARTIFACTS_DIR}/candidate-profile/cpu.pprof" \
+  -memprofile "${ARTIFACTS_DIR}/candidate-profile/delta-heap.pprof" \
+  -benchmem -count 10 -benchtime 2s ./...
+CANDIDATE_END_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/')
+
+mkdir -p "${ARTIFACTS_DIR}/baseline-profile"
+cd "$BASELINE_SRC/ddtrace/tracer/"
+BASELINE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/')
+go test -run=XXX -bench $BENCHMARK_TARGETS \
+  -cpuprofile "${ARTIFACTS_DIR}/baseline-profile/cpu.pprof" \
+  -memprofile "${ARTIFACTS_DIR}/baseline-profile/delta-heap.pprof" \
+  -benchmem -count 10 -benchtime 2s ./...
+BASELINE_END_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/')
+
+PROFILER_RUNS_END_DATE=$(date -d '+1 min' -u --iso-8601=minutes | sed 's/\+.*/Z/')
+
+# Upload profiles to Datadog
+cd "${ARTIFACTS_DIR}/baseline-profile"
+curl -i  \
+  -H "DD-API-KEY: $DD_API_KEY" \
+  -H "DD-EVP-ORIGIN: test-origin"\
+  -H "DD-EVP-ORIGIN-VERSION: test-origin-version"\
+  -F "delta-heap.pprof=@delta-heap.pprof" \
+  -F "cpu.pprof=@cpu.pprof" \
+  -F "event=@-;type=application/json" \
+  https://$INTAKE_API_HOST/api/v2/profile <<END
+{
+ "attachments":[ "cpu.pprof", "delta-heap.pprof" ],
+ "tags_profiler":"service:dd-trace-go-benchmarks,job_id:$CI_JOB_ID,config:baseline",
+ "start":"$BASELINE_START_DATE",
+ "end":"$BASELINE_END_DATE",
+ "family":"go",
+ "version":"4"
+}
+END
+
+cd "${ARTIFACTS_DIR}/candidate-profile"
+curl -i  \
+  -H "DD-API-KEY: $DD_API_KEY" \
+  -H "DD-EVP-ORIGIN: test-origin"\
+  -H "DD-EVP-ORIGIN-VERSION: test-origin-version"\
+  -F "delta-heap.pprof=@delta-heap.pprof" \
+  -F "cpu.pprof=@cpu.pprof" \
+  -F "event=@-;type=application/json" \
+  https://$INTAKE_API_HOST/api/v2/profile <<END
+{
+ "attachments":[ "cpu.pprof", "delta-heap.pprof" ],
+ "tags_profiler":"service:dd-trace-go-benchmarks,job_id:$CI_JOB_ID,config:candidate",
+ "start":"$CANDIDATE_START_DATE",
+ "end":"$CANDIDATE_END_DATE",
+ "family":"go",
+ "version":"4"
+}
+END
+
+echo ""
+echo ""
+echo "Profiles were uploaded to Datadog! Open the following link to see them:"
+echo ""
+echo "https://dd.datad0g.com/profiling/comparison?query=service%3Add-trace-go-benchmarks&compare_end_A=$(date -d"$PROFILER_RUNS_END_DATE" +%s)000&compare_end_B=$(date -d"$PROFILER_RUNS_END_DATE" +%s)000&compare_query_B=service%3Add-trace-go-benchmarks%20config%3Acandidate%20%20job_id%3A${CI_JOB_ID}&compare_query_A=service%3Add-trace-go-benchmarks%20config%3Abaseline%20job_id%3A${CI_JOB_ID}&compare_start_A=$(date -d"$PROFILER_RUNS_START_DATE" +%s)000&compare_start_B=$(date -d"$PROFILER_RUNS_START_DATE" +%s)000&compareValuesMode=absolute&my_code=enabled&viz=flame_graph&paused=true"
+echo ""
+echo ""

.gitlab/scripts/run-benchmarks.sh

@@ -2,10 +2,10 @@
 
 set -ex
 
-CANDIDATE_SRC="/app/candidate/"
+source ./.gitlab/scripts/config-benchmarks.sh
+
 CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME
 CANDIDATE_COMMIT_SHA=$CI_COMMIT_SHA
-BENCHMARK_TARGETS="BenchmarkConcurrentTracing|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkInjectW3C"
 
 # Clone candidate release
 git clone --branch "$CANDIDATE_BRANCH" https://github.com/DataDog/dd-trace-go "$CANDIDATE_SRC" && \
@@ -16,7 +16,6 @@ git clone --branch "$CANDIDATE_BRANCH" https://github.com/DataDog/dd-trace-go "$
 cd "$CANDIDATE_SRC/ddtrace/tracer/"
 go test -run=XXX -bench $BENCHMARK_TARGETS -benchmem -count 10 -benchtime 2s ./... | tee "${ARTIFACTS_DIR}/pr_bench.txt"
 
-BASELINE_SRC="/app/baseline/"
 BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :)
 
 if [ ! -z "$BASELINE_BRANCH" ]; then