diff --git a/.gitlab/scripts/analyze-results.sh b/.gitlab/scripts/analyze-results.sh index 5bc1c02ee2..166518b5a3 100755 --- a/.gitlab/scripts/analyze-results.sh +++ b/.gitlab/scripts/analyze-results.sh @@ -1,11 +1,9 @@ #!/usr/bin/env bash -# Change threshold for detection of regression -# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference -export UNCONFIDENCE_THRESHOLD=2.0 +source ./.gitlab/scripts/config-benchmarks.sh +INITIAL_DIR=$(pwd) CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME -CANDIDATE_SRC="/app/candidate/" cd "$CANDIDATE_SRC" CANDIDATE_COMMIT_SHA=$(git rev-parse --short HEAD) @@ -20,7 +18,6 @@ benchmark_analyzer convert \ --outpath="${ARTIFACTS_DIR}/pr.converted.json" \ "${ARTIFACTS_DIR}/pr_bench.txt" -BASELINE_SRC="/app/baseline/" if [ -d $BASELINE_SRC ]; then BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :) @@ -37,8 +34,24 @@ if [ -d $BASELINE_SRC ]; then --outpath="${ARTIFACTS_DIR}/main.converted.json" \ "${ARTIFACTS_DIR}/main_bench.txt" - benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath "${ARTIFACTS_DIR}/report.md" --format md-nodejs "${ARTIFACTS_DIR}/main.converted.json" "${ARTIFACTS_DIR}/pr.converted.json" - benchmark_analyzer compare pairwise --baseline='{"config":"baseline"}' --candidate='{"config":"candidate"}' --outpath "${ARTIFACTS_DIR}/report_full.html" --format html "${ARTIFACTS_DIR}/main.converted.json" "${ARTIFACTS_DIR}/pr.converted.json" + benchmark_analyzer compare pairwise \ + --baseline='{"config":"baseline"}' \ + --candidate='{"config":"candidate"}' \ + --outpath "${ARTIFACTS_DIR}/report_full.html" \ + --format html \ + "${ARTIFACTS_DIR}/main.converted.json" \ + "${ARTIFACTS_DIR}/pr.converted.json" + + if ! benchmark_analyzer compare pairwise \ + --baseline='{"config":"baseline"}' \ + --candidate='{"config":"candidate"}' \ + --outpath "${ARTIFACTS_DIR}/report.md" \ + --fail_on_regression=True \ + --format md-nodejs \ + "${ARTIFACTS_DIR}/main.converted.json" \ + "${ARTIFACTS_DIR}/pr.converted.json"; then + "$INITIAL_DIR/.gitlab/scripts/run-benchmarks-with-profiler.sh" + fi else benchmark_analyzer analyze --outpath "${ARTIFACTS_DIR}/analysis.html" --format html "${ARTIFACTS_DIR}/pr.converted.json" fi diff --git a/.gitlab/scripts/config-benchmarks.sh b/.gitlab/scripts/config-benchmarks.sh new file mode 100755 index 0000000000..8565188639 --- /dev/null +++ b/.gitlab/scripts/config-benchmarks.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +export CANDIDATE_SRC="/app/candidate/" +export BASELINE_SRC="/app/baseline/" + +# Change threshold for detection of regression +# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference +export UNCONFIDENCE_THRESHOLD=2.0 +export FAIL_ON_REGRESSION_THRESHOLD=$UNCONFIDENCE_THRESHOLD + +export BENCHMARK_TARGETS="BenchmarkConcurrentTracing|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkInjectW3C" diff --git a/.gitlab/scripts/run-benchmarks-with-profiler.sh b/.gitlab/scripts/run-benchmarks-with-profiler.sh new file mode 100755 index 0000000000..dc6a81ac0a --- /dev/null +++ b/.gitlab/scripts/run-benchmarks-with-profiler.sh @@ -0,0 +1,78 @@ +#!/usr/bin/env bash + +set -e + +source ./.gitlab/scripts/config-benchmarks.sh + +DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-go.dd_api_key --with-decryption --query "Parameter.Value" --out text) +INTAKE_API_HOST=intake.profile.datad0g.com + +PROFILER_RUNS_START_DATE=$(date -u --iso-8601=minutes | sed 's/\+.*/Z/') + +# Run candidate and baseline the same way like in run-benchmarks.sh, but with profiler enabled +mkdir -p "${ARTIFACTS_DIR}/candidate-profile" +cd "$CANDIDATE_SRC/ddtrace/tracer/" +CANDIDATE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') +go test -run=XXX -bench $BENCHMARK_TARGETS \ + -cpuprofile "${ARTIFACTS_DIR}/candidate-profile/cpu.pprof" \ + -memprofile "${ARTIFACTS_DIR}/candidate-profile/delta-heap.pprof" \ + -benchmem -count 10 -benchtime 2s ./... +CANDIDATE_END_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') + +mkdir -p "${ARTIFACTS_DIR}/baseline-profile" +cd "$BASELINE_SRC/ddtrace/tracer/" +BASELINE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') +go test -run=XXX -bench $BENCHMARK_TARGETS \ + -cpuprofile "${ARTIFACTS_DIR}/baseline-profile/cpu.pprof" \ + -memprofile "${ARTIFACTS_DIR}/baseline-profile/delta-heap.pprof" \ + -benchmem -count 10 -benchtime 2s ./... +BASELINE_END_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') + +PROFILER_RUNS_END_DATE=$(date -d '+1 min' -u --iso-8601=minutes | sed 's/\+.*/Z/') + +# Upload profiles to Datadog +cd "${ARTIFACTS_DIR}/baseline-profile" +curl -i \ + -H "DD-API-KEY: $DD_API_KEY" \ + -H "DD-EVP-ORIGIN: test-origin"\ + -H "DD-EVP-ORIGIN-VERSION: test-origin-version"\ + -F "delta-heap.pprof=@delta-heap.pprof" \ + -F "cpu.pprof=@cpu.pprof" \ + -F "event=@-;type=application/json" \ + https://$INTAKE_API_HOST/api/v2/profile <