diff --git a/.gitlab/scripts/analyze-results.sh b/.gitlab/scripts/analyze-results.sh index 806dc20b58..166518b5a3 100755 --- a/.gitlab/scripts/analyze-results.sh +++ b/.gitlab/scripts/analyze-results.sh @@ -1,12 +1,9 @@ #!/usr/bin/env bash -# Change threshold for detection of regression -# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference -export UNCONFIDENCE_THRESHOLD=2.0 -export FAIL_ON_REGRESSION_THRESHOLD=2.0 +source ./.gitlab/scripts/config-benchmarks.sh +INITIAL_DIR=$(pwd) CANDIDATE_BRANCH=$CI_COMMIT_REF_NAME -CANDIDATE_SRC="/app/candidate/" cd "$CANDIDATE_SRC" CANDIDATE_COMMIT_SHA=$(git rev-parse --short HEAD) @@ -21,7 +18,6 @@ benchmark_analyzer convert \ --outpath="${ARTIFACTS_DIR}/pr.converted.json" \ "${ARTIFACTS_DIR}/pr_bench.txt" -BASELINE_SRC="/app/baseline/" if [ -d $BASELINE_SRC ]; then BASELINE_BRANCH=$(github-find-merge-into-branch --for-repo="$CI_PROJECT_NAME" --for-pr="$CANDIDATE_BRANCH" || :) @@ -54,7 +50,7 @@ if [ -d $BASELINE_SRC ]; then --format md-nodejs \ "${ARTIFACTS_DIR}/main.converted.json" \ "${ARTIFACTS_DIR}/pr.converted.json"; then - "$ARTIFACTS_DIR/../.gitlab/scripts/run-benchmarks-with-profiler.sh" + "$INITIAL_DIR/.gitlab/scripts/run-benchmarks-with-profiler.sh" fi else benchmark_analyzer analyze --outpath "${ARTIFACTS_DIR}/analysis.html" --format html "${ARTIFACTS_DIR}/pr.converted.json" diff --git a/.gitlab/scripts/config-benchmarks.sh b/.gitlab/scripts/config-benchmarks.sh new file mode 100755 index 0000000000..8565188639 --- /dev/null +++ b/.gitlab/scripts/config-benchmarks.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +export CANDIDATE_SRC="/app/candidate/" +export BASELINE_SRC="/app/baseline/" + +# Change threshold for detection of regression +# @see https://github.com/DataDog/relenv-benchmark-analyzer#what-is-a-significant-difference +export UNCONFIDENCE_THRESHOLD=2.0 +export FAIL_ON_REGRESSION_THRESHOLD=$UNCONFIDENCE_THRESHOLD + +export BENCHMARK_TARGETS="BenchmarkConcurrentTracing|BenchmarkStartSpan|BenchmarkSingleSpanRetention|BenchmarkInjectW3C" diff --git a/.gitlab/scripts/run-benchmarks-with-profiler.sh b/.gitlab/scripts/run-benchmarks-with-profiler.sh index 04a2896603..dc6a81ac0a 100755 --- a/.gitlab/scripts/run-benchmarks-with-profiler.sh +++ b/.gitlab/scripts/run-benchmarks-with-profiler.sh @@ -2,25 +2,27 @@ set -e -CANDIDATE_SRC="/app/candidate/" -BASELINE_SRC="/app/baseline/" +source ./.gitlab/scripts/config-benchmarks.sh -mkdir -p "${ARTIFACTS_DIR}/candidate-profile" -cd "$CANDIDATE_SRC/ddtrace/tracer/" +DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-go.dd_api_key --with-decryption --query "Parameter.Value" --out text) +INTAKE_API_HOST=intake.profile.datad0g.com PROFILER_RUNS_START_DATE=$(date -u --iso-8601=minutes | sed 's/\+.*/Z/') +# Run candidate and baseline the same way like in run-benchmarks.sh, but with profiler enabled +mkdir -p "${ARTIFACTS_DIR}/candidate-profile" +cd "$CANDIDATE_SRC/ddtrace/tracer/" CANDIDATE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') -go test -run=XXX -bench "BenchmarkConcurrentTracing|BenchmarkStartSpan" \ +go test -run=XXX -bench $BENCHMARK_TARGETS \ -cpuprofile "${ARTIFACTS_DIR}/candidate-profile/cpu.pprof" \ -memprofile "${ARTIFACTS_DIR}/candidate-profile/delta-heap.pprof" \ -benchmem -count 10 -benchtime 2s ./... CANDIDATE_END_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') -BASELINE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') mkdir -p "${ARTIFACTS_DIR}/baseline-profile" cd "$BASELINE_SRC/ddtrace/tracer/" -go test -run=XXX -bench "BenchmarkConcurrentTracing|BenchmarkStartSpan" \ +BASELINE_START_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') +go test -run=XXX -bench $BENCHMARK_TARGETS \ -cpuprofile "${ARTIFACTS_DIR}/baseline-profile/cpu.pprof" \ -memprofile "${ARTIFACTS_DIR}/baseline-profile/delta-heap.pprof" \ -benchmem -count 10 -benchtime 2s ./... @@ -28,8 +30,6 @@ BASELINE_END_DATE=$(date -u --iso-8601=seconds | sed 's/\+.*/Z/') PROFILER_RUNS_END_DATE=$(date -d '+1 min' -u --iso-8601=minutes | sed 's/\+.*/Z/') -DD_API_KEY=$(aws ssm get-parameter --region us-east-1 --name ci.dd-trace-go.dd_api_key --with-decryption --query "Parameter.Value" --out text) - # Upload profiles to Datadog cd "${ARTIFACTS_DIR}/baseline-profile" curl -i \ @@ -39,7 +39,7 @@ curl -i \ -F "delta-heap.pprof=@delta-heap.pprof" \ -F "cpu.pprof=@cpu.pprof" \ -F "event=@-;type=application/json" \ - https://intake.profile.datad0g.com/api/v2/profile <