fix(pkg/serverless/daemon): reduce memory consumption in benchmark (#…

…23066)

fix(pkg/serverless/daemon): reduce memory consumption in benchmark

.github/workflows/serverless-benchmarks.yml

@@ -10,73 +10,124 @@ on:
 env:
   DD_API_KEY: must-be-set
 
+concurrency:
+  group: ${{ github.workflow }}/PR#${{ github.event.pull_request.number }}
+  cancel-in-progress: true
+
 jobs:
-  run:
+  baseline:
+    name: Baseline
     runs-on: ubuntu-latest
+    outputs:
+      sha: ${{ steps.prepare.outputs.sha }}
     steps:
+      - name: Checkout ${{ github.base_ref }}
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.base_ref }}
+
       - name: Install Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: stable
 
-      - name: Install benchstat
+      - name: Prepare working tree
+        id: prepare
         run: |
-          go install golang.org/x/perf/cmd/benchstat@latest
+          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
+          go get ./...
 
-      - name: Checkout datadog-agent repository
+      - name: Run benchmark
+        run: |
+          go test -tags=test -run='^$' -bench=StartEndInvocation -count=10 -benchtime=500ms -timeout=60m \
+              ./pkg/serverless/... | tee ${{runner.temp}}/benchmark.log
+
+      - name: Upload result artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: baseline.log
+          path: ${{runner.temp}}/benchmark.log
+          if-no-files-found: error
+
+
+  current:
+    name: Current
+    runs-on: ubuntu-latest
+    outputs:
+      sha: ${{ steps.prepare.outputs.sha }}
+
+    steps:
+      - name: Checkout ${{ github.ref }}
         uses: actions/checkout@v4
         with:
-          path: go/src/github.com/DataDog/datadog-agent
+          ref: ${{ github.sha }}
+
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
 
-      - name: Checkout datadog-agent base branch
-        id: previous
+      - name: Prepare working tree
+        id: prepare
         run: |
-          cd go/src/github.com/DataDog/datadog-agent
-          git fetch origin $GITHUB_BASE_REF --depth 1
-          git checkout $GITHUB_BASE_REF
           echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
-          echo "previous commit: $(git rev-parse HEAD)"
           go get ./...
 
-      - name: Previous benchmark results
+      - name: Run benchmark
         run: |
-          cd go/src/github.com/DataDog/datadog-agent
           go test -tags=test -run='^$' -bench=StartEndInvocation -count=10 -benchtime=500ms -timeout=60m \
-              ./pkg/serverless/... | tee previous
+              ./pkg/serverless/... | tee ${{runner.temp}}/benchmark.log
 
-      - name: Checkout datadog-agent pr branch
-        id: current
-        run: |
-          cd go/src/github.com/DataDog/datadog-agent
-          git fetch origin $GITHUB_SHA --depth 1
-          git checkout $GITHUB_SHA
-          echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
-          echo "current commit: $(git rev-parse HEAD)"
-          go get ./...
+      - name: Upload result artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: current.log
+          path: ${{runner.temp}}/benchmark.log
+          if-no-files-found: error
+
+  summary:
+    name: Summary
+    runs-on: ubuntu-latest
+    needs: [baseline, current]
 
-      - name: Current benchmark results
+    steps:
+      - name: Install Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: stable
+          cache: false
+
+      - name: Install benchstat
         run: |
-          cd go/src/github.com/DataDog/datadog-agent
-          go test -tags=test -run='^$' -bench=StartEndInvocation -count=10 -benchtime=500ms -timeout=60m \
-              ./pkg/serverless/... | tee current
+          go install golang.org/x/perf/cmd/benchstat@latest
+
+      - name: Download baseline artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: baseline.log
+          path: baseline.log
+      - name: Download current artifact
+        uses: actions/download-artifact@v4
+        with:
+          name: current.log
+          path: current.log
 
       - name: Analyze results
         id: analyze
         run: |
-          cd go/src/github.com/DataDog/datadog-agent
-          benchstat -row /event previous current | tee analyze.txt
+          benchstat -row /event baseline.log current.log | tee analyze.txt
           echo "analyze<<EOF" >> $GITHUB_OUTPUT
           cat analyze.txt >> $GITHUB_OUTPUT
           echo "EOF" >> $GITHUB_OUTPUT
 
       - name: Post comment
-        uses: marocchino/sticky-pull-request-comment@v2.5.0
+        uses: marocchino/sticky-pull-request-comment@v2.9.0
         with:
           recreate: true
           message: |
             ## Serverless Benchmark Results
 
-            `BenchmarkStartEndInvocation` comparison between ${{ steps.previous.outputs.sha }} and ${{ steps.current.outputs.sha }}.
+            `BenchmarkStartEndInvocation` comparison between ${{ needs.baseline.outputs.sha }} and ${{ needs.current.outputs.sha }}.
 
             <details>
             <summary>tl;dr</summary>

pkg/serverless/daemon/daemon_test.go

@@ -48,7 +48,13 @@ func TestWaitForDaemonBlocking(t *testing.T) {
 }
 
 func GetValueSyncOnce(so *sync.Once) uint64 {
-	return reflect.ValueOf(so).Elem().FieldByName("done").Uint()
+	val := reflect.ValueOf(so).Elem().FieldByName("done")
+	if val.Kind() == reflect.Struct {
+		// Go >= 1.22 (sync/atomic.Uint32)
+		return val.FieldByName("v").Uint()
+	}
+	// Go <= 1.21 (uint32)
+	return val.Uint()
 }
 
 func TestTellDaemonRuntimeDoneOnceStartOnly(t *testing.T) {

pkg/serverless/daemon/routes_test.go

@@ -16,6 +16,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cihub/seelog"
 	"github.com/stretchr/testify/assert"
 	"golang.org/x/exp/slices"
 
@@ -25,6 +26,7 @@ import (
 	"github.com/DataDog/datadog-agent/pkg/serverless/trace"
 	"github.com/DataDog/datadog-agent/pkg/trace/api"
 	"github.com/DataDog/datadog-agent/pkg/trace/testutil"
+	"github.com/DataDog/datadog-agent/pkg/util/log"
 )
 
 type mockLifecycleProcessor struct {
@@ -340,6 +342,12 @@ func getEventFromFile(filename string) string {
 }
 
 func BenchmarkStartEndInvocation(b *testing.B) {
+	// Set the logger up, so that it does not buffer all entries forever (some of these are BIG as they include the
+	// JSON payload). We're not interested in any output here, so we send it all to `io.Discard`.
+	l, err := seelog.LoggerFromWriterWithMinLevel(io.Discard, seelog.ErrorLvl)
+	assert.Nil(b, err)
+	log.SetupLogger(l, "error")
+
 	// relative to location of this test file
 	payloadFiles, err := os.ReadDir("../trace/testdata/event_samples")
 	if err != nil {
@@ -354,6 +362,7 @@ func BenchmarkStartEndInvocation(b *testing.B) {
 			rr := httptest.NewRecorder()
 
 			d := startAgents()
+			defer d.Stop()
 			start := &StartInvocation{d}
 			end := &EndInvocation{d}
 
@@ -370,7 +379,6 @@ func BenchmarkStartEndInvocation(b *testing.B) {
 				end.ServeHTTP(rr, endReq)
 			}
 			b.StopTimer()
-			d.Stop()
 		})
 	}
 }

pkg/serverless/invocationlifecycle/lifecycle.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"strings"
 	"time"
+	"unsafe"
 
 	json "github.com/json-iterator/go"
 
@@ -87,8 +88,7 @@ func (lp *LifecycleProcessor) OnInvokeStart(startDetails *InvocationStartDetails
 	log.Debug("[lifecycle] ---------------------------------------")
 
 	payloadBytes := ParseLambdaPayload(startDetails.InvokeEventRawPayload)
-	// TODO: avoid the unnecessary copy of payloadBytes when the logger isn't in debug level thanks to a []byte stringer
-	log.Debugf("Parsed payload string: %s", string(payloadBytes))
+	log.Debugf("Parsed payload string: %s", unsafe.String(unsafe.SliceData(payloadBytes), len(payloadBytes)))
 
 	lowercaseEventPayload, err := trigger.Unmarshal(bytes.ToLower(payloadBytes))
 	if err != nil {