[SVLS-4142] Create a cold start span only when a cold start has not a…

…lready occurred (#22202)

* create a Lambda span on timeouts

* don't create a cold start span when the runtime restarts during timeouts

* fix linting

* fix test

* lint: rename name variables

* lint again

* small fixes

* refactor timeout span logic

* add mutexes

* fix span completed check

* revert refactor

* remove cold start span changes

* use mutex over rwmutex

* test routes

* add comment + update tests

* test endExecutionSpan

* add serverless.go test

* add test /hello for route

* prevent additional cold start spans from being created after runtime is killed

* fix bad conflict merge

* fix test

cmd/serverless/main.go

@@ -269,6 +269,7 @@ func runAgent() {
 		TraceAgent:           serverlessDaemon.TraceAgent,
 		StopChan:             make(chan struct{}),
 		ColdStartSpanId:      coldStartSpanId,
+		ColdStartRequestID:   serverlessDaemon.ExecutionContext.GetCurrentState().ColdstartRequestID,
 	}
 
 	log.Debug("Starting ColdStartSpanCreator")

pkg/serverless/trace/cold_start_span_creator.go

@@ -46,6 +46,7 @@ type ColdStartSpanCreator struct {
 	initDuration          float64
 	StopChan              chan struct{}
 	initStartTime         time.Time
+	ColdStartRequestID    string
 }
 
 //nolint:revive // TODO(SERV) Fix revive linter
@@ -143,7 +144,14 @@ func (c *ColdStartSpanCreator) create() {
 		Type:     "serverless",
 	}
 
-	c.createSpan.Do(func() { c.processSpan(coldStartSpan) })
+	c.createSpan.Do(func() {
+		// An unexpected shutdown will reset this sync.Once counter, so we check whether a cold start has already occurred
+		if len(c.ColdStartRequestID) > 0 {
+			log.Debugf("[ColdStartCreator] Cold start span already created for request ID %s", c.ColdStartRequestID)
+			return
+		}
+		c.processSpan(coldStartSpan)
+	})
 }
 
 func (c *ColdStartSpanCreator) processSpan(coldStartSpan *pb.Span) {

pkg/serverless/trace/cold_start_span_creator_test.go

@@ -265,6 +265,65 @@ func TestColdStartSpanCreatorNotColdStart(t *testing.T) {
 	assert.Equal(t, true, timedOut)
 }
 
+func TestColdStartSpanCreatorColdStartExists(t *testing.T) {
+	setupTraceAgentTest(t)
+
+	cfg := config.New()
+	cfg.GlobalTags = map[string]string{}
+	cfg.Endpoints[0].APIKey = "test"
+	ctx, cancel := context.WithCancel(context.Background())
+	defer cancel()
+	agnt := agent.NewAgent(ctx, cfg, telemetry.NewNoopCollector(), &statsd.NoOpClient{})
+	traceAgent := &ServerlessTraceAgent{
+		ta: agnt,
+	}
+	coldStartDuration := 50.0 // Given in millis
+	lambdaSpanChan := make(chan *pb.Span)
+	lambdaInitMetricChan := make(chan *serverlessLog.LambdaInitMetric)
+	initReportStartTime := time.Now().Add(-1 * time.Second)
+	lambdaInitMetricDuration := &serverlessLog.LambdaInitMetric{
+		InitDurationTelemetry: coldStartDuration,
+	}
+	lambdaInitMetricStartTime := &serverlessLog.LambdaInitMetric{
+		InitStartTime: initReportStartTime,
+	}
+	stopChan := make(chan struct{})
+	coldStartSpanID := random.Random.Uint64()
+	coldStartSpanCreator := &ColdStartSpanCreator{
+		TraceAgent:           traceAgent,
+		LambdaSpanChan:       lambdaSpanChan,
+		LambdaInitMetricChan: lambdaInitMetricChan,
+		ColdStartSpanId:      coldStartSpanID,
+		StopChan:             stopChan,
+		ColdStartRequestID:   "test",
+	}
+
+	coldStartSpanCreator.Run()
+	defer coldStartSpanCreator.Stop()
+
+	lambdaSpan := &pb.Span{
+		Service:  "aws.lambda",
+		Name:     "aws.lambda",
+		Start:    time.Now().Unix(),
+		TraceID:  random.Random.Uint64(),
+		SpanID:   random.Random.Uint64(),
+		ParentID: random.Random.Uint64(),
+		Duration: 500,
+	}
+	lambdaSpanChan <- lambdaSpan
+	lambdaInitMetricChan <- lambdaInitMetricDuration
+	lambdaInitMetricChan <- lambdaInitMetricStartTime
+	timeout := time.After(time.Millisecond)
+	timedOut := false
+	select {
+	case ss := <-traceAgent.ta.TraceWriter.In:
+		t.Fatalf("created a coldstart span when we should have passed, %v", ss)
+	case <-timeout:
+		timedOut = true
+	}
+	assert.Equal(t, true, timedOut)
+}
+
 func TestColdStartSpanCreatorCreateValidProvisionedConcurrency(t *testing.T) {
 	setupTraceAgentTest(t)