From e60e8cae27ff0afd22603fd9dbdfe7d0153895b0 Mon Sep 17 00:00:00 2001 From: boks1971 Date: Thu, 5 Jun 2025 16:34:30 -0700 Subject: [PATCH] Log qLen when processing webhook. Seeing p99 of queue length which I am not able to cross-check with a resource that is causing qlen to get that deep. So, adding a qLen log which will make it a bit easier to cross check if the prom stats look right. --- webhook/resource_queue.go | 3 ++- webhook/resource_url_notifier.go | 18 +++++++++++++++--- 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/webhook/resource_queue.go b/webhook/resource_queue.go index dc1d12446..c74cf5044 100644 --- a/webhook/resource_queue.go +++ b/webhook/resource_queue.go @@ -114,8 +114,9 @@ func (r *resourceQueue) worker() { } item := r.items.PopFront() + qLen := r.items.Len() r.mu.Unlock() - r.params.Poster.Process(item.ctx, item.queuedAt, item.event, item.params) + r.params.Poster.Process(item.ctx, item.queuedAt, item.event, item.params, qLen) } } diff --git a/webhook/resource_url_notifier.go b/webhook/resource_url_notifier.go index bd16b6aa0..b4da3e339 100644 --- a/webhook/resource_url_notifier.go +++ b/webhook/resource_url_notifier.go @@ -54,7 +54,13 @@ var DefaultResourceURLNotifierConfig = ResourceURLNotifierConfig{ } type poster interface { - Process(ctx context.Context, queuedAt time.Time, event *livekit.WebhookEvent, params *ResourceURLNotifierParams) + Process( + ctx context.Context, + queuedAt time.Time, + event *livekit.WebhookEvent, + params *ResourceURLNotifierParams, + qLen int, + ) } type resourceQueueInfo struct { @@ -254,11 +260,17 @@ func (r *ResourceURLNotifier) Stop(force bool) { } // poster interface -func (r *ResourceURLNotifier) Process(ctx context.Context, queuedAt time.Time, event *livekit.WebhookEvent, params *ResourceURLNotifierParams) { +func (r *ResourceURLNotifier) Process( + ctx context.Context, + queuedAt time.Time, + event *livekit.WebhookEvent, + params *ResourceURLNotifierParams, + qLen int, +) { fields := logFields(event, params.URL) queueDuration := time.Since(queuedAt) - fields = append(fields, "queueDuration", queueDuration) + fields = append(fields, "queueDuration", queueDuration, "qLen", qLen) if queueDuration > params.Config.MaxAge { fields = append(fields, "reason", "age")