chore(async): track fire-and-forget goroutines, drain on shutdown

internal/asyncutil/asyncutil.go

@@ -0,0 +1,42 @@
+// Package asyncutil tracks detached, fire-and-forget goroutines (email/SMS
+// sends, webhook events, audit logs) that request handlers start and don't
+// wait on. Go recovers panics so a background side-effect can never take
+// down the process; Wait lets graceful shutdown drain them before exiting.
+package asyncutil
+
+import (
+	"sync/atomic"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+var counter atomic.Int64
+
+// Go runs fn in a new goroutine, tracking it so Wait can block until it
+// finishes and recovering any panic fn raises. log may be nil, in which
+// case a recovered panic is dropped (all current call sites pass a logger).
+func Go(log *zerolog.Logger, fn func()) {
+	counter.Add(1)
+	go func() {
+		defer counter.Add(-1)
+		defer func() {
+			if r := recover(); r != nil && log != nil {
+				log.Error().Interface("panic", r).Msg("recovered panic in background goroutine")
+			}
+		}()
+		fn()
+	}()
+}
+
+// Wait blocks until every goroutine started via Go has finished. Intended
+// for use once during graceful shutdown, after listeners have stopped
+// accepting new requests.
+func Wait(log zerolog.Logger) {
+	if n := counter.Load(); n > 0 {
+		log.Info().Int64("active_goroutines", n).Msg("waiting for background goroutines to complete")
+	}
+	for counter.Load() > 0 {
+		time.Sleep(100 * time.Millisecond)
+	}
+}

internal/asyncutil/asyncutil_test.go

@@ -0,0 +1,29 @@
+package asyncutil
+
+import (
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/rs/zerolog"
+)
+
+func TestGoWaitRunsAndDrains(t *testing.T) {
+	var ran atomic.Bool
+	Go(nil, func() {
+		time.Sleep(20 * time.Millisecond)
+		ran.Store(true)
+	})
+	Wait(zerolog.Nop())
+	if !ran.Load() {
+		t.Fatal("Wait returned before the goroutine finished")
+	}
+}
+
+func TestGoRecoversPanic(t *testing.T) {
+	log := zerolog.Nop()
+	Go(&log, func() {
+		panic("boom")
+	})
+	Wait(log) // must not propagate the panic to the test goroutine
+}

internal/audit/provider.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/rs/zerolog"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/storage"
 	"github.com/authorizerdev/authorizer/internal/storage/schemas"
 )
@@ -85,7 +86,7 @@ func metadataWithProtocol(meta, protocol string) string {
 
 // LogEvent asynchronously records an audit log entry.
 func (p *provider) LogEvent(event Event) {
-	go func() {
+	asyncutil.Go(p.deps.Log, func() {
 		log := p.deps.Log.With().Str("func", "LogEvent").Logger()
 		auditLog := &schemas.AuditLog{
 			ActorID:      event.ActorID,
@@ -101,5 +102,5 @@ func (p *provider) LogEvent(event Event) {
 		if err := p.deps.StorageProvider.AddAuditLog(context.Background(), auditLog); err != nil {
 			log.Debug().Err(err).Str("action", event.Action).Msg("Failed to add audit log")
 		}
-	}()
+	})
 }

internal/http_handlers/logout.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/cookie"
@@ -133,15 +134,16 @@ func (h *httpProvider) LogoutHandler() gin.HandlerFunc {
 		// logout response is never blocked by a slow receiver.
 		if strings.TrimSpace(h.Config.BackchannelLogoutURI) != "" {
 			hostname := parsers.GetHost(gc)
-			go func(uri, host, sub, sid string) {
+			uri, host, sub, sid := h.Config.BackchannelLogoutURI, hostname, userID, sessionData.Nonce
+			asyncutil.Go(h.Log, func() {
 				if err := h.TokenProvider.NotifyBackchannelLogout(context.Background(), uri, &token.BackchannelLogoutConfig{
 					HostName:  host,
 					Subject:   sub,
 					SessionID: sid,
 				}); err != nil {
 					log.Debug().Err(err).Msg("backchannel logout notification failed")
 				}
-			}(h.Config.BackchannelLogoutURI, hostname, userID, sessionData.Nonce)
+			})
 		}
 
 		if redirectURL != "" {

internal/http_handlers/oauth_callback.go

@@ -18,6 +18,7 @@ import (
 	"github.com/google/uuid"
 	goredis "github.com/redis/go-redis/v9"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/cookie"
@@ -398,7 +399,7 @@ func (h *httpProvider) OAuthCallbackHandler() gin.HandlerFunc {
 		bgCtx := context.WithoutCancel(ctx)
 		userAgent := utils.GetUserAgent(ctx.Request)
 		ip := utils.GetIP(ctx.Request)
-		go func() {
+		asyncutil.Go(h.Log, func() {
 			if isSignUp {
 				_ = h.EventsProvider.RegisterEvent(bgCtx, constants.UserSignUpWebhookEvent, provider, user)
 				// User is also logged in with signup
@@ -413,7 +414,7 @@ func (h *httpProvider) OAuthCallbackHandler() gin.HandlerFunc {
 			}); err != nil {
 				log.Debug().Err(err).Msg("Failed to add session")
 			}
-		}()
+		})
 		if strings.Contains(redirectURL, "?") {
 			redirectURL = redirectURL + "&" + params
 		} else {

internal/http_handlers/oauth_sso.go

@@ -44,6 +44,7 @@ import (
 	"github.com/go-jose/go-jose/v4"
 	"github.com/golang-jwt/jwt/v4"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/cookie"
@@ -661,15 +662,15 @@ func (h *httpProvider) issueSSOSession(c *gin.Context, flow *ssoFlowState, user
 	bgCtx := context.WithoutCancel(c.Request.Context())
 	userAgent := utils.GetUserAgent(c.Request)
 	ip := utils.GetIP(c.Request)
-	go func() {
+	asyncutil.Go(h.Log, func() {
 		if isSignUp {
 			_ = h.EventsProvider.RegisterEvent(bgCtx, constants.UserSignUpWebhookEvent, constants.AuthRecipeMethodSSO, user)
 		}
 		_ = h.EventsProvider.RegisterEvent(bgCtx, constants.UserLoginWebhookEvent, constants.AuthRecipeMethodSSO, user)
 		if err := h.StorageProvider.AddSession(bgCtx, &schemas.Session{UserID: user.ID, UserAgent: userAgent, IP: ip}); err != nil {
 			h.Log.Debug().Err(err).Msg("failed to add session")
 		}
-	}()
+	})
 
 	params := "state=" + url.QueryEscape(flow.AppState)
 	redirectURL := flow.AppRedirect

internal/http_handlers/saml_sp.go

@@ -54,6 +54,7 @@ import (
 	goredis "github.com/redis/go-redis/v9"
 	"github.com/rs/zerolog"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/cookie"
@@ -543,15 +544,15 @@ func (h *httpProvider) issueSAMLSession(c *gin.Context, slug, orgID, appRedirect
 	bgCtx := context.WithoutCancel(c.Request.Context())
 	userAgent := utils.GetUserAgent(c.Request)
 	ip := utils.GetIP(c.Request)
-	go func() {
+	asyncutil.Go(h.Log, func() {
 		if isSignUp {
 			_ = h.EventsProvider.RegisterEvent(bgCtx, constants.UserSignUpWebhookEvent, constants.AuthRecipeMethodSSO, user)
 		}
 		_ = h.EventsProvider.RegisterEvent(bgCtx, constants.UserLoginWebhookEvent, constants.AuthRecipeMethodSSO, user)
 		if err := h.StorageProvider.AddSession(bgCtx, &schemas.Session{UserID: user.ID, UserAgent: userAgent, IP: ip}); err != nil {
 			h.Log.Debug().Err(err).Msg("failed to add session")
 		}
-	}()
+	})
 
 	params := "state=" + url.QueryEscape(appState)
 	redirectURL := appRedirect

internal/http_handlers/verify_email.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gin-gonic/gin"
 	"github.com/google/uuid"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/cookie"
@@ -192,7 +193,7 @@ func (h *httpProvider) VerifyEmailHandler() gin.HandlerFunc {
 				} else {
 					nonce = authorizeState
 				}
-				go func() { _ = h.MemoryStoreProvider.RemoveState(state) }()
+				asyncutil.Go(h.Log, func() { _ = h.MemoryStoreProvider.RemoveState(state) })
 			}
 		}
 		if nonce == "" {
@@ -267,7 +268,7 @@ func (h *httpProvider) VerifyEmailHandler() gin.HandlerFunc {
 		bgCtx := context.WithoutCancel(c)
 		userAgent := utils.GetUserAgent(c.Request)
 		ip := utils.GetIP(c.Request)
-		go func() {
+		asyncutil.Go(h.Log, func() {
 			if isSignUp {
 				_ = h.EventsProvider.RegisterEvent(bgCtx, constants.UserSignUpWebhookEvent, loginMethod, user)
 				// User is also logged in with signup
@@ -282,7 +283,7 @@ func (h *httpProvider) VerifyEmailHandler() gin.HandlerFunc {
 			}); err != nil {
 				log.Debug().Err(err).Msg("Error adding session")
 			}
-		}()
+		})
 
 		c.Redirect(http.StatusTemporaryRedirect, redirectURL)
 	}

internal/server/server.go

@@ -10,6 +10,7 @@ import (
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/rs/zerolog"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/config"
 	"github.com/authorizerdev/authorizer/internal/gateway"
 	"github.com/authorizerdev/authorizer/internal/graphql"
@@ -148,6 +149,9 @@ func (s *server) Run(ctx context.Context) error {
 	if err := metricsSrv.Shutdown(shCtx); err != nil {
 		s.Dependencies.Log.Error().Err(err).Msg("Metrics server graceful shutdown failed")
 	}
+	// Drain fire-and-forget background work (emails, webhooks, audit logs)
+	// spawned by in-flight requests before the process exits.
+	asyncutil.Wait(*s.Dependencies.Log)
 	return nil
 }
 

internal/service/admin_access.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/graph/model"
@@ -45,11 +46,11 @@ func (p *provider) RevokeAccess(ctx context.Context, meta RequestMetadata, param
 		return nil, nil, err
 	}
 
-	go func() {
+	asyncutil.Go(p.Log, func() {
 		ctx := context.WithoutCancel(ctx)
 		_ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID)
 		_ = p.EventsProvider.RegisterEvent(ctx, constants.UserAccessRevokedWebhookEvent, "", user)
-	}()
+	})
 	p.AuditProvider.LogEvent(audit.Event{
 		Action:   constants.AuditAdminAccessRevokedEvent,
 		Protocol: meta.Protocol, ActorType: constants.AuditActorTypeAdmin,
@@ -92,10 +93,10 @@ func (p *provider) EnableAccess(ctx context.Context, meta RequestMetadata, param
 		log.Debug().Err(err).Msg("Failed to update user")
 		return nil, nil, err
 	}
-	go func() {
+	asyncutil.Go(p.Log, func() {
 		ctx := context.WithoutCancel(ctx)
 		_ = p.EventsProvider.RegisterEvent(ctx, constants.UserAccessEnabledWebhookEvent, "", user)
-	}()
+	})
 	p.AuditProvider.LogEvent(audit.Event{
 		Action:   constants.AuditAdminAccessEnabledEvent,
 		Protocol: meta.Protocol, ActorType: constants.AuditActorTypeAdmin,
@@ -239,13 +240,13 @@ func (p *provider) InviteMembers(ctx context.Context, meta RequestMetadata, para
 		}
 
 		// exec it as go routine so that we can reduce the api latency
-		go func() {
+		asyncutil.Go(p.Log, func() {
 			_ = p.EmailProvider.SendEmail([]string{refs.StringValue(user.Email)}, constants.VerificationTypeInviteMember, map[string]interface{}{
 				"user":             user.ToMap(),
 				"organization":     utils.GetOrganization(p.Config),
 				"verification_url": utils.GetInviteVerificationURL(verifyEmailURL, verificationToken, redirectURL),
 			})
-		}()
+		})
 	}
 
 	invitedUsers := []*model.User{}

internal/service/admin_users.go

@@ -10,6 +10,7 @@ import (
 
 	"github.com/gin-gonic/gin"
 
+	"github.com/authorizerdev/authorizer/internal/asyncutil"
 	"github.com/authorizerdev/authorizer/internal/audit"
 	"github.com/authorizerdev/authorizer/internal/constants"
 	"github.com/authorizerdev/authorizer/internal/graph/model"
@@ -224,7 +225,7 @@ func (p *provider) UpdateUser(ctx context.Context, meta RequestMetadata, params
 			return nil, nil, fmt.Errorf("user with this email address already exists")
 		}
 
-		go func() { _ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID) }()
+		asyncutil.Go(p.Log, func() { _ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID) })
 
 		// gin-shim: parsers.GetHost / GetAppURL still take a *gin.Context.
 		gc := &gin.Context{Request: meta.Request}
@@ -264,13 +265,13 @@ func (p *provider) UpdateUser(ctx context.Context, meta RequestMetadata, params
 		}
 
 		// exec it as go routine so that we can reduce the api latency
-		go func() {
+		asyncutil.Go(p.Log, func() {
 			_ = p.EmailProvider.SendEmail([]string{refs.StringValue(user.Email)}, constants.VerificationTypeBasicAuthSignup, map[string]interface{}{
 				"user":             user.ToMap(),
 				"organization":     utils.GetOrganization(p.Config),
 				"verification_url": utils.GetEmailVerificationURL(verificationToken, hostname, redirectURL),
 			})
-		}()
+		})
 	}
 
 	if params.PhoneNumber != nil && refs.StringValue(user.PhoneNumber) != refs.StringValue(params.PhoneNumber) {
@@ -286,7 +287,7 @@ func (p *provider) UpdateUser(ctx context.Context, meta RequestMetadata, params
 			log.Debug().Str("phone", phone).Msg("User with phone number already exists")
 			return nil, nil, fmt.Errorf("user with this phone number already exists")
 		}
-		go func() { _ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID) }()
+		asyncutil.Go(p.Log, func() { _ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID) })
 		user.PhoneNumber = &phone
 		user.PhoneNumberVerifiedAt = nil
 	}
@@ -311,7 +312,7 @@ func (p *provider) UpdateUser(ctx context.Context, meta RequestMetadata, params
 			rolesToSave = strings.Join(inputRoles, ",")
 		}
 
-		go func() { _ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID) }()
+		asyncutil.Go(p.Log, func() { _ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID) })
 	}
 
 	if rolesToSave != "" {
@@ -381,7 +382,7 @@ func (p *provider) DeleteUser(ctx context.Context, meta RequestMetadata, params
 		Message: `user deleted successfully`,
 	}
 
-	go func() {
+	asyncutil.Go(p.Log, func() {
 		ctx := context.WithoutCancel(ctx)
 		// delete otp for given email
 		otp, err := p.StorageProvider.GetOTPByEmail(ctx, refs.StringValue(user.Email))
@@ -426,7 +427,7 @@ func (p *provider) DeleteUser(ctx context.Context, meta RequestMetadata, params
 
 		_ = p.MemoryStoreProvider.DeleteAllUserSessions(user.ID)
 		_ = p.EventsProvider.RegisterEvent(ctx, constants.UserDeletedWebhookEvent, "", user)
-	}()
+	})
 	p.AuditProvider.LogEvent(audit.Event{
 		Action:   constants.AuditAdminUserDeletedEvent,
 		Protocol: meta.Protocol, ActorType: constants.AuditActorTypeAdmin,