chore: loadtest workflow enhancements

loadtest/mock/webhook/main.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"os"
 	"os/signal"
+	"strconv"
 	"syscall"
 	"time"
 
@@ -19,10 +20,57 @@ func main() {
 		port = envPort
 	}
 
-	// Create the webhook server with default configuration
+	// Parse delay mode configuration from environment
+	delayMode := false
+	if envDelayMode := os.Getenv("DELAY_MODE"); envDelayMode == "true" || envDelayMode == "1" {
+		delayMode = true
+	}
+
+	minDelay := 1 * time.Second
+	if envMinDelay := os.Getenv("MIN_DELAY"); envMinDelay != "" {
+		if d, err := time.ParseDuration(envMinDelay); err == nil {
+			minDelay = d
+		}
+	}
+
+	maxDelay := 2 * time.Second
+	if envMaxDelay := os.Getenv("MAX_DELAY"); envMaxDelay != "" {
+		if d, err := time.ParseDuration(envMaxDelay); err == nil {
+			maxDelay = d
+		}
+	}
+
+	slowDelayMin := 30 * time.Second
+	if envSlowDelayMin := os.Getenv("SLOW_DELAY_MIN"); envSlowDelayMin != "" {
+		if d, err := time.ParseDuration(envSlowDelayMin); err == nil {
+			slowDelayMin = d
+		}
+	}
+
+	slowDelayMax := 35 * time.Second
+	if envSlowDelayMax := os.Getenv("SLOW_DELAY_MAX"); envSlowDelayMax != "" {
+		if d, err := time.ParseDuration(envSlowDelayMax); err == nil {
+			slowDelayMax = d
+		}
+	}
+
+	slowPercent := 0.1 // Default 0.1%
+	if envSlowPercent := os.Getenv("SLOW_PERCENT"); envSlowPercent != "" {
+		if p, err := strconv.ParseFloat(envSlowPercent, 64); err == nil {
+			slowPercent = p
+		}
+	}
+
+	// Create the webhook server with configuration
 	srv := server.NewServer(server.Config{
-		EventTTL: 10 * time.Minute, // Default 10 minutes TTL for events
-		MaxSize:  10000,            // Maximum number of events to store
+		EventTTL:     10 * time.Minute, // Default 10 minutes TTL for events
+		MaxSize:      10000,            // Maximum number of events to store
+		DelayMode:    delayMode,
+		MinDelay:     minDelay,
+		MaxDelay:     maxDelay,
+		SlowDelayMin: slowDelayMin,
+		SlowDelayMax: slowDelayMax,
+		SlowPercent:  slowPercent,
 	})
 
 	// Create HTTP server
@@ -38,6 +86,13 @@ func main() {
 	// Start server in a goroutine
 	go func() {
 		log.Printf("Mock Webhook Server starting on port %s", port)
+		if delayMode {
+			log.Printf("⏱️  DELAY MODE ENABLED:")
+			log.Printf("   - Normal delay: %v - %v", minDelay, maxDelay)
+			log.Printf("   - Slow delay: %v - %v (%.2f%% of requests)", slowDelayMin, slowDelayMax, slowPercent)
+		} else {
+			log.Printf("⚡ DELAY MODE DISABLED - No artificial delays")
+		}
 		if err := httpServer.ListenAndServe(); err != nil && err != http.ErrServerClosed {
 			log.Fatalf("Server error: %v", err)
 		}

loadtest/mock/webhook/server/server.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"io"
 	"log"
+	"math/rand"
 	"net/http"
 	"time"
 
@@ -13,8 +14,14 @@ import (
 
 // Config holds server configuration options
 type Config struct {
-	EventTTL time.Duration
-	MaxSize  int
+	EventTTL     time.Duration
+	MaxSize      int
+	DelayMode    bool          // Enable artificial delays
+	MinDelay     time.Duration // Minimum delay (e.g., 1s)
+	MaxDelay     time.Duration // Maximum delay (e.g., 2s)
+	SlowDelayMin time.Duration // Slow delay minimum (e.g., 30s)
+	SlowDelayMax time.Duration // Slow delay maximum (e.g., 35s)
+	SlowPercent  float64       // Percentage of slow requests (e.g., 0.1 for 0.1%)
 }
 
 // Server handles webhook events and provides APIs to check their delivery
@@ -87,6 +94,33 @@ func (s *Server) Routes() http.Handler {
 
 // handleWebhook processes incoming webhook requests
 func (s *Server) handleWebhook(w http.ResponseWriter, r *http.Request) {
+	// Apply artificial delay if delay mode is enabled
+	if s.config.DelayMode {
+		var delay time.Duration
+
+		// Determine if this request should be slow (0.1% chance)
+		if rand.Float64() < s.config.SlowPercent/100.0 {
+			// Slow request: 30-35s delay
+			slowRange := s.config.SlowDelayMax - s.config.SlowDelayMin
+			if slowRange > 0 {
+				delay = s.config.SlowDelayMin + time.Duration(rand.Int63n(int64(slowRange)))
+			} else {
+				delay = s.config.SlowDelayMin
+			}
+			log.Printf("SLOW REQUEST: Applying delay of %v", delay)
+		} else {
+			// Normal delay: 1-2s
+			normalRange := s.config.MaxDelay - s.config.MinDelay
+			if normalRange > 0 {
+				delay = s.config.MinDelay + time.Duration(rand.Int63n(int64(normalRange)))
+			} else {
+				delay = s.config.MinDelay
+			}
+		}
+
+		time.Sleep(delay)
+	}
+
 	// Extract the event ID from headers - try both supported header formats
 	eventID := r.Header.Get("x-outpost-event-id")
 	if eventID == "" {

loadtest/scripts/test-mock-webhook.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+# Mock Webhook Throughput Test Script
+# This script tests the mock webhook service directly to measure its capacity
+
+set -e
+
+# Default values
+RATE="${RATE:-10}"
+DURATION="${DURATION:-30s}"
+PRE_ALLOCATED_VUS="${PRE_ALLOCATED_VUS:-20}"
+MAX_VUS="${MAX_VUS:-1000}"
+WEBHOOK_URL="${WEBHOOK_URL:-http://localhost:8080/webhook}"
+
+# Colors for output
+GREEN='\033[0;32m'
+YELLOW='\033[1;33m'
+NC='\033[0m' # No Color
+
+# Help function
+show_help() {
+  echo "Usage: $0 [options]"
+  echo ""
+  echo "Test the mock webhook service throughput directly"
+  echo ""
+  echo "Environment Variables:"
+  echo "  RATE                Target requests per second (default: 10)"
+  echo "  DURATION            Test duration (default: 30s)"
+  echo "  PRE_ALLOCATED_VUS   Pre-allocated virtual users (default: 20)"
+  echo "  MAX_VUS             Maximum virtual users (default: 1000)"
+  echo "  WEBHOOK_URL         Mock webhook URL (default: http://localhost:8080/webhook)"
+  echo ""
+  echo "Options:"
+  echo "  --help              Show this help message"
+  echo ""
+  echo "Examples:"
+  echo "  # Test with defaults (10 req/s for 30s)"
+  echo "  $0"
+  echo ""
+  echo "  # Test with 100 req/s for 1 minute"
+  echo "  RATE=100 DURATION=1m $0"
+  echo ""
+  echo "  # Test against production mock webhook"
+  echo "  WEBHOOK_URL=https://webhook-mock-production.up.railway.app/webhook RATE=50 $0"
+  echo ""
+}
+
+# Parse arguments
+for arg in "$@"; do
+  case $arg in
+    --help)
+      show_help
+      exit 0
+      ;;
+    *)
+      echo "Unknown option: $arg"
+      show_help
+      exit 1
+      ;;
+  esac
+done
+
+echo -e "${GREEN}=== Mock Webhook Throughput Test ===${NC}"
+echo -e "${YELLOW}Configuration:${NC}"
+echo "  Webhook URL: $WEBHOOK_URL"
+echo "  Rate: $RATE req/s"
+echo "  Duration: $DURATION"
+echo "  Pre-allocated VUs: $PRE_ALLOCATED_VUS"
+echo "  Max VUs: $MAX_VUS"
+echo ""
+
+# Check if k6 is installed
+if ! command -v k6 &> /dev/null; then
+  echo "Error: k6 is not installed. Please install k6 first:"
+  echo "  https://k6.io/docs/get-started/installation/"
+  exit 1
+fi
+
+# Get the directory of this script
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+PROJECT_ROOT="$( cd "$SCRIPT_DIR/.." && pwd )"
+
+# Run the k6 test
+echo -e "${GREEN}Starting test...${NC}"
+echo ""
+
+cd "$PROJECT_ROOT"
+
+k6 run \
+  -e RATE="$RATE" \
+  -e DURATION="$DURATION" \
+  -e PRE_ALLOCATED_VUS="$PRE_ALLOCATED_VUS" \
+  -e MAX_VUS="$MAX_VUS" \
+  -e WEBHOOK_URL="$WEBHOOK_URL" \
+  src/tests/webhook-mock-throughput.ts
+
+echo ""
+echo -e "${GREEN}Test complete!${NC}"