From 67311e40f2429c75e771de163256c7d062758caa Mon Sep 17 00:00:00 2001 From: Alex Luong Date: Mon, 13 Oct 2025 21:56:55 +0700 Subject: [PATCH] feat: idgen --- internal/app/app.go | 11 + internal/config/config.go | 7 + internal/config/id_template.go | 6 + internal/idgen/benchmark_comparison_test.go | 38 +++ internal/idgen/idgen.go | 126 ++++++++++ internal/idgen/idgen_test.go | 241 ++++++++++++++++++++ internal/publishmq/messagehandler.go | 4 +- internal/services/api/publish_handlers.go | 4 +- internal/services/api/router_test.go | 1 + 9 files changed, 434 insertions(+), 4 deletions(-) create mode 100644 internal/config/id_template.go create mode 100644 internal/idgen/benchmark_comparison_test.go create mode 100644 internal/idgen/idgen.go create mode 100644 internal/idgen/idgen_test.go diff --git a/internal/app/app.go b/internal/app/app.go index 51d8cf4a..885e47a7 100644 --- a/internal/app/app.go +++ b/internal/app/app.go @@ -11,6 +11,7 @@ import ( "time" "github.com/hookdeck/outpost/internal/config" + "github.com/hookdeck/outpost/internal/idgen" "github.com/hookdeck/outpost/internal/infra" "github.com/hookdeck/outpost/internal/logging" "github.com/hookdeck/outpost/internal/migrator" @@ -56,6 +57,16 @@ func run(mainContext context.Context, cfg *config.Config) error { } logger.Info("starting outpost", logFields...) + // Initialize ID generators + logger.Debug("configuring ID generators", + zap.String("event_template", cfg.IDTemplate.Event)) + if err := idgen.Configure(idgen.IDTemplateConfig{ + Event: cfg.IDTemplate.Event, + }); err != nil { + logger.Error("failed to configure ID generators", zap.Error(err)) + return err + } + if err := runMigration(mainContext, cfg, logger); err != nil { return err } diff --git a/internal/config/config.go b/internal/config/config.go index e82b9142..bf65bf87 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -97,6 +97,9 @@ type Config struct { // Alert Alert AlertConfig `yaml:"alert"` + + // ID Template + IDTemplate IDTemplateConfig `yaml:"id_template"` } var ( @@ -184,6 +187,10 @@ func (c *Config) InitDefaults() { HookdeckSourceURL: "https://hkdk.events/yhk665ljz3rn6l", SentryDSN: "https://examplePublicKey@o0.ingest.sentry.io/0", } + + c.IDTemplate = IDTemplateConfig{ + Event: "{{uuidv4}}", + } } func (c *Config) parseConfigFile(flagPath string, osInterface OSInterface) error { diff --git a/internal/config/id_template.go b/internal/config/id_template.go new file mode 100644 index 00000000..66856811 --- /dev/null +++ b/internal/config/id_template.go @@ -0,0 +1,6 @@ +package config + +// IDTemplateConfig is the configuration for ID generation templates +type IDTemplateConfig struct { + Event string `yaml:"event" env:"ID_TEMPLATE_EVENT" desc:"Go template for generating event IDs. Available functions: uuidv4, uuidv7, nanoid. Default: '{{uuidv4}}'" required:"N"` +} diff --git a/internal/idgen/benchmark_comparison_test.go b/internal/idgen/benchmark_comparison_test.go new file mode 100644 index 00000000..0fe8fa2d --- /dev/null +++ b/internal/idgen/benchmark_comparison_test.go @@ -0,0 +1,38 @@ +package idgen + +import ( + "testing" + + "github.com/google/uuid" +) + +// Benchmark direct UUID generation (original approach) +func BenchmarkDirectUUIDv4(b *testing.B) { + for i := 0; i < b.N; i++ { + _ = uuid.New().String() + } +} + +func BenchmarkDirectUUIDv7(b *testing.B) { + for i := 0; i < b.N; i++ { + id, _ := uuid.NewV7() + _ = id.String() + } +} + +// Benchmark template-based generation (new approach) +func BenchmarkTemplateUUIDv4(b *testing.B) { + gen, _ := NewIDGenerator("{{uuidv4}}") + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = gen.Generate() + } +} + +func BenchmarkTemplateUUIDv7(b *testing.B) { + gen, _ := NewIDGenerator("{{uuidv7}}") + b.ResetTimer() + for i := 0; i < b.N; i++ { + _, _ = gen.Generate() + } +} diff --git a/internal/idgen/idgen.go b/internal/idgen/idgen.go new file mode 100644 index 00000000..6da7816d --- /dev/null +++ b/internal/idgen/idgen.go @@ -0,0 +1,126 @@ +package idgen + +import ( + "bytes" + "crypto/rand" + "encoding/base64" + "fmt" + "text/template" + + "github.com/google/uuid" +) + +var ( + eventGenerator *IDGenerator +) + +func init() { + // Initialize with default UUID v4 generator + eventGenerator, _ = NewIDGenerator("{{uuidv4}}") +} + +// IDGenerator generates IDs based on a template +type IDGenerator struct { + template *template.Template +} + +// NewIDGenerator creates a new ID generator with the given template string +func NewIDGenerator(templateStr string) (*IDGenerator, error) { + if templateStr == "" { + templateStr = "{{uuidv4}}" + } + + // Create template with custom functions + tmpl := template.New("id").Funcs(template.FuncMap{ + "uuidv4": func() string { + return uuid.New().String() + }, + "uuidv7": func() string { + id, err := uuid.NewV7() + if err != nil { + // Fallback to v4 if v7 generation fails + return uuid.New().String() + } + return id.String() + }, + "nanoid": func() string { + return generateNanoid(21) // default size of 21 + }, + }) + + // Parse template + parsed, err := tmpl.Parse(templateStr) + if err != nil { + return nil, fmt.Errorf("failed to parse ID template: %w", err) + } + + return &IDGenerator{template: parsed}, nil +} + +// Generate generates a new ID using the template +func (g *IDGenerator) Generate() (string, error) { + var buf bytes.Buffer + if err := g.template.Execute(&buf, nil); err != nil { + return "", fmt.Errorf("failed to generate ID: %w", err) + } + return buf.String(), nil +} + +// generateNanoid generates a nanoid-like ID +// This is a simplified implementation inspired by nanoid +// Uses URL-safe base64 alphabet +func generateNanoid(size int) string { + // URL-safe alphabet (A-Za-z0-9_-) + const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + + bytes := make([]byte, size) + if _, err := rand.Read(bytes); err != nil { + // Fallback to UUID if random generation fails + return uuid.New().String() + } + + // Map random bytes to alphabet + result := make([]byte, size) + for i := 0; i < size; i++ { + result[i] = alphabet[int(bytes[i])%len(alphabet)] + } + + return string(result) +} + +// Helper to encode bytes to base64 URL-safe string +func encodeBase64URL(b []byte) string { + return base64.RawURLEncoding.EncodeToString(b) +} + +// IDTemplateConfig contains ID generation templates for different entity types +type IDTemplateConfig struct { + Event string +} + +// Configure configures all ID generators based on the provided config +// This should be called once at application startup before any concurrent usage +func Configure(cfg IDTemplateConfig) error { + // Configure event generator if template is provided + if cfg.Event != "" { + gen, err := NewIDGenerator(cfg.Event) + if err != nil { + return fmt.Errorf("failed to configure event ID generator: %w", err) + } + eventGenerator = gen + } + + return nil +} + +// Event generates an event ID using the configured generator. +// Defaults to UUID v4 if not configured via Configure(). +func Event() string { + id, err := eventGenerator.Generate() + if err != nil { + // Fallback to UUID v4 on error + return uuid.New().String() + } + + return id +} diff --git a/internal/idgen/idgen_test.go b/internal/idgen/idgen_test.go new file mode 100644 index 00000000..a9cb187e --- /dev/null +++ b/internal/idgen/idgen_test.go @@ -0,0 +1,241 @@ +package idgen + +import ( + "strings" + "testing" + + "github.com/google/uuid" +) + +func TestNewIDGenerator(t *testing.T) { + tests := []struct { + name string + template string + wantErr bool + description string + }{ + { + name: "empty template uses default", + template: "", + wantErr: false, + description: "should use default uuidv4", + }, + { + name: "valid uuidv4 template", + template: "{{uuidv4}}", + wantErr: false, + description: "should accept uuidv4 function", + }, + { + name: "valid uuidv7 template", + template: "{{uuidv7}}", + wantErr: false, + description: "should accept uuidv7 function", + }, + { + name: "valid nanoid template", + template: "{{nanoid}}", + wantErr: false, + description: "should accept nanoid function", + }, + { + name: "composite template", + template: "evt_{{uuidv4}}", + wantErr: false, + description: "should accept composite templates", + }, + { + name: "invalid template syntax", + template: "{{invalid", + wantErr: true, + description: "should fail on invalid template syntax", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gen, err := NewIDGenerator(tt.template) + if (err != nil) != tt.wantErr { + t.Errorf("NewIDGenerator() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !tt.wantErr && gen == nil { + t.Error("NewIDGenerator() returned nil generator without error") + } + }) + } +} + +func TestIDGenerator_Generate(t *testing.T) { + tests := []struct { + name string + template string + validate func(t *testing.T, id string) + }{ + { + name: "uuidv4 generates valid UUID", + template: "{{uuidv4}}", + validate: func(t *testing.T, id string) { + if _, err := uuid.Parse(id); err != nil { + t.Errorf("Generated ID is not a valid UUID: %s", id) + } + // UUIDv4 has version 4 in the correct position + if !strings.Contains(id, "-4") { + t.Errorf("Generated ID is not a UUID v4: %s", id) + } + }, + }, + { + name: "uuidv7 generates valid UUID", + template: "{{uuidv7}}", + validate: func(t *testing.T, id string) { + if _, err := uuid.Parse(id); err != nil { + t.Errorf("Generated ID is not a valid UUID: %s", id) + } + // UUIDv7 has version 7 in the correct position + parsed, _ := uuid.Parse(id) + if parsed.Version() != 7 { + t.Errorf("Generated ID is not a UUID v7: %s (version: %d)", id, parsed.Version()) + } + }, + }, + { + name: "nanoid generates valid ID", + template: "{{nanoid}}", + validate: func(t *testing.T, id string) { + if len(id) != 21 { + t.Errorf("Nanoid should be 21 characters, got %d: %s", len(id), id) + } + // Check it only contains valid characters + const alphabet = "0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz" + for _, c := range id { + if !strings.ContainsRune(alphabet, c) { + t.Errorf("Nanoid contains invalid character: %c", c) + } + } + }, + }, + { + name: "composite template with prefix", + template: "evt_{{uuidv4}}", + validate: func(t *testing.T, id string) { + if !strings.HasPrefix(id, "evt_") { + t.Errorf("ID should have prefix 'evt_', got: %s", id) + } + uuidPart := strings.TrimPrefix(id, "evt_") + if _, err := uuid.Parse(uuidPart); err != nil { + t.Errorf("UUID part is not valid: %s", uuidPart) + } + }, + }, + { + name: "multiple function calls", + template: "{{uuidv4}}_{{nanoid}}", + validate: func(t *testing.T, id string) { + parts := strings.Split(id, "_") + if len(parts) != 2 { + t.Errorf("Expected 2 parts separated by underscore, got %d", len(parts)) + return + } + // First part should be UUID + if _, err := uuid.Parse(parts[0]); err != nil { + t.Errorf("First part is not a valid UUID: %s", parts[0]) + } + // Second part should be nanoid (21 chars) + if len(parts[1]) != 21 { + t.Errorf("Second part should be 21 characters, got %d", len(parts[1])) + } + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gen, err := NewIDGenerator(tt.template) + if err != nil { + t.Fatalf("NewIDGenerator() error = %v", err) + } + + id, err := gen.Generate() + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + + if id == "" { + t.Error("Generate() returned empty string") + } + + tt.validate(t, id) + }) + } +} + +func TestIDGenerator_GenerateUniqueness(t *testing.T) { + gen, err := NewIDGenerator("{{uuidv4}}") + if err != nil { + t.Fatalf("NewIDGenerator() error = %v", err) + } + + // Generate multiple IDs and ensure they're unique + ids := make(map[string]bool) + for i := 0; i < 100; i++ { + id, err := gen.Generate() + if err != nil { + t.Fatalf("Generate() error = %v", err) + } + if ids[id] { + t.Errorf("Generated duplicate ID: %s", id) + } + ids[id] = true + } +} + +func TestEvent(t *testing.T) { + t.Run("generates UUID v4 by default", func(t *testing.T) { + id := Event() + if id == "" { + t.Error("Event() returned empty string") + } + if _, err := uuid.Parse(id); err != nil { + t.Errorf("Event() returned invalid UUID: %s", id) + } + }) + + t.Run("uses configured template", func(t *testing.T) { + err := Configure(IDTemplateConfig{ + Event: "evt_{{uuidv4}}", + }) + if err != nil { + t.Fatalf("Configure() error = %v", err) + } + + id := Event() + if !strings.HasPrefix(id, "evt_") { + t.Errorf("Event() = %v, want prefix 'evt_'", id) + } + }) +} + +func BenchmarkIDGenerator_UUIDv4(b *testing.B) { + gen, _ := NewIDGenerator("{{uuidv4}}") + b.ResetTimer() + for i := 0; i < b.N; i++ { + gen.Generate() + } +} + +func BenchmarkIDGenerator_UUIDv7(b *testing.B) { + gen, _ := NewIDGenerator("{{uuidv7}}") + b.ResetTimer() + for i := 0; i < b.N; i++ { + gen.Generate() + } +} + +func BenchmarkIDGenerator_Nanoid(b *testing.B) { + gen, _ := NewIDGenerator("{{nanoid}}") + b.ResetTimer() + for i := 0; i < b.N; i++ { + gen.Generate() + } +} diff --git a/internal/publishmq/messagehandler.go b/internal/publishmq/messagehandler.go index 557778b0..0f7c506a 100644 --- a/internal/publishmq/messagehandler.go +++ b/internal/publishmq/messagehandler.go @@ -5,8 +5,8 @@ import ( "encoding/json" "time" - "github.com/google/uuid" "github.com/hookdeck/outpost/internal/consumer" + "github.com/hookdeck/outpost/internal/idgen" "github.com/hookdeck/outpost/internal/models" "github.com/hookdeck/outpost/internal/mqs" ) @@ -52,7 +52,7 @@ type PublishedEvent struct { func (p *PublishedEvent) toEvent() models.Event { id := p.ID if id == "" { - id = uuid.New().String() + id = idgen.Event() } eventTime := p.Time if eventTime.IsZero() { diff --git a/internal/services/api/publish_handlers.go b/internal/services/api/publish_handlers.go index 3740b400..69400ddf 100644 --- a/internal/services/api/publish_handlers.go +++ b/internal/services/api/publish_handlers.go @@ -6,8 +6,8 @@ import ( "time" "github.com/gin-gonic/gin" - "github.com/google/uuid" "github.com/hookdeck/outpost/internal/idempotence" + "github.com/hookdeck/outpost/internal/idgen" "github.com/hookdeck/outpost/internal/logging" "github.com/hookdeck/outpost/internal/models" "github.com/hookdeck/outpost/internal/publishmq" @@ -78,7 +78,7 @@ type PublishedEvent struct { func (p *PublishedEvent) toEvent() models.Event { id := p.ID if id == "" { - id = uuid.New().String() + id = idgen.Event() } eventTime := p.Time if eventTime.IsZero() { diff --git a/internal/services/api/router_test.go b/internal/services/api/router_test.go index 495fb256..63e8b74c 100644 --- a/internal/services/api/router_test.go +++ b/internal/services/api/router_test.go @@ -41,6 +41,7 @@ func setupTestRouter(t *testing.T, apiKey, jwtSecret string, funcs ...func(t *te entityStore := setupTestEntityStore(t, redisClient, nil) logStore := setupTestLogStore(t, funcs...) eventHandler := publishmq.NewEventHandler(logger, redisClient, deliveryMQ, entityStore, eventTracer, testutil.TestTopics) + router := api.NewRouter( api.RouterConfig{ ServiceName: "",