From 9f9cc3a0ed2fe2b1fc8d0fe178608bad967a41b7 Mon Sep 17 00:00:00 2001 From: Minh Cung Date: Sun, 29 Mar 2026 11:56:14 +1100 Subject: [PATCH 1/2] . --- Makefile | 4 + README.md | 32 ++ ...3-29-sqlite-performance-benchmark-suite.md | 60 ++++ pkg/rain/sqlite_benchmark_test.go | 292 ++++++++++++++++++ pkg/rain/sqlite_integration_test.go | 18 +- 5 files changed, 397 insertions(+), 9 deletions(-) create mode 100644 docs/adr/2026-03-29-sqlite-performance-benchmark-suite.md create mode 100644 pkg/rain/sqlite_benchmark_test.go diff --git a/Makefile b/Makefile index a3e3eec..a1b22e6 100644 --- a/Makefile +++ b/Makefile @@ -19,6 +19,10 @@ test: ## run tests with coverage go test -race -coverprofile=coverage.out $$(go list ./... | grep -v '/examples/') go tool cover -func=coverage.out | sort -rnk3 +.PHONY: bench +bench: ## run sqlite benchmark suite with allocation metrics + go test -run '^$$' -bench . -benchmem ./pkg/rain + .PHONY: test-json test-json: ## run tests with JSON output (for CI) go test -json -race -coverprofile=coverage.out $$(go list ./... | grep -v '/examples/') > test-report.jsonl diff --git a/README.md b/README.md index 48d259f..a351ce2 100644 --- a/README.md +++ b/README.md @@ -11,6 +11,7 @@ A type-safe, SQL-like ORM for Go inspired by DrizzleORM — lightweight, fast, a * [Project Layout](#project-layout) * [Quick Start](#quick-start) * [Examples](#examples) + * [Performance Benchmarks](#performance-benchmarks) * [Makefile Targets](#makefile-targets) * [Contribute](#contribute) @@ -227,10 +228,41 @@ sqlType := d.DataType(schema.ColumnType{DataType: "string", Size: 255}) // VARC See the [examples/](examples/) directory for complete, runnable examples. +# Performance Benchmarks + +Rain includes a SQLite-first benchmark suite for measuring end-to-end ORM performance and memory usage across representative CRUD and join workloads. + +Run the full suite: + +```sh +make bench +``` + +Run a single workload: + +```sh +go test -run '^$' -bench 'BenchmarkSQLiteSelectJoinScan' -benchmem ./pkg/rain +``` + +Run one workload for one dataset size: + +```sh +go test -run '^$' -bench 'BenchmarkSQLiteSelectJoinScan/medium$' -benchmem ./pkg/rain +``` + +Compare two runs over time by saving the benchmark output and diffing the benchmark lines from the same machine and environment. Use the built-in Go metrics as the primary signals: + +- `ns/op` shows the average execution time per benchmark iteration. +- `B/op` shows the average bytes allocated per iteration. +- `allocs/op` shows the average number of heap allocations per iteration. + +The suite seeds deterministic `small`, `medium`, and `large` SQLite datasets before measurements start so setup cost does not pollute the reported ORM metrics. + # Makefile Targets ```sh $> make +bench run sqlite benchmark suite with allocation metrics bootstrap download tool and module dependencies build build the library (verifies compilation) clean clean up test artifacts diff --git a/docs/adr/2026-03-29-sqlite-performance-benchmark-suite.md b/docs/adr/2026-03-29-sqlite-performance-benchmark-suite.md new file mode 100644 index 0000000..5f14516 --- /dev/null +++ b/docs/adr/2026-03-29-sqlite-performance-benchmark-suite.md @@ -0,0 +1,60 @@ +# ADR: SQLite-first ORM performance benchmark suite + +## Status +Accepted on 2026-03-29. + +## Context +Rain had integration coverage for SQLite-backed ORM flows, but no benchmark suite for measuring end-to-end latency and allocation behavior. That left the project without a repeatable way to inspect how query construction, SQL compilation, execution, and scan paths behave under realistic ORM workloads. + +The first benchmark suite needs to be practical for engineers running locally, deterministic enough to compare runs over time, and narrow enough to ship without introducing multi-dialect infrastructure or profiling workflows that are not yet required. + +## Decision +Add a SQLite-first benchmark suite in `pkg/rain` using Go's native benchmark runner. + +### Scope + +- Measure end-to-end ORM execution rather than builder-only compilation. +- Focus on developer diagnostics instead of pass/fail CI regression thresholds. +- Use Go benchmark metrics as the memory signal: `ns/op`, `B/op`, and `allocs/op`. + +### Workloads + +- Single-row insert via `.Model(...)` +- Single-row insert via `.Set(...)` +- Point lookup select and struct scan +- Filtered select into a slice +- Bulk scan into a slice +- Join scan across aliased `users` and `posts` tables + +### Dataset defaults + +- `small`: 100 users / 1,000 posts +- `medium`: 1,000 users / 10,000 posts +- `large`: 10,000 users / 100,000 posts + +### Harness rules + +- Each benchmark dataset runs against an isolated SQLite database. +- Schema creation and deterministic data seeding happen before `b.ResetTimer()`. +- Seeded row counts are validated before measurements begin. +- Benchmarks stay on the public ORM API and reuse the same table definitions as integration tests. + +## Deferred work + +- Raw `database/sql` baseline comparisons +- `pprof` heap and CPU profile capture +- Postgres and MySQL benchmark backends +- CI thresholds for time or allocation regressions + +## Consequences +### Positive + +- Gives Rain a stable local benchmark entrypoint with realistic ORM workloads. +- Makes allocation behavior visible without adding extra tooling. +- Keeps extension paths open for future dialect backends and baseline comparisons. + +### Negative + +- Benchmark coverage is limited to SQLite in v1. +- Insert benchmarks still include normal table growth during the measured run. +- Results are intended for trend analysis, not absolute cross-machine comparisons. diff --git a/pkg/rain/sqlite_benchmark_test.go b/pkg/rain/sqlite_benchmark_test.go new file mode 100644 index 0000000..ff7b5a3 --- /dev/null +++ b/pkg/rain/sqlite_benchmark_test.go @@ -0,0 +1,292 @@ +package rain_test + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/hyperlocalise/rain-orm/pkg/rain" + "github.com/hyperlocalise/rain-orm/pkg/schema" +) + +type benchmarkDataset struct { + name string + users int + posts int +} + +type benchmarkFixture struct { + db *rain.DB + users *sqliteUsersTable + posts *sqlitePostsTable + target int64 +} + +type benchmarkUserRow struct { + ID int64 `db:"id"` + Email string `db:"email"` + Name string `db:"name"` + Active bool `db:"active"` + Nickname *string `db:"nickname"` +} + +type benchmarkJoinRow struct { + Title string `db:"title"` + Email string `db:"email"` +} + +var benchmarkDatasets = []benchmarkDataset{ + {name: "small", users: 100, posts: 1000}, + {name: "medium", users: 1000, posts: 10000}, + {name: "large", users: 10000, posts: 100000}, +} + +func BenchmarkSQLiteInsertModel(b *testing.B) { + runSQLiteBenchmarkDatasets(b, func(b *testing.B, fixture *benchmarkFixture, dataset benchmarkDataset) { + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + + for idx := range b.N { + nickname := fmt.Sprintf("nickname-%s-%d", dataset.name, idx) + if _, err := fixture.db.Insert(). + Table(fixture.users). + Model(&sqliteInsertModel{ + Email: fmt.Sprintf("model-%s-%d@example.com", dataset.name, idx), + Name: fmt.Sprintf("Model User %d", idx), + Active: idx%2 == 0, + Nickname: &nickname, + }). + Exec(ctx); err != nil { + b.Fatalf("insert model: %v", err) + } + } + }) +} + +func BenchmarkSQLiteInsertSet(b *testing.B) { + runSQLiteBenchmarkDatasets(b, func(b *testing.B, fixture *benchmarkFixture, dataset benchmarkDataset) { + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + + for idx := range b.N { + if _, err := fixture.db.Insert(). + Table(fixture.users). + Set(fixture.users.Email, fmt.Sprintf("set-%s-%d@example.com", dataset.name, idx)). + Set(fixture.users.Name, fmt.Sprintf("Set User %d", idx)). + Set(fixture.users.Active, idx%2 == 0). + Set(fixture.users.Nickname, fmt.Sprintf("set-nick-%d", idx)). + Exec(ctx); err != nil { + b.Fatalf("insert set: %v", err) + } + } + }) +} + +func BenchmarkSQLiteSelectPointLookup(b *testing.B) { + runSQLiteBenchmarkDatasets(b, func(b *testing.B, fixture *benchmarkFixture, _ benchmarkDataset) { + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + var row benchmarkUserRow + if err := fixture.db.Select(). + Table(fixture.users). + Where(fixture.users.ID.Eq(fixture.target)). + Scan(ctx, &row); err != nil { + b.Fatalf("point lookup scan: %v", err) + } + } + }) +} + +func BenchmarkSQLiteSelectFilteredSlice(b *testing.B) { + runSQLiteBenchmarkDatasets(b, func(b *testing.B, fixture *benchmarkFixture, dataset benchmarkDataset) { + ctx := context.Background() + limit := min(dataset.users/2, 500) + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + rows := make([]benchmarkUserRow, 0, limit) + if err := fixture.db.Select(). + Table(fixture.users). + Where(fixture.users.Active.Eq(true)). + OrderBy(fixture.users.ID.Asc()). + Limit(limit). + Scan(ctx, &rows); err != nil { + b.Fatalf("filtered slice scan: %v", err) + } + } + }) +} + +func BenchmarkSQLiteSelectBulkScan(b *testing.B) { + runSQLiteBenchmarkDatasets(b, func(b *testing.B, fixture *benchmarkFixture, dataset benchmarkDataset) { + ctx := context.Background() + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + rows := make([]benchmarkUserRow, 0, dataset.users) + if err := fixture.db.Select(). + Table(fixture.users). + OrderBy(fixture.users.ID.Asc()). + Scan(ctx, &rows); err != nil { + b.Fatalf("bulk scan: %v", err) + } + } + }) +} + +func BenchmarkSQLiteSelectJoinScan(b *testing.B) { + runSQLiteBenchmarkDatasets(b, func(b *testing.B, fixture *benchmarkFixture, dataset benchmarkDataset) { + ctx := context.Background() + u := schema.Alias(fixture.users, "u") + p := schema.Alias(fixture.posts, "p") + expectedRows := min(dataset.posts/2, 1000) + b.ReportAllocs() + b.ResetTimer() + + for range b.N { + rows := make([]benchmarkJoinRow, 0, expectedRows) + if err := fixture.db.Select(). + Table(p). + Column(p.Title, u.Email). + Join(u, p.UserID.EqCol(u.ID)). + Where(u.Active.Eq(true)). + OrderBy(p.ID.Asc()). + Limit(expectedRows). + Scan(ctx, &rows); err != nil { + b.Fatalf("join scan: %v", err) + } + } + }) +} + +func runSQLiteBenchmarkDatasets( + b *testing.B, + run func(b *testing.B, fixture *benchmarkFixture, dataset benchmarkDataset), +) { + b.Helper() + + for _, dataset := range benchmarkDatasets { + dataset := dataset + b.Run(dataset.name, func(b *testing.B) { + fixture := newSQLiteBenchmarkFixture(b, dataset) + run(b, fixture, dataset) + }) + } +} + +func newSQLiteBenchmarkFixture(b *testing.B, dataset benchmarkDataset) *benchmarkFixture { + b.Helper() + + ctx := context.Background() + db := openSQLiteTestDB(b) + users, posts := defineSQLiteTables() + createSQLiteSchema(b, ctx, db) + seedSQLiteBenchmarkData(b, ctx, db, users, posts, dataset) + validateSQLiteBenchmarkData(b, ctx, db, users, posts, dataset) + + return &benchmarkFixture{ + db: db, + users: users, + posts: posts, + target: int64(dataset.users/2 + 1), + } +} + +func seedSQLiteBenchmarkData( + b *testing.B, + ctx context.Context, + db *rain.DB, + users *sqliteUsersTable, + posts *sqlitePostsTable, + dataset benchmarkDataset, +) { + b.Helper() + + tx, err := db.Begin(ctx) + if err != nil { + b.Fatalf("begin seed transaction: %v", err) + } + defer func() { + if tx == nil { + return + } + if rollbackErr := tx.Rollback(); rollbackErr != nil && rollbackErr != rain.ErrNoConnection { + b.Fatalf("rollback seed transaction: %v", rollbackErr) + } + }() + + const batchSize = 500 + createdAt := time.Date(2026, time.March, 29, 0, 0, 0, 0, time.UTC) + + for start := 0; start < dataset.users; start += batchSize { + end := min(start+batchSize, dataset.users) + rows := make([]map[schema.ColumnReference]any, 0, end-start) + for idx := start; idx < end; idx++ { + rows = append(rows, map[schema.ColumnReference]any{ + users.Email: fmt.Sprintf("user-%06d@example.com", idx+1), + users.Name: fmt.Sprintf("User %06d", idx+1), + users.Active: idx%2 == 0, + users.Nickname: fmt.Sprintf("nick-%06d", idx+1), + users.CreatedAt: createdAt.Add(time.Duration(idx) * time.Second), + }) + } + if _, err := tx.Insert().Table(users).Values(rows...).Exec(ctx); err != nil { + b.Fatalf("seed users batch [%d:%d): %v", start, end, err) + } + } + + for start := 0; start < dataset.posts; start += batchSize { + end := min(start+batchSize, dataset.posts) + rows := make([]map[schema.ColumnReference]any, 0, end-start) + for idx := start; idx < end; idx++ { + userID := int64((idx % dataset.users) + 1) + rows = append(rows, map[schema.ColumnReference]any{ + posts.UserID: userID, + posts.Title: fmt.Sprintf("Post %06d for user %06d", idx+1, userID), + }) + } + if _, err := tx.Insert().Table(posts).Values(rows...).Exec(ctx); err != nil { + b.Fatalf("seed posts batch [%d:%d): %v", start, end, err) + } + } + + if err := tx.Commit(); err != nil { + b.Fatalf("commit seed transaction: %v", err) + } + tx = nil +} + +func validateSQLiteBenchmarkData( + b *testing.B, + ctx context.Context, + db *rain.DB, + users *sqliteUsersTable, + posts *sqlitePostsTable, + dataset benchmarkDataset, +) { + b.Helper() + + userCount, err := db.Select().Table(users).Count(ctx) + if err != nil { + b.Fatalf("count users: %v", err) + } + if userCount != int64(dataset.users) { + b.Fatalf("seeded users mismatch: got %d want %d", userCount, dataset.users) + } + + postCount, err := db.Select().Table(posts).Count(ctx) + if err != nil { + b.Fatalf("count posts: %v", err) + } + if postCount != int64(dataset.posts) { + b.Fatalf("seeded posts mismatch: got %d want %d", postCount, dataset.posts) + } +} diff --git a/pkg/rain/sqlite_integration_test.go b/pkg/rain/sqlite_integration_test.go index 7a0281c..94538cb 100644 --- a/pkg/rain/sqlite_integration_test.go +++ b/pkg/rain/sqlite_integration_test.go @@ -271,33 +271,33 @@ func TestSQLiteIntegrationDialectTypeRendering(t *testing.T) { } } -func openSQLiteTestDB(t *testing.T) *rain.DB { - t.Helper() +func openSQLiteTestDB(tb testing.TB) *rain.DB { + tb.Helper() - dbPath := filepath.Join(t.TempDir(), "rain.sqlite") + dbPath := filepath.Join(tb.TempDir(), "rain.sqlite") db, err := rain.Open("sqlite", dbPath) if err != nil { - t.Fatalf("open sqlite db: %v", err) + tb.Fatalf("open sqlite db: %v", err) } - t.Cleanup(func() { + tb.Cleanup(func() { _ = db.Close() }) return db } -func createSQLiteSchema(t *testing.T, ctx context.Context, db *rain.DB) { - t.Helper() +func createSQLiteSchema(tb testing.TB, ctx context.Context, db *rain.DB) { + tb.Helper() users, posts := defineSQLiteTables() for _, table := range []schema.TableReference{users, posts} { statement, err := db.CreateTableSQL(table) if err != nil { - t.Fatalf("compile schema for %q: %v", table.TableDef().Name, err) + tb.Fatalf("compile schema for %q: %v", table.TableDef().Name, err) } if _, err := db.Exec(ctx, statement); err != nil { - t.Fatalf("exec schema statement %q: %v", statement, err) + tb.Fatalf("exec schema statement %q: %v", statement, err) } } } From 07ab619ae2b9048c483e3a2fd58be17bf2d84dea Mon Sep 17 00:00:00 2001 From: Minh Cung Date: Sun, 29 Mar 2026 12:01:50 +1100 Subject: [PATCH 2/2] Update Makefile Co-authored-by: greptile-apps[bot] <165735046+greptile-apps[bot]@users.noreply.github.com> --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index a1b22e6..f009cf4 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,7 @@ test: ## run tests with coverage .PHONY: bench bench: ## run sqlite benchmark suite with allocation metrics - go test -run '^$$' -bench . -benchmem ./pkg/rain + go test -run '^$$' -bench . -benchmem -count=3 ./pkg/rain .PHONY: test-json test-json: ## run tests with JSON output (for CI)