feat(store): Store interface + SQLite reference impl + migration framework

README.md

@@ -24,6 +24,81 @@ go get github.com/mathomhaus/lore@latest
 
 Requires Go 1.23 or newer.
 
+## Usage
+
+### Store: persist and retrieve entries
+
+The `store.Store` interface is the primary write and read surface. Open a
+`*sql.DB` with the `"sqlite"` driver (registered by `modernc.org/sqlite`),
+pass it to `sqlite.New`, and the constructor runs schema migrations
+automatically.
+
+```go
+import (
+    "context"
+    "database/sql"
+    "fmt"
+
+    _ "modernc.org/sqlite"
+
+    "github.com/mathomhaus/lore/pkg/lore"
+    "github.com/mathomhaus/lore/pkg/lore/store/sqlite"
+)
+
+func main() {
+    dsn := "lore.db" +
+        "?_pragma=journal_mode(WAL)" +
+        "&_pragma=busy_timeout(5000)" +
+        "&_pragma=synchronous(NORMAL)" +
+        "&_pragma=foreign_keys(ON)"
+
+    db, err := sql.Open("sqlite", dsn)
+    if err != nil {
+        panic(err)
+    }
+    defer db.Close()
+
+    st, err := sqlite.New(db)
+    if err != nil {
+        panic(err)
+    }
+    defer st.Close(context.Background())
+
+    // Persist a decision.
+    id, err := st.Inscribe(context.Background(), lore.Entry{
+        Project: "myproject",
+        Kind:    lore.KindDecision,
+        Title:   "Use SQLite for local persistence",
+        Body:    "Chosen for zero-dependency deployment and strong ACID guarantees.",
+        Tags:    []string{"adr", "storage"},
+    })
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println("inscribed", id)
+
+    // Retrieve it.
+    entry, err := st.Get(context.Background(), id)
+    if err != nil {
+        panic(err)
+    }
+    fmt.Println(entry.Title)
+
+    // Full-text search.
+    hits, err := st.SearchText(context.Background(), "SQLite persistence", lore.SearchOpts{Limit: 5})
+    if err != nil {
+        panic(err)
+    }
+    for _, h := range hits {
+        fmt.Printf("%.3f  %s\n", h.Score, h.Entry.Title)
+    }
+}
+```
+
+The `store.Store` interface is backend-agnostic. Swap `sqlite.New` for any
+implementation that satisfies the interface to use a different storage engine
+without changing callers.
+
 ## Status: pre-v1.0
 
 Lore is pre-v1.0. The exported surface is stable in shape but may change in

go.mod

@@ -1,3 +1,23 @@
 module github.com/mathomhaus/lore
 
-go 1.23
+go 1.25.0
+
+require (
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
+	github.com/go-logr/logr v1.4.3 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/ncruces/go-strftime v1.0.0 // indirect
+	github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
+	go.opentelemetry.io/auto/sdk v1.2.1 // indirect
+	go.opentelemetry.io/otel v1.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.43.0 // indirect
+	go.opentelemetry.io/otel/trace v1.43.0 // indirect
+	golang.org/x/sys v0.42.0 // indirect
+	modernc.org/libc v1.72.0 // indirect
+	modernc.org/mathutil v1.7.1 // indirect
+	modernc.org/memory v1.11.0 // indirect
+	modernc.org/sqlite v1.50.0 // indirect
+)

go.sum

@@ -0,0 +1,36 @@
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
+github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
+github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
+github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
+github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
+github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
+github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
+go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
+go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
+go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I=
+go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0=
+go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM=
+go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY=
+go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A=
+go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0=
+golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo=
+golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw=
+modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c=
+modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ=
+modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
+modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
+modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI=
+modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
+modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM=
+modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew=

pkg/lore/store/sqlite/migrations.go

@@ -0,0 +1,245 @@
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"io/fs"
+	"log/slog"
+	"regexp"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+)
+
+// migration describes one numbered SQL file under migrations/. Parsed from its
+// filename: NNN_description.up.sql -> version=NNN, description="description".
+type migration struct {
+	version     int
+	description string
+	filename    string
+}
+
+// fileNameRe matches "NNN_description.up.sql" and captures both halves.
+var fileNameRe = regexp.MustCompile(`^(\d+)_([a-z0-9_]+)\.up\.sql$`)
+
+// schemaMigrationsDDL ensures the tracking table exists before any migration
+// logic runs.
+const schemaMigrationsDDL = `
+CREATE TABLE IF NOT EXISTS schema_migrations (
+  version     INTEGER PRIMARY KEY,
+  description TEXT    NOT NULL,
+  applied_at  TEXT    NOT NULL DEFAULT (datetime('now'))
+)`
+
+// Migrate applies every pending migration from the embedded migrations/
+// directory to db, in ascending version order, inside a transaction per
+// migration. Migrations already recorded in schema_migrations are skipped.
+//
+// logger receives one Info log per applied migration at level Info. Pass
+// slog.Default() for standard behavior; pass a discarding logger to silence
+// upgrade notices. Migrate is safe to call on every startup: if no migrations
+// are pending it is a no-op beyond the schema_migrations CREATE IF NOT EXISTS
+// and one SELECT per migration file.
+func Migrate(ctx context.Context, db *sql.DB, logger *slog.Logger) error {
+	if logger == nil {
+		logger = slog.Default()
+	}
+
+	if _, err := db.ExecContext(ctx, schemaMigrationsDDL); err != nil {
+		return fmt.Errorf("sqlite: migrate: create schema_migrations: %w", err)
+	}
+
+	migrations, err := loadMigrations()
+	if err != nil {
+		return fmt.Errorf("sqlite: migrate: load migrations: %w", err)
+	}
+
+	applied, err := appliedVersions(ctx, db)
+	if err != nil {
+		return fmt.Errorf("sqlite: migrate: read applied versions: %w", err)
+	}
+
+	for _, m := range migrations {
+		if applied[m.version] {
+			continue
+		}
+		if err := applyOne(ctx, db, m); err != nil {
+			return err
+		}
+		logger.InfoContext(ctx, "applying lore migration",
+			"version", m.version,
+			"description", m.description,
+		)
+	}
+
+	return nil
+}
+
+// appliedVersions returns the set of migration versions already recorded.
+func appliedVersions(ctx context.Context, db *sql.DB) (map[int]bool, error) {
+	rows, err := db.QueryContext(ctx, `SELECT version FROM schema_migrations`)
+	if err != nil {
+		return nil, fmt.Errorf("query schema_migrations: %w", err)
+	}
+	defer func() { _ = rows.Close() }()
+
+	out := map[int]bool{}
+	for rows.Next() {
+		var v int
+		if err := rows.Scan(&v); err != nil {
+			return nil, fmt.Errorf("scan schema_migrations: %w", err)
+		}
+		out[v] = true
+	}
+	if err := rows.Err(); err != nil {
+		return nil, fmt.Errorf("iterate schema_migrations: %w", err)
+	}
+	return out, nil
+}
+
+// loadMigrations walks migrationFS and returns migrations sorted by version.
+func loadMigrations() ([]migration, error) {
+	entries, err := fs.ReadDir(migrationFS, "migrations")
+	if err != nil {
+		return nil, fmt.Errorf("read embedded migrations dir: %w", err)
+	}
+
+	out := make([]migration, 0, len(entries))
+	seen := map[int]string{}
+	for _, e := range entries {
+		if e.IsDir() {
+			continue
+		}
+		name := e.Name()
+		match := fileNameRe.FindStringSubmatch(name)
+		if match == nil {
+			continue
+		}
+		v, err := strconv.Atoi(match[1])
+		if err != nil {
+			return nil, fmt.Errorf("parse version in %q: %w", name, err)
+		}
+		if prior, dup := seen[v]; dup {
+			return nil, fmt.Errorf("duplicate migration version %d: %q and %q", v, prior, name)
+		}
+		seen[v] = name
+		out = append(out, migration{
+			version:     v,
+			description: strings.ReplaceAll(match[2], "_", " "),
+			filename:    name,
+		})
+	}
+	sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version })
+	return out, nil
+}
+
+// applyOne executes every statement in migration m inside a single transaction
+// and records the version in schema_migrations. All statements land or none do.
+func applyOne(ctx context.Context, db *sql.DB, m migration) error {
+	raw, err := fs.ReadFile(migrationFS, "migrations/"+m.filename)
+	if err != nil {
+		return fmt.Errorf("sqlite: migrate: read %s: %w", m.filename, err)
+	}
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		return fmt.Errorf("sqlite: migrate: begin tx (version %d): %w", m.version, err)
+	}
+	defer func() { _ = tx.Rollback() }()
+
+	for i, stmt := range splitStatements(string(raw)) {
+		if _, err := tx.ExecContext(ctx, stmt); err != nil {
+			return fmt.Errorf("sqlite: migrate: version %d statement %d: %w", m.version, i+1, err)
+		}
+	}
+
+	if _, err := tx.ExecContext(ctx,
+		`INSERT INTO schema_migrations (version, description, applied_at) VALUES (?, ?, ?)`,
+		m.version, m.description, time.Now().UTC().Format(time.RFC3339),
+	); err != nil {
+		return fmt.Errorf("sqlite: migrate: record version %d: %w", m.version, err)
+	}
+
+	if err := tx.Commit(); err != nil {
+		return fmt.Errorf("sqlite: migrate: commit version %d: %w", m.version, err)
+	}
+	return nil
+}
+
+// splitStatements splits a SQL script into individual statements. It handles
+// CREATE TRIGGER bodies (which embed semicolons inside BEGIN...END) by tracking
+// nesting depth: a statement ends at ";" only when depth is zero.
+func splitStatements(script string) []string {
+	var cleaned strings.Builder
+	for _, line := range strings.Split(script, "\n") {
+		trimmed := strings.TrimSpace(line)
+		if strings.HasPrefix(trimmed, "--") {
+			cleaned.WriteByte('\n')
+			continue
+		}
+		if idx := strings.Index(line, " --"); idx >= 0 {
+			line = line[:idx]
+		}
+		cleaned.WriteString(line)
+		cleaned.WriteByte('\n')
+	}
+	src := cleaned.String()
+	upper := strings.ToUpper(src)
+
+	var (
+		stmts []string
+		buf   strings.Builder
+		depth int
+	)
+
+	isBoundary := func(b byte) bool {
+		return b == 0 || b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == ';'
+	}
+
+	for i := 0; i < len(src); i++ {
+		if i+5 <= len(upper) && upper[i:i+5] == "BEGIN" {
+			var next byte
+			if i+5 < len(upper) {
+				next = upper[i+5]
+			}
+			var prev byte
+			if i > 0 {
+				prev = upper[i-1]
+			}
+			if isBoundary(next) && (i == 0 || isBoundary(prev)) {
+				depth++
+			}
+		}
+		if i+3 <= len(upper) && upper[i:i+3] == "END" {
+			var next byte
+			if i+3 < len(upper) {
+				next = upper[i+3]
+			}
+			var prev byte
+			if i > 0 {
+				prev = upper[i-1]
+			}
+			if isBoundary(next) && (i == 0 || isBoundary(prev)) {
+				if depth > 0 {
+					depth--
+				}
+			}
+		}
+
+		if src[i] == ';' && depth == 0 {
+			stmt := strings.TrimSpace(buf.String())
+			if stmt != "" {
+				stmts = append(stmts, stmt)
+			}
+			buf.Reset()
+			continue
+		}
+		buf.WriteByte(src[i])
+	}
+	if stmt := strings.TrimSpace(buf.String()); stmt != "" {
+		stmts = append(stmts, stmt)
+	}
+	return stmts
+}