diff --git a/README.md b/README.md index 7256208..fe5329f 100644 --- a/README.md +++ b/README.md @@ -24,6 +24,81 @@ go get github.com/mathomhaus/lore@latest Requires Go 1.23 or newer. +## Usage + +### Store: persist and retrieve entries + +The `store.Store` interface is the primary write and read surface. Open a +`*sql.DB` with the `"sqlite"` driver (registered by `modernc.org/sqlite`), +pass it to `sqlite.New`, and the constructor runs schema migrations +automatically. + +```go +import ( + "context" + "database/sql" + "fmt" + + _ "modernc.org/sqlite" + + "github.com/mathomhaus/lore/pkg/lore" + "github.com/mathomhaus/lore/pkg/lore/store/sqlite" +) + +func main() { + dsn := "lore.db" + + "?_pragma=journal_mode(WAL)" + + "&_pragma=busy_timeout(5000)" + + "&_pragma=synchronous(NORMAL)" + + "&_pragma=foreign_keys(ON)" + + db, err := sql.Open("sqlite", dsn) + if err != nil { + panic(err) + } + defer db.Close() + + st, err := sqlite.New(db) + if err != nil { + panic(err) + } + defer st.Close(context.Background()) + + // Persist a decision. + id, err := st.Inscribe(context.Background(), lore.Entry{ + Project: "myproject", + Kind: lore.KindDecision, + Title: "Use SQLite for local persistence", + Body: "Chosen for zero-dependency deployment and strong ACID guarantees.", + Tags: []string{"adr", "storage"}, + }) + if err != nil { + panic(err) + } + fmt.Println("inscribed", id) + + // Retrieve it. + entry, err := st.Get(context.Background(), id) + if err != nil { + panic(err) + } + fmt.Println(entry.Title) + + // Full-text search. + hits, err := st.SearchText(context.Background(), "SQLite persistence", lore.SearchOpts{Limit: 5}) + if err != nil { + panic(err) + } + for _, h := range hits { + fmt.Printf("%.3f %s\n", h.Score, h.Entry.Title) + } +} +``` + +The `store.Store` interface is backend-agnostic. Swap `sqlite.New` for any +implementation that satisfies the interface to use a different storage engine +without changing callers. + ## Status: pre-v1.0 Lore is pre-v1.0. The exported surface is stable in shape but may change in diff --git a/go.mod b/go.mod index 797656a..32685b0 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,23 @@ module github.com/mathomhaus/lore -go 1.23 +go 1.25.0 + +require ( + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/go-logr/logr v1.4.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/uuid v1.6.0 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.43.0 // indirect + go.opentelemetry.io/otel/metric v1.43.0 // indirect + go.opentelemetry.io/otel/trace v1.43.0 // indirect + golang.org/x/sys v0.42.0 // indirect + modernc.org/libc v1.72.0 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.50.0 // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..ccfbc91 --- /dev/null +++ b/go.sum @@ -0,0 +1,36 @@ +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI= +github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/otel v1.43.0 h1:mYIM03dnh5zfN7HautFE4ieIig9amkNANT+xcVxAj9I= +go.opentelemetry.io/otel v1.43.0/go.mod h1:JuG+u74mvjvcm8vj8pI5XiHy1zDeoCS2LB1spIq7Ay0= +go.opentelemetry.io/otel/metric v1.43.0 h1:d7638QeInOnuwOONPp4JAOGfbCEpYb+K6DVWvdxGzgM= +go.opentelemetry.io/otel/metric v1.43.0/go.mod h1:RDnPtIxvqlgO8GRW18W6Z/4P462ldprJtfxHxyKd2PY= +go.opentelemetry.io/otel/trace v1.43.0 h1:BkNrHpup+4k4w+ZZ86CZoHHEkohws8AY+WTX09nk+3A= +go.opentelemetry.io/otel/trace v1.43.0/go.mod h1:/QJhyVBUUswCphDVxq+8mld+AvhXZLhe+8WVFxiFff0= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +modernc.org/libc v1.72.0 h1:IEu559v9a0XWjw0DPoVKtXpO2qt5NVLAnFaBbjq+n8c= +modernc.org/libc v1.72.0/go.mod h1:tTU8DL8A+XLVkEY3x5E/tO7s2Q/q42EtnNWda/L5QhQ= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/sqlite v1.50.0 h1:eMowQSWLK0MeiQTdmz3lqoF5dqclujdlIKeJA11+7oM= +modernc.org/sqlite v1.50.0/go.mod h1:m0w8xhwYUVY3H6pSDwc3gkJ/irZT/0YEXwBlhaxQEew= diff --git a/pkg/lore/store/sqlite/migrations.go b/pkg/lore/store/sqlite/migrations.go new file mode 100644 index 0000000..90e147f --- /dev/null +++ b/pkg/lore/store/sqlite/migrations.go @@ -0,0 +1,245 @@ +package sqlite + +import ( + "context" + "database/sql" + "fmt" + "io/fs" + "log/slog" + "regexp" + "sort" + "strconv" + "strings" + "time" +) + +// migration describes one numbered SQL file under migrations/. Parsed from its +// filename: NNN_description.up.sql -> version=NNN, description="description". +type migration struct { + version int + description string + filename string +} + +// fileNameRe matches "NNN_description.up.sql" and captures both halves. +var fileNameRe = regexp.MustCompile(`^(\d+)_([a-z0-9_]+)\.up\.sql$`) + +// schemaMigrationsDDL ensures the tracking table exists before any migration +// logic runs. +const schemaMigrationsDDL = ` +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + description TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +)` + +// Migrate applies every pending migration from the embedded migrations/ +// directory to db, in ascending version order, inside a transaction per +// migration. Migrations already recorded in schema_migrations are skipped. +// +// logger receives one Info log per applied migration at level Info. Pass +// slog.Default() for standard behavior; pass a discarding logger to silence +// upgrade notices. Migrate is safe to call on every startup: if no migrations +// are pending it is a no-op beyond the schema_migrations CREATE IF NOT EXISTS +// and one SELECT per migration file. +func Migrate(ctx context.Context, db *sql.DB, logger *slog.Logger) error { + if logger == nil { + logger = slog.Default() + } + + if _, err := db.ExecContext(ctx, schemaMigrationsDDL); err != nil { + return fmt.Errorf("sqlite: migrate: create schema_migrations: %w", err) + } + + migrations, err := loadMigrations() + if err != nil { + return fmt.Errorf("sqlite: migrate: load migrations: %w", err) + } + + applied, err := appliedVersions(ctx, db) + if err != nil { + return fmt.Errorf("sqlite: migrate: read applied versions: %w", err) + } + + for _, m := range migrations { + if applied[m.version] { + continue + } + if err := applyOne(ctx, db, m); err != nil { + return err + } + logger.InfoContext(ctx, "applying lore migration", + "version", m.version, + "description", m.description, + ) + } + + return nil +} + +// appliedVersions returns the set of migration versions already recorded. +func appliedVersions(ctx context.Context, db *sql.DB) (map[int]bool, error) { + rows, err := db.QueryContext(ctx, `SELECT version FROM schema_migrations`) + if err != nil { + return nil, fmt.Errorf("query schema_migrations: %w", err) + } + defer func() { _ = rows.Close() }() + + out := map[int]bool{} + for rows.Next() { + var v int + if err := rows.Scan(&v); err != nil { + return nil, fmt.Errorf("scan schema_migrations: %w", err) + } + out[v] = true + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate schema_migrations: %w", err) + } + return out, nil +} + +// loadMigrations walks migrationFS and returns migrations sorted by version. +func loadMigrations() ([]migration, error) { + entries, err := fs.ReadDir(migrationFS, "migrations") + if err != nil { + return nil, fmt.Errorf("read embedded migrations dir: %w", err) + } + + out := make([]migration, 0, len(entries)) + seen := map[int]string{} + for _, e := range entries { + if e.IsDir() { + continue + } + name := e.Name() + match := fileNameRe.FindStringSubmatch(name) + if match == nil { + continue + } + v, err := strconv.Atoi(match[1]) + if err != nil { + return nil, fmt.Errorf("parse version in %q: %w", name, err) + } + if prior, dup := seen[v]; dup { + return nil, fmt.Errorf("duplicate migration version %d: %q and %q", v, prior, name) + } + seen[v] = name + out = append(out, migration{ + version: v, + description: strings.ReplaceAll(match[2], "_", " "), + filename: name, + }) + } + sort.Slice(out, func(i, j int) bool { return out[i].version < out[j].version }) + return out, nil +} + +// applyOne executes every statement in migration m inside a single transaction +// and records the version in schema_migrations. All statements land or none do. +func applyOne(ctx context.Context, db *sql.DB, m migration) error { + raw, err := fs.ReadFile(migrationFS, "migrations/"+m.filename) + if err != nil { + return fmt.Errorf("sqlite: migrate: read %s: %w", m.filename, err) + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return fmt.Errorf("sqlite: migrate: begin tx (version %d): %w", m.version, err) + } + defer func() { _ = tx.Rollback() }() + + for i, stmt := range splitStatements(string(raw)) { + if _, err := tx.ExecContext(ctx, stmt); err != nil { + return fmt.Errorf("sqlite: migrate: version %d statement %d: %w", m.version, i+1, err) + } + } + + if _, err := tx.ExecContext(ctx, + `INSERT INTO schema_migrations (version, description, applied_at) VALUES (?, ?, ?)`, + m.version, m.description, time.Now().UTC().Format(time.RFC3339), + ); err != nil { + return fmt.Errorf("sqlite: migrate: record version %d: %w", m.version, err) + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("sqlite: migrate: commit version %d: %w", m.version, err) + } + return nil +} + +// splitStatements splits a SQL script into individual statements. It handles +// CREATE TRIGGER bodies (which embed semicolons inside BEGIN...END) by tracking +// nesting depth: a statement ends at ";" only when depth is zero. +func splitStatements(script string) []string { + var cleaned strings.Builder + for _, line := range strings.Split(script, "\n") { + trimmed := strings.TrimSpace(line) + if strings.HasPrefix(trimmed, "--") { + cleaned.WriteByte('\n') + continue + } + if idx := strings.Index(line, " --"); idx >= 0 { + line = line[:idx] + } + cleaned.WriteString(line) + cleaned.WriteByte('\n') + } + src := cleaned.String() + upper := strings.ToUpper(src) + + var ( + stmts []string + buf strings.Builder + depth int + ) + + isBoundary := func(b byte) bool { + return b == 0 || b == ' ' || b == '\n' || b == '\r' || b == '\t' || b == ';' + } + + for i := 0; i < len(src); i++ { + if i+5 <= len(upper) && upper[i:i+5] == "BEGIN" { + var next byte + if i+5 < len(upper) { + next = upper[i+5] + } + var prev byte + if i > 0 { + prev = upper[i-1] + } + if isBoundary(next) && (i == 0 || isBoundary(prev)) { + depth++ + } + } + if i+3 <= len(upper) && upper[i:i+3] == "END" { + var next byte + if i+3 < len(upper) { + next = upper[i+3] + } + var prev byte + if i > 0 { + prev = upper[i-1] + } + if isBoundary(next) && (i == 0 || isBoundary(prev)) { + if depth > 0 { + depth-- + } + } + } + + if src[i] == ';' && depth == 0 { + stmt := strings.TrimSpace(buf.String()) + if stmt != "" { + stmts = append(stmts, stmt) + } + buf.Reset() + continue + } + buf.WriteByte(src[i]) + } + if stmt := strings.TrimSpace(buf.String()); stmt != "" { + stmts = append(stmts, stmt) + } + return stmts +} diff --git a/pkg/lore/store/sqlite/migrations/001_initial.up.sql b/pkg/lore/store/sqlite/migrations/001_initial.up.sql new file mode 100644 index 0000000..b62685b --- /dev/null +++ b/pkg/lore/store/sqlite/migrations/001_initial.up.sql @@ -0,0 +1,86 @@ +-- 001_initial.up.sql +-- Baseline schema for the lore SQLite store. +-- Applied by pkg/lore/store/sqlite inside a single transaction. + +-- --------------------------------------------------------------------------- +-- Entries: the primary content table. +-- +-- tags and metadata are stored as JSON text: tags as a JSON array of strings +-- (e.g. '["adr","architecture"]') and metadata as a JSON object +-- (e.g. '{"component":"auth"}'). +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS entries ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + project TEXT NOT NULL DEFAULT '', + kind TEXT NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL DEFAULT '', + source TEXT NOT NULL DEFAULT '', + tags TEXT NOT NULL DEFAULT '[]', + metadata TEXT NOT NULL DEFAULT '{}', + created_at TEXT NOT NULL DEFAULT (datetime('now')), + updated_at TEXT NOT NULL DEFAULT (datetime('now')) +); + +CREATE INDEX IF NOT EXISTS idx_entries_project ON entries(project); +CREATE INDEX IF NOT EXISTS idx_entries_kind ON entries(kind); +CREATE INDEX IF NOT EXISTS idx_entries_source ON entries(source) WHERE source != ''; + +-- --------------------------------------------------------------------------- +-- Edges: directed typed links between entries. +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS edges ( + from_id INTEGER NOT NULL, + to_id INTEGER NOT NULL, + relation TEXT NOT NULL, + weight REAL NOT NULL DEFAULT 0, + created_at TEXT NOT NULL DEFAULT (datetime('now')), + PRIMARY KEY (from_id, to_id, relation), + FOREIGN KEY (from_id) REFERENCES entries(id), + FOREIGN KEY (to_id) REFERENCES entries(id) +); + +-- --------------------------------------------------------------------------- +-- FTS5 virtual table for full-text search over title + body. +-- +-- content=entries + content_rowid=id makes this a content shadow index: +-- the FTS table follows entries.id as the rowid and delegates content reads +-- back to entries so content is stored once. +-- --------------------------------------------------------------------------- + +CREATE VIRTUAL TABLE IF NOT EXISTS entries_fts USING fts5( + title, body, + content=entries, content_rowid=id +); + +-- FTS5 sync triggers. Fire on title or body changes only so routine +-- metadata updates (source, tags, metadata) do not re-index the row. + +CREATE TRIGGER IF NOT EXISTS entries_ai AFTER INSERT ON entries BEGIN + INSERT INTO entries_fts(rowid, title, body) + VALUES (new.id, new.title, new.body); +END; + +CREATE TRIGGER IF NOT EXISTS entries_ad AFTER DELETE ON entries BEGIN + INSERT INTO entries_fts(entries_fts, rowid, title, body) + VALUES ('delete', old.id, old.title, old.body); +END; + +CREATE TRIGGER IF NOT EXISTS entries_au AFTER UPDATE OF title, body ON entries BEGIN + INSERT INTO entries_fts(entries_fts, rowid, title, body) + VALUES ('delete', old.id, old.title, old.body); + INSERT INTO entries_fts(rowid, title, body) + VALUES (new.id, new.title, new.body); +END; + +-- --------------------------------------------------------------------------- +-- schema_migrations: tracks applied migrations so Migrate is idempotent. +-- --------------------------------------------------------------------------- + +CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + description TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT (datetime('now')) +); diff --git a/pkg/lore/store/sqlite/schema.go b/pkg/lore/store/sqlite/schema.go new file mode 100644 index 0000000..8941cb9 --- /dev/null +++ b/pkg/lore/store/sqlite/schema.go @@ -0,0 +1,22 @@ +// Package sqlite provides a SQLite-backed implementation of store.Store using +// modernc.org/sqlite, a pure-Go SQLite driver that requires no CGO. +// +// Callers open a *sql.DB themselves (using sql.Open with driver name "sqlite") +// and pass it to New. The store runs pending migrations on construction and +// owns no connection-pool configuration; the caller sets MaxOpenConns, +// MaxIdleConns, and pragmas via the DSN. +// +// Recommended DSN pragmas for every connection in the pool: +// +// ?_pragma=journal_mode(WAL) +// &_pragma=busy_timeout(5000) +// &_pragma=synchronous(NORMAL) +// &_pragma=foreign_keys(ON) +// +// Pass ":memory:" as the DSN path for ephemeral test databases. +package sqlite + +import "embed" + +//go:embed migrations/*.up.sql +var migrationFS embed.FS diff --git a/pkg/lore/store/sqlite/sqlite.go b/pkg/lore/store/sqlite/sqlite.go new file mode 100644 index 0000000..4e29433 --- /dev/null +++ b/pkg/lore/store/sqlite/sqlite.go @@ -0,0 +1,744 @@ +package sqlite + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "log/slog" + "strings" + "sync" + "time" + + // modernc.org/sqlite is a pure-Go SQLite driver (no CGO). The blank import + // registers the "sqlite" driver name with database/sql. + _ "modernc.org/sqlite" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" + + "github.com/mathomhaus/lore/pkg/lore" + "github.com/mathomhaus/lore/pkg/lore/store" +) + +const defaultLimit = 50 +const tracerName = "github.com/mathomhaus/lore" + +// Option configures a Store constructed by New. +type Option func(*sqliteStore) + +// WithLogger sets the logger used for migration progress and warnings. +// Defaults to slog.Default() when not provided. +func WithLogger(l *slog.Logger) Option { + return func(s *sqliteStore) { + if l != nil { + s.logger = l + } + } +} + +// WithTracer sets the OpenTelemetry tracer for span instrumentation. +// Defaults to otel.Tracer(tracerName) when not provided. +func WithTracer(t trace.Tracer) Option { + return func(s *sqliteStore) { + if t != nil { + s.tracer = t + } + } +} + +// WithDefaultLimit sets the default page size for list operations. +// Defaults to 50 when not provided. Values <= 0 are ignored. +func WithDefaultLimit(n int) Option { + return func(s *sqliteStore) { + if n > 0 { + s.defaultLimit = n + } + } +} + +// sqliteStore is the SQLite-backed implementation of store.Store. +type sqliteStore struct { + db *sql.DB + logger *slog.Logger + tracer trace.Tracer + defaultLimit int + + mu sync.RWMutex + closed bool +} + +// New returns a Store backed by the given *sql.DB. The caller owns the *sql.DB +// and is responsible for connection pool configuration and Close ordering. +// New runs pending migrations on the database; pass a fresh empty DB or one +// already at the latest schema_migrations version. +// +// The driver name for modernc.org/sqlite is "sqlite" (not "sqlite3"). A +// minimal DSN that applies all required pragmas per connection looks like: +// +// "file:path.db?_pragma=journal_mode(WAL)&_pragma=busy_timeout(5000)&_pragma=synchronous(NORMAL)&_pragma=foreign_keys(ON)" +func New(db *sql.DB, opts ...Option) (store.Store, error) { + if db == nil { + return nil, fmt.Errorf("sqlite: new: db must not be nil") + } + + s := &sqliteStore{ + db: db, + logger: slog.Default(), + tracer: otel.Tracer(tracerName), + defaultLimit: defaultLimit, + } + for _, o := range opts { + o(s) + } + + if err := Migrate(context.Background(), db, s.logger); err != nil { + return nil, fmt.Errorf("sqlite: new: migrate: %w", err) + } + + return s, nil +} + +// checkClosed returns ErrClosed if the store has been closed. +func (s *sqliteStore) checkClosed() error { + s.mu.RLock() + defer s.mu.RUnlock() + if s.closed { + return fmt.Errorf("sqlite: %w", lore.ErrClosed) + } + return nil +} + +// resolveLimit applies the caller-supplied limit, falling back to the store +// default when zero. +func (s *sqliteStore) resolveLimit(limit int) int { + if limit > 0 { + return limit + } + return s.defaultLimit +} + +// recordError marks a span as errored and logs the failure. +func (s *sqliteStore) recordError(span trace.Span, op string, err error) { + span.RecordError(err) + span.SetStatus(codes.Error, err.Error()) + s.logger.Error("lore store operation failed", "op", op, "error", err) +} + +// ---- Inscribe ---- + +// Inscribe persists a new entry and returns its storage-assigned ID. +func (s *sqliteStore) Inscribe(ctx context.Context, e lore.Entry) (int64, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.inscribe", + trace.WithAttributes( + attribute.String("lore.kind", string(e.Kind)), + attribute.String("lore.source", e.Source), + ), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "inscribe", err) + return 0, err + } + if err := e.Kind.Validate(); err != nil { + s.recordError(span, "inscribe", err) + return 0, fmt.Errorf("sqlite: inscribe: %w", err) + } + if strings.TrimSpace(e.Title) == "" { + err := fmt.Errorf("sqlite: inscribe: %w: title is required", lore.ErrInvalidArgument) + s.recordError(span, "inscribe", err) + return 0, err + } + + tagsJSON, err := marshalTags(e.Tags) + if err != nil { + s.recordError(span, "inscribe", err) + return 0, fmt.Errorf("sqlite: inscribe: marshal tags: %w", err) + } + metaJSON, err := marshalMetadata(e.Metadata) + if err != nil { + s.recordError(span, "inscribe", err) + return 0, fmt.Errorf("sqlite: inscribe: marshal metadata: %w", err) + } + + now := time.Now().UTC().Format(time.RFC3339) + res, err := s.db.ExecContext(ctx, + `INSERT INTO entries (project, kind, title, body, source, tags, metadata, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)`, + e.Project, string(e.Kind), e.Title, e.Body, e.Source, + tagsJSON, metaJSON, now, now, + ) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: inscribe: insert: %w", err) + s.recordError(span, "inscribe", wrappedErr) + return 0, wrappedErr + } + + id, err := res.LastInsertId() + if err != nil { + wrappedErr := fmt.Errorf("sqlite: inscribe: last insert id: %w", err) + s.recordError(span, "inscribe", wrappedErr) + return 0, wrappedErr + } + + span.SetAttributes(attribute.Int64("lore.id", id)) + return id, nil +} + +// ---- Update ---- + +// Update replaces all mutable fields of the entry identified by e.ID. +func (s *sqliteStore) Update(ctx context.Context, e lore.Entry) error { + ctx, span := s.tracer.Start(ctx, "lore.store.update", + trace.WithAttributes( + attribute.Int64("lore.id", e.ID), + attribute.String("lore.kind", string(e.Kind)), + ), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "update", err) + return err + } + if e.ID <= 0 { + err := fmt.Errorf("sqlite: update: %w: id must be positive", lore.ErrInvalidArgument) + s.recordError(span, "update", err) + return err + } + if err := e.Kind.Validate(); err != nil { + s.recordError(span, "update", err) + return fmt.Errorf("sqlite: update: %w", err) + } + if strings.TrimSpace(e.Title) == "" { + err := fmt.Errorf("sqlite: update: %w: title is required", lore.ErrInvalidArgument) + s.recordError(span, "update", err) + return err + } + + tagsJSON, err := marshalTags(e.Tags) + if err != nil { + s.recordError(span, "update", err) + return fmt.Errorf("sqlite: update: marshal tags: %w", err) + } + metaJSON, err := marshalMetadata(e.Metadata) + if err != nil { + s.recordError(span, "update", err) + return fmt.Errorf("sqlite: update: marshal metadata: %w", err) + } + + now := time.Now().UTC().Format(time.RFC3339) + res, err := s.db.ExecContext(ctx, + `UPDATE entries SET project=?, kind=?, title=?, body=?, source=?, tags=?, metadata=?, updated_at=? + WHERE id=?`, + e.Project, string(e.Kind), e.Title, e.Body, e.Source, + tagsJSON, metaJSON, now, e.ID, + ) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: update: %w", err) + s.recordError(span, "update", wrappedErr) + return wrappedErr + } + + n, err := res.RowsAffected() + if err != nil { + wrappedErr := fmt.Errorf("sqlite: update: rows affected: %w", err) + s.recordError(span, "update", wrappedErr) + return wrappedErr + } + if n == 0 { + wrappedErr := fmt.Errorf("sqlite: update: id %d: %w", e.ID, lore.ErrNotFound) + s.recordError(span, "update", wrappedErr) + return wrappedErr + } + return nil +} + +// ---- Get ---- + +// Get returns the entry with the given ID. +func (s *sqliteStore) Get(ctx context.Context, id int64) (lore.Entry, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.get", + trace.WithAttributes(attribute.Int64("lore.id", id)), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "get", err) + return lore.Entry{}, err + } + + row := s.db.QueryRowContext(ctx, + `SELECT id, project, kind, title, body, source, tags, metadata, created_at, updated_at + FROM entries WHERE id = ?`, id) + + e, err := scanEntry(row) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + wrappedErr := fmt.Errorf("sqlite: get: id %d: %w", id, lore.ErrNotFound) + s.recordError(span, "get", wrappedErr) + return lore.Entry{}, wrappedErr + } + wrappedErr := fmt.Errorf("sqlite: get: %w", err) + s.recordError(span, "get", wrappedErr) + return lore.Entry{}, wrappedErr + } + return e, nil +} + +// ---- DeleteBySource ---- + +// DeleteBySource removes all entries whose Source field exactly matches source. +func (s *sqliteStore) DeleteBySource(ctx context.Context, source string) (int, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.delete_by_source", + trace.WithAttributes(attribute.String("lore.source", source)), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "delete_by_source", err) + return 0, err + } + if source == "" { + err := fmt.Errorf("sqlite: delete_by_source: %w: source must not be empty", lore.ErrInvalidArgument) + s.recordError(span, "delete_by_source", err) + return 0, err + } + + res, err := s.db.ExecContext(ctx, `DELETE FROM entries WHERE source = ?`, source) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: delete_by_source: %w", err) + s.recordError(span, "delete_by_source", wrappedErr) + return 0, wrappedErr + } + + n, err := res.RowsAffected() + if err != nil { + wrappedErr := fmt.Errorf("sqlite: delete_by_source: rows affected: %w", err) + s.recordError(span, "delete_by_source", wrappedErr) + return 0, wrappedErr + } + return int(n), nil +} + +// ---- ListByTag ---- + +// ListByTag returns all entries that carry the given tag. +func (s *sqliteStore) ListByTag(ctx context.Context, tag string, opts lore.ListOpts) ([]lore.Entry, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.list_by_tag", + trace.WithAttributes( + attribute.String("lore.tag", tag), + attribute.Int("lore.limit", s.resolveLimit(opts.Limit)), + ), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "list_by_tag", err) + return nil, err + } + if tag == "" { + err := fmt.Errorf("sqlite: list_by_tag: %w: tag must not be empty", lore.ErrInvalidArgument) + s.recordError(span, "list_by_tag", err) + return nil, err + } + if opts.Limit < 0 { + err := fmt.Errorf("sqlite: list_by_tag: %w: limit must not be negative", lore.ErrInvalidArgument) + s.recordError(span, "list_by_tag", err) + return nil, err + } + + limit := s.resolveLimit(opts.Limit) + + // JSON contains match: the tag appears as a quoted value in the JSON array. + // e.g. tag="adr" matches tags='["adr","architecture"]' but not tags='["sadr"]'. + tagPattern := `%"` + strings.ReplaceAll(tag, `"`, `\"`) + `"%` + + rows, err := s.db.QueryContext(ctx, + `SELECT id, project, kind, title, body, source, tags, metadata, created_at, updated_at + FROM entries + WHERE tags LIKE ? + ORDER BY created_at DESC + LIMIT ? OFFSET ?`, + tagPattern, limit, opts.Offset, + ) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: list_by_tag: query: %w", err) + s.recordError(span, "list_by_tag", wrappedErr) + return nil, wrappedErr + } + defer func() { _ = rows.Close() }() + + return scanEntries(rows) +} + +// ---- ListByKind ---- + +// ListByKind returns all entries of the given kind. +func (s *sqliteStore) ListByKind(ctx context.Context, kind lore.Kind, opts lore.ListOpts) ([]lore.Entry, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.list_by_kind", + trace.WithAttributes( + attribute.String("lore.kind", string(kind)), + attribute.Int("lore.limit", s.resolveLimit(opts.Limit)), + ), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "list_by_kind", err) + return nil, err + } + if err := kind.Validate(); err != nil { + s.recordError(span, "list_by_kind", err) + return nil, fmt.Errorf("sqlite: list_by_kind: %w", err) + } + if opts.Limit < 0 { + err := fmt.Errorf("sqlite: list_by_kind: %w: limit must not be negative", lore.ErrInvalidArgument) + s.recordError(span, "list_by_kind", err) + return nil, err + } + + limit := s.resolveLimit(opts.Limit) + + rows, err := s.db.QueryContext(ctx, + `SELECT id, project, kind, title, body, source, tags, metadata, created_at, updated_at + FROM entries + WHERE kind = ? + ORDER BY created_at DESC + LIMIT ? OFFSET ?`, + string(kind), limit, opts.Offset, + ) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: list_by_kind: query: %w", err) + s.recordError(span, "list_by_kind", wrappedErr) + return nil, wrappedErr + } + defer func() { _ = rows.Close() }() + + return scanEntries(rows) +} + +// ---- SearchText ---- + +// SearchText runs a BM25 full-text query against Title and Body. +func (s *sqliteStore) SearchText(ctx context.Context, query string, opts lore.SearchOpts) ([]lore.SearchHit, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.search_text", + trace.WithAttributes( + attribute.Int("lore.query.length", len(query)), + attribute.Int("lore.limit", s.resolveLimit(opts.Limit)), + ), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "search_text", err) + return nil, err + } + if strings.TrimSpace(query) == "" { + err := fmt.Errorf("sqlite: search_text: %w: query must not be empty", lore.ErrInvalidArgument) + s.recordError(span, "search_text", err) + return nil, err + } + if opts.Limit < 0 { + err := fmt.Errorf("sqlite: search_text: %w: limit must not be negative", lore.ErrInvalidArgument) + s.recordError(span, "search_text", err) + return nil, err + } + + limit := s.resolveLimit(opts.Limit) + + var ( + sqlBuf strings.Builder + args []any + ) + + sqlBuf.WriteString(` + SELECT e.id, e.project, e.kind, e.title, e.body, e.source, e.tags, e.metadata, + e.created_at, e.updated_at, + -bm25(entries_fts) AS score + FROM entries_fts f + JOIN entries e ON e.id = f.rowid + WHERE entries_fts MATCH ?`) + args = append(args, query) + + if opts.Project != "" { + sqlBuf.WriteString(` AND e.project = ?`) + args = append(args, opts.Project) + } + + if len(opts.Kinds) > 0 { + placeholders := make([]string, len(opts.Kinds)) + for i, k := range opts.Kinds { + placeholders[i] = "?" + args = append(args, string(k)) + } + sqlBuf.WriteString(` AND e.kind IN (` + strings.Join(placeholders, ",") + `)`) + } + + sqlBuf.WriteString(` ORDER BY score DESC LIMIT ?`) + args = append(args, limit) + + rows, err := s.db.QueryContext(ctx, sqlBuf.String(), args...) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: search_text: query: %w", err) + s.recordError(span, "search_text", wrappedErr) + return nil, wrappedErr + } + defer func() { _ = rows.Close() }() + + var hits []lore.SearchHit + for rows.Next() { + var ( + e lore.Entry + score float64 + ) + if err := scanEntryWithScore(rows, &e, &score); err != nil { + wrappedErr := fmt.Errorf("sqlite: search_text: scan: %w", err) + s.recordError(span, "search_text", wrappedErr) + return nil, wrappedErr + } + hits = append(hits, lore.SearchHit{Entry: e, Score: score}) + } + if err := rows.Err(); err != nil { + wrappedErr := fmt.Errorf("sqlite: search_text: iterate: %w", err) + s.recordError(span, "search_text", wrappedErr) + return nil, wrappedErr + } + return hits, nil +} + +// ---- AddEdge ---- + +// AddEdge persists a directed edge. Re-adding an identical triple is a no-op. +func (s *sqliteStore) AddEdge(ctx context.Context, edge lore.Edge) error { + ctx, span := s.tracer.Start(ctx, "lore.store.add_edge", + trace.WithAttributes( + attribute.Int64("lore.edge.from", edge.FromID), + attribute.Int64("lore.edge.to", edge.ToID), + ), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "add_edge", err) + return err + } + if edge.Relation == "" { + err := fmt.Errorf("sqlite: add_edge: %w: relation must not be empty", lore.ErrInvalidArgument) + s.recordError(span, "add_edge", err) + return err + } + + // Verify both entries exist. + for _, id := range []int64{edge.FromID, edge.ToID} { + var exists int + err := s.db.QueryRowContext(ctx, `SELECT 1 FROM entries WHERE id = ?`, id).Scan(&exists) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + wrappedErr := fmt.Errorf("sqlite: add_edge: entry %d: %w", id, lore.ErrNotFound) + s.recordError(span, "add_edge", wrappedErr) + return wrappedErr + } + wrappedErr := fmt.Errorf("sqlite: add_edge: lookup entry %d: %w", id, err) + s.recordError(span, "add_edge", wrappedErr) + return wrappedErr + } + } + + now := time.Now().UTC().Format(time.RFC3339) + _, err := s.db.ExecContext(ctx, + `INSERT OR IGNORE INTO edges (from_id, to_id, relation, weight, created_at) + VALUES (?, ?, ?, ?, ?)`, + edge.FromID, edge.ToID, edge.Relation, edge.Weight, now, + ) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: add_edge: insert: %w", err) + s.recordError(span, "add_edge", wrappedErr) + return wrappedErr + } + return nil +} + +// ---- ListEdges ---- + +// ListEdges returns all edges whose FromID equals fromID. +func (s *sqliteStore) ListEdges(ctx context.Context, fromID int64) ([]lore.Edge, error) { + ctx, span := s.tracer.Start(ctx, "lore.store.list_edges", + trace.WithAttributes(attribute.Int64("lore.id", fromID)), + ) + defer span.End() + + if err := s.checkClosed(); err != nil { + s.recordError(span, "list_edges", err) + return nil, err + } + if fromID <= 0 { + err := fmt.Errorf("sqlite: list_edges: %w: fromID must be positive", lore.ErrInvalidArgument) + s.recordError(span, "list_edges", err) + return nil, err + } + + rows, err := s.db.QueryContext(ctx, + `SELECT from_id, to_id, relation, weight, created_at + FROM edges WHERE from_id = ? + ORDER BY created_at ASC`, + fromID, + ) + if err != nil { + wrappedErr := fmt.Errorf("sqlite: list_edges: query: %w", err) + s.recordError(span, "list_edges", wrappedErr) + return nil, wrappedErr + } + defer func() { _ = rows.Close() }() + + var out []lore.Edge + for rows.Next() { + var ( + e lore.Edge + createdAt string + ) + if err := rows.Scan(&e.FromID, &e.ToID, &e.Relation, &e.Weight, &createdAt); err != nil { + wrappedErr := fmt.Errorf("sqlite: list_edges: scan: %w", err) + s.recordError(span, "list_edges", wrappedErr) + return nil, wrappedErr + } + e.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + out = append(out, e) + } + if err := rows.Err(); err != nil { + wrappedErr := fmt.Errorf("sqlite: list_edges: iterate: %w", err) + s.recordError(span, "list_edges", wrappedErr) + return nil, wrappedErr + } + return out, nil +} + +// ---- Close ---- + +// Close is idempotent and returns nil after the first call. +func (s *sqliteStore) Close(_ context.Context) error { + s.mu.Lock() + defer s.mu.Unlock() + s.closed = true + return nil +} + +// ---- scan helpers ---- + +// rowScanner is satisfied by both *sql.Row and *sql.Rows. +type rowScanner interface { + Scan(dest ...any) error +} + +// scanEntry reads one row of the entries SELECT column list into a lore.Entry. +func scanEntry(row rowScanner) (lore.Entry, error) { + var ( + e lore.Entry + tagsJSON string + metaJSON string + createdAt string + updatedAt string + ) + if err := row.Scan( + &e.ID, &e.Project, &e.Kind, &e.Title, &e.Body, &e.Source, + &tagsJSON, &metaJSON, &createdAt, &updatedAt, + ); err != nil { + return lore.Entry{}, err + } + + if err := json.Unmarshal([]byte(tagsJSON), &e.Tags); err != nil { + e.Tags = nil + } + if len(e.Tags) == 0 { + e.Tags = nil + } + + if metaJSON != "" && metaJSON != "{}" { + var m map[string]string + if err := json.Unmarshal([]byte(metaJSON), &m); err == nil { + e.Metadata = m + } + } + + e.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + e.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt) + return e, nil +} + +// scanEntryWithScore reads a row that includes a trailing score column. +func scanEntryWithScore(row rowScanner, e *lore.Entry, score *float64) error { + var ( + tagsJSON string + metaJSON string + createdAt string + updatedAt string + ) + if err := row.Scan( + &e.ID, &e.Project, &e.Kind, &e.Title, &e.Body, &e.Source, + &tagsJSON, &metaJSON, &createdAt, &updatedAt, score, + ); err != nil { + return err + } + + if err := json.Unmarshal([]byte(tagsJSON), &e.Tags); err != nil { + e.Tags = nil + } + if len(e.Tags) == 0 { + e.Tags = nil + } + + if metaJSON != "" && metaJSON != "{}" { + var m map[string]string + if err := json.Unmarshal([]byte(metaJSON), &m); err == nil { + e.Metadata = m + } + } + + e.CreatedAt, _ = time.Parse(time.RFC3339, createdAt) + e.UpdatedAt, _ = time.Parse(time.RFC3339, updatedAt) + return nil +} + +// scanEntries iterates *sql.Rows and returns all scanned entries. +func scanEntries(rows *sql.Rows) ([]lore.Entry, error) { + var out []lore.Entry + for rows.Next() { + e, err := scanEntry(rows) + if err != nil { + return nil, fmt.Errorf("scan entry: %w", err) + } + out = append(out, e) + } + if err := rows.Err(); err != nil { + return nil, fmt.Errorf("iterate entries: %w", err) + } + return out, nil +} + +// ---- JSON marshal helpers ---- + +func marshalTags(tags []string) (string, error) { + if len(tags) == 0 { + return "[]", nil + } + b, err := json.Marshal(tags) + if err != nil { + return "", err + } + return string(b), nil +} + +func marshalMetadata(m map[string]string) (string, error) { + if len(m) == 0 { + return "{}", nil + } + b, err := json.Marshal(m) + if err != nil { + return "", err + } + return string(b), nil +} diff --git a/pkg/lore/store/sqlite/sqlite_test.go b/pkg/lore/store/sqlite/sqlite_test.go new file mode 100644 index 0000000..e34f678 --- /dev/null +++ b/pkg/lore/store/sqlite/sqlite_test.go @@ -0,0 +1,618 @@ +package sqlite_test + +import ( + "context" + "database/sql" + "errors" + "testing" + + _ "modernc.org/sqlite" + + "github.com/mathomhaus/lore/pkg/lore" + "github.com/mathomhaus/lore/pkg/lore/store" + "github.com/mathomhaus/lore/pkg/lore/store/sqlite" +) + +// openMemoryStore opens a fresh :memory: SQLite database and returns a Store +// backed by it. The test owns both the Store and the *sql.DB; it must call +// st.Close then db.Close when done. +func openMemoryStore(t *testing.T) (store.Store, *sql.DB) { + t.Helper() + db, err := sql.Open("sqlite", ":memory:?_pragma=foreign_keys(ON)") + if err != nil { + t.Fatalf("sql.Open: %v", err) + } + st, err := sqlite.New(db) + if err != nil { + _ = db.Close() + t.Fatalf("sqlite.New: %v", err) + } + return st, db +} + +func closeStore(t *testing.T, st store.Store, db *sql.DB) { + t.Helper() + if err := st.Close(context.Background()); err != nil { + t.Errorf("Store.Close: %v", err) + } + if err := db.Close(); err != nil { + t.Errorf("db.Close: %v", err) + } +} + +// sampleEntry returns a valid lore.Entry suitable for test inserts. +func sampleEntry(overrides ...func(*lore.Entry)) lore.Entry { + e := lore.Entry{ + Project: "testproject", + Kind: lore.KindDecision, + Title: "Use SQLite for local storage", + Body: "We chose SQLite because it is embeddable and requires no separate process.", + Source: "https://example.com/adr-001", + Tags: []string{"adr", "storage"}, + } + for _, fn := range overrides { + fn(&e) + } + return e +} + +// TestInscribe_Roundtrip verifies that inscribing an entry and fetching it by +// ID returns an entry with matching fields. +func TestInscribe_Roundtrip(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + in := sampleEntry() + + id, err := st.Inscribe(ctx, in) + if err != nil { + t.Fatalf("Inscribe: %v", err) + } + if id <= 0 { + t.Fatalf("expected positive id, got %d", id) + } + + got, err := st.Get(ctx, id) + if err != nil { + t.Fatalf("Get: %v", err) + } + + if got.ID != id { + t.Errorf("ID: got %d, want %d", got.ID, id) + } + if got.Project != in.Project { + t.Errorf("Project: got %q, want %q", got.Project, in.Project) + } + if got.Kind != in.Kind { + t.Errorf("Kind: got %q, want %q", got.Kind, in.Kind) + } + if got.Title != in.Title { + t.Errorf("Title: got %q, want %q", got.Title, in.Title) + } + if got.Body != in.Body { + t.Errorf("Body: got %q, want %q", got.Body, in.Body) + } + if got.Source != in.Source { + t.Errorf("Source: got %q, want %q", got.Source, in.Source) + } + if len(got.Tags) != len(in.Tags) { + t.Errorf("Tags length: got %d, want %d", len(got.Tags), len(in.Tags)) + } else { + for i := range in.Tags { + if got.Tags[i] != in.Tags[i] { + t.Errorf("Tags[%d]: got %q, want %q", i, got.Tags[i], in.Tags[i]) + } + } + } +} + +// TestInscribe_RejectsInvalidKind verifies that an entry with an unknown kind +// returns ErrInvalidKind and no row is written. +func TestInscribe_RejectsInvalidKind(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + e := sampleEntry(func(e *lore.Entry) { + e.Kind = "boguskind" + }) + + _, err := st.Inscribe(ctx, e) + if err == nil { + t.Fatal("expected error for invalid kind, got nil") + } + if !errors.Is(err, lore.ErrInvalidKind) { + t.Errorf("expected ErrInvalidKind, got %v", err) + } +} + +// TestInscribe_RejectsEmptyTitle verifies that an entry with an empty title +// returns ErrInvalidArgument. +func TestInscribe_RejectsEmptyTitle(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + e := sampleEntry(func(e *lore.Entry) { e.Title = " " }) + + _, err := st.Inscribe(ctx, e) + if err == nil { + t.Fatal("expected error for empty title, got nil") + } + if !errors.Is(err, lore.ErrInvalidArgument) { + t.Errorf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestUpdate_RoundTrip verifies that an update replaces entry fields. +func TestUpdate_RoundTrip(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + id, err := st.Inscribe(ctx, sampleEntry()) + if err != nil { + t.Fatalf("Inscribe: %v", err) + } + + updated := sampleEntry(func(e *lore.Entry) { + e.ID = id + e.Title = "Updated title" + e.Body = "Updated body" + e.Kind = lore.KindPrinciple + }) + if err := st.Update(ctx, updated); err != nil { + t.Fatalf("Update: %v", err) + } + + got, err := st.Get(ctx, id) + if err != nil { + t.Fatalf("Get after Update: %v", err) + } + if got.Title != "Updated title" { + t.Errorf("Title: got %q, want %q", got.Title, "Updated title") + } + if got.Kind != lore.KindPrinciple { + t.Errorf("Kind: got %q, want %q", got.Kind, lore.KindPrinciple) + } +} + +// TestUpdate_NotFound verifies that updating a non-existent ID returns +// ErrNotFound. +func TestUpdate_NotFound(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + e := sampleEntry(func(e *lore.Entry) { e.ID = 99999 }) + + err := st.Update(ctx, e) + if err == nil { + t.Fatal("expected ErrNotFound, got nil") + } + if !errors.Is(err, lore.ErrNotFound) { + t.Errorf("expected ErrNotFound, got %v", err) + } +} + +// TestGet_NotFound verifies that fetching a non-existent ID returns ErrNotFound. +func TestGet_NotFound(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + _, err := st.Get(ctx, 99999) + if err == nil { + t.Fatal("expected ErrNotFound, got nil") + } + if !errors.Is(err, lore.ErrNotFound) { + t.Errorf("expected ErrNotFound, got %v", err) + } +} + +// TestDeleteBySource_Multiple verifies that all entries with a given source are +// deleted and the correct count is returned. +func TestDeleteBySource_Multiple(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + src := "https://example.com/shared-source" + + for i := 0; i < 3; i++ { + e := sampleEntry(func(e *lore.Entry) { e.Source = src }) + if _, err := st.Inscribe(ctx, e); err != nil { + t.Fatalf("Inscribe[%d]: %v", i, err) + } + } + // One extra with a different source. + other := sampleEntry(func(e *lore.Entry) { e.Source = "https://example.com/other" }) + otherID, err := st.Inscribe(ctx, other) + if err != nil { + t.Fatalf("Inscribe other: %v", err) + } + + deleted, err := st.DeleteBySource(ctx, src) + if err != nil { + t.Fatalf("DeleteBySource: %v", err) + } + if deleted != 3 { + t.Errorf("deleted count: got %d, want 3", deleted) + } + + // The other entry should still be accessible. + if _, err := st.Get(ctx, otherID); err != nil { + t.Errorf("Get other entry after delete: %v", err) + } +} + +// TestDeleteBySource_Empty verifies that an empty source returns +// ErrInvalidArgument. +func TestDeleteBySource_Empty(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + _, err := st.DeleteBySource(context.Background(), "") + if !errors.Is(err, lore.ErrInvalidArgument) { + t.Errorf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestListByTag_Filter verifies that only entries carrying the queried tag are +// returned and entries without the tag are excluded. +func TestListByTag_Filter(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + + e1 := sampleEntry(func(e *lore.Entry) { + e.Title = "Entry with adr tag" + e.Tags = []string{"adr", "architecture"} + }) + e2 := sampleEntry(func(e *lore.Entry) { + e.Title = "Entry with only architecture tag" + e.Tags = []string{"architecture"} + }) + e3 := sampleEntry(func(e *lore.Entry) { + e.Title = "Entry with no tags" + e.Tags = nil + }) + + for _, e := range []lore.Entry{e1, e2, e3} { + if _, err := st.Inscribe(ctx, e); err != nil { + t.Fatalf("Inscribe: %v", err) + } + } + + hits, err := st.ListByTag(ctx, "adr", lore.ListOpts{}) + if err != nil { + t.Fatalf("ListByTag: %v", err) + } + if len(hits) != 1 { + t.Fatalf("expected 1 result, got %d", len(hits)) + } + if hits[0].Title != e1.Title { + t.Errorf("Title: got %q, want %q", hits[0].Title, e1.Title) + } +} + +// TestListByKind_Filter verifies that only entries of the queried kind are +// returned. +func TestListByKind_Filter(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + + decisions := []lore.Entry{ + sampleEntry(func(e *lore.Entry) { e.Kind = lore.KindDecision; e.Title = "Decision A" }), + sampleEntry(func(e *lore.Entry) { e.Kind = lore.KindDecision; e.Title = "Decision B" }), + } + principles := []lore.Entry{ + sampleEntry(func(e *lore.Entry) { e.Kind = lore.KindPrinciple; e.Title = "Principle A" }), + } + + for _, e := range append(decisions, principles...) { + if _, err := st.Inscribe(ctx, e); err != nil { + t.Fatalf("Inscribe: %v", err) + } + } + + got, err := st.ListByKind(ctx, lore.KindDecision, lore.ListOpts{}) + if err != nil { + t.Fatalf("ListByKind: %v", err) + } + if len(got) != 2 { + t.Errorf("expected 2 decisions, got %d", len(got)) + } + for _, e := range got { + if e.Kind != lore.KindDecision { + t.Errorf("unexpected kind %q in results", e.Kind) + } + } +} + +// TestListByKind_InvalidKind verifies that an invalid kind returns ErrInvalidKind. +func TestListByKind_InvalidKind(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + _, err := st.ListByKind(context.Background(), "notakind", lore.ListOpts{}) + if !errors.Is(err, lore.ErrInvalidKind) { + t.Errorf("expected ErrInvalidKind, got %v", err) + } +} + +// TestSearchText_BM25Ranking verifies that a known-relevant document ranks +// above an unrelated document. +func TestSearchText_BM25Ranking(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + + relevant := sampleEntry(func(e *lore.Entry) { + e.Title = "SQLite full text search with BM25" + e.Body = "BM25 ranking in SQLite FTS5 provides excellent relevance scoring for text retrieval." + }) + unrelated := sampleEntry(func(e *lore.Entry) { + e.Title = "Kubernetes deployment strategy" + e.Body = "Rolling updates minimize downtime during deployments in kubernetes clusters." + }) + + for _, e := range []lore.Entry{relevant, unrelated} { + if _, err := st.Inscribe(ctx, e); err != nil { + t.Fatalf("Inscribe: %v", err) + } + } + + hits, err := st.SearchText(ctx, "BM25 SQLite", lore.SearchOpts{Limit: 10}) + if err != nil { + t.Fatalf("SearchText: %v", err) + } + if len(hits) == 0 { + t.Fatal("expected at least one hit, got zero") + } + + // The first (highest-scoring) hit should be the relevant document. + if hits[0].Entry.Title != relevant.Title { + t.Errorf("top hit title: got %q, want %q", hits[0].Entry.Title, relevant.Title) + } + + // Scores must be positive (we negate bm25() which returns negative values). + for i, h := range hits { + if h.Score <= 0 { + t.Errorf("hit[%d] score %f is not positive", i, h.Score) + } + } +} + +// TestSearchText_EmptyQuery verifies that an empty query returns +// ErrInvalidArgument. +func TestSearchText_EmptyQuery(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + _, err := st.SearchText(context.Background(), " ", lore.SearchOpts{}) + if !errors.Is(err, lore.ErrInvalidArgument) { + t.Errorf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestAddEdge_RoundTrip verifies that an added edge is returned by ListEdges. +func TestAddEdge_RoundTrip(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + id1, err := st.Inscribe(ctx, sampleEntry(func(e *lore.Entry) { e.Title = "Entry One" })) + if err != nil { + t.Fatalf("Inscribe id1: %v", err) + } + id2, err := st.Inscribe(ctx, sampleEntry(func(e *lore.Entry) { e.Title = "Entry Two" })) + if err != nil { + t.Fatalf("Inscribe id2: %v", err) + } + + edge := lore.Edge{ + FromID: id1, + ToID: id2, + Relation: "informs", + Weight: 1.0, + } + if err := st.AddEdge(ctx, edge); err != nil { + t.Fatalf("AddEdge: %v", err) + } + + edges, err := st.ListEdges(ctx, id1) + if err != nil { + t.Fatalf("ListEdges: %v", err) + } + if len(edges) != 1 { + t.Fatalf("expected 1 edge, got %d", len(edges)) + } + if edges[0].FromID != id1 { + t.Errorf("FromID: got %d, want %d", edges[0].FromID, id1) + } + if edges[0].ToID != id2 { + t.Errorf("ToID: got %d, want %d", edges[0].ToID, id2) + } + if edges[0].Relation != "informs" { + t.Errorf("Relation: got %q, want %q", edges[0].Relation, "informs") + } +} + +// TestAddEdge_Idempotent verifies that adding the same edge twice is a no-op. +func TestAddEdge_Idempotent(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + id1, _ := st.Inscribe(ctx, sampleEntry(func(e *lore.Entry) { e.Title = "A" })) + id2, _ := st.Inscribe(ctx, sampleEntry(func(e *lore.Entry) { e.Title = "B" })) + + edge := lore.Edge{FromID: id1, ToID: id2, Relation: "informs"} + if err := st.AddEdge(ctx, edge); err != nil { + t.Fatalf("first AddEdge: %v", err) + } + if err := st.AddEdge(ctx, edge); err != nil { + t.Fatalf("second AddEdge (should be idempotent): %v", err) + } + + edges, err := st.ListEdges(ctx, id1) + if err != nil { + t.Fatalf("ListEdges: %v", err) + } + if len(edges) != 1 { + t.Errorf("expected 1 edge after two identical adds, got %d", len(edges)) + } +} + +// TestAddEdge_NotFound verifies that adding an edge with a non-existent entry +// returns ErrNotFound. +func TestAddEdge_NotFound(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + id1, _ := st.Inscribe(ctx, sampleEntry()) + + err := st.AddEdge(ctx, lore.Edge{FromID: id1, ToID: 99999, Relation: "informs"}) + if !errors.Is(err, lore.ErrNotFound) { + t.Errorf("expected ErrNotFound, got %v", err) + } +} + +// TestListEdges_FromID verifies that edges are scoped to the given FromID. +func TestListEdges_FromID(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + ids := make([]int64, 3) + for i := range ids { + id, err := st.Inscribe(ctx, sampleEntry(func(e *lore.Entry) { + e.Title = "Entry" + })) + if err != nil { + t.Fatalf("Inscribe[%d]: %v", i, err) + } + ids[i] = id + } + + // Add edges from ids[0] to ids[1] and ids[2]. + for _, toID := range ids[1:] { + if err := st.AddEdge(ctx, lore.Edge{FromID: ids[0], ToID: toID, Relation: "informs"}); err != nil { + t.Fatalf("AddEdge: %v", err) + } + } + // Add edge from ids[1] to ids[2]. Should NOT appear in ids[0] list. + if err := st.AddEdge(ctx, lore.Edge{FromID: ids[1], ToID: ids[2], Relation: "informs"}); err != nil { + t.Fatalf("AddEdge ids[1]->ids[2]: %v", err) + } + + edges, err := st.ListEdges(ctx, ids[0]) + if err != nil { + t.Fatalf("ListEdges: %v", err) + } + if len(edges) != 2 { + t.Errorf("expected 2 edges from ids[0], got %d", len(edges)) + } + for _, e := range edges { + if e.FromID != ids[0] { + t.Errorf("unexpected FromID %d in results", e.FromID) + } + } +} + +// TestListEdges_InvalidFromID verifies that fromID <= 0 returns ErrInvalidArgument. +func TestListEdges_InvalidFromID(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + _, err := st.ListEdges(context.Background(), 0) + if !errors.Is(err, lore.ErrInvalidArgument) { + t.Errorf("expected ErrInvalidArgument for fromID=0, got %v", err) + } +} + +// TestClose_Idempotent verifies that calling Close multiple times returns nil. +func TestClose_Idempotent(t *testing.T) { + st, db := openMemoryStore(t) + defer func() { _ = db.Close() }() + + ctx := context.Background() + if err := st.Close(ctx); err != nil { + t.Fatalf("first Close: %v", err) + } + if err := st.Close(ctx); err != nil { + t.Errorf("second Close (should be idempotent): %v", err) + } +} + +// TestAfterClose_ReturnsErrClosed verifies that method calls after Close return +// ErrClosed. +func TestAfterClose_ReturnsErrClosed(t *testing.T) { + st, db := openMemoryStore(t) + defer func() { _ = db.Close() }() + + ctx := context.Background() + if err := st.Close(ctx); err != nil { + t.Fatalf("Close: %v", err) + } + + _, err := st.Inscribe(ctx, sampleEntry()) + if !errors.Is(err, lore.ErrClosed) { + t.Errorf("Inscribe after Close: expected ErrClosed, got %v", err) + } + + _, err = st.Get(ctx, 1) + if !errors.Is(err, lore.ErrClosed) { + t.Errorf("Get after Close: expected ErrClosed, got %v", err) + } +} + +// TestListByTag_EmptyTag verifies that an empty tag returns ErrInvalidArgument. +func TestListByTag_EmptyTag(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + _, err := st.ListByTag(context.Background(), "", lore.ListOpts{}) + if !errors.Is(err, lore.ErrInvalidArgument) { + t.Errorf("expected ErrInvalidArgument, got %v", err) + } +} + +// TestMetadata_RoundTrip verifies that metadata key-value pairs survive a +// write-read cycle. +func TestMetadata_RoundTrip(t *testing.T) { + st, db := openMemoryStore(t) + defer closeStore(t, st, db) + + ctx := context.Background() + in := sampleEntry(func(e *lore.Entry) { + e.Metadata = map[string]string{ + "component": "auth", + "owner": "platform-team", + } + }) + + id, err := st.Inscribe(ctx, in) + if err != nil { + t.Fatalf("Inscribe: %v", err) + } + + got, err := st.Get(ctx, id) + if err != nil { + t.Fatalf("Get: %v", err) + } + + if got.Metadata["component"] != "auth" { + t.Errorf("metadata component: got %q, want %q", got.Metadata["component"], "auth") + } + if got.Metadata["owner"] != "platform-team" { + t.Errorf("metadata owner: got %q, want %q", got.Metadata["owner"], "platform-team") + } +} diff --git a/pkg/lore/store/store.go b/pkg/lore/store/store.go new file mode 100644 index 0000000..8e9cd12 --- /dev/null +++ b/pkg/lore/store/store.go @@ -0,0 +1,124 @@ +// Package store defines the Store interface that all lore persistence backends +// must satisfy. The interface is small and deliberate: every method maps to a +// distinct access pattern (point read, scan, text search, edge traversal) so +// callers can reason about cost without understanding implementation internals. +// +// Error contract: +// - Every method returns a non-nil error on failure. +// - Callers may test returned errors with errors.Is against the sentinel +// values in the parent lore package: ErrNotFound, ErrDuplicate, +// ErrInvalidKind, ErrInvalidArgument, ErrConflict, ErrUnsupported, +// ErrClosed. +// - Implementations wrap sentinels with fmt.Errorf("operation: %w", sentinel) +// so the full operation context is available via err.Error() while +// errors.Is still resolves the sentinel. +// +// Lifecycle contract: +// - Callers own the underlying *sql.DB (or equivalent resource). They +// configure pool sizing, set pragmas, and call Close on the DB after they +// have called Store.Close. +// - Store.Close is idempotent: calling it more than once returns nil. +// - After Close, all other methods return ErrClosed. +package store + +import ( + "context" + + "github.com/mathomhaus/lore/pkg/lore" +) + +// Store persists lore entries and edges. Implementations are caller-managed: +// constructors accept a caller-owned *sql.DB (or equivalent); consumers manage +// pool configuration and lifecycle. New runs pending migrations automatically. +// +// All methods accept a context.Context as their first argument. Cancellation +// or deadline expiry propagates to the underlying database driver and returns +// the context error wrapped with operation context. +type Store interface { + // Inscribe persists a new entry and returns its storage-assigned ID. + // + // Errors: + // - ErrInvalidKind when e.Kind is outside the canonical taxonomy. + // - ErrInvalidArgument when e.Title or e.Body is empty. + // - ErrDuplicate when the implementation enforces a uniqueness + // constraint and it would be violated. + // - ErrClosed when the store has been closed. + Inscribe(ctx context.Context, e lore.Entry) (id int64, err error) + + // Update replaces all mutable fields of the entry identified by e.ID. + // The caller must supply a fully-populated Entry (not a partial patch). + // + // Errors: + // - ErrNotFound when e.ID does not match any persisted entry. + // - ErrInvalidKind when e.Kind is outside the canonical taxonomy. + // - ErrInvalidArgument when e.Title or e.Body is empty. + // - ErrClosed when the store has been closed. + Update(ctx context.Context, e lore.Entry) error + + // Get returns the entry with the given ID. + // + // Errors: + // - ErrNotFound when id does not match any persisted entry. + // - ErrClosed when the store has been closed. + Get(ctx context.Context, id int64) (lore.Entry, error) + + // DeleteBySource removes all entries whose Source field exactly matches + // source and returns the count of deleted rows. Deleting a non-existent + // source is not an error: deleted returns 0 and err is nil. + // + // Errors: + // - ErrInvalidArgument when source is empty. + // - ErrClosed when the store has been closed. + DeleteBySource(ctx context.Context, source string) (deleted int, err error) + + // ListByTag returns all entries that carry the given tag, subject to opts. + // The tag match is exact: "adr" does not match "adr-2024". Results are + // ordered by created_at descending (newest first). + // + // Errors: + // - ErrInvalidArgument when tag is empty or opts.Limit is negative. + // - ErrClosed when the store has been closed. + ListByTag(ctx context.Context, tag string, opts lore.ListOpts) ([]lore.Entry, error) + + // ListByKind returns all entries of the given kind, subject to opts. + // Results are ordered by created_at descending (newest first). + // + // Errors: + // - ErrInvalidKind when kind is outside the canonical taxonomy. + // - ErrInvalidArgument when opts.Limit is negative. + // - ErrClosed when the store has been closed. + ListByKind(ctx context.Context, kind lore.Kind, opts lore.ListOpts) ([]lore.Entry, error) + + // SearchText runs a full-text query against Title and Body and returns + // ranked hits. The query string is passed to the FTS engine as-is; callers + // are responsible for any tokenization pre-processing. Higher Score values + // are better; scores are not comparable across queries or implementations. + // + // Errors: + // - ErrInvalidArgument when query is empty or opts.Limit is negative. + // - ErrClosed when the store has been closed. + SearchText(ctx context.Context, query string, opts lore.SearchOpts) ([]lore.SearchHit, error) + + // AddEdge persists a directed edge from edge.FromID to edge.ToID labeled + // edge.Relation. Re-adding an identical (FromID, ToID, Relation) triple is + // a no-op and returns nil (idempotent). + // + // Errors: + // - ErrNotFound when FromID or ToID does not match any persisted entry. + // - ErrInvalidArgument when Relation is empty. + // - ErrClosed when the store has been closed. + AddEdge(ctx context.Context, edge lore.Edge) error + + // ListEdges returns all edges whose FromID equals fromID. The result may + // be empty when no edges have been added from fromID. + // + // Errors: + // - ErrInvalidArgument when fromID is zero or negative. + // - ErrClosed when the store has been closed. + ListEdges(ctx context.Context, fromID int64) ([]lore.Edge, error) + + // Close releases any resources held by the store. After Close returns, all + // subsequent calls to store methods return ErrClosed. Close is idempotent: + // calling it more than once returns nil. + Close(ctx context.Context) error +}