From 9f54d8d7fffd1f389c40e1a3cc21b6649ce3353a Mon Sep 17 00:00:00 2001 From: Aaron Stannard Date: Mon, 27 Apr 2026 21:09:45 +0000 Subject: [PATCH] Add lightweight database migration system (#44) Replace inline schema constant in DatabaseInitializer with numbered SQL migration files and a SchemaMigrator that tracks applied versions in a schema_version table. Existing databases are handled safely via IF NOT EXISTS clauses in migration 001. --- src/SkillServer/Data/DatabaseInitializer.cs | 104 +----------------- src/SkillServer/Data/SchemaMigrator.cs | 98 +++++++++++++++++ src/SkillServer/SkillServer.csproj | 6 + .../migrations/001_initial_schema.sql | 88 +++++++++++++++ 4 files changed, 194 insertions(+), 102 deletions(-) create mode 100644 src/SkillServer/Data/SchemaMigrator.cs create mode 100644 src/SkillServer/migrations/001_initial_schema.sql diff --git a/src/SkillServer/Data/DatabaseInitializer.cs b/src/SkillServer/Data/DatabaseInitializer.cs index d303408..86623c7 100644 --- a/src/SkillServer/Data/DatabaseInitializer.cs +++ b/src/SkillServer/Data/DatabaseInitializer.cs @@ -3,13 +3,11 @@ // Copyright (C) 2026 - 2026 Petabridge, LLC // // ----------------------------------------------------------------------- -using Dapper; -using Microsoft.Data.Sqlite; namespace SkillServer.Data; /// -/// Initializes the SQLite database schema. +/// Initializes the SQLite database schema via numbered migrations. /// public sealed class DatabaseInitializer { @@ -28,104 +26,6 @@ public DatabaseInitializer(IConfiguration configuration, ILogger +// Copyright (C) 2026 - 2026 Petabridge, LLC +// +// ----------------------------------------------------------------------- + +using Microsoft.Data.Sqlite; + +namespace SkillServer.Data; + +public static class SchemaMigrator +{ + public static async Task MigrateAsync(string connectionString, ILogger logger, + CancellationToken ct = default) + { + logger.LogInformation("Starting database schema migration..."); + + await using var conn = new SqliteConnection(connectionString); + await conn.OpenAsync(ct); + + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = """ + CREATE TABLE IF NOT EXISTS schema_version ( + version INT PRIMARY KEY, + name TEXT NOT NULL, + applied_at TEXT NOT NULL + ); + """; + await cmd.ExecuteNonQueryAsync(ct); + } + + var appliedMigrations = new HashSet(); + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = "SELECT version FROM schema_version"; + await using var reader = await cmd.ExecuteReaderAsync(ct); + while (await reader.ReadAsync(ct)) + { + appliedMigrations.Add(reader.GetInt32(0)); + } + } + + var migrationsDir = Path.Combine(AppContext.BaseDirectory, "migrations"); + if (!Directory.Exists(migrationsDir)) + { + logger.LogWarning("Migrations directory not found: {MigrationsDir}", migrationsDir); + return; + } + + var migrationFiles = Directory.GetFiles(migrationsDir, "*.sql") + .Select(f => new { Path = f, Name = Path.GetFileName(f) }) + .Select(f => new { f.Path, f.Name, Version = ParseVersion(f.Name) }) + .Where(f => f.Version != null) + .OrderBy(f => f.Version) + .ToList(); + + foreach (var migration in migrationFiles) + { + var version = migration.Version!.Value; + if (appliedMigrations.Contains(version)) + { + logger.LogInformation("Migration {Name} already applied", migration.Name); + continue; + } + + logger.LogInformation("Applying migration: {Name}", migration.Name); + var sql = await File.ReadAllTextAsync(migration.Path, ct); + + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = sql; + await cmd.ExecuteNonQueryAsync(ct); + } + + await using (var cmd = conn.CreateCommand()) + { + cmd.CommandText = "INSERT INTO schema_version (version, name, applied_at) VALUES (@version, @name, @appliedAt)"; + cmd.Parameters.AddWithValue("@version", version); + cmd.Parameters.AddWithValue("@name", migration.Name); + cmd.Parameters.AddWithValue("@appliedAt", DateTimeOffset.UtcNow.ToString("o")); + await cmd.ExecuteNonQueryAsync(ct); + } + + logger.LogInformation("Migration {Name} applied successfully", migration.Name); + } + + logger.LogInformation("Database schema migration completed successfully"); + } + + internal static int? ParseVersion(string fileName) + { + var parts = fileName.Split('_', 2); + if (parts.Length < 2) return null; + if (int.TryParse(parts[0], out var version)) return version; + return null; + } +} diff --git a/src/SkillServer/SkillServer.csproj b/src/SkillServer/SkillServer.csproj index 90ef5d7..054ab1b 100644 --- a/src/SkillServer/SkillServer.csproj +++ b/src/SkillServer/SkillServer.csproj @@ -39,4 +39,10 @@ + + + PreserveNewest + + + diff --git a/src/SkillServer/migrations/001_initial_schema.sql b/src/SkillServer/migrations/001_initial_schema.sql new file mode 100644 index 0000000..9749118 --- /dev/null +++ b/src/SkillServer/migrations/001_initial_schema.sql @@ -0,0 +1,88 @@ +-- 001_initial_schema.sql: Initial schema (retrofitted from DatabaseInitializer.cs) + +CREATE TABLE IF NOT EXISTS skills ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + name TEXT NOT NULL UNIQUE COLLATE NOCASE, + created_at TEXT NOT NULL, + updated_at TEXT NOT NULL +); + +CREATE TABLE IF NOT EXISTS skill_versions ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + skill_id INTEGER NOT NULL REFERENCES skills(id) ON DELETE CASCADE, + version TEXT NOT NULL, + description TEXT NOT NULL, + category TEXT, + skill_type TEXT NOT NULL CHECK(skill_type IN ('skill-md', 'archive')), + sha256 TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + published_at TEXT NOT NULL, + is_latest INTEGER NOT NULL DEFAULT 0, + UNIQUE(skill_id, version) +); + +CREATE TABLE IF NOT EXISTS skill_files ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + skill_version_id INTEGER NOT NULL REFERENCES skill_versions(id) ON DELETE CASCADE, + relative_path TEXT NOT NULL, + sha256 TEXT NOT NULL, + size_bytes INTEGER NOT NULL, + UNIQUE(skill_version_id, relative_path) +); + +CREATE INDEX IF NOT EXISTS idx_skill_versions_latest + ON skill_versions(skill_id) WHERE is_latest = 1; + +CREATE INDEX IF NOT EXISTS idx_skill_versions_sha256 + ON skill_versions(sha256); + +CREATE INDEX IF NOT EXISTS idx_skill_files_sha256 + ON skill_files(sha256); + +CREATE TABLE IF NOT EXISTS api_keys ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + label TEXT NOT NULL, + key_hash TEXT NOT NULL UNIQUE, + created_at TEXT NOT NULL, + expires_at TEXT +); + +CREATE INDEX IF NOT EXISTS idx_api_keys_hash + ON api_keys(key_hash); + +-- Full-text search index for skill discovery +CREATE VIRTUAL TABLE IF NOT EXISTS skills_fts USING fts5( + name, + description, + category +); + +-- Keep FTS in sync when new versions are published +CREATE TRIGGER IF NOT EXISTS trg_skills_fts_insert +AFTER INSERT ON skill_versions +WHEN NEW.is_latest = 1 +BEGIN + DELETE FROM skills_fts WHERE rowid = NEW.skill_id; + INSERT INTO skills_fts(rowid, name, description, category) + SELECT NEW.skill_id, s.name, NEW.description, COALESCE(NEW.category, '') + FROM skills s WHERE s.id = NEW.skill_id; +END; + +-- Keep FTS in sync when versions are deleted +CREATE TRIGGER IF NOT EXISTS trg_skills_fts_delete +AFTER DELETE ON skill_versions +BEGIN + DELETE FROM skills_fts WHERE rowid = OLD.skill_id; + INSERT INTO skills_fts(rowid, name, description, category) + SELECT s.id, s.name, sv.description, COALESCE(sv.category, '') + FROM skills s + JOIN skill_versions sv ON sv.skill_id = s.id AND sv.is_latest = 1 + WHERE s.id = OLD.skill_id; +END; + +-- Backfill FTS from existing data (idempotent for existing databases) +INSERT INTO skills_fts(rowid, name, description, category) +SELECT s.id, s.name, sv.description, COALESCE(sv.category, '') +FROM skills s +JOIN skill_versions sv ON sv.skill_id = s.id AND sv.is_latest = 1 +WHERE s.id NOT IN (SELECT rowid FROM skills_fts);