feat: add db migration tool

go.work

@@ -24,5 +24,6 @@ use (
 	./sdk/go/examples/time
 	./sdk/go/examples/vectors
 	./sdk/go/templates/default
+	./tools/modus-dbmigrate
 	./tools/modus-keygen
 )

runtime/Makefile

@@ -19,12 +19,16 @@ tidy:
 	go mod tidy -v
 	go fmt ./...
 
+.PHONY: copy-db-migrations
+copy-db-migrations:
+	cp db/migrations/*.sql ../tools/modus-dbmigrate/migrations/
+
 .PHONY: build-explorer
 build-explorer:
 	cd explorer/content && npm ci && npm run build
 
 .PHONY: build
-build: build-explorer
+build: build-explorer copy-db-migrations
 	go build -o $(EXECUTABLE) -ldflags "$(LDFLAGS)" .
 
 .PHONY: run

tools/modus-dbmigrate/.gitignore

@@ -0,0 +1,2 @@
+modus-dbmigrate
+modus-dbmigrate.exe

tools/modus-dbmigrate/README.md

@@ -0,0 +1,25 @@
+# DB Migrate Tool
+
+This is a small utility that will apply database migrations for the Modus runtime's PostgreSQL DB.
+
+## Usage
+
+You must have Go v1.25 or newer installed locally. We currently do not release this tool as a
+binary.
+
+Before running, set the `MODUS_DB` environment variable to the connection string URI for the
+PostgreSQL database you want to target.
+
+You can run the tool in one of two ways:
+
+- Clone this repo, then from the `/tools/modus-dbmigrate` directory, run the utility via: `go run .`
+- Install it with `go install github.com/hypermodeinc/modus/tools/modus-dbmigrate@latest` then run
+  `modus-dbmigrate`
+
+The tool will connect to the database and apply any migrations required to bring the schema up to
+date.
+
+## Note
+
+The migration tool embeds the appropriate migration scripts. You do not need to copy them separately
+or install any other tool.

tools/modus-dbmigrate/go.mod

@@ -0,0 +1,11 @@
+module github.com/hypermodeinc/modus/tools/modus-dbmigrate
+
+go 1.25.0
+
+require github.com/golang-migrate/migrate/v4 v4.19.0
+
+require (
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/lib/pq v1.10.9 // indirect
+)

tools/modus-dbmigrate/go.sum

@@ -0,0 +1,67 @@
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0=
+github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E=
+github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY=
+github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU=
+github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI=
+github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M=
+github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE=
+github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/dhui/dktest v0.4.6 h1:+DPKyScKSEp3VLtbMDHcUq6V5Lm5zfZZVb0Sk7Ahom4=
+github.com/dhui/dktest v0.4.6/go.mod h1:JHTSYDtKkvFNFHJKqCzVzqXecyv+tKt8EzceOmQOgbU=
+github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk=
+github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E=
+github.com/docker/docker v28.3.3+incompatible h1:Dypm25kh4rmk49v1eiVbsAtpAsYURjYkaKubwuBdxEI=
+github.com/docker/docker v28.3.3+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk=
+github.com/docker/go-connections v0.5.0 h1:USnMq7hx7gwdVZq1L49hLXaFtUdTADjXGp+uj1Br63c=
+github.com/docker/go-connections v0.5.0/go.mod h1:ov60Kzw0kKElRwhNs9UlUHAE/F9Fe6GLaXnqyDdmEXc=
+github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4=
+github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg=
+github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
+github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
+github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
+github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
+github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
+github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
+github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q=
+github.com/golang-migrate/migrate/v4 v4.19.0 h1:RcjOnCGz3Or6HQYEJ/EEVLfWnmw9KnoigPSjzhCuaSE=
+github.com/golang-migrate/migrate/v4 v4.19.0/go.mod h1:9dyEcu+hO+G9hPSw8AIg50yg622pXJsoHItQnDGZkI0=
+github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/errwrap v1.1.0 h1:OxrOeh75EUXMY8TBjag2fzXGZ40LB6IKw45YeGUDY2I=
+github.com/hashicorp/errwrap v1.1.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
+github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo=
+github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM=
+github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw=
+github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o=
+github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0=
+github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo=
+github.com/moby/term v0.5.0 h1:xt8Q1nalod/v7BqbG21f8mQPqH+xAaC9C3N3wfWbVP0=
+github.com/moby/term v0.5.0/go.mod h1:8FzsFHVUBGZdbDsJw/ot+X+d5HLUbvklYLJ9uGfcI3Y=
+github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
+github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
+github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug=
+github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM=
+github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
+github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
+github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+go.opentelemetry.io/auto/sdk v1.1.0 h1:cH53jehLUN6UFLY71z+NDOiNJqDdPRaXzTel0sJySYA=
+go.opentelemetry.io/auto/sdk v1.1.0/go.mod h1:3wSPjt5PWp2RhlCcmmOial7AvC4DQqZb7a7wCow3W8A=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk=
+go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8=
+go.opentelemetry.io/otel v1.37.0 h1:9zhNfelUvx0KBfu/gb+ZgeAfAgtWrfHJZcAqFC228wQ=
+go.opentelemetry.io/otel v1.37.0/go.mod h1:ehE/umFRLnuLa/vSccNq9oS1ErUlkkK71gMcN34UG8I=
+go.opentelemetry.io/otel/metric v1.37.0 h1:mvwbQS5m0tbmqML4NqK+e3aDiO02vsf/WgbsdpcPoZE=
+go.opentelemetry.io/otel/metric v1.37.0/go.mod h1:04wGrZurHYKOc+RKeye86GwKiTb9FKm1WHtO+4EVr2E=
+go.opentelemetry.io/otel/trace v1.37.0 h1:HLdcFNbRQBE2imdSEgm/kwqmQj1Or1l/7bW6mxVK7z4=
+go.opentelemetry.io/otel/trace v1.37.0/go.mod h1:TlgrlQ+PtQO5XFerSPUYG0JSgGyryXewPGyayAWSBS0=
+golang.org/x/sys v0.31.0 h1:ioabZlmFYtWhL+TRYpcnNlLwhyxaM9kWTDEmfnprqik=
+golang.org/x/sys v0.31.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
+gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

tools/modus-dbmigrate/main.go

@@ -0,0 +1,58 @@
+package main
+
+import (
+	"embed"
+	"errors"
+	"fmt"
+	"os"
+
+	"github.com/golang-migrate/migrate/v4"
+	_ "github.com/golang-migrate/migrate/v4/database/postgres"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+)
+
+//go:embed migrations/*.sql
+var migrationsFS embed.FS
+
+func main() {
+	connStr := os.Getenv("MODUS_DB")
+	if connStr == "" {
+		fmt.Fprintln(os.Stderr, "MODUS_DB environment variable is not set.")
+		os.Exit(1)
+	}
+
+	if err := migrateDB(connStr); errors.Is(err, migrate.ErrNoChange) {
+		fmt.Println("Database schema is already up to date.")
+	} else if err != nil {
+		fmt.Fprintf(os.Stderr, "Error running migrations: %v\n", err)
+		os.Exit(1)
+	} else {
+		fmt.Println("Migrations applied successfully")
+	}
+}
+
+func migrateDB(connStr string) error {
+	d, err := iofs.New(migrationsFS, "migrations")
+	if err != nil {
+		return fmt.Errorf("error creating iofs source: %w", err)
+	}
+
+	m, err := migrate.NewWithSourceInstance("iofs", d, connStr)
+	if err != nil {
+		return err
+	}
+	m.Log = &migrateLogger{}
+
+	return m.Up()
+}
+
+type migrateLogger struct {
+}
+
+func (l *migrateLogger) Printf(format string, v ...interface{}) {
+	fmt.Printf(format, v...)
+}
+
+func (l *migrateLogger) Verbose() bool {
+	return false
+}

tools/modus-dbmigrate/migrations/000001_init_inference_history.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS "inferences";

tools/modus-dbmigrate/migrations/000001_init_inference_history.up.sql

@@ -0,0 +1,10 @@
+CREATE TABLE IF NOT EXISTS "inferences" (
+    "id" UUID PRIMARY KEY,
+    "model_hash" TEXT NOT NULL,
+    "input" JSONB NOT NULL,
+    "output" JSONB NOT NULL,
+    "started_at" TIMESTAMP(3) WITH TIME ZONE NOT NULL,
+    "duration_ms" INTEGER NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS model_hash_started_at_idx ON inferences (model_hash, started_at);

tools/modus-dbmigrate/migrations/000002_create_collections.down.sql

@@ -0,0 +1,2 @@
+DROP TABLE IF EXISTS "collection_vectors";
+DROP TABLE IF EXISTS "collection_texts";

tools/modus-dbmigrate/migrations/000002_create_collections.up.sql

@@ -0,0 +1,19 @@
+CREATE TABLE IF NOT EXISTS "collection_texts" (
+    "id" BIGSERIAL PRIMARY KEY,
+    "collection" TEXT NOT NULL,
+    "key" TEXT NOT NULL,
+    "text" TEXT NOT NULL,
+    "created_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+    );
+
+CREATE INDEX IF NOT EXISTS collection_texts_collection_key_idx ON collection_texts (collection, key);
+
+CREATE TABLE IF NOT EXISTS "collection_vectors" (
+    "id" BIGSERIAL PRIMARY KEY,
+    "text_id" BIGINT REFERENCES collection_texts(id) ON DELETE CASCADE,
+    "search_method" TEXT NOT NULL,
+    "vector" REAL[] NOT NULL,
+    "created_at" TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT NOW()
+    );
+
+CREATE INDEX IF NOT EXISTS collection_vectors_search_method_text_id_idx ON collection_vectors (search_method, text_id);

tools/modus-dbmigrate/migrations/000003_update_inference_history.down.sql

@@ -0,0 +1,12 @@
+ALTER INDEX IF EXISTS inferences_model_hash_started_at_idx RENAME TO model_hash_started_at_idx;
+
+DROP INDEX IF EXISTS inferences_plugin_id_idx;
+DROP INDEX IF EXISTS inferences_plugin_id_function_idx;
+DROP INDEX IF EXISTS inferences_function_idx;
+
+ALTER TABLE IF EXISTS "inferences"
+DROP CONSTRAINT inferences_plugins_fkey,
+DROP COLUMN "plugin_id",
+DROP COLUMN "function";
+
+DROP TABLE IF EXISTS "plugins";

tools/modus-dbmigrate/migrations/000003_update_inference_history.up.sql

@@ -0,0 +1,25 @@
+CREATE TABLE IF NOT EXISTS "plugins" (
+    "id" UUID PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "version" TEXT,
+    "language" TEXT NOT NULL,
+    "sdk_version" TEXT NOT NULL,
+    "build_id" TEXT UNIQUE NOT NULL,
+    "build_time" TIMESTAMP(3) WITH TIME ZONE NOT NULL,
+    "git_repo" TEXT,
+    "git_commit" TEXT
+);
+
+CREATE INDEX IF NOT EXISTS plugins_build_id_idx ON plugins (build_id);
+CREATE INDEX IF NOT EXISTS plugins_build_time_idx ON plugins (build_time);
+
+ALTER TABLE IF EXISTS "inferences"
+ADD COLUMN "plugin_id" UUID,
+ADD COLUMN "function" TEXT,
+ADD CONSTRAINT inferences_plugins_fkey FOREIGN KEY (plugin_id) REFERENCES plugins (id);
+
+CREATE INDEX IF NOT EXISTS inferences_plugin_id_idx ON inferences (plugin_id);
+CREATE INDEX IF NOT EXISTS inferences_plugin_id_function_idx ON inferences (plugin_id, function);
+CREATE INDEX IF NOT EXISTS inferences_function_idx ON inferences (function);
+
+ALTER INDEX IF EXISTS model_hash_started_at_idx RENAME TO inferences_model_hash_started_at_idx;

tools/modus-dbmigrate/migrations/000004_add_collection_labels.down.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE collection_texts DROP COLUMN labels;
+
+COMMIT;

tools/modus-dbmigrate/migrations/000004_add_collection_labels.up.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE collection_texts ADD COLUMN labels TEXT[];
+
+COMMIT;

tools/modus-dbmigrate/migrations/000005_add_tenancy.down.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE collection_texts DROP COLUMN IF EXISTS namespace;
+
+COMMIT;

tools/modus-dbmigrate/migrations/000005_add_tenancy.up.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+ALTER TABLE collection_texts ADD COLUMN namespace TEXT DEFAULT '' NOT NULL;
+
+COMMIT;

tools/modus-dbmigrate/migrations/000006_add_started_at_index.down.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+DROP INDEX IF EXISTS idx_started_at_desc;
+
+COMMIT;

tools/modus-dbmigrate/migrations/000006_add_started_at_index.up.sql

@@ -0,0 +1,5 @@
+BEGIN;
+
+CREATE INDEX idx_started_at_desc ON inferences (started_at DESC);
+
+COMMIT;

tools/modus-dbmigrate/migrations/000007_add_agent_state.down.sql

@@ -0,0 +1 @@
+DROP TABLE IF EXISTS "agents";

tools/modus-dbmigrate/migrations/000007_add_agent_state.up.sql

@@ -0,0 +1,16 @@
+BEGIN;
+
+CREATE TABLE IF NOT EXISTS "agents" (
+    "id" TEXT NOT NULL PRIMARY KEY,
+    "name" TEXT NOT NULL,
+    "status" TEXT NOT NULL,
+    "data" TEXT,
+    "updated" TIMESTAMP(3) WITH TIME ZONE NOT NULL
+);
+
+CREATE INDEX IF NOT EXISTS agents_name_idx ON agents (name);
+CREATE INDEX IF NOT EXISTS agents_status_idx ON agents (status);
+CREATE INDEX IF NOT EXISTS agents_updated_idx ON agents (updated);
+CREATE INDEX IF NOT EXISTS agents_status_updated_idx ON agents (status, updated);
+
+COMMIT;