diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index f58af89..0ebce5b 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -40,7 +40,7 @@ crates/ ├── kd6-core/ # Domain types, OmsError, OmsProvider trait (pure -- no I/O deps) ├── kd6-sqlite/ # SQLite SPI implementation (sqlx), migrations in migrations/ ├── kd6-server/ # Axum HTTP server, route handlers, custom extractors -└── kd6-mcp/ # MCP server (rmcp), 9 tools, Streamable HTTP + stdio transports +└── kd6-mcp/ # MCP server (rmcp), 10 tools, Streamable HTTP + stdio transports ``` - **kd6-core** is the shared contract. All other crates depend on it. @@ -127,11 +127,11 @@ REST API rooted under `/v1/stores/{store_id}/`: ### MCP Server -The MCP server (kd6-mcp) exposes 9 tools via the Model Context Protocol using +The MCP server (kd6-mcp) exposes 10 tools via the Model Context Protocol using rmcp. It supports Streamable HTTP transport (default, port 8081) and stdio transport. Tools: `create_store`, `list_stores`, `create_memory`, `get_memory`, `search_memories`, `delete_memory`, `create_edge`, `traverse_graph`, -`gdpr_purge`. +`gdpr_purge`, `store_stats`. ### Environment Variables diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index dd7cd8e..4edfa19 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,14 +23,5 @@ jobs: - uses: Swatinem/rust-cache@v2 - - name: Format check - run: cargo fmt -- --check - - - name: Clippy - run: cargo clippy --all-targets -- -D warnings - - - name: Build - run: cargo build --all-targets - - - name: Test - run: cargo test + - name: CI + run: make ci diff --git a/.gitignore b/.gitignore index ad67955..1254c30 100644 --- a/.gitignore +++ b/.gitignore @@ -19,3 +19,4 @@ target # and can be added to the global gitignore or merged into this file. For a more nuclear # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +.fastembed_cache/ diff --git a/Cargo.lock b/Cargo.lock index bb19d69..174c7a9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2,6 +2,26 @@ # It is not intended for manual editing. version = 4 +[[package]] +name = "adler2" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" + +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "getrandom 0.3.4", + "once_cell", + "serde", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -11,6 +31,24 @@ dependencies = [ "memchr", ] +[[package]] +name = "aligned" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee4508988c62edf04abd8d92897fca0c2995d907ce1dfeaf369dac3716a40685" +dependencies = [ + "as-slice", +] + +[[package]] +name = "aligned-vec" +version = "0.6.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc890384c8602f339876ded803c97ad529f3842aba97f6392b3dba0dd171769b" +dependencies = [ + "equator", +] + [[package]] name = "allocator-api2" version = "0.2.21" @@ -32,6 +70,38 @@ version = "1.0.102" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f202df86484c868dbad7eaa557ef785d5c66295e41b460ef922eca0723b842c" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" + +[[package]] +name = "arg_enum_proc_macro" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ae92a5119aa49cdbcf6b9f893fe4e1d98b04ccbf82ee0584ad948a44a734dea" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "arrayvec" +version = "0.7.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" + +[[package]] +name = "as-slice" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "516b6b4f0e40d50dcda9365d53964ec74560ad4284da2e7fc97122cd83174516" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "assert-json-diff" version = "2.0.2" @@ -80,6 +150,49 @@ version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" +[[package]] +name = "av-scenechange" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f321d77c20e19b92c39e7471cf986812cbb46659d2af674adc4331ef3f18394" +dependencies = [ + "aligned", + "anyhow", + "arg_enum_proc_macro", + "arrayvec", + "log", + "num-rational", + "num-traits", + "pastey 0.1.1", + "rayon", + "thiserror", + "v_frame", + "y4m", +] + +[[package]] +name = "av1-grain" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8cfddb07216410377231960af4fcab838eaa12e013417781b78bd95ee22077f8" +dependencies = [ + "anyhow", + "arrayvec", + "log", + "nom 8.0.0", + "num-rational", + "v_frame", +] + +[[package]] +name = "avif-serialize" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7178fe5f7d460b13895ebb9dcb28a3a6216d2df2574a0806cb51b555d297f38" +dependencies = [ + "arrayvec", +] + [[package]] name = "axum" version = "0.8.9" @@ -162,6 +275,12 @@ dependencies = [ "url", ] +[[package]] +name = "base64" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e1b586273c5702936fe7b7d6896644d8be71e6314cfe09d3167c95f712589e8" + [[package]] name = "base64" version = "0.22.1" @@ -174,6 +293,12 @@ version = "1.8.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2af50177e190e07a26ab74f8b1efbfe2ef87da2116221318cb1c2e82baf7de06" +[[package]] +name = "bit_field" +version = "0.10.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e4b40c7323adcfc0a41c4b88143ed58346ff65a288fc144329c5c45e05d70c6" + [[package]] name = "bitflags" version = "2.11.1" @@ -183,6 +308,15 @@ dependencies = [ "serde_core", ] +[[package]] +name = "bitstream-io" +version = "4.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7eff00be299a18769011411c9def0d827e8f2d7bf0c3dbf53633147a8867fd1f" +dependencies = [ + "no_std_io2", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -192,18 +326,36 @@ dependencies = [ "generic-array", ] +[[package]] +name = "built" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c0e531d93d39c34eef561e929e8a7f86d77a5af08aac4f6d6e39976c51858e9" + [[package]] name = "bumpalo" version = "3.20.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "byteorder" version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -216,6 +368,15 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" +[[package]] +name = "castaway" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" +dependencies = [ + "rustversion", +] + [[package]] name = "cc" version = "1.2.63" @@ -223,6 +384,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "556e016178bb5662a08681bbe0f00f8e17631781a4dfc8c45e466e4b185ec27f" dependencies = [ "find-msvc-tools", + "jobserver", + "libc", "shlex", ] @@ -257,6 +420,27 @@ dependencies = [ "windows-link", ] +[[package]] +name = "color_quant" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d7b894f5411737b7867f4827955924d7c254fc9f4d91a6aad6b097804b1018b" + +[[package]] +name = "compact_str" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dfdd1c2274d9aa354115b09dc9a901d6c5576818cdf70d14cae2bdb47df00ab" +dependencies = [ + "castaway", + "cfg-if", + "itoa", + "rustversion", + "ryu", + "serde", + "static_assertions", +] + [[package]] name = "concurrent-queue" version = "2.5.0" @@ -266,6 +450,19 @@ dependencies = [ "crossbeam-utils", ] +[[package]] +name = "console" +version = "0.15.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "054ccb5b10f9f2cbf51eb355ca1d05c2d279ce1804688d0db74b4733a5aeafd8" +dependencies = [ + "encode_unicode", + "libc", + "once_cell", + "unicode-width", + "windows-sys 0.59.0", +] + [[package]] name = "const-oid" version = "0.9.6" @@ -282,6 +479,26 @@ dependencies = [ "version_check", ] +[[package]] +name = "core-foundation" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" +dependencies = [ + "core-foundation-sys", + "libc", +] + +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -321,6 +538,34 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "217698eaf96b4a3f0bc4f3662aaa55bdf913cd54d7204591faa790070c6d0853" +[[package]] +name = "crc32fast" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +dependencies = [ + "cfg-if", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-queue" version = "0.3.12" @@ -336,6 +581,12 @@ version = "0.8.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -346,14 +597,38 @@ dependencies = [ "typenum", ] +[[package]] +name = "darling" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" +dependencies = [ + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + [[package]] name = "darling" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25ae13da2f202d56bd7f91c25fba009e7717a1e4a1cc98a76d844b65ae912e9d" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.23.0", + "darling_macro 0.23.0", +] + +[[package]] +name = "darling_core" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d00b9596d185e565c2207a0b01f8bd1a135483d02d9b7b0a54b11da8d53412e" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn", ] [[package]] @@ -369,17 +644,55 @@ dependencies = [ "syn", ] +[[package]] +name = "darling_macro" +version = "0.20.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" +dependencies = [ + "darling_core 0.20.11", + "quote", + "syn", +] + [[package]] name = "darling_macro" version = "0.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3984ec7bd6cfa798e62b4a642426a5be0e68f9401cfc2a01e3fa9ea2fcdb8d" dependencies = [ - "darling_core", + "darling_core 0.23.0", "quote", "syn", ] +[[package]] +name = "dary_heap" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b1e3a325bc115f096c8b77bbf027a7c2592230e70be2d985be950d3d5e60ebe" +dependencies = [ + "serde", +] + +[[package]] +name = "deadpool" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0be2b1d1d6ec8d846f05e137292d0b89133caf95ef33695424c09568bdd39b1b" +dependencies = [ + "deadpool-runtime", + "lazy_static", + "num_cpus", + "tokio", +] + +[[package]] +name = "deadpool-runtime" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "092966b41edc516079bdf31ec78a2e0588d1d0c08f78b91d8307215928642b2b" + [[package]] name = "der" version = "0.7.10" @@ -400,6 +713,37 @@ dependencies = [ "powerfmt", ] +[[package]] +name = "derive_builder" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "507dfb09ea8b7fa618fcf76e953f4f5e192547945816d5358edffe39f6f94947" +dependencies = [ + "derive_builder_macro", +] + +[[package]] +name = "derive_builder_core" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" +dependencies = [ + "darling 0.20.11", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "derive_builder_macro" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" +dependencies = [ + "derive_builder_core", + "syn", +] + [[package]] name = "diff" version = "0.1.13" @@ -418,6 +762,27 @@ dependencies = [ "subtle", ] +[[package]] +name = "dirs" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3e8aa94d75141228480295a7d0e7feb620b1a5ad9f12bc40be62411e38cce4e" +dependencies = [ + "dirs-sys", +] + +[[package]] +name = "dirs-sys" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e01a3366d27ee9890022452ee61b2b63a67e6f13f58900b651ff5665f0bb1fab" +dependencies = [ + "libc", + "option-ext", + "redox_users", + "windows-sys 0.61.2", +] + [[package]] name = "displaydoc" version = "0.2.6" @@ -451,30 +816,71 @@ dependencies = [ ] [[package]] -name = "equivalent" -version = "1.0.2" +name = "encode_unicode" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0" [[package]] -name = "errno" -version = "0.3.14" +name = "encoding_rs" +version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "libc", - "windows-sys 0.61.2", + "cfg-if", ] [[package]] -name = "etcetera" -version = "0.8.0" +name = "equator" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +checksum = "4711b213838dfee0117e3be6ac926007d7f433d7bbe33595975d4190cb07e6fc" dependencies = [ - "cfg-if", - "home", - "windows-sys 0.48.0", + "equator-macro", +] + +[[package]] +name = "equator-macro" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "44f23cf4b44bfce11a86ace86f8a73ffdec849c9fd00a386a53d278bd9e81fb3" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "errno" +version = "0.3.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" +dependencies = [ + "libc", + "windows-sys 0.61.2", +] + +[[package]] +name = "esaxx-rs" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d817e038c30374a4bcb22f94d0a8a0e216958d4c3dcde369b1439fec4bdda6e6" + +[[package]] +name = "etcetera" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" +dependencies = [ + "cfg-if", + "home", + "windows-sys 0.48.0", ] [[package]] @@ -488,12 +894,85 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "exr" +version = "1.74.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4300e043a56aa2cb633c01af81ca8f699a321879a7854d3896a0ba89056363be" +dependencies = [ + "bit_field", + "half", + "lebe", + "miniz_oxide", + "rayon-core", + "smallvec", + "zune-inflate", +] + +[[package]] +name = "fastembed" +version = "4.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04c269a76bfc6cea69553b7d040acb16c793119cebd97c756d21e08d0f075ff8" +dependencies = [ + "anyhow", + "hf-hub", + "image", + "ndarray", + "ort", + "ort-sys", + "rayon", + "serde_json", + "tokenizers", +] + +[[package]] +name = "fastrand" +version = "2.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9f1f227452a390804cdb637b74a86990f2a7d7ba4b7d5693aac9b4dd6defd8d6" + +[[package]] +name = "fax" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf1079563223d5d59d83c85886a56e586cfd5c1a26292e971a0fa266531ac5a" + +[[package]] +name = "fdeflate" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e6853b52649d4ac5c0bd02320cddc5ba956bdb407c4b75a2c6b75bf51500f8c" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "filetime" +version = "0.2.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c287a33c7f0a620c38e641e7f60827713987b3c0f26e8ddc9462cc69cf75759" +dependencies = [ + "cfg-if", + "libc", +] + [[package]] name = "find-msvc-tools" version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flate2" +version = "1.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "843fba2746e448b37e26a819579957415c8cef339bf08564fe8b7ddbd959573c" +dependencies = [ + "crc32fast", + "miniz_oxide", +] + [[package]] name = "flume" version = "0.11.1" @@ -505,12 +984,33 @@ dependencies = [ "spin", ] +[[package]] +name = "fnv" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" + [[package]] name = "foldhash" version = "0.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +[[package]] +name = "foreign-types" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f6f339eb8adc052cd2ca78910fda869aefa38d22d5cb648e6485e4d3fc06f3b1" +dependencies = [ + "foreign-types-shared", +] + +[[package]] +name = "foreign-types-shared" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" + [[package]] name = "form_urlencoded" version = "1.2.2" @@ -668,6 +1168,46 @@ dependencies = [ "wasip3", ] +[[package]] +name = "gif" +version = "0.14.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee8cfcc411d9adbbaba82fb72661cc1bcca13e8bba98b364e62b2dba8f960159" +dependencies = [ + "color_quant", + "weezl", +] + +[[package]] +name = "h2" +version = "0.4.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "171fefbc92fe4a4de27e0698d6a5b392d6a0e333506bc49133760b3bcf948733" +dependencies = [ + "atomic-waker", + "bytes", + "fnv", + "futures-core", + "futures-sink", + "http", + "indexmap", + "slab", + "tokio", + "tokio-util", + "tracing", +] + +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -700,12 +1240,39 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" + [[package]] name = "hex" version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" +[[package]] +name = "hf-hub" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629d8f3bbeda9d148036d6b0de0a3ab947abd08ce90626327fc3547a49d59d97" +dependencies = [ + "dirs", + "http", + "indicatif", + "libc", + "log", + "native-tls", + "rand 0.9.4", + "reqwest", + "serde", + "serde_json", + "thiserror", + "ureq", + "windows-sys 0.60.2", +] + [[package]] name = "hkdf" version = "0.12.4" @@ -788,6 +1355,7 @@ dependencies = [ "bytes", "futures-channel", "futures-core", + "h2", "http", "http-body", "httparse", @@ -799,24 +1367,60 @@ dependencies = [ "want", ] +[[package]] +name = "hyper-rustls" +version = "0.27.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" +dependencies = [ + "http", + "hyper", + "hyper-util", + "rustls", + "tokio", + "tokio-rustls", + "tower-service", +] + +[[package]] +name = "hyper-tls" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "70206fc6890eaca9fde8a0bf71caa2ddfc9fe045ac9e5c70df101a7dbde866e0" +dependencies = [ + "bytes", + "http-body-util", + "hyper", + "hyper-util", + "native-tls", + "tokio", + "tokio-native-tls", + "tower-service", +] + [[package]] name = "hyper-util" version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "96547c2556ec9d12fb1578c4eaf448b04993e7fb79cbaad930a656880a6bdfa0" dependencies = [ + "base64 0.22.1", "bytes", "futures-channel", "futures-util", "http", "http-body", "hyper", + "ipnet", "libc", + "percent-encoding", "pin-project-lite", "socket2", + "system-configuration", "tokio", "tower-service", "tracing", + "windows-registry", ] [[package]] @@ -958,6 +1562,46 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck", + "byteorder-lite", + "color_quant", + "exr", + "gif", + "image-webp", + "moxcms", + "num-traits", + "png", + "qoi", + "ravif", + "rayon", + "rgb", + "tiff", + "zune-core", + "zune-jpeg", +] + +[[package]] +name = "image-webp" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "525e9ff3e1a4be2fbea1fdf0e98686a6d98b4d8f937e1bf7402245af1909e8c3" +dependencies = [ + "byteorder-lite", + "quick-error", +] + +[[package]] +name = "imgref" +version = "1.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40fac9d56ed6437b198fddba683305e8e2d651aa42647f00f5ae542e7f5c94a2" + [[package]] name = "indexmap" version = "2.14.0" @@ -970,12 +1614,61 @@ dependencies = [ "serde_core", ] +[[package]] +name = "indicatif" +version = "0.17.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "183b3088984b400f4cfac3620d5e076c84da5364016b4f49473de574b2586235" +dependencies = [ + "console", + "number_prefix", + "portable-atomic", + "unicode-width", + "web-time", +] + +[[package]] +name = "interpolate_name" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c34819042dc3d3971c46c2190835914dfbe0c3c13f61449b2997f4e9722dfa60" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "ipnet" +version = "2.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d98f6fed1fde3f8c21bc40a1abb88dd75e67924f9cffc3ef95607bad8017f8e2" + +[[package]] +name = "itertools" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b192c782037fadd9cfa75548310488aabdbf3d2da73885b31bd0abd03351285" +dependencies = [ + "either", +] + [[package]] name = "itoa" version = "1.0.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", +] + [[package]] name = "js-sys" version = "0.3.99" @@ -994,7 +1687,7 @@ version = "9.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a87cc7a48537badeae96744432de36f4be2b4a34a05a5ef32e9dd8a1c169dde" dependencies = [ - "base64", + "base64 0.22.1", "js-sys", "pem", "ring", @@ -1012,9 +1705,25 @@ dependencies = [ "serde", "serde_json", "thiserror", + "tokio", "uuid", ] +[[package]] +name = "kd6-embed" +version = "0.1.0" +dependencies = [ + "async-trait", + "fastembed", + "kd6-core", + "reqwest", + "serde", + "serde_json", + "tokio", + "tracing", + "wiremock", +] + [[package]] name = "kd6-mcp" version = "0.1.0" @@ -1024,6 +1733,7 @@ dependencies = [ "axum", "chrono", "kd6-core", + "kd6-embed", "kd6-sqlite", "rmcp", "schemars", @@ -1040,11 +1750,13 @@ name = "kd6-server" version = "0.1.0" dependencies = [ "anyhow", + "async-trait", "axum", "axum-test", "chrono", "jsonwebtoken", "kd6-core", + "kd6-embed", "kd6-sqlite", "serde", "serde_json", @@ -1089,12 +1801,28 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lebe" +version = "0.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a79a3332a6609480d7d0c9eab957bca6b455b91bb84e66d19f5ff66294b85b8" + [[package]] name = "libc" version = "0.2.186" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "68ab91017fe16c622486840e4c83c9a37afeff978bd239b5293d61ece587de66" +[[package]] +name = "libfuzzer-sys" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f12a681b7dd8ce12bff52488013ba614b869148d54dd79836ab85aafdd53f08d" +dependencies = [ + "arbitrary", + "cc", +] + [[package]] name = "libm" version = "0.2.16" @@ -1124,6 +1852,12 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "linux-raw-sys" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" + [[package]] name = "litemap" version = "0.8.2" @@ -1145,6 +1879,31 @@ version = "0.4.30" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "616ec5685824bcc94416c6d4a7a446eea774a31efd7062c8480ba6fd06d7a6e5" +[[package]] +name = "loop9" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fae87c125b03c1d2c0150c90365d7d6bcc53fb73a9acaef207d2d065860f062" +dependencies = [ + "imgref", +] + +[[package]] +name = "macro_rules_attribute" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65049d7923698040cd0b1ddcced9b0eb14dd22c5f86ae59c3740eab64a676520" +dependencies = [ + "macro_rules_attribute-proc_macro", + "paste", +] + +[[package]] +name = "macro_rules_attribute-proc_macro" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "670fdfda89751bc4a84ac13eaa63e205cf0fd22b4c9a5fbfa085b63c1f1d3a30" + [[package]] name = "matchers" version = "0.2.0" @@ -1160,6 +1919,26 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47e1ffaa40ddd1f3ed91f717a33c8c0ee23fff369e3aa8772b9605cc1d22f4c3" +[[package]] +name = "matrixmultiply" +version = "0.3.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a06de3016e9fae57a36fd14dba131fccf49f74b40b7fbdb472f96e361ec71a08" +dependencies = [ + "autocfg", + "rawpointer", +] + +[[package]] +name = "maybe-rayon" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ea1f30cedd69f0a2954655f7188c6a834246d2bcf1e315e2ac40c4b24dc9519" +dependencies = [ + "cfg-if", + "rayon", +] + [[package]] name = "md-5" version = "0.10.6" @@ -1182,6 +1961,22 @@ version = "0.3.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6877bb514081ee2a7ff5ef9de3281f14a4dd4bceac4c09388074a6b5df8a139a" +[[package]] +name = "minimal-lexical" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" + +[[package]] +name = "miniz_oxide" +version = "0.8.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" +dependencies = [ + "adler2", + "simd-adler32", +] + [[package]] name = "mio" version = "1.2.1" @@ -1193,6 +1988,110 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "monostate" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3341a273f6c9d5bef1908f17b7267bbab0e95c9bf69a0d4dcf8e9e1b2c76ef67" +dependencies = [ + "monostate-impl", + "serde", + "serde_core", +] + +[[package]] +name = "monostate-impl" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4db6d5580af57bf992f59068d4ea26fd518574ff48d7639b255a36f9de6e7e9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "native-tls" +version = "0.2.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "465500e14ea162429d264d44189adc38b199b62b1c21eea9f69e4b73cb03bbf2" +dependencies = [ + "libc", + "log", + "openssl", + "openssl-probe", + "openssl-sys", + "schannel", + "security-framework", + "security-framework-sys", + "tempfile", +] + +[[package]] +name = "ndarray" +version = "0.16.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "882ed72dce9365842bf196bdeedf5055305f11fc8c03dee7bb0194a6cad34841" +dependencies = [ + "matrixmultiply", + "num-complex", + "num-integer", + "num-traits", + "portable-atomic", + "portable-atomic-util", + "rawpointer", +] + +[[package]] +name = "new_debug_unreachable" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" + +[[package]] +name = "no_std_io2" +version = "0.9.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "418abd1b6d34fbf6cae440dc874771b0525a604428704c76e48b29a5e67b8003" +dependencies = [ + "memchr", +] + +[[package]] +name = "nom" +version = "7.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d273983c5a657a70a3e8f2a01329822f3b8c8172b73826411a55751e404a0a4a" +dependencies = [ + "memchr", + "minimal-lexical", +] + +[[package]] +name = "nom" +version = "8.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df9761775871bdef83bee530e60050f7e54b1105350d6884eb0fb4f46c2f9405" +dependencies = [ + "memchr", +] + +[[package]] +name = "noop_proc_macro" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0676bb32a98c1a483ce53e500a81ad9c3d5b3f7c920c28c24e9cb0980d0b5bc8" + [[package]] name = "nu-ansi-term" version = "0.50.3" @@ -1228,12 +2127,32 @@ dependencies = [ "zeroize", ] +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + [[package]] name = "num-conv" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521739c6d2bac4aa25192232afe6841231376b2b26d4d9fae5ecf8ca5772e441" +[[package]] +name = "num-derive" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed3955f1a9c7c0c15e092f9c887db08b1fc683305fdf6eb6684f22555355e202" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "num-integer" version = "0.1.46" @@ -1254,6 +2173,17 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + [[package]] name = "num-traits" version = "0.2.19" @@ -1264,12 +2194,123 @@ dependencies = [ "libm", ] +[[package]] +name = "num_cpus" +version = "1.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" +dependencies = [ + "hermit-abi", + "libc", +] + +[[package]] +name = "number_prefix" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830b246a0e5f20af87141b25c173cd1b609bd7779a4617d6ec582abaf90870f3" + [[package]] name = "once_cell" version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" +[[package]] +name = "onig" +version = "6.5.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0cc3cbf698f9438986c11a880c90a6d04b9de27575afd28bbf45b154b6c709e2" +dependencies = [ + "bitflags", + "libc", + "once_cell", + "onig_sys", +] + +[[package]] +name = "onig_sys" +version = "69.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e68317604e77e53b85896388e1a803c1d21b74c899ec9e5e1112db90735edd7" +dependencies = [ + "cc", + "pkg-config", +] + +[[package]] +name = "openssl" +version = "0.10.80" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a45fa2aa886c42762255da344f0a0d313e254066c46aad76f300c3d3da62d967" +dependencies = [ + "bitflags", + "cfg-if", + "foreign-types", + "libc", + "openssl-macros", + "openssl-sys", +] + +[[package]] +name = "openssl-macros" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "openssl-probe" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" + +[[package]] +name = "openssl-sys" +version = "0.9.116" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" +dependencies = [ + "cc", + "libc", + "pkg-config", + "vcpkg", +] + +[[package]] +name = "option-ext" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" + +[[package]] +name = "ort" +version = "2.0.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52afb44b6b0cffa9bf45e4d37e5a4935b0334a51570658e279e9e3e6cf324aa5" +dependencies = [ + "ndarray", + "ort-sys", + "tracing", +] + +[[package]] +name = "ort-sys" +version = "2.0.0-rc.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c41d7757331aef2d04b9cb09b45583a59217628beaf91895b7e76187b6e8c088" +dependencies = [ + "flate2", + "pkg-config", + "sha2", + "tar", + "ureq", +] + [[package]] name = "parking" version = "2.2.1" @@ -1299,6 +2340,18 @@ dependencies = [ "windows-link", ] +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + [[package]] name = "pastey" version = "0.2.3" @@ -1311,7 +2364,7 @@ version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d30c53c26bc5b31a98cd02d20f25a7c8567146caf63ed593a9d87b2775291be" dependencies = [ - "base64", + "base64 0.22.1", "serde_core", ] @@ -1369,6 +2422,34 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6" +[[package]] +name = "png" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60769b8b31b2a9f263dae2776c37b1b28ae246943cf719eb6946a1db05128a61" +dependencies = [ + "bitflags", + "crc32fast", + "fdeflate", + "flate2", + "miniz_oxide", +] + +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.5" @@ -1422,6 +2503,46 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "profiling" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d595e54a326bc53c1c197b32d295e14b169e3cfeaa8dc82b529f947fba6bcf5" +dependencies = [ + "profiling-procmacros", +] + +[[package]] +name = "profiling-procmacros" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4488a4a36b9a4ba6b9334a32a39971f77c1436ec82c38707bce707699cc3bbcb" +dependencies = [ + "quote", + "syn", +] + +[[package]] +name = "pxfm" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e0c5ccf5294c6ccd63a74f1565028353830a9c2f5eb0c682c355c471726a6e3f" + +[[package]] +name = "qoi" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f6d64c71eb498fe9eae14ce4ec935c555749aef511cca85b5568910d6e48001" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "quick-error" +version = "2.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a993555f31e5a609f617c12db6250dedcac1b0a85076912c436e6fc9b2c8e6a3" + [[package]] name = "quote" version = "1.0.45" @@ -1519,6 +2640,93 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" +[[package]] +name = "rav1e" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43b6dd56e85d9483277cde964fd1bdb0428de4fec5ebba7540995639a21cb32b" +dependencies = [ + "aligned-vec", + "arbitrary", + "arg_enum_proc_macro", + "arrayvec", + "av-scenechange", + "av1-grain", + "bitstream-io", + "built", + "cfg-if", + "interpolate_name", + "itertools", + "libc", + "libfuzzer-sys", + "log", + "maybe-rayon", + "new_debug_unreachable", + "noop_proc_macro", + "num-derive", + "num-traits", + "paste", + "profiling", + "rand 0.9.4", + "rand_chacha 0.9.0", + "simd_helpers", + "thiserror", + "v_frame", + "wasm-bindgen", +] + +[[package]] +name = "ravif" +version = "0.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e52310197d971b0f5be7fe6b57530dcd27beb35c1b013f29d66c1ad73fbbcc45" +dependencies = [ + "avif-serialize", + "imgref", + "loop9", + "quick-error", + "rav1e", + "rayon", + "rgb", +] + +[[package]] +name = "rawpointer" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "60a357793950651c4ed0f3f52338f53b2f809f32d83a07f72909fa13e4c6c1e3" + +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-cond" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2964d0cf57a3e7a06e8183d14a8b527195c706b7983549cd5462d5aa3747438f" +dependencies = [ + "either", + "itertools", + "rayon", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -1537,6 +2745,17 @@ dependencies = [ "bitflags", ] +[[package]] +name = "redox_users" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4e608c6638b9c18977b00b475ac1f28d14e84b27d8d42f70e0bf1e3dec127ac" +dependencies = [ + "getrandom 0.2.17", + "libredox", + "thiserror", +] + [[package]] name = "ref-cast" version = "1.0.25" @@ -1557,6 +2776,18 @@ dependencies = [ "syn", ] +[[package]] +name = "regex" +version = "1.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e10754a14b9137dd7b1e3e5b0493cc9171fdd105e0ab477f51b72e7f3ac0e276" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + [[package]] name = "regex-automata" version = "0.4.14" @@ -1574,6 +2805,49 @@ version = "0.8.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dc897dd8d9e8bd1ed8cdad82b5966c3e0ecae09fb1907d58efaa013543185d0a" +[[package]] +name = "reqwest" +version = "0.12.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" +dependencies = [ + "base64 0.22.1", + "bytes", + "encoding_rs", + "futures-core", + "futures-util", + "h2", + "http", + "http-body", + "http-body-util", + "hyper", + "hyper-rustls", + "hyper-tls", + "hyper-util", + "js-sys", + "log", + "mime", + "native-tls", + "percent-encoding", + "pin-project-lite", + "rustls-pki-types", + "serde", + "serde_json", + "serde_urlencoded", + "sync_wrapper", + "tokio", + "tokio-native-tls", + "tokio-util", + "tower", + "tower-http", + "tower-service", + "url", + "wasm-bindgen", + "wasm-bindgen-futures", + "wasm-streams", + "web-sys", +] + [[package]] name = "reserve-port" version = "2.4.0" @@ -1583,6 +2857,12 @@ dependencies = [ "thiserror", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" + [[package]] name = "ring" version = "0.17.14" @@ -1604,14 +2884,14 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0810a9f717d9828f475fe1f629f4c305c8464b7f496c3a854b58d29e65f4058e" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "bytes", "chrono", "futures", "http", "http-body", "http-body-util", - "pastey", + "pastey 0.2.3", "pin-project-lite", "rand 0.10.1", "rmcp-macros", @@ -1634,7 +2914,7 @@ version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6aefac48c364756e97f04c0401ba3231e8607882c7c1d92da0437dc16307904d" dependencies = [ - "darling", + "darling 0.23.0", "proc-macro2", "quote", "serde_json", @@ -1676,6 +2956,54 @@ dependencies = [ "thiserror", ] +[[package]] +name = "rustix" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +dependencies = [ + "bitflags", + "errno", + "libc", + "linux-raw-sys", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef86cd5876211988985292b91c96a8f2d298df24e75989a43a3c73f2d4d8168b" +dependencies = [ + "log", + "once_cell", + "ring", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-pki-types" +version = "1.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30a7197ae7eb376e574fe940d068c30fe0462554a3ddbe4eca7838e049c937a9" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-webpki" +version = "0.103.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61c429a8649f110dddef65e2a5ad240f747e85f7758a6bccc7e5777bd33f756e" +dependencies = [ + "ring", + "rustls-pki-types", + "untrusted", +] + [[package]] name = "rustversion" version = "1.0.22" @@ -1688,6 +3016,15 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "schannel" +version = "0.1.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91c1b7e4904c873ef0710c1f407dde2e6287de2bebc1bbbf7d430bb7cbffd939" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "schemars" version = "1.2.1" @@ -1720,6 +3057,29 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "security-framework" +version = "3.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" +dependencies = [ + "bitflags", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ce2691df843ecc5d231c0b14ece2acc3efb62c0a398c7e1d875f3983ce020e3" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "semver" version = "1.0.28" @@ -1860,6 +3220,21 @@ dependencies = [ "rand_core 0.6.4", ] +[[package]] +name = "simd-adler32" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "703d5c7ef118737c72f1af64ad2f6f8c5e1921f818cdcb97b8fe6fc69bf66214" + +[[package]] +name = "simd_helpers" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95890f873bec569a0362c235787f3aca6e1e887302ba4840839bcc6459c42da6" +dependencies = [ + "quote", +] + [[package]] name = "simple_asn1" version = "0.6.4" @@ -1897,6 +3272,17 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "socks" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0c3dbbd9ae980613c6dd8e28a9407b50509d3803b57624d5dfe8315218cd58b" +dependencies = [ + "byteorder", + "libc", + "winapi", +] + [[package]] name = "spin" version = "0.9.8" @@ -1916,6 +3302,18 @@ dependencies = [ "der", ] +[[package]] +name = "spm_precompiled" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5851699c4033c63636f7ea4cf7b7c1f1bf06d0cc03cfb42e711de5a5c46cf326" +dependencies = [ + "base64 0.13.1", + "nom 7.1.3", + "serde", + "unicode-segmentation", +] + [[package]] name = "sqlx" version = "0.8.6" @@ -1935,7 +3333,7 @@ version = "0.8.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee6798b1838b6a0f69c007c133b8df5866302197e404e8b6ee8ed3e3a5e68dc6" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "crc", @@ -2010,7 +3408,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa003f0038df784eb8fecbbac13affe3da23b45194bd57dba231c8f48199c526" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "bytes", @@ -2054,7 +3452,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "db58fcd5a53cf07c184b154801ff91347e4c30d17a3562a635ff028ad5deda46" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags", "byteorder", "chrono", @@ -2131,6 +3529,12 @@ version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" +[[package]] +name = "static_assertions" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" + [[package]] name = "stringprep" version = "0.1.5" @@ -2152,34 +3556,82 @@ checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" name = "subtle" version = "2.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" +checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" + +[[package]] +name = "syn" +version = "2.0.117" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "sync_wrapper" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +dependencies = [ + "futures-core", +] + +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "system-configuration" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" +dependencies = [ + "bitflags", + "core-foundation 0.9.4", + "system-configuration-sys", +] [[package]] -name = "syn" -version = "2.0.117" +name = "system-configuration-sys" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e665b8803e7b1d2a727f4023456bbbbe74da67099c585258af0ad9c5013b9b99" +checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ - "proc-macro2", - "quote", - "unicode-ident", + "core-foundation-sys", + "libc", ] [[package]] -name = "sync_wrapper" -version = "1.0.2" +name = "tar" +version = "0.4.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0bf256ce5efdfa370213c1dabab5935a12e49f2c58d15e9eac2870d3b4f27263" +checksum = "3f6221d9a6003c78398e3b239969f352578258df48c8eb051caadae0015bc840" +dependencies = [ + "filetime", + "libc", + "xattr", +] [[package]] -name = "synstructure" -version = "0.13.2" +name = "tempfile" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ - "proc-macro2", - "quote", - "syn", + "fastrand", + "getrandom 0.4.2", + "once_cell", + "rustix", + "windows-sys 0.61.2", ] [[package]] @@ -2211,6 +3663,20 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiff" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b63feaf3343d35b6ca4d50483f94843803b0f51634937cc2ec519fc32232bc52" +dependencies = [ + "fax", + "flate2", + "half", + "quick-error", + "weezl", + "zune-jpeg", +] + [[package]] name = "time" version = "0.3.47" @@ -2267,6 +3733,39 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" +[[package]] +name = "tokenizers" +version = "0.21.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a620b996116a59e184c2fa2dfd8251ea34a36d0a514758c6f966386bd2e03476" +dependencies = [ + "ahash", + "aho-corasick", + "compact_str", + "dary_heap", + "derive_builder", + "esaxx-rs", + "getrandom 0.3.4", + "itertools", + "log", + "macro_rules_attribute", + "monostate", + "onig", + "paste", + "rand 0.9.4", + "rayon", + "rayon-cond", + "regex", + "regex-syntax", + "serde", + "serde_json", + "spm_precompiled", + "thiserror", + "unicode-normalization-alignments", + "unicode-segmentation", + "unicode_categories", +] + [[package]] name = "tokio" version = "1.52.3" @@ -2295,6 +3794,26 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-native-tls" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbae76ab933c85776efabc971569dd6119c580d8f5d448769dec1764bf796ef2" +dependencies = [ + "native-tls", + "tokio", +] + +[[package]] +name = "tokio-rustls" +version = "0.26.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" +dependencies = [ + "rustls", + "tokio", +] + [[package]] name = "tokio-stream" version = "0.1.18" @@ -2343,12 +3862,15 @@ checksum = "4cfcf7e2740e6fc6d4d688b4ef00650406bb94adf4731e43c096c3a19fe40840" dependencies = [ "bitflags", "bytes", + "futures-util", "http", "http-body", "pin-project-lite", + "tower", "tower-layer", "tower-service", "tracing", + "url", ] [[package]] @@ -2458,24 +3980,71 @@ dependencies = [ "tinyvec", ] +[[package]] +name = "unicode-normalization-alignments" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "43f613e4fa046e69818dd287fdc4bc78175ff20331479dab6e1b0f98d57062de" +dependencies = [ + "smallvec", +] + [[package]] name = "unicode-properties" version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7df058c713841ad818f1dc5d3fd88063241cc61f49f5fbea4b951e8cf5a8d71d" +[[package]] +name = "unicode-segmentation" +version = "1.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c6f5d3c3b1bf09027a88a6bc961fc00497d651009560b5463668dc81b0fa87a8" + +[[package]] +name = "unicode-width" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" + [[package]] name = "unicode-xid" version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" +[[package]] +name = "unicode_categories" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39ec24b3121d976906ece63c9daad25b85969647682eee313cb5779fdd69e14e" + [[package]] name = "untrusted" version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" +[[package]] +name = "ureq" +version = "2.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02d1a66277ed75f640d608235660df48c8e3c19f3b4edb6a263315626cc3c01d" +dependencies = [ + "base64 0.22.1", + "flate2", + "log", + "native-tls", + "once_cell", + "rustls", + "rustls-pki-types", + "serde", + "serde_json", + "socks", + "url", + "webpki-roots 0.26.11", +] + [[package]] name = "url" version = "2.5.8" @@ -2506,6 +4075,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v_frame" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "666b7727c8875d6ab5db9533418d7c764233ac9c0cff1d469aec8fa127597be2" +dependencies = [ + "aligned-vec", + "num-traits", + "wasm-bindgen", +] + [[package]] name = "valuable" version = "0.1.1" @@ -2576,6 +4156,16 @@ dependencies = [ "wasm-bindgen-shared", ] +[[package]] +name = "wasm-bindgen-futures" +version = "0.4.72" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9473dbd2991ae90b6291c3c32c30c6187ac49aa32f9905d1cce280ec1e110b0f" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "wasm-bindgen-macro" version = "0.2.122" @@ -2630,6 +4220,19 @@ dependencies = [ "wasmparser", ] +[[package]] +name = "wasm-streams" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65" +dependencies = [ + "futures-util", + "js-sys", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", +] + [[package]] name = "wasmparser" version = "0.244.0" @@ -2642,6 +4245,50 @@ dependencies = [ "semver", ] +[[package]] +name = "web-sys" +version = "0.3.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d621441cfc37b84979402712047321980c178f299193a3589d05b99e8763436" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "web-time" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a6580f308b1fad9207618087a65c04e7a10bc77e02c8e84e9b00dd4b12fa0bb" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + +[[package]] +name = "webpki-roots" +version = "0.26.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" +dependencies = [ + "webpki-roots 1.0.7", +] + +[[package]] +name = "webpki-roots" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "weezl" +version = "0.1.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a28ac98ddc8b9274cb41bb4d9d4d5c425b6020c50c46f25559911905610b4a88" + [[package]] name = "whoami" version = "1.6.1" @@ -2652,6 +4299,28 @@ dependencies = [ "wasite", ] +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows-core" version = "0.62.2" @@ -2693,6 +4362,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" +[[package]] +name = "windows-registry" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "02752bf7fbdcce7f2a27a742f798510f3e5ad88dbe84871e5168e2120c3d5720" +dependencies = [ + "windows-link", + "windows-result", + "windows-strings", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -2729,6 +4409,24 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.59.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +dependencies = [ + "windows-targets 0.52.6", +] + +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -2762,13 +4460,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.48.5" @@ -2781,6 +4496,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.48.5" @@ -2793,6 +4514,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.48.5" @@ -2805,12 +4532,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.48.5" @@ -2823,6 +4562,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.48.5" @@ -2835,6 +4580,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.48.5" @@ -2847,6 +4598,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.48.5" @@ -2859,6 +4616,35 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + +[[package]] +name = "wiremock" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08db1edfb05d9b3c1542e521aea074442088292f00b5f28e435c714a98f85031" +dependencies = [ + "assert-json-diff", + "base64 0.22.1", + "deadpool", + "futures", + "http", + "http-body-util", + "hyper", + "hyper-util", + "log", + "once_cell", + "regex", + "serde", + "serde_json", + "tokio", + "url", +] + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -2959,6 +4745,22 @@ version = "0.6.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ffae5123b2d3fc086436f8834ae3ab053a283cfac8fe0a0b8eaae044768a4c4" +[[package]] +name = "xattr" +version = "1.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32e45ad4206f6d2479085147f02bc2ef834ac85886624a23575ae137c8aa8156" +dependencies = [ + "libc", + "rustix", +] + +[[package]] +name = "y4m" +version = "0.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a5a4b21e1a62b67a2970e6831bc091d7b87e119e7f9791aef9702e3bef04448" + [[package]] name = "yansi" version = "1.0.1" @@ -3073,3 +4875,27 @@ name = "zmij" version = "1.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8848ee67ecc8aedbaf3e4122217aff892639231befc6a1b58d29fff4c2cabaa" + +[[package]] +name = "zune-core" +version = "0.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cb8a0807f7c01457d0379ba880ba6322660448ddebc890ce29bb64da71fb40f9" + +[[package]] +name = "zune-inflate" +version = "0.2.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73ab332fe2f6680068f3582b16a24f90ad7096d5d39b974d1c0aff0125116f02" +dependencies = [ + "simd-adler32", +] + +[[package]] +name = "zune-jpeg" +version = "0.5.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27bc9d5b815bc103f142aa054f561d9187d191692ec7c2d1e2b4737f8dbd7296" +dependencies = [ + "zune-core", +] diff --git a/Cargo.toml b/Cargo.toml index 1d8f087..fa4d4b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -2,6 +2,7 @@ resolver = "2" members = [ "crates/kd6-core", + "crates/kd6-embed", "crates/kd6-sqlite", "crates/kd6-server", "crates/kd6-mcp", @@ -34,6 +35,8 @@ tower-http = { version = "0.6", features = ["cors", "trace"] } jsonwebtoken = "9" tracing-subscriber = { version = "0.3", features = ["env-filter"] } +reqwest = { version = "0.12", features = ["json"], default-features = false } + # MCP server rmcp = { version = "1", features = ["server", "transport-io", "transport-streamable-http-server", "macros", "schemars"] } schemars = "1" diff --git a/Makefile b/Makefile new file mode 100644 index 0000000..08dda98 --- /dev/null +++ b/Makefile @@ -0,0 +1,39 @@ +.PHONY: fmt lint build test ci clean run-server run-mcp + +# Format code +fmt: + cargo fmt + +# Check formatting (same as CI) +fmt-check: + cargo fmt -- --check + +# Lint with clippy (same as CI: warnings are errors) +lint: + cargo clippy --all-targets -- -D warnings + +# Build all targets +build: + cargo build --all-targets + +# Run tests +test: + cargo test + +# Run the full CI pipeline locally — use before every push +ci: fmt-check lint build test + +# Auto-format then run CI checks +fix: fmt lint build test + +# Remove build artifacts +clean: + cargo clean + +# Run the HTTP API server (port 8080) +run-server: + KD6_DATABASE_URL="sqlite:kd6.db?mode=rwc" cargo run -p kd6-server + +# Run the MCP server (Streamable HTTP, port 8081) +run-mcp: + KD6_DATABASE_URL="sqlite:kd6.db?mode=rwc" cargo run -p kd6-mcp diff --git a/crates/kd6-core/Cargo.toml b/crates/kd6-core/Cargo.toml index 64b8f1a..97a6e7d 100644 --- a/crates/kd6-core/Cargo.toml +++ b/crates/kd6-core/Cargo.toml @@ -11,3 +11,6 @@ uuid = { workspace = true } chrono = { workspace = true } thiserror = { workspace = true } async-trait = { workspace = true } + +[dev-dependencies] +tokio = { workspace = true } diff --git a/crates/kd6-core/src/embedding.rs b/crates/kd6-core/src/embedding.rs new file mode 100644 index 0000000..4c975d7 --- /dev/null +++ b/crates/kd6-core/src/embedding.rs @@ -0,0 +1,414 @@ +use async_trait::async_trait; + +use crate::error::OmsError; + +/// Computes vector embeddings from text content. +/// +/// Implementations may use a local model (e.g., ONNX via fastembed), +/// a remote API (e.g., OpenAI, Azure OpenAI, Ollama), or any +/// OpenAI-compatible embedding endpoint. +#[async_trait] +pub trait EmbeddingProvider: Send + Sync { + /// Compute embeddings for one or more text strings. + /// + /// Returns a vector of embedding vectors, one per input text. + /// All vectors MUST have the same dimensionality. + async fn embed_texts(&self, texts: &[String]) -> Result>, OmsError>; + + /// Compute a single embedding for a search query. + /// + /// Some models use different prefixes for queries vs. documents + /// (e.g., "query: " vs. "passage: " in E5/nomic models). + async fn embed_query(&self, query: &str) -> Result, OmsError>; + + /// Return the dimensionality of embeddings produced by this provider. + fn dimensions(&self) -> usize; + + /// Return a stable identifier for the embedding model. + fn model_id(&self) -> &str; +} + +/// A no-op embedding provider that never computes embeddings. +/// +/// Used when no embedding provider is configured (pass-through mode). +/// Callers must supply their own embeddings for vector search; +/// keyword search remains available. +pub struct NoopEmbedder; + +#[async_trait] +impl EmbeddingProvider for NoopEmbedder { + async fn embed_texts(&self, _texts: &[String]) -> Result>, OmsError> { + Err(OmsError::InvalidInput( + "no embedding provider configured; supply embeddings in the request or \ + configure KD6_EMBEDDING_PROVIDER" + .into(), + )) + } + + async fn embed_query(&self, _query: &str) -> Result, OmsError> { + Err(OmsError::InvalidInput( + "no embedding provider configured; supply embedding in the search request or \ + configure KD6_EMBEDDING_PROVIDER" + .into(), + )) + } + + fn dimensions(&self) -> usize { + 0 + } + + fn model_id(&self) -> &str { + "none" + } +} + +/// Returns `true` if the provider is the no-op placeholder. +pub fn is_noop(provider: &dyn EmbeddingProvider) -> bool { + provider.model_id() == "none" && provider.dimensions() == 0 +} + +// --------------------------------------------------------------------------- +// Automatic embedding helpers (OMS spec section 8.4) +// +// Shared by kd6-server and kd6-mcp so that both entry points apply the same +// server-side embedding logic. +// --------------------------------------------------------------------------- + +/// Extract embeddable text from a JSON content value. +/// +/// - String values are used directly. +/// - Object and array values have their string-typed leaves concatenated +/// (recursive traversal). +/// - Other types are serialized to their JSON representation. +pub fn content_to_text(content: &serde_json::Value) -> String { + match content { + serde_json::Value::String(s) => s.clone(), + serde_json::Value::Object(_) | serde_json::Value::Array(_) => { + let mut parts = Vec::new(); + collect_string_leaves(content, &mut parts); + if parts.is_empty() { + serde_json::to_string(content).unwrap_or_default() + } else { + parts.join(" ") + } + } + other => serde_json::to_string(other).unwrap_or_default(), + } +} + +/// Recursively collect string leaves from any JSON value. +fn collect_string_leaves(value: &serde_json::Value, out: &mut Vec) { + match value { + serde_json::Value::String(s) => out.push(s.clone()), + serde_json::Value::Object(map) => { + for v in map.values() { + collect_string_leaves(v, out); + } + } + serde_json::Value::Array(arr) => { + for v in arr { + collect_string_leaves(v, out); + } + } + _ => {} + } +} + +/// Compute an embedding for content if no embedding was provided and the +/// provider is active. Returns the embedding to use (either caller-provided +/// or freshly computed). +/// +/// Implements OMS spec section 8.4.1 (write) behavior. +pub async fn auto_embed_content( + provider: &dyn EmbeddingProvider, + content: &serde_json::Value, + existing_embedding: Option>, +) -> Result>, OmsError> { + if let Some(emb) = existing_embedding { + // Validate dimensionality if provider is configured + if !is_noop(provider) && emb.len() != provider.dimensions() { + return Err(OmsError::InvalidInput(format!( + "embedding has {} dimensions, expected {} for model {}", + emb.len(), + provider.dimensions(), + provider.model_id() + ))); + } + return Ok(Some(emb)); + } + + if is_noop(provider) { + return Ok(None); + } + + let text = content_to_text(content); + if text.is_empty() { + return Ok(None); + } + + let embeddings = provider.embed_texts(&[text]).await?; + Ok(embeddings.into_iter().next()) +} + +/// Compute a query embedding if none was provided and the provider is active. +/// +/// Implements OMS spec section 8.4.3 (search) behavior. +pub async fn auto_embed_query( + provider: &dyn EmbeddingProvider, + query: &str, + existing_embedding: Option>, +) -> Result>, OmsError> { + if let Some(emb) = existing_embedding { + if !is_noop(provider) && emb.len() != provider.dimensions() { + return Err(OmsError::InvalidInput(format!( + "query embedding has {} dimensions, expected {} for model {}", + emb.len(), + provider.dimensions(), + provider.model_id() + ))); + } + return Ok(Some(emb)); + } + + if is_noop(provider) { + return Ok(None); + } + + if query.trim().is_empty() { + return Ok(None); + } + + let embedding = provider.embed_query(query).await?; + Ok(Some(embedding)) +} + +/// Compute the embedding for a memory update request. +/// +/// Implements OMS spec section 8.4.2 (update) behavior. Handles the +/// three-state `Option>>` semantics: +/// +/// - `None` — caller didn't mention embedding: auto-compute if content changed +/// - `Some(None)` — explicitly clear the embedding +/// - `Some(Some(v))` — explicitly set a new embedding (validated) +pub async fn auto_embed_update( + provider: &dyn EmbeddingProvider, + new_content: Option<&serde_json::Value>, + embedding_field: Option>>, +) -> Result>>, OmsError> { + match embedding_field { + // Caller explicitly provided a new embedding — validate and use it + Some(Some(emb)) => { + if !is_noop(provider) && emb.len() != provider.dimensions() { + return Err(OmsError::InvalidInput(format!( + "embedding has {} dimensions, expected {} for model {}", + emb.len(), + provider.dimensions(), + provider.model_id() + ))); + } + Ok(Some(Some(emb))) + } + // Caller explicitly cleared the embedding — respect it + Some(None) => Ok(Some(None)), + // Caller didn't mention embedding — auto-compute if content changed + None => { + if let Some(content) = new_content { + let computed = auto_embed_content(provider, content, None).await?; + Ok(computed.map(Some)) + } else { + // No content change, no embedding change — preserve existing + Ok(None) + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use serde_json::json; + + // --- Deterministic fake embedder for tests --- + + struct FakeEmbedder; + + #[async_trait] + impl EmbeddingProvider for FakeEmbedder { + async fn embed_texts(&self, texts: &[String]) -> Result>, OmsError> { + Ok(texts + .iter() + .map(|t| vec![t.len() as f32, 0.0, 1.0]) + .collect()) + } + async fn embed_query(&self, query: &str) -> Result, OmsError> { + Ok(vec![query.len() as f32, 0.0, 1.0]) + } + fn dimensions(&self) -> usize { + 3 + } + fn model_id(&self) -> &str { + "fake-3d" + } + } + + // --- content_to_text --- + + #[test] + fn content_to_text_string() { + assert_eq!(content_to_text(&json!("hello")), "hello"); + } + + #[test] + fn content_to_text_object_extracts_string_leaves() { + let val = json!({"text": "hello", "nested": {"msg": "world"}}); + let text = content_to_text(&val); + assert!(text.contains("hello")); + assert!(text.contains("world")); + } + + #[test] + fn content_to_text_array_extracts_string_leaves() { + let val = json!(["alpha", {"inner": "beta"}, "gamma"]); + let text = content_to_text(&val); + assert!(text.contains("alpha")); + assert!(text.contains("beta")); + assert!(text.contains("gamma")); + } + + #[test] + fn content_to_text_number_serializes() { + assert_eq!(content_to_text(&json!(42)), "42"); + } + + #[test] + fn content_to_text_empty_object_falls_back_to_json() { + let val = json!({"count": 5}); + let text = content_to_text(&val); + // No string leaves, so falls back to JSON serialization + assert!(text.contains("count")); + } + + // --- is_noop --- + + #[test] + fn noop_detected_correctly() { + assert!(is_noop(&NoopEmbedder)); + assert!(!is_noop(&FakeEmbedder)); + } + + // --- auto_embed_content --- + + #[tokio::test] + async fn auto_embed_content_uses_provided_embedding() { + let result = auto_embed_content(&FakeEmbedder, &json!("text"), Some(vec![1.0, 2.0, 3.0])) + .await + .unwrap(); + assert_eq!(result, Some(vec![1.0, 2.0, 3.0])); + } + + #[tokio::test] + async fn auto_embed_content_rejects_wrong_dimensions() { + let result = auto_embed_content(&FakeEmbedder, &json!("text"), Some(vec![1.0, 2.0])).await; + assert!(result.is_err()); + } + + #[tokio::test] + async fn auto_embed_content_computes_when_none_provided() { + let result = auto_embed_content(&FakeEmbedder, &json!("hello"), None) + .await + .unwrap(); + // FakeEmbedder returns [len, 0.0, 1.0] — "hello" is 5 chars + assert_eq!(result, Some(vec![5.0, 0.0, 1.0])); + } + + #[tokio::test] + async fn auto_embed_content_returns_none_for_noop() { + let result = auto_embed_content(&NoopEmbedder, &json!("text"), None) + .await + .unwrap(); + assert_eq!(result, None); + } + + #[tokio::test] + async fn auto_embed_content_noop_passes_through_provided() { + let result = auto_embed_content(&NoopEmbedder, &json!("text"), Some(vec![1.0, 2.0])) + .await + .unwrap(); + assert_eq!(result, Some(vec![1.0, 2.0])); + } + + #[tokio::test] + async fn auto_embed_content_returns_none_for_empty_text() { + let result = auto_embed_content(&FakeEmbedder, &json!(""), None) + .await + .unwrap(); + assert_eq!(result, None); + } + + // --- auto_embed_query --- + + #[tokio::test] + async fn auto_embed_query_computes_when_none_provided() { + let result = auto_embed_query(&FakeEmbedder, "search term", None) + .await + .unwrap(); + assert_eq!(result, Some(vec![11.0, 0.0, 1.0])); + } + + #[tokio::test] + async fn auto_embed_query_uses_provided_embedding() { + let result = auto_embed_query(&FakeEmbedder, "query", Some(vec![9.0, 8.0, 7.0])) + .await + .unwrap(); + assert_eq!(result, Some(vec![9.0, 8.0, 7.0])); + } + + #[tokio::test] + async fn auto_embed_query_returns_none_for_empty_query() { + let result = auto_embed_query(&FakeEmbedder, " ", None).await.unwrap(); + assert_eq!(result, None); + } + + // --- auto_embed_update --- + + #[tokio::test] + async fn auto_embed_update_explicit_embedding_accepted() { + let result = auto_embed_update( + &FakeEmbedder, + Some(&json!("new content")), + Some(Some(vec![1.0, 2.0, 3.0])), + ) + .await + .unwrap(); + assert_eq!(result, Some(Some(vec![1.0, 2.0, 3.0]))); + } + + #[tokio::test] + async fn auto_embed_update_explicit_clear() { + let result = auto_embed_update(&FakeEmbedder, Some(&json!("new")), Some(None)) + .await + .unwrap(); + assert_eq!(result, Some(None)); + } + + #[tokio::test] + async fn auto_embed_update_auto_computes_on_content_change() { + let result = auto_embed_update(&FakeEmbedder, Some(&json!("new")), None) + .await + .unwrap(); + assert_eq!(result, Some(Some(vec![3.0, 0.0, 1.0]))); + } + + #[tokio::test] + async fn auto_embed_update_preserves_existing_when_no_content_change() { + let result = auto_embed_update(&FakeEmbedder, None, None).await.unwrap(); + assert_eq!(result, None); + } + + #[tokio::test] + async fn auto_embed_update_rejects_wrong_dimensions() { + let result = + auto_embed_update(&FakeEmbedder, Some(&json!("x")), Some(Some(vec![1.0]))).await; + assert!(result.is_err()); + } +} diff --git a/crates/kd6-core/src/lib.rs b/crates/kd6-core/src/lib.rs index 9f4f356..97e5988 100644 --- a/crates/kd6-core/src/lib.rs +++ b/crates/kd6-core/src/lib.rs @@ -1,6 +1,11 @@ +pub mod embedding; pub mod error; pub mod models; pub mod provider; +pub use embedding::{ + auto_embed_content, auto_embed_query, auto_embed_update, content_to_text, EmbeddingProvider, + NoopEmbedder, +}; pub use error::OmsError; pub use provider::OmsProvider; diff --git a/crates/kd6-core/src/models/audit.rs b/crates/kd6-core/src/models/audit.rs index da74a41..bf454c1 100644 --- a/crates/kd6-core/src/models/audit.rs +++ b/crates/kd6-core/src/models/audit.rs @@ -16,6 +16,12 @@ pub struct AuditEntry { #[serde(skip_serializing_if = "Option::is_none")] pub details: Option, pub created_at: DateTime, + /// When true, this entry has been anonymized by GDPR purge. + /// The `entry_hash` was computed from original (pre-redaction) content, + /// so content-hash verification should be skipped, but chain verification + /// (prev_hash links) remains valid. + #[serde(default, skip_serializing_if = "std::ops::Not::not")] + pub redacted: bool, } /// Filter for querying audit logs. diff --git a/crates/kd6-core/src/models/entry.rs b/crates/kd6-core/src/models/entry.rs index 1caa560..7f3d75b 100644 --- a/crates/kd6-core/src/models/entry.rs +++ b/crates/kd6-core/src/models/entry.rs @@ -40,6 +40,9 @@ pub struct MemoryEntry { // --- Graph metadata (Level 3) --- #[serde(skip_serializing_if = "Option::is_none")] pub entity_type: Option, + // --- Upsert support (see OMS spec 4.3.2) --- + #[serde(skip_serializing_if = "Option::is_none")] + pub upsert_key: Option, } #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] diff --git a/crates/kd6-core/src/models/store.rs b/crates/kd6-core/src/models/store.rs index 33955f9..a2c3be3 100644 --- a/crates/kd6-core/src/models/store.rs +++ b/crates/kd6-core/src/models/store.rs @@ -121,6 +121,10 @@ pub struct CreateMemoryRequest { // --- Graph metadata (Level 3) --- #[serde(skip_serializing_if = "Option::is_none")] pub entity_type: Option, + // --- Upsert support (see OMS spec 4.3.2) --- + /// When set, enables atomic create-or-replace within the same store, layer, and scope. + #[serde(skip_serializing_if = "Option::is_none")] + pub upsert_key: Option, } fn default_layer() -> MemoryLayer { diff --git a/crates/kd6-core/src/provider.rs b/crates/kd6-core/src/provider.rs index 56b5fea..2c4021e 100644 --- a/crates/kd6-core/src/provider.rs +++ b/crates/kd6-core/src/provider.rs @@ -39,6 +39,22 @@ pub trait OmsProvider: Send + Sync { async fn delete_store(&self, tenant_id: &str, store_id: Uuid) -> Result<(), OmsError>; + /// Atomically get an existing store by name, or create it if it doesn't exist. + /// Used for `_default` store auto-provisioning (OMS spec 4.1.1). + async fn get_or_create_store( + &self, + tenant_id: &str, + name: &str, + request: CreateStoreRequest, + ) -> Result { + // Default implementation: list + create (non-atomic, override for atomicity) + let stores = self.list_stores(tenant_id).await?; + if let Some(store) = stores.into_iter().find(|s| s.name == name) { + return Ok(store); + } + self.create_store(tenant_id, request).await + } + // --- Memory CRUD --- async fn create_memory( diff --git a/crates/kd6-embed/Cargo.toml b/crates/kd6-embed/Cargo.toml new file mode 100644 index 0000000..e598237 --- /dev/null +++ b/crates/kd6-embed/Cargo.toml @@ -0,0 +1,26 @@ +[package] +name = "kd6-embed" +version.workspace = true +edition.workspace = true +license.workspace = true + +[features] +default = ["local"] +local = ["fastembed"] + +[dependencies] +kd6-core = { path = "../kd6-core" } +async-trait = { workspace = true } +tracing = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +reqwest = { workspace = true } +tokio = { workspace = true } + +# Local embedding via ONNX (optional, default on) +fastembed = { version = "4", optional = true } + +[dev-dependencies] +tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } +wiremock = "0.6" +serde_json = { workspace = true } diff --git a/crates/kd6-embed/src/lib.rs b/crates/kd6-embed/src/lib.rs new file mode 100644 index 0000000..346fbcc --- /dev/null +++ b/crates/kd6-embed/src/lib.rs @@ -0,0 +1,7 @@ +#[cfg(feature = "local")] +mod local; +mod openai_compatible; + +#[cfg(feature = "local")] +pub use local::LocalEmbedder; +pub use openai_compatible::OpenAiCompatibleEmbedder; diff --git a/crates/kd6-embed/src/local.rs b/crates/kd6-embed/src/local.rs new file mode 100644 index 0000000..bd69d8c --- /dev/null +++ b/crates/kd6-embed/src/local.rs @@ -0,0 +1,86 @@ +use async_trait::async_trait; +use fastembed::{EmbeddingModel, InitOptions, TextEmbedding}; +use std::sync::Arc; +use tokio::sync::Mutex; + +use kd6_core::embedding::EmbeddingProvider; +use kd6_core::OmsError; + +/// In-process embedding provider using fastembed (ONNX Runtime). +/// +/// Downloads the model on first use (~25MB for the default model) +/// and caches it locally. No external services or API keys required. +pub struct LocalEmbedder { + model: Arc>, + model_id: String, + dimensions: usize, +} + +impl LocalEmbedder { + /// Create a new local embedder with the default model (all-MiniLM-L6-v2, 384 dimensions). + pub fn new() -> Result { + Self::with_model(EmbeddingModel::AllMiniLML6V2) + } + + /// Create a local embedder with a specific fastembed model. + pub fn with_model(model: EmbeddingModel) -> Result { + let info = TextEmbedding::get_model_info(&model) + .map_err(|e| OmsError::Internal(format!("unknown embedding model: {e}")))?; + let model_id = info.model_code.clone(); + let dimensions = info.dim; + + tracing::info!( + model = %model_id, + dimensions, + "initializing local embedding model" + ); + + let embedder = + TextEmbedding::try_new(InitOptions::new(model).with_show_download_progress(true)) + .map_err(|e| { + OmsError::Internal(format!("failed to initialize embedding model: {e}")) + })?; + + Ok(Self { + model: Arc::new(Mutex::new(embedder)), + model_id, + dimensions, + }) + } +} + +#[async_trait] +impl EmbeddingProvider for LocalEmbedder { + async fn embed_texts(&self, texts: &[String]) -> Result>, OmsError> { + let texts = texts.to_vec(); + let model = Arc::clone(&self.model); + + // fastembed is synchronous; run on a blocking thread to avoid + // starving the async runtime. + tokio::task::spawn_blocking(move || { + let model = model.blocking_lock(); + let str_refs: Vec<&str> = texts.iter().map(|s| s.as_str()).collect(); + model + .embed(str_refs, None) + .map_err(|e| OmsError::Internal(format!("embedding failed: {e}"))) + }) + .await + .map_err(|e| OmsError::Internal(format!("embedding task panicked: {e}")))? + } + + async fn embed_query(&self, query: &str) -> Result, OmsError> { + let results = self.embed_texts(&[query.to_string()]).await?; + results + .into_iter() + .next() + .ok_or_else(|| OmsError::Internal("embedding returned no results".into())) + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + fn model_id(&self) -> &str { + &self.model_id + } +} diff --git a/crates/kd6-embed/src/openai_compatible.rs b/crates/kd6-embed/src/openai_compatible.rs new file mode 100644 index 0000000..984c93f --- /dev/null +++ b/crates/kd6-embed/src/openai_compatible.rs @@ -0,0 +1,177 @@ +use std::time::Duration; + +use async_trait::async_trait; + +use kd6_core::embedding::EmbeddingProvider; +use kd6_core::OmsError; + +/// Request timeout for embedding API calls (30 seconds). +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// Connection timeout for embedding API calls (10 seconds). +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); + +/// Embedding provider that calls any OpenAI-compatible `/v1/embeddings` endpoint. +/// +/// Works with OpenAI, Azure OpenAI, Ollama, vLLM, LiteLLM, and any +/// service that speaks the same protocol. +pub struct OpenAiCompatibleEmbedder { + client: reqwest::Client, + endpoint: String, + model: String, + api_key: Option, + dimensions: usize, +} + +#[derive(serde::Serialize)] +struct EmbedRequest<'a> { + model: &'a str, + input: &'a [String], +} + +#[derive(serde::Deserialize)] +struct EmbedResponse { + data: Vec, +} + +#[derive(serde::Deserialize)] +struct EmbedData { + embedding: Vec, + index: usize, +} + +impl OpenAiCompatibleEmbedder { + /// Create a new OpenAI-compatible embedding provider. + /// + /// - `endpoint`: Base URL (e.g., `https://api.openai.com/v1`) + /// - `model`: Model name (e.g., `text-embedding-3-small`) + /// - `api_key`: Optional API key (not needed for local providers like Ollama) + /// - `dimensions`: Expected embedding dimensionality + pub fn new( + endpoint: String, + model: String, + api_key: Option, + dimensions: usize, + ) -> Result { + tracing::info!( + endpoint = %endpoint, + model = %model, + dimensions, + "configured OpenAI-compatible embedding provider" + ); + + let client = reqwest::Client::builder() + .timeout(REQUEST_TIMEOUT) + .connect_timeout(CONNECT_TIMEOUT) + .build() + .map_err(|e| OmsError::Internal(format!("failed to build HTTP client: {e}")))?; + + Ok(Self { + client, + endpoint: endpoint.trim_end_matches('/').to_string(), + model, + api_key, + dimensions, + }) + } +} + +#[async_trait] +impl EmbeddingProvider for OpenAiCompatibleEmbedder { + async fn embed_texts(&self, texts: &[String]) -> Result>, OmsError> { + if texts.is_empty() { + return Ok(vec![]); + } + + let url = format!("{}/embeddings", self.endpoint); + + let body = EmbedRequest { + model: &self.model, + input: texts, + }; + + let mut req = self.client.post(&url).json(&body); + if let Some(key) = &self.api_key { + req = req.bearer_auth(key); + } + + let response = req.send().await.map_err(|e| { + OmsError::Internal(format!( + "embedding request to {} failed: {e}", + self.endpoint + )) + })?; + + if !response.status().is_success() { + let status = response.status(); + let body = response + .text() + .await + .unwrap_or_else(|_| "failed to read response body".into()); + return Err(if status.as_u16() == 429 { + OmsError::Internal(format!("embedding endpoint rate limited (429): {body}")) + } else if status.is_client_error() { + OmsError::InvalidInput(format!( + "embedding endpoint rejected request ({status}): {body}" + )) + } else { + OmsError::Internal(format!("embedding endpoint returned {status}: {body}")) + }); + } + + let embed_response: EmbedResponse = response + .json() + .await + .map_err(|e| OmsError::Internal(format!("failed to parse embedding response: {e}")))?; + + if embed_response.data.len() != texts.len() { + return Err(OmsError::Internal(format!( + "embedding endpoint returned {} vectors for {} inputs", + embed_response.data.len(), + texts.len() + ))); + } + + // Sort by index to maintain input order (providers may return out-of-order) + let mut sorted = embed_response.data; + sorted.sort_by_key(|d| d.index); + + // Validate index set is exactly 0..N (no duplicates, gaps, or out-of-range) + for (i, item) in sorted.iter().enumerate() { + if item.index != i { + return Err(OmsError::Internal(format!( + "embedding response has invalid index sequence: expected {i}, got {}", + item.index + ))); + } + } + + // Validate dimensionality of all results + for (i, item) in sorted.iter().enumerate() { + if item.embedding.len() != self.dimensions { + return Err(OmsError::Internal(format!( + "embedding at index {i} has {} dimensions, expected {}", + item.embedding.len(), + self.dimensions + ))); + } + } + + Ok(sorted.into_iter().map(|d| d.embedding).collect()) + } + + async fn embed_query(&self, query: &str) -> Result, OmsError> { + let results = self.embed_texts(&[query.to_string()]).await?; + results + .into_iter() + .next() + .ok_or_else(|| OmsError::Internal("embedding returned no results".into())) + } + + fn dimensions(&self) -> usize { + self.dimensions + } + + fn model_id(&self) -> &str { + &self.model + } +} diff --git a/crates/kd6-embed/tests/openai_compatible.rs b/crates/kd6-embed/tests/openai_compatible.rs new file mode 100644 index 0000000..8e1c866 --- /dev/null +++ b/crates/kd6-embed/tests/openai_compatible.rs @@ -0,0 +1,257 @@ +use kd6_core::embedding::EmbeddingProvider; +use kd6_embed::OpenAiCompatibleEmbedder; +use serde_json::json; +use wiremock::matchers::{method, path}; +use wiremock::{Mock, MockServer, ResponseTemplate}; + +/// Helper to start a mock server and create an embedder pointing at it. +async fn mock_embedder(dims: usize) -> (MockServer, OpenAiCompatibleEmbedder) { + let server = MockServer::start().await; + let embedder = + OpenAiCompatibleEmbedder::new(server.uri(), "test-model".into(), None, dims).unwrap(); + (server, embedder) +} + +fn embedding_response(data: Vec<(usize, Vec)>) -> ResponseTemplate { + let data: Vec<_> = data + .into_iter() + .map(|(idx, emb)| json!({"index": idx, "embedding": emb})) + .collect(); + ResponseTemplate::new(200).set_body_json(json!({ + "object": "list", + "data": data, + "model": "test-model", + "usage": {"prompt_tokens": 10, "total_tokens": 10} + })) +} + +#[tokio::test] +async fn test_single_text_embedding() { + let (server, embedder) = mock_embedder(3).await; + + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![(0, vec![0.1, 0.2, 0.3])])) + .mount(&server) + .await; + + let result = embedder.embed_texts(&["hello".into()]).await.unwrap(); + assert_eq!(result.len(), 1); + assert_eq!(result[0], vec![0.1, 0.2, 0.3]); +} + +#[tokio::test] +async fn test_batch_embedding() { + let (server, embedder) = mock_embedder(2).await; + + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![ + (0, vec![1.0, 2.0]), + (1, vec![3.0, 4.0]), + (2, vec![5.0, 6.0]), + ])) + .mount(&server) + .await; + + let texts: Vec = vec!["a".into(), "b".into(), "c".into()]; + let result = embedder.embed_texts(&texts).await.unwrap(); + assert_eq!(result.len(), 3); + assert_eq!(result[0], vec![1.0, 2.0]); + assert_eq!(result[2], vec![5.0, 6.0]); +} + +#[tokio::test] +async fn test_out_of_order_indexes_sorted() { + let (server, embedder) = mock_embedder(2).await; + + // Server returns results out of order (index 1 before 0) + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![ + (1, vec![3.0, 4.0]), + (0, vec![1.0, 2.0]), + ])) + .mount(&server) + .await; + + let result = embedder + .embed_texts(&["first".into(), "second".into()]) + .await + .unwrap(); + // Should be sorted by index + assert_eq!(result[0], vec![1.0, 2.0]); + assert_eq!(result[1], vec![3.0, 4.0]); +} + +#[tokio::test] +async fn test_empty_input_returns_empty() { + let (_server, embedder) = mock_embedder(3).await; + // No mock needed — empty input short-circuits before HTTP call + let result = embedder.embed_texts(&[]).await.unwrap(); + assert!(result.is_empty()); +} + +#[tokio::test] +async fn test_embed_query_delegates_to_embed_texts() { + let (server, embedder) = mock_embedder(2).await; + + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![(0, vec![9.0, 8.0])])) + .mount(&server) + .await; + + let result = embedder.embed_query("question").await.unwrap(); + assert_eq!(result, vec![9.0, 8.0]); +} + +#[tokio::test] +async fn test_dimension_mismatch_error() { + let (server, embedder) = mock_embedder(3).await; + + // Return 2-dim when 3-dim expected + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![(0, vec![1.0, 2.0])])) + .mount(&server) + .await; + + let err = embedder + .embed_texts(&["test".into()]) + .await + .expect_err("should reject wrong dimensions"); + let msg = format!("{err}"); + assert!(msg.contains("2 dimensions"), "error: {msg}"); + assert!(msg.contains("expected 3"), "error: {msg}"); +} + +#[tokio::test] +async fn test_count_mismatch_error() { + let (server, embedder) = mock_embedder(2).await; + + // Return 1 vector for 2 inputs + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![(0, vec![1.0, 2.0])])) + .mount(&server) + .await; + + let err = embedder + .embed_texts(&["a".into(), "b".into()]) + .await + .expect_err("should reject count mismatch"); + let msg = format!("{err}"); + assert!(msg.contains("1 vectors"), "error: {msg}"); + assert!(msg.contains("2 inputs"), "error: {msg}"); +} + +#[tokio::test] +async fn test_duplicate_index_error() { + let (server, embedder) = mock_embedder(2).await; + + // Duplicate index 0 + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(embedding_response(vec![ + (0, vec![1.0, 2.0]), + (0, vec![3.0, 4.0]), + ])) + .mount(&server) + .await; + + let err = embedder + .embed_texts(&["a".into(), "b".into()]) + .await + .expect_err("should reject duplicate indexes"); + let msg = format!("{err}"); + assert!(msg.contains("invalid index"), "error: {msg}"); +} + +#[tokio::test] +async fn test_server_error_500() { + let (server, embedder) = mock_embedder(3).await; + + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(500).set_body_string("internal server error")) + .mount(&server) + .await; + + let err = embedder + .embed_texts(&["test".into()]) + .await + .expect_err("should propagate 500"); + let msg = format!("{err}"); + assert!(msg.contains("500"), "error: {msg}"); +} + +#[tokio::test] +async fn test_client_error_400() { + let (server, embedder) = mock_embedder(3).await; + + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(400).set_body_string("bad request")) + .mount(&server) + .await; + + let err = embedder + .embed_texts(&["test".into()]) + .await + .expect_err("should propagate 400"); + let msg = format!("{err}"); + assert!(msg.contains("rejected request"), "error: {msg}"); +} + +#[tokio::test] +async fn test_rate_limit_429() { + let (server, embedder) = mock_embedder(3).await; + + Mock::given(method("POST")) + .and(path("/embeddings")) + .respond_with(ResponseTemplate::new(429).set_body_string("rate limited")) + .mount(&server) + .await; + + let err = embedder + .embed_texts(&["test".into()]) + .await + .expect_err("should propagate 429"); + let msg = format!("{err}"); + assert!(msg.contains("rate limited"), "error: {msg}"); +} + +#[tokio::test] +async fn test_api_key_sent_as_bearer() { + let server = MockServer::start().await; + let embedder = OpenAiCompatibleEmbedder::new( + server.uri(), + "test-model".into(), + Some("sk-test-key".into()), + 2, + ) + .unwrap(); + + Mock::given(method("POST")) + .and(path("/embeddings")) + .and(wiremock::matchers::header( + "authorization", + "Bearer sk-test-key", + )) + .respond_with(embedding_response(vec![(0, vec![1.0, 2.0])])) + .mount(&server) + .await; + + let result = embedder.embed_texts(&["hello".into()]).await.unwrap(); + assert_eq!(result.len(), 1); +} + +#[tokio::test] +async fn test_model_id_and_dimensions() { + let embedder = + OpenAiCompatibleEmbedder::new("http://localhost:1".into(), "my-model".into(), None, 768) + .unwrap(); + assert_eq!(embedder.model_id(), "my-model"); + assert_eq!(embedder.dimensions(), 768); +} diff --git a/crates/kd6-mcp/Cargo.toml b/crates/kd6-mcp/Cargo.toml index dd1466b..9b4da26 100644 --- a/crates/kd6-mcp/Cargo.toml +++ b/crates/kd6-mcp/Cargo.toml @@ -9,6 +9,7 @@ description = "MCP (Model Context Protocol) server for KD6 OMS" [dependencies] kd6-core = { path = "../kd6-core" } kd6-sqlite = { path = "../kd6-sqlite" } +kd6-embed = { path = "../kd6-embed" } rmcp = { workspace = true } schemars = { workspace = true } @@ -26,3 +27,4 @@ anyhow = "1" [dev-dependencies] tokio = { workspace = true, features = ["macros", "rt-multi-thread"] } kd6-sqlite = { path = "../kd6-sqlite" } +async-trait = { workspace = true } diff --git a/crates/kd6-mcp/src/lib.rs b/crates/kd6-mcp/src/lib.rs index 2babae2..1e41288 100644 --- a/crates/kd6-mcp/src/lib.rs +++ b/crates/kd6-mcp/src/lib.rs @@ -1,6 +1,7 @@ mod tools; pub use tools::{ - CreateEdgeParams, CreateMemoryParams, CreateStoreParams, DeleteMemoryParams, GetMemoryParams, - GraphTraverseParams, ListStoresParams, Kd6McpServer, SearchMemoriesParams, StoreStatsParams, + CreateEdgeParams, CreateMemoryParams, CreateStoreParams, DeleteMemoryParams, GdprPurgeParams, + GetMemoryParams, GraphTraverseParams, Kd6McpServer, ListStoresParams, SearchMemoriesParams, + StoreStatsParams, }; diff --git a/crates/kd6-mcp/src/main.rs b/crates/kd6-mcp/src/main.rs index 287f692..322b12d 100644 --- a/crates/kd6-mcp/src/main.rs +++ b/crates/kd6-mcp/src/main.rs @@ -1,6 +1,6 @@ use std::sync::Arc; -use kd6_core::OmsProvider; +use kd6_core::{EmbeddingProvider, NoopEmbedder, OmsProvider}; use kd6_sqlite::SqliteProvider; use rmcp::transport::streamable_http_server::session::local::LocalSessionManager; use rmcp::transport::streamable_http_server::StreamableHttpService; @@ -16,27 +16,71 @@ async fn main() -> anyhow::Result<()> { .with_writer(std::io::stderr) .init(); - let db_url = std::env::var("KD6_DATABASE_URL") - .unwrap_or_else(|_| "sqlite:kd6.db?mode=rwc".to_string()); + let db_url = + std::env::var("KD6_DATABASE_URL").unwrap_or_else(|_| "sqlite:kd6.db?mode=rwc".to_string()); let provider = Arc::new(SqliteProvider::new(&db_url).await?) as Arc; - let transport = std::env::var("KD6_MCP_TRANSPORT") - .unwrap_or_else(|_| "http".to_string()); + // --- Embedding provider (same pattern as kd6-server) --- + let embedding_provider = + std::env::var("KD6_EMBEDDING_PROVIDER").unwrap_or_else(|_| "local".into()); + + let embedder: Arc = match embedding_provider.as_str() { + "local" => { + tracing::info!("embedding provider: local (fastembed, in-process ONNX)"); + Arc::new(kd6_embed::LocalEmbedder::new()?) + } + "openai-compatible" => { + let endpoint = std::env::var("KD6_EMBEDDING_ENDPOINT").expect( + "KD6_EMBEDDING_ENDPOINT is required when KD6_EMBEDDING_PROVIDER=openai-compatible", + ); + let model = std::env::var("KD6_EMBEDDING_MODEL").expect( + "KD6_EMBEDDING_MODEL is required when KD6_EMBEDDING_PROVIDER=openai-compatible", + ); + let api_key = std::env::var("KD6_EMBEDDING_API_KEY").ok(); + let dimensions: usize = std::env::var("KD6_EMBEDDING_DIMENSIONS") + .unwrap_or_else(|_| "1536".into()) + .parse() + .expect("KD6_EMBEDDING_DIMENSIONS must be a positive integer"); + tracing::info!( + "embedding provider: openai-compatible (endpoint={endpoint}, model={model})" + ); + Arc::new( + kd6_embed::OpenAiCompatibleEmbedder::new(endpoint, model, api_key, dimensions) + .expect("failed to create OpenAI-compatible embedding provider"), + ) + } + "none" => { + tracing::info!( + "embedding provider: none (pass-through, callers must supply embeddings)" + ); + Arc::new(NoopEmbedder) + } + other => { + anyhow::bail!("unknown KD6_EMBEDDING_PROVIDER: {other}. Valid options: local, openai-compatible, none"); + } + }; + + tracing::info!( + model_id = embedder.model_id(), + dimensions = embedder.dimensions(), + "embedding provider ready" + ); + + let transport = std::env::var("KD6_MCP_TRANSPORT").unwrap_or_else(|_| "http".to_string()); match transport.as_str() { "stdio" => { tracing::info!("KD6 MCP server starting on stdio"); - let server = Kd6McpServer::new(provider); + let server = Kd6McpServer::new(provider, embedder); let service = server.serve(rmcp::transport::stdio()).await?; service.waiting().await?; } _ => { - let addr = std::env::var("KD6_MCP_ADDR") - .unwrap_or_else(|_| "0.0.0.0:8081".to_string()); + let addr = std::env::var("KD6_MCP_ADDR").unwrap_or_else(|_| "0.0.0.0:8081".to_string()); let service = StreamableHttpService::new( - move || Ok(Kd6McpServer::new(provider.clone())), + move || Ok(Kd6McpServer::new(provider.clone(), embedder.clone())), LocalSessionManager::default().into(), Default::default(), ); diff --git a/crates/kd6-mcp/src/tools.rs b/crates/kd6-mcp/src/tools.rs index dcc36dc..7bde36c 100644 --- a/crates/kd6-mcp/src/tools.rs +++ b/crates/kd6-mcp/src/tools.rs @@ -6,6 +6,7 @@ use rmcp::{tool, tool_handler, tool_router, ServerHandler}; use schemars::JsonSchema; use serde::{Deserialize, Serialize}; +use kd6_core::embedding::{auto_embed_content, auto_embed_query, EmbeddingProvider}; use kd6_core::models::{ AccessControl, CreateEdgeRequest, CreateMemoryRequest, CreateStoreRequest, GraphTraversalRequest, MemoryLayer, MemoryScope, SearchQuery, StoreConfig, @@ -16,17 +17,24 @@ use kd6_core::OmsProvider; #[derive(Clone)] pub struct Kd6McpServer { provider: Arc, + embedder: Arc, #[allow(dead_code)] tool_router: ToolRouter, } impl Kd6McpServer { - pub fn new(provider: Arc) -> Self { + pub fn new(provider: Arc, embedder: Arc) -> Self { Self { provider, + embedder, tool_router: Self::tool_router(), } } + + /// Return the list of registered MCP tools (useful for testing/introspection). + pub fn list_tools(&self) -> Vec { + self.tool_router.list_all() + } } #[tool_handler( @@ -52,18 +60,22 @@ pub struct ListStoresParams { pub tenant_id: String, } -#[derive(Debug, Deserialize, JsonSchema)] +#[derive(Debug, Default, Deserialize, JsonSchema)] pub struct CreateMemoryParams { /// Tenant identifier. + #[serde(default)] pub tenant_id: String, /// Store ID to create the memory in. + #[serde(default)] pub store_id: String, /// Memory layer: working, episodic, semantic, procedural, or archival. #[serde(default = "default_layer_str")] pub layer: String, /// The memory content as a JSON string or plain text. - pub content: String, + #[serde(default)] + pub content: serde_json::Value, /// Agent that owns this memory. + #[serde(default)] pub owner_agent_id: String, /// Tags for categorization. #[serde(default)] @@ -71,6 +83,16 @@ pub struct CreateMemoryParams { /// Optional entity type for graph nodes. #[serde(default)] pub entity_type: Option, + /// Optional upsert key for atomic create-or-replace (see OMS spec 4.3.2). + #[serde(default)] + pub upsert_key: Option, + /// Optional scope fields for finer-grained visibility. + #[serde(default)] + pub scope_user_id: Option, + #[serde(default)] + pub scope_agent_id: Option, + #[serde(default)] + pub scope_session_id: Option, } fn default_layer_str() -> String { @@ -170,6 +192,30 @@ pub struct StoreStatsParams { pub store_id: String, } +#[derive(Debug, Default, Deserialize, JsonSchema)] +pub struct GdprPurgeParams { + /// Tenant identifier. + pub tenant_id: String, + /// Store ID. + pub store_id: String, + /// Scope to purge. All memories matching this scope will be permanently deleted + /// and associated audit entries will be anonymized. + #[serde(default)] + pub scope_org_id: Option, + #[serde(default)] + pub scope_team_id: Option, + #[serde(default)] + pub scope_project_id: Option, + #[serde(default)] + pub scope_user_id: Option, + #[serde(default)] + pub scope_agent_id: Option, + #[serde(default)] + pub scope_session_id: Option, + #[serde(default)] + pub scope_run_id: Option, +} + // --- Tool result helper --- #[derive(Serialize)] @@ -277,8 +323,12 @@ impl Kd6McpServer { Err(e) => return err_json(e), }; - let content = serde_json::from_str(&p.content) - .unwrap_or_else(|_| serde_json::json!({"text": p.content})); + let content: serde_json::Value = + if p.content.is_null() || p.content == serde_json::Value::String(String::new()) { + serde_json::Value::String(String::new()) + } else { + p.content + }; let request = CreateMemoryRequest { layer, @@ -287,6 +337,9 @@ impl Kd6McpServer { owner_agent_id: p.owner_agent_id, scope: MemoryScope { tenant_id: p.tenant_id.clone(), + user_id: p.scope_user_id, + agent_id: p.scope_agent_id, + session_id: p.scope_session_id, ..Default::default() }, tags: p.tags, @@ -299,6 +352,21 @@ impl Kd6McpServer { valid_until: None, confidence: None, entity_type: p.entity_type, + upsert_key: p.upsert_key, + }; + + // Auto-embed content (OMS spec section 8.4.1) + let embedding = + match auto_embed_content(&*self.embedder, &request.content, request.embedding.clone()) + .await + { + Ok(emb) => emb, + Err(e) => return err_json(e.to_string()), + }; + + let request = CreateMemoryRequest { + embedding, + ..request }; match self @@ -345,9 +413,15 @@ impl Kd6McpServer { Err(e) => return err_json(e), }; + // Auto-embed query (OMS spec section 8.4.3) + let embedding = match auto_embed_query(&*self.embedder, &p.query, None).await { + Ok(emb) => emb, + Err(e) => return err_json(e.to_string()), + }; + let query = SearchQuery { query: p.query, - embedding: None, + embedding, layers: vec![], scope: None, top_k: p.top_k, @@ -423,9 +497,10 @@ impl Kd6McpServer { } #[tool( + name = "traverse_graph", description = "Traverse the knowledge graph starting from a memory, following relationship edges" )] - pub async fn graph_traverse(&self, Parameters(p): Parameters) -> String { + pub async fn traverse_graph(&self, Parameters(p): Parameters) -> String { if let Err(e) = validate_tenant_id(&p.tenant_id) { return err_json(e); } @@ -469,6 +544,39 @@ impl Kd6McpServer { Err(e) => err_json(e.to_string()), } } + + #[tool( + description = "GDPR purge: permanently delete all memories matching the given scope and anonymize related audit entries" + )] + pub async fn gdpr_purge(&self, Parameters(p): Parameters) -> String { + if let Err(e) = validate_tenant_id(&p.tenant_id) { + return err_json(e); + } + let store_id = match parse_uuid(&p.store_id) { + Ok(id) => id, + Err(e) => return err_json(e), + }; + + let scope = MemoryScope { + tenant_id: p.tenant_id.clone(), + org_id: p.scope_org_id, + team_id: p.scope_team_id, + project_id: p.scope_project_id, + user_id: p.scope_user_id, + agent_id: p.scope_agent_id, + session_id: p.scope_session_id, + run_id: p.scope_run_id, + }; + + match self + .provider + .gdpr_purge(&p.tenant_id, store_id, scope) + .await + { + Ok(deleted) => ok_json(serde_json::json!({ "deleted": deleted })), + Err(e) => err_json(e.to_string()), + } + } } #[cfg(test)] diff --git a/crates/kd6-mcp/tests/integration.rs b/crates/kd6-mcp/tests/integration.rs index 25bb402..52ee85b 100644 --- a/crates/kd6-mcp/tests/integration.rs +++ b/crates/kd6-mcp/tests/integration.rs @@ -1,19 +1,24 @@ use std::sync::Arc; -use kd6_core::OmsProvider; +use kd6_core::{EmbeddingProvider, NoopEmbedder, OmsProvider}; use kd6_mcp::{ - CreateEdgeParams, CreateMemoryParams, CreateStoreParams, DeleteMemoryParams, GetMemoryParams, - GraphTraverseParams, ListStoresParams, Kd6McpServer, SearchMemoriesParams, StoreStatsParams, + CreateEdgeParams, CreateMemoryParams, CreateStoreParams, DeleteMemoryParams, GdprPurgeParams, + GetMemoryParams, GraphTraverseParams, Kd6McpServer, ListStoresParams, SearchMemoriesParams, + StoreStatsParams, }; use kd6_sqlite::SqliteProvider; use rmcp::handler::server::wrapper::Parameters; +use rmcp::ServerHandler; use serde_json::{json, Value}; const TENANT_ID: &str = "tenant-1"; async fn test_server() -> Kd6McpServer { let provider = SqliteProvider::new("sqlite::memory:").await.unwrap(); - Kd6McpServer::new(Arc::new(provider) as Arc) + Kd6McpServer::new( + Arc::new(provider) as Arc, + Arc::new(NoopEmbedder), + ) } fn parse_response(response: String) -> Value { @@ -43,10 +48,10 @@ async fn create_memory( tenant_id: tenant_id.to_string(), store_id: store_id.to_string(), layer: "working".to_string(), - content: content.to_string(), + content: json!(content), owner_agent_id: "agent-1".to_string(), tags: vec!["test".to_string()], - entity_type: None, + ..Default::default() })) .await, ) @@ -101,10 +106,7 @@ async fn create_memory_returns_created_entry() { assert_eq!(response["success"], json!(true)); assert_eq!(response["data"]["store_id"], json!(store_id)); - assert_eq!( - response["data"]["content"], - json!({"text": "remember this"}) - ); + assert_eq!(response["data"]["content"], json!("remember this")); } #[tokio::test] @@ -127,7 +129,7 @@ async fn get_memory_returns_created_entry() { assert_eq!(fetched["success"], json!(true)); assert_eq!(fetched["data"]["id"], json!(memory_id)); - assert_eq!(fetched["data"]["content"], json!({"text": "remember me"})); + assert_eq!(fetched["data"]["content"], json!("remember me")); } #[tokio::test] @@ -153,7 +155,7 @@ async fn search_memories_finds_keyword_match() { assert_eq!(response["data"].as_array().unwrap().len(), 1); assert_eq!( response["data"][0]["entry"]["content"], - json!({"text": "alpha keyword match"}) + json!("alpha keyword match") ); } @@ -244,7 +246,7 @@ async fn graph_traverse_returns_neighboring_nodes() { let response = parse_response( server - .graph_traverse(Parameters(GraphTraverseParams { + .traverse_graph(Parameters(GraphTraverseParams { tenant_id: TENANT_ID.to_string(), store_id: store_id.to_string(), start_memory_id: source_id.to_string(), @@ -309,3 +311,249 @@ async fn create_memory_rejects_invalid_store_id() { .unwrap() .contains("invalid UUID 'not-a-uuid'")); } + +// --------------------------------------------------------------------------- +// Embedding-aware MCP tests +// --------------------------------------------------------------------------- + +/// Deterministic fake embedder for tests (avoids slow model download). +/// Produces 3-dimensional vectors based on text length. +struct FakeEmbedder; + +#[async_trait::async_trait] +impl EmbeddingProvider for FakeEmbedder { + async fn embed_texts(&self, texts: &[String]) -> Result>, kd6_core::OmsError> { + Ok(texts + .iter() + .map(|t| { + let len = t.len() as f32; + vec![len, len * 0.5, 1.0] + }) + .collect()) + } + async fn embed_query(&self, query: &str) -> Result, kd6_core::OmsError> { + let len = query.len() as f32; + Ok(vec![len, len * 0.5, 1.0]) + } + fn dimensions(&self) -> usize { + 3 + } + fn model_id(&self) -> &str { + "fake-3d" + } +} + +async fn test_server_with_embedder() -> Kd6McpServer { + let provider = SqliteProvider::new("sqlite::memory:").await.unwrap(); + Kd6McpServer::new( + Arc::new(provider) as Arc, + Arc::new(FakeEmbedder) as Arc, + ) +} + +#[tokio::test] +async fn mcp_create_memory_produces_embedding() { + let server = test_server_with_embedder().await; + let store = create_store(&server, TENANT_ID, "embed-store").await; + let store_id = store["data"]["id"].as_str().unwrap(); + + let response = create_memory( + &server, + TENANT_ID, + store_id, + "embeddings should be auto-computed", + ) + .await; + + assert_eq!(response["success"], json!(true)); + assert!( + response["data"]["embedding"].is_array(), + "MCP create_memory should auto-embed when embedder is configured" + ); + let dims = response["data"]["embedding"].as_array().unwrap().len(); + assert_eq!(dims, 3, "FakeEmbedder produces 3-dim vectors"); +} + +#[tokio::test] +async fn mcp_vector_search_returns_results() { + let server = test_server_with_embedder().await; + let store = create_store(&server, TENANT_ID, "search-embed-store").await; + let store_id = store["data"]["id"].as_str().unwrap(); + + // Add documents with varying content lengths for distinct embeddings + for text in [ + "short", + "a medium length document about many topics", + "another quite different and longer document for testing purposes here", + ] { + create_memory(&server, TENANT_ID, store_id, text).await; + } + + // Vector search + let response = parse_response( + server + .search_memories(Parameters(SearchMemoriesParams { + tenant_id: TENANT_ID.to_string(), + store_id: store_id.to_string(), + query: "short".to_string(), + top_k: 3, + keyword: false, + })) + .await, + ); + + assert_eq!(response["success"], json!(true)); + let results = response["data"].as_array().unwrap(); + assert!( + !results.is_empty(), + "vector search should return results with FakeEmbedder" + ); +} + +#[tokio::test] +async fn mcp_noop_embedder_allows_keyword_search() { + // With NoopEmbedder, keyword search should still work + let server = test_server().await; + let store = create_store(&server, TENANT_ID, "keyword-only-store").await; + let store_id = store["data"]["id"].as_str().unwrap(); + + create_memory(&server, TENANT_ID, store_id, "unique keyword testphrase").await; + + let response = parse_response( + server + .search_memories(Parameters(SearchMemoriesParams { + tenant_id: TENANT_ID.to_string(), + store_id: store_id.to_string(), + query: "testphrase".to_string(), + top_k: 10, + keyword: true, + })) + .await, + ); + + assert_eq!(response["success"], json!(true)); + let results = response["data"].as_array().unwrap(); + assert_eq!(results.len(), 1, "keyword search should find the document"); +} + +#[tokio::test] +async fn mcp_gdpr_purge_deletes_scoped_memories() { + let server = test_server().await; + + // Create a store + let store_response = parse_response( + server + .create_store(Parameters(CreateStoreParams { + tenant_id: TENANT_ID.to_string(), + name: "gdpr-test".to_string(), + })) + .await, + ); + let store_id = store_response["data"]["id"].as_str().unwrap().to_string(); + + // Create two memories with different scopes + let _ = parse_response( + server + .create_memory(Parameters(CreateMemoryParams { + tenant_id: TENANT_ID.to_string(), + store_id: store_id.clone(), + content: json!("user-a data"), + layer: "working".to_string(), + owner_agent_id: "test-agent".to_string(), + scope_user_id: Some("user-a".to_string()), + ..Default::default() + })) + .await, + ); + let _ = parse_response( + server + .create_memory(Parameters(CreateMemoryParams { + tenant_id: TENANT_ID.to_string(), + store_id: store_id.clone(), + content: json!("user-b data"), + layer: "working".to_string(), + owner_agent_id: "test-agent".to_string(), + scope_user_id: Some("user-b".to_string()), + ..Default::default() + })) + .await, + ); + + // Purge user-a + let purge_response = parse_response( + server + .gdpr_purge(Parameters(GdprPurgeParams { + tenant_id: TENANT_ID.to_string(), + store_id: store_id.clone(), + scope_user_id: Some("user-a".to_string()), + ..Default::default() + })) + .await, + ); + assert_eq!(purge_response["success"], json!(true)); + assert_eq!(purge_response["data"]["deleted"], json!(1)); + + // Verify user-b's data still exists via search + let search_response = parse_response( + server + .search_memories(Parameters(SearchMemoriesParams { + tenant_id: TENANT_ID.to_string(), + store_id: store_id.clone(), + query: "user-b".to_string(), + top_k: 10, + keyword: true, + })) + .await, + ); + assert_eq!(search_response["success"], json!(true)); + let results = search_response["data"].as_array().unwrap(); + assert_eq!(results.len(), 1, "user-b data should survive purge"); +} + +// --------------------------------------------------------------------------- +// MCP transport smoke tests +// --------------------------------------------------------------------------- + +/// Verify the server registers exactly the expected 10 tools. +#[tokio::test] +async fn test_mcp_tool_registration() { + let server = test_server().await; + let tools = server.list_tools(); + let tool_names: std::collections::BTreeSet = + tools.iter().map(|t| t.name.to_string()).collect(); + + let expected: std::collections::BTreeSet = [ + "create_store", + "list_stores", + "store_stats", + "create_memory", + "get_memory", + "search_memories", + "delete_memory", + "create_edge", + "traverse_graph", + "gdpr_purge", + ] + .iter() + .map(|s| s.to_string()) + .collect(); + + assert_eq!( + tool_names, expected, + "registered tools mismatch: got {tool_names:?}" + ); +} + +/// Verify server metadata (name, version) is set correctly. +#[tokio::test] +async fn test_mcp_server_info() { + let server = test_server().await; + let info = server.get_info(); + + assert_eq!(info.server_info.name, "kd6"); + assert_eq!(info.server_info.version, "0.1.0"); + assert!( + info.instructions.is_some(), + "server should have instructions" + ); +} diff --git a/crates/kd6-server/Cargo.toml b/crates/kd6-server/Cargo.toml index bd09eab..7cb6496 100644 --- a/crates/kd6-server/Cargo.toml +++ b/crates/kd6-server/Cargo.toml @@ -6,6 +6,7 @@ license.workspace = true [dependencies] kd6-core = { path = "../kd6-core" } +kd6-embed = { path = "../kd6-embed" } kd6-sqlite = { path = "../kd6-sqlite" } axum = { workspace = true } tokio = { workspace = true } @@ -23,4 +24,6 @@ anyhow = "1" [dev-dependencies] axum-test = "17" +async-trait = { workspace = true } +kd6-embed = { path = "../kd6-embed" } kd6-sqlite = { path = "../kd6-sqlite" } diff --git a/crates/kd6-server/src/embed.rs b/crates/kd6-server/src/embed.rs new file mode 100644 index 0000000..5c83016 --- /dev/null +++ b/crates/kd6-server/src/embed.rs @@ -0,0 +1,2 @@ +// Re-export embedding helpers from kd6-core for use by route handlers. +pub use kd6_core::embedding::{auto_embed_content, auto_embed_query, auto_embed_update}; diff --git a/crates/kd6-server/src/extract.rs b/crates/kd6-server/src/extract.rs index 08ab282..3cc942f 100644 --- a/crates/kd6-server/src/extract.rs +++ b/crates/kd6-server/src/extract.rs @@ -6,8 +6,19 @@ use axum::response::{IntoResponse, Response}; use axum::Json; use serde::de::DeserializeOwned; use serde_json::json; +use uuid::Uuid; + +use crate::state::AppState; + +/// The well-known default tenant identifier (OMS spec 4.4.1). +pub const DEFAULT_TENANT: &str = "_default"; + +/// The well-known default store alias (OMS spec 4.1.1). +pub const DEFAULT_STORE_ALIAS: &str = "_default"; /// Extracts tenant identity from `X-Tenant-ID` header. +/// Falls back to `_default` when the header is absent and default tenant +/// resolution is enabled in ServerConfig (OMS spec 4.4.1). pub struct TenantId(pub String); pub struct TenantIdRejection(&'static str); @@ -18,18 +29,49 @@ impl IntoResponse for TenantIdRejection { } } -impl FromRequestParts for TenantId { +impl FromRequestParts for TenantId { type Rejection = TenantIdRejection; - async fn from_request_parts(parts: &mut Parts, _state: &S) -> Result { - parts + async fn from_request_parts( + parts: &mut Parts, + state: &AppState, + ) -> Result { + let header_value = parts .headers .get("X-Tenant-ID") .and_then(|v| v.to_str().ok()) .map(|s| s.trim()) .filter(|s| !s.is_empty()) - .map(|s| TenantId(s.to_string())) - .ok_or(TenantIdRejection("X-Tenant-ID header is required")) + .map(|s| s.to_string()); + + match header_value { + Some(tenant) => Ok(TenantId(tenant)), + None if state.config.default_tenant => Ok(TenantId(DEFAULT_TENANT.to_string())), + None => Err(TenantIdRejection("X-Tenant-ID header is required")), + } + } +} + +/// Resolved store identifier -- either a concrete UUID or the `_default` alias. +#[derive(Debug, Clone)] +pub enum StoreRef { + Id(Uuid), + Default, +} + +impl<'de> serde::Deserialize<'de> for StoreRef { + fn deserialize(deserializer: D) -> Result + where + D: serde::Deserializer<'de>, + { + let s = String::deserialize(deserializer)?; + if s == DEFAULT_STORE_ALIAS { + Ok(StoreRef::Default) + } else { + Uuid::parse_str(&s) + .map(StoreRef::Id) + .map_err(serde::de::Error::custom) + } } } @@ -121,3 +163,36 @@ where Ok(PathId(value)) } } + +/// Resolve a `StoreRef` to a concrete UUID. When the reference is `_default` +/// and auto-provisioning is enabled, the default store is created on demand. +pub async fn resolve_store( + store_ref: &StoreRef, + tenant_id: &str, + state: &AppState, +) -> Result { + match store_ref { + StoreRef::Id(id) => Ok(*id), + StoreRef::Default => { + if !state.config.auto_provision { + return Err(kd6_core::OmsError::InvalidInput( + "_default store alias is disabled; create a store explicitly".into(), + )); + } + let store = state + .provider + .get_or_create_store( + tenant_id, + DEFAULT_STORE_ALIAS, + kd6_core::models::CreateStoreRequest { + name: DEFAULT_STORE_ALIAS.to_string(), + region: None, + config: Default::default(), + metadata: Default::default(), + }, + ) + .await?; + Ok(store.id) + } + } +} diff --git a/crates/kd6-server/src/lib.rs b/crates/kd6-server/src/lib.rs index 449b503..d9f75e2 100644 --- a/crates/kd6-server/src/lib.rs +++ b/crates/kd6-server/src/lib.rs @@ -1,3 +1,4 @@ +pub mod embed; pub mod error; pub mod extract; pub mod routes; diff --git a/crates/kd6-server/src/main.rs b/crates/kd6-server/src/main.rs index 6ea23a3..d2a9dab 100644 --- a/crates/kd6-server/src/main.rs +++ b/crates/kd6-server/src/main.rs @@ -1,6 +1,7 @@ use std::sync::Arc; -use kd6_server::state::AppState; +use kd6_core::NoopEmbedder; +use kd6_server::state::{AppState, ServerConfig}; use tokio::net::TcpListener; use tracing_subscriber::EnvFilter; @@ -14,14 +15,73 @@ async fn main() -> anyhow::Result<()> { std::env::var("KD6_DATABASE_URL").unwrap_or_else(|_| "sqlite:kd6.db?mode=rwc".into()); let provider = kd6_sqlite::SqliteProvider::new(&db_url).await?; + let auto_provision = std::env::var("KD6_AUTO_PROVISION") + .map(|v| v != "false" && v != "0") + .unwrap_or(true); + let default_tenant = std::env::var("KD6_DEFAULT_TENANT") + .map(|v| v != "false" && v != "0") + .unwrap_or(true); + + // --- Embedding provider --- + let embedding_provider = + std::env::var("KD6_EMBEDDING_PROVIDER").unwrap_or_else(|_| "local".into()); + + let embedder: Arc = match embedding_provider.as_str() { + "local" => { + tracing::info!("embedding provider: local (fastembed, in-process ONNX)"); + Arc::new(kd6_embed::LocalEmbedder::new()?) + } + "openai-compatible" => { + let endpoint = std::env::var("KD6_EMBEDDING_ENDPOINT").expect( + "KD6_EMBEDDING_ENDPOINT is required when KD6_EMBEDDING_PROVIDER=openai-compatible", + ); + let model = std::env::var("KD6_EMBEDDING_MODEL").expect( + "KD6_EMBEDDING_MODEL is required when KD6_EMBEDDING_PROVIDER=openai-compatible", + ); + let api_key = std::env::var("KD6_EMBEDDING_API_KEY").ok(); + let dimensions: usize = std::env::var("KD6_EMBEDDING_DIMENSIONS") + .unwrap_or_else(|_| "1536".into()) + .parse() + .expect("KD6_EMBEDDING_DIMENSIONS must be a positive integer"); + tracing::info!( + "embedding provider: openai-compatible (endpoint={endpoint}, model={model})" + ); + Arc::new( + kd6_embed::OpenAiCompatibleEmbedder::new(endpoint, model, api_key, dimensions) + .expect("failed to create OpenAI-compatible embedding provider"), + ) + } + "none" => { + tracing::info!( + "embedding provider: none (pass-through, callers must supply embeddings)" + ); + Arc::new(NoopEmbedder) + } + other => { + anyhow::bail!("unknown KD6_EMBEDDING_PROVIDER: {other}. Valid options: local, openai-compatible, none"); + } + }; + + tracing::info!( + model_id = embedder.model_id(), + dimensions = embedder.dimensions(), + "embedding provider ready" + ); + let state = AppState { provider: Arc::new(provider), + embedder, + config: ServerConfig { + auto_provision, + default_tenant, + }, }; let app = kd6_server::build_router(state); let addr = std::env::var("LISTEN_ADDR").unwrap_or_else(|_| "0.0.0.0:8080".into()); tracing::info!("listening on {addr}"); + tracing::info!("auto_provision={auto_provision}, default_tenant={default_tenant}"); let listener = TcpListener::bind(&addr).await?; axum::serve(listener, app).await?; diff --git a/crates/kd6-server/src/routes/audit.rs b/crates/kd6-server/src/routes/audit.rs index 76bb5d9..12a6231 100644 --- a/crates/kd6-server/src/routes/audit.rs +++ b/crates/kd6-server/src/routes/audit.rs @@ -5,15 +5,16 @@ use uuid::Uuid; use kd6_core::models::{AuditEntry, AuditFilter, Page}; use crate::error::ApiError; -use crate::extract::{PathId, TenantId}; +use crate::extract::{resolve_store, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn store_audit_log( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, Query(filter): Query, ) -> Result>, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let page = state.provider.audit_log(&tenant, store_id, filter).await?; Ok(Json(page)) } @@ -21,8 +22,9 @@ pub async fn store_audit_log( pub async fn memory_audit_log( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, memory_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, memory_id)): PathId<(StoreRef, Uuid)>, ) -> Result>, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let filter = AuditFilter { memory_id: Some(memory_id), ..Default::default() diff --git a/crates/kd6-server/src/routes/batch.rs b/crates/kd6-server/src/routes/batch.rs index 40f03e8..8378134 100644 --- a/crates/kd6-server/src/routes/batch.rs +++ b/crates/kd6-server/src/routes/batch.rs @@ -1,22 +1,30 @@ use axum::extract::State; use axum::http::StatusCode; use axum::Json; -use uuid::Uuid; use kd6_core::models::{ BatchCreateRequest, BatchCreateResponse, BatchDeleteRequest, BatchDeleteResponse, }; +use crate::embed::auto_embed_content; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn batch_create( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, - JsonBody(request): JsonBody, + PathId(store_ref): PathId, + JsonBody(mut request): JsonBody, ) -> Result<(StatusCode, Json), ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; + + // Auto-embed each entry in the batch (OMS spec section 8.4.1) + for entry in &mut request.entries { + entry.embedding = + auto_embed_content(&*state.embedder, &entry.content, entry.embedding.take()).await?; + } + let response = state .provider .batch_create_memories(&tenant, store_id, request) @@ -27,9 +35,10 @@ pub async fn batch_create( pub async fn batch_delete( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let response = state .provider .batch_delete_memories(&tenant, store_id, request) diff --git a/crates/kd6-server/src/routes/gdpr.rs b/crates/kd6-server/src/routes/gdpr.rs index 2f9cf1c..827e830 100644 --- a/crates/kd6-server/src/routes/gdpr.rs +++ b/crates/kd6-server/src/routes/gdpr.rs @@ -1,12 +1,11 @@ use axum::extract::State; use axum::Json; use serde::Serialize; -use uuid::Uuid; use kd6_core::models::MemoryScope; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; #[derive(Serialize)] @@ -17,9 +16,10 @@ pub struct GdprPurgeResponse { pub async fn gdpr_purge( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(scope): JsonBody, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let deleted = state.provider.gdpr_purge(&tenant, store_id, scope).await?; Ok(Json(GdprPurgeResponse { deleted })) } diff --git a/crates/kd6-server/src/routes/graph.rs b/crates/kd6-server/src/routes/graph.rs index 282c77f..f547818 100644 --- a/crates/kd6-server/src/routes/graph.rs +++ b/crates/kd6-server/src/routes/graph.rs @@ -3,20 +3,19 @@ use axum::http::StatusCode; use axum::Json; use uuid::Uuid; -use kd6_core::models::{ - CreateEdgeRequest, GraphEdge, GraphTraversalRequest, GraphTraversalResult, -}; +use kd6_core::models::{CreateEdgeRequest, GraphEdge, GraphTraversalRequest, GraphTraversalResult}; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn create_edge( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result<(StatusCode, Json), ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let edge = state .provider .create_edge(&tenant, store_id, request) @@ -27,8 +26,9 @@ pub async fn create_edge( pub async fn delete_edge( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, edge_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, edge_id)): PathId<(StoreRef, Uuid)>, ) -> Result { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; state .provider .delete_edge(&tenant, store_id, edge_id) @@ -39,9 +39,10 @@ pub async fn delete_edge( pub async fn traverse( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let result = state .provider .graph_traverse(&tenant, store_id, request) diff --git a/crates/kd6-server/src/routes/inheritance.rs b/crates/kd6-server/src/routes/inheritance.rs index 3c4cbff..182be20 100644 --- a/crates/kd6-server/src/routes/inheritance.rs +++ b/crates/kd6-server/src/routes/inheritance.rs @@ -6,15 +6,16 @@ use uuid::Uuid; use kd6_core::models::{BubbleUpRequest, CreateInheritanceRequest, InheritanceSpec, MemoryEntry}; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn create_inheritance( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result<(StatusCode, Json), ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let spec = state .provider .create_inheritance(&tenant, store_id, request) @@ -25,8 +26,9 @@ pub async fn create_inheritance( pub async fn delete_inheritance( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, inheritance_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, inheritance_id)): PathId<(StoreRef, Uuid)>, ) -> Result { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; state .provider .delete_inheritance(&tenant, store_id, inheritance_id) @@ -37,9 +39,10 @@ pub async fn delete_inheritance( pub async fn bubble_up( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result>, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let entries = state.provider.bubble_up(&tenant, store_id, request).await?; Ok(Json(entries)) } diff --git a/crates/kd6-server/src/routes/lifecycle.rs b/crates/kd6-server/src/routes/lifecycle.rs index 542705d..f011fba 100644 --- a/crates/kd6-server/src/routes/lifecycle.rs +++ b/crates/kd6-server/src/routes/lifecycle.rs @@ -1,10 +1,9 @@ use axum::extract::State; use axum::Json; use serde::Serialize; -use uuid::Uuid; use crate::error::ApiError; -use crate::extract::{PathId, TenantId}; +use crate::extract::{resolve_store, PathId, StoreRef, TenantId}; use crate::state::AppState; #[derive(Serialize)] @@ -15,8 +14,9 @@ pub struct PurgeResponse { pub async fn purge_expired( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let deleted = state.provider.purge_expired(&tenant, store_id).await?; Ok(Json(PurgeResponse { deleted })) } @@ -24,8 +24,9 @@ pub async fn purge_expired( pub async fn lifecycle_stats( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let stats = state.provider.stats(&tenant, store_id).await?; Ok(Json(stats)) } diff --git a/crates/kd6-server/src/routes/memories.rs b/crates/kd6-server/src/routes/memories.rs index e8b9f8f..950a3f5 100644 --- a/crates/kd6-server/src/routes/memories.rs +++ b/crates/kd6-server/src/routes/memories.rs @@ -7,16 +7,23 @@ use kd6_core::models::{ CreateMemoryRequest, ListMemoriesFilter, MemoryEntry, Page, UpdateMemoryRequest, }; +use crate::embed::{auto_embed_content, auto_embed_update}; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn create_memory( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, - JsonBody(request): JsonBody, + PathId(store_ref): PathId, + JsonBody(mut request): JsonBody, ) -> Result<(StatusCode, Json), ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; + + // Auto-embed if no embedding provided (OMS spec section 8.4.1) + request.embedding = + auto_embed_content(&*state.embedder, &request.content, request.embedding).await?; + let entry = state .provider .create_memory(&tenant, store_id, request) @@ -27,8 +34,9 @@ pub async fn create_memory( pub async fn get_memory( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, memory_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, memory_id)): PathId<(StoreRef, Uuid)>, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let entry = state .provider .get_memory(&tenant, store_id, memory_id) @@ -39,9 +47,10 @@ pub async fn get_memory( pub async fn list_memories( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, Query(filter): Query, ) -> Result>, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let page = state .provider .list_memories(&tenant, store_id, filter) @@ -52,9 +61,19 @@ pub async fn list_memories( pub async fn update_memory( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, memory_id)): PathId<(Uuid, Uuid)>, - JsonBody(request): JsonBody, + PathId((store_ref, memory_id)): PathId<(StoreRef, Uuid)>, + JsonBody(mut request): JsonBody, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; + + // Auto-embed if content changed but no new embedding provided (OMS spec section 8.4.2) + request.embedding = auto_embed_update( + &*state.embedder, + request.content.as_ref(), + request.embedding, + ) + .await?; + let entry = state .provider .update_memory(&tenant, store_id, memory_id, request) @@ -65,8 +84,9 @@ pub async fn update_memory( pub async fn delete_memory( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, memory_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, memory_id)): PathId<(StoreRef, Uuid)>, ) -> Result { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; state .provider .delete_memory(&tenant, store_id, memory_id) diff --git a/crates/kd6-server/src/routes/search.rs b/crates/kd6-server/src/routes/search.rs index 275ec38..cffeeda 100644 --- a/crates/kd6-server/src/routes/search.rs +++ b/crates/kd6-server/src/routes/search.rs @@ -1,19 +1,24 @@ use axum::extract::State; use axum::Json; -use uuid::Uuid; use kd6_core::models::{SearchQuery, SearchResult}; +use crate::embed::auto_embed_query; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn search( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, - JsonBody(query): JsonBody, + PathId(store_ref): PathId, + JsonBody(mut query): JsonBody, ) -> Result>, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; + + // Auto-embed query if no embedding provided (OMS spec section 8.4.3) + query.embedding = auto_embed_query(&*state.embedder, &query.query, query.embedding).await?; + let results = state.provider.search(&tenant, store_id, query).await?; Ok(Json(results)) } diff --git a/crates/kd6-server/src/routes/shared_spaces.rs b/crates/kd6-server/src/routes/shared_spaces.rs index eaaa4ad..876aeef 100644 --- a/crates/kd6-server/src/routes/shared_spaces.rs +++ b/crates/kd6-server/src/routes/shared_spaces.rs @@ -8,15 +8,16 @@ use kd6_core::models::{ }; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn create_shared_space( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result<(StatusCode, Json), ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let space = state .provider .create_shared_space(&tenant, store_id, request) @@ -27,8 +28,9 @@ pub async fn create_shared_space( pub async fn list_shared_spaces( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, ) -> Result>, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let spaces = state.provider.list_shared_spaces(&tenant, store_id).await?; Ok(Json(spaces)) } @@ -36,8 +38,9 @@ pub async fn list_shared_spaces( pub async fn get_shared_space( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, space_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, space_id)): PathId<(StoreRef, Uuid)>, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let space = state .provider .get_shared_space(&tenant, store_id, space_id) @@ -48,9 +51,10 @@ pub async fn get_shared_space( pub async fn join_shared_space( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, space_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, space_id)): PathId<(StoreRef, Uuid)>, JsonBody(request): JsonBody, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let space = state .provider .join_shared_space(&tenant, store_id, space_id, request) @@ -61,9 +65,10 @@ pub async fn join_shared_space( pub async fn leave_shared_space( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, space_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, space_id)): PathId<(StoreRef, Uuid)>, JsonBody(request): JsonBody, ) -> Result { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; state .provider .leave_shared_space(&tenant, store_id, space_id, request) @@ -74,8 +79,9 @@ pub async fn leave_shared_space( pub async fn delete_shared_space( State(state): State, TenantId(tenant): TenantId, - PathId((store_id, space_id)): PathId<(Uuid, Uuid)>, + PathId((store_ref, space_id)): PathId<(StoreRef, Uuid)>, ) -> Result { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; state .provider .delete_shared_space(&tenant, store_id, space_id) diff --git a/crates/kd6-server/src/routes/stores.rs b/crates/kd6-server/src/routes/stores.rs index dde1f9a..a017d1b 100644 --- a/crates/kd6-server/src/routes/stores.rs +++ b/crates/kd6-server/src/routes/stores.rs @@ -1,12 +1,11 @@ use axum::extract::State; use axum::http::StatusCode; use axum::Json; -use uuid::Uuid; use kd6_core::models::{CreateStoreRequest, MemoryStore, UpdateStoreRequest}; use crate::error::ApiError; -use crate::extract::{JsonBody, PathId, TenantId}; +use crate::extract::{resolve_store, JsonBody, PathId, StoreRef, TenantId}; use crate::state::AppState; pub async fn create_store( @@ -21,8 +20,9 @@ pub async fn create_store( pub async fn get_store( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let store = state.provider.get_store(&tenant, store_id).await?; Ok(Json(store)) } @@ -38,9 +38,10 @@ pub async fn list_stores( pub async fn update_store( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, JsonBody(request): JsonBody, ) -> Result, ApiError> { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; let store = state .provider .update_store(&tenant, store_id, request) @@ -51,8 +52,9 @@ pub async fn update_store( pub async fn delete_store( State(state): State, TenantId(tenant): TenantId, - PathId(store_id): PathId, + PathId(store_ref): PathId, ) -> Result { + let store_id = resolve_store(&store_ref, &tenant, &state).await?; state.provider.delete_store(&tenant, store_id).await?; Ok(StatusCode::NO_CONTENT) } diff --git a/crates/kd6-server/src/state.rs b/crates/kd6-server/src/state.rs index e8d7be0..63df502 100644 --- a/crates/kd6-server/src/state.rs +++ b/crates/kd6-server/src/state.rs @@ -1,8 +1,29 @@ use std::sync::Arc; +use kd6_core::embedding::EmbeddingProvider; use kd6_core::OmsProvider; +/// Configuration for optional spec features (OMS spec 4.1.1, 4.4.1). +#[derive(Debug, Clone)] +pub struct ServerConfig { + /// Enable `_default` store alias and auto-provisioning of stores on first write. + pub auto_provision: bool, + /// Enable default tenant resolution when `X-Tenant-ID` header is absent. + pub default_tenant: bool, +} + +impl Default for ServerConfig { + fn default() -> Self { + Self { + auto_provision: true, + default_tenant: true, + } + } +} + #[derive(Clone)] pub struct AppState { pub provider: Arc, + pub embedder: Arc, + pub config: ServerConfig, } diff --git a/crates/kd6-server/tests/integration.rs b/crates/kd6-server/tests/integration.rs index fcf02de..5fbf349 100644 --- a/crates/kd6-server/tests/integration.rs +++ b/crates/kd6-server/tests/integration.rs @@ -4,13 +4,32 @@ use axum::http::{HeaderName, HeaderValue, StatusCode}; use axum_test::TestServer; use serde_json::{json, Value}; -use kd6_server::state::AppState; +use kd6_core::NoopEmbedder; +use kd6_server::state::{AppState, ServerConfig}; use kd6_sqlite::SqliteProvider; async fn test_app() -> TestServer { let provider = SqliteProvider::new("sqlite::memory:").await.unwrap(); let state = AppState { provider: Arc::new(provider), + embedder: Arc::new(NoopEmbedder), + config: ServerConfig::default(), + }; + + let app = kd6_server::build_router(state); + TestServer::new(app).unwrap() +} + +/// Test app with default tenant and auto-provisioning disabled (strict mode). +async fn test_app_strict() -> TestServer { + let provider = SqliteProvider::new("sqlite::memory:").await.unwrap(); + let state = AppState { + provider: Arc::new(provider), + embedder: Arc::new(NoopEmbedder), + config: ServerConfig { + auto_provision: false, + default_tenant: false, + }, }; let app = kd6_server::build_router(state); @@ -37,7 +56,7 @@ async fn test_health_endpoint() { #[tokio::test] async fn test_missing_tenant_header_returns_401_json() { - let server = test_app().await; + let server = test_app_strict().await; let response = server.get("/v1/stores").await; response.assert_status(StatusCode::UNAUTHORIZED); @@ -47,7 +66,7 @@ async fn test_missing_tenant_header_returns_401_json() { #[tokio::test] async fn test_empty_tenant_header_returns_401() { - let server = test_app().await; + let server = test_app_strict().await; let response = tenant_header(server.get("/v1/stores"), "").await; response.assert_status(StatusCode::UNAUTHORIZED); @@ -57,7 +76,7 @@ async fn test_empty_tenant_header_returns_401() { #[tokio::test] async fn test_whitespace_tenant_header_returns_401() { - let server = test_app().await; + let server = test_app_strict().await; let response = tenant_header(server.get("/v1/stores"), " ").await; response.assert_status(StatusCode::UNAUTHORIZED); @@ -834,3 +853,1042 @@ async fn test_malformed_json_body_returns_json_error() { let body: Value = response.json(); assert!(body["error"].is_string(), "expected JSON error response"); } + +// --- Tests for OMS spec 4.1.1, 4.3.2, 4.4.1 --- + +#[tokio::test] +async fn test_default_tenant_creates_store_without_header() { + let server = test_app().await; + + let response = server + .post("/v1/stores") + .json(&json!({ "name": "auto-store" })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + assert_eq!(body["name"], "auto-store"); + assert_eq!(body["tenant_id"], "_default"); +} + +#[tokio::test] +async fn test_default_store_alias_auto_creates_store() { + let server = test_app().await; + + let response = server + .post("/v1/stores/_default/memories") + .json(&json!({ + "content": "test memory via default store", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": "_default" } + })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + assert_eq!(body["content"], "test memory via default store"); + assert_eq!(body["owner_agent_id"], "agent-1"); +} + +#[tokio::test] +async fn test_default_store_reuses_existing_default() { + let server = test_app().await; + + // First write auto-creates the default store + let first = server + .post("/v1/stores/_default/memories") + .json(&json!({ + "content": "first", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": "_default" } + })) + .await; + first.assert_status(StatusCode::CREATED); + let first_store_id = first.json::()["store_id"].clone(); + + // Second write reuses the same default store + let second = server + .post("/v1/stores/_default/memories") + .json(&json!({ + "content": "second", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": "_default" } + })) + .await; + second.assert_status(StatusCode::CREATED); + let second_store_id = second.json::()["store_id"].clone(); + + assert_eq!(first_store_id, second_store_id); +} + +#[tokio::test] +async fn test_default_store_disabled_returns_error() { + let server = test_app_strict().await; + let tenant = "test-tenant"; + + let response = tenant_header(server.post("/v1/stores/_default/memories"), tenant) + .json(&json!({ + "content": "should fail", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant } + })) + .await; + + response.assert_status(StatusCode::BAD_REQUEST); + let body: Value = response.json(); + assert!(body["error"].as_str().unwrap().contains("_default")); +} + +#[tokio::test] +async fn test_upsert_creates_new_entry() { + let server = test_app().await; + let tenant = "test-tenant"; + + // Create a store + let store_resp = tenant_header(server.post("/v1/stores"), tenant) + .json(&json!({ "name": "upsert-store" })) + .await; + let store_id = store_resp.json::()["id"] + .as_str() + .unwrap() + .to_string(); + + // Create memory with upsert_key + let response = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": "dark mode", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant }, + "upsert_key": "preference:theme" + })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + assert_eq!(body["content"], "dark mode"); + assert_eq!(body["version"], 1); + assert_eq!(body["upsert_key"], "preference:theme"); +} + +#[tokio::test] +async fn test_upsert_replaces_existing_entry() { + let server = test_app().await; + let tenant = "test-tenant"; + + let store_resp = tenant_header(server.post("/v1/stores"), tenant) + .json(&json!({ "name": "upsert-store" })) + .await; + let store_id = store_resp.json::()["id"] + .as_str() + .unwrap() + .to_string(); + + // First upsert + let first = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": "dark mode", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant }, + "upsert_key": "preference:theme" + })) + .await; + first.assert_status(StatusCode::CREATED); + let first_id = first.json::()["id"].as_str().unwrap().to_string(); + + // Second upsert with same key -- should replace + let second = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": "light mode", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant }, + "upsert_key": "preference:theme" + })) + .await; + second.assert_status(StatusCode::CREATED); + let second_body: Value = second.json(); + + // Same ID, incremented version, new content + assert_eq!(second_body["id"], first_id); + assert_eq!(second_body["content"], "light mode"); + assert_eq!(second_body["version"], 2); +} + +#[tokio::test] +async fn test_upsert_different_keys_create_separate_entries() { + let server = test_app().await; + let tenant = "test-tenant"; + + let store_resp = tenant_header(server.post("/v1/stores"), tenant) + .json(&json!({ "name": "upsert-store" })) + .await; + let store_id = store_resp.json::()["id"] + .as_str() + .unwrap() + .to_string(); + + let first = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": "dark mode", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant }, + "upsert_key": "preference:theme" + })) + .await; + + let second = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": "english", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant }, + "upsert_key": "preference:language" + })) + .await; + + let first_id = first.json::()["id"].as_str().unwrap().to_string(); + let second_id = second.json::()["id"].as_str().unwrap().to_string(); + + assert_ne!(first_id, second_id); +} + +#[tokio::test] +async fn test_plain_string_content_round_trips() { + let server = test_app().await; + let tenant = "test-tenant"; + + let store_resp = tenant_header(server.post("/v1/stores"), tenant) + .json(&json!({ "name": "content-test" })) + .await; + let store_id = store_resp.json::()["id"] + .as_str() + .unwrap() + .to_string(); + + // Create with plain string content + let created = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": "just a plain string", + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant } + })) + .await; + created.assert_status(StatusCode::CREATED); + let memory_id = created.json::()["id"].as_str().unwrap().to_string(); + + // Read it back + let fetched = tenant_header( + server.get(&format!("/v1/stores/{store_id}/memories/{memory_id}")), + tenant, + ) + .await; + let body: Value = fetched.json(); + + // Content should be a plain string, not wrapped in {"text": "..."} + assert_eq!(body["content"], "just a plain string"); +} + +#[tokio::test] +async fn test_structured_content_round_trips() { + let server = test_app().await; + let tenant = "test-tenant"; + + let store_resp = tenant_header(server.post("/v1/stores"), tenant) + .json(&json!({ "name": "content-test" })) + .await; + let store_id = store_resp.json::()["id"] + .as_str() + .unwrap() + .to_string(); + + let structured = json!({"key": "value", "nested": {"a": 1}}); + let created = tenant_header( + server.post(&format!("/v1/stores/{store_id}/memories")), + tenant, + ) + .json(&json!({ + "content": structured, + "owner_agent_id": "agent-1", + "scope": { "tenant_id": tenant } + })) + .await; + created.assert_status(StatusCode::CREATED); + let memory_id = created.json::()["id"].as_str().unwrap().to_string(); + + let fetched = tenant_header( + server.get(&format!("/v1/stores/{store_id}/memories/{memory_id}")), + tenant, + ) + .await; + let body: Value = fetched.json(); + assert_eq!(body["content"], structured); +} + +#[tokio::test] +async fn test_zero_setup_write_with_defaults() { + let server = test_app().await; + + // No tenant header, no store creation -- just write a memory + let response = server + .post("/v1/stores/_default/memories") + .json(&json!({ + "content": "zero setup memory", + "owner_agent_id": "agent-1", + "scope": {} + })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + assert_eq!(body["content"], "zero setup memory"); + // Scope should have been normalized to the default tenant + assert_eq!(body["scope"]["tenant_id"], "_default"); +} + +// ── Embedding integration tests ───────────────────────────────────────────── + +/// Deterministic fake embedder for tests (avoids slow model download). +struct FakeEmbedder; + +#[async_trait::async_trait] +impl kd6_core::EmbeddingProvider for FakeEmbedder { + async fn embed_texts(&self, texts: &[String]) -> Result>, kd6_core::OmsError> { + Ok(texts + .iter() + .map(|t| { + let len = t.len() as f32; + vec![len, len * 0.5, 1.0] + }) + .collect()) + } + async fn embed_query(&self, query: &str) -> Result, kd6_core::OmsError> { + let len = query.len() as f32; + Ok(vec![len, len * 0.5, 1.0]) + } + fn dimensions(&self) -> usize { + 3 + } + fn model_id(&self) -> &str { + "fake-3d" + } +} + +/// Test app with deterministic fake embedder for embedding tests. +async fn test_app_with_embedder() -> TestServer { + let provider = SqliteProvider::new("sqlite::memory:").await.unwrap(); + let state = AppState { + provider: Arc::new(provider), + embedder: Arc::new(FakeEmbedder), + config: ServerConfig::default(), + }; + + let app = kd6_server::build_router(state); + TestServer::new(app).unwrap() +} + +#[tokio::test] +async fn test_auto_embed_on_write() { + let server = test_app_with_embedder().await; + + // Create a store + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-test"), + ) + .json(&json!({ + "name": "embed-store", + "region": "local" + })) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Create memory WITHOUT providing embedding + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-test"), + ) + .json(&json!({ + "content": "The user prefers dark mode in all applications", + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + + // Embedding should have been auto-computed + let embedding = body["embedding"] + .as_array() + .expect("embedding should be present"); + assert!(!embedding.is_empty(), "embedding should not be empty"); + assert_eq!(embedding.len(), 3, "FakeEmbedder produces 3-dim vectors"); +} + +#[tokio::test] +async fn test_auto_embed_on_search() { + let server = test_app_with_embedder().await; + + // Create store + memory (embedding auto-computed on write) + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-search"), + ) + .json(&json!({ + "name": "search-store", + "region": "local" + })) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Create a few memories + for content in [ + "The team decided to use PostgreSQL for the database", + "User interface should follow Material Design guidelines", + "Authentication will use OAuth 2.0 with JWT tokens", + ] { + server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-search"), + ) + .json(&json!({ + "content": content, + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + } + + // Search WITHOUT providing embedding — should auto-embed the query + let response = server + .post(&format!("/v1/stores/{store_id}/search")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-search"), + ) + .json(&json!({ + "query": "database choice", + "top_k": 3, + "threshold": 0.0 + })) + .await; + + response.assert_status_ok(); + let results: Vec = response.json(); + + // Should find results via vector similarity + assert!(!results.is_empty(), "vector search should return results"); +} + +#[tokio::test] +async fn test_auto_embed_preserves_caller_embedding() { + let server = test_app_with_embedder().await; + + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-preserve"), + ) + .json(&json!({ + "name": "preserve-store", + "region": "local" + })) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Provide a custom 3-dim embedding (matches FakeEmbedder dimensions) + let custom_embedding: Vec = vec![42.0, 21.0, 1.0]; + + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-preserve"), + ) + .json(&json!({ + "content": "test content", + "owner_agent_id": "test-agent", + "scope": {}, + "embedding": custom_embedding + })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + + // Should use the caller-provided embedding, not auto-compute + let stored: Vec = body["embedding"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect(); + assert_eq!(stored.len(), 3); + // Check first value matches what we sent + assert!((stored[0] - 42.0).abs() < 0.001); + assert!((stored[1] - 21.0).abs() < 0.001); +} + +#[tokio::test] +async fn test_auto_embed_rejects_wrong_dimensions() { + let server = test_app_with_embedder().await; + + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-dim"), + ) + .json(&json!({ + "name": "dim-store", + "region": "local" + })) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Send a 100-dim embedding when model expects 3 + let wrong_embedding: Vec = vec![0.1; 100]; + + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-dim"), + ) + .json(&json!({ + "content": "test content", + "owner_agent_id": "test-agent", + "scope": {}, + "embedding": wrong_embedding + })) + .await; + + // Should be rejected due to dimensionality mismatch + response.assert_status(StatusCode::BAD_REQUEST); +} + +#[tokio::test] +async fn test_auto_embed_on_update() { + let server = test_app_with_embedder().await; + + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-update"), + ) + .json(&json!({ + "name": "update-store", + "region": "local" + })) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Create memory + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-update"), + ) + .json(&json!({ + "content": "cats are great pets", + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + let memory_id = response.json::()["id"].as_str().unwrap().to_string(); + let original_embedding: Vec = response.json::()["embedding"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect(); + + // Update content — embedding should be recomputed + let response = server + .patch(&format!("/v1/stores/{store_id}/memories/{memory_id}")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("embed-update"), + ) + .json(&json!({ + "content": "quantum computing breakthroughs in 2026" + })) + .await; + + response.assert_status_ok(); + let updated_embedding: Vec = response.json::()["embedding"] + .as_array() + .unwrap() + .iter() + .map(|v| v.as_f64().unwrap()) + .collect(); + + assert_eq!(updated_embedding.len(), 3); + // Embedding should be different because content changed + assert_ne!( + original_embedding, updated_embedding, + "embedding should change when content changes" + ); +} + +#[tokio::test] +async fn test_noop_embedder_passthrough() { + // Using test_app() which has NoopEmbedder + let server = test_app().await; + + let response = server + .post("/v1/stores/_default/memories") + .json(&json!({ + "content": "no embedding provider configured", + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + // With NoopEmbedder, no embedding should be computed + assert!( + body["embedding"].is_null(), + "noop embedder should not produce embeddings" + ); +} + +// --------------------------------------------------------------------------- +// Vector search via HTTP +// --------------------------------------------------------------------------- + +#[tokio::test] +async fn test_vector_search_returns_ranked_results() { + let server = test_app_with_embedder().await; + + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("vsearch-test"), + ) + .json(&json!({"name": "vsearch-store", "region": "local"})) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Add semantically diverse documents + for text in [ + "The Eiffel Tower is a landmark in Paris, France", + "Machine learning models require training data", + "The Louvre Museum contains the Mona Lisa", + "Rust is a systems programming language", + ] { + server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("vsearch-test"), + ) + .json(&json!({ + "content": text, + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + } + + // Vector search (no keyword=true, so pure vector similarity) + let response = server + .post(&format!("/v1/stores/{store_id}/search")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("vsearch-test"), + ) + .json(&json!({"query": "Paris landmarks and attractions", "top_k": 4})) + .await; + response.assert_status_ok(); + let results: Vec = response.json(); + + assert!(!results.is_empty(), "vector search should return results"); + // All results should have scores + for r in &results { + assert!(r["score"].as_f64().is_some(), "result should have a score"); + } +} + +#[tokio::test] +async fn test_structured_content_with_arrays_embeds_correctly() { + let server = test_app_with_embedder().await; + + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("array-test"), + ) + .json(&json!({"name": "array-store", "region": "local"})) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Create memory with array content containing string values + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("array-test"), + ) + .json(&json!({ + "content": { + "title": "Array Test", + "items": ["artificial intelligence", "machine learning", "deep learning"], + "nested": {"tags": ["neural networks"]} + }, + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + response.assert_status(StatusCode::CREATED); + let body: Value = response.json(); + // Should have an embedding (arrays were traversed for text) + assert!( + body["embedding"].is_array(), + "structured content with arrays should produce embedding" + ); +} + +#[tokio::test] +async fn test_update_memory_explicit_clear_embedding() { + let server = test_app_with_embedder().await; + + let response = server + .post("/v1/stores") + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("clear-test"), + ) + .json(&json!({"name": "clear-store", "region": "local"})) + .await; + let store_id = response.json::()["id"].as_str().unwrap().to_string(); + + // Create memory (auto-embedded) + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("clear-test"), + ) + .json(&json!({ + "content": "this will be embedded", + "owner_agent_id": "test-agent", + "scope": {} + })) + .await; + response.assert_status(StatusCode::CREATED); + let created: Value = response.json(); + let memory_id = created["id"].as_str().unwrap(); + assert!(created["embedding"].is_array(), "should have auto-embedded"); + + // Update with content change but no explicit embedding — should auto-recompute + let response = server + .patch(&format!("/v1/stores/{store_id}/memories/{memory_id}")) + .add_header( + HeaderName::from_static("x-tenant-id"), + HeaderValue::from_static("clear-test"), + ) + .json(&json!({ + "content": "updated content for re-embedding", + "version": 1 + })) + .await; + response.assert_status_ok(); + let updated: Value = response.json(); + assert!( + updated["embedding"].is_array(), + "should have re-computed embedding on content change" + ); +} + +#[tokio::test] +async fn test_inheritance_create_and_delete() { + let server = test_app().await; + + let response = server + .post("/v1/stores") + .json(&json!({ "name": "inheritance-store" })) + .await; + response.assert_status(StatusCode::CREATED); + let store: Value = response.json(); + let store_id = store["id"].as_str().unwrap(); + + let response = server + .post(&format!("/v1/stores/{store_id}/inherit")) + .json(&json!({ + "parent_agent_id": "parent-agent", + "child_agent_id": "child-agent", + "inherit_layers": ["working", "episodic"], + "filter": {}, + "bubble_up": {"enabled": true, "layers": ["working"]}, + "access": "read_only" + })) + .await; + response.assert_status(StatusCode::CREATED); + let inheritance: Value = response.json(); + let inheritance_id = inheritance["id"].as_str().unwrap(); + + assert_eq!(inheritance["store_id"], store_id); + assert_eq!(inheritance["tenant_id"], "_default"); + assert_eq!(inheritance["parent_agent_id"], "parent-agent"); + assert_eq!(inheritance["child_agent_id"], "child-agent"); + assert_eq!( + inheritance["inherit_layers"], + json!(["working", "episodic"]) + ); + assert_eq!(inheritance["bubble_up"]["enabled"], true); + assert_eq!(inheritance["bubble_up"]["layers"], json!(["working"])); + assert_eq!(inheritance["access"], "read_only"); + + let response = server + .delete(&format!("/v1/stores/{store_id}/inherit/{inheritance_id}")) + .await; + response.assert_status(StatusCode::NO_CONTENT); +} + +#[tokio::test] +async fn test_bubble_up_creates_parent_memories() { + let server = test_app().await; + + let response = server + .post("/v1/stores") + .json(&json!({ "name": "bubble-up-store" })) + .await; + response.assert_status(StatusCode::CREATED); + let store: Value = response.json(); + let store_id = store["id"].as_str().unwrap(); + + let response = server + .post(&format!("/v1/stores/{store_id}/inherit")) + .json(&json!({ + "parent_agent_id": "parent-agent", + "child_agent_id": "child-agent", + "inherit_layers": ["working", "episodic"], + "filter": {}, + "bubble_up": {"enabled": true, "layers": ["working"]}, + "access": "read_only" + })) + .await; + response.assert_status(StatusCode::CREATED); + + for content in ["child memory one", "child memory two"] { + let response = server + .post(&format!("/v1/stores/{store_id}/memories")) + .json(&json!({ + "layer": "working", + "content": { "text": content }, + "owner_agent_id": "child-agent", + "scope": { "agent_id": "child-agent" } + })) + .await; + response.assert_status(StatusCode::CREATED); + } + + let response = server + .post(&format!("/v1/stores/{store_id}/bubble-up")) + .json(&json!({ + "parent_agent_id": "parent-agent", + "child_agent_id": "child-agent", + "layers": ["working"] + })) + .await; + response.assert_status_ok(); + let bubbled: Value = response.json(); + let items = bubbled.as_array().unwrap(); + assert_eq!(items.len(), 2); + assert!(items + .iter() + .all(|item| item["owner_agent_id"] == "parent-agent")); + assert!(items + .iter() + .all(|item| item["scope"]["agent_id"] == "parent-agent")); + assert!(items.iter().all(|item| item["source"]["uri"] + .as_str() + .unwrap() + .starts_with("bubble_up:"))); + + let response = server + .get(&format!( + "/v1/stores/{store_id}/memories?owner_agent_id=parent-agent" + )) + .await; + response.assert_status_ok(); + let memories: Value = response.json(); + let parent_items = memories["items"].as_array().unwrap(); + assert_eq!(memories["total"], 2); + assert_eq!(parent_items.len(), 2); + assert!(parent_items + .iter() + .all(|item| item["content"]["text"] == "child memory one" + || item["content"]["text"] == "child memory two")); +} + +#[tokio::test] +async fn test_shared_space_lifecycle() { + let server = test_app().await; + + let response = server + .post("/v1/stores") + .json(&json!({ "name": "shared-space-store" })) + .await; + response.assert_status(StatusCode::CREATED); + let store: Value = response.json(); + let store_id = store["id"].as_str().unwrap(); + + let response = server + .post(&format!("/v1/stores/{store_id}/shared-spaces")) + .json(&json!({ + "name": "test-space", + "description": "A test shared space", + "allowed_layers": ["working", "semantic"], + "creator_agent_id": "agent-1", + "scope": {}, + "layer": "working" + })) + .await; + response.assert_status(StatusCode::CREATED); + let space: Value = response.json(); + let space_id = space["id"].as_str().unwrap(); + + assert_eq!(space["name"], "test-space"); + assert_eq!(space["tenant_id"], "_default"); + assert_eq!(space["layer"], "working"); + assert_eq!(space["scope"]["tenant_id"], "_default"); + assert!(space["participants"].as_array().unwrap().is_empty()); + + let response = server + .get(&format!("/v1/stores/{store_id}/shared-spaces")) + .await; + response.assert_status_ok(); + let spaces: Value = response.json(); + let listed_spaces = spaces.as_array().unwrap(); + assert_eq!(listed_spaces.len(), 1); + assert_eq!(listed_spaces[0]["id"], space_id); + + let response = server + .get(&format!("/v1/stores/{store_id}/shared-spaces/{space_id}")) + .await; + response.assert_status_ok(); + let fetched: Value = response.json(); + assert_eq!(fetched["id"], space_id); + assert_eq!(fetched["name"], "test-space"); + assert_eq!(fetched["layer"], "working"); + + let response = server + .post(&format!( + "/v1/stores/{store_id}/shared-spaces/{space_id}/join" + )) + .json(&json!({ + "agent_id": "agent-2", + "access": "read_write" + })) + .await; + response.assert_status_ok(); + let joined: Value = response.json(); + let participants = joined["participants"].as_array().unwrap(); + assert_eq!(participants.len(), 1); + assert_eq!(participants[0]["agent_id"], "agent-2"); + assert_eq!(participants[0]["access"], "read_write"); + + let response = server + .post(&format!( + "/v1/stores/{store_id}/shared-spaces/{space_id}/leave" + )) + .json(&json!({ + "agent_id": "agent-2" + })) + .await; + response.assert_status(StatusCode::NO_CONTENT); + + let response = server + .delete(&format!("/v1/stores/{store_id}/shared-spaces/{space_id}")) + .await; + response.assert_status(StatusCode::NO_CONTENT); + + let response = server + .get(&format!("/v1/stores/{store_id}/shared-spaces")) + .await; + response.assert_status_ok(); + let spaces: Value = response.json(); + assert!(spaces.as_array().unwrap().is_empty()); +} + +#[tokio::test] +async fn test_inheritance_not_found_returns_error() { + let server = test_app().await; + + let response = server + .post("/v1/stores") + .json(&json!({ "name": "missing-inheritance-store" })) + .await; + response.assert_status(StatusCode::CREATED); + let store: Value = response.json(); + let store_id = store["id"].as_str().unwrap(); + + let response = server + .delete(&format!( + "/v1/stores/{store_id}/inherit/00000000-0000-0000-0000-000000000001" + )) + .await; + response.assert_status(StatusCode::BAD_REQUEST); + let body: Value = response.json(); + assert!(body["error"] + .as_str() + .unwrap() + .contains("inheritance not found")); +} + +#[tokio::test] +async fn test_shared_space_not_found_returns_error() { + let server = test_app().await; + + let response = server + .post("/v1/stores") + .json(&json!({ "name": "missing-space-store" })) + .await; + response.assert_status(StatusCode::CREATED); + let store: Value = response.json(); + let store_id = store["id"].as_str().unwrap(); + + let response = server + .get(&format!( + "/v1/stores/{store_id}/shared-spaces/00000000-0000-0000-0000-000000000001" + )) + .await; + response.assert_status(StatusCode::BAD_REQUEST); + let body: Value = response.json(); + assert!(body["error"] + .as_str() + .unwrap() + .contains("shared space not found")); +} diff --git a/crates/kd6-sqlite/migrations/20260602000000_upsert_and_autoprovision.sql b/crates/kd6-sqlite/migrations/20260602000000_upsert_and_autoprovision.sql new file mode 100644 index 0000000..ffa6aaf --- /dev/null +++ b/crates/kd6-sqlite/migrations/20260602000000_upsert_and_autoprovision.sql @@ -0,0 +1,7 @@ +-- Add upsert_key column to memories (OMS spec 4.3.2) +ALTER TABLE memories ADD COLUMN upsert_key TEXT; + +-- Index for efficient upsert lookups: store + layer + scope + upsert_key +CREATE INDEX IF NOT EXISTS idx_memories_upsert + ON memories(store_id, layer, scope_tenant_id, upsert_key) + WHERE upsert_key IS NOT NULL; diff --git a/crates/kd6-sqlite/migrations/20260603000000_fix_upsert_and_audit.sql b/crates/kd6-sqlite/migrations/20260603000000_fix_upsert_and_audit.sql new file mode 100644 index 0000000..b0ee30d --- /dev/null +++ b/crates/kd6-sqlite/migrations/20260603000000_fix_upsert_and_audit.sql @@ -0,0 +1,21 @@ +-- Fix upsert index to match full scope (not just scope_tenant_id). +-- Two memories with the same upsert_key but different scopes are distinct entries. +DROP INDEX IF EXISTS idx_memories_upsert; +CREATE INDEX IF NOT EXISTS idx_memories_upsert + ON memories(store_id, layer, scope_tenant_id, scope_org_id, scope_team_id, + scope_project_id, scope_user_id, scope_agent_id, scope_session_id, + scope_run_id, upsert_key) + WHERE upsert_key IS NOT NULL; + +-- Add composite index for memory pagination (list_memories ORDER BY created_at DESC). +CREATE INDEX IF NOT EXISTS idx_memories_store_tenant_created + ON memories(store_id, tenant_id, created_at DESC); + +-- Add redacted flag to audit_log for GDPR anonymization. +-- When TRUE, the entry's content has been anonymized but its hash chain +-- (entry_hash, prev_hash) remains intact for cryptographic verification. +ALTER TABLE audit_log ADD COLUMN redacted INTEGER NOT NULL DEFAULT 0; + +-- Add unique constraint for store names within a tenant (prevents _default race). +CREATE UNIQUE INDEX IF NOT EXISTS idx_stores_tenant_name + ON stores(tenant_id, name); diff --git a/crates/kd6-sqlite/src/provider.rs b/crates/kd6-sqlite/src/provider.rs index 43b59fa..f311bfa 100644 --- a/crates/kd6-sqlite/src/provider.rs +++ b/crates/kd6-sqlite/src/provider.rs @@ -265,6 +265,7 @@ fn row_to_audit(row: &sqlx::sqlite::SqliteRow) -> Result { created_at: DateTime::parse_from_rfc3339(&created_at) .map_err(|e| OmsError::Internal(format!("invalid audit created_at: {e}")))? .with_timezone(&Utc), + redacted: row.try_get::("redacted").unwrap_or(0) != 0, }) } @@ -429,6 +430,7 @@ fn row_to_memory(row: &sqlx::sqlite::SqliteRow) -> Result }, confidence: row.get("confidence"), entity_type: row.get("entity_type"), + upsert_key: row.get("upsert_key"), }) } @@ -602,6 +604,148 @@ impl SqliteProvider { Ok(()) } + /// Insert a single memory entry on an existing connection/transaction. + /// Does NOT manage transactions — caller is responsible for BEGIN/COMMIT/ROLLBACK. + /// Returns the created `MemoryEntry`. + async fn insert_memory_on_conn( + &self, + conn: &mut SqliteConnection, + tenant_id: &str, + store_id: Uuid, + request: CreateMemoryRequest, + ) -> Result { + let mut request = request; + request.scope = request.scope.normalize(tenant_id); + + let id = Uuid::new_v4(); + let now = Utc::now(); + let now_str = now.to_rfc3339(); + let embedding_blob = request.embedding.as_ref().map(|v| embedding_to_bytes(v)); + let tags_json = serde_json::to_string(&request.tags) + .map_err(|e| OmsError::Internal(format!("failed to serialize tags: {e}")))?; + let categories_json = serde_json::to_string(&request.categories) + .map_err(|e| OmsError::Internal(format!("failed to serialize categories: {e}")))?; + let source_json = request + .source + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| OmsError::Internal(format!("failed to serialize source: {e}")))?; + let access_policy = access_policy_to_str(&request.access_control.policy); + let allowed_agents_json = request + .access_control + .allowed_agents + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| OmsError::Internal(format!("failed to serialize allowed_agents: {e}")))?; + let allowed_scopes_json = request + .access_control + .allowed_scopes + .as_ref() + .map(serde_json::to_string) + .transpose() + .map_err(|e| OmsError::Internal(format!("failed to serialize allowed_scopes: {e}")))?; + let content_json = serde_json::to_string(&request.content) + .map_err(|e| OmsError::Internal(format!("failed to serialize content: {e}")))?; + let layer_str = request.layer.to_string(); + let expires_str = request.expires_at.map(|t| t.to_rfc3339()); + let valid_from_str = request.valid_from.as_ref().map(DateTime::to_rfc3339); + let valid_until_str = request.valid_until.as_ref().map(DateTime::to_rfc3339); + + sqlx::query( + "INSERT INTO memories ( + id, store_id, tenant_id, layer, content_json, embedding, + owner_agent_id, + scope_tenant_id, scope_org_id, scope_team_id, scope_project_id, + scope_user_id, scope_agent_id, scope_session_id, scope_run_id, + tags_json, categories_json, source_json, + access_policy, allowed_agents_json, allowed_scopes_json, + created_at, updated_at, expires_at, immutable, version, + valid_from, valid_until, confidence, entity_type, upsert_key + ) VALUES ( + ?, ?, ?, ?, ?, ?, + ?, + ?, ?, ?, ?, + ?, ?, ?, ?, + ?, ?, ?, + ?, ?, ?, + ?, ?, ?, ?, 1, + ?, ?, ?, ?, ? + )", + ) + .bind(id.to_string()) + .bind(store_id.to_string()) + .bind(tenant_id) + .bind(&layer_str) + .bind(&content_json) + .bind(&embedding_blob) + .bind(&request.owner_agent_id) + .bind(&request.scope.tenant_id) + .bind(&request.scope.org_id) + .bind(&request.scope.team_id) + .bind(&request.scope.project_id) + .bind(&request.scope.user_id) + .bind(&request.scope.agent_id) + .bind(&request.scope.session_id) + .bind(&request.scope.run_id) + .bind(&tags_json) + .bind(&categories_json) + .bind(&source_json) + .bind(access_policy) + .bind(&allowed_agents_json) + .bind(&allowed_scopes_json) + .bind(&now_str) + .bind(&now_str) + .bind(&expires_str) + .bind(request.immutable) + .bind(&valid_from_str) + .bind(&valid_until_str) + .bind(request.confidence) + .bind(&request.entity_type) + .bind(&request.upsert_key) + .execute(&mut *conn) + .await + .map_err(|e| OmsError::Internal(format!("failed to insert memory: {e}")))?; + + let entry = MemoryEntry { + id, + store_id, + layer: request.layer, + content: request.content, + embedding: request.embedding, + owner_agent_id: request.owner_agent_id, + scope: request.scope, + tags: request.tags, + categories: request.categories, + source: request.source, + access_control: request.access_control, + created_at: now, + updated_at: now, + expires_at: request.expires_at, + immutable: request.immutable, + version: 1, + valid_from: request.valid_from, + valid_until: request.valid_until, + confidence: request.confidence, + entity_type: request.entity_type, + upsert_key: request.upsert_key, + }; + + self.log_audit_on_conn( + conn, + tenant_id, + store_id, + Some(entry.id), + "create", + Some(entry.owner_agent_id.as_str()), + Some(serde_json::json!({"version": entry.version})), + ) + .await?; + + Ok(entry) + } + async fn get_space_participants( &self, tenant_id: &str, @@ -711,6 +855,74 @@ impl OmsProvider for SqliteProvider { rows.iter().map(|row| self.row_to_store(row)).collect() } + async fn get_or_create_store( + &self, + tenant_id: &str, + name: &str, + request: CreateStoreRequest, + ) -> Result { + // Try to find existing store first + if let Some(row) = sqlx::query("SELECT * FROM stores WHERE tenant_id = ? AND name = ?") + .bind(tenant_id) + .bind(name) + .fetch_optional(&self.pool) + .await + .map_err(|e| OmsError::Internal(format!("failed to query store by name: {e}")))? + { + return self.row_to_store(&row); + } + + // Attempt atomic insert; unique index prevents duplicates + let id = Uuid::new_v4(); + let now = Utc::now(); + let now_str = now.to_rfc3339(); + let config_json = serde_json::to_string(&request.config) + .map_err(|e| OmsError::Internal(format!("failed to serialize config: {e}")))?; + let metadata_json = serde_json::to_string(&request.metadata) + .map_err(|e| OmsError::Internal(format!("failed to serialize metadata: {e}")))?; + + let result = sqlx::query( + "INSERT OR IGNORE INTO stores (id, name, tenant_id, region, config_json, metadata_json, created_at, updated_at) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)", + ) + .bind(id.to_string()) + .bind(name) + .bind(tenant_id) + .bind(&request.region) + .bind(&config_json) + .bind(&metadata_json) + .bind(&now_str) + .bind(&now_str) + .execute(&self.pool) + .await + .map_err(|e| OmsError::Internal(format!("failed to insert store: {e}")))?; + + if result.rows_affected() == 0 { + // Another request created it concurrently — fetch it + let row = sqlx::query("SELECT * FROM stores WHERE tenant_id = ? AND name = ?") + .bind(tenant_id) + .bind(name) + .fetch_one(&self.pool) + .await + .map_err(|e| { + OmsError::Internal(format!("failed to fetch concurrent store: {e}")) + })?; + return self.row_to_store(&row); + } + + Ok(MemoryStore { + id, + name: name.to_string(), + tenant_id: tenant_id.to_string(), + region: request.region, + config: request.config, + sovereignty: SovereigntyConfig::default(), + metadata: request.metadata, + created_at: now, + updated_at: now, + }) + } + async fn update_store( &self, tenant_id: &str, @@ -822,6 +1034,150 @@ impl OmsProvider for SqliteProvider { .await .map_err(|e| OmsError::Internal(format!("failed to begin transaction: {e}")))?; + // Upsert: if upsert_key is set, look for an existing entry to replace. + // Match on full normalized scope so that the same upsert_key in different + // scopes creates distinct entries. + if let Some(ref upsert_key) = request.upsert_key { + let existing: Option = sqlx::query( + "SELECT id, version, content_json, created_at FROM memories + WHERE store_id = ? AND layer = ? AND upsert_key = ? + AND scope_tenant_id = ? + AND COALESCE(scope_org_id, '') = COALESCE(?, '') + AND COALESCE(scope_team_id, '') = COALESCE(?, '') + AND COALESCE(scope_project_id, '') = COALESCE(?, '') + AND COALESCE(scope_user_id, '') = COALESCE(?, '') + AND COALESCE(scope_agent_id, '') = COALESCE(?, '') + AND COALESCE(scope_session_id, '') = COALESCE(?, '') + AND COALESCE(scope_run_id, '') = COALESCE(?, '') + LIMIT 1", + ) + .bind(store_id.to_string()) + .bind(&layer_str) + .bind(upsert_key) + .bind(&request.scope.tenant_id) + .bind(&request.scope.org_id) + .bind(&request.scope.team_id) + .bind(&request.scope.project_id) + .bind(&request.scope.user_id) + .bind(&request.scope.agent_id) + .bind(&request.scope.session_id) + .bind(&request.scope.run_id) + .fetch_optional(&mut *conn) + .await + .map_err(|e| OmsError::Internal(format!("failed to check upsert key: {e}")))?; + + if let Some(row) = existing { + let existing_id_str: String = row.get("id"); + let existing_id = Uuid::parse_str(&existing_id_str) + .map_err(|e| OmsError::Internal(format!("invalid existing id: {e}")))?; + let existing_version: i64 = row.get("version"); + let prev_content: String = row.get("content_json"); + let original_created_at: String = row.get("created_at"); + let new_version = existing_version + 1; + + if let Err(e) = sqlx::query( + "UPDATE memories SET + content_json = ?, embedding = ?, tags_json = ?, categories_json = ?, + source_json = ?, access_policy = ?, allowed_agents_json = ?, + allowed_scopes_json = ?, + scope_tenant_id = ?, scope_org_id = ?, scope_team_id = ?, + scope_project_id = ?, scope_user_id = ?, scope_agent_id = ?, + scope_session_id = ?, scope_run_id = ?, + updated_at = ?, expires_at = ?, + valid_from = ?, valid_until = ?, confidence = ?, entity_type = ?, + owner_agent_id = ?, version = ? + WHERE id = ?", + ) + .bind(&content_json) + .bind(&embedding_blob) + .bind(&tags_json) + .bind(&categories_json) + .bind(&source_json) + .bind(access_policy) + .bind(&allowed_agents_json) + .bind(&allowed_scopes_json) + // Persist scope columns (ensures normalization changes are applied) + .bind(&request.scope.tenant_id) + .bind(&request.scope.org_id) + .bind(&request.scope.team_id) + .bind(&request.scope.project_id) + .bind(&request.scope.user_id) + .bind(&request.scope.agent_id) + .bind(&request.scope.session_id) + .bind(&request.scope.run_id) + .bind(&now_str) + .bind(&expires_str) + .bind(&valid_from_str) + .bind(&valid_until_str) + .bind(request.confidence) + .bind(&request.entity_type) + .bind(&request.owner_agent_id) + .bind(new_version) + .bind(&existing_id_str) + .execute(&mut *conn) + .await + { + let _ = sqlx::query("ROLLBACK").execute(&mut *conn).await; + return Err(OmsError::Internal(format!("failed to upsert memory: {e}"))); + } + + // Return DB-truth: use original created_at, not current time + let original_created = DateTime::parse_from_rfc3339(&original_created_at) + .map(|dt| dt.with_timezone(&Utc)) + .unwrap_or(now); + + let entry = MemoryEntry { + id: existing_id, + store_id, + layer: request.layer, + content: request.content, + embedding: request.embedding, + owner_agent_id: request.owner_agent_id, + scope: request.scope, + tags: request.tags, + categories: request.categories, + source: request.source, + access_control: request.access_control, + created_at: original_created, + updated_at: now, + expires_at: request.expires_at, + immutable: request.immutable, + version: new_version, + valid_from: request.valid_from, + valid_until: request.valid_until, + confidence: request.confidence, + entity_type: request.entity_type, + upsert_key: request.upsert_key, + }; + + if let Err(e) = self + .log_audit_on_conn( + &mut conn, + tenant_id, + store_id, + Some(entry.id), + "upsert", + Some(entry.owner_agent_id.as_str()), + Some(serde_json::json!({ + "version": entry.version, + "previous_content": prev_content, + })), + ) + .await + { + let _ = sqlx::query("ROLLBACK").execute(&mut *conn).await; + return Err(e); + } + + sqlx::query("COMMIT") + .execute(&mut *conn) + .await + .map_err(|e| OmsError::Internal(format!("failed to commit: {e}")))?; + + return Ok(entry); + } + } + if let Err(e) = sqlx::query( "INSERT INTO memories ( id, store_id, tenant_id, layer, content_json, embedding, @@ -831,7 +1187,7 @@ impl OmsProvider for SqliteProvider { tags_json, categories_json, source_json, access_policy, allowed_agents_json, allowed_scopes_json, created_at, updated_at, expires_at, immutable, version, - valid_from, valid_until, confidence, entity_type + valid_from, valid_until, confidence, entity_type, upsert_key ) VALUES ( ?, ?, ?, ?, ?, ?, ?, @@ -840,7 +1196,7 @@ impl OmsProvider for SqliteProvider { ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 1, - ?, ?, ?, ? + ?, ?, ?, ?, ? )", ) .bind(id.to_string()) @@ -872,6 +1228,7 @@ impl OmsProvider for SqliteProvider { .bind(&valid_until_str) .bind(request.confidence) .bind(&request.entity_type) + .bind(&request.upsert_key) .execute(&mut *conn) .await { @@ -900,6 +1257,7 @@ impl OmsProvider for SqliteProvider { valid_until: request.valid_until, confidence: request.confidence, entity_type: request.entity_type, + upsert_key: request.upsert_key, }; if let Err(e) = self @@ -1414,9 +1772,11 @@ impl OmsProvider for SqliteProvider { let now_str = Utc::now().to_rfc3339(); - let mut conn = self.pool.acquire().await.map_err(|e| { - OmsError::Internal(format!("failed to acquire connection: {e}")) - })?; + let mut conn = self + .pool + .acquire() + .await + .map_err(|e| OmsError::Internal(format!("failed to acquire connection: {e}")))?; sqlx::query("BEGIN IMMEDIATE") .execute(&mut *conn) @@ -1719,122 +2079,158 @@ impl OmsProvider for SqliteProvider { OmsError::Internal(format!("failed to query child memories for bubble up: {e}")) })?; - // Check which source memories have already been bubbled up to this parent - // by looking for existing memories with a source reference pointing back. - let mut already_bubbled = std::collections::HashSet::new(); - { - let existing_sql = - "SELECT source_json FROM memories WHERE store_id = ? AND tenant_id = ? AND scope_agent_id = ? AND source_json IS NOT NULL"; - let rows = sqlx::query(existing_sql) - .bind(store_id.to_string()) - .bind(tenant_id) - .bind(&request.parent_agent_id) - .fetch_all(&self.pool) - .await - .map_err(|e| { - OmsError::Internal(format!("failed to check existing bubbled memories: {e}")) - })?; - for row in &rows { - let source_str: Option = row.get("source_json"); - if let Some(json_str) = source_str { - if let Ok(src) = serde_json::from_str::(&json_str) { - if let Some(ref uri) = src.uri { - if let Some(ref_id) = uri.strip_prefix("bubble_up:") { - if let Ok(uid) = Uuid::parse_str(ref_id) { - already_bubbled.insert(uid); + let mut created = Vec::new(); + + // Run all reads AND inserts under a single transaction for atomicity + // and to prevent concurrent duplicates. + let mut conn = self + .pool + .acquire() + .await + .map_err(|e| OmsError::Internal(format!("failed to acquire connection: {e}")))?; + + sqlx::query("BEGIN IMMEDIATE") + .execute(&mut *conn) + .await + .map_err(|e| { + OmsError::Internal(format!("failed to begin bubble_up transaction: {e}")) + })?; + + let result: Result<(), OmsError> = async { + // Check which source memories have already been bubbled up to this parent + // (inside the transaction to prevent concurrent duplicates). + let mut already_bubbled = std::collections::HashSet::new(); + { + let existing_sql = + "SELECT source_json FROM memories WHERE store_id = ? AND tenant_id = ? AND scope_agent_id = ? AND source_json IS NOT NULL"; + let rows = sqlx::query(existing_sql) + .bind(store_id.to_string()) + .bind(tenant_id) + .bind(&request.parent_agent_id) + .fetch_all(&mut *conn) + .await + .map_err(|e| { + OmsError::Internal(format!("failed to check existing bubbled memories: {e}")) + })?; + for row in &rows { + let source_str: Option = row.get("source_json"); + if let Some(json_str) = source_str { + if let Ok(src) = serde_json::from_str::(&json_str) { + if let Some(ref uri) = src.uri { + if let Some(ref_id) = uri.strip_prefix("bubble_up:") { + if let Ok(uid) = Uuid::parse_str(ref_id) { + already_bubbled.insert(uid); + } } } } } } } - } - let mut created = Vec::new(); - for row in &source_rows { - let source = row_to_memory(row)?; + for row in &source_rows { + let source = row_to_memory(row)?; - // Skip if already bubbled up - if already_bubbled.contains(&source.id) { - continue; - } + if already_bubbled.contains(&source.id) { + continue; + } - let mut parent_scope = source.scope.clone(); - parent_scope.agent_id = Some(request.parent_agent_id.clone()); - parent_scope.session_id = None; - parent_scope.run_id = None; + let mut parent_scope = source.scope.clone(); + parent_scope.agent_id = Some(request.parent_agent_id.clone()); + parent_scope.session_id = None; + parent_scope.run_id = None; - let memory = self - .create_memory( - tenant_id, - store_id, - CreateMemoryRequest { - layer: source.layer, - content: source.content, - embedding: source.embedding, - owner_agent_id: request.parent_agent_id.clone(), - scope: parent_scope, - tags: source.tags, - categories: source.categories, - source: Some(SourceReference { - conversation_id: None, - document_id: None, - uri: Some(format!("bubble_up:{}", source.id)), - }), - access_control: source.access_control, - expires_at: source.expires_at, - immutable: source.immutable, - valid_from: source.valid_from, - valid_until: source.valid_until, - confidence: source.confidence, - entity_type: source.entity_type, - }, - ) - .await?; - created.push(memory); - } - - if let Some(summary) = request.summary { - let layer = requested_layers - .first() - .copied() - .unwrap_or(MemoryLayer::Working); - let summary_entry = self - .create_memory( - tenant_id, - store_id, - CreateMemoryRequest { - layer, - content: summary, - embedding: None, - owner_agent_id: request.parent_agent_id.clone(), - scope: MemoryScope { - tenant_id: tenant_id.to_string(), - org_id: None, - team_id: None, - project_id: None, - user_id: None, - agent_id: Some(request.parent_agent_id.clone()), - session_id: None, - run_id: None, + let memory = self + .insert_memory_on_conn( + &mut conn, + tenant_id, + store_id, + CreateMemoryRequest { + layer: source.layer, + content: source.content, + embedding: source.embedding, + owner_agent_id: request.parent_agent_id.clone(), + scope: parent_scope, + tags: source.tags, + categories: source.categories, + source: Some(SourceReference { + conversation_id: None, + document_id: None, + uri: Some(format!("bubble_up:{}", source.id)), + }), + access_control: source.access_control, + expires_at: source.expires_at, + immutable: source.immutable, + valid_from: source.valid_from, + valid_until: source.valid_until, + confidence: source.confidence, + entity_type: source.entity_type, + upsert_key: source.upsert_key, }, - tags: vec!["bubble_up".into(), "summary".into()], - categories: vec!["summary".into()], - source: None, - access_control: AccessControl::default(), - expires_at: None, - immutable: false, - valid_from: None, - valid_until: None, - confidence: None, - entity_type: None, - }, - ) - .await?; - created.push(summary_entry); - } + ) + .await?; + created.push(memory); + } + + if let Some(summary) = request.summary { + let layer = requested_layers + .first() + .copied() + .unwrap_or(MemoryLayer::Working); + let summary_entry = self + .insert_memory_on_conn( + &mut conn, + tenant_id, + store_id, + CreateMemoryRequest { + layer, + content: summary, + embedding: None, + owner_agent_id: request.parent_agent_id.clone(), + scope: MemoryScope { + tenant_id: tenant_id.to_string(), + org_id: None, + team_id: None, + project_id: None, + user_id: None, + agent_id: Some(request.parent_agent_id.clone()), + session_id: None, + run_id: None, + }, + tags: vec!["bubble_up".into(), "summary".into()], + categories: vec!["summary".into()], + source: None, + access_control: AccessControl::default(), + expires_at: None, + immutable: false, + valid_from: None, + valid_until: None, + confidence: None, + entity_type: None, + upsert_key: None, + }, + ) + .await?; + created.push(summary_entry); + } - Ok(created) + Ok(()) + } + .await; + + match result { + Ok(()) => { + sqlx::query("COMMIT") + .execute(&mut *conn) + .await + .map_err(|e| OmsError::Internal(format!("failed to commit bubble_up: {e}")))?; + Ok(created) + } + Err(e) => { + let _ = sqlx::query("ROLLBACK").execute(&mut *conn).await; + Err(e) + } + } } // --- Level 2: Shared Spaces --- @@ -2308,11 +2704,13 @@ impl OmsProvider for SqliteProvider { // Anonymize audit log entries referencing purged memories (GDPR Art. 17). // We retain the audit entry for compliance proof but strip PII fields. + // The `redacted` flag signals that entry_hash will no longer match current + // row content, but the hash chain (prev_hash links) remains intact. if !purged_ids.is_empty() { for chunk in purged_ids.chunks(500) { let placeholders: Vec<&str> = chunk.iter().map(|_| "?").collect(); let anon_sql = format!( - "UPDATE audit_log SET agent_id = NULL, details_json = NULL \ + "UPDATE audit_log SET agent_id = NULL, details_json = NULL, redacted = 1 \ WHERE store_id = ? AND tenant_id = ? AND memory_id IN ({})", placeholders.join(",") ); @@ -2713,6 +3111,7 @@ mod tests { valid_until: None, confidence: None, entity_type: None, + upsert_key: None, } } @@ -3838,4 +4237,176 @@ mod tests { kd6_core::models::sovereignty::SovereigntyMode::Any ); } + + #[tokio::test] + async fn upsert_same_key_same_scope_updates_in_place() { + let (provider, store) = setup_with_store().await; + let mut req = make_memory_request("agent-1"); + req.upsert_key = Some("dedup-1".into()); + req.content = serde_json::json!({"text": "first version"}); + + let first = provider + .create_memory("tenant-1", store.id, req.clone()) + .await + .unwrap(); + assert_eq!(first.version, 1); + + // Second create with same key should upsert (update in place) + req.content = serde_json::json!({"text": "second version"}); + let second = provider + .create_memory("tenant-1", store.id, req) + .await + .unwrap(); + + assert_eq!(second.id, first.id, "should reuse same ID"); + assert_eq!(second.version, 2); + assert_eq!( + second.content, + serde_json::json!({"text": "second version"}) + ); + // created_at should be preserved from the original insert + assert_eq!(second.created_at, first.created_at); + } + + #[tokio::test] + async fn upsert_same_key_different_scope_creates_distinct_entries() { + let (provider, store) = setup_with_store().await; + let mut req1 = make_memory_request("agent-1"); + req1.upsert_key = Some("dedup-2".into()); + req1.scope.user_id = Some("user-a".into()); + req1.content = serde_json::json!({"text": "user A data"}); + + let mut req2 = make_memory_request("agent-1"); + req2.upsert_key = Some("dedup-2".into()); + req2.scope.user_id = Some("user-b".into()); + req2.content = serde_json::json!({"text": "user B data"}); + + let entry_a = provider + .create_memory("tenant-1", store.id, req1) + .await + .unwrap(); + let entry_b = provider + .create_memory("tenant-1", store.id, req2) + .await + .unwrap(); + + assert_ne!( + entry_a.id, entry_b.id, + "different scopes should create distinct entries" + ); + assert_eq!(entry_a.scope.user_id, Some("user-a".into())); + assert_eq!(entry_b.scope.user_id, Some("user-b".into())); + } + + #[tokio::test] + async fn upsert_persists_scope_columns() { + let (provider, store) = setup_with_store().await; + let mut req = make_memory_request("agent-1"); + req.upsert_key = Some("scope-test".into()); + req.scope.user_id = Some("user-x".into()); + req.scope.team_id = Some("team-y".into()); + + provider + .create_memory("tenant-1", store.id, req.clone()) + .await + .unwrap(); + + // Upsert with same scope + req.content = serde_json::json!({"text": "updated"}); + let updated = provider + .create_memory("tenant-1", store.id, req) + .await + .unwrap(); + + // Fetch from DB to verify scope is actually persisted (not just request echo) + let fetched = provider + .get_memory("tenant-1", store.id, updated.id) + .await + .unwrap(); + assert_eq!(fetched.scope.user_id, Some("user-x".into())); + assert_eq!(fetched.scope.team_id, Some("team-y".into())); + assert_eq!(fetched.content, serde_json::json!({"text": "updated"})); + } + + #[tokio::test] + async fn gdpr_purge_sets_redacted_flag_on_audit_entries() { + let provider = test_provider().await; + let tenant = "t-gdpr-redact"; + let store = provider + .create_store( + tenant, + CreateStoreRequest { + name: "redact-store".into(), + region: None, + config: StoreConfig::default(), + metadata: Default::default(), + }, + ) + .await + .unwrap(); + + let mut req = make_memory_request("agent-1"); + req.scope.tenant_id = tenant.into(); + req.scope.user_id = Some("user-purge".into()); + let entry = provider.create_memory(tenant, store.id, req).await.unwrap(); + + // There should be audit entries for the create + let audit_before = provider + .audit_log( + tenant, + store.id, + AuditFilter { + memory_id: Some(entry.id), + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(!audit_before.items.is_empty()); + assert!(!audit_before.items[0].redacted); + + // Purge user-purge's data + provider + .gdpr_purge( + tenant, + store.id, + kd6_core::models::MemoryScope { + tenant_id: tenant.into(), + user_id: Some("user-purge".into()), + ..Default::default() + }, + ) + .await + .unwrap(); + + // Audit entries for the purged memory should be redacted + let audit_after = provider + .audit_log( + tenant, + store.id, + AuditFilter { + memory_id: Some(entry.id), + ..Default::default() + }, + ) + .await + .unwrap(); + assert!(!audit_after.items.is_empty()); + for audit_entry in &audit_after.items { + if audit_entry.action != "gdpr_purge" { + assert!( + audit_entry.redacted, + "audit entry should be marked redacted" + ); + assert!( + audit_entry.agent_id.is_none(), + "agent_id should be anonymized" + ); + assert!( + audit_entry.details.is_none(), + "details should be anonymized" + ); + } + } + } } diff --git a/spec/oms-spec.md b/spec/oms-spec.md index 95266be..4e40775 100644 --- a/spec/oms-spec.md +++ b/spec/oms-spec.md @@ -28,12 +28,20 @@ - 5.6 [Audit & Compliance](#56-audit--compliance) 6. [Authentication & Multi-Tenancy](#6-authentication--multi-tenancy) 7. [Protocol Integration](#7-protocol-integration) -8. [Backend Provider Interface (SPI)](#8-backend-provider-interface-spi) -9. [Data Sovereignty](#9-data-sovereignty) -10. [Conformance Levels](#10-conformance-levels) -11. [Reference Implementation Guidance](#11-reference-implementation-guidance) -12. [Relationship to Existing Work](#12-relationship-to-existing-work) -13. [References](#13-references) +8. [Embedding Provider Interface](#8-embedding-provider-interface) + - 8.1 [Motivation](#81-motivation) + - 8.2 [Embedding Provider SPI](#82-embedding-provider-spi) + - 8.3 [Store-Level Embedding Configuration](#83-store-level-embedding-configuration) + - 8.4 [Automatic Embedding Behavior](#84-automatic-embedding-behavior) + - 8.5 [Embedding Model Lifecycle](#85-embedding-model-lifecycle) + - 8.6 [Built-in Provider Requirements](#86-built-in-provider-requirements) + - 8.7 [Capability Advertisement](#87-capability-advertisement) +9. [Backend Provider Interface (SPI)](#9-backend-provider-interface-spi) +10. [Data Sovereignty](#10-data-sovereignty) +11. [Conformance Levels](#11-conformance-levels) +12. [Reference Implementation Guidance](#12-reference-implementation-guidance) +13. [Relationship to Existing Work](#13-relationship-to-existing-work) +14. [References](#14-references) --- @@ -96,13 +104,32 @@ MemoryStore: backend: BackendConfig # pluggable backend configuration default_ttl: duration | null # default TTL for new entries default_sharing_policy: SharingPolicy # default access policy - embedding_model: string | null # model for vector embeddings + embedding: # embedding provider config (see section 8) + provider: string | null # provider identifier (e.g., "openai", "ollama") + model: string | null # model name (e.g., "text-embedding-3-small") + dimensions: int | null # override dimensions (if supported by model) compaction_policy: CompactionPolicy | null metadata: map # arbitrary user metadata created_at: timestamp updated_at: timestamp ``` +#### 4.1.1 Default Store and Auto-Provisioning + +Implementations MAY support a **default store** identified by the well-known alias `_default`. When enabled, any API call that would normally require a `store_id` path parameter MAY use `_default` as the store identifier. + +When auto-provisioning is enabled and a write operation (memory creation, upsert) targets a store or tenant that does not yet exist, the service MUST lazily create the missing entities before completing the write: + +1. If the resolved tenant does not exist, create it with implementation-defined defaults. +2. If the target store does not exist within the resolved tenant (whether `_default` or an explicitly named store), create it with implementation-defined default configuration. +3. Return the write result as normal. The caller does not need to distinguish between a pre-existing store and a newly provisioned one. + +This lazy provisioning means an agent's very first memory write is self-sufficient. No setup calls are required. Read operations against non-existent stores or tenants MUST return empty results (not errors), so that search-before-write patterns also work without prior provisioning. + +This feature is intended for development, single-agent, and evaluation scenarios where explicit store and tenant management adds unnecessary ceremony. Implementations that enable it SHOULD document the default configuration applied to auto-provisioned stores. + +> **Security consideration:** In multi-tenant production deployments, auto-provisioning may allow any authenticated agent to implicitly create tenants or provision storage. Implementations MUST provide a configuration flag to disable auto-provisioning and MUST disable it by default when the deployment is configured for multi-tenant operation. When disabled, writes targeting non-existent stores MUST be rejected with `404 Not Found` and requests with unrecognized tenant context MUST be rejected with `403 Forbidden`. + ### 4.2 Memory Layers Every store supports up to five memory layers, each with independent configuration. The five-layer model is grounded in cognitive science research on human memory systems [1][2][3][4]: @@ -128,8 +155,9 @@ MemoryEntry: id: string (UUID) store_id: string layer: "working" | "episodic" | "semantic" | "procedural" | "archival" - content: string | object # the memory content - embedding: float[] | null # vector representation (computed by service or provided) + content: string | object # the memory content (see 4.3.1) + embedding: float[] | null # vector representation (server-computed or caller-provided; see section 8) + upsert_key: string | null # optional idempotency key for upsert semantics (see 4.3.2) metadata: owner_agent_id: string # which agent created this entry scope: MemoryScope # visibility and sharing scope @@ -139,7 +167,7 @@ MemoryEntry: temporal: # optional temporal metadata (inspired by Zep [10]) valid_from: timestamp | null # when this fact became true valid_until: timestamp | null # when this fact became false (null = still valid) - confidence: float | null # confidence score (0.0–1.0) + confidence: float | null # confidence score (0.0-1.0) access_control: policy: "private" | "inherit" | "shared" | "public_read" allowed_agents: string[] | null # specific agents granted access @@ -155,13 +183,34 @@ MemoryEntry: relationships: Relationship[] | null # edges to other entities ``` +#### 4.3.1 Content Format + +The `content` field accepts both plain strings and structured JSON objects. When a client provides a plain string, the service MUST accept it as the full memory content and return it as a string in subsequent reads. Implementations MUST NOT require clients to wrap text in a JSON envelope (e.g., `{"text": "..."}`) when the content is unstructured text. + +Structured content is appropriate when the memory carries typed fields, key-value metadata, or nested data that the client wants to preserve with schema. The service MUST round-trip structured content without modification. + +#### 4.3.2 Upsert Semantics + +The `upsert_key` field enables atomic create-or-replace behavior. When a `POST` to the memory creation endpoint includes a non-null `upsert_key`, the service MUST apply the following logic: + +1. Search for an existing memory entry within the same `store_id`, `layer`, and `scope` that has a matching `upsert_key`. +2. If a match exists: replace its `content`, increment its `version`, update `updated_at`, and return the updated entry. The previous content MUST be recorded in the audit trail. +3. If no match exists: create a new entry as normal. + +This operation MUST be atomic. Concurrent upserts with the same key MUST serialize and produce a consistent result (last writer wins). The `upsert_key` is scoped to the combination of store, layer, and scope to prevent unintended collisions across organizational boundaries. + +**Use cases:** +- Storing agent preferences that should have exactly one current value (e.g., `upsert_key: "preference:theme"`) +- Maintaining singleton facts that are periodically refreshed (e.g., `upsert_key: "agent-status:summarizer"`) +- Avoiding search-then-delete-then-create race conditions in concurrent agent systems + ### 4.4 Memory Scope (Hierarchical) Scopes define visibility boundaries and enable parent-child memory inheritance. The design follows a hierarchical model where more specific scopes inherit from broader ones: ```yaml MemoryScope: - tenant_id: string # REQUIRED — the hard isolation boundary + tenant_id: string # REQUIRED -- the hard isolation boundary org_id: string | null # organizational unit within tenant team_id: string | null # team within org project_id: string | null # project within team @@ -171,6 +220,18 @@ MemoryScope: run_id: string | null # specific execution run ``` +#### 4.4.1 Default Tenant + +Implementations MAY support a **default tenant** identified by the well-known value `_default`. When enabled, requests that do not provide explicit tenant context (no `X-Tenant-ID` header, no `tenant_id` JWT claim) are resolved to the default tenant. + +When combined with auto-provisioning (see 4.1.1), the default tenant is lazily created on the first write operation that resolves to it. This allows agents and frameworks that have no concept of multi-tenancy to issue their first memory write with zero prior setup. + +This feature is intended for local development, single-user tools, and evaluation environments where tenant management is unnecessary overhead. + +> **Security consideration:** Enabling default tenant resolution in a multi-tenant production deployment constitutes a security risk. A request that omits tenant context would silently resolve to the default tenant rather than being rejected, potentially causing data to land in the wrong isolation boundary. Implementations MUST provide a configuration flag to disable default tenant support and MUST disable it by default when the deployment is configured for multi-tenant operation. + +When default tenant support is disabled, requests that omit tenant context MUST be rejected with `400 Bad Request`. + **Scope resolution rules:** A memory entry is visible to any agent whose scope is **equal to or more specific than** the entry's scope: @@ -253,13 +314,15 @@ DELETE /v1/stores/{store_id} # Delete store (with policy enforcem ### 5.2 Memory CRUD ``` -POST /v1/stores/{store_id}/memories # Create memory entry +POST /v1/stores/{store_id}/memories # Create memory entry (supports upsert, see 4.3.2) GET /v1/stores/{store_id}/memories/{memory_id} # Get by ID PATCH /v1/stores/{store_id}/memories/{memory_id} # Update entry (versioned) DELETE /v1/stores/{store_id}/memories/{memory_id} # Delete entry (audited) GET /v1/stores/{store_id}/memories # List with filters + pagination ``` +The `store_id` path parameter accepts either a concrete store UUID or the well-known alias `_default` when default store support is enabled (see 4.1.1). This alias resolution applies to all store-scoped endpoints across the API surface. + ### 5.3 Memory Search ```http @@ -268,6 +331,7 @@ Content-Type: application/json { "query": "string", // natural language or structured query + "embedding": [0.1, 0.2, ...] | null, // optional pre-computed query embedding "layers": ["episodic", "semantic"], // which layers to search "scope": { ... }, // scope filter "top_k": 10, @@ -285,6 +349,8 @@ Content-Type: application/json } ``` +When the store has a configured embedding provider (see section 8), the `embedding` field is optional — the service computes a query embedding from `query` automatically. When no embedding provider is configured, callers must supply `embedding` for vector search or set `keyword: true` for text-only search. + ### 5.4 Memory Lifecycle ``` @@ -325,12 +391,12 @@ POST /v1/stores/{store_id}/purge # GDPR right-to-era ## 6. Authentication & Multi-Tenancy Every request MUST include: -- **Tenant context** — via `X-Tenant-ID` header or `tenant_id` claim in JWT -- **Agent identity** — via `X-Agent-ID` header or `agent_id` claim in JWT -- **Authentication** — OAuth 2.1 bearer token or mTLS +- **Tenant context** -- via `X-Tenant-ID` header or `tenant_id` claim in JWT. Implementations that support default tenant resolution (see 4.4.1) MAY omit this requirement for requests targeting the default tenant. +- **Agent identity** -- via `X-Agent-ID` header or `agent_id` claim in JWT +- **Authentication** -- OAuth 2.1 bearer token or mTLS The service MUST enforce: -- **Tenant isolation** at the data layer — requests MUST NOT cross tenant boundaries under any circumstances +- **Tenant isolation** at the data layer -- requests MUST NOT cross tenant boundaries under any circumstances - **Agent-level authorization** per the entry's `access_control` policy - **Rate limiting** per tenant and per agent - **Audit logging** for all write operations @@ -371,7 +437,270 @@ When agents delegate tasks across boundaries [19]: --- -## 8. Backend Provider Interface (SPI) +## 8. Embedding Provider Interface + +Server-side embedding is a core capability that allows OMS implementations to automatically compute vector representations for memory content. This eliminates the requirement for callers to supply pre-computed embeddings and enables seamless integration with agent frameworks that treat the memory backend as a "text in, relevance out" service. + +### 8.1 Motivation + +Most agent frameworks (LangChain, CrewAI, Google ADK, Squad) send plain text to their memory backend and expect the backend to handle vectorization. Without server-side embedding, every client adapter must independently manage an embedding model — introducing latency (extra network hop), cost (duplicate model loading), and complexity (version skew between embedding models across clients). This was identified as the P0 integration gap blocking LangChain and other major frameworks from using OMS backends. + +By defining embedding as a first-class concern in the OMS spec, implementations gain: +- **Zero-config integration** — agents write text, the service handles the rest +- **Consistency** — all memories in a store use the same embedding model and dimensions +- **Upgradeability** — the store owner can upgrade the embedding model without changing clients +- **Hybrid search** — keyword (BM25) and vector search merge naturally when embeddings are always present + +### 8.1.1 Client-Side vs. Server-Side Embedding + +OMS supports two embedding strategies. The choice is made **per-request**, not per-store, and the two strategies can coexist within the same store: + +| Strategy | How it works | When to use | +|---|---|---| +| **Server-side (recommended)** | Caller sends plain text. The OMS service computes the embedding using its configured provider before storing or searching. | Default path. Simplest for callers. Guarantees model consistency across all entries. Required for framework integrations (LangChain, CrewAI, etc.) that send text only. | +| **Client-side** | Caller computes the embedding externally and includes it in the `embedding` field of the request. The service validates dimensionality but does **not** re-embed. | Advanced use cases where the caller controls the model (e.g., fine-tuned domain embeddings, multi-modal embeddings, or embeddings computed during an upstream pipeline step). | + +**Design principles:** + +1. **Server-side is the default.** When the `embedding` field is absent from a write or search request, the service MUST compute it from content/query text. This makes the simplest possible API call — `{"content": "some text"}` — fully functional with vector search. + +2. **Client-side is an opt-in override.** When the caller provides an `embedding` field, the service uses it as-is. This respects caller expertise while still enforcing dimensionality constraints. + +3. **Consistency within a store.** All embeddings in a store MUST share the same dimensionality, regardless of whether they were computed server-side or client-side. A client-provided embedding with the wrong dimensionality MUST be rejected. Callers who provide their own embeddings are responsible for using a model that produces vectors in the same semantic space as the store's configured provider — mixing incompatible models degrades search quality. + +4. **No embedding function required in client adapters.** Framework integrations (e.g., LangChain VectorStore, CrewAI RAGStorage) SHOULD default to server-side embedding, making the embedding parameter optional in client constructors. This is the key usability win: `store = KD6VectorStore(base_url="http://localhost:8080")` works with no embedding model configuration on the client side. + +### 8.2 Embedding Provider SPI + +Implementations MUST support a pluggable embedding provider interface. The interface is intentionally minimal to accommodate local models, remote APIs, and managed services: + +```python +class EmbeddingProvider(ABC): + """Computes vector embeddings from text content. + + Implementations may call a local model (e.g., ONNX, Sentence Transformers), + a remote API (e.g., OpenAI, Azure OpenAI, Cohere, Voyager), or a managed + service (e.g., Vertex AI, Amazon Bedrock). + """ + + @abstractmethod + def embed_texts(self, texts: list[str]) -> list[list[float]]: + """Compute embeddings for one or more text strings. + + Args: + texts: The input strings to embed. Each string may be a plain + memory content string or a query string. + + Returns: + A list of embedding vectors, one per input text. + All vectors MUST have the same dimensionality. + + Raises: + EmbeddingError: If the provider is unavailable or the input + exceeds provider-specific limits. + """ + ... + + @abstractmethod + def embed_query(self, query: str) -> list[float]: + """Compute a single embedding for a search query. + + Some embedding models use different prefixes or instructions for + queries vs. documents (e.g., "query: " vs. "passage: " in E5 models). + This method allows the provider to apply query-specific preprocessing. + + Default implementations MAY delegate to embed_texts([query])[0]. + """ + ... + + @abstractmethod + def dimensions(self) -> int: + """Return the dimensionality of embeddings produced by this provider. + + This value MUST be constant for the lifetime of the provider instance + and MUST match the length of all vectors returned by embed_texts + and embed_query. + """ + ... + + @abstractmethod + def model_id(self) -> str: + """Return a stable identifier for the embedding model. + + Used for tracking which model produced stored embeddings, enabling + migration detection when a store's model is upgraded. + + Examples: "text-embedding-3-small", "all-MiniLM-L6-v2", + "voyage-3", "text-embedding-004" + """ + ... +``` + +### 8.3 Store-Level Embedding Configuration + +Each memory store MAY be configured with an embedding provider. The `embedding` field in `StoreConfig` specifies the provider and its parameters: + +```yaml +StoreConfig: + embedding: + provider: string # provider identifier (e.g., "openai", "ollama", "sentence-transformers") + model: string # model name within the provider (e.g., "text-embedding-3-small") + dimensions: int | null # override dimensions (for models that support variable dimensions) + options: map # provider-specific options (e.g., base_url, api_version) +``` + +When `embedding` is `null` or omitted, the store operates in **pass-through mode**: callers MUST supply pre-computed embeddings for vector search, and keyword search remains available without embeddings. This preserves backward compatibility and supports use cases where the caller controls the embedding model. + +### 8.4 Automatic Embedding Behavior + +When a store has a configured embedding provider, the service MUST apply the following rules. These rules implement the client-side/server-side coexistence described in section 8.1.1: server-side embedding is the default, and client-provided embeddings are accepted as an opt-in override. + +#### 8.4.1 On Memory Write (`POST /v1/stores/{store_id}/memories`) + +1. **Client-side override:** If the request includes a non-null `embedding` field, use the caller-provided embedding as-is. The service MUST validate that the dimensionality matches the store's configured model and reject mismatches with `422 Unprocessable Entity`. +2. **Server-side default:** If the request omits `embedding` (or sets it to `null`) and a provider is configured, the service MUST extract text from `content` and compute an embedding before storing: + - For string content: embed the string directly. + - For structured (JSON object) content: serialize to a canonical text representation. Implementations SHOULD concatenate string-typed leaf values. Implementations MAY accept a `content_text_field` store config option to specify which field(s) to embed (e.g., `"text"`, `"body"`). +3. **Pass-through mode:** If the request omits `embedding` and no provider is configured (pass-through mode), the memory is stored without an embedding. Vector search will not return this entry, but keyword search will. +4. The computed or provided embedding MUST be stored alongside the memory entry and returned in subsequent reads (when the `embedding` field is requested). +5. Batch write operations (`POST /v1/stores/{store_id}/batch`) MUST apply the same rules per entry. Implementations SHOULD batch embedding calls to the provider for efficiency. + +#### 8.4.2 On Memory Update (`PATCH /v1/stores/{store_id}/memories/{memory_id}`) + +1. If the update modifies `content` and includes a new `embedding`, use the caller-provided embedding (client-side override). Validate dimensionality. +2. If the update modifies `content` but omits `embedding`, the service MUST recompute the embedding from the new content using the configured provider (server-side default). +3. If the update does not modify `content`, the existing embedding MUST be preserved regardless of whether `embedding` is present in the request. + +#### 8.4.3 On Search (`POST /v1/stores/{store_id}/search`) + +1. **Client-side override:** If the request includes a non-null `embedding` field, use it as the query vector for similarity search. The service does not embed the `query` string. +2. **Server-side default:** If the request omits `embedding`, the service MUST compute a query embedding from the `query` string using the provider's `embed_query` method before performing similarity search. +3. **Pass-through mode:** If the request omits `embedding` and no provider is configured, vector similarity search is not available. The service MUST fall back to keyword-only search if a `query` string is present, or reject the request with `400 Bad Request` if keyword search is also not applicable. +4. When `keyword: true` is also set, the service performs both keyword and vector search and merges results using the existing merge strategy (see section 5.3). + +This means a minimal search request — `{"query": "user preferences", "top_k": 10}` — performs full hybrid search (keyword + vector) when the store has an embedding provider and keyword search is available. No embedding knowledge is required on the caller side. + +### 8.5 Embedding Model Lifecycle + +#### 8.5.1 Model Versioning + +Each stored embedding SHOULD be tagged with the `model_id` that produced it. Implementations MAY store this as metadata on the memory entry or as a store-level field in the database schema. + +When a store's embedding model is changed (via `PATCH /v1/stores/{store_id}`), the service MUST NOT silently mix embeddings from different models in search results, as this produces meaningless similarity scores. Implementations MUST choose one of the following strategies: + +1. **Lazy re-embedding (recommended):** Mark all existing entries as needing re-embedding. Recompute embeddings in the background or on next read. Search results during the migration period may exclude stale entries or return them with degraded scores. +2. **Eager re-embedding:** Immediately recompute all embeddings in the store. This may be expensive for large stores but ensures instant consistency. +3. **Reject model change:** Refuse to change the embedding model on a store that contains entries. Require the caller to create a new store and migrate entries explicitly. + +Implementations MUST document which strategy they use. + +#### 8.5.2 Dimensionality Constraints + +All embeddings within a single store MUST have the same dimensionality. The dimensionality is determined by the store's configured embedding provider (or by the first caller-provided embedding if no provider is configured). Subsequent writes with mismatched dimensionality MUST be rejected with `422 Unprocessable Entity`. + +### 8.6 Built-in Provider Requirements + +Implementations at Level 1 conformance are NOT required to include any embedding provider — pass-through mode with keyword search is sufficient. + +Implementations at Level 2 and above SHOULD provide at least one built-in embedding provider or document how to configure an external one. Recommended provider categories: + +| Category | Examples | Trade-offs | +|---|---|---| +| **Local model** | ONNX Runtime, Sentence Transformers, FastEmbed | No network dependency, no API costs; requires CPU/GPU on the OMS host | +| **Remote API** | OpenAI, Azure OpenAI, Cohere, Voyager, Google Vertex AI | High quality, no local resources; adds latency and API costs | +| **Sidecar** | Ollama, vLLM, TEI (Text Embeddings Inference) | Decoupled scaling, GPU isolation; requires separate deployment | + +### 8.7 Capability Advertisement + +The `ProviderCapabilities` object (see section 9) MUST include embedding-related fields: + +```python +@dataclass +class EmbeddingCapabilities: + server_side_embedding: bool # true if the provider has a configured embedding provider + model_id: str | None # the active embedding model identifier + dimensions: int | None # dimensionality of embeddings produced + max_batch_size: int | None # maximum texts per embed_texts call (null = unlimited) + supports_query_prefix: bool # true if embed_query applies distinct preprocessing +``` + +This allows clients to discover which embedding strategy to use: + +```http +GET /v1/stores/{store_id} +→ { "capabilities": { "embedding": { "server_side_embedding": true, "model_id": "text-embedding-3-small", "dimensions": 1536 } } } +``` + +**Client behavior based on capabilities:** + +- `server_side_embedding: true` — Clients SHOULD omit the `embedding` field in requests and let the server handle vectorization. This is the simplest integration path and is required for framework adapters (LangChain, CrewAI, etc.) that do not manage embedding models. Clients MAY still provide embeddings for override purposes. +- `server_side_embedding: false` — Clients MUST supply their own embeddings for vector search, or restrict to keyword-only search. Framework adapters that cannot provide embeddings SHOULD document this limitation clearly. + +### 8.8 Reference Implementation (KD6) + +The KD6 reference implementation provides two embedding providers selected via the `KD6_EMBEDDING_PROVIDER` environment variable: + +| Provider | `KD6_EMBEDDING_PROVIDER` | Description | +|---|---|---| +| **Local (default)** | `local` | In-process ONNX inference via [fastembed-rs](https://github.com/Anush008/fastembed-rs). Default model: `all-MiniLM-L6-v2` (384 dimensions, ~25MB). Downloads on first use, cached thereafter. No API keys or external services required. | +| **OpenAI-compatible** | `openai-compatible` | Calls any endpoint implementing the OpenAI `/v1/embeddings` API: OpenAI, Azure OpenAI, Ollama, vLLM, LiteLLM, etc. | +| **None** | `none` | Pass-through mode. No embeddings are computed. Callers must supply embeddings in requests or use keyword-only search. | + +**Environment variables:** + +```bash +# Local provider (default — no configuration needed) +KD6_EMBEDDING_PROVIDER=local + +# OpenAI-compatible remote provider +KD6_EMBEDDING_PROVIDER=openai-compatible +KD6_EMBEDDING_ENDPOINT=https://api.openai.com/v1 # required +KD6_EMBEDDING_MODEL=text-embedding-3-small # required +KD6_EMBEDDING_API_KEY=sk-... # optional (not needed for Ollama) +KD6_EMBEDDING_DIMENSIONS=1536 # optional (default: 1536) + +# Disable embedding +KD6_EMBEDDING_PROVIDER=none +``` + +**Behavior:** +- On write: if the request omits `embedding`, the configured provider computes it from `content` before storing. Caller-provided embeddings are used as-is but validated for correct dimensionality. +- On search: if the request omits `embedding`, the provider computes a query embedding from `query` before performing vector similarity search. +- On update: if `content` changes and no new `embedding` is provided, the provider recomputes the embedding. +- Dimensionality mismatches between caller-provided embeddings and the configured model are rejected with `400 Bad Request`. + +### 8.9 Framework Integration Pattern + +The client-side/server-side embedding design enables a clean integration pattern for agent frameworks. Because the OMS service handles embedding, client adapters do not need an embedding model — they are thin HTTP clients that map framework APIs to OMS REST calls. + +**Example: LangChain VectorStore adapter** + +LangChain's `VectorStore` interface expects `add_texts(texts)` to handle vectorization and `similarity_search(query)` to return relevant documents. With server-side embedding, the adapter simply forwards text to the OMS API: + +```python +from langchain_core.vectorstores import VectorStore + +class KD6VectorStore(VectorStore): + def __init__(self, base_url="http://localhost:8080", embedding=None): + # embedding parameter is optional — server handles it by default + self._base_url = base_url + self._embedding = embedding # client-side override (optional) + + def add_texts(self, texts, metadatas=None, **kwargs): + entries = [{"content": text, ...} for text in texts] + # No embedding computation needed — KD6 does it server-side + return self._post("/memories/batch", {"entries": entries}) + + def similarity_search(self, query, k=4, **kwargs): + # Just send the query string — KD6 embeds and searches + return self._post("/search", {"query": query, "top_k": k}) +``` + +This pattern applies to any framework: the adapter maps the framework's text-based API to OMS REST calls, and the server handles embedding transparently. The optional `embedding` parameter allows advanced callers to provide pre-computed vectors when they need control over the embedding model. + +--- + +## 9. Backend Provider Interface (SPI) For customers who want to implement their own memory backend, the spec defines a **Service Provider Interface (SPI):** @@ -394,7 +723,11 @@ class OMSProvider(ABC): # --- Memory CRUD --- @abstractmethod - def put(self, store_id: str, entry: MemoryEntry) -> MemoryEntry: ... + def put(self, store_id: str, entry: MemoryEntry) -> MemoryEntry: + """Create a memory entry. If entry.upsert_key is set and a matching + entry exists in the same store, layer, and scope, the existing entry + is replaced atomically (see 4.3.2).""" + ... @abstractmethod def get(self, store_id: str, memory_id: str) -> MemoryEntry: ... @abstractmethod @@ -445,6 +778,7 @@ class ProviderCapabilities: pub_sub_notifications: bool # supports real-time change notifications encryption_at_rest: bool audit_log: bool + embedding: EmbeddingCapabilities | None # server-side embedding support (see section 8) ``` This enables: @@ -455,7 +789,7 @@ This enables: --- -## 9. Data Sovereignty +## 10. Data Sovereignty Data sovereignty is a first-class concern in the OMS spec: @@ -482,7 +816,7 @@ SovereigntyConfig: --- -## 10. Conformance Levels +## 11. Conformance Levels To enable incremental adoption and community contribution, the spec defines three conformance levels: @@ -497,6 +831,15 @@ An implementation at this level provides basic memory functionality: - Tenant isolation via scope (at minimum, `tenant_id` enforcement) - Authentication (OAuth 2.1 bearer token) - Capabilities discovery endpoint +- Plain string content support (see 4.3.1) +- Upsert semantics via `upsert_key` (see 4.3.2) + +**Optional at Level 1:** +- Default store alias `_default` with auto-provisioning (see 4.1.1) +- Default tenant resolution (see 4.4.1) +- Server-side embedding via a configured `EmbeddingProvider` (see section 8). When not configured, callers must supply pre-computed embeddings for vector search. Keyword-only search remains available without an embedding provider. + +These optional features lower the adoption barrier for single-agent and development scenarios. When both are enabled, an agent's first memory write requires zero setup calls. Implementations that support them MUST document their security implications and provide configuration to disable them in production. **Estimated implementation effort:** 2–4 weeks for a team familiar with vector databases. @@ -512,6 +855,7 @@ All of Level 1, plus: - Keyword/BM25 search - Batch operations - Hierarchical scoping (at minimum: tenant → agent → session) +- Server-side embedding with at least one built-in or configurable `EmbeddingProvider` (see section 8) **Estimated implementation effort:** 2–3 months. @@ -533,7 +877,7 @@ All of Level 2, plus: --- -## 11. Reference Implementation Guidance +## 12. Reference Implementation Guidance A reference implementation targeting Azure SHOULD use the following technology mapping: @@ -546,12 +890,12 @@ A reference implementation targeting Azure SHOULD use the following technology m | Archival | MinIO / S3 | Azure Blob Storage | Lifecycle policies for cost optimization | | Metadata / Config | PostgreSQL | Cosmos DB for PostgreSQL | Tenant registry, store config | | Audit Log | Append-only log | Azure Event Hubs → Data Explorer | Partitioned by tenant | -| Embedding | Local model (e.g., Sentence Transformers) | Azure OpenAI (text-embedding-3-*) | Configurable per store | +| Embedding | FastEmbed (ONNX) / Local model | Azure OpenAI (text-embedding-3-*) | Local default; remote override (see section 8.8) | | Tenant Orchestration | Kubernetes + Helm | AKS (namespace-per-tenant) | Tiered isolation model | --- -## 12. Relationship to Existing Work +## 13. Relationship to Existing Work The OMS spec draws from and complements existing work: @@ -570,7 +914,7 @@ The OMS spec draws from and complements existing work: --- -## 13. References +## 14. References [1] Tulving, E. (1972). "Episodic and Semantic Memory." In E. Tulving & W. Donaldson (Eds.), *Organization of Memory* (pp. 381–403). Academic Press. diff --git a/validation/squad-kd6-provider.ts b/validation/squad-kd6-provider.ts new file mode 100644 index 0000000..f3fa0fe --- /dev/null +++ b/validation/squad-kd6-provider.ts @@ -0,0 +1,327 @@ +/** + * Kd6MemoryProvider — Squad MemoryProvider backed by KD6's HTTP API. + * + * This file validates that Squad can integrate with KD6 using the + * zero-setup API (default tenant, default store, upsert, plain strings). + * + * Usage: + * npx tsx validation/squad-kd6-provider.ts + * + * Requires a running KD6 server at http://127.0.0.1:18080 + */ + +// ── Squad types (inlined to avoid dependency) ────────────────────────────── + +type MemoryClass = 'TRANSIENT' | 'LOCAL' | 'DECISION' | 'POLICY' | 'COPILOT_MEMORY' | 'FORBIDDEN'; +type MemoryLoadGuidance = 'ALWAYS' | 'ON-DEMAND' | 'ARCHIVE' | 'NEVER'; + +interface MemoryClassification { + class: MemoryClass; + allowed: boolean; + reason: string; + destination: 'none' | 'local' | 'decision-inbox' | 'policy-inbox' | 'external-semantic'; + loadGuidance: MemoryLoadGuidance; +} + +interface CopilotMemoryProviderWriteRequest { + content: string; + title: string; + author?: string; + metadata?: Record; + classification: MemoryClassification; +} + +interface CopilotMemoryProviderWriteResult { + id: string; + path?: string; +} + +interface MemoryProviderSearchResult { + id: string; + title: string; + snippet: string; + path?: string; + class: MemoryClass; + loadGuidance: MemoryLoadGuidance; + score?: number; +} + +interface MemoryProviderStatus { + id: string; + name: string; + available: boolean; + reason?: string; +} + +interface MemoryProvider { + readonly id: string; + readonly name: string; + readonly supportedClasses: ReadonlyArray; + status(): Promise; + write(request: CopilotMemoryProviderWriteRequest): Promise; + search(query: string): Promise; + delete(id: string): Promise; +} + +// ── KD6 layer mapping ────────────────────────────────────────────────────── + +const CLASS_TO_LAYER: Record = { + LOCAL: 'episodic', + DECISION: 'semantic', + POLICY: 'procedural', +}; + +const LAYER_TO_CLASS: Record = { + episodic: 'LOCAL', + semantic: 'DECISION', + procedural: 'POLICY', + working: 'LOCAL', +}; + +const LAYER_TO_GUIDANCE: Record = { + episodic: 'ON-DEMAND', + semantic: 'ALWAYS', + procedural: 'ALWAYS', + working: 'ON-DEMAND', +}; + +// ── Kd6MemoryProvider ────────────────────────────────────────────────────── + +class Kd6MemoryProvider implements MemoryProvider { + readonly id = 'kd6'; + readonly name = 'KD6 Open Memory Service'; + readonly supportedClasses: ReadonlyArray = ['LOCAL', 'DECISION', 'POLICY']; + + constructor( + private readonly baseUrl: string = 'http://127.0.0.1:18080', + private readonly store: string = '_default', + ) {} + + async status(): Promise { + try { + const res = await fetch(`${this.baseUrl}/health`); + const body = await res.json() as { status: string }; + return { + id: this.id, + name: this.name, + available: body.status === 'ok', + }; + } catch (err) { + return { + id: this.id, + name: this.name, + available: false, + reason: String(err), + }; + } + } + + async write(request: CopilotMemoryProviderWriteRequest): Promise { + const layer = CLASS_TO_LAYER[request.classification.class] ?? 'working'; + const upsertKey = request.metadata?.['upsert_key']; + + const body: Record = { + content: request.content, + owner_agent_id: request.author ?? 'squad', + layer, + scope: {}, + tags: [request.classification.class.toLowerCase()], + }; + if (upsertKey) body.upsert_key = upsertKey; + + const res = await fetch(`${this.baseUrl}/v1/stores/${this.store}/memories`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify(body), + }); + + if (!res.ok) { + const text = await res.text(); + throw new Error(`KD6 write failed (${res.status}): ${text}`); + } + + const entry = await res.json() as { id: string }; + return { + id: entry.id, + path: `kd6:${this.store}:${entry.id}`, + }; + } + + async search(query: string): Promise { + const res = await fetch(`${this.baseUrl}/v1/stores/${this.store}/search`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ query, keyword: true, top_k: 20 }), + }); + + if (!res.ok) return []; + + const results = await res.json() as Array<{ + entry: { + id: string; + layer: string; + content: string; + tags: string[]; + }; + score: number; + }>; + + return results.map(r => ({ + id: r.entry.id, + title: r.entry.tags[0] ?? r.entry.layer, + snippet: typeof r.entry.content === 'string' + ? r.entry.content.slice(0, 240) + : JSON.stringify(r.entry.content).slice(0, 240), + path: `kd6:${this.store}:${r.entry.id}`, + class: LAYER_TO_CLASS[r.entry.layer] ?? 'LOCAL', + loadGuidance: LAYER_TO_GUIDANCE[r.entry.layer] ?? 'ON-DEMAND', + score: r.score, + })); + } + + async delete(id: string): Promise { + const res = await fetch(`${this.baseUrl}/v1/stores/${this.store}/memories/${id}`, { + method: 'DELETE', + }); + return res.ok; + } +} + +// ── Validation tests ─────────────────────────────────────────────────────── + +async function run() { + const provider = new Kd6MemoryProvider(); + let passed = 0; + let failed = 0; + + function assert(condition: boolean, msg: string) { + if (condition) { + console.log(` ✅ ${msg}`); + passed++; + } else { + console.log(` ❌ ${msg}`); + failed++; + } + } + + // 1. Status check + console.log('\n📋 Test: status()'); + const status = await provider.status(); + assert(status.available, 'KD6 is available'); + assert(status.id === 'kd6', 'Provider ID is kd6'); + + // 2. Write LOCAL memory + console.log('\n📋 Test: write LOCAL memory'); + const localResult = await provider.write({ + content: 'The project uses Next.js 14 with App Router', + title: 'Framework choice', + author: 'squad-architect', + classification: { + class: 'LOCAL', + allowed: true, + reason: 'episodic context', + destination: 'local', + loadGuidance: 'ON-DEMAND', + }, + }); + assert(!!localResult.id, `Got memory ID: ${localResult.id}`); + assert(localResult.path?.startsWith('kd6:'), `Path starts with kd6: ${localResult.path}`); + + // 3. Write DECISION memory + console.log('\n📋 Test: write DECISION memory'); + const decisionResult = await provider.write({ + content: 'Use Tailwind CSS for styling instead of CSS modules', + title: 'Styling decision', + author: 'squad-ux-agent', + classification: { + class: 'DECISION', + allowed: true, + reason: 'architectural decision', + destination: 'decision-inbox', + loadGuidance: 'ALWAYS', + }, + }); + assert(!!decisionResult.id, `Got DECISION ID: ${decisionResult.id}`); + + // 4. Write POLICY memory + console.log('\n📋 Test: write POLICY memory'); + const policyResult = await provider.write({ + content: 'All API endpoints must validate input with zod schemas', + title: 'Validation policy', + author: 'squad-security', + classification: { + class: 'POLICY', + allowed: true, + reason: 'security policy', + destination: 'policy-inbox', + loadGuidance: 'ALWAYS', + }, + }); + assert(!!policyResult.id, `Got POLICY ID: ${policyResult.id}`); + + // 5. Search + console.log('\n📋 Test: search()'); + const searchResults = await provider.search('Tailwind CSS'); + assert(searchResults.length > 0, `Found ${searchResults.length} results for "Tailwind CSS"`); + const tailwind = searchResults.find(r => r.snippet.includes('Tailwind')); + assert(!!tailwind, 'Found the Tailwind decision'); + assert(tailwind?.class === 'DECISION', `Mapped class is DECISION (got ${tailwind?.class})`); + assert(tailwind?.loadGuidance === 'ALWAYS', `Guidance is ALWAYS (got ${tailwind?.loadGuidance})`); + + // 6. Upsert (supersede pattern) + console.log('\n📋 Test: upsert (supersede)'); + const v1 = await provider.write({ + content: 'Deploy to Azure App Service', + title: 'Deploy target', + author: 'squad-devops', + metadata: { upsert_key: 'deploy-target' }, + classification: { + class: 'DECISION', + allowed: true, + reason: 'deployment decision', + destination: 'decision-inbox', + loadGuidance: 'ALWAYS', + }, + }); + + const v2 = await provider.write({ + content: 'Deploy to Azure Container Apps instead', + title: 'Deploy target', + author: 'squad-devops', + metadata: { upsert_key: 'deploy-target' }, + classification: { + class: 'DECISION', + allowed: true, + reason: 'deployment decision updated', + destination: 'decision-inbox', + loadGuidance: 'ALWAYS', + }, + }); + + assert(v1.id === v2.id, `Upsert reused same ID: ${v1.id} === ${v2.id}`); + + const deploySearch = await provider.search('Container Apps'); + const containerEntry = deploySearch.find(r => r.snippet.includes('Container Apps')); + assert(!!containerEntry, 'Upserted content is searchable'); + + // 7. Delete + console.log('\n📋 Test: delete()'); + const deleted = await provider.delete(policyResult.id); + assert(deleted, `Deleted policy memory ${policyResult.id}`); + + const searchAfterDelete = await provider.search('zod schemas'); + const stillExists = searchAfterDelete.find(r => r.id === policyResult.id); + assert(!stillExists, 'Deleted memory no longer appears in search'); + + // Summary + console.log(`\n${'═'.repeat(50)}`); + console.log(`Results: ${passed} passed, ${failed} failed`); + console.log(`${'═'.repeat(50)}\n`); + + process.exit(failed > 0 ? 1 : 0); +} + +run().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +});