diff --git a/.cargo/config.toml b/.cargo/config.toml index 357bc0fa3..a4264eb67 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -40,9 +40,8 @@ rustflags = [] [build] incremental = true -rustc-wrapper = "sccache" # Parallel compilation will use default (all available cores) # Use sparse registry protocol for faster dependency resolution [registries.crates-io] -protocol = "sparse" \ No newline at end of file +protocol = "sparse" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 4d1a582e2..35c5a3a1d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -1381,6 +1381,8 @@ jobs: echo "✅ Multi-arch manifest created for ${REPO}:${VERSION}, ${REPO}:${VERSION_COMMIT_TAG}, and ${REPO}:latest" - name: Update Docker Hub repository README + id: dockerhub_readme + continue-on-error: true uses: peter-evans/dockerhub-description@v4 with: username: ${{ secrets.DOCKERHUB_USERNAME }} @@ -1388,6 +1390,12 @@ jobs: repository: ${{ steps.vars.outputs.docker_repo }} readme-filepath: ./docker/REPO-README.md + - name: Warn if Docker Hub README update failed + if: steps.dockerhub_readme.outcome == 'failure' + shell: bash + run: | + echo "::warning::Docker image publish succeeded, but Docker Hub README update failed. Ensure DOCKERHUB_USERNAME is a repo admin and DOCKERHUB_TOKEN has read/write/delete scope." + # ═══════════════════════════════════════════════════════════════════════════ # INTEGRATION TESTS - Smoke + TypeScript SDK + Dart SDK against Docker image # ═══════════════════════════════════════════════════════════════════════════ diff --git a/Cargo.lock b/Cargo.lock index 680d5bdd4..7fc98b929 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -443,9 +443,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.21" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" +checksum = "824a212faf96e9acacdbd09febd34438f8f711fb84e09a8916013cd7815ca28d" dependencies = [ "anstyle", "anstyle-parse", @@ -464,9 +464,9 @@ checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.7" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" +checksum = "52ce7f38b242319f7cabaa6813055467063ecdc9d355bbb4ce0c68908cd8130e" dependencies = [ "utf8parse", ] @@ -515,15 +515,6 @@ dependencies = [ "object", ] -[[package]] -name = "arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" -dependencies = [ - "derive_arbitrary", -] - [[package]] name = "arrayvec" version = "0.7.6" @@ -754,9 +745,9 @@ dependencies = [ [[package]] name = "assert_cmd" -version = "2.1.2" +version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c5bcfa8749ac45dd12cb11055aeeb6b27a3895560d60d71e3c23bf979e60514" +checksum = "9a686bbee5efb88a82df0621b236e74d925f470e5445d3220a5648b892ec99c9" dependencies = [ "anstyle", "bstr", @@ -1136,9 +1127,9 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "cc" -version = "1.2.56" +version = "1.2.57" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aebf35691d1bfb0ac386a69bac2fde4dd276fb618cf8bf4f5318fe285e821bb2" +checksum = "7a0dd1ca384932ff3641c8718a02769f1698e7563dc6974ffd03346116310423" dependencies = [ "find-msvc-tools", "jobserver", @@ -1257,9 +1248,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2797f34da339ce31042b27d23607e051786132987f595b02ba4f6a6dffb7030a" +checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" dependencies = [ "clap_builder", "clap_derive", @@ -1267,9 +1258,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.5.60" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24a241312cea5059b13574bb9b3861cabf758b879c15190b37b6d6fd63ab6876" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" dependencies = [ "anstream", "anstyle", @@ -1279,9 +1270,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.5.55" +version = "4.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a92793da1a46a5f2a02a6f4c46c6496b28c43638adea8306fcb0caa1634f24e5" +checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" dependencies = [ "heck", "proc-macro2", @@ -1796,9 +1787,9 @@ checksum = "d7a1e2f27636f116493b8b860f5546edb47c8d8f8ea73e1d2a20be88e28d1fea" [[package]] name = "datafusion" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "503f1f4a9060ae6e650d3dff5dc7a21266fea1302d890768d45b4b28586e830f" +checksum = "ea28305c211e3541c9cfcf06a23d0d8c7c824b4502ed1fdf0a6ff4ad24ee531c" dependencies = [ "arrow", "arrow-schema", @@ -1847,9 +1838,9 @@ dependencies = [ [[package]] name = "datafusion-catalog" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14417a3ee4ae3d092b56cd6c1d32e8ff3e2c9ec130ecb2276ec91c89fd599399" +checksum = "78ab99b6df5f60a6ddbc515e4c05caee1192d395cf3cb67ce5d1c17e3c9b9b74" dependencies = [ "arrow", "async-trait", @@ -1872,9 +1863,9 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9d0eba824adb45a4b3ac6f0251d40df3f6a9382371cad136f4f14ac9ebc6bc10" +checksum = "77ae3d14912c0d779ada98d30dc60f3244f3c26c2446b87394629ea5c076a31c" dependencies = [ "arrow", "async-trait", @@ -1895,9 +1886,9 @@ dependencies = [ [[package]] name = "datafusion-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0039deefbd00c56adf5168b7ca58568fb058e4ba4c5a03b09f8be371b4e434b6" +checksum = "ea2df29b9592a5d55b8238eaf67d2f21963d5a08cd1a8b7670134405206caabd" dependencies = [ "ahash 0.8.12", "arrow", @@ -1919,9 +1910,9 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1ec7e3e60b813048331f8fb9673583173e5d2dd8fef862834ee871fc98b57ca7" +checksum = "42639baa0049d5fffd7e283504b9b5e7b9b2e7a2dea476eed60ab0d40d999b85" dependencies = [ "futures", "log", @@ -1930,9 +1921,9 @@ dependencies = [ [[package]] name = "datafusion-datasource" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "802068957f620302ecf05f84ff4019601aeafd36f5f3f1334984af2e34265129" +checksum = "25951b617bb22a9619e1520450590cb2004bfcad10bcb396b961f4a1a10dcec5" dependencies = [ "arrow", "async-trait", @@ -1959,9 +1950,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90fc387d5067c62d494a6647d29c5ad4fcdd5a6e50ab4ea1d2568caa2d66f2cc" +checksum = "dc0b28226960ba99c50d78ac6f736ebe09eb5cb3bb9bb58194266278000ca41f" dependencies = [ "arrow", "arrow-ipc", @@ -1983,9 +1974,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "efd5e20579bb6c8bd4e6c620253972fb723822030c280dd6aa047f660d09eeba" +checksum = "f538b57b052a678b1ce860181c65d3ace5a8486312dc50b41c01dd585a773a51" dependencies = [ "arrow", "async-trait", @@ -2006,9 +1997,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0788b0d48fcef31880a02013ea3cc18e5a4e0eacc3b0abdd2cd0597b99dc96e" +checksum = "89fbc1d32b1b03c9734e27c0c5f041232b68621c8455f22769838634750a196c" dependencies = [ "arrow", "async-trait", @@ -2028,9 +2019,9 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66639b70f1f363f5f0950733170100e588f1acfacac90c1894e231194aa35957" +checksum = "203271d31fe5613a5943181db70ec98162121d1de94a9a300d5e5f19f9500a32" dependencies = [ "arrow", "async-trait", @@ -2058,15 +2049,15 @@ dependencies = [ [[package]] name = "datafusion-doc" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e44b41f3e8267c6cf3eec982d63f34db9f1dd5f30abfd2e1f124f0871708952e" +checksum = "5b6450dc702b3d39e8ced54c3356abb453bd2f3cea86d90d555a4b92f7a38462" [[package]] name = "datafusion-execution" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e456f60e5d38db45335e84617006d90af14a8c8c5b8e959add708b2daaa0e2c" +checksum = "e66a02fa601de49da5181dbdcf904a18b16a184db2b31f5e5534552ea2d5e660" dependencies = [ "arrow", "async-trait", @@ -2085,9 +2076,9 @@ dependencies = [ [[package]] name = "datafusion-expr" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6507c719804265a58043134580c1c20767e7c23ba450724393f03ec982769ad9" +checksum = "cdf59a9b308a1a07dc2eb2f85e6366bc0226dc390b40f3aa0a72d79f1cfe2465" dependencies = [ "arrow", "async-trait", @@ -2108,9 +2099,9 @@ dependencies = [ [[package]] name = "datafusion-expr-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a413caa9c5885072b539337aed68488f0291653e8edd7d676c92df2480f6cab0" +checksum = "bd99eac4c6538c708638db43e7a3bd88e0e57955ddb722d420fb9a6d38dfc28f" dependencies = [ "arrow", "datafusion-common", @@ -2121,9 +2112,9 @@ dependencies = [ [[package]] name = "datafusion-functions" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "189256495dc9cbbb8e20dbcf161f60422e628d201a78df8207e44bd4baefadb6" +checksum = "11aa2c492ac046397b36d57c62a72982aad306495bbcbcdbcabd424d4a2fe245" dependencies = [ "arrow", "arrow-buffer", @@ -2148,9 +2139,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12e73dfee4cd67c4a507ffff4c5a711d39983adf544adbc09c09bf06f789f413" +checksum = "325a00081898945d48d6194d9ca26120e523c993be3bb7c084061a5a2a72e787" dependencies = [ "ahash 0.8.12", "arrow", @@ -2169,9 +2160,9 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87727bd9e65f4f9ac6d608c9810b7da9eaa3b18b26a4a4b76520592d49020acf" +checksum = "809bbcb1e0dbec5d0ce30d493d135aea7564f1ba4550395f7f94321223df2dae" dependencies = [ "ahash 0.8.12", "arrow", @@ -2182,9 +2173,9 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e5ef761359224b7c2b5a1bfad6296ac63225f8583d08ad18af9ba1a89ac3887" +checksum = "29ebaa5d7024ef45973e0a7db1e9aeaa647936496f4d4061c0448f23d77d6320" dependencies = [ "arrow", "arrow-ord", @@ -2205,9 +2196,9 @@ dependencies = [ [[package]] name = "datafusion-functions-table" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b17dac25dfda2d2a90ff0ad1c054a11fb1523766226bec6e9bd8c410daee2ae" +checksum = "60eab6f39df9ee49a2c7fa38eddc01fa0086ee31b29c7d19f38e72f479609752" dependencies = [ "arrow", "async-trait", @@ -2221,9 +2212,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c594a29ddb22cbdbce500e4d99b5b2392c5cecb4c1086298b41d1ffec14dbb77" +checksum = "e00b2c15e342a90e65a846199c9e49293dd09fe1bcd63d8be2544604892f7eb8" dependencies = [ "arrow", "datafusion-common", @@ -2239,9 +2230,9 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9aa1b15ed81c7543f62264a30dd49dec4b1b0b698053b968f53be32dfba4f729" +checksum = "493e2e1d1f4753dfc139a5213f1b5d0b97eea46a82d9bda3c7908aa96981b74b" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2249,9 +2240,9 @@ dependencies = [ [[package]] name = "datafusion-macros" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c00c31c4795597aa25b74cab5174ac07a53051f27ce1e011ecaffa9eaeecef81" +checksum = "ba01c55ade8278a791b429f7bf5cb1de64de587a342d084b18245edfae7096e2" dependencies = [ "datafusion-doc", "quote", @@ -2260,9 +2251,9 @@ dependencies = [ [[package]] name = "datafusion-optimizer" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80ccf60767c09302b2e0fc3afebb3761a6d508d07316fab8c5e93312728a21bb" +checksum = "a80c6dfbba6a2163a9507f6353ac78c69d8deb26232c9e419160e58ff7c3e047" dependencies = [ "arrow", "chrono", @@ -2280,9 +2271,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64b7f277556944e4edd3558da01d9e9ff9f5416f1c0aa7fee088e57bd141a7e" +checksum = "5d3a86264bb9163e7360b6622e789bc7fcbb43672e78a8493f0bc369a41a57c6" dependencies = [ "ahash 0.8.12", "arrow", @@ -2304,9 +2295,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7abaee372ea2d19c016ee9ef8629c4415257d291cdd152bc7f0b75f28af1b63" +checksum = "3f5e00e524ac33500be6c5eeac940bd3f6b984ba9b7df0cd5f6c34a8a2cc4d6b" dependencies = [ "arrow", "datafusion-common", @@ -2319,9 +2310,9 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42237efe621f92adc22d111b531fdbc2cc38ca9b5e02327535628fb103ae2157" +checksum = "2ae769ea5d688b4e74e9be5cad6f9d9f295b540825355868a3ab942380dd97ce" dependencies = [ "ahash 0.8.12", "arrow", @@ -2336,9 +2327,9 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fd093498bd1319c6e5c76e9dfa905e78486f01b34579ce97f2e3a49f84c37fac" +checksum = "f3588753ab2b47b0e43cd823fe5e7944df6734dabd6dafb72e2cc1c2a22f1944" dependencies = [ "arrow", "datafusion-common", @@ -2355,9 +2346,9 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cbe61b12daf81a9f20ba03bd3541165d51f86e004ef37426b11881330eed261" +checksum = "79949cbb109c2a45c527bfe0d956b9f2916807c05d4d2e66f3fd0af827ac2b61" dependencies = [ "ahash 0.8.12", "arrow", @@ -2386,9 +2377,9 @@ dependencies = [ [[package]] name = "datafusion-pruning" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0124331116db7f79df92ebfd2c3b11a8f90240f253555c9bb084f10b6fecf1dd" +checksum = "6434e2ee8a39d04b95fed688ff34dc251af6e4a0c2e1714716b6e3846690d589" dependencies = [ "arrow", "datafusion-common", @@ -2403,9 +2394,9 @@ dependencies = [ [[package]] name = "datafusion-session" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1673e3c58ba618a6ea0568672f00664087b8982c581e9afd5aa6c3c79c9b431f" +checksum = "c91efb8302b4877d499c37e9a71886b90236ab27d9cc42fd51112febf341abd6" dependencies = [ "async-trait", "datafusion-common", @@ -2417,9 +2408,9 @@ dependencies = [ [[package]] name = "datafusion-sql" -version = "52.2.0" +version = "52.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5272d256dab5347bb39d2040589f45d8c6b715b27edcb5fffe88cc8b9c3909cb" +checksum = "3f01eef7bcf4d00e87305b55f1b75792384e130fe0258bac02cd48378ae5ff87" dependencies = [ "arrow", "bigdecimal", @@ -2453,17 +2444,6 @@ dependencies = [ "powerfmt", ] -[[package]] -name = "derive_arbitrary" -version = "1.4.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.117", -] - [[package]] name = "derive_more" version = "0.99.20" @@ -3656,7 +3636,7 @@ dependencies = [ "kalamdb-configs", "kalamdb-server", "log", - "miniz_oxide 0.9.0", + "miniz_oxide 0.9.1", "reqwest 0.13.2", "serde", "serde-wasm-bindgen", @@ -4527,9 +4507,9 @@ dependencies = [ [[package]] name = "miniz_oxide" -version = "0.9.0" +version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5faa9f23e86bd5768d76def086192ff5f869fb088da12a976ea21e9796b975f6" +checksum = "b63fbc4a50860e98e7b2aa7804ded1db5cbc3aff9193adaff57a6931bf7c4b4c" dependencies = [ "adler2", ] @@ -4746,18 +4726,18 @@ dependencies = [ [[package]] name = "objc2-core-foundation" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c10c2894a6fed806ade6027bcd50662746363a9589d3ec9d9bef30a4e4bc166" +checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.11.0", ] [[package]] name = "objc2-io-kit" -version = "0.3.1" +version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "71c1c64d6120e51cd86033f67176b1cb66780c2efe34dec55176f77befd93c0a" +checksum = "33fafba39597d6dc1fb709123dfa8289d39406734be322956a69f0931c73bb15" dependencies = [ "libc", "objc2-core-foundation", @@ -4836,9 +4816,9 @@ dependencies = [ [[package]] name = "once_cell" -version = "1.21.3" +version = "1.21.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +checksum = "9f7c3e4beb33f85d45ae3e3a1792185706c8e16d043238c593331cc7cd313b50" [[package]] name = "once_cell_polyfill" @@ -6519,9 +6499,9 @@ dependencies = [ [[package]] name = "sysinfo" -version = "0.38.3" +version = "0.38.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d03c61d2a49c649a15c407338afe7accafde9dac869995dccb73e5f7ef7d9034" +checksum = "92ab6a2f8bfe508deb3c6406578252e491d299cbbf3bc0529ecc3313aee4a52f" dependencies = [ "libc", "memchr", @@ -6545,9 +6525,9 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.26.0" +version = "3.27.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "82a72c767771b47409d2345987fda8628641887d5466101319899796367354a0" +checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", "getrandom 0.4.2", @@ -7087,9 +7067,9 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.22" +version = "0.3.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f30143827ddab0d256fd843b7a66d164e9f271cfa0dde49142c5ca0ca291f1e" +checksum = "cb7f578e5945fb242538965c2d0b04418d38ec25c79d160cd279bf0731c8d319" dependencies = [ "matchers", "nu-ansi-term", @@ -7161,6 +7141,12 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" +[[package]] +name = "typed-path" +version = "0.12.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e28f89b80c87b8fb0cf04ab448d5dd0dd0ade2f8891bae878de66a75a28600e" + [[package]] name = "typenum" version = "1.19.0" @@ -8142,15 +8128,15 @@ dependencies = [ [[package]] name = "zip" -version = "4.6.1" +version = "8.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "caa8cd6af31c3b31c6631b8f483848b91589021b28fffe50adada48d4f4d2ed1" +checksum = "b680f2a0cd479b4cff6e1233c483fdead418106eae419dc60200ae9850f6d004" dependencies = [ - "arbitrary", "crc32fast", "flate2", "indexmap", "memchr", + "typed-path", "zopfli", ] diff --git a/Cargo.toml b/Cargo.toml index 9b71d6f29..a134c6463 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -70,13 +70,13 @@ rocksdb = { version = "0.24.0", default-features = false, features = ["snappy", # Apache Arrow ecosystem # IMPORTANT: arrow, arrow-schema and parquet MUST match the version used by DataFusion. -# DataFusion 52.x uses arrow 57.x — using a different major version causes duplicate -# arrow_array crates in the binary and type-mismatch compilation errors. -# To upgrade arrow, first upgrade DataFusion to a version that uses the desired arrow version. +# DataFusion 52.3.0 resolves to Arrow 57.3.0; mixing in Arrow 58 pulls a second +# arrow_array crate into the graph and causes FixedSizeListArray type mismatches. +# Upgrade DataFusion first before bumping Arrow/Parquet beyond this family. arrow = { version = "57.3.0", default-features = false, features = ["prettyprint", "json"] } arrow-schema = { version = "57.3.0" } -datafusion = { version = "52.2.0", default-features = false, features = ["sql", "parquet", "recursive_protection", "nested_expressions"] } -datafusion-common = { version = "52.2.0" } +datafusion = { version = "52.3.0", default-features = false, features = ["sql", "parquet", "recursive_protection", "nested_expressions"] } +datafusion-common = { version = "52.3.0" } sqlparser = { version = "0.61.0" } parquet = { version = "57.3.0", default-features = false, features = ["snap", "zstd", "arrow"] } @@ -105,16 +105,16 @@ toml = "0.9.8" dotenv = "0.15" # CLI tools -clap = { version = "4.5.60", features = ["derive", "color"] } +clap = { version = "4.6.0", features = ["derive", "color"] } rustyline = { version = "17.0.2" } # System num_cpus = "1.16" -sysinfo = "0.38.2" +sysinfo = "0.38.4" hostname = "0.4" # Testing -tempfile = "3.25.0" +tempfile = "3.27.0" wait-timeout = "0.2" serial_test = "3.3.1" @@ -144,20 +144,20 @@ rpassword = "7.3" dirs = "6.0" term_size = "0.3" crossterm = "0.29.0" -assert_cmd = "2.1.2" +assert_cmd = "2.2.0" predicates = "3.1.4" futures-util = { version = "0.3.32", default-features = false } base64 = "0.22" flatbuffers = "25.12.19" flexbuffers = "25.9.23" async-trait = "0.1.74" -once_cell = "1.20" +once_cell = "1.21.4" regex = "1.12.3" dashmap = "6.1" bytes = "1.11.1" http-body = "1.0.0" http-body-util = "0.1.2" -cc = "1.2.56" +cc = "1.2.57" proc-macro2 = "1.0.106" quote = "1.0.44" syn = { version = "2.0.117", features = ["full", "extra-traits"] } @@ -188,8 +188,8 @@ tsify-next = { version = "0.5", default-features = false, features = ["js"] } serde-wasm-bindgen = "0.6" flate2 = "1.1.9" tar = "0.4" -zip = { version = "4.2", default-features = false, features = ["deflate"] } -miniz_oxide = "0.9.0" +zip = { version = "8.2.0", default-features = false, features = ["deflate"] } +miniz_oxide = "0.9.1" flutter_rust_bridge = "=2.11.1" rust-embed = { version = "8.11.0", features = ["compression"] } mime_guess = "2.0" @@ -198,7 +198,7 @@ kalamdb-server = { path = "backend" } # Tracing / observability tracing = "0.1" -tracing-subscriber = { version = "0.3", features = ["env-filter", "json", "fmt"] } +tracing-subscriber = { version = "0.3.23", features = ["env-filter", "json", "fmt"] } tracing-actix-web = "0.7.21" tracing-log = "0.2" tracing-opentelemetry = "0.32.1" diff --git a/backend/crates/kalamdb-auth/src/helpers/ip_extractor.rs b/backend/crates/kalamdb-auth/src/helpers/ip_extractor.rs index af0446fb9..329a92355 100644 --- a/backend/crates/kalamdb-auth/src/helpers/ip_extractor.rs +++ b/backend/crates/kalamdb-auth/src/helpers/ip_extractor.rs @@ -43,16 +43,16 @@ static TRUSTED_PROXY_RANGES: Lazy>> = Lazy::new(|| RwLock::new /// ``` pub fn init_trusted_proxy_ranges(entries: &[String]) -> anyhow::Result<()> { let parsed = kalamdb_configs::parse_trusted_proxy_entries(entries)?; - *TRUSTED_PROXY_RANGES - .write() - .expect("trusted proxy ranges lock poisoned") = parsed; + *TRUSTED_PROXY_RANGES.write().expect("trusted proxy ranges lock poisoned") = parsed; Ok(()) } -pub fn extract_client_ip_addr_secure(peer_addr: Option, headers: &HeaderMap) -> Option { - let trusted_proxy_ranges = TRUSTED_PROXY_RANGES - .read() - .expect("trusted proxy ranges lock poisoned"); +pub fn extract_client_ip_addr_secure( + peer_addr: Option, + headers: &HeaderMap, +) -> Option { + let trusted_proxy_ranges = + TRUSTED_PROXY_RANGES.read().expect("trusted proxy ranges lock poisoned"); extract_client_ip_addr_with_trusted_ranges(peer_addr, headers, &trusted_proxy_ranges) } @@ -68,7 +68,9 @@ fn extract_client_ip_addr_with_trusted_ranges( trusted_proxy_ranges: &[IpNet], ) -> Option { if peer_addr.is_some_and(|ip| is_trusted_proxy_peer(ip, trusted_proxy_ranges)) { - if let Some(ip) = extract_proxy_header_ip(headers.get("X-Forwarded-For"), true, "X-Forwarded-For") { + if let Some(ip) = + extract_proxy_header_ip(headers.get("X-Forwarded-For"), true, "X-Forwarded-For") + { return Some(ip); } @@ -76,10 +78,7 @@ fn extract_client_ip_addr_with_trusted_ranges( return Some(ip); } } else if headers.contains_key("X-Forwarded-For") || headers.contains_key("X-Real-IP") { - warn!( - "Security: Ignoring proxy headers from untrusted peer {:?}", - peer_addr - ); + warn!("Security: Ignoring proxy headers from untrusted peer {:?}", peer_addr); } peer_addr @@ -215,10 +214,8 @@ mod tests { HeaderName::from_static("x-forwarded-for"), HeaderValue::from_static("203.0.113.8"), ); - let trusted_ranges = kalamdb_configs::parse_trusted_proxy_entries(&[ - "10.0.0.0/8".to_string(), - ]) - .unwrap(); + let trusted_ranges = + kalamdb_configs::parse_trusted_proxy_entries(&["10.0.0.0/8".to_string()]).unwrap(); let ip = extract_client_ip_addr_with_trusted_ranges( Some("10.0.1.9".parse().unwrap()), @@ -253,10 +250,8 @@ mod tests { HeaderName::from_static("x-forwarded-for"), HeaderValue::from_static("127.0.0.1"), ); - let trusted_ranges = kalamdb_configs::parse_trusted_proxy_entries(&[ - "10.0.0.0/8".to_string(), - ]) - .unwrap(); + let trusted_ranges = + kalamdb_configs::parse_trusted_proxy_entries(&["10.0.0.0/8".to_string()]).unwrap(); let ip = extract_client_ip_addr_with_trusted_ranges( Some("10.0.1.9".parse().unwrap()), diff --git a/backend/crates/kalamdb-commons/src/storage_key.rs b/backend/crates/kalamdb-commons/src/storage_key.rs index 1646b090b..0f6b1db9c 100644 --- a/backend/crates/kalamdb-commons/src/storage_key.rs +++ b/backend/crates/kalamdb-commons/src/storage_key.rs @@ -107,9 +107,14 @@ pub fn decode_key(bytes: &[u8]) -> Result { /// Compute the next lexicographic key after the provided encoded bytes. /// -/// This is used to make range scans exclusive of a specific key. It appends -/// a null byte which is safe for storekey-encoded values because the encoded -/// bytes are prefix-free for complete keys. +/// This is used to make range scans exclusive of a specific complete key. It +/// appends a null byte which is safe for storekey-encoded values because the +/// encoded bytes are prefix-free for complete keys. +/// +/// Do not use this as an upper bound for encoded prefixes. For composite +/// storekey prefixes like `encode_prefix(&(user_id, pk))`, the resulting bytes +/// are not guaranteed to sort after every longer key that starts with that +/// prefix. pub fn next_storage_key_bytes(bytes: &[u8]) -> Vec { let mut next = Vec::with_capacity(bytes.len() + 1); next.extend_from_slice(bytes); diff --git a/backend/crates/kalamdb-configs/src/config/loader.rs b/backend/crates/kalamdb-configs/src/config/loader.rs index ca6239da2..21237c6cb 100644 --- a/backend/crates/kalamdb-configs/src/config/loader.rs +++ b/backend/crates/kalamdb-configs/src/config/loader.rs @@ -1,5 +1,5 @@ -use super::types::ServerConfig; use super::trusted_proxies::parse_trusted_proxy_entries; +use super::types::ServerConfig; use crate::file_helpers::normalize_dir_path; use std::fs; use std::path::Path; @@ -108,10 +108,7 @@ impl ServerConfig { } parse_trusted_proxy_entries(&self.security.trusted_proxy_ranges).map_err(|error| { - anyhow::anyhow!( - "Invalid security.trusted_proxy_ranges configuration: {}", - error - ) + anyhow::anyhow!("Invalid security.trusted_proxy_ranges configuration: {}", error) })?; Ok(()) diff --git a/backend/crates/kalamdb-configs/src/config/mod.rs b/backend/crates/kalamdb-configs/src/config/mod.rs index 9407a5fa3..2bb580b72 100644 --- a/backend/crates/kalamdb-configs/src/config/mod.rs +++ b/backend/crates/kalamdb-configs/src/config/mod.rs @@ -6,6 +6,6 @@ pub mod overrides; pub mod trusted_proxies; pub mod types; -pub use trusted_proxies::*; pub use cluster::*; +pub use trusted_proxies::*; pub use types::*; diff --git a/backend/crates/kalamdb-configs/src/config/override.rs b/backend/crates/kalamdb-configs/src/config/override.rs index 11d5625b1..864ea1b5f 100644 --- a/backend/crates/kalamdb-configs/src/config/override.rs +++ b/backend/crates/kalamdb-configs/src/config/override.rs @@ -371,10 +371,7 @@ mod tests { #[test] fn test_env_override_trusted_proxy_ranges() { let _guard = acquire_env_lock(); - env::set_var( - "KALAMDB_SECURITY_TRUSTED_PROXY_RANGES", - "10.0.1.9,192.168.0.0/24", - ); + env::set_var("KALAMDB_SECURITY_TRUSTED_PROXY_RANGES", "10.0.1.9,192.168.0.0/24"); let mut config = ServerConfig::default(); config.apply_env_overrides().unwrap(); diff --git a/backend/crates/kalamdb-configs/src/config/trusted_proxies.rs b/backend/crates/kalamdb-configs/src/config/trusted_proxies.rs index fe74a2195..5ae720a38 100644 --- a/backend/crates/kalamdb-configs/src/config/trusted_proxies.rs +++ b/backend/crates/kalamdb-configs/src/config/trusted_proxies.rs @@ -55,8 +55,6 @@ mod tests { #[test] fn rejects_invalid_entries() { let error = parse_trusted_proxy_entries(&["not-an-ip".to_string()]).unwrap_err(); - assert!(error - .to_string() - .contains("invalid trusted proxy entry 'not-an-ip'")); + assert!(error.to_string().contains("invalid trusted proxy entry 'not-an-ip'")); } -} \ No newline at end of file +} diff --git a/backend/crates/kalamdb-core/src/applier/executor/dml.rs b/backend/crates/kalamdb-core/src/applier/executor/dml.rs index 99047371d..bbbd0aee7 100644 --- a/backend/crates/kalamdb-core/src/applier/executor/dml.rs +++ b/backend/crates/kalamdb-core/src/applier/executor/dml.rs @@ -104,19 +104,8 @@ impl DmlExecutor { .ok_or_else(|| ApplierError::not_found("Table provider", table_id))?; if let Some(provider) = provider_arc.as_any().downcast_ref::() { - let prior_row = match find_row_by_pk(provider, Some(user_id), pk_value).await { - Ok(Some((_key, row))) => Some(row.fields), - Ok(None) => None, - Err(err) => { - log::warn!( - "Failed to load row for update file cleanup in {} (pk={}): {}", - table_id, - pk_value, - err - ); - None - }, - }; + let prior_row = + self.load_user_row_for_cleanup(provider, table_id, user_id, pk_value).await; let replaced_refs = prior_row.as_ref().map_or_else(Vec::new, |row| { collect_replaced_file_refs_for_update( @@ -175,21 +164,12 @@ impl DmlExecutor { if let Some(provider) = provider_arc.as_any().downcast_ref::() { let mut deleted_count = 0; for pk_value in pk_values { - let file_refs = match find_row_by_pk(provider, Some(user_id), pk_value).await { - Ok(Some((_key, row))) => { - collect_file_refs_from_row(self.app_context.as_ref(), table_id, &row.fields) - }, - Ok(None) => Vec::new(), - Err(err) => { - log::warn!( - "Failed to load row for file cleanup in {} (pk={}): {}", - table_id, - pk_value, - err - ); - Vec::new() - }, - }; + let file_refs = self + .load_user_row_for_cleanup(provider, table_id, user_id, pk_value) + .await + .map_or_else(Vec::new, |row| { + collect_file_refs_from_row(self.app_context.as_ref(), table_id, &row) + }); if provider .delete_by_id_field(user_id, pk_value) @@ -289,19 +269,7 @@ impl DmlExecutor { let system_user = UserId::system(); let update_row = updates[0].clone(); - let prior_row = match find_row_by_pk(provider, None, pk_value).await { - Ok(Some((_key, row))) => Some(row.fields), - Ok(None) => None, - Err(err) => { - log::warn!( - "Failed to load row for update file cleanup in {} (pk={}): {}", - table_id, - pk_value, - err - ); - None - }, - }; + let prior_row = self.load_shared_row_for_cleanup(provider, table_id, pk_value).await; let replaced_refs = prior_row.as_ref().map_or_else(Vec::new, |row| { collect_replaced_file_refs_for_update( @@ -360,21 +328,12 @@ impl DmlExecutor { let mut deleted_count = 0; for pk_value in pk_values { - let file_refs = match find_row_by_pk(provider, None, pk_value).await { - Ok(Some((_key, row))) => { - collect_file_refs_from_row(self.app_context.as_ref(), table_id, &row.fields) - }, - Ok(None) => Vec::new(), - Err(err) => { - log::warn!( - "Failed to load row for file cleanup in {} (pk={}): {}", - table_id, - pk_value, - err - ); - Vec::new() - }, - }; + let file_refs = self + .load_shared_row_for_cleanup(provider, table_id, pk_value) + .await + .map_or_else(Vec::new, |row| { + collect_file_refs_from_row(self.app_context.as_ref(), table_id, &row) + }); if provider .delete_by_id_field(&system_user, pk_value) @@ -407,6 +366,135 @@ impl DmlExecutor { // Helper Methods // ========================================================================= + async fn load_user_row_for_cleanup( + &self, + provider: &UserTableProvider, + table_id: &TableId, + user_id: &UserId, + pk_value: &str, + ) -> Option { + let pk_name = provider.primary_key_field_name(); + let schema = provider.schema_ref(); + let pk_field = match schema.field_with_name(pk_name) { + Ok(field) => field, + Err(err) => { + log::warn!( + "Failed to resolve PK field for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + return None; + }, + }; + + let pk_scalar = match kalamdb_commons::conversions::parse_string_as_scalar( + pk_value, + pk_field.data_type(), + ) { + Ok(value) => value, + Err(err) => { + log::warn!( + "Failed to parse PK value for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + return None; + }, + }; + + match provider.find_by_pk(user_id, &pk_scalar).await { + Ok(Some((_key, row))) => Some(row.fields), + Ok(None) => match find_row_by_pk(provider, Some(user_id), pk_value).await { + Ok(Some((_key, row))) => Some(row.fields), + Ok(None) => None, + Err(err) => { + log::warn!( + "Failed to load cold row for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + None + }, + }, + Err(err) => { + log::warn!( + "Failed to load hot row for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + None + }, + } + } + + async fn load_shared_row_for_cleanup( + &self, + provider: &SharedTableProvider, + table_id: &TableId, + pk_value: &str, + ) -> Option { + let pk_name = provider.primary_key_field_name(); + let schema = provider.schema_ref(); + let pk_field = match schema.field_with_name(pk_name) { + Ok(field) => field, + Err(err) => { + log::warn!( + "Failed to resolve PK field for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + return None; + }, + }; + + let pk_scalar = match kalamdb_commons::conversions::parse_string_as_scalar( + pk_value, + pk_field.data_type(), + ) { + Ok(value) => value, + Err(err) => { + log::warn!( + "Failed to parse PK value for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + return None; + }, + }; + + match provider.find_by_pk(&pk_scalar).await { + Ok(Some((_key, row))) => Some(row.fields), + Ok(None) => match find_row_by_pk(provider, None, pk_value).await { + Ok(Some((_key, row))) => Some(row.fields), + Ok(None) => None, + Err(err) => { + log::warn!( + "Failed to load cold row for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + None + }, + }, + Err(err) => { + log::warn!( + "Failed to load hot row for file cleanup in {} (pk={}): {}", + table_id, + pk_value, + err + ); + None + }, + } + } + /// Update with fallback for UserTableProvider async fn update_user_provider( &self, diff --git a/backend/crates/kalamdb-core/src/live/helpers/filter_eval.rs b/backend/crates/kalamdb-core/src/live/helpers/filter_eval.rs index 02bf19b41..cc97fda46 100644 --- a/backend/crates/kalamdb-core/src/live/helpers/filter_eval.rs +++ b/backend/crates/kalamdb-core/src/live/helpers/filter_eval.rs @@ -14,6 +14,7 @@ use datafusion::sql::sqlparser::ast::{BinaryOperator, Expr, Statement, Value}; use datafusion::sql::sqlparser::dialect::PostgreSqlDialect; use datafusion::sql::sqlparser::parser::Parser; use kalamdb_commons::models::rows::Row; +use regex::RegexBuilder; /// Parse a WHERE clause string into an Expr AST /// @@ -136,6 +137,38 @@ fn evaluate_expr(expr: &Expr, row_data: &Row, depth: usize) -> Result evaluate_like_pattern( + expr, + pattern, + row_data, + *negated, + *any, + parse_like_escape_char(escape_char.as_ref())?, + false, + ), + + Expr::ILike { + negated, + any, + expr, + pattern, + escape_char, + } => evaluate_like_pattern( + expr, + pattern, + row_data, + *negated, + *any, + parse_like_escape_char(escape_char.as_ref())?, + true, + ), + _ => Err(KalamDbError::InvalidOperation(format!( "Unsupported expression type: {:?}", expr @@ -214,6 +247,113 @@ fn compare_numeric( }) } +fn evaluate_like_pattern( + expr: &Expr, + pattern: &Expr, + row_data: &Row, + negated: bool, + any: bool, + escape_char: Option, + case_insensitive: bool, +) -> Result { + if any { + return Err(KalamDbError::InvalidOperation( + "LIKE ANY expressions are not supported in live query filters".to_string(), + )); + } + + let value = extract_value(expr, row_data)?; + let pattern_value = extract_value(pattern, row_data)?; + + let value_str = as_str(&value).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Cannot evaluate LIKE against non-string value: {:?}", + value + )) + })?; + let pattern_str = as_str(&pattern_value).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "LIKE pattern must be a string literal or column value, got: {:?}", + pattern_value + )) + })?; + + let regex_pattern = build_like_regex_pattern(pattern_str, escape_char)?; + let regex = RegexBuilder::new(®ex_pattern) + .case_insensitive(case_insensitive) + .build() + .map_err(|error| { + KalamDbError::InvalidOperation(format!( + "Invalid LIKE pattern {:?}: {}", + pattern_str, error + )) + })?; + + let matches = regex.is_match(value_str); + Ok(if negated { !matches } else { matches }) +} + +fn build_like_regex_pattern( + pattern: &str, + escape_char: Option, +) -> Result { + let mut regex_pattern = String::with_capacity(pattern.len() + 2); + regex_pattern.push('^'); + + let mut escaped = false; + for ch in pattern.chars() { + if escaped { + regex_pattern.push_str(®ex::escape(&ch.to_string())); + escaped = false; + continue; + } + + if Some(ch) == escape_char { + escaped = true; + continue; + } + + match ch { + '%' => regex_pattern.push_str(".*"), + '_' => regex_pattern.push('.'), + _ => regex_pattern.push_str(®ex::escape(&ch.to_string())), + } + } + + if escaped { + return Err(KalamDbError::InvalidOperation(format!( + "LIKE pattern has dangling escape character: {:?}", + pattern + ))); + } + + regex_pattern.push('$'); + Ok(regex_pattern) +} + +fn parse_like_escape_char(escape_char: Option<&Value>) -> Result, KalamDbError> { + match escape_char { + None => Ok(None), + Some(Value::SingleQuotedString(value)) | Some(Value::DoubleQuotedString(value)) => { + let mut chars = value.chars(); + let ch = chars.next().ok_or_else(|| { + KalamDbError::InvalidOperation("LIKE ESCAPE cannot be empty".to_string()) + })?; + if chars.next().is_some() { + return Err(KalamDbError::InvalidOperation(format!( + "LIKE ESCAPE must be a single character, got {:?}", + value + ))); + } + Ok(Some(ch)) + }, + Some(other) => Err(KalamDbError::InvalidOperation(format!( + "Unsupported LIKE ESCAPE value: {:?}", + other + ))), + } +} + /// Extract a value from an expression /// /// Handles: @@ -398,6 +538,28 @@ mod tests { assert!(!matches(&expr, &non_matching_row2).unwrap()); } + #[test] + fn test_like_filter() { + let expr = parse_where_clause("metric_name LIKE 'open_files_%'").unwrap(); + + let matching_row = to_row(json!({"metric_name": "open_files_other"})); + assert!(matches(&expr, &matching_row).unwrap()); + + let non_matching_row = to_row(json!({"metric_name": "closed_files_other"})); + assert!(!matches(&expr, &non_matching_row).unwrap()); + } + + #[test] + fn test_ilike_filter() { + let expr = parse_where_clause("metric_name ILIKE 'OPEN_FILES_%'").unwrap(); + + let matching_row = to_row(json!({"metric_name": "open_files_other"})); + assert!(matches(&expr, &matching_row).unwrap()); + + let non_matching_row = to_row(json!({"metric_name": "closed_files_other"})); + assert!(!matches(&expr, &non_matching_row).unwrap()); + } + #[test] fn test_deeply_nested_filter_rejected() { // Build a deeply nested expression that exceeds MAX_EXPR_DEPTH diff --git a/backend/crates/kalamdb-sql/src/ddl/alter_table.rs b/backend/crates/kalamdb-sql/src/ddl/alter_table.rs index 13c3298cc..b3524eaf0 100644 --- a/backend/crates/kalamdb-sql/src/ddl/alter_table.rs +++ b/backend/crates/kalamdb-sql/src/ddl/alter_table.rs @@ -404,10 +404,12 @@ fn extract_access_level(option: &SqlOption) -> DdlResult> { "PRIVATE" => TableAccess::Private, "RESTRICTED" => TableAccess::Restricted, "DBA" => TableAccess::Dba, - other => return Err(format!( + other => { + return Err(format!( "Invalid ACCESS_LEVEL '{}'. Supported values: PUBLIC, PRIVATE, RESTRICTED, DBA", other - )), + )) + }, }; return Ok(Some(access_level)); } diff --git a/backend/crates/kalamdb-sql/src/execute_as.rs b/backend/crates/kalamdb-sql/src/execute_as.rs index dcf56d172..7004d7ad4 100644 --- a/backend/crates/kalamdb-sql/src/execute_as.rs +++ b/backend/crates/kalamdb-sql/src/execute_as.rs @@ -322,11 +322,10 @@ mod tests { #[test] fn parse_escaped_quote_in_username() { - let result = parse_execute_as( - "EXECUTE AS USER 'alice''o' (INSERT INTO default.t VALUES (1))", - ) - .expect("should parse") - .expect("should be an envelope"); + let result = + parse_execute_as("EXECUTE AS USER 'alice''o' (INSERT INTO default.t VALUES (1))") + .expect("should parse") + .expect("should be an envelope"); assert_eq!(result.username, "alice'o"); assert_eq!(result.inner_sql, "INSERT INTO default.t VALUES (1)"); @@ -341,10 +340,7 @@ mod tests { .expect("should be an envelope"); assert_eq!(result.username, "alice"); - assert_eq!( - result.inner_sql, - "INSERT INTO default.t VALUES ('hello (', 'done')" - ); + assert_eq!(result.inner_sql, "INSERT INTO default.t VALUES ('hello (', 'done')"); } #[test] @@ -352,10 +348,7 @@ mod tests { let inner = extract_inner_sql( "EXECUTE AS USER 'alice' (INSERT INTO default.t VALUES ('hello (', 'done'))", ); - assert_eq!( - inner.as_deref(), - Some("INSERT INTO default.t VALUES ('hello (', 'done')") - ); + assert_eq!(inner.as_deref(), Some("INSERT INTO default.t VALUES ('hello (', 'done')")); } #[test] diff --git a/backend/crates/kalamdb-store/src/indexed_store.rs b/backend/crates/kalamdb-store/src/indexed_store.rs index 76086c114..472c113a7 100644 --- a/backend/crates/kalamdb-store/src/indexed_store.rs +++ b/backend/crates/kalamdb-store/src/indexed_store.rs @@ -762,6 +762,34 @@ where Ok(Box::new(mapped)) } + /// Returns the newest entity matching an index prefix. + /// + /// This uses reverse index iteration so hot-path PK lookups can fetch the + /// latest MVCC version without walking every historical version. + pub fn get_latest_by_index_prefix( + &self, + index_idx: usize, + prefix: &[u8], + ) -> Result> { + let index_partition = self + .index_partitions + .get(index_idx) + .ok_or_else(|| StorageError::Other(format!("Index {} not found", index_idx)))? + .clone(); + let mut iter = self.backend.scan_reverse(&index_partition, Some(prefix), None, Some(1))?; + + while let Some((_index_key, primary_key_bytes)) = iter.next() { + let primary_key = K::from_storage_key(&primary_key_bytes) + .map_err(StorageError::SerializationError)?; + + if let Some(entity) = self.get(&primary_key)? { + return Ok(Some((primary_key, entity))); + } + } + + Ok(None) + } + /// Scans an index and returns only the primary keys (no entity fetch). /// /// More efficient when you only need the keys. @@ -1041,6 +1069,16 @@ where .map_err(|e| StorageError::Other(format!("spawn_blocking error: {}", e)))? } + /// Async version of `insert_batch()`. + /// + /// Uses `spawn_blocking` to avoid blocking the async runtime. + pub async fn insert_batch_async(&self, entries: Vec<(K, V)>) -> Result<()> { + let store = self.clone(); + tokio::task::spawn_blocking(move || store.insert_batch(&entries)) + .await + .map_err(|e| StorageError::Other(format!("spawn_blocking error: {}", e)))? + } + /// Async version of `update()`. /// /// Uses `spawn_blocking` to avoid blocking the async runtime. @@ -1078,6 +1116,34 @@ where .map_err(|e| StorageError::Other(format!("spawn_blocking error: {}", e)))? } + /// Async version of `get_latest_by_index_prefix()`. + /// + /// Uses `spawn_blocking` to avoid blocking the async runtime. + pub async fn get_latest_by_index_prefix_async( + &self, + index_idx: usize, + prefix: Vec, + ) -> Result> { + let store = self.clone(); + tokio::task::spawn_blocking(move || store.get_latest_by_index_prefix(index_idx, &prefix)) + .await + .map_err(|e| StorageError::Other(format!("spawn_blocking error: {}", e)))? + } + + /// Async version of `insert_batch_preencoded()`. + /// + /// Uses `spawn_blocking` to avoid blocking the async runtime. + pub async fn insert_batch_preencoded_async( + &self, + entries: Vec<(K, V)>, + encoded_values: Vec>, + ) -> Result<()> { + let store = self.clone(); + tokio::task::spawn_blocking(move || store.insert_batch_preencoded(&entries, encoded_values)) + .await + .map_err(|e| StorageError::Other(format!("spawn_blocking error: {}", e)))? + } + /// Async version of `get()` from EntityStore. /// /// Uses `spawn_blocking` to avoid blocking the async runtime. @@ -1337,6 +1403,33 @@ mod tests { assert_eq!(new_jobs.len(), 0); } + #[test] + fn test_get_latest_by_index_prefix_returns_last_matching_entry() { + let backend: Arc = Arc::new(InMemoryBackend::new()); + let store = + IndexedEntityStore::new(backend.clone(), "test_jobs", vec![Arc::new(TestStatusIndex)]); + + let mut older = create_test_job("job-old", JobStatus::Running); + older.created_at = 100; + older.updated_at = 100; + + let mut newer = create_test_job("job-new", JobStatus::Running); + newer.created_at = 200; + newer.updated_at = 200; + + let mut completed = create_test_job("job-done", JobStatus::Completed); + completed.created_at = 300; + completed.updated_at = 300; + + store.insert(&older.job_id, &older).unwrap(); + store.insert(&newer.job_id, &newer).unwrap(); + store.insert(&completed.job_id, &completed).unwrap(); + + let latest = store.get_latest_by_index_prefix(0, &[2]).unwrap().expect("latest running"); + assert_eq!(latest.0, newer.job_id); + assert_eq!(latest.1.created_at, 200); + } + #[tokio::test] async fn test_async_operations() { let backend: Arc = Arc::new(InMemoryBackend::new()); @@ -1357,6 +1450,23 @@ mod tests { let running = store.scan_by_index_async(0, Some(vec![2]), None).await.unwrap(); assert_eq!(running.len(), 1); + let latest = store.get_latest_by_index_prefix_async(0, vec![2]).await.unwrap(); + assert_eq!(latest.expect("latest running").0, job_id); + + let queued_job = create_test_job("job2", JobStatus::Queued); + let completed_job = create_test_job("job3", JobStatus::Completed); + store + .insert_batch_async(vec![ + (queued_job.job_id.clone(), queued_job.clone()), + (completed_job.job_id.clone(), completed_job.clone()), + ]) + .await + .unwrap(); + + let queued = store.scan_by_index_async(0, Some(vec![1]), None).await.unwrap(); + assert_eq!(queued.len(), 1); + assert_eq!(queued[0].0, queued_job.job_id); + // Async delete store.delete_async(job_id.clone()).await.unwrap(); diff --git a/backend/crates/kalamdb-store/src/rocksdb_impl.rs b/backend/crates/kalamdb-store/src/rocksdb_impl.rs index eb640923b..64faa0231 100644 --- a/backend/crates/kalamdb-store/src/rocksdb_impl.rs +++ b/backend/crates/kalamdb-store/src/rocksdb_impl.rs @@ -245,6 +245,87 @@ impl StorageBackend for RocksDBBackend { Ok(Box::new(iter)) } + fn scan_reverse( + &self, + partition: &Partition, + prefix: Option<&[u8]>, + start_key: Option<&[u8]>, + limit: Option, + ) -> Result, Vec)> + Send + '_>> { + let _span = tracing::debug_span!( + "rocksdb.scan_reverse", + partition = %partition.name(), + has_prefix = prefix.is_some(), + limit = ?limit + ) + .entered(); + use rocksdb::Direction; + + let cf = self.get_cf(partition)?; + let snapshot = self.db.snapshot(); + let prefix_vec = prefix.map(|p| p.to_vec()); + + let mut readopts = rocksdb::ReadOptions::default(); + readopts.set_snapshot(&snapshot); + if let Some(p) = &prefix_vec { + readopts.set_iterate_range(PrefixRange(p.clone())); + } + + // When a prefix bound is present, IteratorMode::End starts at the high end + // of that bounded range. This is required for storekey-encoded composite + // prefixes where appending a null byte does not produce a valid upper bound. + let iter_mode = if let Some(start) = start_key { + IteratorMode::From(start, Direction::Reverse) + } else { + IteratorMode::End + }; + + let inner = self.db.iterator_cf_opt(&cf, readopts, iter_mode); + + struct SnapshotReverseScanIter<'a, D: rocksdb::DBAccess> { + _snapshot: rocksdb::SnapshotWithThreadMode<'a, D>, + inner: rocksdb::DBIteratorWithThreadMode<'a, D>, + prefix: Option>, + remaining: Option, + } + + impl<'a, D: rocksdb::DBAccess> Iterator for SnapshotReverseScanIter<'a, D> { + type Item = (Vec, Vec); + + fn next(&mut self) -> Option { + if let Some(0) = self.remaining { + return None; + } + + match self.inner.next()? { + Ok((k, v)) => { + if let Some(ref prefix) = self.prefix { + if !k.starts_with(prefix) { + return None; + } + } + if let Some(ref mut left) = self.remaining { + if *left > 0 { + *left -= 1; + } + } + Some((k.to_vec(), v.to_vec())) + }, + Err(_) => None, + } + } + } + + let iter = SnapshotReverseScanIter:: { + _snapshot: snapshot, + inner, + prefix: prefix_vec, + remaining: limit, + }; + + Ok(Box::new(iter)) + } + fn partition_exists(&self, partition: &Partition) -> bool { self.db.cf_handle(partition.name()).is_some() } @@ -428,6 +509,7 @@ impl StorageBackend for RocksDBBackend { #[cfg(test)] mod tests { use super::*; + use kalamdb_commons::{encode_key, encode_prefix}; use tempfile::TempDir; fn create_test_db() -> (Arc, TempDir) { @@ -563,6 +645,33 @@ mod tests { assert_eq!(results.len(), 2); } + #[test] + fn test_scan_reverse_with_storekey_prefix_returns_latest_match() { + let (db, _temp) = create_test_db(); + let backend = RocksDBBackend::new(db); + + let partition = Partition::new("test_cf"); + backend.create_partition(&partition).unwrap(); + + let prefix = encode_prefix(&("user1", 42_i64)); + let older_key = encode_key(&("user1", 42_i64, 100_i64)); + let newer_key = encode_key(&("user1", 42_i64, 200_i64)); + let other_key = encode_key(&("user1", 99_i64, 300_i64)); + + backend.put(&partition, &older_key, b"older").unwrap(); + backend.put(&partition, &newer_key, b"newer").unwrap(); + backend.put(&partition, &other_key, b"other").unwrap(); + + let results: Vec<_> = backend + .scan_reverse(&partition, Some(&prefix), None, Some(1)) + .unwrap() + .collect(); + + assert_eq!(results.len(), 1); + assert_eq!(results[0].0, newer_key); + assert_eq!(results[0].1, b"newer".to_vec()); + } + #[test] fn test_list_partitions() { let (db, _temp) = create_test_db(); diff --git a/backend/crates/kalamdb-store/src/storage_trait.rs b/backend/crates/kalamdb-store/src/storage_trait.rs index eb58d2390..71e4269e5 100644 --- a/backend/crates/kalamdb-store/src/storage_trait.rs +++ b/backend/crates/kalamdb-store/src/storage_trait.rs @@ -119,6 +119,43 @@ pub trait StorageBackend: Send + Sync { limit: Option, ) -> Result>; + /// Scans keys in reverse order, optionally filtered by prefix and limit. + /// + /// This is used by hot-path latest-version lookups where the newest entry is + /// stored last in the keyspace. Backends with native reverse iterators should + /// override this; the default implementation falls back to a forward scan and + /// reverses the collected rows. + fn scan_reverse( + &self, + partition: &Partition, + prefix: Option<&[u8]>, + start_key: Option<&[u8]>, + limit: Option, + ) -> Result> { + if matches!(limit, Some(0)) { + return Ok(Box::new(std::iter::empty())); + } + + let mut items: Vec<(Vec, Vec)> = + self.scan(partition, prefix, None, None)?.collect(); + + if let Some(start) = start_key { + items.retain(|(key, _)| key.as_slice() <= start); + } + + if let Some(prefix) = prefix { + items.retain(|(key, _)| key.starts_with(prefix)); + } + + items.reverse(); + + if let Some(limit) = limit { + items.truncate(limit); + } + + Ok(Box::new(items.into_iter())) + } + /// Checks if a partition exists. fn partition_exists(&self, partition: &Partition) -> bool; diff --git a/backend/crates/kalamdb-tables/src/shared_tables/shared_table_provider.rs b/backend/crates/kalamdb-tables/src/shared_tables/shared_table_provider.rs index 0b6a13d87..ac32c67e9 100644 --- a/backend/crates/kalamdb-tables/src/shared_tables/shared_table_provider.rs +++ b/backend/crates/kalamdb-tables/src/shared_tables/shared_table_provider.rs @@ -40,7 +40,8 @@ use kalamdb_session::{check_shared_table_access, check_shared_table_write_access use kalamdb_store::EntityStore; use kalamdb_system::VectorMetric; use kalamdb_vector::{ - new_indexed_shared_vector_hot_store, SharedVectorHotOpId, VectorHotOp, VectorHotOpType, + new_indexed_shared_vector_hot_store, SharedVectorHotOpId, SharedVectorHotStore, VectorHotOp, + VectorHotOpType, }; use std::any::Any; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -69,6 +70,9 @@ pub struct SharedTableProvider { /// Embedding columns tracked by vector hot staging: (column_name, dimensions). vector_columns: Vec<(String, u32)>, + + /// Cached vector staging stores keyed by embedding column name. + vector_stores: HashMap>, } impl SharedTableProvider { @@ -79,7 +83,7 @@ impl SharedTableProvider { /// * `store` - SharedTableIndexedStore for this table pub fn new(core: Arc, store: Arc) -> Self { let pk_index = SharedTablePkIndex::new(core.table_id(), core.primary_key_field_name()); - let vector_columns = core + let vector_columns: Vec<(String, u32)> = core .table_def() .columns .iter() @@ -90,12 +94,27 @@ impl SharedTableProvider { _ => None, }) .collect(); + let backend = store.backend().clone(); + let vector_stores: HashMap> = vector_columns + .iter() + .map(|(column_name, _)| { + ( + column_name.clone(), + Arc::new(new_indexed_shared_vector_hot_store( + backend.clone(), + core.table_id(), + column_name, + )), + ) + }) + .collect(); Self { core, store, pk_index, vector_columns, + vector_stores, } } @@ -233,14 +252,17 @@ impl SharedTableProvider { } } - fn stage_vector_upsert(&self, seq: SharedTableRowId, row: &Row) -> Result<(), KalamDbError> { + async fn stage_vector_upsert( + &self, + seq: SharedTableRowId, + row: &Row, + ) -> Result<(), KalamDbError> { if self.vector_columns.is_empty() { return Ok(()); } let pk = crate::utils::unified_dml::extract_user_pk_value(row, self.primary_key_field_name())?; - let backend = self.store.backend().clone(); for (column_name, dimensions) in &self.vector_columns { let Some(value) = row.get(column_name.as_str()) else { @@ -250,11 +272,12 @@ impl SharedTableProvider { continue; }; - let store = new_indexed_shared_vector_hot_store( - backend.clone(), - self.core.table_id(), - column_name, - ); + let store = self.vector_stores.get(column_name).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Missing cached vector store for column '{}'", + column_name + )) + })?; let key = SharedVectorHotOpId::new(seq, pk.clone()); let op = VectorHotOp::new( self.core.table_id().clone(), @@ -266,7 +289,7 @@ impl SharedTableProvider { *dimensions, VectorMetric::Cosine, ); - store.insert(&key, &op).map_err(|e| { + store.insert_async(key, op).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to stage vector upsert op: {}", e)) })?; } @@ -274,18 +297,81 @@ impl SharedTableProvider { Ok(()) } - fn stage_vector_delete(&self, seq: SharedTableRowId, pk: &str) -> Result<(), KalamDbError> { + async fn stage_vector_upsert_batch( + &self, + entries: &[(SharedTableRowId, SharedTableRow)], + ) -> Result<(), KalamDbError> { + if self.vector_columns.is_empty() || entries.is_empty() { + return Ok(()); + } + + let mut ops_by_column: HashMap> = + HashMap::new(); + + for (row_key, entity) in entries { + let pk = crate::utils::unified_dml::extract_user_pk_value( + &entity.fields, + self.primary_key_field_name(), + )?; + + for (column_name, dimensions) in &self.vector_columns { + let Some(value) = entity.fields.get(column_name.as_str()) else { + continue; + }; + let Some(vector) = Self::extract_embedding_vector(value, *dimensions) else { + continue; + }; + + ops_by_column.entry(column_name.clone()).or_default().push(( + SharedVectorHotOpId::new(*row_key, pk.clone()), + VectorHotOp::new( + self.core.table_id().clone(), + column_name.clone(), + pk.clone(), + VectorHotOpType::Upsert, + Some(vector), + None, + *dimensions, + VectorMetric::Cosine, + ), + )); + } + } + + for (column_name, ops) in ops_by_column { + let store = self.vector_stores.get(&column_name).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Missing cached vector store for column '{}'", + column_name + )) + })?; + store.insert_batch_async(ops).await.map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to batch stage vector upsert ops for column '{}': {}", + column_name, e + )) + })?; + } + + Ok(()) + } + + async fn stage_vector_delete( + &self, + seq: SharedTableRowId, + pk: &str, + ) -> Result<(), KalamDbError> { if self.vector_columns.is_empty() { return Ok(()); } - let backend = self.store.backend().clone(); for (column_name, dimensions) in &self.vector_columns { - let store = new_indexed_shared_vector_hot_store( - backend.clone(), - self.core.table_id(), - column_name, - ); + let store = self.vector_stores.get(column_name).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Missing cached vector store for column '{}'", + column_name + )) + })?; let key = SharedVectorHotOpId::new(seq, pk.to_string()); let op = VectorHotOp::new( self.core.table_id().clone(), @@ -297,7 +383,7 @@ impl SharedTableProvider { *dimensions, VectorMetric::Cosine, ); - store.insert(&key, &op).map_err(|e| { + store.insert_async(key, op).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to stage vector delete op: {}", e)) })?; } @@ -338,33 +424,28 @@ impl SharedTableProvider { /// /// This method uses the PK index to find all versions of a row with the given PK value, /// then returns the latest non-deleted version. - fn find_by_pk( + async fn latest_hot_pk_entry( &self, pk_value: &ScalarValue, ) -> Result, KalamDbError> { - // Build index prefix for this PK value let prefix = self.pk_index.build_prefix_for_pk(pk_value); + self.store + .get_latest_by_index_prefix_async(0, prefix) + .await + .into_kalamdb_error("PK index scan failed") + } - // Use scan_by_index on the IndexedEntityStore (index 0 is the PK index) - // scan_by_index returns (key, entity) pairs directly - let results = self - .store - .scan_by_index(0, Some(&prefix), None) - .into_kalamdb_error("PK index scan failed")?; - - if results.is_empty() { - return Ok(None); - } - - if let Some((row_id, row)) = results.into_iter().max_by_key(|(row_id, _)| row_id.as_i64()) { + pub async fn find_by_pk( + &self, + pk_value: &ScalarValue, + ) -> Result, KalamDbError> { + Ok(self.latest_hot_pk_entry(pk_value).await?.and_then(|(row_id, row)| { if row._deleted { - Ok(None) + None } else { - Ok(Some((row_id, row))) + Some((row_id, row)) } - } else { - Ok(None) - } + })) } /// Returns true if the latest hot-storage version of this PK is a tombstone @@ -373,15 +454,10 @@ impl SharedTableProvider { /// /// Used in the PK fast-path of `scan_rows` to prevent cold storage (Parquet) /// from surfacing a row that has already been deleted in hot storage. - fn pk_tombstoned_in_hot(&self, pk_value: &ScalarValue) -> Result { - let prefix = self.pk_index.build_prefix_for_pk(pk_value); - let results = self - .store - .scan_by_index(0, Some(&prefix), None) - .into_kalamdb_error("PK index scan failed")?; - Ok(results - .into_iter() - .max_by_key(|(row_id, _)| row_id.as_i64()) + async fn pk_tombstoned_in_hot(&self, pk_value: &ScalarValue) -> Result { + Ok(self + .latest_hot_pk_entry(pk_value) + .await? .map(|(_, row)| row._deleted) .unwrap_or(false)) } @@ -614,21 +690,13 @@ impl BaseTableProvider for SharedTableProvider // Use shared helper to parse PK value let pk_value = crate::utils::pk::parse_pk_value(id_value); - // Fast path: return the latest non-deleted row from hot storage if it exists. - if let Some((row_id, _row)) = self.find_by_pk(&pk_value)? { + if let Some((row_id, row)) = self.latest_hot_pk_entry(&pk_value).await? { + if row._deleted { + return Ok(None); + } return Ok(Some(row_id)); } - // If hot storage has entries but they're all deleted, allow PK reuse. - let prefix = self.pk_index.build_prefix_for_pk(&pk_value); - let hot_has_versions = self - .store - .exists_by_index(0, &prefix) - .into_kalamdb_error("PK index scan failed")?; - if hot_has_versions { - return Ok(None); - } - // Not found in hot storage - check cold storage using optimized manifest-based lookup // This uses column_stats to prune segments that can't contain the PK let pk_name = self.primary_key_field_name(); @@ -681,13 +749,13 @@ impl BaseTableProvider for SharedTableProvider let row_key = seq_id; // Store the entity in RocksDB (hot storage) using insert() to update PK index - self.store.insert(&row_key, &entity).map_err(|e| { + self.store.insert_async(row_key, entity.clone()).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to insert shared table row: {}", e)) })?; log::debug!("Inserted shared table row with _seq {}", seq_id); - if let Err(e) = self.stage_vector_upsert(seq_id, &entity.fields) { + if let Err(e) = self.stage_vector_upsert(seq_id, &entity.fields).await { log::warn!( "Failed to stage vector upsert for table={}, seq={}: {}", self.core.table_id(), @@ -774,70 +842,43 @@ impl BaseTableProvider for SharedTableProvider // Batch PK validation: collect all user-provided PK values let pk_name = self.primary_key_field_name(); - let mut pk_values_to_check: Vec = Vec::new(); + let mut pk_values_to_check: Vec<(String, ScalarValue)> = Vec::new(); + let mut seen_batch_pks = HashSet::new(); for row_data in &coerced_rows { if let Some(pk_value) = row_data.get(pk_name) { if !matches!(pk_value, ScalarValue::Null) { let pk_str = crate::utils::unified_dml::extract_user_pk_value(row_data, pk_name)?; - pk_values_to_check.push(pk_str); + if !seen_batch_pks.insert(pk_str.clone()) { + return Err(KalamDbError::AlreadyExists(format!( + "Primary key violation: value '{}' appears multiple times in the insert batch for column '{}'", + pk_str, pk_name + ))); + } + pk_values_to_check.push((pk_str, pk_value.clone())); } } } - // OPTIMIZED: Check PK existence in hot storage only (cold storage handled below) - // For small batches (≤2 rows), use direct lookups. For larger batches, use batch scan. + // Check only the latest hot version for each PK. Tombstoned latest + // versions are reusable and should not fail the insert. if !pk_values_to_check.is_empty() { - if pk_values_to_check.len() <= 2 { - // Small batch: individual O(1) lookups are efficient - for pk_str in &pk_values_to_check { - let prefix = self.pk_index.build_pk_prefix(pk_str); - if self - .store - .exists_by_index(0, &prefix) - .into_kalamdb_error("PK index check failed")? - { + for (pk_str, pk_value) in &pk_values_to_check { + if let Some((_row_id, row)) = self.latest_hot_pk_entry(pk_value).await? { + if !row._deleted { return Err(KalamDbError::AlreadyExists(format!( "Primary key violation: value '{}' already exists in column '{}'", pk_str, pk_name ))); } } - } else { - // Larger batch: build all prefixes and use batch scan - let prefixes: Vec> = pk_values_to_check - .iter() - .map(|pk_str| self.pk_index.build_pk_prefix(pk_str)) - .collect(); - - // Use empty common prefix for shared tables (no user scoping) - let existing = self - .store - .exists_batch_by_index(0, &[], &prefixes) - .into_kalamdb_error("Batch PK index check failed")?; - - if !existing.is_empty() { - // Find which PK matched - for pk_str in &pk_values_to_check { - let prefix = self.pk_index.build_pk_prefix(pk_str); - if existing.contains(&prefix) { - return Err(KalamDbError::AlreadyExists(format!( - "Primary key violation: value '{}' already exists in column '{}'", - pk_str, pk_name - ))); - } - } - // Fallback if we couldn't match the prefix back (shouldn't happen) - return Err(KalamDbError::AlreadyExists(format!( - "Primary key violation: a value already exists in column '{}'", - pk_name - ))); - } } // OPTIMIZED: Batch cold storage check - O(files) instead of O(files × N) // This reads Parquet files ONCE for all PK values instead of N times let pk_column_id = self.core.primary_key_column_id(); + let pk_values: Vec = + pk_values_to_check.iter().map(|(pk, _)| pk.clone()).collect(); if let Some(found_pk) = base::pk_exists_batch_in_cold( &self.core, self.core.table_id(), @@ -845,7 +886,7 @@ impl BaseTableProvider for SharedTableProvider None, // No user scoping for shared tables pk_name, pk_column_id, - &pk_values_to_check, + &pk_values, ) .await? { @@ -896,22 +937,22 @@ impl BaseTableProvider for SharedTableProvider let entries: Vec<(SharedTableRowId, SharedTableRow)> = row_keys.iter().copied().zip(shared_rows.into_iter()).collect(); - self.store.insert_batch_preencoded(&entries, encoded_values).map_err(|e| { - KalamDbError::InvalidOperation(format!( - "Failed to batch insert shared table rows: {}", - e - )) - })?; - - for (row_key, entity) in &entries { - if let Err(e) = self.stage_vector_upsert(*row_key, &entity.fields) { - log::warn!( - "Failed to stage vector upsert for table={}, seq={}: {}", - self.core.table_id(), - row_key.as_i64(), + self.store + .insert_batch_preencoded_async(entries.clone(), encoded_values) + .await + .map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to batch insert shared table rows: {}", e - ); - } + )) + })?; + + if let Err(e) = self.stage_vector_upsert_batch(&entries).await { + log::warn!( + "Failed to batch stage vector upserts for table={}: {}", + self.core.table_id(), + e + ); } // Mark manifest as having pending writes (hot data needs to be flushed) @@ -1035,22 +1076,28 @@ impl BaseTableProvider for SharedTableProvider // Resolve latest per PK - first try hot storage (O(1) via PK index), // then fall back to cold storage (Parquet scan) - let (_latest_key, latest_row) = if let Some(result) = self.find_by_pk(&pk_value_scalar)? { - result - } else { - // Not in hot storage, check cold storage - log::debug!( - "[UPDATE] PK {} not found in hot storage, querying cold storage for pk={}", - pk_name, - pk_value - ); - base::find_row_by_pk(self, None, pk_value).await?.ok_or_else(|| { - KalamDbError::NotFound(format!( - "Row with {}={} not found (checked both hot and cold storage)", + let (_latest_key, latest_row) = + if let Some(result) = self.find_by_pk(&pk_value_scalar).await? { + result + } else if self.pk_tombstoned_in_hot(&pk_value_scalar).await? { + return Err(KalamDbError::NotFound(format!( + "Row with {}={} was deleted", pk_name, pk_value - )) - })? - }; + ))); + } else { + // Not in hot storage, check cold storage + log::debug!( + "[UPDATE] PK {} not found in hot storage, querying cold storage for pk={}", + pk_name, + pk_value + ); + base::find_row_by_pk(self, None, pk_value).await?.ok_or_else(|| { + KalamDbError::NotFound(format!( + "Row with {}={} not found (checked both hot and cold storage)", + pk_name, pk_value + )) + })? + }; let mut merged = latest_row.fields.values.clone(); for (k, v) in &updates.values { @@ -1069,11 +1116,11 @@ impl BaseTableProvider for SharedTableProvider }; let row_key = seq_id; // Use insert() to update PK index for the new MVCC version - self.store.insert(&row_key, &entity).map_err(|e| { + self.store.insert_async(row_key, entity.clone()).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to update shared table row: {}", e)) })?; - if let Err(e) = self.stage_vector_upsert(seq_id, &entity.fields) { + if let Err(e) = self.stage_vector_upsert(seq_id, &entity.fields).await { log::warn!( "Failed to stage vector upsert for table={}, seq={}: {}", self.core.table_id(), @@ -1178,8 +1225,10 @@ impl BaseTableProvider for SharedTableProvider // Find latest resolved row for this PK // First try hot storage (O(1) via PK index), then fall back to cold storage (Parquet scan) - let latest_row = if let Some((_key, row)) = self.find_by_pk(&pk_value_scalar)? { + let latest_row = if let Some((_key, row)) = self.find_by_pk(&pk_value_scalar).await? { row + } else if self.pk_tombstoned_in_hot(&pk_value_scalar).await? { + return Ok(false); } else { // Not in hot storage, check cold storage match base::find_row_by_pk(self, None, pk_value).await? { @@ -1215,11 +1264,11 @@ impl BaseTableProvider for SharedTableProvider seq_id.as_i64() ); // Use insert() to update PK index for the tombstone record - self.store.insert(&row_key, &entity).map_err(|e| { + self.store.insert_async(row_key, entity.clone()).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to delete shared table row: {}", e)) })?; - if let Err(e) = self.stage_vector_delete(seq_id, pk_value) { + if let Err(e) = self.stage_vector_delete(seq_id, pk_value).await { log::warn!( "Failed to stage vector delete for table={}, seq={}, pk={}: {}", self.core.table_id(), @@ -1295,7 +1344,7 @@ impl BaseTableProvider for SharedTableProvider }; // Try hot storage PK index (O(1)) - let found = self.find_by_pk(&pk_scalar)?; + let found = self.find_by_pk(&pk_scalar).await?; if let Some((row_id, row)) = found { log::debug!( "[SharedProvider] PK fast-path hit for {}={}, _seq={}", @@ -1314,7 +1363,7 @@ impl BaseTableProvider for SharedTableProvider // Not in hot storage — check if it is tombstoned before trying cold storage. // A tombstone in hot storage means the row was deleted; falling back to Parquet // would surface a stale version and violate MVCC visibility rules. - if self.pk_tombstoned_in_hot(&pk_scalar)? { + if self.pk_tombstoned_in_hot(&pk_scalar).await? { log::debug!( "[SharedProvider] PK fast-path tombstone for {}={}", pk_name, diff --git a/backend/crates/kalamdb-tables/src/user_tables/user_table_provider.rs b/backend/crates/kalamdb-tables/src/user_tables/user_table_provider.rs index b12062eed..bd0e0ff6f 100644 --- a/backend/crates/kalamdb-tables/src/user_tables/user_table_provider.rs +++ b/backend/crates/kalamdb-tables/src/user_tables/user_table_provider.rs @@ -39,7 +39,8 @@ use kalamdb_session::{can_read_all_users, check_user_table_access, check_user_ta use kalamdb_store::EntityStore; use kalamdb_system::VectorMetric; use kalamdb_vector::{ - new_indexed_user_vector_hot_store, UserVectorHotOpId, VectorHotOp, VectorHotOpType, + new_indexed_user_vector_hot_store, UserVectorHotOpId, UserVectorHotStore, VectorHotOp, + VectorHotOpType, }; use std::any::Any; use std::collections::{BTreeMap, HashMap, HashSet}; @@ -70,6 +71,9 @@ pub struct UserTableProvider { /// Embedding columns tracked by vector hot staging: (column_name, dimensions). vector_columns: Vec<(String, u32)>, + + /// Cached vector staging stores keyed by embedding column name. + vector_stores: HashMap>, } impl UserTableProvider { @@ -80,7 +84,7 @@ impl UserTableProvider { /// * `store` - IndexedEntityStore with PK index for this table pub fn new(core: Arc, store: Arc) -> Self { let pk_index = UserTablePkIndex::new(core.table_id(), core.primary_key_field_name()); - let vector_columns = core + let vector_columns: Vec<(String, u32)> = core .table_def() .columns .iter() @@ -91,6 +95,20 @@ impl UserTableProvider { _ => None, }) .collect(); + let backend = store.backend().clone(); + let vector_stores: HashMap> = vector_columns + .iter() + .map(|(column_name, _)| { + ( + column_name.clone(), + Arc::new(new_indexed_user_vector_hot_store( + backend.clone(), + core.table_id(), + column_name, + )), + ) + }) + .collect(); if log::log_enabled!(log::Level::Debug) { let field_names: Vec<_> = core.schema().fields().iter().map(|f| f.name()).collect(); @@ -106,6 +124,7 @@ impl UserTableProvider { store, pk_index, vector_columns, + vector_stores, } } @@ -188,7 +207,7 @@ impl UserTableProvider { } } - fn stage_vector_upsert( + async fn stage_vector_upsert( &self, user_id: &UserId, seq: SeqId, @@ -200,8 +219,6 @@ impl UserTableProvider { let pk = crate::utils::unified_dml::extract_user_pk_value(row, self.primary_key_field_name())?; - let backend = self.store.backend().clone(); - for (column_name, dimensions) in &self.vector_columns { let Some(value) = row.get(column_name.as_str()) else { continue; @@ -210,11 +227,12 @@ impl UserTableProvider { continue; }; - let store = new_indexed_user_vector_hot_store( - backend.clone(), - self.core.table_id(), - column_name, - ); + let store = self.vector_stores.get(column_name).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Missing cached vector store for column '{}'", + column_name + )) + })?; let key = UserVectorHotOpId::new(user_id.clone(), seq, pk.clone()); let op = VectorHotOp::new( self.core.table_id().clone(), @@ -226,7 +244,7 @@ impl UserTableProvider { *dimensions, VectorMetric::Cosine, ); - store.insert(&key, &op).map_err(|e| { + store.insert_async(key, op).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to stage vector upsert op: {}", e)) })?; } @@ -234,7 +252,67 @@ impl UserTableProvider { Ok(()) } - fn stage_vector_delete( + async fn stage_vector_upsert_batch( + &self, + user_id: &UserId, + entries: &[(UserTableRowId, UserTableRow)], + ) -> Result<(), KalamDbError> { + if self.vector_columns.is_empty() || entries.is_empty() { + return Ok(()); + } + + let mut ops_by_column: HashMap> = + HashMap::new(); + + for (row_key, entity) in entries { + let pk = crate::utils::unified_dml::extract_user_pk_value( + &entity.fields, + self.primary_key_field_name(), + )?; + + for (column_name, dimensions) in &self.vector_columns { + let Some(value) = entity.fields.get(column_name.as_str()) else { + continue; + }; + let Some(vector) = Self::extract_embedding_vector(value, *dimensions) else { + continue; + }; + + ops_by_column.entry(column_name.clone()).or_default().push(( + UserVectorHotOpId::new(user_id.clone(), row_key.seq, pk.clone()), + VectorHotOp::new( + self.core.table_id().clone(), + column_name.clone(), + pk.clone(), + VectorHotOpType::Upsert, + Some(vector), + None, + *dimensions, + VectorMetric::Cosine, + ), + )); + } + } + + for (column_name, ops) in ops_by_column { + let store = self.vector_stores.get(&column_name).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Missing cached vector store for column '{}'", + column_name + )) + })?; + store.insert_batch_async(ops).await.map_err(|e| { + KalamDbError::InvalidOperation(format!( + "Failed to batch stage vector upsert ops for column '{}': {}", + column_name, e + )) + })?; + } + + Ok(()) + } + + async fn stage_vector_delete( &self, user_id: &UserId, seq: SeqId, @@ -244,13 +322,13 @@ impl UserTableProvider { return Ok(()); } - let backend = self.store.backend().clone(); for (column_name, dimensions) in &self.vector_columns { - let store = new_indexed_user_vector_hot_store( - backend.clone(), - self.core.table_id(), - column_name, - ); + let store = self.vector_stores.get(column_name).ok_or_else(|| { + KalamDbError::InvalidOperation(format!( + "Missing cached vector store for column '{}'", + column_name + )) + })?; let key = UserVectorHotOpId::new(user_id.clone(), seq, pk.to_string()); let op = VectorHotOp::new( self.core.table_id().clone(), @@ -262,7 +340,7 @@ impl UserTableProvider { *dimensions, VectorMetric::Cosine, ); - store.insert(&key, &op).map_err(|e| { + store.insert_async(key, op).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to stage vector delete op: {}", e)) })?; } @@ -297,34 +375,30 @@ impl UserTableProvider { /// /// # Returns /// Option<(UserTableRowId, UserTableRow)> if found - pub fn find_by_pk( + async fn latest_hot_pk_entry( &self, user_id: &UserId, pk_value: &ScalarValue, ) -> Result, KalamDbError> { - // Build prefix for PK index scan let prefix = self.pk_index.build_prefix_for_pk(user_id, pk_value); + self.store + .get_latest_by_index_prefix_async(0, prefix) + .await + .into_kalamdb_error("PK index scan failed") + } - // Scan index for all versions with this PK - let index_results = self - .store - .scan_by_index(0, Some(&prefix), None) - .into_kalamdb_error("PK index scan failed")?; - - if index_results.is_empty() { - return Ok(None); - } - - if let Some((row_id, row)) = index_results.into_iter().max_by_key(|(row_id, _)| row_id.seq) - { + pub async fn find_by_pk( + &self, + user_id: &UserId, + pk_value: &ScalarValue, + ) -> Result, KalamDbError> { + Ok(self.latest_hot_pk_entry(user_id, pk_value).await?.and_then(|(row_id, row)| { if row._deleted { - Ok(None) + None } else { - Ok(Some((row_id, row))) + Some((row_id, row)) } - } else { - Ok(None) - } + })) } /// Returns true if the latest hot-storage version of this PK is a tombstone @@ -333,19 +407,14 @@ impl UserTableProvider { /// /// Used in the PK fast-path of `scan_rows` to prevent cold storage (Parquet) /// from surfacing a row that has already been deleted in hot storage. - fn pk_tombstoned_in_hot( + async fn pk_tombstoned_in_hot( &self, user_id: &UserId, pk_value: &ScalarValue, ) -> Result { - let prefix = self.pk_index.build_prefix_for_pk(user_id, pk_value); - let index_results = self - .store - .scan_by_index(0, Some(&prefix), None) - .into_kalamdb_error("PK index scan failed")?; - Ok(index_results - .into_iter() - .max_by_key(|(row_id, _)| row_id.seq) + Ok(self + .latest_hot_pk_entry(user_id, pk_value) + .await? .map(|(_, row)| row._deleted) .unwrap_or(false)) } @@ -441,25 +510,7 @@ impl UserTableProvider { state: &dyn Session, filters: &[Expr], ) -> DataFusionResult> { - let (user_id, _role) = - extract_user_context(state).map_err(|e| DataFusionError::Execution(e.to_string()))?; - - let kvs = self - .scan_with_version_resolution_to_kvs_async(user_id, None, None, None, false) - .await - .map_err(|e| DataFusionError::Execution(e.to_string()))?; - - let schema = self.schema_ref(); - let mut matched = Vec::new(); - for (_row_key, row) in kvs { - let fields = Self::build_notification_row(&row); - if crate::utils::datafusion_dml::row_matches_filters(state, &schema, &fields, filters)? - { - matched.push(fields); - } - } - - Ok(matched) + crate::utils::datafusion_dml::collect_matching_rows(self, state, filters).await } } @@ -500,26 +551,15 @@ impl BaseTableProvider for UserTableProvider { // Use shared helper to parse PK value let pk_value = crate::utils::pk::parse_pk_value(id_value); - // Fast path: check hot storage for a non-deleted version - if let Some((row_id, _row)) = self.find_by_pk(user_id, &pk_value)? { + if let Some((row_id, row)) = self.latest_hot_pk_entry(user_id, &pk_value).await? { + if row._deleted { + log::trace!("[UserTableProvider] PK {} latest hot version is tombstoned", id_value); + return Ok(None); + } log::trace!("[UserTableProvider] PK collision in hot storage: id={}", id_value); return Ok(Some(row_id)); } - // If hot storage has entries but all are deleted, the PK can be reused. - let hot_prefix = self.pk_index.build_prefix_for_pk(user_id, &pk_value); - let hot_has_versions = self - .store - .exists_by_index(0, &hot_prefix) - .into_kalamdb_error("PK index scan failed")?; - if hot_has_versions { - log::trace!( - "[UserTableProvider] PK {} has only deleted versions in hot storage", - id_value - ); - return Ok(None); - } - log::trace!("[UserTableProvider] PK {} not in hot storage, checking cold", id_value); // Not found in hot storage - check cold storage using optimized manifest-based lookup @@ -584,13 +624,13 @@ impl BaseTableProvider for UserTableProvider { // user_id.as_str(), seq_id); // Store the entity in RocksDB (hot storage) with PK index maintenance - self.store.insert(&row_key, &entity).map_err(|e| { + self.store.insert_async(row_key.clone(), entity.clone()).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to insert user table row: {}", e)) })?; log::debug!("Inserted user table row for user {} with _seq {}", user_id.as_str(), seq_id); - if let Err(e) = self.stage_vector_upsert(user_id, seq_id, &entity.fields) { + if let Err(e) = self.stage_vector_upsert(user_id, seq_id, &entity.fields).await { log::warn!( "Failed to stage vector upsert for table={}, user={}, seq={}: {}", self.core.table_id(), @@ -694,47 +734,33 @@ impl BaseTableProvider for UserTableProvider { // Batch PK validation: collect all user-provided PK values and their prefixes tracing::debug!(row_count, "insert_batch.pk_validation start"); let pk_name = self.primary_key_field_name(); - let mut pk_values_to_check: Vec<(String, Vec)> = Vec::new(); + let mut pk_values_to_check: Vec<(String, ScalarValue)> = Vec::new(); + let mut seen_batch_pks = HashSet::new(); { for row_data in &coerced_rows { if let Some(pk_value) = row_data.get(pk_name) { if !matches!(pk_value, ScalarValue::Null) { let pk_str = crate::utils::unified_dml::extract_user_pk_value(row_data, pk_name)?; - let prefix = self.pk_index.build_prefix_for_pk(user_id, &pk_value); - pk_values_to_check.push((pk_str, prefix)); - } - } - } - - // OPTIMIZED: Check all PKs in single index scan + HashSet lookup - // For small batches (1-2 PKs), use individual lookups to avoid scan overhead - if !pk_values_to_check.is_empty() { - if pk_values_to_check.len() <= 2 { - // Small batch: individual lookups are faster - for (pk_str, _prefix) in &pk_values_to_check { - if self.find_row_key_by_id_field(user_id, pk_str).await?.is_some() { + if !seen_batch_pks.insert(pk_str.clone()) { return Err(KalamDbError::AlreadyExists(format!( - "Primary key violation: value '{}' already exists in column '{}'", + "Primary key violation: value '{}' appears multiple times in the insert batch for column '{}'", pk_str, pk_name ))); } + pk_values_to_check.push((pk_str, pk_value.clone())); } - } else { - // Larger batch: use batch index scan for efficiency - // Build common prefix for this user's PKs - let user_prefix = self.pk_index.build_user_prefix(user_id); - let prefixes: Vec> = - pk_values_to_check.iter().map(|(_, p)| p.clone()).collect(); - - let existing = self - .store - .exists_batch_by_index(0, &user_prefix, &prefixes) - .into_kalamdb_error("Batch PK index scan failed")?; - - // Check if any of the requested PKs already exist - for (pk_str, prefix) in &pk_values_to_check { - if existing.contains(prefix) { + } + } + + // Check only the latest hot version for each requested PK. Tombstoned + // latest versions are reusable and should not fail the insert. + if !pk_values_to_check.is_empty() { + for (pk_str, pk_value) in &pk_values_to_check { + if let Some((_row_id, row)) = + self.latest_hot_pk_entry(user_id, pk_value).await? + { + if !row._deleted { return Err(KalamDbError::AlreadyExists(format!( "Primary key violation: value '{}' already exists in column '{}'", pk_str, pk_name @@ -742,6 +768,26 @@ impl BaseTableProvider for UserTableProvider { } } } + + let pk_column_id = self.core.primary_key_column_id(); + let pk_values: Vec = + pk_values_to_check.iter().map(|(pk, _)| pk.clone()).collect(); + if let Some(found_pk) = base::pk_exists_batch_in_cold( + &self.core, + self.core.table_id(), + self.core.table_type(), + Some(user_id), + pk_name, + pk_column_id, + &pk_values, + ) + .await? + { + return Err(KalamDbError::AlreadyExists(format!( + "Primary key violation: value '{}' already exists in column '{}'", + found_pk, pk_name + ))); + } } } tracing::debug!(row_count, "insert_batch.pk_validation done"); @@ -784,26 +830,23 @@ impl BaseTableProvider for UserTableProvider { let entries: Vec<(UserTableRowId, UserTableRow)> = row_keys.iter().cloned().zip(user_rows.into_iter()).collect(); - { - let _write_span = tracing::info_span!("insert_batch.store_write", row_count).entered(); - self.store.insert_batch_preencoded(&entries, encoded_values).map_err(|e| { + self.store + .insert_batch_preencoded_async(entries.clone(), encoded_values) + .await + .map_err(|e| { KalamDbError::InvalidOperation(format!( "Failed to batch insert user table rows: {}", e )) })?; - } - for (row_key, entity) in &entries { - if let Err(e) = self.stage_vector_upsert(user_id, row_key.seq, &entity.fields) { - log::warn!( - "Failed to stage vector upsert for table={}, user={}, seq={}: {}", - self.core.table_id(), - user_id.as_str(), - row_key.seq.as_i64(), - e - ); - } + if let Err(e) = self.stage_vector_upsert_batch(user_id, &entries).await { + log::warn!( + "Failed to batch stage vector upserts for table={}, user={}: {}", + self.core.table_id(), + user_id.as_str(), + e + ); } // Mark manifest as having pending writes (hot data needs to be flushed) @@ -937,8 +980,13 @@ impl BaseTableProvider for UserTableProvider { // Find latest resolved row for this PK under same user // First try hot storage (O(1) via PK index), then fall back to cold storage (Parquet scan) let (_latest_key, latest_row) = - if let Some(result) = self.find_by_pk(user_id, &pk_value_scalar)? { + if let Some(result) = self.find_by_pk(user_id, &pk_value_scalar).await? { result + } else if self.pk_tombstoned_in_hot(user_id, &pk_value_scalar).await? { + return Err(KalamDbError::NotFound(format!( + "Row with {}={} was deleted", + pk_name, pk_value + ))); } else { // Not in hot storage, check cold storage log::debug!( @@ -988,11 +1036,11 @@ impl BaseTableProvider for UserTableProvider { pk_value, seq_id.as_i64() ); - self.store.insert(&row_key, &entity).map_err(|e| { + self.store.insert_async(row_key.clone(), entity.clone()).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to update user table row: {}", e)) })?; - if let Err(e) = self.stage_vector_upsert(user_id, seq_id, &entity.fields) { + if let Err(e) = self.stage_vector_upsert(user_id, seq_id, &entity.fields).await { log::warn!( "Failed to stage vector upsert for table={}, user={}, seq={}: {}", self.core.table_id(), @@ -1099,22 +1147,25 @@ impl BaseTableProvider for UserTableProvider { // Find latest resolved row for this PK under same user // First try hot storage (O(1) via PK index), then fall back to cold storage (Parquet scan) - let latest_row = if let Some((_key, row)) = self.find_by_pk(user_id, &pk_value_scalar)? { - row - } else { - // Not in hot storage, check cold storage - match base::find_row_by_pk(self, Some(user_id), pk_value).await? { - Some((_key, row)) => row, - None => { - log::trace!( - "[UserProvider DELETE_BY_PK] Row with {}={} not found", - pk_name, - pk_value - ); - return Ok(false); - }, - } - }; + let latest_row = + if let Some((_key, row)) = self.find_by_pk(user_id, &pk_value_scalar).await? { + row + } else if self.pk_tombstoned_in_hot(user_id, &pk_value_scalar).await? { + return Ok(false); + } else { + // Not in hot storage, check cold storage + match base::find_row_by_pk(self, Some(user_id), pk_value).await? { + Some((_key, row)) => row, + None => { + log::trace!( + "[UserProvider DELETE_BY_PK] Row with {}={} not found", + pk_name, + pk_value + ); + return Ok(false); + }, + } + }; let sys_cols = self.core.services.system_columns.clone(); let seq_id = sys_cols.generate_seq_id().map_err(|e| { @@ -1139,11 +1190,11 @@ impl BaseTableProvider for UserTableProvider { seq_id.as_i64() ); // Insert tombstone version (MVCC - all writes are inserts with new SeqId) - self.store.insert(&row_key, &entity).map_err(|e| { + self.store.insert_async(row_key.clone(), entity.clone()).await.map_err(|e| { KalamDbError::InvalidOperation(format!("Failed to delete user table row: {}", e)) })?; - if let Err(e) = self.stage_vector_delete(user_id, seq_id, pk_value) { + if let Err(e) = self.stage_vector_delete(user_id, seq_id, pk_value).await { log::warn!( "Failed to stage vector delete for table={}, user={}, seq={}, pk={}: {}", self.core.table_id(), @@ -1229,7 +1280,7 @@ impl BaseTableProvider for UserTableProvider { }; // Hot storage PK index (O(1)) - let found = self.find_by_pk(user_id, &pk_scalar)?; + let found = self.find_by_pk(user_id, &pk_scalar).await?; if let Some((row_id, row)) = found { log::debug!( "[UserProvider] PK fast-path hit for {}={}, user={}", @@ -1249,7 +1300,7 @@ impl BaseTableProvider for UserTableProvider { // If so, the row has been deleted — do NOT fall back to cold storage. // Returning the Parquet version would surface a row whose latest // version is a tombstone, violating MVCC visibility rules. - if self.pk_tombstoned_in_hot(user_id, &pk_scalar)? { + if self.pk_tombstoned_in_hot(user_id, &pk_scalar).await? { log::debug!( "[UserProvider] PK fast-path tombstone for {}={}, user={}", pk_name, diff --git a/backend/src/main.rs b/backend/src/main.rs index 0594bc1bf..db41f17bc 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -283,7 +283,9 @@ async fn main() -> Result<()> { &config.logging.format, &config.logging.otlp, ) - .map_err(|error| anyhow::anyhow!("Failed to initialize logging at '{}': {}", server_log_path, error))?; + .map_err(|error| { + anyhow::anyhow!("Failed to initialize logging at '{}': {}", server_log_path, error) + })?; // Display enhanced version information info!("KalamDB Server v{:<10} | Build: {}", SERVER_VERSION, BUILD_DATE); diff --git a/backend/tests/testserver/subscription/test_live_query_updates.rs b/backend/tests/testserver/subscription/test_live_query_updates.rs index c04d8aab7..e4da3ded3 100644 --- a/backend/tests/testserver/subscription/test_live_query_updates.rs +++ b/backend/tests/testserver/subscription/test_live_query_updates.rs @@ -126,3 +126,126 @@ async fn test_live_query_detects_updates() -> anyhow::Result<()> { } Ok(()) } + +/// Test UPDATE detection for subscriptions that use LIKE filters +#[tokio::test] +#[ntest::timeout(15000)] +async fn test_live_query_detects_updates_with_like_filter() -> anyhow::Result<()> { + let server = super::test_support::http_server::get_global_server().await; + let ns = unique_namespace("test_updates_like"); + let table = "metrics"; + + let resp = server.execute_sql(&format!("CREATE NAMESPACE {}", ns)).await?; + assert_eq!(resp.status, ResponseStatus::Success); + + let resp = server + .execute_sql(&format!( + "CREATE TABLE {}.{} ( + id TEXT PRIMARY KEY, + metric_name TEXT, + metric_value BIGINT, + updated_at BIGINT + ) WITH ( + TYPE = 'USER', + STORAGE_ID = 'local' + )", + ns, table + )) + .await?; + assert_eq!(resp.status, ResponseStatus::Success); + + let resp = server + .execute_sql(&format!( + "INSERT INTO {}.{} (id, metric_name, metric_value, updated_at) VALUES ('metric1', 'open_files_other', 10, 1000)", + ns, table + )) + .await?; + assert_eq!(resp.status, ResponseStatus::Success); + + let client = server.link_client("root"); + let sql = format!( + "SELECT * FROM {}.{} WHERE metric_name LIKE 'open_files_%'", + ns, table + ); + let mut subscription = client.subscribe(&sql).await.expect("Failed to subscribe"); + + let timeout = tokio::time::sleep(Duration::from_secs(5)); + tokio::pin!(timeout); + + loop { + tokio::select! { + event = subscription.next() => { + match event { + Some(Ok(ChangeEvent::InitialDataBatch { rows, batch_control, .. })) => { + assert_eq!(rows.len(), 1, "Should have 1 initial matching row"); + assert_eq!( + rows[0].get("metric_name").and_then(|value| value.as_str()), + Some("open_files_other") + ); + if !batch_control.has_more { + break; + } + } + Some(Ok(ChangeEvent::Ack { .. })) => {} + Some(Ok(_)) => {} + Some(Err(error)) => panic!("Error receiving initial data: {:?}", error), + None => panic!("Stream ended before initial data"), + } + } + _ = &mut timeout => panic!("Timed out waiting for initial data"), + } + } + + let resp = server + .execute_sql(&format!( + "UPDATE {}.{} SET metric_value = 11, updated_at = 2000 WHERE id = 'metric1'", + ns, table + )) + .await?; + assert_eq!(resp.status, ResponseStatus::Success); + + let timeout = tokio::time::sleep(Duration::from_secs(5)); + tokio::pin!(timeout); + + loop { + tokio::select! { + event = subscription.next() => { + match event { + Some(Ok(ChangeEvent::Update { rows, old_rows, .. })) => { + assert_eq!(rows.len(), 1, "Should receive one matching update row"); + + let new_row = &rows[0]; + + assert_eq!( + new_row.get("metric_name").and_then(|value| value.as_str()), + Some("open_files_other") + ); + assert_eq!( + new_row + .get("metric_value") + .and_then(|value| value.as_str()) + .and_then(|value| value.parse::().ok()), + Some(11) + ); + if let Some(old_row) = old_rows.first() { + assert_eq!( + old_row + .get("metric_value") + .and_then(|value| value.as_str()) + .and_then(|value| value.parse::().ok()), + Some(10) + ); + } + break; + } + Some(Ok(_)) => {} + Some(Err(error)) => panic!("Error receiving update: {:?}", error), + None => panic!("Stream ended before update"), + } + } + _ = &mut timeout => panic!("Timed out waiting for filtered update"), + } + } + + Ok(()) +} diff --git a/batch_docker_ubuntu24_build.txt b/batch_docker_ubuntu24_build.txt new file mode 100644 index 000000000..66a267393 --- /dev/null +++ b/batch_docker_ubuntu24_build.txt @@ -0,0 +1,152 @@ +#0 building with "desktop-linux" instance using docker driver + +#1 [internal] load build definition from Dockerfile.prebuilt +#1 transferring dockerfile: 3.80kB done +#1 DONE 0.0s + +#2 [context binaries] load .dockerignore +#2 transferring binaries: 2B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/ubuntu:24.04 +#3 ... + +#4 [internal] load metadata for docker.io/library/busybox:1.36.1-musl +#4 DONE 2.4s + +#3 [internal] load metadata for docker.io/library/ubuntu:24.04 +#3 DONE 3.2s + +#5 [internal] load .dockerignore +#5 transferring context: 555B 0.0s done +#5 DONE 0.0s + +#6 [runtime-prep 1/4] FROM docker.io/library/busybox:1.36.1-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 +#6 resolve docker.io/library/busybox:1.36.1-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 0.0s done +#6 DONE 0.0s + +#7 [internal] load build context +#7 transferring context: 77B done +#7 DONE 0.0s + +#8 [stage-1 1/9] FROM docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 +#8 resolve docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 0.0s done +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 0B / 28.87MB 0.2s +#8 ... + +#9 [context binaries] load from client +#9 transferring binaries: 10.04MB 0.4s done +#9 DONE 0.4s + +#8 [stage-1 1/9] FROM docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 2.10MB / 28.87MB 0.6s +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 6.20MB / 28.87MB 0.8s +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 11.76MB / 28.87MB 0.9s +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 18.87MB / 28.87MB 1.1s +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 28.87MB / 28.87MB 1.2s +#8 sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 28.87MB / 28.87MB 1.3s done +#8 extracting sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 +#8 ... + +#10 [context binaries] load from client +#10 transferring binaries: 170.72MB 2.6s done +#10 DONE 2.7s + +#11 [runtime-prep 2/4] RUN mkdir -p /runtime/usr/local/bin /runtime/data/rocksdb /runtime/data/storage /runtime/data/logs /runtime/config && cp /bin/busybox /runtime/usr/local/bin/busybox +#11 CACHED + +#12 [runtime-prep 3/4] COPY backend/server.example.toml /runtime/config/server.toml +#12 CACHED + +#13 [runtime-prep 4/4] RUN sed -i 's|data_path = "\./data"|data_path = "/data"|g' /runtime/config/server.toml +#13 CACHED + +#8 [stage-1 1/9] FROM docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 +#8 extracting sha256:66a4bbbfab887561d75f1fdb3c6221c974346f82c9229f5ef99f96b7e6c25704 1.6s done +#8 DONE 3.0s + +#14 [stage-1 2/9] RUN apt-get update && apt-get install -y --no-install-recommends bash ca-certificates && rm -rf /var/lib/apt/lists/* +#14 0.675 Get:1 http://ports.ubuntu.com/ubuntu-ports noble InRelease [256 kB] +#14 1.168 Get:2 http://ports.ubuntu.com/ubuntu-ports noble-updates InRelease [126 kB] +#14 1.290 Get:3 http://ports.ubuntu.com/ubuntu-ports noble-backports InRelease [126 kB] +#14 1.411 Get:4 http://ports.ubuntu.com/ubuntu-ports noble-security InRelease [126 kB] +#14 1.536 Get:5 http://ports.ubuntu.com/ubuntu-ports noble/main arm64 Packages [1776 kB] +#14 1.828 Get:6 http://ports.ubuntu.com/ubuntu-ports noble/multiverse arm64 Packages [274 kB] +#14 1.834 Get:7 http://ports.ubuntu.com/ubuntu-ports noble/universe arm64 Packages [19.0 MB] +#14 2.328 Get:8 http://ports.ubuntu.com/ubuntu-ports noble/restricted arm64 Packages [113 kB] +#14 2.335 Get:9 http://ports.ubuntu.com/ubuntu-ports noble-updates/restricted arm64 Packages [4990 kB] +#14 2.448 Get:10 http://ports.ubuntu.com/ubuntu-ports noble-updates/multiverse arm64 Packages [41.7 kB] +#14 2.450 Get:11 http://ports.ubuntu.com/ubuntu-ports noble-updates/main arm64 Packages [2437 kB] +#14 2.508 Get:12 http://ports.ubuntu.com/ubuntu-ports noble-updates/universe arm64 Packages [1977 kB] +#14 2.546 Get:13 http://ports.ubuntu.com/ubuntu-ports noble-backports/main arm64 Packages [49.5 kB] +#14 2.546 Get:14 http://ports.ubuntu.com/ubuntu-ports noble-backports/universe arm64 Packages [36.1 kB] +#14 2.548 Get:15 http://ports.ubuntu.com/ubuntu-ports noble-backports/multiverse arm64 Packages [695 B] +#14 2.548 Get:16 http://ports.ubuntu.com/ubuntu-ports noble-security/restricted arm64 Packages [4797 kB] +#14 2.669 Get:17 http://ports.ubuntu.com/ubuntu-ports noble-security/multiverse arm64 Packages [42.1 kB] +#14 2.669 Get:18 http://ports.ubuntu.com/ubuntu-ports noble-security/universe arm64 Packages [1262 kB] +#14 2.699 Get:19 http://ports.ubuntu.com/ubuntu-ports noble-security/main arm64 Packages [2075 kB] +#14 3.904 Fetched 39.5 MB in 3s (11.6 MB/s) +#14 3.904 Reading package lists... +#14 5.737 Reading package lists... +#14 6.891 Building dependency tree... +#14 7.063 Reading state information... +#14 7.386 bash is already the newest version (5.2.21-2ubuntu4). +#14 7.386 The following NEW packages will be installed: +#14 7.386 ca-certificates openssl +#14 7.592 0 upgraded, 2 newly installed, 0 to remove and 11 not upgraded. +#14 7.592 Need to get 1144 kB of archives. +#14 7.592 After this operation, 2338 kB of additional disk space will be used. +#14 7.592 Get:1 http://ports.ubuntu.com/ubuntu-ports noble-updates/main arm64 openssl arm64 3.0.13-0ubuntu3.7 [985 kB] +#14 8.276 Get:2 http://ports.ubuntu.com/ubuntu-ports noble/main arm64 ca-certificates all 20240203 [159 kB] +#14 8.427 debconf: delaying package configuration, since apt-utils is not installed +#14 8.456 Fetched 1144 kB in 1s (1297 kB/s) +#14 8.475 Selecting previously unselected package openssl. +#14 8.475 (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 4376 files and directories currently installed.) +#14 8.497 Preparing to unpack .../openssl_3.0.13-0ubuntu3.7_arm64.deb ... +#14 8.499 Unpacking openssl (3.0.13-0ubuntu3.7) ... +#14 8.547 Selecting previously unselected package ca-certificates. +#14 8.548 Preparing to unpack .../ca-certificates_20240203_all.deb ... +#14 8.551 Unpacking ca-certificates (20240203) ... +#14 8.591 Setting up openssl (3.0.13-0ubuntu3.7) ... +#14 8.598 Setting up ca-certificates (20240203) ... +#14 8.884 Updating certificates in /etc/ssl/certs... +#14 9.218 146 added, 0 removed; done. +#14 9.232 Processing triggers for ca-certificates (20240203) ... +#14 9.237 Updating certificates in /etc/ssl/certs... +#14 9.468 0 added, 0 removed; done. +#14 9.468 Running hooks in /etc/ca-certificates/update.d... +#14 9.469 done. +#14 DONE 9.9s + +#15 [stage-1 3/9] COPY --from=binaries --chmod=755 kalamdb-server /usr/local/bin/kalamdb-server +#15 DONE 0.8s + +#16 [stage-1 4/9] COPY --from=binaries --chmod=755 kalam /usr/local/bin/kalam-cli +#16 DONE 0.1s + +#17 [stage-1 5/9] COPY --from=binaries --chmod=755 kalam /usr/local/bin/kalam +#17 DONE 0.1s + +#18 [stage-1 6/9] COPY --from=runtime-prep --chmod=755 /runtime/usr/local/bin/busybox /usr/local/bin/busybox +#18 DONE 0.1s + +#19 [stage-1 7/9] COPY --from=runtime-prep --chown=65532:65532 /runtime/data /data +#19 DONE 0.0s + +#20 [stage-1 8/9] COPY --from=runtime-prep --chown=65532:65532 /runtime/config /config +#20 DONE 0.0s + +#21 [stage-1 9/9] WORKDIR /data +#21 DONE 0.0s + +#22 exporting to image +#22 exporting layers +#22 exporting layers 4.2s done +#22 exporting manifest sha256:be260f2ab639d4794856da7989f7dedcc3367c7c2cc387dec80e47cc69e06cbd done +#22 exporting config sha256:59df2e187e82a52ff1168d8f0991817f0507bd5fdc6718bb053872d582ad12df done +#22 exporting attestation manifest sha256:ae1702f716b828e672467f792ed5f1e967972b8b24385200e3af0405aac0bcba done +#22 exporting manifest list sha256:4f91bf74e7c84467d796ff34725110c1d99e0ad83719370b1eeb601e5d5716eb done +#22 naming to docker.io/library/kalamdb:ubuntu24-scan done +#22 unpacking to docker.io/library/kalamdb:ubuntu24-scan +#22 unpacking to docker.io/library/kalamdb:ubuntu24-scan 2.0s done +#22 DONE 6.2s diff --git a/batch_docker_ubuntu24_lean_build.txt b/batch_docker_ubuntu24_lean_build.txt new file mode 100644 index 000000000..540175284 --- /dev/null +++ b/batch_docker_ubuntu24_lean_build.txt @@ -0,0 +1,111 @@ +#0 building with "desktop-linux" instance using docker driver + +#1 [internal] load build definition from Dockerfile.prebuilt +#1 transferring dockerfile: 4.46kB done +#1 DONE 0.0s + +#2 [context binaries] load .dockerignore +#2 transferring binaries: 2B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/busybox:1.36.1-musl +#3 ... + +#4 [internal] load metadata for docker.io/library/ubuntu:24.04 +#4 DONE 4.1s + +#3 [internal] load metadata for docker.io/library/busybox:1.36.1-musl +#3 DONE 4.1s + +#5 [internal] load .dockerignore +#5 transferring context: 555B done +#5 DONE 0.0s + +#6 [binary-prep 1/6] FROM docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 +#6 resolve docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 0.0s done +#6 DONE 0.0s + +#7 [runtime-prep 1/4] FROM docker.io/library/busybox:1.36.1-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 +#7 resolve docker.io/library/busybox:1.36.1-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 0.0s done +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 77B done +#8 DONE 0.0s + +#9 [stage-2 2/9] RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +#9 CACHED + +#10 [runtime-prep 2/4] RUN mkdir -p /runtime/usr/local/bin /runtime/data/rocksdb /runtime/data/storage /runtime/data/logs /runtime/config && cp /bin/busybox /runtime/usr/local/bin/busybox +#10 CACHED + +#11 [runtime-prep 3/4] COPY backend/server.example.toml /runtime/config/server.toml +#11 CACHED + +#12 [runtime-prep 4/4] RUN sed -i 's|data_path = "\./data"|data_path = "/data"|g' /runtime/config/server.toml +#12 CACHED + +#13 [stage-2 3/9] COPY --from=runtime-prep --chmod=755 /runtime/usr/local/bin/busybox /usr/local/bin/busybox +#13 CACHED + +#14 [context binaries] load from client +#14 ... + +#15 [context binaries] load from client +#15 transferring binaries: 10.04MB 0.2s done +#15 DONE 0.2s + +#14 [context binaries] load from client +#14 transferring binaries: 170.72MB 1.8s done +#14 DONE 1.8s + +#16 [binary-prep 3/6] RUN mkdir -p /staged/usr/local/bin +#16 CACHED + +#17 [binary-prep 4/6] COPY --from=binaries kalamdb-server /staged/usr/local/bin/kalamdb-server +#17 CACHED + +#18 [binary-prep 2/6] RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends binutils && rm -rf /var/lib/apt/lists/* +#18 CACHED + +#19 [binary-prep 5/6] COPY --from=binaries kalam /staged/usr/local/bin/kalam-cli +#19 CACHED + +#20 [binary-prep 6/6] RUN chmod 755 /staged/usr/local/bin/kalamdb-server /staged/usr/local/bin/kalam-cli && strip --strip-unneeded /staged/usr/local/bin/kalamdb-server /staged/usr/local/bin/kalam-cli || true +#20 1.690 strip: /staged/usr/local/bin/kalamdb-server: file format not recognized +#20 1.690 strip: /staged/usr/local/bin/kalam-cli: file format not recognized +#20 DONE 1.9s + +#12 [runtime-prep 4/4] RUN sed -i 's|data_path = "\./data"|data_path = "/data"|g' /runtime/config/server.toml +#12 CACHED + +#21 [stage-2 4/9] COPY --from=binary-prep --chmod=755 /staged/usr/local/bin/kalamdb-server /usr/local/bin/kalamdb-server +#21 DONE 1.4s + +#22 [stage-2 5/9] COPY --from=binary-prep --chmod=755 /staged/usr/local/bin/kalam-cli /usr/local/bin/kalam-cli +#22 DONE 0.1s + +#23 [stage-2 6/9] RUN ln -sf /usr/local/bin/kalam-cli /usr/local/bin/kalam +#23 DONE 0.3s + +#24 [stage-2 7/9] COPY --from=runtime-prep --chown=65532:65532 /runtime/data /data +#24 DONE 0.0s + +#25 [stage-2 8/9] COPY --from=runtime-prep --chown=65532:65532 /runtime/config /config +#25 DONE 0.0s + +#26 [stage-2 9/9] WORKDIR /data +#26 DONE 0.0s + +#27 exporting to image +#27 exporting layers +#27 exporting layers 4.5s done +#27 exporting manifest sha256:5229e1d9ec81ded4e7bb4f4cf9eebfdebf1320988914140d267361577d2530ff +#27 exporting manifest sha256:5229e1d9ec81ded4e7bb4f4cf9eebfdebf1320988914140d267361577d2530ff done +#27 exporting config sha256:9b77d0d1ccd00043f3b380f76ece18248b8734a54094a3f2c7653bd93ab92a0a done +#27 exporting attestation manifest sha256:6d37d0a46fe54ed5f709653accf25d5afc343d8e04c233cbec375042daee26c7 done +#27 exporting manifest list sha256:3d3656c2f348784af05847bec475545c94cff5f5e70adcb9af22944f6086b4f5 done +#27 naming to docker.io/library/kalamdb:ubuntu24-lts-lean done +#27 unpacking to docker.io/library/kalamdb:ubuntu24-lts-lean +#27 unpacking to docker.io/library/kalamdb:ubuntu24-lts-lean 1.0s done +#27 DONE 5.5s diff --git a/batch_docker_ubuntu24_min_build.txt b/batch_docker_ubuntu24_min_build.txt new file mode 100644 index 000000000..430e51e82 --- /dev/null +++ b/batch_docker_ubuntu24_min_build.txt @@ -0,0 +1,138 @@ +#0 building with "desktop-linux" instance using docker driver + +#1 [internal] load build definition from Dockerfile.prebuilt +#1 transferring dockerfile: 3.90kB done +#1 DONE 0.0s + +#2 [context binaries] load .dockerignore +#2 transferring binaries: 2B done +#2 DONE 0.0s + +#3 [internal] load metadata for docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 +#3 DONE 0.0s + +#4 [internal] load metadata for docker.io/library/busybox:1.36.1-musl +#4 DONE 760.5s + +#5 [internal] load .dockerignore +#5 transferring context: 555B done +#5 DONE 0.0s + +#6 [stage-1 1/9] FROM docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 +#6 resolve docker.io/library/ubuntu:24.04@sha256:d1e2e92c075e5ca139d51a140fff46f84315c0fdce203eab2807c7e495eff4f9 0.0s done +#6 CACHED + +#7 [runtime-prep 1/4] FROM docker.io/library/busybox:1.36.1-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 +#7 resolve docker.io/library/busybox:1.36.1-musl@sha256:3c6ae8008e2c2eedd141725c30b20d9c36b026eb796688f88205845ef17aa213 0.0s done +#7 DONE 0.0s + +#8 [internal] load build context +#8 transferring context: 77B done +#8 DONE 0.0s + +#9 [context binaries] load from client +#9 transferring binaries: 5.08MB 0.1s +#9 transferring binaries: 10.04MB 0.2s done +#9 DONE 0.2s + +#10 [context binaries] load from client +#10 transferring binaries: 170.72MB 1.7s done +#10 DONE 1.8s + +#11 [runtime-prep 3/4] COPY backend/server.example.toml /runtime/config/server.toml +#11 CACHED + +#12 [runtime-prep 2/4] RUN mkdir -p /runtime/usr/local/bin /runtime/data/rocksdb /runtime/data/storage /runtime/data/logs /runtime/config && cp /bin/busybox /runtime/usr/local/bin/busybox +#12 CACHED + +#13 [runtime-prep 4/4] RUN sed -i 's|data_path = "\./data"|data_path = "/data"|g' /runtime/config/server.toml +#13 CACHED + +#14 [stage-1 2/9] RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && rm -rf /var/lib/apt/lists/* +#14 0.485 Get:1 http://ports.ubuntu.com/ubuntu-ports noble InRelease [256 kB] +#14 0.980 Get:2 http://ports.ubuntu.com/ubuntu-ports noble-updates InRelease [126 kB] +#14 1.097 Get:3 http://ports.ubuntu.com/ubuntu-ports noble-backports InRelease [126 kB] +#14 1.214 Get:4 http://ports.ubuntu.com/ubuntu-ports noble-security InRelease [126 kB] +#14 1.332 Get:5 http://ports.ubuntu.com/ubuntu-ports noble/universe arm64 Packages [19.0 MB] +#14 2.039 Get:6 http://ports.ubuntu.com/ubuntu-ports noble/multiverse arm64 Packages [274 kB] +#14 2.039 Get:7 http://ports.ubuntu.com/ubuntu-ports noble/restricted arm64 Packages [113 kB] +#14 2.039 Get:8 http://ports.ubuntu.com/ubuntu-ports noble/main arm64 Packages [1776 kB] +#14 2.086 Get:9 http://ports.ubuntu.com/ubuntu-ports noble-updates/universe arm64 Packages [1977 kB] +#14 2.122 Get:10 http://ports.ubuntu.com/ubuntu-ports noble-updates/restricted arm64 Packages [4990 kB] +#14 2.238 Get:11 http://ports.ubuntu.com/ubuntu-ports noble-updates/main arm64 Packages [2437 kB] +#14 2.444 Get:12 http://ports.ubuntu.com/ubuntu-ports noble-updates/multiverse arm64 Packages [41.7 kB] +#14 2.446 Get:13 http://ports.ubuntu.com/ubuntu-ports noble-backports/multiverse arm64 Packages [695 B] +#14 2.446 Get:14 http://ports.ubuntu.com/ubuntu-ports noble-backports/universe arm64 Packages [36.1 kB] +#14 2.446 Get:15 http://ports.ubuntu.com/ubuntu-ports noble-backports/main arm64 Packages [49.5 kB] +#14 2.447 Get:16 http://ports.ubuntu.com/ubuntu-ports noble-security/universe arm64 Packages [1262 kB] +#14 2.533 Get:17 http://ports.ubuntu.com/ubuntu-ports noble-security/multiverse arm64 Packages [42.1 kB] +#14 2.535 Get:18 http://ports.ubuntu.com/ubuntu-ports noble-security/restricted arm64 Packages [4797 kB] +#14 2.638 Get:19 http://ports.ubuntu.com/ubuntu-ports noble-security/main arm64 Packages [2075 kB] +#14 3.370 Fetched 39.5 MB in 3s (12.9 MB/s) +#14 3.370 Reading package lists... +#14 4.319 Reading package lists... +#14 5.327 Building dependency tree... +#14 5.480 Reading state information... +#14 5.724 The following additional packages will be installed: +#14 5.724 openssl +#14 5.757 The following NEW packages will be installed: +#14 5.758 ca-certificates openssl +#14 5.943 0 upgraded, 2 newly installed, 0 to remove and 11 not upgraded. +#14 5.943 Need to get 1144 kB of archives. +#14 5.943 After this operation, 2338 kB of additional disk space will be used. +#14 5.943 Get:1 http://ports.ubuntu.com/ubuntu-ports noble-updates/main arm64 openssl arm64 3.0.13-0ubuntu3.7 [985 kB] +#14 6.566 Get:2 http://ports.ubuntu.com/ubuntu-ports noble/main arm64 ca-certificates all 20240203 [159 kB] +#14 6.772 debconf: delaying package configuration, since apt-utils is not installed +#14 6.807 Fetched 1144 kB in 1s (1425 kB/s) +#14 6.823 Selecting previously unselected package openssl. +#14 6.823 (Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 4376 files and directories currently installed.) +#14 6.829 Preparing to unpack .../openssl_3.0.13-0ubuntu3.7_arm64.deb ... +#14 6.830 Unpacking openssl (3.0.13-0ubuntu3.7) ... +#14 6.864 Selecting previously unselected package ca-certificates. +#14 6.865 Preparing to unpack .../ca-certificates_20240203_all.deb ... +#14 6.867 Unpacking ca-certificates (20240203) ... +#14 6.918 Setting up openssl (3.0.13-0ubuntu3.7) ... +#14 6.924 Setting up ca-certificates (20240203) ... +#14 7.171 Updating certificates in /etc/ssl/certs... +#14 7.458 146 added, 0 removed; done. +#14 7.469 Processing triggers for ca-certificates (20240203) ... +#14 7.472 Updating certificates in /etc/ssl/certs... +#14 7.676 0 added, 0 removed; done. +#14 7.676 Running hooks in /etc/ca-certificates/update.d... +#14 7.677 done. +#14 DONE 7.8s + +#15 [stage-1 3/9] COPY --from=binaries --chmod=755 kalamdb-server /usr/local/bin/kalamdb-server +#15 DONE 0.4s + +#16 [stage-1 4/9] COPY --from=binaries --chmod=755 kalam /usr/local/bin/kalam-cli +#16 DONE 0.1s + +#17 [stage-1 5/9] COPY --from=binaries --chmod=755 kalam /usr/local/bin/kalam +#17 DONE 0.1s + +#18 [stage-1 6/9] COPY --from=runtime-prep --chmod=755 /runtime/usr/local/bin/busybox /usr/local/bin/busybox +#18 DONE 0.0s + +#19 [stage-1 7/9] COPY --from=runtime-prep --chown=65532:65532 /runtime/data /data +#19 DONE 0.0s + +#20 [stage-1 8/9] COPY --from=runtime-prep --chown=65532:65532 /runtime/config /config +#20 DONE 0.0s + +#21 [stage-1 9/9] WORKDIR /data +#21 DONE 0.0s + +#22 exporting to image +#22 exporting layers +#22 exporting layers 5.0s done +#22 exporting manifest sha256:459dbb6e8d6c61039a4494a29e9020ced28f9be0cbc530ecad900d357fdd273b +#22 exporting manifest sha256:459dbb6e8d6c61039a4494a29e9020ced28f9be0cbc530ecad900d357fdd273b 0.0s done +#22 exporting config sha256:4b9e143b4f0ec0d80c63a79f9311fd1a2ff1d5417f2fdb234f3bffad9c409bee 0.0s done +#22 exporting attestation manifest sha256:7f31ed67287c27603324cf735b56082b6811accf233f9c13cc60904a88c64353 0.0s done +#22 exporting manifest list sha256:1bc8d7fc05244a56b43f3990ff9b349ced76d4dc809103acb40c2d2c87919b29 0.0s done +#22 naming to docker.io/library/kalamdb:ubuntu24-lts-min +#22 naming to docker.io/library/kalamdb:ubuntu24-lts-min done +#22 unpacking to docker.io/library/kalamdb:ubuntu24-lts-min +#22 unpacking to docker.io/library/kalamdb:ubuntu24-lts-min 1.5s done +#22 DONE 6.7s diff --git a/binaries-amd64/kalam b/binaries-amd64/kalam new file mode 100755 index 000000000..e798b3378 Binary files /dev/null and b/binaries-amd64/kalam differ diff --git a/binaries-amd64/kalamdb-server b/binaries-amd64/kalamdb-server new file mode 100755 index 000000000..01ce127f1 Binary files /dev/null and b/binaries-amd64/kalamdb-server differ diff --git a/cli/src/session.rs b/cli/src/session.rs index 21064cc98..e8f346c47 100644 --- a/cli/src/session.rs +++ b/cli/src/session.rs @@ -13,8 +13,9 @@ use clap::ValueEnum; use colored::*; use indicatif::{ProgressBar, ProgressStyle}; use kalam_link::{ + credentials::{CredentialStore, Credentials}, AuthProvider, ConnectionOptions, KalamLinkClient, KalamLinkError, KalamLinkTimeouts, - SubscriptionConfig, SubscriptionOptions, TimestampFormatter, UploadProgress, + QueryResponse, SubscriptionConfig, SubscriptionOptions, TimestampFormatter, UploadProgress, UploadProgressCallback, }; use rustyline::completion::Completer; @@ -374,6 +375,91 @@ impl CLISession { }) } + async fn execute_query_with_auth_retry( + &mut self, + sql: &str, + ) -> std::result::Result { + let result = self.client.execute_query(sql, None, None, None).await; + match result { + Err(error) if Self::is_expired_token_error(&error) => { + if self.try_refresh_session_token().await { + self.client.execute_query(sql, None, None, None).await + } else { + Err(error) + } + }, + other => other, + } + } + + fn is_expired_token_error(error: &KalamLinkError) -> bool { + fn message_matches(message: &str) -> bool { + let lowered = message.to_ascii_lowercase(); + lowered.contains("token expired") || lowered.contains("expired token") + } + + match error { + KalamLinkError::ServerError { + status_code, + message, + } => *status_code == 401 && message_matches(message), + KalamLinkError::AuthenticationError(message) => message_matches(message), + _ => false, + } + } + + async fn try_refresh_session_token(&mut self) -> bool { + let Some(instance) = self.instance.clone() else { + return false; + }; + + let Some(creds) = self + .credential_store + .as_ref() + .and_then(|store| store.get_credentials(&instance).ok().flatten()) + else { + return false; + }; + + if !creds.can_refresh() { + return false; + } + + let Some(refresh_token) = creds.refresh_token.as_deref() else { + return false; + }; + + let Ok(login_response) = self.client.refresh_access_token(refresh_token).await else { + return false; + }; + + let username = login_response.user.username.clone(); + let access_token = login_response.access_token.clone(); + let refreshed_creds = Credentials::with_refresh_token( + instance.clone(), + access_token.clone(), + username.clone(), + login_response.expires_at.clone(), + creds.server_url.clone().or_else(|| Some(self.server_url.clone())), + login_response.refresh_token.clone().or_else(|| creds.refresh_token.clone()), + login_response + .refresh_expires_at + .clone() + .or_else(|| creds.refresh_expires_at.clone()), + ); + + if let Some(store) = self.credential_store.as_mut() { + if let Err(error) = store.set_credentials(&refreshed_creds) { + eprintln!("Warning: Could not save refreshed credentials: {}", error); + } + } + + self.client.set_auth(AuthProvider::jwt_token(access_token)); + self.username = username; + self.credentials_loaded = true; + true + } + /// Execute a SQL query with loading indicator /// /// **Implements T092**: Execute SQL via kalam-link client @@ -430,7 +516,7 @@ impl CLISession { // Execute the query let result = if upload_parts.is_empty() { - self.client.execute_query(sql, None, None, None).await + self.execute_query_with_auth_retry(&sql_to_send).await } else { let mut parts_for_send = Vec::with_capacity(upload_parts.len()); for part in upload_parts.iter_mut() { @@ -2817,6 +2903,213 @@ impl Helper for CLIHelper {} #[cfg(test)] mod tests { use super::*; + use crate::credentials::FileCredentialStore; + use kalam_link::credentials::{CredentialStore, Credentials}; + use ntest::timeout; + use serde_json::json; + use std::collections::HashMap; + use std::sync::Arc; + use tempfile::TempDir; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + use tokio::sync::Mutex as AsyncMutex; + + #[derive(Debug, Default)] + struct TestServerState { + sql_authorization_headers: Vec, + refresh_authorization_headers: Vec, + } + + struct TestServer { + base_url: String, + state: Arc>, + task: tokio::task::JoinHandle<()>, + } + + impl TestServer { + async fn spawn() -> Self { + let listener = + TcpListener::bind("127.0.0.1:0").await.expect("bind test server listener"); + let address = listener.local_addr().expect("read local addr"); + let state = Arc::new(AsyncMutex::new(TestServerState::default())); + let state_clone = Arc::clone(&state); + + let task = tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + let state = Arc::clone(&state_clone); + tokio::spawn(async move { + let _ = handle_test_connection(stream, state).await; + }); + } + }); + + Self { + base_url: format!("http://{}", address), + state, + task, + } + } + } + + impl Drop for TestServer { + fn drop(&mut self) { + self.task.abort(); + } + } + + async fn handle_test_connection( + mut stream: TcpStream, + state: Arc>, + ) -> std::io::Result<()> { + let request = read_http_request(&mut stream).await?; + let authorization = request.headers.get("authorization").cloned(); + + let (status_line, body) = match request.path.as_str() { + "/v1/api/healthcheck" => ( + "HTTP/1.1 200 OK", + json!({ + "status": "healthy", + "version": "test", + "api_version": "v1", + "build_date": null + }) + .to_string(), + ), + "/v1/api/sql" => { + if let Some(header) = authorization.clone() { + state.lock().await.sql_authorization_headers.push(header.clone()); + match header.as_str() { + "Bearer expired-token" => { + ("HTTP/1.1 401 Unauthorized", "Token expired".to_string()) + }, + "Bearer fresh-token" => ( + "HTTP/1.1 200 OK", + json!({ + "status": "success", + "results": [{ + "schema": [{ + "name": "count", + "data_type": "BigInt", + "index": 0 + }], + "rows": [["1"]], + "row_count": 1 + }], + "took": 1.0 + }) + .to_string(), + ), + _ => ("HTTP/1.1 401 Unauthorized", "Unauthorized".to_string()), + } + } else { + ("HTTP/1.1 401 Unauthorized", "Missing authorization".to_string()) + } + }, + "/v1/api/auth/refresh" => { + if let Some(header) = authorization.clone() { + state.lock().await.refresh_authorization_headers.push(header.clone()); + if header == "Bearer refresh-token" { + ( + "HTTP/1.1 200 OK", + json!({ + "user": { + "id": "user-1", + "username": "admin", + "role": "dba", + "email": null, + "created_at": "2026-03-17T00:00:00Z", + "updated_at": "2026-03-17T00:00:00Z" + }, + "expires_at": "2099-01-01T00:00:00Z", + "access_token": "fresh-token", + "refresh_token": "fresh-refresh-token", + "refresh_expires_at": "2099-02-01T00:00:00Z" + }) + .to_string(), + ) + } else { + ("HTTP/1.1 401 Unauthorized", "Invalid refresh token".to_string()) + } + } else { + ("HTTP/1.1 401 Unauthorized", "Missing authorization".to_string()) + } + }, + _ => ("HTTP/1.1 404 Not Found", "Not found".to_string()), + }; + + let response = format!( + "{status_line}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).await?; + stream.shutdown().await + } + + struct TestHttpRequest { + path: String, + headers: HashMap, + } + + async fn read_http_request(stream: &mut TcpStream) -> std::io::Result { + let mut buffer = Vec::new(); + let mut temp = [0_u8; 1024]; + let mut header_end = None; + let mut content_length = 0_usize; + + loop { + let bytes_read = stream.read(&mut temp).await?; + if bytes_read == 0 { + break; + } + buffer.extend_from_slice(&temp[..bytes_read]); + + if header_end.is_none() { + if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + header_end = Some(position + 4); + let header_text = String::from_utf8_lossy(&buffer[..position]); + for line in header_text.lines().skip(1) { + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + } + } + + if let Some(end) = header_end { + if buffer.len() >= end + content_length { + break; + } + } + } + + let header_end = header_end.expect("request should include headers"); + let header_text = String::from_utf8_lossy(&buffer[..header_end - 4]); + let mut lines = header_text.lines(); + let request_line = lines.next().expect("request line"); + let path = request_line.split_whitespace().nth(1).expect("request path").to_string(); + + let mut headers = HashMap::new(); + for line in lines { + if let Some((name, value)) = line.split_once(':') { + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string()); + } + } + + Ok(TestHttpRequest { path, headers }) + } + + fn create_temp_store() -> (FileCredentialStore, TempDir) { + let temp_dir = TempDir::new().expect("create temp dir"); + let store_path = temp_dir.path().join("credentials.toml"); + let store = FileCredentialStore::with_path(store_path).expect("create credential store"); + (store, temp_dir) + } #[test] fn test_output_format() { @@ -2845,4 +3138,66 @@ mod tests { assert_eq!(sql, "SELECT * FROM table"); assert!(options.is_some()); } + + #[tokio::test] + #[timeout(5000)] + async fn test_execute_refreshes_expired_token_during_active_session() { + let server = TestServer::spawn().await; + let (mut store, _temp_dir) = create_temp_store(); + let creds = Credentials::with_refresh_token( + "local".to_string(), + "expired-token".to_string(), + "admin".to_string(), + "2000-01-01T00:00:00Z".to_string(), + Some(server.base_url.clone()), + Some("refresh-token".to_string()), + Some("2099-01-01T00:00:00Z".to_string()), + ); + store.set_credentials(&creds).expect("store initial credentials"); + + let mut session = CLISession::with_auth_and_instance( + server.base_url.clone(), + AuthProvider::jwt_token("expired-token".to_string()), + OutputFormat::Json, + false, + Some("local".to_string()), + Some(store), + Some("admin".to_string()), + Some(0), + false, + Some(Duration::from_secs(2)), + None, + None, + CLIConfiguration::default(), + crate::config::default_config_path(), + true, + ) + .await + .expect("create session"); + + session + .execute("SELECT count(*) FROM dba.stats") + .await + .expect("query should succeed after token refresh"); + + let stored = session + .credential_store + .as_ref() + .expect("credential store available") + .get_credentials("local") + .expect("load refreshed credentials") + .expect("stored credentials present"); + assert_eq!(stored.jwt_token, "fresh-token"); + assert_eq!(stored.refresh_token.as_deref(), Some("fresh-refresh-token")); + + let state = server.state.lock().await; + assert_eq!( + state.sql_authorization_headers, + vec![ + "Bearer expired-token".to_string(), + "Bearer fresh-token".to_string() + ] + ); + assert_eq!(state.refresh_authorization_headers, vec!["Bearer refresh-token".to_string()]); + } } diff --git a/docker/README.md b/docker/README.md index 337aaa456..82de11628 100644 --- a/docker/README.md +++ b/docker/README.md @@ -182,7 +182,7 @@ Important environment variables used by the compose files: - `KALAMDB_JWT_SECRET`: required for safe non-localhost deployments - `KALAMDB_ROOT_PASSWORD`: optional root password for immediate login - `KALAMDB_PORT`: host port override for the single-node compose setup -- `RUST_LOG`: server log level +- `KALAMDB_LOG_LEVEL`: server log level ## Persistence diff --git a/docker/REPO-README.md b/docker/REPO-README.md index 8589c470d..fcbd967ee 100644 --- a/docker/REPO-README.md +++ b/docker/REPO-README.md @@ -185,7 +185,7 @@ These are the most useful environment variables for setup and day-to-day use. | `KALAMDB_SERVER_HOST` | Bind address inside the container | `0.0.0.0` | | `KALAMDB_ROOT_PASSWORD` | Root user password for initial login | `kalamdb123` | | `KALAMDB_JWT_SECRET` | JWT signing secret, should be at least 32 chars | `super-secret-value` | -| `RUST_LOG` | Logging level | `info` | +| `KALAMDB_LOG_LEVEL` | Logging level | `info` | | `KALAMDB_PORT` | Host port override in single-node compose | `8080` | | `KALAMDB_ALLOW_REMOTE_SETUP` | Allow initial remote setup in Docker-based development flows | `true` | | `KALAMDB_CLUSTER_ID` | Cluster identifier for multi-node deployments | `docker-cluster` | diff --git a/docker/build/Dockerfile.prebuilt b/docker/build/Dockerfile.prebuilt index 994b4b79d..de5b7a912 100644 --- a/docker/build/Dockerfile.prebuilt +++ b/docker/build/Dockerfile.prebuilt @@ -16,12 +16,29 @@ RUN mkdir -p /runtime/usr/local/bin /runtime/data/rocksdb /runtime/data/storage COPY backend/server.example.toml /runtime/config/server.toml RUN sed -i 's|data_path = "\./data"|data_path = "/data"|g' /runtime/config/server.toml -# Debian 13 slim keeps the runtime minimal while still providing glibc and bash. -# This trades a slightly larger image for a usable interactive shell and standard userland. -FROM debian:13-slim +# Prepare release-built Linux binaries for the runtime image. +# Strip debug/symbol data when possible to keep the final image smaller. +FROM ubuntu:24.04 AS binary-prep RUN apt-get update && \ - apt-get install -y --no-install-recommends bash ca-certificates && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends binutils && \ + rm -rf /var/lib/apt/lists/* + +RUN mkdir -p /staged/usr/local/bin + +COPY --from=binaries kalamdb-server /staged/usr/local/bin/kalamdb-server +COPY --from=binaries kalam /staged/usr/local/bin/kalam-cli + +RUN chmod 755 /staged/usr/local/bin/kalamdb-server /staged/usr/local/bin/kalam-cli && \ + strip --strip-unneeded /staged/usr/local/bin/kalamdb-server /staged/usr/local/bin/kalam-cli || true + +# Ubuntu 24.04 is the current LTS release. The base image already includes bash, +# so only install CA certificates to keep the runtime as small as possible while +# preserving HTTPS support for CLI and runtime tooling. +FROM ubuntu:24.04 + +RUN apt-get update && \ + DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends ca-certificates && \ rm -rf /var/lib/apt/lists/* ARG OCI_IMAGE_TITLE="KalamDB" @@ -48,12 +65,11 @@ LABEL org.opencontainers.image.title="${OCI_IMAGE_TITLE}" \ org.opencontainers.image.revision="${OCI_IMAGE_REVISION}" \ org.opencontainers.image.created="${OCI_IMAGE_CREATED}" -# Copy pre-built binaries from build context (provided via --build-context binaries=...) -# The build context should contain kalamdb-server and kalam binaries -COPY --from=binaries --chmod=755 kalamdb-server /usr/local/bin/kalamdb-server -COPY --from=binaries --chmod=755 kalam /usr/local/bin/kalam-cli -COPY --from=binaries --chmod=755 kalam /usr/local/bin/kalam +# Copy pre-built binaries from the prepared binary stage. COPY --from=runtime-prep --chmod=755 /runtime/usr/local/bin/busybox /usr/local/bin/busybox +COPY --from=binary-prep --chmod=755 /staged/usr/local/bin/kalamdb-server /usr/local/bin/kalamdb-server +COPY --from=binary-prep --chmod=755 /staged/usr/local/bin/kalam-cli /usr/local/bin/kalam-cli +RUN ln -sf /usr/local/bin/kalam-cli /usr/local/bin/kalam # Copy writable runtime paths and the normalized config with non-root ownership. COPY --from=runtime-prep --chown=65532:65532 /runtime/data /data diff --git a/link/sdks/dart/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so b/link/sdks/dart/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so index e9a18d4c4..99f1a6a8f 100755 Binary files a/link/sdks/dart/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so and b/link/sdks/dart/android/src/main/jniLibs/arm64-v8a/libkalam_link_dart.so differ diff --git a/link/sdks/dart/android/src/main/jniLibs/x86_64/libkalam_link_dart.so b/link/sdks/dart/android/src/main/jniLibs/x86_64/libkalam_link_dart.so index 4ac4c37f7..1f12c20cf 100755 Binary files a/link/sdks/dart/android/src/main/jniLibs/x86_64/libkalam_link_dart.so and b/link/sdks/dart/android/src/main/jniLibs/x86_64/libkalam_link_dart.so differ diff --git a/link/sdks/dart/ios/Frameworks/libkalam_link_dart.a b/link/sdks/dart/ios/Frameworks/libkalam_link_dart.a index aa892f93a..f0ad40a11 100644 Binary files a/link/sdks/dart/ios/Frameworks/libkalam_link_dart.a and b/link/sdks/dart/ios/Frameworks/libkalam_link_dart.a differ diff --git a/link/sdks/dart/web/pkg/kalam_link_dart.d.ts b/link/sdks/dart/web/pkg/kalam_link_dart.d.ts index 9900dc33b..f4b42b93d 100644 --- a/link/sdks/dart/web/pkg/kalam_link_dart.d.ts +++ b/link/sdks/dart/web/pkg/kalam_link_dart.d.ts @@ -1191,6 +1191,13 @@ export type InitInput = RequestInfo | URL | Response | BufferSource | WebAssembl export interface InitOutput { readonly memory: WebAssembly.Memory; + readonly __wbg_wasmtimestampformatter_free: (a: number, b: number) => void; + readonly parseIso8601: (a: number, b: number) => [number, number, number]; + readonly timestampNow: () => number; + readonly wasmtimestampformatter_format: (a: number, b: number, c: number) => [number, number]; + readonly wasmtimestampformatter_formatRelative: (a: number, b: number) => [number, number]; + readonly wasmtimestampformatter_new: () => number; + readonly wasmtimestampformatter_withFormat: (a: number, b: number) => [number, number, number]; readonly __wbg_kalamclient_free: (a: number, b: number) => void; readonly kalamclient_ack: (a: number, b: number, c: number, d: number, e: number, f: number, g: bigint) => any; readonly kalamclient_anonymous: (a: number, b: number) => [number, number, number]; @@ -1229,21 +1236,14 @@ export interface InitOutput { readonly kalamclient_subscribeWithSql: (a: number, b: number, c: number, d: number, e: number, f: any) => any; readonly kalamclient_unsubscribe: (a: number, b: number, c: number) => any; readonly kalamclient_withJwt: (a: number, b: number, c: number, d: number) => [number, number, number]; - readonly __wbg_wasmtimestampformatter_free: (a: number, b: number) => void; - readonly parseIso8601: (a: number, b: number) => [number, number, number]; - readonly timestampNow: () => number; - readonly wasmtimestampformatter_format: (a: number, b: number, c: number) => [number, number]; - readonly wasmtimestampformatter_formatRelative: (a: number, b: number) => [number, number]; - readonly wasmtimestampformatter_new: () => number; - readonly wasmtimestampformatter_withFormat: (a: number, b: number) => [number, number, number]; - readonly wasm_bindgen__closure__destroy__h3082ecb8e5442115: (a: number, b: number) => void; - readonly wasm_bindgen__closure__destroy__hc4784aa82de56652: (a: number, b: number) => void; - readonly wasm_bindgen__convert__closures_____invoke__h6537501fed6ccdff: (a: number, b: number, c: any) => [number, number]; - readonly wasm_bindgen__convert__closures_____invoke__h11188c184bbe4ba9: (a: number, b: number, c: any, d: any) => void; - readonly wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471: (a: number, b: number, c: any) => void; - readonly wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_1: (a: number, b: number, c: any) => void; - readonly wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_2: (a: number, b: number, c: any) => void; - readonly wasm_bindgen__convert__closures_____invoke__h3bc7e3c5702cdef5: (a: number, b: number) => void; + readonly wasm_bindgen__closure__destroy__h28a5275c8860f744: (a: number, b: number) => void; + readonly wasm_bindgen__closure__destroy__hb0865e1ee6322b2b: (a: number, b: number) => void; + readonly wasm_bindgen__convert__closures_____invoke__hcd5247acc038309e: (a: number, b: number, c: any) => [number, number]; + readonly wasm_bindgen__convert__closures_____invoke__h44aa07c92bd0562c: (a: number, b: number, c: any, d: any) => void; + readonly wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a: (a: number, b: number, c: any) => void; + readonly wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_1: (a: number, b: number, c: any) => void; + readonly wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_2: (a: number, b: number, c: any) => void; + readonly wasm_bindgen__convert__closures_____invoke__ha38bbadd4cfeb566: (a: number, b: number) => void; readonly __wbindgen_malloc: (a: number, b: number) => number; readonly __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; readonly __wbindgen_exn_store: (a: number) => void; diff --git a/link/sdks/dart/web/pkg/kalam_link_dart.js b/link/sdks/dart/web/pkg/kalam_link_dart.js index 200e3774f..d15446926 100644 --- a/link/sdks/dart/web/pkg/kalam_link_dart.js +++ b/link/sdks/dart/web/pkg/kalam_link_dart.js @@ -1141,7 +1141,7 @@ function __wbg_get_imports() { const a = state0.a; state0.a = 0; try { - return wasm_bindgen__convert__closures_____invoke__h11188c184bbe4ba9(a, state0.b, arg0, arg1); + return wasm_bindgen__convert__closures_____invoke__h44aa07c92bd0562c(a, state0.b, arg0, arg1); } finally { state0.a = a; } @@ -1163,7 +1163,7 @@ function __wbg_get_imports() { const a = state0.a; state0.a = 0; try { - return wasm_bindgen__convert__closures_____invoke__h11188c184bbe4ba9(a, state0.b, arg0, arg1); + return wasm_bindgen__convert__closures_____invoke__h44aa07c92bd0562c(a, state0.b, arg0, arg1); } finally { state0.a = a; } @@ -1299,28 +1299,28 @@ function __wbg_get_imports() { return ret; }, __wbindgen_cast_0000000000000001: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { dtor_idx: 134, function: Function { arguments: [NamedExternref("CloseEvent")], shim_idx: 135, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h3082ecb8e5442115, wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471); + // Cast intrinsic for `Closure(Closure { dtor_idx: 104, function: Function { arguments: [NamedExternref("CloseEvent")], shim_idx: 105, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. + const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h28a5275c8860f744, wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a); return ret; }, __wbindgen_cast_0000000000000002: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { dtor_idx: 134, function: Function { arguments: [NamedExternref("ErrorEvent")], shim_idx: 135, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h3082ecb8e5442115, wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_1); + // Cast intrinsic for `Closure(Closure { dtor_idx: 104, function: Function { arguments: [NamedExternref("ErrorEvent")], shim_idx: 105, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. + const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h28a5275c8860f744, wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_1); return ret; }, __wbindgen_cast_0000000000000003: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { dtor_idx: 134, function: Function { arguments: [NamedExternref("MessageEvent")], shim_idx: 135, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h3082ecb8e5442115, wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_2); + // Cast intrinsic for `Closure(Closure { dtor_idx: 104, function: Function { arguments: [NamedExternref("MessageEvent")], shim_idx: 105, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. + const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h28a5275c8860f744, wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_2); return ret; }, __wbindgen_cast_0000000000000004: function(arg0, arg1) { - // Cast intrinsic for `Closure(Closure { dtor_idx: 134, function: Function { arguments: [], shim_idx: 138, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h3082ecb8e5442115, wasm_bindgen__convert__closures_____invoke__h3bc7e3c5702cdef5); + // Cast intrinsic for `Closure(Closure { dtor_idx: 104, function: Function { arguments: [], shim_idx: 107, ret: Unit, inner_ret: Some(Unit) }, mutable: true }) -> Externref`. + const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__h28a5275c8860f744, wasm_bindgen__convert__closures_____invoke__ha38bbadd4cfeb566); return ret; }, __wbindgen_cast_0000000000000005: function(arg0, arg1) { // Cast intrinsic for `Closure(Closure { dtor_idx: 237, function: Function { arguments: [Externref], shim_idx: 238, ret: Result(Unit), inner_ret: Some(Result(Unit)) }, mutable: true }) -> Externref`. - const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__hc4784aa82de56652, wasm_bindgen__convert__closures_____invoke__h6537501fed6ccdff); + const ret = makeMutClosure(arg0, arg1, wasm.wasm_bindgen__closure__destroy__hb0865e1ee6322b2b, wasm_bindgen__convert__closures_____invoke__hcd5247acc038309e); return ret; }, __wbindgen_cast_0000000000000006: function(arg0) { @@ -1359,31 +1359,31 @@ function __wbg_get_imports() { }; } -function wasm_bindgen__convert__closures_____invoke__h3bc7e3c5702cdef5(arg0, arg1) { - wasm.wasm_bindgen__convert__closures_____invoke__h3bc7e3c5702cdef5(arg0, arg1); +function wasm_bindgen__convert__closures_____invoke__ha38bbadd4cfeb566(arg0, arg1) { + wasm.wasm_bindgen__convert__closures_____invoke__ha38bbadd4cfeb566(arg0, arg1); } -function wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471(arg0, arg1, arg2) { - wasm.wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471(arg0, arg1, arg2); +function wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a(arg0, arg1, arg2) { + wasm.wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a(arg0, arg1, arg2); } -function wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_1(arg0, arg1, arg2) { - wasm.wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_1(arg0, arg1, arg2); +function wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_1(arg0, arg1, arg2) { + wasm.wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_1(arg0, arg1, arg2); } -function wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_2(arg0, arg1, arg2) { - wasm.wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_2(arg0, arg1, arg2); +function wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_2(arg0, arg1, arg2) { + wasm.wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_2(arg0, arg1, arg2); } -function wasm_bindgen__convert__closures_____invoke__h6537501fed6ccdff(arg0, arg1, arg2) { - const ret = wasm.wasm_bindgen__convert__closures_____invoke__h6537501fed6ccdff(arg0, arg1, arg2); +function wasm_bindgen__convert__closures_____invoke__hcd5247acc038309e(arg0, arg1, arg2) { + const ret = wasm.wasm_bindgen__convert__closures_____invoke__hcd5247acc038309e(arg0, arg1, arg2); if (ret[1]) { throw takeFromExternrefTable0(ret[0]); } } -function wasm_bindgen__convert__closures_____invoke__h11188c184bbe4ba9(arg0, arg1, arg2, arg3) { - wasm.wasm_bindgen__convert__closures_____invoke__h11188c184bbe4ba9(arg0, arg1, arg2, arg3); +function wasm_bindgen__convert__closures_____invoke__h44aa07c92bd0562c(arg0, arg1, arg2, arg3) { + wasm.wasm_bindgen__convert__closures_____invoke__h44aa07c92bd0562c(arg0, arg1, arg2, arg3); } diff --git a/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm b/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm index e2a445cd5..227df6d61 100644 Binary files a/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm and b/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm differ diff --git a/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm.d.ts b/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm.d.ts index 82cc3e8bc..cd5791844 100644 --- a/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm.d.ts +++ b/link/sdks/dart/web/pkg/kalam_link_dart_bg.wasm.d.ts @@ -1,6 +1,13 @@ /* tslint:disable */ /* eslint-disable */ export const memory: WebAssembly.Memory; +export const __wbg_wasmtimestampformatter_free: (a: number, b: number) => void; +export const parseIso8601: (a: number, b: number) => [number, number, number]; +export const timestampNow: () => number; +export const wasmtimestampformatter_format: (a: number, b: number, c: number) => [number, number]; +export const wasmtimestampformatter_formatRelative: (a: number, b: number) => [number, number]; +export const wasmtimestampformatter_new: () => number; +export const wasmtimestampformatter_withFormat: (a: number, b: number) => [number, number, number]; export const __wbg_kalamclient_free: (a: number, b: number) => void; export const kalamclient_ack: (a: number, b: number, c: number, d: number, e: number, f: number, g: bigint) => any; export const kalamclient_anonymous: (a: number, b: number) => [number, number, number]; @@ -39,21 +46,14 @@ export const kalamclient_subscribe: (a: number, b: number, c: number, d: any) => export const kalamclient_subscribeWithSql: (a: number, b: number, c: number, d: number, e: number, f: any) => any; export const kalamclient_unsubscribe: (a: number, b: number, c: number) => any; export const kalamclient_withJwt: (a: number, b: number, c: number, d: number) => [number, number, number]; -export const __wbg_wasmtimestampformatter_free: (a: number, b: number) => void; -export const parseIso8601: (a: number, b: number) => [number, number, number]; -export const timestampNow: () => number; -export const wasmtimestampformatter_format: (a: number, b: number, c: number) => [number, number]; -export const wasmtimestampformatter_formatRelative: (a: number, b: number) => [number, number]; -export const wasmtimestampformatter_new: () => number; -export const wasmtimestampformatter_withFormat: (a: number, b: number) => [number, number, number]; -export const wasm_bindgen__closure__destroy__h3082ecb8e5442115: (a: number, b: number) => void; -export const wasm_bindgen__closure__destroy__hc4784aa82de56652: (a: number, b: number) => void; -export const wasm_bindgen__convert__closures_____invoke__h6537501fed6ccdff: (a: number, b: number, c: any) => [number, number]; -export const wasm_bindgen__convert__closures_____invoke__h11188c184bbe4ba9: (a: number, b: number, c: any, d: any) => void; -export const wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471: (a: number, b: number, c: any) => void; -export const wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_1: (a: number, b: number, c: any) => void; -export const wasm_bindgen__convert__closures_____invoke__hc08ba1fc1bd11471_2: (a: number, b: number, c: any) => void; -export const wasm_bindgen__convert__closures_____invoke__h3bc7e3c5702cdef5: (a: number, b: number) => void; +export const wasm_bindgen__closure__destroy__h28a5275c8860f744: (a: number, b: number) => void; +export const wasm_bindgen__closure__destroy__hb0865e1ee6322b2b: (a: number, b: number) => void; +export const wasm_bindgen__convert__closures_____invoke__hcd5247acc038309e: (a: number, b: number, c: any) => [number, number]; +export const wasm_bindgen__convert__closures_____invoke__h44aa07c92bd0562c: (a: number, b: number, c: any, d: any) => void; +export const wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a: (a: number, b: number, c: any) => void; +export const wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_1: (a: number, b: number, c: any) => void; +export const wasm_bindgen__convert__closures_____invoke__h21f99aac9a68416a_2: (a: number, b: number, c: any) => void; +export const wasm_bindgen__convert__closures_____invoke__ha38bbadd4cfeb566: (a: number, b: number) => void; export const __wbindgen_malloc: (a: number, b: number) => number; export const __wbindgen_realloc: (a: number, b: number, c: number, d: number) => number; export const __wbindgen_exn_store: (a: number) => void; diff --git a/link/src/client.rs b/link/src/client.rs index f0a381d69..a7c8e4b71 100644 --- a/link/src/client.rs +++ b/link/src/client.rs @@ -426,6 +426,7 @@ impl KalamLinkClient { /// [`DynamicAuthProvider`]: crate::auth::DynamicAuthProvider pub fn set_auth(&mut self, auth: AuthProvider) { self.auth = auth.clone(); + self.query_executor.set_auth(auth.clone()); let resolved = ResolvedAuth::Static(auth); self.resolved_auth = resolved.clone(); // Update the shared source so the background connection task picks up @@ -1093,6 +1094,12 @@ struct HealthCheckCache { #[cfg(test)] mod tests { use super::*; + use serde_json::json; + use std::collections::HashMap; + use std::sync::Arc; + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + use tokio::net::{TcpListener, TcpStream}; + use tokio::sync::Mutex; #[test] fn test_builder_pattern() { @@ -1133,4 +1140,154 @@ mod tests { assert!(client.connection_options.ws_lazy_connect); } + + #[derive(Debug, Default)] + struct QueryAuthState { + headers: Vec, + } + + #[tokio::test] + async fn test_set_auth_updates_http_query_executor() { + let listener = TcpListener::bind("127.0.0.1:0").await.expect("bind listener"); + let address = listener.local_addr().expect("read local addr"); + let state = Arc::new(Mutex::new(QueryAuthState::default())); + let state_clone = Arc::clone(&state); + + let server = tokio::spawn(async move { + loop { + let Ok((stream, _)) = listener.accept().await else { + break; + }; + let state = Arc::clone(&state_clone); + tokio::spawn(async move { + let _ = handle_test_request(stream, state).await; + }); + } + }); + + let mut client = KalamLinkClient::builder() + .base_url(format!("http://{}", address)) + .jwt_token("expired-token") + .build() + .expect("build client"); + + let first_error = client + .execute_query("SELECT 1", None, None, None) + .await + .expect_err("expired token should fail"); + assert!(matches!( + first_error, + KalamLinkError::ServerError { + status_code: 401, + .. + } + )); + + client.set_auth(AuthProvider::jwt_token("fresh-token".to_string())); + + let response = client + .execute_query("SELECT 1", None, None, None) + .await + .expect("fresh token should succeed"); + assert!(response.success()); + + let recorded_headers = state.lock().await.headers.clone(); + assert_eq!( + recorded_headers, + vec![ + "Bearer expired-token".to_string(), + "Bearer fresh-token".to_string() + ] + ); + + server.abort(); + } + + async fn handle_test_request( + mut stream: TcpStream, + state: Arc>, + ) -> std::io::Result<()> { + let request = read_test_request(&mut stream).await?; + let authorization = request.headers.get("authorization").cloned().unwrap_or_default(); + state.lock().await.headers.push(authorization.clone()); + + let (status_line, body) = match authorization.as_str() { + "Bearer fresh-token" => ( + "HTTP/1.1 200 OK", + json!({ + "status": "success", + "results": [{ + "schema": [{ + "name": "value", + "data_type": "BigInt", + "index": 0 + }], + "rows": [["1"]], + "row_count": 1 + }], + "took": 1.0 + }) + .to_string(), + ), + _ => ("HTTP/1.1 401 Unauthorized", "Token expired".to_string()), + }; + + let response = format!( + "{status_line}\r\ncontent-type: application/json\r\ncontent-length: {}\r\nconnection: close\r\n\r\n{}", + body.len(), + body + ); + stream.write_all(response.as_bytes()).await?; + stream.shutdown().await + } + + struct TestRequest { + headers: HashMap, + } + + async fn read_test_request(stream: &mut TcpStream) -> std::io::Result { + let mut buffer = Vec::new(); + let mut temp = [0_u8; 1024]; + let mut header_end = None; + let mut content_length = 0_usize; + + loop { + let bytes_read = stream.read(&mut temp).await?; + if bytes_read == 0 { + break; + } + buffer.extend_from_slice(&temp[..bytes_read]); + + if header_end.is_none() { + if let Some(position) = buffer.windows(4).position(|window| window == b"\r\n\r\n") { + header_end = Some(position + 4); + let header_text = String::from_utf8_lossy(&buffer[..position]); + for line in header_text.lines().skip(1) { + if let Some((name, value)) = line.split_once(':') { + if name.eq_ignore_ascii_case("content-length") { + content_length = value.trim().parse().unwrap_or(0); + } + } + } + } + } + + if let Some(end) = header_end { + if buffer.len() >= end + content_length { + break; + } + } + } + + let header_end = header_end.expect("request should include headers"); + let header_text = String::from_utf8_lossy(&buffer[..header_end - 4]); + let mut headers = HashMap::new(); + for line in header_text.lines().skip(1) { + if let Some((name, value)) = line.split_once(':') { + headers.insert(name.trim().to_ascii_lowercase(), value.trim().to_string()); + } + } + + Ok(TestRequest { headers }) + } } diff --git a/link/src/query/executor.rs b/link/src/query/executor.rs index f7a793f55..e7337094f 100644 --- a/link/src/query/executor.rs +++ b/link/src/query/executor.rs @@ -81,6 +81,10 @@ impl QueryExecutor { } } + pub(crate) fn set_auth(&mut self, auth: AuthProvider) { + self.auth = auth; + } + fn is_retry_safe_sql(sql: &str) -> bool { matches!( Self::first_keyword(sql).as_deref(), diff --git a/specs/004-system-improvements-and/agent-files/FLUSH_STATUS.md b/specs/004-system-improvements-and/agent-files/FLUSH_STATUS.md index c9202842b..8d9a2ed92 100644 --- a/specs/004-system-improvements-and/agent-files/FLUSH_STATUS.md +++ b/specs/004-system-improvements-and/agent-files/FLUSH_STATUS.md @@ -96,7 +96,7 @@ To fix this, the scheduler needs access to: ### Test Direct Flush Execution ```bash cd backend -$env:RUST_LOG="kalamdb_core::flush=debug" +$env:KALAMDB_LOG_LEVEL="kalamdb_core::flush=debug" cargo test test_user_table_flush_single_user --lib -- --nocapture ``` @@ -108,7 +108,7 @@ Once the scheduler is wired: 1. Start server with debug logging: ```bash cd backend -$env:RUST_LOG="debug" +$env:KALAMDB_LOG_LEVEL="debug" cargo run --bin kalamdb-server ``` diff --git a/specs/014-live-queries-websocket/Security-Risks.md b/specs/014-live-queries-websocket/Security-Risks.md index 81c291fd5..40a35296d 100644 --- a/specs/014-live-queries-websocket/Security-Risks.md +++ b/specs/014-live-queries-websocket/Security-Risks.md @@ -2137,7 +2137,7 @@ pub fn setup_test_environment() { env: KALAMDB_ENV: test KALAMDB_JWT_SECRET: ${{ secrets.TEST_JWT_SECRET }} - RUST_LOG: warn,kalamdb=debug + KALAMDB_LOG_LEVEL: warn,kalamdb=debug jobs: test: diff --git a/ui/src/components/auth/ProtectedRoute.test.tsx b/ui/src/components/auth/ProtectedRoute.test.tsx new file mode 100644 index 000000000..efc66c751 --- /dev/null +++ b/ui/src/components/auth/ProtectedRoute.test.tsx @@ -0,0 +1,78 @@ +// @vitest-environment jsdom + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { fireEvent, render, screen } from "@testing-library/react"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import ProtectedRoute from "@/components/auth/ProtectedRoute"; + +const mockUseAuth = vi.fn(); + +vi.mock("@/lib/auth", () => ({ + useAuth: () => mockUseAuth(), +})); + +function renderProtectedRoute(initialPath = "/dashboard") { + return render( + + + Login screen} /> + +
Secret dashboard
+ + )} + /> +
+
, + ); +} + +describe("ProtectedRoute", () => { + beforeEach(() => { + mockUseAuth.mockReset(); + }); + + it("redirects unauthenticated users to the login route", async () => { + mockUseAuth.mockReturnValue({ + isAuthenticated: false, + isLoading: false, + user: null, + logout: vi.fn(), + }); + + renderProtectedRoute(); + + expect(await screen.findByText("Login screen")).toBeTruthy(); + }); + + it("shows access denied for authenticated non-admin users and lets them logout", () => { + const logout = vi.fn(); + mockUseAuth.mockReturnValue({ + isAuthenticated: true, + isLoading: false, + user: { role: "user" }, + logout, + }); + + renderProtectedRoute(); + + expect(screen.getByText("Access Denied")).toBeTruthy(); + fireEvent.click(screen.getByRole("button", { name: /logout and switch user/i })); + expect(logout).toHaveBeenCalledTimes(1); + }); + + it("renders protected content for system and dba roles", () => { + mockUseAuth.mockReturnValue({ + isAuthenticated: true, + isLoading: false, + user: { role: "system" }, + logout: vi.fn(), + }); + + renderProtectedRoute(); + + expect(screen.getByText("Secret dashboard")).toBeTruthy(); + }); +}); diff --git a/ui/src/components/auth/SetupGuard.test.tsx b/ui/src/components/auth/SetupGuard.test.tsx new file mode 100644 index 000000000..79c85c8b9 --- /dev/null +++ b/ui/src/components/auth/SetupGuard.test.tsx @@ -0,0 +1,141 @@ +// @vitest-environment jsdom + +import { afterEach, describe, expect, it, vi } from "vitest"; +import { cleanup, render, screen } from "@testing-library/react"; +import { configureStore } from "@reduxjs/toolkit"; +import { Provider } from "react-redux"; +import { MemoryRouter, Route, Routes } from "react-router-dom"; +import SetupGuard from "@/components/auth/SetupGuard"; +import setupReducer from "@/store/setupSlice"; + +const mockCheckSetupStatus = vi.fn(() => ({ type: "setup/checkStatus/mock" })); + +vi.mock("@/store/setupSlice", async () => { + const actual = await vi.importActual("@/store/setupSlice"); + return { + ...actual, + checkSetupStatus: () => mockCheckSetupStatus(), + }; +}); + +function renderSetupGuard({ + initialPath, + setupState, +}: { + initialPath: string; + setupState: { + needsSetup: boolean | null; + isCheckingStatus: boolean; + isSubmitting: boolean; + setupComplete: boolean; + error: string | null; + createdUsername: string | null; + }; +}) { + const store = configureStore({ + reducer: { + setup: setupReducer, + }, + preloadedState: { + setup: setupState, + }, + }); + + return render( + + + + Login screen} /> + +
Setup wizard
+ + )} + /> + +
Protected admin content
+ + )} + /> +
+
+
, + ); +} + +describe("SetupGuard", () => { + afterEach(() => { + cleanup(); + vi.clearAllMocks(); + }); + + it("shows the loading state while setup status is being checked", () => { + renderSetupGuard({ + initialPath: "/dashboard", + setupState: { + needsSetup: null, + isCheckingStatus: true, + isSubmitting: false, + setupComplete: false, + error: null, + createdUsername: null, + }, + }); + + expect(screen.getByText("Checking server status...")).toBeTruthy(); + expect(mockCheckSetupStatus).toHaveBeenCalledTimes(1); + }); + + it("redirects to the setup page when the server still needs setup", async () => { + renderSetupGuard({ + initialPath: "/dashboard", + setupState: { + needsSetup: true, + isCheckingStatus: false, + isSubmitting: false, + setupComplete: false, + error: null, + createdUsername: null, + }, + }); + + expect(await screen.findByText("Setup wizard")).toBeTruthy(); + }); + + it("redirects away from the setup page after setup is complete", async () => { + renderSetupGuard({ + initialPath: "/setup", + setupState: { + needsSetup: false, + isCheckingStatus: false, + isSubmitting: false, + setupComplete: false, + error: null, + createdUsername: null, + }, + }); + + expect(await screen.findByText("Login screen")).toBeTruthy(); + }); + + it("renders children when setup is complete and the user is on an app route", () => { + renderSetupGuard({ + initialPath: "/dashboard", + setupState: { + needsSetup: false, + isCheckingStatus: false, + isSubmitting: false, + setupComplete: false, + error: null, + createdUsername: null, + }, + }); + + expect(screen.getByText("Protected admin content")).toBeTruthy(); + }); +}); diff --git a/ui/src/components/datatype-display/NumberDisplay.tsx b/ui/src/components/datatype-display/NumberDisplay.tsx index e53cbb0e4..a45f2bba2 100644 --- a/ui/src/components/datatype-display/NumberDisplay.tsx +++ b/ui/src/components/datatype-display/NumberDisplay.tsx @@ -1,5 +1,5 @@ interface NumberDisplayProps { - value: number; + value: number | string | bigint; dataType?: 'INT' | 'BIGINT' | 'SMALLINT' | 'FLOAT' | 'DOUBLE' | 'DECIMAL'; } @@ -16,7 +16,7 @@ export function NumberDisplay({ value, dataType }: NumberDisplayProps) { maximumFractionDigits: 6, }); } else if (dataType === 'BIGINT') { - // Show BigInt as raw number without thousand separators + // Preserve the exact textual representation for 64-bit integers. formatted = String(value); } else { formatted = Number(value).toLocaleString(); diff --git a/ui/src/components/datatype-display/index.test.tsx b/ui/src/components/datatype-display/index.test.tsx new file mode 100644 index 000000000..9eb8e2e80 --- /dev/null +++ b/ui/src/components/datatype-display/index.test.tsx @@ -0,0 +1,38 @@ +// @vitest-environment jsdom + +import { describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { CellDisplay } from "./index"; + +vi.mock("kalam-link", () => { + class KalamCellValue { + private value: unknown; + + constructor(value: unknown) { + this.value = value; + } + + static from(value: unknown) { + return new KalamCellValue(value); + } + + toJson() { + return this.value; + } + } + + return { KalamCellValue }; +}); + +describe("CellDisplay", () => { + it("renders bigint-like strings without losing precision", () => { + render( + , + ); + + expect(screen.getByText("9223372036854775807")).toBeTruthy(); + }); +}); diff --git a/ui/src/components/datatype-display/index.tsx b/ui/src/components/datatype-display/index.tsx index 4fae9fc1c..79de7c81a 100644 --- a/ui/src/components/datatype-display/index.tsx +++ b/ui/src/components/datatype-display/index.tsx @@ -90,6 +90,10 @@ export function CellDisplay({ value, dataType, namespace, tableName }: CellDispl } else if (normalizedType.includes('16') || normalizedType.includes('8') || normalizedType === 'SMALLINT') { displayType = 'SMALLINT'; } + + if (displayType === 'BIGINT' && (typeof rawValue === 'string' || typeof rawValue === 'bigint')) { + return ; + } const numericValue = typeof rawValue === 'number' diff --git a/ui/src/components/sql-studio-v2/CellContextMenu.tsx b/ui/src/components/sql-studio-v2/CellContextMenu.tsx index 31ac7e58e..bba94f938 100644 --- a/ui/src/components/sql-studio-v2/CellContextMenu.tsx +++ b/ui/src/components/sql-studio-v2/CellContextMenu.tsx @@ -10,6 +10,7 @@ export interface CellContextMenuState { value: unknown; rowStatus: "deleted" | "edited" | null; cellEdited: boolean; + canMutate: boolean; } interface CellContextMenuProps { @@ -62,6 +63,7 @@ export function CellContextMenu({ const isDeleted = context.rowStatus === "deleted"; const isEdited = context.rowStatus === "edited"; + const canMutate = context.canMutate; const hasViewableData = context.value !== null && (typeof context.value === "object" || @@ -83,7 +85,7 @@ export function CellContextMenu({
- {!isDeleted && ( + {canMutate && !isDeleted && ( )} - {!isDeleted && ( + {canMutate && !isDeleted && ( )} - {isDeleted && ( + {canMutate && isDeleted && ( )} - {isEdited && !isDeleted && ( + {canMutate && isEdited && !isDeleted && ( - +
diff --git a/ui/src/components/sql-studio-v2/StudioResultsGrid.tsx b/ui/src/components/sql-studio-v2/StudioResultsGrid.tsx index 5a01d844f..2fdb7a02e 100644 --- a/ui/src/components/sql-studio-v2/StudioResultsGrid.tsx +++ b/ui/src/components/sql-studio-v2/StudioResultsGrid.tsx @@ -111,6 +111,31 @@ function compareValues(left: unknown, right: unknown): number { return String(left).localeCompare(String(right), undefined, { numeric: true, sensitivity: "base" }); } +function unwrapCellValue(value: unknown): unknown { + return value instanceof KalamCellValue ? value.toJson() : value; +} + +function parseSeqValue(value: unknown): bigint | null { + const raw = unwrapCellValue(value); + if (typeof raw === "bigint") { + return raw; + } + if (typeof raw === "number" && Number.isInteger(raw)) { + return BigInt(raw); + } + if (typeof raw === "string") { + const trimmed = raw.trim(); + if (/^-?\d+$/.test(trimmed)) { + try { + return BigInt(trimmed); + } catch { + return null; + } + } + } + return null; +} + export function StudioResultsGrid({ result, isRunning, @@ -220,6 +245,10 @@ export function StudioResultsGrid({ const parsedTableContext = useMemo(() => extractTableContext(activeSql), [activeSql]); const cellNamespace = parsedTableContext?.namespace ?? selectedTable?.namespace; const cellTableName = parsedTableContext?.tableName ?? selectedTable?.name; + const isSystemTable = cellNamespace?.toLowerCase() === "system"; + const isSuccess = !isRunning && result?.status === "success"; + const hasTabularResults = isSuccess && schema.length > 0; + const canMutateRows = hasTabularResults && !isLiveMode && !isSystemTable; const sortedRowIndices = useMemo(() => { const indices = sourceRows.map((_, rowIndex) => rowIndex); @@ -237,30 +266,24 @@ export function StudioResultsGrid({ } if (isLiveMode) { - // Live mode: sort by PK with new inserts at top - const pkColumns = schema - .filter((f) => f.isPrimaryKey) - .sort((a, b) => a.index - b.index) - .map((f) => f.name); - + // Live mode: sort newest-first by _seq when present. indices.sort((leftIndex, rightIndex) => { const leftRow = sourceRows[leftIndex]; const rightRow = sourceRows[rightIndex]; + const leftSeq = parseSeqValue(leftRow?._seq); + const rightSeq = parseSeqValue(rightRow?._seq); + + if (leftSeq !== null && rightSeq !== null) { + if (leftSeq === rightSeq) return 0; + return leftSeq > rightSeq ? -1 : 1; + } + if (leftSeq !== null) return -1; + if (rightSeq !== null) return 1; + const leftChangeType = leftRow?.[LIVE_META.CHANGE_TYPE] as string | undefined; const rightChangeType = rightRow?.[LIVE_META.CHANGE_TYPE] as string | undefined; - - // New inserts go to top - const leftIsInsert = leftChangeType === "insert"; - const rightIsInsert = rightChangeType === "insert"; - if (leftIsInsert && !rightIsInsert) return -1; - if (!leftIsInsert && rightIsInsert) return 1; - - // Otherwise sort by PK ascending - for (const pkCol of pkColumns) { - const leftValue = leftRow?.[pkCol]; - const rightValue = rightRow?.[pkCol]; - const comparison = compareValues(leftValue, rightValue); - if (comparison !== 0) return comparison; + if (leftChangeType !== rightChangeType) { + return String(rightChangeType ?? "").localeCompare(String(leftChangeType ?? "")); } return 0; }); @@ -291,6 +314,14 @@ export function StudioResultsGrid({ } }, [selectedCell, currentPageRows]); + useEffect(() => { + if (canMutateRows) { + return; + } + + setSelectedRows(new Set()); + }, [canMutateRows]); + useEffect(() => { if (!isLiveMode) { return; @@ -349,6 +380,9 @@ export function StudioResultsGrid({ }; const handleEditCell = (rowIndex: number, columnName: string, currentValue: unknown) => { + if (!canMutateRows) { + return; + } openCellViewer(currentValue, columnName, rowIndex, true); }; @@ -358,12 +392,21 @@ export function StudioResultsGrid({ oldValue: unknown, newValue: unknown, ) => { + if (!canMutateRows) { + setInlineEditContext(null); + return; + } + const pkValues = getPrimaryKeyValues(rowIndex); editCell(rowIndex, columnName, oldValue, newValue, pkValues); setInlineEditContext(null); }; const handleDeleteRow = (rowIndex: number) => { + if (!canMutateRows) { + return; + } + const row = sourceRows[rowIndex]; if (!row) { return; @@ -379,6 +422,10 @@ export function StudioResultsGrid({ }; const handleDeleteSelectedRows = () => { + if (!canMutateRows) { + return; + } + const targetRows = Array.from(selectedRows); targetRows.forEach((rowIndex) => { const row = sourceRows[rowIndex]; @@ -403,10 +450,10 @@ export function StudioResultsGrid({ dataType, rowIndex, columnName, - canEdit: editable && !isLiveMode, + canEdit: editable && canMutateRows, }); }, - [schema, isLiveMode], + [canMutateRows, schema], ); const moveSelectionByArrow = useCallback( @@ -477,6 +524,10 @@ export function StudioResultsGrid({ }, [selectedCell, moveSelectionByArrow]); const handleReviewChanges = () => { + if (!canMutateRows) { + return; + } + const parsed = extractTableContext(activeSql); const namespace = parsed?.namespace ?? selectedTable?.namespace; const tableName = parsed?.tableName ?? selectedTable?.name; @@ -510,10 +561,8 @@ export function StudioResultsGrid({ const editCount = edits.size; const deleteCount = deletions.size; - const isSuccess = !isRunning && result?.status === "success"; - const hasTabularResults = isSuccess && schema.length > 0; const logCount = result?.logs.length ?? 0; - const showTableEditorBars = hasTabularResults && resultView === "results"; + const showResultsTable = hasTabularResults && resultView === "results"; return (
@@ -554,7 +603,7 @@ export function StudioResultsGrid({ showing first {MAX_RENDERED_ROWS.toLocaleString()} )} - {resultView === "results" && selectedRows.size > 0 && ( + {canMutateRows && resultView === "results" && selectedRows.size > 0 && ( {selectedRows.size} selected @@ -567,9 +616,9 @@ export function StudioResultsGrid({
)} - {!isRunning && showTableEditorBars && ( + {!isRunning && showResultsTable && ( <> -
-
- {changeCount === 0 - ? "No pending table changes" - : `${changeCount} change${changeCount === 1 ? "" : "s"} • ${editCount} edit${editCount === 1 ? "" : "s"} • ${deleteCount} delete${deleteCount === 1 ? "" : "s"}`} -
-
- - + {canMutateRows && ( +
+
+ {changeCount === 0 + ? "No pending table changes" + : `${changeCount} change${changeCount === 1 ? "" : "s"} • ${editCount} edit${editCount === 1 ? "" : "s"} • ${deleteCount} delete${deleteCount === 1 ? "" : "s"}`} +
+
+ + +
-
+ )}
@@ -654,7 +705,7 @@ export function StudioResultsGrid({ Change
- ) : ( + ) : canMutateRows ? (
+ ) : ( +
)} {schema.map((field) => { @@ -784,7 +837,7 @@ export function StudioResultsGrid({ )}
- ) : ( + ) : canMutateRows ? (
+ ) : ( +
)} @@ -823,7 +878,7 @@ export function StudioResultsGrid({ }); }} onDoubleClick={() => { - openCellViewer(value, field.name, rowIndex, true); + openCellViewer(value, field.name, rowIndex, canMutateRows); }} onContextMenu={(event) => { event.preventDefault(); @@ -839,6 +894,7 @@ export function StudioResultsGrid({ value, rowStatus, cellEdited, + canMutate: canMutateRows, }); }} className={cn( diff --git a/ui/src/components/sql-studio-v2/types.ts b/ui/src/components/sql-studio-v2/types.ts index 3c5c5b2ba..819f3f5f9 100644 --- a/ui/src/components/sql-studio-v2/types.ts +++ b/ui/src/components/sql-studio-v2/types.ts @@ -1,7 +1,7 @@ export interface LiveSubscriptionOptions { batch_size?: number; last_rows?: number; - from?: number | string; + from?: string; } export interface StudioColumn { diff --git a/ui/src/components/sql-studio-v2/workspaceState.ts b/ui/src/components/sql-studio-v2/workspaceState.ts index 3d47ef6dd..59e768e83 100644 --- a/ui/src/components/sql-studio-v2/workspaceState.ts +++ b/ui/src/components/sql-studio-v2/workspaceState.ts @@ -5,7 +5,7 @@ type PanelLayout = [number, number]; interface PersistedSubscriptionOptions { batch_size?: number; last_rows?: number; - from?: number | string; + from?: string; } export interface SqlStudioPersistedQueryTab { @@ -88,7 +88,7 @@ function normalizeSubscriptionOptions(value: unknown): PersistedSubscriptionOpti result.last_rows = record.last_rows; } if (typeof record.from === "number" || typeof record.from === "string") { - result.from = record.from; + result.from = String(record.from); } return Object.keys(result).length > 0 ? result : undefined; } diff --git a/ui/src/components/ui/code-block.test.tsx b/ui/src/components/ui/code-block.test.tsx new file mode 100644 index 000000000..00eb48cda --- /dev/null +++ b/ui/src/components/ui/code-block.test.tsx @@ -0,0 +1,51 @@ +// @vitest-environment jsdom + +import { describe, expect, it, vi } from "vitest"; +import { render, screen } from "@testing-library/react"; +import { CodeBlock } from "./code-block"; + +vi.mock("kalam-link", () => { + class KalamCellValue { + private value: unknown; + + constructor(value: unknown) { + this.value = value; + } + + static from(value: unknown) { + return new KalamCellValue(value); + } + + toJson() { + return this.value; + } + } + + return { KalamCellValue }; +}); + +describe("CodeBlock", () => { + it("renders nested KalamCellValue payloads with their underlying values", () => { + render( + "9223372036854775807", + }, + name: { + toJson: () => "metric-a", + }, + }, + ], + }} + jsonPreferred + />, + ); + + expect(screen.getByText(/9223372036854775807/)).toBeTruthy(); + expect(screen.getByText(/metric-a/)).toBeTruthy(); + }); +}); diff --git a/ui/src/components/ui/code-block.tsx b/ui/src/components/ui/code-block.tsx index c6ccb3731..1d2b842dd 100644 --- a/ui/src/components/ui/code-block.tsx +++ b/ui/src/components/ui/code-block.tsx @@ -14,9 +14,38 @@ interface NormalizedCode { isJson: boolean; } +function unwrapSerializableValue(value: unknown): unknown { + if (value instanceof KalamCellValue) { + return unwrapSerializableValue(value.toJson()); + } + + if (Array.isArray(value)) { + return value.map((item) => unwrapSerializableValue(item)); + } + + if (value && typeof value === "object") { + const maybeSerializable = value as { toJson?: () => unknown }; + if (typeof maybeSerializable.toJson === "function") { + try { + return unwrapSerializableValue(maybeSerializable.toJson()); + } catch { + // Fall through to entry-wise unwrap. + } + } + + return Object.fromEntries( + Object.entries(value as Record).map(([key, entry]) => [ + key, + unwrapSerializableValue(entry), + ]), + ); + } + + return value; +} + function normalizeCode(value: unknown, jsonPreferred: boolean): NormalizedCode { - // Unwrap KalamCellValue wrappers before normalizing - const raw = value instanceof KalamCellValue ? value.toJson() : value; + const raw = unwrapSerializableValue(value); if (raw === null || raw === undefined) { return { text: "null", isJson: true }; @@ -90,8 +119,8 @@ export function CodeBlock({ const highlighted = normalized.isJson ? highlightJson(normalized.text) : null; return ( -
- +
+ {normalized.isJson && highlighted ? (
             
diff --git a/ui/src/features/sql-studio/components/ExplorerTableContextMenu.tsx b/ui/src/features/sql-studio/components/ExplorerTableContextMenu.tsx
index cf38ee42a..03637b80a 100644
--- a/ui/src/features/sql-studio/components/ExplorerTableContextMenu.tsx
+++ b/ui/src/features/sql-studio/components/ExplorerTableContextMenu.tsx
@@ -24,12 +24,13 @@ export function ExplorerTableContextMenu({
   }
 
   const table = contextMenu.table;
+  const itemClassName = "w-full px-3 py-2 text-left text-sm text-popover-foreground transition-colors hover:bg-accent hover:text-accent-foreground";
 
   return (
     <>
       
@@ -37,21 +38,21 @@ export function ExplorerTableContextMenu({