From 50e3439424acf6be8097ede8228491dab001e4ba Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Sat, 23 May 2026 10:27:16 -0700 Subject: [PATCH 1/5] feat(query): fetch results as Arrow IPC instead of JSON Switch the async query path and poll command from JSON to Arrow IPC (application/vnd.apache.arrow.stream) for result fetching. Arrow is ~39% smaller on the wire and ~150 ms faster on large result sets. - Add `get_bytes()` to ApiClient for binary HTTP responses - Add Arrow decoding: `arrow_cell`, `arrow_ipc_to_query_response`, `fetch_arrow_result` in query.rs - Inline async polling (500 ms interval with spinner) so callers never need to re-run the command for slow queries - `poll` subcommand updated to use Arrow fetch The sync fast-path (200 from /query) remains JSON since the server returns the result inline and does not support Arrow there. Co-Authored-By: Claude Sonnet 4.6 --- Cargo.lock | 474 +++++++++++++++++++++++++++++++++++++++++++++++++-- Cargo.toml | 1 + src/api.rs | 26 +++ src/query.rs | 152 +++++++++++++++-- 4 files changed, 620 insertions(+), 33 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 3681f51..8fa723b 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -8,6 +8,20 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" +[[package]] +name = "ahash" +version = "0.8.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" +dependencies = [ + "cfg-if", + "const-random", + "getrandom 0.3.4", + "once_cell", + "version_check", + "zerocopy", +] + [[package]] name = "aho-corasick" version = "1.1.4" @@ -17,6 +31,15 @@ dependencies = [ "memchr", ] +[[package]] +name = "android_system_properties" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "819e7219dbd41043ac279b19830f2efc897156490d7fd6ea916720117ee66311" +dependencies = [ + "libc", +] + [[package]] name = "ansi-str" version = "0.9.0" @@ -98,6 +121,174 @@ version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" +[[package]] +name = "arrow" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5ec52ba94edeed950e4a41f75d35376df196e8cb04437f7280a5aa49f20f796" +dependencies = [ + "arrow-arith", + "arrow-array", + "arrow-buffer", + "arrow-cast", + "arrow-data", + "arrow-ipc", + "arrow-ord", + "arrow-row", + "arrow-schema", + "arrow-select", + "arrow-string", +] + +[[package]] +name = "arrow-arith" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8fc766fdacaf804cb10c7c70580254fcdb5d55cdfda2bc57b02baf5223a3af9e" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "num", +] + +[[package]] +name = "arrow-array" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a12fcdb3f1d03f69d3ec26ac67645a8fe3f878d77b5ebb0b15d64a116c212985" +dependencies = [ + "ahash", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "chrono", + "half", + "hashbrown 0.15.5", + "num", +] + +[[package]] +name = "arrow-buffer" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "263f4801ff1839ef53ebd06f99a56cecd1dbaf314ec893d93168e2e860e0291c" +dependencies = [ + "bytes", + "half", + "num", +] + +[[package]] +name = "arrow-cast" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede6175fbc039dfc946a61c1b6d42fd682fcecf5ab5d148fbe7667705798cac9" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "atoi", + "base64", + "chrono", + "half", + "lexical-core", + "num", + "ryu", +] + +[[package]] +name = "arrow-data" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "61cfdd7d99b4ff618f167e548b2411e5dd2c98c0ddebedd7df433d34c20a4429" +dependencies = [ + "arrow-buffer", + "arrow-schema", + "half", + "num", +] + +[[package]] +name = "arrow-ipc" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62ff528658b521e33905334723b795ee56b393dbe9cf76c8b1f64b648c65a60c" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "flatbuffers", +] + +[[package]] +name = "arrow-ord" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f0a3334a743bd2a1479dbc635540617a3923b4b2f6870f37357339e6b5363c21" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", +] + +[[package]] +name = "arrow-row" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8d1d7a7291d2c5107e92140f75257a99343956871f3d3ab33a7b41532f79cb68" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "half", +] + +[[package]] +name = "arrow-schema" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "39cfaf5e440be44db5413b75b72c2a87c1f8f0627117d110264048f2969b99e9" + +[[package]] +name = "arrow-select" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69efcd706420e52cd44f5c4358d279801993846d1c2a8e52111853d61d55a619" +dependencies = [ + "ahash", + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "num", +] + +[[package]] +name = "arrow-string" +version = "54.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a21546b337ab304a32cfc0770f671db7411787586b45b78b4593ae78e64e2b03" +dependencies = [ + "arrow-array", + "arrow-buffer", + "arrow-data", + "arrow-schema", + "arrow-select", + "memchr", + "num", + "regex", + "regex-syntax", +] + [[package]] name = "ascii" version = "1.1.0" @@ -114,18 +305,39 @@ dependencies = [ "serde_json", ] +[[package]] +name = "atoi" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f28d99ec8bfea296261ca1af174f24225171fea9664ba9003cbebee704810528" +dependencies = [ + "num-traits", +] + [[package]] name = "atomic-waker" version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1505bd5d3d116872e7271a6d4e16d81d0c8570876c8de68093a09ac269d8aac0" +[[package]] +name = "autocfg" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2032f911046de80f0a198e0901378627c33f59ea0ac00e363d481118bd70a53" + [[package]] name = "base64" version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.11.0" @@ -187,6 +399,17 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" +[[package]] +name = "chrono" +version = "0.4.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c673075a2e0e5f4a1dde27ce9dee1ea4558c7ffe648f576438a20ca1d2acc4b0" +dependencies = [ + "iana-time-zone", + "num-traits", + "windows-link", +] + [[package]] name = "chunked_transfer" version = "1.5.0" @@ -270,6 +493,26 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "const-random" +version = "0.1.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87e00182fe74b066627d63b85fd550ac2998d4b0bd86bfed477a0ae4c7c71359" +dependencies = [ + "const-random-macro", +] + +[[package]] +name = "const-random-macro" +version = "0.1.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f9d839f2a20b0aee515dc581a6172f2321f96cab76c1a38a4c584a194955390e" +dependencies = [ + "getrandom 0.2.17", + "once_cell", + "tiny-keccak", +] + [[package]] name = "convert_case" version = "0.10.0" @@ -369,7 +612,7 @@ version = "0.28.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "829d955a0bb380ef178a640b91779e3987da38c9aea133b20614cfed8cdea9c6" dependencies = [ - "bitflags", + "bitflags 2.11.0", "crossterm_winapi", "mio", "parking_lot", @@ -385,7 +628,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d8b9f2e4c67f833b660cdb0a3523065869fb35570177239812ed4c905aeff87b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "crossterm_winapi", "derive_more", "document-features", @@ -406,6 +649,12 @@ dependencies = [ "winapi", ] +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "crypto-common" version = "0.1.7" @@ -561,6 +810,16 @@ version = "0.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5baebc0774151f905a1a2cc41989300b1e6fbb29aff0ceffa1064fdd3088d582" +[[package]] +name = "flatbuffers" +version = "24.12.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f1baf0dbf96932ec9a3038d57900329c015b0bfb7b63d904f3bc27e2b02a096" +dependencies = [ + "bitflags 1.3.2", + "rustc_version", +] + [[package]] name = "flate2" version = "1.1.9" @@ -734,6 +993,18 @@ dependencies = [ "tracing", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "num-traits", + "zerocopy", +] + [[package]] name = "hashbrown" version = "0.15.5" @@ -760,6 +1031,7 @@ name = "hotdata-cli" version = "0.2.9" dependencies = [ "anstyle", + "arrow", "base64", "clap", "clap_complete", @@ -918,6 +1190,30 @@ dependencies = [ "windows-registry", ] +[[package]] +name = "iana-time-zone" +version = "0.1.65" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e31bc9ad994ba00e440a8aa5c9ef0ec67d5cb5e5cb0cc7f8b744a35b389cc470" +dependencies = [ + "android_system_properties", + "core-foundation-sys", + "iana-time-zone-haiku", + "js-sys", + "log", + "wasm-bindgen", + "windows-core", +] + +[[package]] +name = "iana-time-zone-haiku" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f31827a206f56af32e590ba56d5d2d085f558508192593743f16b2306495269f" +dependencies = [ + "cc", +] + [[package]] name = "icu_collections" version = "2.1.1" @@ -1057,7 +1353,7 @@ version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6654738b8024300cf062d04a1c13c10c8e2cea598ec1c47dc9b6641159429756" dependencies = [ - "bitflags", + "bitflags 2.11.0", "crossterm 0.29.0", "dyn-clone", "fuzzy-matcher", @@ -1128,19 +1424,82 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" +[[package]] +name = "lexical-core" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" +dependencies = [ + "lexical-parse-float", + "lexical-parse-integer", + "lexical-util", + "lexical-write-float", + "lexical-write-integer", +] + +[[package]] +name = "lexical-parse-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" +dependencies = [ + "lexical-parse-integer", + "lexical-util", +] + +[[package]] +name = "lexical-parse-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" +dependencies = [ + "lexical-util", +] + +[[package]] +name = "lexical-util" +version = "1.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" + +[[package]] +name = "lexical-write-float" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" +dependencies = [ + "lexical-util", + "lexical-write-integer", +] + +[[package]] +name = "lexical-write-integer" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" +dependencies = [ + "lexical-util", +] + [[package]] name = "libc" version = "0.2.182" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6800badb6cb2082ffd7b6a67e6125bb39f18782f793520caee8cb8846be06112" +[[package]] +name = "libm" +version = "0.2.16" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" + [[package]] name = "libredox" version = "0.1.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1744e39d1d6a9948f4f388969627434e31128196de472883b39f148769bfe30a" dependencies = [ - "bitflags", + "bitflags 2.11.0", "libc", "plain", "redox_syscall 0.7.3", @@ -1289,7 +1648,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "cfg_aliases", "libc", @@ -1314,6 +1673,80 @@ dependencies = [ "winapi", ] +[[package]] +name = "num" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35bd024e8b2ff75562e5f34e7f4905839deb4b22955ef5e73d2fea1b9813cb23" +dependencies = [ + "num-bigint", + "num-complex", + "num-integer", + "num-iter", + "num-rational", + "num-traits", +] + +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + +[[package]] +name = "num-complex" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73f88a1307638156682bada9d7604135552957b7818057dcef22705b4d509495" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-integer" +version = "0.1.46" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7969661fd2958a5cb096e56c8e1ad0444ac2bbcd0061bd28660485a44879858f" +dependencies = [ + "num-traits", +] + +[[package]] +name = "num-iter" +version = "0.1.45" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1429034a0490724d0075ebb2bc9e875d6503c3cf69e235a8941aa757d83ef5bf" +dependencies = [ + "autocfg", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-rational" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f83d14da390562dca69fc84082e73e548e1ad308d24accdedd2720017cb37824" +dependencies = [ + "num-bigint", + "num-integer", + "num-traits", +] + +[[package]] +name = "num-traits" +version = "0.2.19" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" +dependencies = [ + "autocfg", + "libm", +] + [[package]] name = "number_prefix" version = "0.4.0" @@ -1326,7 +1759,7 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -1368,7 +1801,7 @@ version = "0.10.79" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf0b434746ee2832f4f0baf10137e1cabb18cbe6912c69e2e33263c45250f542" dependencies = [ - "bitflags", + "bitflags 2.11.0", "cfg-if", "foreign-types", "libc", @@ -1718,7 +2151,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -1727,7 +2160,7 @@ version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ce70a74e890531977d37e532c34d45e9055d2409ed08ddba14529471ed0be16" dependencies = [ - "bitflags", + "bitflags 2.11.0", ] [[package]] @@ -1851,7 +2284,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys 0.4.15", @@ -1864,7 +2297,7 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags", + "bitflags 2.11.0", "errno", "libc", "linux-raw-sys 0.12.1", @@ -1939,7 +2372,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.10.1", "core-foundation-sys", "libc", @@ -2230,7 +2663,7 @@ version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags", + "bitflags 2.11.0", "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2334,6 +2767,15 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "tiny-keccak" +version = "2.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2c9d3793400a45f954c52e73d068316d76b6f4e36977e3fcebb13a2721e80237" +dependencies = [ + "crunchy", +] + [[package]] name = "tiny_http" version = "0.12.0" @@ -2481,7 +2923,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags", + "bitflags 2.11.0", "bytes", "futures-util", "http", @@ -2750,7 +3192,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags", + "bitflags 2.11.0", "hashbrown 0.15.5", "indexmap", "semver", @@ -3160,7 +3602,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags", + "bitflags 2.11.0", "indexmap", "log", "serde", diff --git a/Cargo.toml b/Cargo.toml index 3d1cf5a..76422ef 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,6 +19,7 @@ reqwest = { version = "0.12", features = ["blocking", "json"] } rayon = "1.10" serde = { version = "1", features = ["derive"] } serde_json = "1" +arrow = { version = "54", default-features = false, features = ["ipc"] } serde_yaml = "0.9" base64 = "0.22" crossterm = "0.28" diff --git a/src/api.rs b/src/api.rs index df3ef71..2083b2b 100644 --- a/src/api.rs +++ b/src/api.rs @@ -277,6 +277,32 @@ impl ApiClient { self.send(req, None) } + /// GET with a custom Accept header; returns raw bytes instead of decoded text. + /// Used for binary result formats such as Arrow IPC streams. + pub fn get_bytes(&self, path: &str, accept: &str) -> (reqwest::StatusCode, Vec) { + let url = format!("{}{path}", self.api_url); + let req = self.build_request(reqwest::Method::GET, &url).header("Accept", accept); + match req.build() { + Ok(request) => { + match self.client.execute(request) { + Ok(resp) => { + let status = resp.status(); + let bytes = resp.bytes().unwrap_or_default().to_vec(); + (status, bytes) + } + Err(e) => { + eprintln!("error connecting to API: {e}"); + std::process::exit(1); + } + } + } + Err(e) => { + eprintln!("error building request: {e}"); + std::process::exit(1); + } + } + } + /// POST request with JSON body, exits on error, returns raw (status, body). pub fn post_raw(&self, path: &str, body: &serde_json::Value) -> (reqwest::StatusCode, String) { let url = format!("{}{path}", self.api_url); diff --git a/src/query.rs b/src/query.rs index 82d275c..dd505ca 100644 --- a/src/query.rs +++ b/src/query.rs @@ -2,6 +2,8 @@ use crate::api::ApiClient; use serde::Deserialize; use serde_json::Value; +const ACCEPT_ARROW: &str = "application/vnd.apache.arrow.stream"; + #[derive(Deserialize)] pub struct QueryResponse { pub result_id: Option, @@ -45,6 +47,103 @@ fn value_to_string(v: &Value) -> String { } } +/// Convert one cell of an Arrow array to a `serde_json::Value`. +fn arrow_cell(col: &dyn arrow::array::Array, row: usize) -> Value { + use arrow::array::*; + use arrow::datatypes::DataType::*; + use serde_json::Number; + + if col.is_null(row) { + return Value::Null; + } + + match col.data_type() { + Boolean => Value::Bool(col.as_any().downcast_ref::().unwrap().value(row)), + Int8 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + Int16 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + Int32 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + Int64 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + UInt8 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + UInt16 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + UInt32 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + UInt64 => Value::Number(col.as_any().downcast_ref::().unwrap().value(row).into()), + Float32 => { + let v = col.as_any().downcast_ref::().unwrap().value(row) as f64; + Number::from_f64(v).map(Value::Number).unwrap_or(Value::Null) + } + Float64 => { + let v = col.as_any().downcast_ref::().unwrap().value(row); + Number::from_f64(v).map(Value::Number).unwrap_or(Value::Null) + } + Utf8 => Value::String( + col.as_any().downcast_ref::().unwrap().value(row).to_owned(), + ), + LargeUtf8 => Value::String( + col.as_any().downcast_ref::().unwrap().value(row).to_owned(), + ), + // Dates, timestamps, decimals, etc. — format via Arrow's display helper. + _ => { + use arrow::util::display::{ArrayFormatter, FormatOptions}; + let opts = FormatOptions::default(); + ArrayFormatter::try_new(col, &opts) + .map(|f| Value::String(f.value(row).to_string())) + .unwrap_or(Value::Null) + } + } +} + +/// Decode an Arrow IPC stream into a `QueryResponse` suitable for display. +fn arrow_ipc_to_query_response(bytes: Vec, result_id: String) -> QueryResponse { + use arrow::ipc::reader::StreamReader; + use std::io::Cursor; + + let reader = match StreamReader::try_new(Cursor::new(&bytes), None) { + Ok(r) => r, + Err(e) => { + eprintln!("error reading Arrow IPC stream: {e}"); + std::process::exit(1); + } + }; + + let columns: Vec = reader.schema().fields().iter().map(|f| f.name().clone()).collect(); + let mut rows: Vec> = Vec::new(); + + for batch_result in reader { + let batch = match batch_result { + Ok(b) => b, + Err(e) => { + eprintln!("error reading Arrow batch: {e}"); + std::process::exit(1); + } + }; + for row in 0..batch.num_rows() { + rows.push((0..batch.num_columns()).map(|c| arrow_cell(batch.column(c).as_ref(), row)).collect()); + } + } + + let row_count = rows.len() as u64; + QueryResponse { + result_id: Some(result_id), + columns, + rows, + row_count, + execution_time_ms: 0, + warning: None, + } +} + +/// Fetch `/results/{result_id}` as Arrow IPC and return a `QueryResponse`. +fn fetch_arrow_result(api: &ApiClient, result_id: &str) -> QueryResponse { + let (status, bytes) = api.get_bytes(&format!("/results/{result_id}"), ACCEPT_ARROW); + if !status.is_success() { + use crossterm::style::Stylize; + let msg = String::from_utf8_lossy(&bytes); + eprintln!("{}", format!("error fetching result: {status} {msg}").red()); + std::process::exit(1); + } + arrow_ipc_to_query_response(bytes, result_id.to_owned()) +} + pub fn execute( sql: &str, workspace_id: &str, @@ -67,11 +166,12 @@ pub fn execute( } let spinner = crate::util::spinner("running query..."); - let (status, resp_body) = api.post_raw("/query", &body); spinner.finish_and_clear(); if status.as_u16() == 202 { + // Query didn't complete within async_after_ms — poll until done, then + // fetch the result as Arrow IPC. let async_resp: AsyncResponse = match serde_json::from_str(&resp_body) { Ok(r) => r, Err(e) => { @@ -79,21 +179,38 @@ pub fn execute( std::process::exit(1); } }; - use crossterm::style::Stylize; - eprintln!( - "{}", - format!("query still running (status: {})", async_resp.status).yellow() - ); - eprintln!("query_run_id: {}", async_resp.query_run_id); - eprintln!( - "{}", - format!( - "Poll with: hotdata query status {}", - async_resp.query_run_id - ) - .dark_grey() - ); - std::process::exit(2); + + let run_id = &async_resp.query_run_id; + let spinner = crate::util::spinner("waiting for query..."); + + loop { + std::thread::sleep(std::time::Duration::from_millis(500)); + let run: QueryRunResponse = api.get(&format!("/query-runs/{run_id}")); + match run.status.as_str() { + "succeeded" => { + spinner.finish_and_clear(); + match run.result_id { + Some(ref result_id) => { + let result = fetch_arrow_result(&api, result_id); + print_result(&result, format); + } + None => { + use crossterm::style::Stylize; + println!("{}", "Query succeeded but no result available.".yellow()); + } + } + return; + } + "failed" => { + spinner.finish_and_clear(); + use crossterm::style::Stylize; + let err = run.error.as_deref().unwrap_or("unknown error"); + eprintln!("{}", format!("query failed: {err}").red()); + std::process::exit(1); + } + _ => continue, + } + } } if !status.is_success() { @@ -106,6 +223,7 @@ pub fn execute( std::process::exit(1); } + // Fast path: query completed synchronously — response is JSON from /query. let result: QueryResponse = match serde_json::from_str(&resp_body) { Ok(r) => r, Err(e) => { @@ -126,7 +244,7 @@ pub fn poll(query_run_id: &str, workspace_id: &str, format: &str) { match run.status.as_str() { "succeeded" => match run.result_id { Some(ref result_id) => { - let result: QueryResponse = api.get(&format!("/results/{result_id}")); + let result = fetch_arrow_result(&api, result_id); print_result(&result, format); } None => { From 9af5114a7f82191a51d28aeb9b72acdb9a6692af Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Sat, 23 May 2026 10:53:35 -0700 Subject: [PATCH 2/5] fix(query): address four code review issues in Arrow IPC path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Route get_bytes through send_debug_bytes so Arrow requests appear in --debug output (was bypassing debug logging entirely) - Replace unwrap_or_default() with resp.bytes()? to propagate body read errors instead of silently returning empty bytes - Remove dead req.build() error arm; send_debug_bytes handles it - Change execution_time_ms to Option; async Arrow results show "—" in the table footer instead of a misleading "0 ms" - Add 5-minute polling timeout to the async execute loop with a hint to re-check via `hotdata query status ` Co-Authored-By: Claude Sonnet 4.6 --- src/api.rs | 18 +++--------------- src/query.rs | 25 ++++++++++++++++++++----- src/util.rs | 31 +++++++++++++++++++++++++++++++ 3 files changed, 54 insertions(+), 20 deletions(-) diff --git a/src/api.rs b/src/api.rs index 2083b2b..1cef44d 100644 --- a/src/api.rs +++ b/src/api.rs @@ -282,22 +282,10 @@ impl ApiClient { pub fn get_bytes(&self, path: &str, accept: &str) -> (reqwest::StatusCode, Vec) { let url = format!("{}{path}", self.api_url); let req = self.build_request(reqwest::Method::GET, &url).header("Accept", accept); - match req.build() { - Ok(request) => { - match self.client.execute(request) { - Ok(resp) => { - let status = resp.status(); - let bytes = resp.bytes().unwrap_or_default().to_vec(); - (status, bytes) - } - Err(e) => { - eprintln!("error connecting to API: {e}"); - std::process::exit(1); - } - } - } + match util::send_debug_bytes(&self.client, req) { + Ok(pair) => pair, Err(e) => { - eprintln!("error building request: {e}"); + eprintln!("error connecting to API: {e}"); std::process::exit(1); } } diff --git a/src/query.rs b/src/query.rs index dd505ca..09d90d1 100644 --- a/src/query.rs +++ b/src/query.rs @@ -10,8 +10,7 @@ pub struct QueryResponse { pub columns: Vec, pub rows: Vec>, pub row_count: u64, - #[serde(default)] - pub execution_time_ms: u64, + pub execution_time_ms: Option, pub warning: Option, } @@ -127,7 +126,7 @@ fn arrow_ipc_to_query_response(bytes: Vec, result_id: String) -> QueryRespon columns, rows, row_count, - execution_time_ms: 0, + execution_time_ms: None, warning: None, } } @@ -182,9 +181,20 @@ pub fn execute( let run_id = &async_resp.query_run_id; let spinner = crate::util::spinner("waiting for query..."); + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(300); loop { std::thread::sleep(std::time::Duration::from_millis(500)); + if std::time::Instant::now() > deadline { + spinner.finish_and_clear(); + use crossterm::style::Stylize; + eprintln!("{}", "query timed out after 5 minutes".red()); + eprintln!( + "{}", + format!("Check status with: hotdata query status {run_id}").dark_grey() + ); + std::process::exit(1); + } let run: QueryRunResponse = api.get(&format!("/query-runs/{run_id}")); match run.status.as_str() { "succeeded" => { @@ -312,13 +322,17 @@ pub fn print_result(result: &QueryResponse, format: &str) { .as_deref() .map(|id| format!(" [result-id: {id}]")) .unwrap_or_default(); + let time_part = match result.execution_time_ms { + Some(ms) => format!("{ms} ms"), + None => "\u{2014}".to_string(), // em dash + }; eprintln!( "{}", format!( - "\n{} row{} ({} ms){}", + "\n{} row{} ({}){}", result.row_count, if result.row_count == 1 { "" } else { "s" }, - result.execution_time_ms, + time_part, id_part ) .dark_grey() @@ -327,3 +341,4 @@ pub fn print_result(result: &QueryResponse, format: &str) { _ => unreachable!(), } } + diff --git a/src/util.rs b/src/util.rs index a5ee4cb..8bf3b8b 100644 --- a/src/util.rs +++ b/src/util.rs @@ -132,6 +132,37 @@ pub fn send_debug( send_debug_with_redaction(client, builder, body_for_log, &[]) } +/// Like `send_debug` but for binary (non-UTF-8) responses such as Arrow IPC. +/// Logs the request and response status in debug mode; prints the byte count +/// instead of attempting to parse the body as JSON. +pub fn send_debug_bytes( + client: &reqwest::blocking::Client, + builder: reqwest::blocking::RequestBuilder, +) -> reqwest::Result<(reqwest::StatusCode, Vec)> { + let request = builder.build()?; + if is_debug() { + log_request_struct(&request, None); + } + let resp = client.execute(request)?; + let status = resp.status(); + let bytes = resp.bytes()?; + if is_debug() { + use crossterm::style::Stylize; + let status_str = format!( + "<<< {} {}", + status.as_u16(), + status.canonical_reason().unwrap_or("") + ); + if status.is_success() { + eprintln!("{}", status_str.dark_green()); + } else { + eprintln!("{}", status_str.dark_red()); + } + eprintln!("{}", format!("[binary: {} bytes]", bytes.len()).dark_grey()); + } + Ok((status, bytes.to_vec())) +} + /// Like `send_debug` but masks the named JSON keys in the printed /// response body. The returned body string is always unredacted. pub fn send_debug_with_redaction( From 94269464656d122d8bb258892c569442f665b3ac Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Sat, 23 May 2026 10:56:00 -0700 Subject: [PATCH 3/5] fix(query): handle unknown poll statuses instead of spinning forever MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the `_ => continue` catch-all in the async polling loop with explicit known in-flight states (`running | queued | pending`) and an unknown-status arm that clears the spinner, prints the unexpected status, and exits with code 2 — matching the behaviour of `poll()`. This prevents infinite loops on terminal statuses like `cancelled` or `timed_out` that the server may add in the future. Co-Authored-By: Claude Sonnet 4.6 --- src/query.rs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/query.rs b/src/query.rs index 09d90d1..30bb29d 100644 --- a/src/query.rs +++ b/src/query.rs @@ -218,7 +218,17 @@ pub fn execute( eprintln!("{}", format!("query failed: {err}").red()); std::process::exit(1); } - _ => continue, + "running" | "queued" | "pending" => continue, + status => { + spinner.finish_and_clear(); + use crossterm::style::Stylize; + eprintln!("{}", format!("query status: {status}").yellow()); + eprintln!( + "{}", + format!("Check status with: hotdata query status {run_id}").dark_grey() + ); + std::process::exit(2); + } } } } From d4a95e6356c445d5cc61d0e5e0e76bf6aa13d9bb Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Sat, 23 May 2026 10:59:14 -0700 Subject: [PATCH 4/5] fix(query): two issues from Codex review - Rename QueryRunResponse.error -> error_message to match the actual server field name; previously failed async queries always printed "query failed: unknown error" instead of the real message - Make fetch_arrow_result pub(crate) and use it in results::get so that `hotdata results get ` fetches Arrow IPC like the rest of the query path, rather than falling back to JSON Co-Authored-By: Claude Sonnet 4.6 --- src/query.rs | 8 ++++---- src/results.rs | 5 +---- 2 files changed, 5 insertions(+), 8 deletions(-) diff --git a/src/query.rs b/src/query.rs index 30bb29d..c37a36c 100644 --- a/src/query.rs +++ b/src/query.rs @@ -26,7 +26,7 @@ struct QueryRunResponse { status: String, result_id: Option, #[serde(default)] - error: Option, + error_message: Option, } fn value_to_string(v: &Value) -> String { @@ -132,7 +132,7 @@ fn arrow_ipc_to_query_response(bytes: Vec, result_id: String) -> QueryRespon } /// Fetch `/results/{result_id}` as Arrow IPC and return a `QueryResponse`. -fn fetch_arrow_result(api: &ApiClient, result_id: &str) -> QueryResponse { +pub(crate) fn fetch_arrow_result(api: &ApiClient, result_id: &str) -> QueryResponse { let (status, bytes) = api.get_bytes(&format!("/results/{result_id}"), ACCEPT_ARROW); if !status.is_success() { use crossterm::style::Stylize; @@ -214,7 +214,7 @@ pub fn execute( "failed" => { spinner.finish_and_clear(); use crossterm::style::Stylize; - let err = run.error.as_deref().unwrap_or("unknown error"); + let err = run.error_message.as_deref().unwrap_or("unknown error"); eprintln!("{}", format!("query failed: {err}").red()); std::process::exit(1); } @@ -274,7 +274,7 @@ pub fn poll(query_run_id: &str, workspace_id: &str, format: &str) { }, "failed" => { use crossterm::style::Stylize; - let err = run.error.as_deref().unwrap_or("unknown error"); + let err = run.error_message.as_deref().unwrap_or("unknown error"); eprintln!("{}", format!("query failed: {err}").red()); std::process::exit(1); } diff --git a/src/results.rs b/src/results.rs index e76ce15..d1ff56f 100644 --- a/src/results.rs +++ b/src/results.rs @@ -64,9 +64,6 @@ pub fn list(workspace_id: &str, limit: Option, offset: Option, format: pub fn get(result_id: &str, workspace_id: &str, format: &str) { let api = ApiClient::new(Some(workspace_id)); - - let path = format!("/results/{result_id}"); - let result: crate::query::QueryResponse = api.get(&path); - + let result = crate::query::fetch_arrow_result(&api, result_id); crate::query::print_result(&result, format); } From 99e7736a16d13abdf2ed1e2cb8f741773fae4719 Mon Sep 17 00:00:00 2001 From: Eddie A Tejeda <669988+eddietejeda@users.noreply.github.com> Date: Sat, 23 May 2026 11:08:53 -0700 Subject: [PATCH 5/5] feat(cli): fix three commands that failed smoke test skills list - Add List variant to SkillCommands as an alias for status databases show - Add Show subcommand to DatabasesCommands; dispatches to databases::get() - Bare positional (hotdata databases ) preserved for backward compat databases tables - Make Tables subcommand optional; bare positional db_id triggers tables_list - hotdata databases tables now lists tables without requiring `list` subcommand - hotdata databases tables list --database still works (backward compat) - hotdata databases tables with no args prints help Co-Authored-By: Claude Sonnet 4.6 --- src/command.rs | 17 ++++++++++++++++- src/main.rs | 51 ++++++++++++++++++++++++++++++++++++-------------- 2 files changed, 53 insertions(+), 15 deletions(-) diff --git a/src/command.rs b/src/command.rs index bfc2171..9f36e77 100644 --- a/src/command.rs +++ b/src/command.rs @@ -547,6 +547,16 @@ pub enum DatabasesCommands { output: String, }, + /// Show details for a specific managed database + Show { + /// Database name or ID + name_or_id: String, + + /// Output format + #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])] + output: String, + }, + /// Create a new managed database Create { /// Optional display label (not unique, not an identifier — databases are addressed by id) @@ -604,8 +614,11 @@ pub enum DatabasesCommands { /// Manage tables inside a managed database Tables { + /// Database id or description — shorthand for `tables list` when no subcommand is given + database: Option, + #[command(subcommand)] - command: DatabaseTablesCommands, + command: Option, }, } @@ -738,6 +751,8 @@ pub enum SkillCommands { }, /// Show the installation status of the hotdata skill Status, + /// List installed skills and their versions (alias for status) + List, } #[derive(Subcommand)] diff --git a/src/main.rs b/src/main.rs index 7cd9167..e0474ea 100644 --- a/src/main.rs +++ b/src/main.rs @@ -390,6 +390,9 @@ fn main() { Some(DatabasesCommands::List { output }) => { databases::list(&workspace_id, &output) } + Some(DatabasesCommands::Show { name_or_id, output }) => { + databases::get(&workspace_id, &name_or_id, &output) + } Some(DatabasesCommands::Create { description, schema, @@ -427,43 +430,63 @@ fn main() { upload_id.as_deref(), ) } - Some(DatabasesCommands::Tables { command }) => match command { - DatabaseTablesCommands::List { - database, + Some(DatabasesCommands::Tables { database, command }) => match command { + Some(DatabaseTablesCommands::List { + database: db_flag, schema, output, - } => databases::tables_list( + }) => databases::tables_list( &workspace_id, - database.as_deref(), + db_flag.as_deref().or(database.as_deref()), schema.as_deref(), &output, ), - DatabaseTablesCommands::Load { - database, + Some(DatabaseTablesCommands::Load { + database: db_flag, table, schema, file, url, upload_id, - } => databases::tables_load( + }) => databases::tables_load( &workspace_id, - database.as_deref(), + db_flag.as_deref().or(database.as_deref()), &table, Some(schema.as_str()), file.as_deref(), url.as_deref(), upload_id.as_deref(), ), - DatabaseTablesCommands::Delete { - database, + Some(DatabaseTablesCommands::Delete { + database: db_flag, table, schema, - } => databases::tables_delete( + }) => databases::tables_delete( &workspace_id, - database.as_deref(), + db_flag.as_deref().or(database.as_deref()), &table, Some(schema.as_str()), ), + None => { + if let Some(ref db) = database { + databases::tables_list( + &workspace_id, + Some(db.as_str()), + None, + "table", + ) + } else { + use clap::CommandFactory; + let mut cmd = Cli::command(); + cmd.build(); + cmd.find_subcommand_mut("databases") + .unwrap() + .find_subcommand_mut("tables") + .unwrap() + .print_help() + .unwrap(); + } + } }, None => { use clap::CommandFactory; @@ -507,7 +530,7 @@ fn main() { skill::install() } } - SkillCommands::Status => skill::status(), + SkillCommands::Status | SkillCommands::List => skill::status(), }, Commands::Results { result_id,