From 0785c5c514ce929312bba2e8bd9e49df91468920 Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Thu, 23 Jul 2026 18:05:48 +0530 Subject: [PATCH 1/4] feat(databases): drain cursor for paginated databases list GET /v1/databases is now paginated (one capped page + next_cursor); the SDK's databases().list gained limit/cursor in hotdata 0.10.0. Bump the dep and follow the cursor to the end so 'databases list', name/ catalog resolution, and the whole-workspace indexes-list id scan still see every database. --- Cargo.lock | 4 +-- Cargo.toml | 2 +- src/commands/databases.rs | 56 +++++++++++++++++++++++++++++++-------- 3 files changed, 48 insertions(+), 14 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 62de7f5..2ffd345 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1186,9 +1186,9 @@ checksum = "7f24254aa9a54b5c858eaee2f5bccdb46aaf0e486a595ed5fd8f86ba55232a70" [[package]] name = "hotdata" -version = "0.9.1" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a1bda44232e3c33228b57881f00cd6c784cda596e88d0886c1807baf6b63ada" +checksum = "0385ea667c306c33223aba8e7765261f455284f9a549c067a09e6c1742a732c7" dependencies = [ "arrow-array", "arrow-ipc", diff --git a/Cargo.toml b/Cargo.toml index 53288a3..f5edc25 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,7 +16,7 @@ path = "src/main.rs" # behind a shared multi-thread tokio runtime. The `arrow` feature backs result # decode; `upload_file` runs the presigned direct-to-storage upload flow # (see src/sdk.rs::Api::upload). -hotdata = { version = "0.9.1", features = ["arrow"] } +hotdata = { version = "0.10.0", features = ["arrow"] } # Shared multi-thread runtime for the sync wrapper; block_on is called # concurrently from rayon worker threads (see src/indexes.rs). The presigned # upload (src/sdk.rs::Api::upload) drives the SDK's async `upload_file` on this diff --git a/src/commands/databases.rs b/src/commands/databases.rs index a4b6e53..747dab9 100644 --- a/src/commands/databases.rs +++ b/src/commands/databases.rs @@ -411,15 +411,50 @@ pub(crate) fn get_database(api: &Api, id: &str) -> Result { block(api.client().databases().get(id)).map(Database::from) } -/// List databases through the SDK's typed `databases().list` handle, mapped -/// into the CLI's summary rows. +/// Drain every page of the paginated databases list into the full set of +/// summary rows. /// -/// Routed through [`block_with_wakeup`] so a cold KEDA start surfaces a "waking -/// up worker" hint instead of an unexplained pause — `databases list` is a -/// common first command against an idle workspace. +/// `GET /v1/databases` returns one capped page plus a `next_cursor`; the CLI +/// shows and resolves against the whole workspace, so follow the cursor to the +/// end. When `spinner` is `Some`, the first page is fetched via +/// [`block_with_wakeup`] so a cold KEDA start surfaces a "waking up worker" +/// hint; the remaining pages (and the spinner-less id-only caller) use +/// [`block`]. +fn list_all_databases( + api: &Api, + spinner: Option<&str>, +) -> Result, ApiError> { + let mut all = Vec::new(); + let mut cursor: Option = None; + let mut first = true; + loop { + let resp = match (first, spinner) { + (true, Some(msg)) => block_with_wakeup( + api, + msg, + api.client().databases().list(None, cursor.as_deref()), + )?, + _ => block(api.client().databases().list(None, cursor.as_deref()))?, + }; + first = false; + let next = resp.next_cursor.clone().flatten(); + all.extend(resp.databases); + // Stop at the last page, or if the server ever returns a non-advancing + // cursor (defensive — never loop forever). + match next { + Some(c) if !c.is_empty() && Some(&c) != cursor.as_ref() => cursor = Some(c), + _ => break, + } + } + Ok(all) +} + +/// List databases, mapped into the CLI's summary rows. Drains all pages so +/// `databases list` shows the whole workspace and name/catalog resolution sees +/// every database (see [`list_all_databases`]). fn list_database_summaries(api: &Api) -> Result, ApiError> { - block_with_wakeup(api, "Loading databases...", api.client().databases().list()) - .map(|r| r.databases.into_iter().map(DatabaseSummary::from).collect()) + list_all_databases(api, Some("Loading databases...")) + .map(|dbs| dbs.into_iter().map(DatabaseSummary::from).collect()) } /// List the ids of every managed database in the workspace. @@ -431,11 +466,10 @@ fn list_database_summaries(api: &Api) -> Result, ApiError> /// `default_connection_id` via [`get_database`]. The list summary omits the /// connection id, hence ids only. /// -/// Deliberately spinner-less (plain [`block`], unlike -/// [`list_database_summaries`]): the caller owns its own "Loading indexes…" -/// spinner, and two indicatif bars would fight over the same line. +/// Deliberately spinner-less (the caller owns its own "Loading indexes…" +/// spinner, and two indicatif bars would fight over the same line). pub(crate) fn list_database_ids(api: &Api) -> Result, ApiError> { - block(api.client().databases().list()).map(|r| r.databases.into_iter().map(|d| d.id).collect()) + list_all_databases(api, None).map(|dbs| dbs.into_iter().map(|d| d.id).collect()) } fn fetch_database(api: &Api, id: &str) -> Database { From d51a4c17e91c4bc8037349fbaad045fc41e1614d Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Thu, 23 Jul 2026 18:21:30 +0530 Subject: [PATCH 2/4] feat(databases): page 'databases list' with --limit/--cursor Show one capped page + a 'use --cursor ' hint (matching tables/queries list) instead of draining the whole workspace into memory. Name/catalog resolution and the whole-workspace indexes-list scan still drain every page internally (list_all_databases). --- src/commands/databases.rs | 45 ++++++++++++++++++++++++++++++++++++--- src/main.rs | 8 ++++--- 2 files changed, 47 insertions(+), 6 deletions(-) diff --git a/src/commands/databases.rs b/src/commands/databases.rs index 747dab9..f7f4bb0 100644 --- a/src/commands/databases.rs +++ b/src/commands/databases.rs @@ -8,6 +8,14 @@ use std::path::Path; pub enum DatabasesCommands { /// List managed databases in the workspace List { + /// Maximum number of databases to return + #[arg(long)] + limit: Option, + + /// Pagination cursor from a previous response + #[arg(long)] + cursor: Option, + /// Output format #[arg(long = "output", short = 'o', default_value = "table", value_parser = ["table", "json", "yaml"])] output: String, @@ -917,10 +925,29 @@ fn collect_tables( (out, false, None) } -pub fn list(workspace_id: &str, format: &str) { +pub fn list(workspace_id: &str, format: &str, limit: Option, cursor: Option<&str>) { let api = Api::new(Some(workspace_id)); - let mut databases = list_database_summaries(&api).unwrap_or_else(|e| e.exit()); - databases.sort_by(|a, b| a.created_at.cmp(&b.created_at)); + // One page, user-paginated (like `tables list` / `queries list`). Name/ + // catalog resolution and the whole-workspace `indexes list` scan still see + // every database — they drain all pages internally via list_all_databases; + // only this display command pages. + let resp = block_with_wakeup( + &api, + "Loading databases...", + api.client() + .databases() + .list(limit.map(|l| l as i32), cursor), + ) + .unwrap_or_else(|e| e.exit()); + let has_more = resp.has_more.flatten().unwrap_or(false); + let next_cursor = resp.next_cursor.clone().flatten(); + // Server returns newest-first; keep that order so the cursor continuation is + // coherent (no client re-sort across pages). + let databases: Vec = resp + .databases + .into_iter() + .map(DatabaseSummary::from) + .collect(); match format { "json" => println!("{}", serde_json::to_string_pretty(&databases).unwrap()), @@ -959,6 +986,18 @@ pub fn list(workspace_id: &str, format: &str) { } _ => unreachable!(), } + + if has_more { + use crossterm::style::Stylize; + eprintln!( + "{}", + format!( + "More results available. Use --cursor {} to fetch the next page.", + next_cursor.as_deref().unwrap_or("") + ) + .dark_grey() + ); + } } pub fn get(workspace_id: &str, id_or_name: &str, format: &str) { diff --git a/src/main.rs b/src/main.rs index fb28e1d..f76806b 100644 --- a/src/main.rs +++ b/src/main.rs @@ -257,9 +257,11 @@ fn main() { databases::get(&workspace_id, &name_or_id, &output); } else { match command { - Some(DatabasesCommands::List { output }) => { - databases::list(&workspace_id, &output) - } + Some(DatabasesCommands::List { + output, + limit, + cursor, + }) => databases::list(&workspace_id, &output, limit, cursor.as_deref()), Some(DatabasesCommands::Show { name_or_id, output }) => { databases::get(&workspace_id, &name_or_id, &output) } From 3c386eb01cb3653433043d7953a6341df79c7a64 Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Thu, 23 Jul 2026 18:34:36 +0530 Subject: [PATCH 3/4] test(databases): cover multi-page cursor drain; drop clone Address review nits on #239: - add list_all_databases_follows_cursor: two mocked pages exercise the cursor-following drain (the prior mocks were single-page) and the non-advancing-cursor guard. - drop the unnecessary next_cursor clone in list_all_databases by moving the page rows out first (partial move). --- src/commands/databases.rs | 36 +++++++++++++++++++++++++++++++++++- 1 file changed, 35 insertions(+), 1 deletion(-) diff --git a/src/commands/databases.rs b/src/commands/databases.rs index f7f4bb0..5f07b86 100644 --- a/src/commands/databases.rs +++ b/src/commands/databases.rs @@ -445,8 +445,10 @@ fn list_all_databases( _ => block(api.client().databases().list(None, cursor.as_deref()))?, }; first = false; - let next = resp.next_cursor.clone().flatten(); + // Move the page rows out first (partial move), then read next_cursor — + // no clone needed. all.extend(resp.databases); + let next = resp.next_cursor.flatten(); // Stop at the last page, or if the server ever returns a non-advancing // cursor (defensive — never loop forever). match next { @@ -2403,6 +2405,38 @@ mod tests { assert_eq!(tables[1].table, "b"); } + #[test] + fn list_all_databases_follows_cursor() { + let mut server = mockito::Server::new(); + // Page 2 (reached via ?cursor=cur2): the last database, no next_cursor. + let page2 = server + .mock("GET", "/v1/databases") + .match_query(mockito::Matcher::UrlEncoded("cursor".into(), "cur2".into())) + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"databases":[{"id":"db2","name":"second","default_catalog":"c2","default_schema":"main"}],"count":1,"limit":1,"has_more":false,"next_cursor":null}"#, + ) + .create(); + // Page 1 (first request, no cursor): first database + a next_cursor. + let page1 = server + .mock("GET", "/v1/databases") + .with_status(200) + .with_header("content-type", "application/json") + .with_body( + r#"{"databases":[{"id":"db1","name":"first","default_catalog":"c1","default_schema":"main"}],"count":1,"limit":1,"has_more":true,"next_cursor":"cur2"}"#, + ) + .create(); + + let api = Api::test_new(&server.url(), "k", Some("ws")); + let dbs = list_all_databases(&api, None).unwrap(); + page1.assert(); + page2.assert(); + assert_eq!(dbs.len(), 2, "drain must reassemble both pages"); + assert_eq!(dbs[0].id, "db1"); + assert_eq!(dbs[1].id, "db2"); + } + #[test] fn collect_tables_with_limit_makes_single_request_and_returns_cursor() { let mut server = mockito::Server::new(); From 00b25ccba0346a3f0c0614c0b769621f13b1464a Mon Sep 17 00:00:00 2001 From: Anoop Narang Date: Thu, 23 Jul 2026 18:46:27 +0530 Subject: [PATCH 4/4] refactor(databases): drop clone in list display too Same next_cursor.clone() avoidance as list_all_databases: read the cursor before moving the page rows out (partial move). --- src/commands/databases.rs | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/commands/databases.rs b/src/commands/databases.rs index 5f07b86..eeb6657 100644 --- a/src/commands/databases.rs +++ b/src/commands/databases.rs @@ -942,7 +942,8 @@ pub fn list(workspace_id: &str, format: &str, limit: Option, cursor: Option ) .unwrap_or_else(|e| e.exit()); let has_more = resp.has_more.flatten().unwrap_or(false); - let next_cursor = resp.next_cursor.clone().flatten(); + // Take next_cursor before moving the rows out below — no clone needed. + let next_cursor = resp.next_cursor.flatten(); // Server returns newest-first; keep that order so the cursor continuation is // coherent (no client re-sort across pages). let databases: Vec = resp