Skip to content

fix(repo): Migrating away from using the cursor field for ordering + pagination #494

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

fix(repo): Migrating away from using the cursor field for ordering + pagination #494

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
5b76216
feat: migrate the pagination `order_by` cursor to a multi-field system
petertonysmith94 Jul 10, 2025
d429428
feat: migrated away from cursor-based pagination
petertonysmith94 Jul 10, 2025
58ea54e
refactor: cleaning up the `apply_pagination` logic
petertonysmith94 Jul 10, 2025
fb4d018
refactor: re-added the join_prefix
petertonysmith94 Jul 14, 2025
2d93d05
Merge branch 'main' into ps/chore/migrate-away-from-cursor
luizstacio Jul 14, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions crates/domains/src/blocks/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,19 +71,21 @@ impl QueryParamsBuilder for BlocksQuery {
conditions.push(format!("block_height = {}", height));
}

let cursor_fields = &["block_height"];

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"block_height",
cursor_fields,
None,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"block_height",
cursor_fields,
None,
);

Expand Down
8 changes: 7 additions & 1 deletion crates/domains/src/infra/db/cursor.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,16 +12,22 @@ use serde::{Deserialize, Serialize};
pub struct Cursor(Cow<'static, str>);

impl Cursor {
const SEPARATOR: &str = "-";

pub fn new(fields: &[&dyn ToString]) -> Self {
Self(Cow::Owned(
fields
.iter()
.map(|f| f.to_string())
.collect::<Vec<_>>()
.join("-"),
.join(Self::SEPARATOR),
))
}

pub fn split(&self) -> Vec<&str> {
self.0.split(Self::SEPARATOR).collect()
}

pub fn from_static(s: &'static str) -> Self {
Self(Cow::Borrowed(s))
}
Expand Down
140 changes: 109 additions & 31 deletions crates/domains/src/infra/repository/query_builder.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ pub trait QueryParamsBuilder {
conditions: &mut Vec<String>,
options: &QueryOptions,
pagination: &QueryPagination,
cursor_field: &str,
cursor_fields: &[&str],
join_prefix: Option<&str>,
) {
if let Some(timestamp) = &options.timestamp {
Expand Down Expand Up @@ -107,27 +107,23 @@ pub trait QueryParamsBuilder {
}

if let Some(after) = pagination.after.as_ref() {
let field = Self::prefix_field(cursor_field, join_prefix);
if cursor_field == "block_height" {
// When using block height as the cursor field,
// we need to compare the block height as a number,
// not a string.
conditions.push(format!("{field} > {}", after));
} else {
conditions.push(format!("{field} > '{after}'"));
}
let after_conditions = Self::create_pagination_conditions(
cursor_fields,
after,
">",
join_prefix,
);
conditions.push(after_conditions);
}

if let Some(before) = pagination.before.as_ref() {
let field = Self::prefix_field(cursor_field, join_prefix);
if cursor_field == "block_height" {
// When using block height as the cursor field,
// we need to compare the block height as a number,
// not a string.
conditions.push(format!("{field} < {}", before));
} else {
conditions.push(format!("{field} < '{before}'"));
}
let before_conditions = Self::create_pagination_conditions(
cursor_fields,
before,
"<",
join_prefix,
);
conditions.push(before_conditions);
}

if !conditions.is_empty() {
Expand All @@ -136,36 +132,118 @@ pub trait QueryParamsBuilder {
}
}

/// Forms the required SQL clauses to replace the cursor with the values
///
/// Example:
///
/// ```md
/// block_height transaction_index receipt_index
///
/// 0001 0 0
/// 0001 0 1 <---
/// 0001 0 2
/// 0001 1 0
/// 0001 2 0
/// 0002 0 0
/// 0002 0 1
/// ```
///
/// ```sql
/// WHERE (
/// (block_height = 0001 AND transaction_index = 0 AND receipt_index > 1) OR
/// (block_height = 0001 AND transaction_index > 0) OR
/// (block_height > 0001)
/// )
/// ```
fn create_pagination_conditions(
cursor_fields: &[&str],
cursor: &Cursor,
operation: &str,
join_prefix: Option<&str>,
) -> String {
if cursor_fields.is_empty() || cursor.is_empty() {
return String::new();
}

let cursor_fields = cursor_fields
.iter()
.map(|f| Self::prefix_field(f, join_prefix))
.collect::<Vec<_>>();
let cursor_values = cursor.split();

let result = (0..cursor_values.len())
.rev()
.map(|i| {
let equality_conditions = (0..i)
.map(|j| {
format!("{} = {}", cursor_fields[j], cursor_values[j])
})
.collect::<Vec<_>>();

let operation_condition = format!(
"{} {} {}",
cursor_fields[i], operation, cursor_values[i]
);

let mut all_conditions = equality_conditions;
all_conditions.push(operation_condition);

format!("({})", all_conditions.join(" AND "))
})
.collect::<Vec<_>>()
.join(" OR ");
format!("({})", result)
}

fn apply_pagination(
query_builder: &mut QueryBuilder<Postgres>,
pagination: &QueryPagination,
cursor_field: &str,
cursor_fields: &[&str],
join_prefix: Option<&str>,
Copy link
Member

@luizstacio luizstacio Jul 11, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can we remove the prefix here? I believe some places use it to create pagination over join query. Like predicates endpoint

Copy link
Author

@petertonysmith94 petertonysmith94 Jul 14, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I had mitigated this by putting the join_prefix within the cursor_fields like this.

However, I believe it's better to have for consistency (63ec002)

) {
let field = Self::prefix_field(cursor_field, join_prefix);
let order_by: OrderBy;
let limit: i32;

match (pagination.first, pagination.last) {
(Some(first), None) => {
query_builder.push(format!(" ORDER BY {field} ASC"));
query_builder.push(format!(" LIMIT {first} "));
return;
order_by = OrderBy::Asc;
limit = first;
}
(None, Some(last)) => {
query_builder.push(format!(" ORDER BY {field} DESC"));
query_builder.push(format!(" LIMIT {last} "));
return;
order_by = OrderBy::Desc;
limit = last;
}
_ => {
limit = pagination.limit.unwrap_or(DEFAULT_LIMIT);
order_by =
pagination.order_by.to_owned().unwrap_or(OrderBy::Desc);
}
_ => {}
}

let limit = pagination.limit.unwrap_or(DEFAULT_LIMIT);
let order_by = pagination.order_by.to_owned().unwrap_or(OrderBy::Desc);
query_builder.push(format!(" ORDER BY {field} {order_by}"));
let order_by_sql =
Self::order_by_statement(cursor_fields, order_by, join_prefix);
query_builder.push(order_by_sql);
query_builder.push(format!(" LIMIT {limit}"));
if let Some(offset) = pagination.offset {
query_builder.push(format!(" OFFSET {offset}"));
}
}

fn order_by_statement(
order_by_fields: &[&str],
order_by: OrderBy,
join_prefix: Option<&str>,
) -> String {
let fields = order_by_fields
.iter()
.map(|field| Self::prefix_field(field, join_prefix))
.map(|field| format!("{field} {order_by}"))
.collect::<Vec<_>>()
.join(", ");

format!(" ORDER BY {fields}")
}

fn prefix_field(field: &str, prefix: Option<&str>) -> String {
match prefix {
Some(prefix) => format!("{prefix}{field}"),
Expand Down
6 changes: 4 additions & 2 deletions crates/domains/src/inputs/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,19 +137,21 @@ impl QueryParamsBuilder for InputsQuery {
));
}

let cursor_fields = &["block_height", "tx_index", "input_index"];

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Expand Down
6 changes: 4 additions & 2 deletions crates/domains/src/messages/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -106,19 +106,21 @@ impl QueryParamsBuilder for MessagesQuery {
));
}

let cursor_fields = &["block_height", "message_index"];

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Expand Down
6 changes: 4 additions & 2 deletions crates/domains/src/outputs/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -117,19 +117,21 @@ impl QueryParamsBuilder for OutputsQuery {
conditions.push(format!("to_address = '{}'", address));
}

let cursor_fields = &["block_height", "tx_index", "output_index"];

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Expand Down
11 changes: 7 additions & 4 deletions crates/domains/src/predicates/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,20 +86,23 @@ impl QueryParamsBuilder for PredicatesQuery {
conditions.push(format!("pt.asset_id = '{}'", asset));
}

let cursor_fields = &["block_height", "tx_index", "input_index"];
let join_prefix = Some("pt.");

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"cursor",
Some("pt."),
cursor_fields,
join_prefix,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"cursor",
Some("pt."),
cursor_fields,
join_prefix,
);

query_builder
Expand Down
6 changes: 4 additions & 2 deletions crates/domains/src/receipts/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -137,19 +137,21 @@ impl QueryParamsBuilder for ReceiptsQuery {
));
}

let cursor_fields = &["block_height", "tx_index", "receipt_index"];

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Expand Down
6 changes: 4 additions & 2 deletions crates/domains/src/transactions/query_params.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -114,19 +114,21 @@ impl QueryParamsBuilder for TransactionsQuery {
));
}

let cursor_fields = &["block_height", "tx_index"];

Self::apply_conditions(
&mut query_builder,
&mut conditions,
&self.options,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Self::apply_pagination(
&mut query_builder,
&self.pagination,
"cursor",
cursor_fields,
None,
);

Expand Down
Loading
Loading