From fa52187ac3866a4015839559a04bb95144d16396 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Fri, 18 Jul 2025 23:29:13 +0200 Subject: [PATCH 1/4] fix: speed up migrations --- .../bucket/migrations/19-transactions-fill-pcv/up.sql | 6 ++++-- .../storage/bucket/migrations/27-fix-invalid-pcv/up.sql | 6 ++++-- .../bucket/migrations/28-fix-pcv-missing-asset/up.sql | 6 ++++-- .../bucket/migrations/31-fix-transaction-updated-at/up.sql | 7 ++++--- 4 files changed, 16 insertions(+), 9 deletions(-) diff --git a/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql b/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql index d7865df07d..ad304ce7bd 100644 --- a/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql +++ b/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql @@ -7,10 +7,10 @@ do $$ drop table if exists moves_view; - create temp table moves_view as + create table moves_view as select transactions_seq, public.aggregate_objects(jsonb_build_object(accounts_address, volumes)) as volumes from ( - select transactions_seq::numeric, accounts_address, public.aggregate_objects(json_build_object(asset, json_build_object('input', (post_commit_volumes).inputs, 'output', (post_commit_volumes).outputs))::jsonb) as volumes + select transactions_seq, accounts_address, public.aggregate_objects(json_build_object(asset, json_build_object('input', (post_commit_volumes).inputs, 'output', (post_commit_volumes).outputs))::jsonb) as volumes from ( SELECT DISTINCT ON (moves.transactions_seq, accounts_address, asset) moves.transactions_seq, accounts_address, asset, first_value(post_commit_volumes) OVER ( @@ -27,6 +27,8 @@ do $$ group by transactions_seq; create index moves_view_idx on moves_view(transactions_seq); + -- speed up hash join when updating rows later + alter table moves_view add foreign key(transactions_seq) references transactions(seq); if (select count(*) from moves_view) = 0 then return; diff --git a/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql b/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql index 33d9751a72..9068369f38 100644 --- a/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql +++ b/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql @@ -7,10 +7,10 @@ do $$ drop table if exists moves_view; - create temp table moves_view as + create table moves_view as select transactions_seq, public.aggregate_objects(jsonb_build_object(accounts_address, volumes)) as volumes from ( - select transactions_seq::numeric, accounts_address, public.aggregate_objects(json_build_object(asset, json_build_object('input', (post_commit_volumes).inputs, 'output', (post_commit_volumes).outputs))::jsonb) as volumes + select transactions_seq, accounts_address, public.aggregate_objects(json_build_object(asset, json_build_object('input', (post_commit_volumes).inputs, 'output', (post_commit_volumes).outputs))::jsonb) as volumes from ( SELECT DISTINCT ON (moves.transactions_seq, accounts_address, asset) moves.transactions_seq, accounts_address, asset, first_value(post_commit_volumes) OVER ( @@ -27,6 +27,8 @@ do $$ group by transactions_seq; create index moves_view_idx on moves_view(transactions_seq); + -- speed up hash join when updating rows later + alter table moves_view add foreign key(transactions_seq) references transactions(seq); if (select count(*) from moves_view) = 0 then return; diff --git a/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql b/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql index c8bc4f21f4..57669be2d4 100644 --- a/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql +++ b/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql @@ -7,10 +7,10 @@ do $$ drop table if exists moves_view; - create temp table moves_view as + create table moves_view as select transactions_seq, public.aggregate_objects(jsonb_build_object(accounts_address, volumes)) as volumes from ( - select transactions_seq::numeric, accounts_address, public.aggregate_objects(json_build_object(asset, json_build_object('input', (post_commit_volumes).inputs, 'output', (post_commit_volumes).outputs))::jsonb) as volumes + select transactions_seq, accounts_address, public.aggregate_objects(json_build_object(asset, json_build_object('input', (post_commit_volumes).inputs, 'output', (post_commit_volumes).outputs))::jsonb) as volumes from ( SELECT DISTINCT ON (moves.transactions_seq, accounts_address, asset) moves.transactions_seq, accounts_address, asset, first_value(post_commit_volumes) OVER ( @@ -27,6 +27,8 @@ do $$ group by transactions_seq; create index moves_view_idx on moves_view(transactions_seq); + -- speed up hash join when updating rows later + alter table moves_view add foreign key(transactions_seq) references transactions(seq); if (select count(*) from moves_view) = 0 then return; diff --git a/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql b/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql index 8fdf3bdedd..57172b0d17 100644 --- a/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql +++ b/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql @@ -7,7 +7,7 @@ do $$ drop table if exists txs_view; - create temp table txs_view as + create table txs_view as select * from transactions where updated_at is null; @@ -15,6 +15,8 @@ do $$ if (select count(*) from txs_view) = 0 then return; end if; + -- speed up hash join when updating rows later + alter table txs_view add foreign key(seq) references transactions(seq); perform pg_notify('migrations-{{ .Schema }}', 'init: ' || (select count(*) from txs_view)); @@ -29,8 +31,7 @@ do $$ update transactions set updated_at = transactions.inserted_at from data - where transactions.seq = data.seq and - transactions.ledger = data.ledger; + where transactions.seq = data.seq; exit when not found; From 350af1f6142fce7b539dadb258646af0833c85a6 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Sat, 19 Jul 2025 15:54:08 +0200 Subject: [PATCH 2/4] chore: clean temporary tables --- .../bucket/migrations/19-transactions-fill-pcv/up.sql | 6 +++++- .../storage/bucket/migrations/27-fix-invalid-pcv/up.sql | 6 +++++- .../bucket/migrations/28-fix-pcv-missing-asset/up.sql | 6 +++++- .../bucket/migrations/31-fix-transaction-updated-at/up.sql | 6 +++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql b/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql index ad304ce7bd..a57ae4cd1b 100644 --- a/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql +++ b/internal/storage/bucket/migrations/19-transactions-fill-pcv/up.sql @@ -31,6 +31,7 @@ do $$ alter table moves_view add foreign key(transactions_seq) references transactions(seq); if (select count(*) from moves_view) = 0 then + drop table moves_view; return; end if; @@ -48,7 +49,10 @@ do $$ from data where transactions.seq = data.transactions_seq; - exit when not found; + if not found then + drop table moves_view; + exit; + end if; _offset = _offset + _batch_size; diff --git a/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql b/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql index 9068369f38..de742a279c 100644 --- a/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql +++ b/internal/storage/bucket/migrations/27-fix-invalid-pcv/up.sql @@ -31,6 +31,7 @@ do $$ alter table moves_view add foreign key(transactions_seq) references transactions(seq); if (select count(*) from moves_view) = 0 then + drop table moves_view; return; end if; @@ -48,7 +49,10 @@ do $$ from data where transactions.seq = data.transactions_seq; - exit when not found; + if not found then + drop table moves_view; + exit; + end if; _offset = _offset + _batch_size; diff --git a/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql b/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql index 57669be2d4..25166375bb 100644 --- a/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql +++ b/internal/storage/bucket/migrations/28-fix-pcv-missing-asset/up.sql @@ -31,6 +31,7 @@ do $$ alter table moves_view add foreign key(transactions_seq) references transactions(seq); if (select count(*) from moves_view) = 0 then + drop table moves_view; return; end if; @@ -48,7 +49,10 @@ do $$ from data where transactions.seq = data.transactions_seq; - exit when not found; + if not found then + drop table moves_view; + exit; + end if; _offset = _offset + _batch_size; diff --git a/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql b/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql index 57172b0d17..918b95649b 100644 --- a/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql +++ b/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql @@ -13,6 +13,7 @@ do $$ where updated_at is null; if (select count(*) from txs_view) = 0 then + drop table txs_view; return; end if; -- speed up hash join when updating rows later @@ -33,7 +34,10 @@ do $$ from data where transactions.seq = data.seq; - exit when not found; + if not found then + drop table txs_view; + exit; + end if; _offset = _offset + _batch_size; From b4e350d58a226711b9f6b35f7cfc1ab3febf8000 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Sat, 19 Jul 2025 19:02:44 +0200 Subject: [PATCH 3/4] feat: optimize another migration --- .../storage/bucket/migrations/34-fix-memento-format/up.sql | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/internal/storage/bucket/migrations/34-fix-memento-format/up.sql b/internal/storage/bucket/migrations/34-fix-memento-format/up.sql index 023b33bbfc..ea262eb72f 100644 --- a/internal/storage/bucket/migrations/34-fix-memento-format/up.sql +++ b/internal/storage/bucket/migrations/34-fix-memento-format/up.sql @@ -1,7 +1,7 @@ do $$ declare _offset integer := 0; - _batch_size integer := 1000; + _batch_size integer := 10000; begin set search_path = '{{ .Schema }}'; @@ -15,9 +15,8 @@ do $$ with data as ( select * from logs + where seq >= _offset and seq < _offset + _batch_size order by seq - offset _offset - limit _batch_size ) update logs set memento = convert_to( From 160d73717611e4fbb2f704d5974265a1b9770a89 Mon Sep 17 00:00:00 2001 From: Geoffrey Ragot Date: Sat, 19 Jul 2025 20:42:54 +0200 Subject: [PATCH 4/4] chore: fix some potential error and improve a migration --- .../bucket/migrations/31-fix-transaction-updated-at/up.sql | 1 + .../storage/bucket/migrations/34-fix-memento-format/up.sql | 6 +++--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql b/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql index 918b95649b..3b0e83fd4b 100644 --- a/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql +++ b/internal/storage/bucket/migrations/31-fix-transaction-updated-at/up.sql @@ -17,6 +17,7 @@ do $$ return; end if; -- speed up hash join when updating rows later + create index txs_view_seq_idx on txs_view(seq); alter table txs_view add foreign key(seq) references transactions(seq); perform pg_notify('migrations-{{ .Schema }}', 'init: ' || (select count(*) from txs_view)); diff --git a/internal/storage/bucket/migrations/34-fix-memento-format/up.sql b/internal/storage/bucket/migrations/34-fix-memento-format/up.sql index ea262eb72f..3972f03be4 100644 --- a/internal/storage/bucket/migrations/34-fix-memento-format/up.sql +++ b/internal/storage/bucket/migrations/34-fix-memento-format/up.sql @@ -81,7 +81,9 @@ do $$ from data where logs.seq = data.seq; - exit when not found; + if _offset >= (select max(seq) from logs) then + exit; + end if; _offset = _offset + _batch_size; @@ -89,7 +91,5 @@ do $$ commit; end loop; - - drop table if exists txs_view; end $$; \ No newline at end of file