Skip to content

Commit

Permalink
Changes to facilitate on-the-fly calculation of pool_history data for…
Browse files Browse the repository at this point in the history 
… most recent 3 epochs to deal with reward table population lag (#282)

Added helper function for fetching history for a given epoch range for
one to all pools, can be utilized for populating cache and is used in
updated pool_history function that performs calculations on the fly for
the most recent 3 epochs (to facilitate reward table update lag)

---------

Co-authored-by: rdlrt <3169068+rdlrt@users.noreply.github.com>
  • Loading branch information
@hodlonaut @rdlrt
hodlonaut and rdlrt committed May 21, 2024
1 parent e7b28b5 commit a333019
Show file tree
Hide file tree
Showing 3 changed files with 266 additions and 201 deletions.
185 changes: 5 additions & 180 deletions files/grest/rpc/01_cached_tables/pool_history_cache.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ BEGIN
END IF;

SELECT COALESCE(MAX(epoch_no), 0) INTO _latest_epoch_no_in_cache FROM grest.pool_history_cache;
-- Split into 100 epochs at a time to avoid hours spent on a single query (which can be risky if that query is killed)
-- Split into 500 epochs at a time to avoid hours spent on a single query (which can be risky if that query is killed)
SELECT LEAST( 500 , (MAX(no) - _latest_epoch_no_in_cache) ) + _latest_epoch_no_in_cache INTO _curr_epoch FROM epoch;

IF _epoch_no_to_insert_from IS NULL THEN
Expand All @@ -55,7 +55,7 @@ BEGIN
RETURN;
END IF;
-- no-op IF we already have data up until second most recent epoch
IF _latest_epoch_no_in_cache >= (_curr_epoch - 1) THEN
IF _latest_epoch_no_in_cache >= (_curr_epoch - 2) THEN
INSERT INTO grest.control_table (key, last_value)
VALUES ('pool_history_cache_last_updated', NOW() AT TIME ZONE 'utc')
ON CONFLICT (key)
Expand All @@ -69,186 +69,11 @@ BEGIN
DELETE FROM grest.pool_history_cache
WHERE epoch_no >= _epoch_no_to_insert_from;

INSERT INTO grest.pool_history_cache (
WITH
blockcounts AS (
SELECT
sl.pool_hash_id,
b.epoch_no,
COUNT(*) AS block_cnt
FROM block AS b,
slot_leader AS sl
WHERE b.slot_leader_id = sl.id
AND b.epoch_no >= _epoch_no_to_insert_from
GROUP BY
sl.pool_hash_id,
b.epoch_no
),

leadertotals AS (
SELECT
r.pool_id,
r.earned_epoch,
COALESCE(SUM(r.amount), 0) AS leadertotal
FROM reward AS r
WHERE r.type = 'leader'
AND r.earned_epoch >= _epoch_no_to_insert_from
GROUP BY
r.pool_id,
r.earned_epoch
),

membertotals AS (
SELECT
r.pool_id,
r.earned_epoch,
COALESCE(SUM(r.amount), 0) AS memtotal
FROM reward AS r
WHERE r.type = 'member'
AND r.earned_epoch >= _epoch_no_to_insert_from
GROUP BY
r.pool_id,
r.earned_epoch
),
RAISE NOTICE 'inserting data from % to %', _epoch_no_to_insert_from, _curr_epoch;

activeandfees AS (
SELECT
act.pool_id,
act.epoch_no,
act.amount AS active_stake,
(
SELECT margin
FROM
pool_update
WHERE
id = (
SELECT MAX(pup2.id)
FROM pool_hash AS ph,
pool_update AS pup2
WHERE pup2.hash_id = ph.id
AND ph.view = act.pool_id
AND pup2.active_epoch_no <= act.epoch_no
)
) AS pool_fee_variable,
(
SELECT fixed_cost
FROM pool_update
WHERE id = (
SELECT MAX(pup2.id)
FROM pool_update AS pup2,
pool_hash AS ph
WHERE ph.view = act.pool_id
AND pup2.hash_id = ph.id
AND pup2.active_epoch_no <= act.epoch_no)
) AS pool_fee_fixed,
(act.amount / (
SELECT NULLIF(act.amount, 0)
FROM grest.epoch_active_stake_cache AS easc
WHERE easc.epoch_no = act.epoch_no
)
) * 100 AS active_stake_pct,
ROUND(
(act.amount / (
SELECT supply::bigint / (
SELECT ep.optimal_pool_count
FROM epoch_param AS ep
WHERE ep.epoch_no = act.epoch_no
)
FROM grest.totals (act.epoch_no)
) * 100
), 2
) AS saturation_pct
FROM grest.pool_active_stake_cache AS act
WHERE act.epoch_no >= _epoch_no_to_insert_from
AND act.epoch_no <= _curr_epoch
),

delegators AS (
SELECT
es.pool_id,
es.epoch_no,
COUNT(1) AS delegator_cnt
FROM epoch_stake AS es
WHERE es.epoch_no >= _epoch_no_to_insert_from
AND es.epoch_no <= _curr_epoch
GROUP BY
es.pool_id,
es.epoch_no
)

SELECT
ph.view AS pool_id,
actf.epoch_no,
actf.active_stake,
actf.active_stake_pct,
actf.saturation_pct,
COALESCE(b.block_cnt, 0) AS block_cnt,
del.delegator_cnt,
actf.pool_fee_variable,
actf.pool_fee_fixed,
-- for debugging: m.memtotal,
-- for debugging: l.leadertotal,
CASE COALESCE(b.block_cnt, 0)
WHEN 0 THEN
0
ELSE
-- special CASE for WHEN reward information is not available yet
CASE COALESCE(l.leadertotal, 0) + COALESCE(m.memtotal, 0)
WHEN 0 THEN NULL
ELSE
CASE
WHEN COALESCE(l.leadertotal, 0) < actf.pool_fee_fixed THEN COALESCE(l.leadertotal, 0)
ELSE ROUND(actf.pool_fee_fixed + (((COALESCE(m.memtotal, 0) + COALESCE(l.leadertotal, 0)) - actf.pool_fee_fixed) * actf.pool_fee_variable))
END
END
END AS pool_fees,
CASE COALESCE(b.block_cnt, 0)
WHEN 0 THEN
0
ELSE
-- special CASE for WHEN reward information is not available yet
CASE COALESCE(l.leadertotal, 0) + COALESCE(m.memtotal, 0)
WHEN 0 THEN NULL
ELSE
CASE
WHEN COALESCE(l.leadertotal, 0) < actf.pool_fee_fixed THEN COALESCE(m.memtotal, 0)
ELSE ROUND(COALESCE(m.memtotal, 0) + (COALESCE(l.leadertotal, 0) - (actf.pool_fee_fixed + (((COALESCE(m.memtotal, 0) + COALESCE(l.leadertotal, 0)) - actf.pool_fee_fixed) * actf.pool_fee_variable))))
END
END
END AS deleg_rewards,
CASE COALESCE(b.block_cnt, 0)
WHEN 0 THEN 0
ELSE
CASE COALESCE(m.memtotal, 0)
WHEN 0 THEN NULL
ELSE COALESCE(m.memtotal, 0)
END
END AS member_rewards,
CASE COALESCE(b.block_cnt, 0)
WHEN 0 THEN 0
ELSE
-- special CASE for WHEN reward information is not available yet
CASE COALESCE(l.leadertotal, 0) + COALESCE(m.memtotal, 0)
WHEN 0 THEN NULL
ELSE
CASE
WHEN COALESCE(l.leadertotal, 0) < actf.pool_fee_fixed THEN ROUND((((POW((LEAST(((COALESCE(m.memtotal, 0)) / (NULLIF(actf.active_stake, 0))), 1000) + 1), 73) - 1)) * 100)::numeric, 9)
-- using LEAST AS a way to prevent overflow, in CASE of dodgy database data (e.g. giant rewards / tiny active stake)
ELSE ROUND((((POW((LEAST((((COALESCE(m.memtotal, 0) + (COALESCE(l.leadertotal, 0) - (actf.pool_fee_fixed + (((COALESCE(m.memtotal, 0)
+ COALESCE(l.leadertotal, 0)) - actf.pool_fee_fixed) * actf.pool_fee_variable))))) / (NULLIF(actf.active_stake, 0))), 1000) + 1), 73) - 1)) * 100)::numeric, 9)
END
END
END AS epoch_ros
FROM pool_hash AS ph
INNER JOIN activeandfees AS actf ON actf.pool_id = ph.view
LEFT JOIN blockcounts AS b ON ph.id = b.pool_hash_id
AND actf.epoch_no = b.epoch_no
LEFT JOIN leadertotals AS l ON ph.id = l.pool_id
AND actf.epoch_no = l.earned_epoch
LEFT JOIN membertotals AS m ON ph.id = m.pool_id
AND actf.epoch_no = m.earned_epoch
LEFT JOIN delegators AS del ON ph.id = del.pool_id
AND actf.epoch_no = del.epoch_no
INSERT INTO grest.pool_history_cache (
select * from grest.get_pool_history_data_bulk(_epoch_no_to_insert_from::word31type, null::text [], _curr_epoch::word31type)
);

INSERT INTO grest.control_table (key, last_value)
Expand Down
1 change: 0 additions & 1 deletion files/grest/rpc/pool/pool_delegators.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -132,4 +132,3 @@ $$;


COMMENT ON FUNCTION grest.pool_delegators_list IS 'Return brief variant of information about live delegators for a given pool, needed by pool_info endpoint.'; --noqa: LT01

0 comments on commit a333019

Please sign in to comment.