From 60133f32b8eb8f3810f6a288930b16b52284fcba Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Wed, 15 Nov 2023 23:15:29 -0600 Subject: [PATCH 1/2] feat: chain tip table --- migrations/1700071472495_chain-tip-table.js | 142 ++++++++++++++++++++ src/datastore/pg-write-store.ts | 82 ++++++----- src/event-stream/event-server.ts | 2 +- 3 files changed, 181 insertions(+), 45 deletions(-) create mode 100644 migrations/1700071472495_chain-tip-table.js diff --git a/migrations/1700071472495_chain-tip-table.js b/migrations/1700071472495_chain-tip-table.js new file mode 100644 index 0000000000..dce518a530 --- /dev/null +++ b/migrations/1700071472495_chain-tip-table.js @@ -0,0 +1,142 @@ +/* eslint-disable camelcase */ + +exports.shorthands = undefined; + +exports.up = pgm => { + pgm.dropMaterializedView('chain_tip'); + pgm.createTable('chain_tip', { + id: { + type: 'bool', + primaryKey: true, + default: true, + }, + block_height: { + type: 'integer', + notNull: true, + }, + block_count: { + type: 'integer', + notNull: true, + }, + block_hash: { + type: 'bytea', + notNull: true, + }, + index_block_hash: { + type: 'bytea', + notNull: true, + }, + burn_block_height: { + type: 'integer', + notNull: true, + }, + microblock_hash: { + type: 'bytea', + }, + microblock_sequence: { + type: 'integer', + }, + microblock_count: { + type: 'integer', + notNull: true, + }, + tx_count: { + type: 'integer', + notNull: true, + }, + tx_count_unanchored: { + type: 'integer', + notNull: true, + }, + }); + pgm.addConstraint('chain_tip', 'chain_tip_one_row', 'CHECK(id)'); + pgm.sql(` + WITH block_tip AS ( + SELECT block_height, block_hash, index_block_hash, burn_block_height + FROM blocks + WHERE block_height = (SELECT MAX(block_height) FROM blocks WHERE canonical = TRUE) + ), + microblock_tip AS ( + SELECT microblock_hash, microblock_sequence + FROM microblocks, block_tip + WHERE microblocks.parent_index_block_hash = block_tip.index_block_hash + AND microblock_canonical = true AND canonical = true + ORDER BY microblock_sequence DESC + LIMIT 1 + ), + microblock_count AS ( + SELECT COUNT(*)::INTEGER AS microblock_count + FROM microblocks + WHERE canonical = TRUE AND microblock_canonical = TRUE + ), + tx_count AS ( + SELECT COUNT(*)::INTEGER AS tx_count + FROM txs + WHERE canonical = TRUE AND microblock_canonical = TRUE + AND block_height <= (SELECT MAX(block_height) FROM blocks WHERE canonical = TRUE) + ), + tx_count_unanchored AS ( + SELECT COUNT(*)::INTEGER AS tx_count_unanchored + FROM txs + WHERE canonical = TRUE AND microblock_canonical = TRUE + ) + INSERT INTO chain_tip (block_height, block_hash, index_block_hash, burn_block_height, + block_count, microblock_hash, microblock_sequence, microblock_count, tx_count, + tx_count_unanchored) + VALUES ( + COALESCE((SELECT block_height FROM block_tip), 0), + COALESCE((SELECT block_hash FROM block_tip), ''), + COALESCE((SELECT index_block_hash FROM block_tip), ''), + COALESCE((SELECT burn_block_height FROM block_tip), 0), + COALESCE((SELECT block_height FROM block_tip), 0), + (SELECT microblock_hash FROM microblock_tip), + (SELECT microblock_sequence FROM microblock_tip), + COALESCE((SELECT microblock_count FROM microblock_count), 0), + COALESCE((SELECT tx_count FROM tx_count), 0), + COALESCE((SELECT tx_count_unanchored FROM tx_count_unanchored), 0) + ) + `); +}; + +exports.down = pgm => { + pgm.dropTable('chain_tip'); + pgm.createMaterializedView('chain_tip', {}, ` + WITH block_tip AS ( + SELECT block_height, block_hash, index_block_hash, burn_block_height + FROM blocks + WHERE block_height = (SELECT MAX(block_height) FROM blocks WHERE canonical = TRUE) + ), + microblock_tip AS ( + SELECT microblock_hash, microblock_sequence + FROM microblocks, block_tip + WHERE microblocks.parent_index_block_hash = block_tip.index_block_hash + AND microblock_canonical = true AND canonical = true + ORDER BY microblock_sequence DESC + LIMIT 1 + ), + microblock_count AS ( + SELECT COUNT(*)::INTEGER AS microblock_count + FROM microblocks + WHERE canonical = TRUE AND microblock_canonical = TRUE + ), + tx_count AS ( + SELECT COUNT(*)::INTEGER AS tx_count + FROM txs + WHERE canonical = TRUE AND microblock_canonical = TRUE + AND block_height <= (SELECT MAX(block_height) FROM blocks WHERE canonical = TRUE) + ), + tx_count_unanchored AS ( + SELECT COUNT(*)::INTEGER AS tx_count_unanchored + FROM txs + WHERE canonical = TRUE AND microblock_canonical = TRUE + ) + SELECT *, block_tip.block_height AS block_count + FROM block_tip + LEFT JOIN microblock_tip ON TRUE + LEFT JOIN microblock_count ON TRUE + LEFT JOIN tx_count ON TRUE + LEFT JOIN tx_count_unanchored ON TRUE + LIMIT 1 + `); + pgm.createIndex('chain_tip', 'block_height', { unique: true }); +}; diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index 6cd51a0ce3..5f95acabfb 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -141,33 +141,6 @@ export class PgWriteStore extends PgStore { return store; } - async getChainTip(sql: PgSqlClient, useMaterializedView = true): Promise { - if (!this.isEventReplay && useMaterializedView) { - return super.getChainTip(sql); - } - // The `chain_tip` materialized view is not available during event replay. - // Since `getChainTip()` is used heavily during event ingestion, we'll fall back to - // a classic query. - const currentTipBlock = await sql< - { - block_height: number; - block_hash: string; - index_block_hash: string; - burn_block_height: number; - }[] - >` - SELECT block_height, block_hash, index_block_hash, burn_block_height - FROM blocks - WHERE canonical = true AND block_height = (SELECT MAX(block_height) FROM blocks) - `; - return { - blockHeight: currentTipBlock[0]?.block_height ?? 0, - blockHash: currentTipBlock[0]?.block_hash ?? '', - indexBlockHash: currentTipBlock[0]?.index_block_hash ?? '', - burnBlockHeight: currentTipBlock[0]?.burn_block_height ?? 0, - }; - } - async storeRawEventRequest(eventPath: string, payload: PgJsonb): Promise { // To avoid depending on the DB more than once and to allow the query transaction to settle, // we'll take the complete insert result and move that to the output TSV file instead of taking @@ -198,7 +171,7 @@ export class PgWriteStore extends PgStore { const contractLogEvents: DbSmartContractEvent[] = []; await this.sqlWriteTransaction(async sql => { - const chainTip = await this.getChainTip(sql, false); + const chainTip = await this.getChainTip(sql); await this.handleReorg(sql, data.block, chainTip.blockHeight); // If the incoming block is not of greater height than current chain tip, then store data as non-canonical. const isCanonical = data.block.block_height > chainTip.blockHeight; @@ -417,12 +390,27 @@ export class PgWriteStore extends PgStore { const mempoolStats = await this.getMempoolStatsInternal({ sql }); this.eventEmitter.emit('mempoolStatsUpdate', mempoolStats); } + if (isCanonical) + await sql` + WITH new_tx_count AS ( + SELECT tx_count + ${data.txs.length} AS tx_count FROM chain_tip + ) + UPDATE chain_tip SET + block_height = ${data.block.block_height}, + block_hash = ${data.block.block_hash}, + index_block_hash = ${data.block.index_block_hash}, + burn_block_height = ${data.block.burn_block_height}, + microblock_hash = NULL, + microblock_sequence = NULL, + block_count = ${data.block.block_height}, + tx_count = (SELECT tx_count FROM new_tx_count), + tx_count_unanchored = (SELECT tx_count FROM new_tx_count) + `; }); // Do we have an IBD height defined in ENV? If so, check if this block update reached it. const ibdHeight = getIbdBlockHeight(); this.isIbdBlockHeightReached = ibdHeight ? data.block.block_height > ibdHeight : true; - await this.refreshMaterializedView('chain_tip'); await this.refreshMaterializedView('mempool_digest'); // Skip sending `PgNotifier` updates altogether if we're in the genesis block since this block is the @@ -597,10 +585,10 @@ export class PgWriteStore extends PgStore { const contractLogEvents: DbSmartContractEvent[] = []; await this.sqlWriteTransaction(async sql => { - // Sanity check: ensure incoming microblocks have a `parent_index_block_hash` that matches the API's - // current known canonical chain tip. We assume this holds true so incoming microblock data is always - // treated as being built off the current canonical anchor block. - const chainTip = await this.getChainTip(sql, false); + // Sanity check: ensure incoming microblocks have a `parent_index_block_hash` that matches the + // API's current known canonical chain tip. We assume this holds true so incoming microblock + // data is always treated as being built off the current canonical anchor block. + const chainTip = await this.getChainTip(sql); const nonCanonicalMicroblock = data.microblocks.find( mb => mb.parent_index_block_hash !== chainTip.indexBlockHash ); @@ -722,9 +710,20 @@ export class PgWriteStore extends PgStore { const mempoolStats = await this.getMempoolStatsInternal({ sql }); this.eventEmitter.emit('mempoolStatsUpdate', mempoolStats); } + if (currentMicroblockTip.microblock_canonical) + await sql` + UPDATE chain_tip SET + microblock_hash = ${currentMicroblockTip.microblock_hash}, + microblock_sequence = ${currentMicroblockTip.microblock_sequence}, + microblock_count = microblock_count + ${data.microblocks.length}, + tx_count_unanchored = ${ + currentMicroblockTip.microblock_sequence === 0 + ? sql`tx_count + ${data.txs.length}` + : sql`tx_count_unanchored + ${data.txs.length}` + } + `; }); - await this.refreshMaterializedView('chain_tip'); await this.refreshMaterializedView('mempool_digest'); if (this.notifier) { @@ -1660,7 +1659,7 @@ export class PgWriteStore extends PgStore { } } - async updateTx(sql: PgSqlClient, tx: DbTxRaw): Promise { + async updateTx(sql: PgSqlClient, tx: DbTxRaw, microblock: boolean = false): Promise { const values: TxInsertValues = { tx_id: tx.tx_id, raw_tx: tx.raw_tx, @@ -1767,7 +1766,7 @@ export class PgWriteStore extends PgStore { async updateMempoolTxs({ mempoolTxs: txs }: { mempoolTxs: DbMempoolTxRaw[] }): Promise { const updatedTxIds: string[] = []; await this.sqlWriteTransaction(async sql => { - const chainTip = await this.getChainTip(sql, false); + const chainTip = await this.getChainTip(sql); for (const tx of txs) { const inserted = await this.insertDbMempoolTx(tx, chainTip, sql); if (inserted) { @@ -2134,7 +2133,7 @@ export class PgWriteStore extends PgStore { } for (const entry of txs) { - const rowsUpdated = await this.updateTx(sql, entry.tx); + const rowsUpdated = await this.updateTx(sql, entry.tx, true); if (rowsUpdated !== 1) { throw new Error( `Unexpected amount of rows updated for microblock tx insert: ${rowsUpdated}` @@ -2944,13 +2943,8 @@ export class PgWriteStore extends PgStore { * Called when a full event import is complete. */ async finishEventReplay() { - if (!this.isEventReplay) { - return; - } - await this.sqlWriteTransaction(async sql => { - await this.refreshMaterializedView('chain_tip', sql, false); - await this.refreshMaterializedView('mempool_digest', sql, false); - }); + if (!this.isEventReplay) return; + await this.refreshMaterializedView('mempool_digest', this.sql, false); } /** diff --git a/src/event-stream/event-server.ts b/src/event-stream/event-server.ts index c767a44163..09562c2501 100644 --- a/src/event-stream/event-server.ts +++ b/src/event-stream/event-server.ts @@ -845,7 +845,7 @@ export async function startEventServer(opts: { if (ibdHeight) { app.use(IBD_PRUNABLE_ROUTES, async (req, res, next) => { try { - const chainTip = await db.getChainTip(db.sql, false); + const chainTip = await db.getChainTip(db.sql); if (chainTip.blockHeight > ibdHeight) { next(); } else { From 87f0ab888bd905467fe6083e945196f9279741d5 Mon Sep 17 00:00:00 2001 From: Rafael Cardenas Date: Thu, 16 Nov 2023 11:18:12 -0600 Subject: [PATCH 2/2] fix: handle reorgs --- src/api/controllers/cache-controller.ts | 6 +- src/api/routes/status.ts | 16 ++-- src/datastore/common.ts | 23 +++--- src/datastore/pg-store.ts | 73 +++++-------------- src/datastore/pg-write-store.ts | 42 +++++++---- src/event-stream/event-server.ts | 4 +- src/tests-event-replay/import-export-tests.ts | 38 +++------- .../poison-microblock-tests.ts | 9 +-- src/tests/cache-control-tests.ts | 13 ++-- src/tests/datastore-tests.ts | 51 +++++++++---- src/tests/mempool-tests.ts | 2 +- src/tests/microblock-tests.ts | 26 +++---- 12 files changed, 140 insertions(+), 163 deletions(-) diff --git a/src/api/controllers/cache-controller.ts b/src/api/controllers/cache-controller.ts index 6d2f84fc77..470aeceba7 100644 --- a/src/api/controllers/cache-controller.ts +++ b/src/api/controllers/cache-controller.ts @@ -252,13 +252,13 @@ async function calculateETag( switch (etagType) { case ETagType.chainTip: try { - const chainTip = await db.getUnanchoredChainTip(); - if (!chainTip.found) { + const chainTip = await db.getChainTip(); + if (chainTip.block_height === 0) { // This should never happen unless the API is serving requests before it has synced any // blocks. return; } - return chainTip.result.microblockHash ?? chainTip.result.indexBlockHash; + return chainTip.microblock_hash ?? chainTip.index_block_hash; } catch (error) { logger.error(error, 'Unable to calculate chain_tip ETag'); return; diff --git a/src/api/routes/status.ts b/src/api/routes/status.ts index eb2cadfccc..328cb327ed 100644 --- a/src/api/routes/status.ts +++ b/src/api/routes/status.ts @@ -18,15 +18,15 @@ export function createStatusRouter(db: PgStore): express.Router { response.pox_v1_unlock_height = poxForceUnlockHeights.result.pox1UnlockHeight as number; response.pox_v2_unlock_height = poxForceUnlockHeights.result.pox2UnlockHeight as number; } - const chainTip = await db.getUnanchoredChainTip(); - if (chainTip.found) { + const chainTip = await db.getChainTip(); + if (chainTip.block_height > 0) { response.chain_tip = { - block_height: chainTip.result.blockHeight, - block_hash: chainTip.result.blockHash, - index_block_hash: chainTip.result.indexBlockHash, - microblock_hash: chainTip.result.microblockHash, - microblock_sequence: chainTip.result.microblockSequence, - burn_block_height: chainTip.result.burnBlockHeight, + block_height: chainTip.block_height, + block_hash: chainTip.block_hash, + index_block_hash: chainTip.index_block_hash, + microblock_hash: chainTip.microblock_hash, + microblock_sequence: chainTip.microblock_sequence, + burn_block_height: chainTip.burn_block_height, }; } setETagCacheHeaders(res); diff --git a/src/datastore/common.ts b/src/datastore/common.ts index be4fd1ee98..5ff49a4ba3 100644 --- a/src/datastore/common.ts +++ b/src/datastore/common.ts @@ -744,15 +744,6 @@ export type BlockIdentifier = | { burnBlockHash: string } | { burnBlockHeight: number }; -export interface DbChainTip { - blockHeight: number; - indexBlockHash: string; - blockHash: string; - microblockHash?: string; - microblockSequence?: number; - burnBlockHeight: number; -} - export interface BlockQueryResult { block_hash: string; index_block_hash: string; @@ -1461,10 +1452,16 @@ export interface SmartContractInsertValues { } export interface DbChainTip { - blockHeight: number; - blockHash: string; - indexBlockHash: string; - burnBlockHeight: number; + block_height: number; + block_count: number; + block_hash: string; + index_block_hash: string; + burn_block_height: number; + microblock_hash?: string; + microblock_sequence?: number; + microblock_count: number; + tx_count: number; + tx_count_unanchored: number; } export enum IndexesState { diff --git a/src/datastore/pg-store.ts b/src/datastore/pg-store.ts index 1e2e6327ca..f5f888c640 100644 --- a/src/datastore/pg-store.ts +++ b/src/datastore/pg-store.ts @@ -202,26 +202,20 @@ export class PgStore extends BasePgStore { }); } - async getChainTip(sql: PgSqlClient): Promise<{ - blockHeight: number; - blockHash: string; - indexBlockHash: string; - burnBlockHeight: number; - }> { - const currentTipBlock = await sql< - { - block_height: number; - block_hash: string; - index_block_hash: string; - burn_block_height: number; - }[] - >`SELECT block_height, block_hash, index_block_hash, burn_block_height FROM chain_tip`; - const height = currentTipBlock[0]?.block_height ?? 0; + async getChainTip(): Promise { + const tipResult = await this.sql`SELECT * FROM chain_tip`; + const tip = tipResult[0]; return { - blockHeight: height, - blockHash: currentTipBlock[0]?.block_hash ?? '', - indexBlockHash: currentTipBlock[0]?.index_block_hash ?? '', - burnBlockHeight: currentTipBlock[0]?.burn_block_height ?? 0, + block_height: tip?.block_height ?? 0, + block_count: tip?.block_count ?? 0, + block_hash: tip?.block_hash ?? '', + index_block_hash: tip?.index_block_hash ?? '', + burn_block_height: tip?.burn_block_height ?? 0, + microblock_hash: tip?.microblock_hash ?? undefined, + microblock_sequence: tip?.microblock_sequence ?? undefined, + microblock_count: tip?.microblock_count ?? 0, + tx_count: tip?.tx_count ?? 0, + tx_count_unanchored: tip?.tx_count_unanchored ?? 0, }; } @@ -316,33 +310,6 @@ export class PgStore extends BasePgStore { return this.getPoxForcedUnlockHeightsInternal(this.sql); } - async getUnanchoredChainTip(): Promise> { - const result = await this.sql< - { - block_height: number; - index_block_hash: string; - block_hash: string; - microblock_hash: string | null; - microblock_sequence: number | null; - burn_block_height: number; - }[] - >`SELECT block_height, index_block_hash, block_hash, microblock_hash, microblock_sequence, burn_block_height - FROM chain_tip`; - if (result.length === 0) { - return { found: false } as const; - } - const row = result[0]; - const chainTipResult: DbChainTip = { - blockHeight: row.block_height, - indexBlockHash: row.index_block_hash, - blockHash: row.block_hash, - microblockHash: row.microblock_hash === null ? undefined : row.microblock_hash, - microblockSequence: row.microblock_sequence === null ? undefined : row.microblock_sequence, - burnBlockHeight: row.burn_block_height, - }; - return { found: true, result: chainTipResult }; - } - async getBlock(blockIdentifer: BlockIdentifier): Promise> { return this.getBlockInternal(this.sql, blockIdentifer); } @@ -626,8 +593,8 @@ export class PgStore extends BasePgStore { async getUnanchoredTxsInternal(sql: PgSqlClient): Promise<{ txs: DbTx[] }> { // Get transactions that have been streamed in microblocks but not yet accepted or rejected in an anchor block. - const { blockHeight } = await this.getChainTip(sql); - const unanchoredBlockHeight = blockHeight + 1; + const { block_height } = await this.getChainTip(); + const unanchoredBlockHeight = block_height + 1; const query = await sql` SELECT ${unsafeCols(sql, [...TX_COLUMNS, abiColumn()])} FROM txs @@ -1372,11 +1339,11 @@ export class PgStore extends BasePgStore { sql: PgSqlClient, { includeUnanchored }: { includeUnanchored: boolean } ): Promise { - const chainTip = await this.getChainTip(sql); + const chainTip = await this.getChainTip(); if (includeUnanchored) { - return chainTip.blockHeight + 1; + return chainTip.block_height + 1; } else { - return chainTip.blockHeight; + return chainTip.block_height; } } @@ -2159,9 +2126,9 @@ export class PgStore extends BasePgStore { async getStxBalanceAtBlock(stxAddress: string, blockHeight: number): Promise { return await this.sqlTransaction(async sql => { - const chainTip = await this.getChainTip(sql); + const chainTip = await this.getChainTip(); const blockHeightToQuery = - blockHeight > chainTip.blockHeight ? chainTip.blockHeight : blockHeight; + blockHeight > chainTip.block_height ? chainTip.block_height : blockHeight; const blockQuery = await this.getBlockByHeightInternal(sql, blockHeightToQuery); if (!blockQuery.found) { throw new Error(`Could not find block at height: ${blockHeight}`); diff --git a/src/datastore/pg-write-store.ts b/src/datastore/pg-write-store.ts index 5f95acabfb..891e41d9aa 100644 --- a/src/datastore/pg-write-store.ts +++ b/src/datastore/pg-write-store.ts @@ -171,10 +171,10 @@ export class PgWriteStore extends PgStore { const contractLogEvents: DbSmartContractEvent[] = []; await this.sqlWriteTransaction(async sql => { - const chainTip = await this.getChainTip(sql); - await this.handleReorg(sql, data.block, chainTip.blockHeight); + const chainTip = await this.getChainTip(); + await this.handleReorg(sql, data.block, chainTip.block_height); // If the incoming block is not of greater height than current chain tip, then store data as non-canonical. - const isCanonical = data.block.block_height > chainTip.blockHeight; + const isCanonical = data.block.block_height > chainTip.block_height; if (!isCanonical) { data.block = { ...data.block, canonical: false }; data.microblocks = data.microblocks.map(mb => ({ ...mb, canonical: false })); @@ -588,9 +588,9 @@ export class PgWriteStore extends PgStore { // Sanity check: ensure incoming microblocks have a `parent_index_block_hash` that matches the // API's current known canonical chain tip. We assume this holds true so incoming microblock // data is always treated as being built off the current canonical anchor block. - const chainTip = await this.getChainTip(sql); + const chainTip = await this.getChainTip(); const nonCanonicalMicroblock = data.microblocks.find( - mb => mb.parent_index_block_hash !== chainTip.indexBlockHash + mb => mb.parent_index_block_hash !== chainTip.index_block_hash ); // Note: the stacks-node event emitter can send old microblocks that have already been processed by a previous anchor block. // Log warning and return, nothing to do. @@ -598,13 +598,13 @@ export class PgWriteStore extends PgStore { logger.info( `Failure in microblock ingestion, microblock ${nonCanonicalMicroblock.microblock_hash} ` + `points to parent index block hash ${nonCanonicalMicroblock.parent_index_block_hash} rather ` + - `than the current canonical tip's index block hash ${chainTip.indexBlockHash}.` + `than the current canonical tip's index block hash ${chainTip.index_block_hash}.` ); return; } // The block height is just one after the current chain tip height - const blockHeight = chainTip.blockHeight + 1; + const blockHeight = chainTip.block_height + 1; dbMicroblocks = data.microblocks.map(mb => { const dbMicroBlock: DbMicroblock = { canonical: true, @@ -617,8 +617,8 @@ export class PgWriteStore extends PgStore { parent_burn_block_hash: mb.parent_burn_block_hash, parent_burn_block_time: mb.parent_burn_block_time, block_height: blockHeight, - parent_block_height: chainTip.blockHeight, - parent_block_hash: chainTip.blockHash, + parent_block_height: chainTip.block_height, + parent_block_hash: chainTip.block_hash, index_block_hash: '', // Empty until microblock is confirmed in an anchor block block_hash: '', // Empty until microblock is confirmed in an anchor block }; @@ -630,7 +630,7 @@ export class PgWriteStore extends PgStore { // block with that data doesn't yet exist. const dbTx: DbTxRaw = { ...entry.tx, - parent_block_hash: chainTip.blockHash, + parent_block_hash: chainTip.block_hash, block_height: blockHeight, }; @@ -1659,7 +1659,7 @@ export class PgWriteStore extends PgStore { } } - async updateTx(sql: PgSqlClient, tx: DbTxRaw, microblock: boolean = false): Promise { + async updateTx(sql: PgSqlClient, tx: DbTxRaw): Promise { const values: TxInsertValues = { tx_id: tx.tx_id, raw_tx: tx.raw_tx, @@ -1727,7 +1727,7 @@ export class PgWriteStore extends PgStore { anchor_mode: tx.anchor_mode, status: tx.status, receipt_time: tx.receipt_time, - receipt_block_height: chainTip.blockHeight, + receipt_block_height: chainTip.block_height, post_conditions: tx.post_conditions, nonce: tx.nonce, fee_rate: tx.fee_rate, @@ -1766,7 +1766,7 @@ export class PgWriteStore extends PgStore { async updateMempoolTxs({ mempoolTxs: txs }: { mempoolTxs: DbMempoolTxRaw[] }): Promise { const updatedTxIds: string[] = []; await this.sqlWriteTransaction(async sql => { - const chainTip = await this.getChainTip(sql); + const chainTip = await this.getChainTip(); for (const tx of txs) { const inserted = await this.insertDbMempoolTx(tx, chainTip, sql); if (inserted) { @@ -2133,7 +2133,7 @@ export class PgWriteStore extends PgStore { } for (const entry of txs) { - const rowsUpdated = await this.updateTx(sql, entry.tx, true); + const rowsUpdated = await this.updateTx(sql, entry.tx); if (rowsUpdated !== 1) { throw new Error( `Unexpected amount of rows updated for microblock tx insert: ${rowsUpdated}` @@ -2244,6 +2244,12 @@ export class PgWriteStore extends PgStore { }); } + // Update unanchored tx count in `chain_tip` table + const txCountDelta = updatedMbTxs.length * (args.isMicroCanonical ? 1 : -1); + await sql` + UPDATE chain_tip SET tx_count_unanchored = tx_count_unanchored + ${txCountDelta} + `; + return { updatedTxs: updatedMbTxs }; } @@ -2859,6 +2865,14 @@ export class PgWriteStore extends PgStore { await this.restoreOrphanedChain(sql, parentResult[0].index_block_hash, updatedEntities); this.logReorgResultInfo(updatedEntities); } + // Reflect updated transaction totals in `chain_tip` table. + const txCountDelta = + updatedEntities.markedCanonical.txs - updatedEntities.markedNonCanonical.txs; + await sql` + UPDATE chain_tip SET + tx_count = tx_count + ${txCountDelta}, + tx_count_unanchored = tx_count_unanchored + ${txCountDelta} + `; } return updatedEntities; } diff --git a/src/event-stream/event-server.ts b/src/event-stream/event-server.ts index 09562c2501..c0ec3c86c0 100644 --- a/src/event-stream/event-server.ts +++ b/src/event-stream/event-server.ts @@ -845,8 +845,8 @@ export async function startEventServer(opts: { if (ibdHeight) { app.use(IBD_PRUNABLE_ROUTES, async (req, res, next) => { try { - const chainTip = await db.getChainTip(db.sql); - if (chainTip.blockHeight > ibdHeight) { + const chainTip = await db.getChainTip(); + if (chainTip.block_height > ibdHeight) { next(); } else { handleRawEventRequest(req, res, next); diff --git a/src/tests-event-replay/import-export-tests.ts b/src/tests-event-replay/import-export-tests.ts index a5d52b7167..ed2fe9d03f 100644 --- a/src/tests-event-replay/import-export-tests.ts +++ b/src/tests-event-replay/import-export-tests.ts @@ -28,13 +28,12 @@ describe('import/export tests', () => { test('event import and export cycle', async () => { // Import from mocknet TSV await importEventsFromTsv('src/tests-event-replay/tsv/mocknet.tsv', 'archival', true, true); - const chainTip = await db.getUnanchoredChainTip(); - expect(chainTip.found).toBe(true); - expect(chainTip.result?.blockHeight).toBe(28); - expect(chainTip.result?.indexBlockHash).toBe( + const chainTip = await db.getChainTip(); + expect(chainTip.block_height).toBe(28); + expect(chainTip.index_block_hash).toBe( '0x76cd67a65c0dfd5ea450bb9efe30da89fa125bfc077c953802f718353283a533' ); - expect(chainTip.result?.blockHash).toBe( + expect(chainTip.block_hash).toBe( '0x7682af212d3c1ef62613412f9b5a727269b4548f14eca2e3f941f7ad8b3c11b2' ); @@ -51,13 +50,12 @@ describe('import/export tests', () => { // Re-import with exported TSV and check that chain tip matches. try { await importEventsFromTsv(`${tmpDir}/export.tsv`, 'archival', true, true); - const newChainTip = await db.getUnanchoredChainTip(); - expect(newChainTip.found).toBe(true); - expect(newChainTip.result?.blockHeight).toBe(28); - expect(newChainTip.result?.indexBlockHash).toBe( + const newChainTip = await db.getChainTip(); + expect(newChainTip.block_height).toBe(28); + expect(newChainTip.index_block_hash).toBe( '0x76cd67a65c0dfd5ea450bb9efe30da89fa125bfc077c953802f718353283a533' ); - expect(newChainTip.result?.blockHash).toBe( + expect(newChainTip.block_hash).toBe( '0x7682af212d3c1ef62613412f9b5a727269b4548f14eca2e3f941f7ad8b3c11b2' ); } finally { @@ -198,30 +196,14 @@ describe('IBD', () => { process.env.IBD_MODE_UNTIL_BLOCK = '1000'; // TSV has 1 microblock message. await expect(getIbdInterceptCountFromTsvEvents()).resolves.toBe(1); - await expect(db.getChainTip(client, false)).resolves.toHaveProperty('blockHeight', 28); + await expect(db.getChainTip()).resolves.toHaveProperty('block_height', 28); }); test('IBD mode does NOT block certain API routes once the threshold number of blocks are ingested', async () => { process.env.IBD_MODE_UNTIL_BLOCK = '1'; // Microblock processed normally. await expect(getIbdInterceptCountFromTsvEvents()).resolves.toBe(0); - await expect(db.getChainTip(client, false)).resolves.toHaveProperty('blockHeight', 28); - }); - - test('IBD mode prevents refreshing materialized views', async () => { - process.env.IBD_MODE_UNTIL_BLOCK = '1000'; - await getIbdInterceptCountFromTsvEvents(); - await db.refreshMaterializedView('chain_tip', client); - const res = await db.sql<{ block_height: number }[]>`SELECT * FROM chain_tip`; - expect(res.count).toBe(0); - }); - - test('IBD mode allows refreshing materialized views after height has passed', async () => { - process.env.IBD_MODE_UNTIL_BLOCK = '10'; - await getIbdInterceptCountFromTsvEvents(); - await db.refreshMaterializedView('chain_tip', client); - const res = await db.sql<{ block_height: number }[]>`SELECT * FROM chain_tip`; - expect(res[0].block_height).toBe(28); + await expect(db.getChainTip()).resolves.toHaveProperty('block_height', 28); }); test('IBD mode covers prune mode', async () => { diff --git a/src/tests-event-replay/poison-microblock-tests.ts b/src/tests-event-replay/poison-microblock-tests.ts index dd6d0ef741..5bd511ae56 100644 --- a/src/tests-event-replay/poison-microblock-tests.ts +++ b/src/tests-event-replay/poison-microblock-tests.ts @@ -25,22 +25,21 @@ describe('poison microblock for height 80743', () => { true ); const poisonTxId = '0x58ffe62029f94f7101b959536ea4953b9bce0ec3f6e2a06254c511bdd5cfa9e7'; - const chainTip = await db.getUnanchoredChainTip(); + const chainTip = await db.getChainTip(); // query the txs table and check the transaction type const searchResult = await db.searchHash({ hash: poisonTxId }); let entityData: any; if (searchResult.result?.entity_data) { entityData = searchResult.result?.entity_data; } - expect(chainTip.found).toBe(true); // check the transaction type to be contract call for this poison block expect(entityData.type_id).toBe(DbTxTypeId.ContractCall); expect(searchResult.found).toBe(true); - expect(chainTip.result?.blockHeight).toBe(1); - expect(chainTip.result?.indexBlockHash).toBe( + expect(chainTip.block_height).toBe(1); + expect(chainTip.index_block_hash).toBe( '0x05ca75b9949195da435e6e36d731dbaa10bb75fda576a52263e25164990bfdaa' ); - expect(chainTip.result?.blockHash).toBe( + expect(chainTip.block_hash).toBe( '0x6b83b44571365e6e530d679536578c71d6c376b07666f3671786b6fd8fac049c' ); }); diff --git a/src/tests/cache-control-tests.ts b/src/tests/cache-control-tests.ts index 482884b1ed..da9e0e826b 100644 --- a/src/tests/cache-control-tests.ts +++ b/src/tests/cache-control-tests.ts @@ -318,13 +318,12 @@ describe('cache-control tests', () => { ], }); - const chainTip2 = await db.getUnanchoredChainTip(); - expect(chainTip2.found).toBeTruthy(); - expect(chainTip2.result?.blockHash).toBe(block1.block_hash); - expect(chainTip2.result?.blockHeight).toBe(block1.block_height); - expect(chainTip2.result?.indexBlockHash).toBe(block1.index_block_hash); - expect(chainTip2.result?.microblockHash).toBe(mb1.microblock_hash); - expect(chainTip2.result?.microblockSequence).toBe(mb1.microblock_sequence); + const chainTip2 = await db.getChainTip(); + expect(chainTip2.block_hash).toBe(block1.block_hash); + expect(chainTip2.block_height).toBe(block1.block_height); + expect(chainTip2.index_block_hash).toBe(block1.index_block_hash); + expect(chainTip2.microblock_hash).toBe(mb1.microblock_hash); + expect(chainTip2.microblock_sequence).toBe(mb1.microblock_sequence); const expectedResp2 = { burn_block_time: 1594647996, diff --git a/src/tests/datastore-tests.ts b/src/tests/datastore-tests.ts index 197ef2f1ce..6889b0a25c 100644 --- a/src/tests/datastore-tests.ts +++ b/src/tests/datastore-tests.ts @@ -3733,6 +3733,7 @@ describe('postgres datastore', () => { contract_name: 'pox', }; + // Start canonical chain await db.update({ block: block1, microblocks: [], @@ -3953,6 +3954,7 @@ describe('postgres datastore', () => { abi: '{"thing":1}', }; + // Insert non-canonical block await db.update({ block: block2b, microblocks: [], @@ -4050,12 +4052,18 @@ describe('postgres datastore', () => { const blockQuery1 = await db.getBlock({ hash: block2b.block_hash }); expect(blockQuery1.result?.canonical).toBe(false); - const chainTip1 = await db.getChainTip(client); + const chainTip1 = await db.getChainTip(); expect(chainTip1).toEqual({ - blockHash: '0x33', - blockHeight: 3, - indexBlockHash: '0xcc', - burnBlockHeight: 123, + block_hash: '0x33', + block_height: 3, + index_block_hash: '0xcc', + burn_block_height: 123, + block_count: 3, + microblock_count: 0, + microblock_hash: undefined, + microblock_sequence: undefined, + tx_count: 2, // Tx from block 2b does not count + tx_count_unanchored: 2, }); const namespaces = await db.getNamespaceList({ includeUnanchored: false }); expect(namespaces.results.length).toBe(1); @@ -4109,12 +4117,19 @@ describe('postgres datastore', () => { await db.update({ block: block3b, microblocks: [], minerRewards: [], txs: [] }); const blockQuery2 = await db.getBlock({ hash: block3b.block_hash }); expect(blockQuery2.result?.canonical).toBe(false); - const chainTip2 = await db.getChainTip(client); + // Chain tip doesn't change yet. + const chainTip2 = await db.getChainTip(); expect(chainTip2).toEqual({ - blockHash: '0x33', - blockHeight: 3, - indexBlockHash: '0xcc', - burnBlockHeight: 123, + block_hash: '0x33', + block_height: 3, + index_block_hash: '0xcc', + burn_block_height: 123, + block_count: 3, + microblock_count: 0, + microblock_hash: undefined, + microblock_sequence: undefined, + tx_count: 2, + tx_count_unanchored: 2, }); const block4b: DbBlock = { @@ -4152,12 +4167,18 @@ describe('postgres datastore', () => { const blockQuery3 = await db.getBlock({ hash: block3b.block_hash }); expect(blockQuery3.result?.canonical).toBe(true); - const chainTip3 = await db.getChainTip(client); + const chainTip3 = await db.getChainTip(); expect(chainTip3).toEqual({ - blockHash: '0x44bb', - blockHeight: 4, - indexBlockHash: '0xddbb', - burnBlockHeight: 123, + block_count: 4, + block_hash: '0x44bb', + block_height: 4, + burn_block_height: 123, + index_block_hash: '0xddbb', + microblock_count: 0, + microblock_hash: undefined, + microblock_sequence: undefined, + tx_count: 2, // Tx from block 2b now counts, but compensates with tx from block 2 + tx_count_unanchored: 2, }); const b1 = await db.getBlock({ hash: block1.block_hash }); diff --git a/src/tests/mempool-tests.ts b/src/tests/mempool-tests.ts index d8d135d692..961d4d0756 100644 --- a/src/tests/mempool-tests.ts +++ b/src/tests/mempool-tests.ts @@ -1539,7 +1539,7 @@ describe('mempool tests', () => { // Simulate the bug with a txs being in the mempool at confirmed at the same time by // directly inserting the mempool-tx and mined-tx, bypassing the normal update functions. await db.updateBlock(db.sql, dbBlock1); - const chainTip = await db.getChainTip(db.sql); + const chainTip = await db.getChainTip(); await db.insertDbMempoolTx(mempoolTx, chainTip, db.sql); await db.updateTx(db.sql, dbTx1); diff --git a/src/tests/microblock-tests.ts b/src/tests/microblock-tests.ts index a6918d99ed..06b27304aa 100644 --- a/src/tests/microblock-tests.ts +++ b/src/tests/microblock-tests.ts @@ -384,13 +384,12 @@ describe('microblock tests', () => { ], }); - const chainTip1 = await db.getUnanchoredChainTip(); - expect(chainTip1.found).toBeTruthy(); - expect(chainTip1.result?.blockHash).toBe(block1.block_hash); - expect(chainTip1.result?.blockHeight).toBe(block1.block_height); - expect(chainTip1.result?.indexBlockHash).toBe(block1.index_block_hash); - expect(chainTip1.result?.microblockHash).toBeUndefined(); - expect(chainTip1.result?.microblockSequence).toBeUndefined(); + const chainTip1 = await db.getChainTip(); + expect(chainTip1.block_hash).toBe(block1.block_hash); + expect(chainTip1.block_height).toBe(block1.block_height); + expect(chainTip1.index_block_hash).toBe(block1.index_block_hash); + expect(chainTip1.microblock_hash).toBeUndefined(); + expect(chainTip1.microblock_sequence).toBeUndefined(); const mb1: DbMicroblockPartial = { microblock_hash: '0xff01', @@ -546,13 +545,12 @@ describe('microblock tests', () => { ], }); - const chainTip2 = await db.getUnanchoredChainTip(); - expect(chainTip2.found).toBeTruthy(); - expect(chainTip2.result?.blockHash).toBe(block1.block_hash); - expect(chainTip2.result?.blockHeight).toBe(block1.block_height); - expect(chainTip2.result?.indexBlockHash).toBe(block1.index_block_hash); - expect(chainTip2.result?.microblockHash).toBe(mb1.microblock_hash); - expect(chainTip2.result?.microblockSequence).toBe(mb1.microblock_sequence); + const chainTip2 = await db.getChainTip(); + expect(chainTip2.block_hash).toBe(block1.block_hash); + expect(chainTip2.block_height).toBe(block1.block_height); + expect(chainTip2.index_block_hash).toBe(block1.index_block_hash); + expect(chainTip2.microblock_hash).toBe(mb1.microblock_hash); + expect(chainTip2.microblock_sequence).toBe(mb1.microblock_sequence); const txListResult1 = await supertest(api.server).get(`/extended/v1/tx`); const { body: txListBody1 }: { body: TransactionResults } = txListResult1;