feat: nakamoto block timestamps (#1886)

* feat: nakamoto block timestamps

* fix: `CONNECTION_DESTROYED` during background mempool stats update

* fix: proper shutdown of debounce background query in pg-write-store

* chore: fix tests

* chore: fix tests

* fix: use `isEventReplay` to determine block time

* chore: rename stacks_block_time to block_time

docs/entities/blocks/block.schema.json

@@ -7,6 +7,8 @@
     "canonical",
     "height",
     "hash",
+    "block_time",
+    "block_time_iso",
     "index_block_hash",
     "parent_block_hash",
     "txs",
@@ -39,6 +41,14 @@
       "type": "string",
       "description": "Hash representing the block"
     },
+    "block_time": {
+      "type": "number",
+      "description": "Unix timestamp (in seconds) indicating when this block was mined."
+    },
+    "block_time_iso": {
+      "type": "string",
+      "description": "An ISO 8601 (YYYY-MM-DDTHH:mm:ss.sssZ) indicating when this block was mined."
+    },
     "index_block_hash": {
       "type": "string",
       "description": "The only hash that can uniquely identify an anchored block or an unconfirmed state trie"

docs/entities/transactions/abstract-transaction.schema.json

@@ -11,6 +11,8 @@
         "tx_index",
         "block_hash",
         "block_height",
+        "block_time",
+        "block_time_iso",
         "burn_block_time",
         "burn_block_time_iso",
         "parent_burn_block_time",
@@ -41,6 +43,14 @@
           "type": "integer",
           "description": "Height of the block this transactions was associated with"
         },
+        "block_time": {
+          "type": "number",
+          "description": "Unix timestamp (in seconds) indicating when this block was mined."
+        },
+        "block_time_iso": {
+          "type": "string",
+          "description": "An ISO 8601 (YYYY-MM-DDTHH:mm:ss.sssZ) indicating when this block was mined."
+        },
         "burn_block_time": {
           "type": "integer",
           "description": "Unix timestamp (in seconds) indicating when this block was mined"

docs/generated.d.ts

@@ -349,6 +349,14 @@ export type AbstractTransaction = BaseTransaction & {
    * Height of the block this transactions was associated with
    */
   block_height: number;
+  /**
+   * Unix timestamp (in seconds) indicating when this block was mined.
+   */
+  block_time: number;
+  /**
+   * An ISO 8601 (YYYY-MM-DDTHH:mm:ss.sssZ) indicating when this block was mined.
+   */
+  block_time_iso: string;
   /**
    * Unix timestamp (in seconds) indicating when this block was mined
    */
@@ -1179,6 +1187,14 @@ export interface Block {
    * Hash representing the block
    */
   hash: string;
+  /**
+   * Unix timestamp (in seconds) indicating when this block was mined.
+   */
+  block_time: number;
+  /**
+   * An ISO 8601 (YYYY-MM-DDTHH:mm:ss.sssZ) indicating when this block was mined.
+   */
+  block_time_iso: string;
   /**
    * The only hash that can uniquely identify an anchored block or an unconfirmed state trie
    */

migrations/1710443563029_stacks_block_time.js

@@ -0,0 +1,21 @@
+/* eslint-disable camelcase */
+
+/** @param { import("node-pg-migrate").MigrationBuilder } pgm */
+exports.up = pgm => {
+  pgm.addColumn('txs', {
+    block_time: {
+      type: 'integer',
+      notNull: true,
+      default: '0'
+    }
+  });
+
+  pgm.addColumn('blocks', {
+    block_time: {
+      type: 'integer',
+      notNull: true,
+      default: '0'
+    }
+  });
+
+};

src/api/controllers/db-controller.ts

@@ -617,6 +617,11 @@ function parseDbBlock(
     canonical: dbBlock.canonical,
     height: dbBlock.block_height,
     hash: dbBlock.block_hash,
+    block_time: dbBlock.block_time > 0 ? dbBlock.block_time : dbBlock.burn_block_time,
+    block_time_iso:
+      dbBlock.block_time > 0
+        ? unixEpochToIso(dbBlock.block_time)
+        : unixEpochToIso(dbBlock.burn_block_time),
     index_block_hash: dbBlock.index_block_hash,
     parent_block_hash: dbBlock.parent_block_hash,
     burn_block_time: dbBlock.burn_block_time,
@@ -1039,6 +1044,12 @@ function parseDbAbstractTx(dbTx: DbTx, baseTx: BaseTransaction): AbstractTransac
     block_hash: dbTx.block_hash,
     parent_block_hash: dbTx.parent_block_hash,
     block_height: dbTx.block_height,
+    block_time: dbTx.block_time || dbTx.burn_block_time,
+    block_time_iso: dbTx.block_time
+      ? unixEpochToIso(dbTx.block_time)
+      : dbTx.burn_block_time > 0
+      ? unixEpochToIso(dbTx.burn_block_time)
+      : '',
     burn_block_time: dbTx.burn_block_time,
     burn_block_time_iso: dbTx.burn_block_time > 0 ? unixEpochToIso(dbTx.burn_block_time) : '',
     parent_burn_block_time: dbTx.parent_burn_block_time,

src/datastore/common.ts

@@ -24,6 +24,7 @@ export interface DbBlock {
   execution_cost_write_count: number;
   execution_cost_write_length: number;
   tx_count: number;
+  block_time: number;
 }
 
 /** An interface representing the microblock data that can be constructed _only_ from the /new_microblocks payload */
@@ -170,6 +171,7 @@ export interface DbTx extends BaseTx {
   block_height: number;
   burn_block_time: number;
   parent_burn_block_time: number;
+  block_time: number;
 
   tx_index: number;
 
@@ -795,6 +797,7 @@ export interface BlockQueryResult {
   parent_microblock_hash: string;
   parent_microblock_sequence: number;
   block_height: number;
+  block_time: number;
   burn_block_time: number;
   burn_block_hash: string;
   burn_block_height: number;
@@ -894,6 +897,7 @@ export interface TxQueryResult {
   index_block_hash: string;
   parent_index_block_hash: string;
   block_hash: string;
+  block_time: number;
   parent_block_hash: string;
   block_height: number;
   burn_block_time: number;
@@ -1044,6 +1048,7 @@ export interface TxInsertValues {
   block_hash: PgBytea;
   parent_block_hash: PgBytea;
   block_height: number;
+  block_time: number;
   burn_block_time: number;
   parent_burn_block_time: number;
   type_id: number;

src/datastore/helpers.ts

@@ -76,6 +76,7 @@ export const TX_COLUMNS = [
   'block_hash',
   'parent_block_hash',
   'block_height',
+  'block_time',
   'burn_block_time',
   'parent_burn_block_time',
   'type_id',
@@ -170,6 +171,7 @@ export const BLOCK_COLUMNS = [
   'parent_microblock_hash',
   'parent_microblock_sequence',
   'block_height',
+  'block_time',
   'burn_block_time',
   'burn_block_hash',
   'burn_block_height',
@@ -214,6 +216,7 @@ export const TX_METADATA_TABLES = [
   'names',
   'namespaces',
   'subdomains',
+  // TODO: add pox_set table here
 ] as const;
 
 export const POX_SYNTHETIC_EVENT_COLUMNS = [
@@ -333,6 +336,7 @@ export function parseTxQueryResult(result: ContractTxQueryResult): DbTx {
     index_block_hash: result.index_block_hash,
     parent_index_block_hash: result.parent_index_block_hash,
     block_hash: result.block_hash,
+    block_time: result.block_time || result.burn_block_time,
     parent_block_hash: result.parent_block_hash,
     block_height: result.block_height,
     burn_block_time: result.burn_block_time,
@@ -450,6 +454,7 @@ export function parseBlockQueryResult(row: BlockQueryResult): DbBlock {
     parent_microblock_hash: row.parent_microblock_hash,
     parent_microblock_sequence: row.parent_microblock_sequence,
     block_height: row.block_height,
+    block_time: row.block_time || row.burn_block_time,
     burn_block_time: row.burn_block_time,
     burn_block_hash: row.burn_block_hash,
     burn_block_height: row.burn_block_height,
@@ -1128,6 +1133,7 @@ export function createDbTxFromCoreMsg(msg: CoreNodeParsedTxMessage): DbTxRaw {
     block_height: msg.block_height,
     burn_block_time: msg.burn_block_time,
     parent_burn_block_time: msg.parent_burn_block_time,
+    block_time: msg.block_time,
     type_id: parseEnum(DbTxTypeId, parsedTx.payload.type_id as number),
     anchor_mode: parseEnum(DbTxAnchorMode, parsedTx.anchor_mode as number),
     status: getTxDbStatus(coreTx.status),

src/datastore/pg-write-store.ts

@@ -1593,6 +1593,7 @@ export class PgWriteStore extends PgStore {
       block_hash: tx.block_hash,
       parent_block_hash: tx.parent_block_hash,
       block_height: tx.block_height,
+      block_time: tx.block_time,
       burn_block_time: tx.burn_block_time,
       parent_burn_block_time: tx.parent_burn_block_time,
       type_id: tx.type_id,
@@ -1790,10 +1791,22 @@ export class PgWriteStore extends PgStore {
       this._debounceMempoolStat.running = true;
       this._debounceMempoolStat.triggeredAt = null;
       try {
-        const mempoolStats = await this.getMempoolStatsInternal({ sql: this.sql });
+        const mempoolStats = await this.sqlTransaction(async sql => {
+          return await this.getMempoolStatsInternal({ sql });
+        });
         this.eventEmitter.emit('mempoolStatsUpdate', mempoolStats);
-      } catch (e) {
-        logger.error(e, `failed to run mempool stats update`);
+      } catch (e: unknown) {
+        const connectionError = e as Error & { code: string };
+        if (
+          connectionError instanceof Error &&
+          ['CONNECTION_ENDED', 'CONNECTION_DESTROYED', 'CONNECTION_CLOSED'].includes(
+            connectionError.code
+          )
+        ) {
+          logger.info(`Skipping mempool stats query because ${connectionError.code}`);
+        } else {
+          logger.error(e, `failed to run mempool stats update`);
+        }
       } finally {
         this._debounceMempoolStat.running = false;
         this._debounceMempoolStat.debounce = null;
@@ -2969,6 +2982,7 @@ export class PgWriteStore extends PgStore {
       block_hash: tx.block_hash,
       parent_block_hash: tx.parent_block_hash,
       block_height: tx.block_height,
+      block_time: tx.block_time,
       burn_block_time: tx.burn_block_time,
       parent_burn_block_time: tx.parent_burn_block_time,
       type_id: tx.type_id,
@@ -3161,4 +3175,11 @@ export class PgWriteStore extends PgStore {
       }
     }
   }
+
+  async close(args?: { timeout?: number }): Promise<void> {
+    if (this._debounceMempoolStat.debounce) {
+      clearTimeout(this._debounceMempoolStat.debounce);
+    }
+    await super.close(args);
+  }
 }

src/event-stream/core-node-message.ts

@@ -255,6 +255,7 @@ export interface CoreNodeBlockMessage {
   pox_v1_unlock_height?: number;
   pox_v2_unlock_height?: number;
   pox_v3_unlock_height?: number;
+  block_time: number;
 }
 
 export interface CoreNodeParsedTxMessage {
@@ -274,6 +275,7 @@ export interface CoreNodeParsedTxMessage {
   burn_block_time: number;
   parent_burn_block_time: number;
   parent_burn_block_hash: string;
+  block_time: number;
 }
 
 export interface CoreNodeBurnBlockMessage {

src/event-stream/event-server.ts

@@ -183,6 +183,7 @@ async function handleMicroblockMessage(
       time: msg.burn_block_timestamp,
     },
   });
+  const stacksBlockReceiptDate = Math.round(Date.now() / 1000);
   const parsedTxs: CoreNodeParsedTxMessage[] = [];
   msg.transactions.forEach(tx => {
     const blockData: CoreNodeMsgBlockData = {
@@ -197,6 +198,7 @@ async function handleMicroblockMessage(
       burn_block_height: -1,
       index_block_hash: '',
       block_hash: '',
+      block_time: stacksBlockReceiptDate,
 
       // These properties can be determined with a db query, they are set while the db is inserting them.
       block_height: -1,
@@ -218,6 +220,7 @@ async function handleMicroblockMessage(
       {
         block_height: -1, // TODO: fill during initial db insert
         index_block_hash: '',
+        block_time: stacksBlockReceiptDate,
       },
       chainId
     ),
@@ -232,10 +235,20 @@ async function handleBlockMessage(
 ): Promise<void> {
   const ingestionTimer = stopwatch();
   const counts = newCoreNoreBlockEventCounts();
+  // If running in IBD mode, we use the parent burn block timestamp as the receipt date,
+  // otherwise, use the current timestamp.
   const parsedTxs: CoreNodeParsedTxMessage[] = [];
   const blockData: CoreNodeMsgBlockData = {
     ...msg,
   };
+  if (!blockData.block_time) {
+    // TODO: if the core node can give use the stacks-block count/index for the current tenure/burn_block then we should
+    // increment this by that value so that timestampts increase monotonically.
+    const stacksBlockReceiptDate = db.isEventReplay
+      ? msg.parent_burn_block_timestamp
+      : Math.round(Date.now() / 1000);
+    blockData.block_time = stacksBlockReceiptDate;
+  }
   msg.transactions.forEach(item => {
     const parsedTx = parseMessageTransaction(chainId, item, blockData, msg.events);
     if (parsedTx) {
@@ -296,6 +309,7 @@ async function handleBlockMessage(
     execution_cost_write_count: 0,
     execution_cost_write_length: 0,
     tx_count: msg.transactions.length,
+    block_time: msg.block_time,
   };
 
   logger.debug(`Received block ${msg.block_hash} (${msg.block_height}) from node`, dbBlock);
@@ -349,7 +363,7 @@ async function handleBlockMessage(
     block: dbBlock,
     microblocks: dbMicroblocks,
     minerRewards: dbMinerRewards,
-    txs: parseDataStoreTxEventData(parsedTxs, msg.events, msg, chainId),
+    txs: parseDataStoreTxEventData(parsedTxs, msg.events, dbBlock, chainId),
     pox_v1_unlock_height: msg.pox_v1_unlock_height,
     pox_v2_unlock_height: msg.pox_v2_unlock_height,
     pox_v3_unlock_height: msg.pox_v3_unlock_height,
@@ -369,6 +383,7 @@ function parseDataStoreTxEventData(
   blockData: {
     block_height: number;
     index_block_hash: string;
+    block_time: number;
   },
   chainId: ChainID
 ): DataStoreTxEventData[] {
@@ -1091,6 +1106,7 @@ export function parseNewBlockMessage(chainId: ChainID, msg: CoreNodeBlockMessage
     burn_block_time: msg.burn_block_time,
     burn_block_hash: msg.burn_block_hash,
     burn_block_height: msg.burn_block_height,
+    block_time: msg.block_time,
     miner_txid: msg.miner_txid,
     execution_cost_read_count: totalCost.execution_cost_read_count,
     execution_cost_read_length: totalCost.execution_cost_read_length,

src/event-stream/reader.ts

@@ -588,6 +588,7 @@ export interface CoreNodeMsgBlockData {
   block_height: number;
   burn_block_time: number;
   burn_block_height: number;
+  block_time: number;
 }
 
 export function parseMicroblocksFromTxs(args: {
@@ -744,6 +745,7 @@ export function parseMessageTransaction(
       raw_tx: coreTx.raw_tx,
       parsed_tx: rawTx,
       block_hash: blockData.block_hash,
+      block_time: blockData.block_time,
       index_block_hash: blockData.index_block_hash,
       parent_index_block_hash: blockData.parent_index_block_hash,
       parent_block_hash: blockData.parent_block_hash,