support mid-block pagination (#80)

src/index.ts

@@ -10,13 +10,14 @@ import { Pool } from "pg";
 const semverCompare = require("semver-compare");
 
 import { connectionHandler} from "./ws-server"; 
-import { applyMiddleware, applyRoutes, Route } from "./utils";
+import { applyMiddleware, applyRoutes, Route, UtilEither, errMsgs } from "./utils";
 import * as utils from "./utils";
 import * as middleware from "./middleware";
 
 import { askBestBlock } from "./services/bestblock";
 import { utxoForAddresses } from "./services/utxoForAddress";
 import { askBlockNumByHash, askBlockNumByTxHash, askTransactionHistory } from "./services/transactionHistory";
+import type { BlockNumByTxHashFrag } from "./services/transactionHistory";
 import { filterUsedAddresses } from "./services/filterUsedAddress";
 import { askUtxoSumForAddresses } from "./services/utxoSumForAddress";
 import { handleSignedTx } from "./services/signedTransaction";
@@ -103,6 +104,28 @@ const utxoSumForAddresses = async (req:  Request, res:Response) => {
   }
 };
 
+const getOrDefaultAfterParam = (
+  result: UtilEither<BlockNumByTxHashFrag>
+): {
+  blockNumber: number,
+  txIndex: number,
+} => {
+  if (result.kind !== "ok") {
+    if (result.errMsg === errMsgs.noValue) {
+      // default value since this is an optional field
+      return {
+        blockNumber: -1,
+        txIndex: -1,
+      };
+    }
+    throw new Error(result.errMsg);
+  }
+  return {
+    blockNumber: result.value.block.number,
+    txIndex: result.value.blockIndex,
+  };
+};
+
 const txHistory = async (req: Request, res: Response) => {
   if(!req.body){
     throw new Error("error, no body");
@@ -116,23 +139,29 @@ const txHistory = async (req: Request, res: Response) => {
     const [referenceTx, referenceBlock] = (body.after && [body.after.tx, body.after.block]) || [];
     const referenceBestBlock = body.untilBlock;
     const untilBlockNum = await askBlockNumByHash(referenceBestBlock);
-    const afterBlockNum = await askBlockNumByTxHash(referenceTx );
+    const afterBlockInfo = await askBlockNumByTxHash(referenceTx);
 
-    if(untilBlockNum.kind === "error" && untilBlockNum.errMsg !== utils.errMsgs.noValue ){
+    if(untilBlockNum.kind === "error" && untilBlockNum.errMsg !== utils.errMsgs.noValue){
       throw new Error("REFERENCE_BEST_BLOCK_MISMATCH");
       return;
     }
-    if(afterBlockNum.kind === "error" && typeof referenceTx !== "undefined") {
+    if(afterBlockInfo.kind === "error" && typeof referenceTx !== "undefined") {
       throw new Error("REFERENCE_TX_NOT_FOUND");
       return;
     }
 
-    if(afterBlockNum.kind === "ok" && afterBlockNum.value.block.hash !== referenceBlock) {
+    if(afterBlockInfo.kind === "ok" && afterBlockInfo.value.block.hash !== referenceBlock) {
       throw new Error("REFERENCE_BLOCK_MISMATCH");
       return;
     }
 
-    const maybeTxs = await askTransactionHistory(pool, limit, body.addresses, afterBlockNum, untilBlockNum);
+    // when things are running smoothly, we would never hit this case case
+    if (untilBlockNum.kind !== "ok") {
+      throw new Error(untilBlockNum.errMsg);
+    }
+    const afterInfo = getOrDefaultAfterParam(afterBlockInfo);
+
+    const maybeTxs = await askTransactionHistory(pool, limit, body.addresses, afterInfo, untilBlockNum.value);
     switch(maybeTxs.kind) {
     case "ok":{
       const txs = maybeTxs.value.map( tx => ({

src/services/transactionHistory.ts

@@ -1,6 +1,6 @@
 import axios from "axios";
 
-import {  assertNever, contentTypeHeaders, errMsgs, graphqlEndpoint, UtilEither} from "../utils";
+import {  contentTypeHeaders, errMsgs, graphqlEndpoint, UtilEither} from "../utils";
 
 import { rowToCertificate, BlockEra, BlockFrag, Certificate, TransInputFrag, TransOutputFrag, TransactionFrag} from "../Transactions/types";
 
@@ -47,7 +47,7 @@ const askTransactionSqlQuery = `
               ON source_tx_out.tx_id = source_tx.id
 
             WHERE source_tx_out.address = ANY(($1)::varchar array)
-               OR source_tx_out.payment_cred = ANY(($5)::bytea array)
+               OR source_tx_out.payment_cred = ANY(($6)::bytea array)
 
           UNION
             ${/* 2) Get all outputs for the transaction */""}
@@ -58,7 +58,7 @@ const askTransactionSqlQuery = `
               on tx.id = tx_out.tx_id
 
             where tx_out.address = ANY(($1)::varchar array)
-              or tx_out.payment_cred = ANY(($6)::bytea array)
+              or tx_out.payment_cred = ANY(($7)::bytea array)
 
           UNION
             ${/* 3) Get all certificates for the transaction */""}
@@ -70,7 +70,7 @@ const askTransactionSqlQuery = `
             where certs."formalType" in ('CertRegKey', 'CertDeregKey','CertDelegate')
               and certs."stakeCred" = any(
                 ${/* stakeCred is encoded as a string, so we have to convert from a byte array to a hex string */""}
-                (SELECT array_agg(encode(addr, 'hex')) from UNNEST($6) as addr)::varchar array
+                (SELECT array_agg(encode(addr, 'hex')) from UNNEST($7) as addr)::varchar array
               )
 
           UNION
@@ -85,7 +85,7 @@ const askTransactionSqlQuery = `
             JOIN stake_address as addr
             on w.addr_id = addr.id
 
-            where addr.hash_raw = any(($6)::bytea array)
+            where addr.hash_raw = any(($7)::bytea array)
            ) hashes
     )
   select tx.hash
@@ -136,11 +136,20 @@ const askTransactionSqlQuery = `
   LEFT JOIN pool_meta_data 
     on tx.id = pool_meta_data.registered_tx_id 
 
-  where     block.block_no <= $2
-        and block.block_no > $3 
+  where 
+        ${/* is within untilBlock (inclusive) */""}
+        block.block_no <= $2
+        and (
+          ${/* Either: */""}
+          ${/* 1) comes in block strict after the "after" field */""}
+          block.block_no > $3
+            or
+          ${/* 2) Is in the same block as the "after" field, but is tx that appears afterwards */""}
+          (block.block_no = $3 and tx.block_index > $4)
+        ) 
 
   order by block.time asc, tx.block_index asc
-  limit $4;
+  limit $5;
 `;
 
 
@@ -251,16 +260,21 @@ export const askTransactionHistory = async (
   pool: Pool
   , limit: number
   , addresses: string[]
-  , afterNum: UtilEither<BlockNumByTxHashFrag>
-  , untilNum: UtilEither<number>) : Promise<UtilEither<TransactionFrag[]>> => {
+  , after: {
+    blockNumber: number,
+    txIndex: number,
+  }
+  , untilNum: number
+) : Promise<UtilEither<TransactionFrag[]>> => {
   const addressTypes = getAddressesByType(addresses);
   const ret = await pool.query(askTransactionSqlQuery, [
       [
         ...addressTypes.legacyAddr,
         ...addressTypes.bech32,
       ]
-    , untilNum.kind === "ok" ? untilNum.value : 0
-    , afterNum.kind === "ok" ? afterNum.value.block.number : 0
+    , untilNum
+    , after.blockNumber
+    , after.txIndex
     , limit
     , addressTypes.paymentCreds
     , addressTypes.stakingKeys
@@ -314,9 +328,10 @@ export const askTransactionHistory = async (
 
 };
 
-interface BlockNumByTxHashFrag {
+export interface BlockNumByTxHashFrag {
   block: BlockByTxHashFrag;
   hash: string;
+  blockIndex: number, // this is actually the index of the transaction in the block
 }
 interface BlockByTxHashFrag {
   hash: string;
@@ -334,6 +349,7 @@ export const askBlockNumByTxHash = async (hash : string|undefined): Promise<Util
                   }
                 }
               ) {
+                blockIndex
                 hash
                 block {
                   number

tests/txHistory.test.ts

@@ -222,7 +222,23 @@ describe("/txs/history", function() {
     const result2 = await axios.post(testableUri, dataRepeatHistory );
     expect(result1.data).to.be.eql(result2.data);
   });
-
+  it("untilBlock should limit the response", async() => {
+    const result = await axios.post(testableUri, {
+      addresses: [
+        "addr1q84shx6jr9s258r9m45ujeyde7u4z7tthkedezjm5kdr4um64gv6jqqncjd205c540fgu5450tzvu27n9lk8ulm3s99spva2ru"
+      ]
+      // make sure if we as for after txIndex 0, txIndex 1 is included in the response
+      // AKA support pagination mid-block
+      , after: {
+        tx: "f07d7d5cb0126da7da9f6a067aee00fd42efae94891a42544abfd1759248019d",
+        block: "728ceadf2d949281591175a6d1641f10f2307eff80eaf59c5300dbd4a5f83554",
+      }
+      // make sure untilBlock is inclusive
+      , untilBlock: "728ceadf2d949281591175a6d1641f10f2307eff80eaf59c5300dbd4a5f83554"
+    });
+    expect(result.data).to.have.lengthOf(1);
+    expect(result.data[0].hash).to.equal("00d6d64b251514c48a9ad75940c5e7031bae5f0d002e9be7f6caf4cc1a78b57f");
+  });
   it("untilBlock should limit the response", async() => {
     const data = R.merge(dataForAddresses, { untilBlock: hashForOlderBlock } );
     const result = await axios.post(testableUri, data);