feat(btc|service): support internal query data caching in TxBuilder.payFee()

.changeset/afraid-pianos-do.md

@@ -0,0 +1,5 @@
+---
+"@rgbpp-sdk/btc": minor
+---
+
+Support query data caching internally in TxBuilder/DataSource, preventing query from the BtcAssetsApi too often when paying fee 

packages/btc/README.md

@@ -372,6 +372,8 @@ interface DataSource {
     allowInsufficient?: boolean;
     onlyNonRgbppUtxos?: boolean;
     onlyConfirmedUtxos?: boolean;
+    noAssetsApiCache?: boolean;
+    internalCacheKey?: string;
     excludeUtxos?: {
       txid: string;
       vout: number;

packages/btc/src/query/cache.ts

@@ -0,0 +1,62 @@
+import { Utxo } from '../transaction/utxo';
+
+export class DataCache {
+  private utxos: Map<string, Utxo[]>; // Map<Key, Utxo[]>
+  private hasRgbppAssets: Map<string, boolean>; // Map<`{txid}:{vout}`, HasAssets>
+
+  constructor() {
+    this.utxos = new Map();
+    this.hasRgbppAssets = new Map();
+  }
+
+  setUtxos(key: string, utxos: Utxo[]) {
+    this.utxos.set(key, utxos);
+  }
+  getUtxos(key: string): Utxo[] | undefined {
+    return this.utxos.get(key);
+  }
+  cleanUtxos(key: string) {
+    if (this.utxos.has(key)) {
+      this.utxos.delete(key);
+    }
+  }
+  async optionalCacheUtxos(props: { key?: string; getter: () => Promise<Utxo[]> | Utxo[] }): Promise<Utxo[]> {
+    if (props.key && this.utxos.has(props.key)) {
+      return this.getUtxos(props.key) as Utxo[];
+    }
+
+    const utxos = await props.getter();
+    if (props.key) {
+      this.setUtxos(props.key, utxos);
+    }
+
+    return utxos;
+  }
+
+  setHasRgbppAssets(key: string, hasAssets: boolean) {
+    this.hasRgbppAssets.set(key, hasAssets);
+  }
+  getHasRgbppAssets(key: string): boolean | undefined {
+    return this.hasRgbppAssets.get(key);
+  }
+  cleanHasRgbppAssets(key: string) {
+    if (this.hasRgbppAssets.has(key)) {
+      this.hasRgbppAssets.delete(key);
+    }
+  }
+  async optionalCacheHasRgbppAssets(props: {
+    key?: string;
+    getter: () => Promise<boolean> | boolean;
+  }): Promise<boolean> {
+    if (props.key && this.hasRgbppAssets.has(props.key)) {
+      return this.getHasRgbppAssets(props.key) as boolean;
+    }
+
+    const hasRgbppAssets = await props.getter();
+    if (props.key) {
+      this.setHasRgbppAssets(props.key, hasRgbppAssets);
+    }
+
+    return hasRgbppAssets;
+  }
+}

packages/btc/src/query/source.ts

@@ -6,14 +6,17 @@ import { TxAddressOutput } from '../transaction/build';
 import { isOpReturnScriptPubkey } from '../transaction/embed';
 import { addressToScriptPublicKeyHex, getAddressType } from '../address';
 import { remove0x } from '../utils';
+import { DataCache } from './cache';
 
 export class DataSource {
+  public cache: DataCache;
   public service: BtcAssetsApi;
   public networkType: NetworkType;
 
   constructor(service: BtcAssetsApi, networkType: NetworkType) {
     this.service = service;
     this.networkType = networkType;
+    this.cache = new DataCache();
   }
 
   // Query a UTXO from the service.
@@ -102,6 +105,8 @@ export class DataSource {
     allowInsufficient?: boolean;
     onlyNonRgbppUtxos?: boolean;
     onlyConfirmedUtxos?: boolean;
+    noAssetsApiCache?: boolean;
+    internalCacheKey?: string;
     excludeUtxos?: {
       txid: string;
       vout: number;
@@ -117,12 +122,20 @@ export class DataSource {
       minUtxoSatoshi,
       onlyConfirmedUtxos,
       onlyNonRgbppUtxos,
+      noAssetsApiCache,
+      internalCacheKey,
       allowInsufficient = false,
       excludeUtxos = [],
     } = props;
-    const utxos = await this.getUtxos(address, {
-      only_confirmed: onlyConfirmedUtxos,
-      min_satoshi: minUtxoSatoshi,
+
+    const utxos = await this.cache.optionalCacheUtxos({
+      key: internalCacheKey,
+      getter: () =>
+        this.getUtxos(address, {
+          only_confirmed: onlyConfirmedUtxos,
+          min_satoshi: minUtxoSatoshi,
+          no_cache: noAssetsApiCache,
+        }),
     });
 
     const collected = [];
@@ -140,8 +153,14 @@ export class DataSource {
         }
       }
       if (onlyNonRgbppUtxos) {
-        const ckbRgbppAssets = await this.service.getRgbppAssetsByBtcUtxo(utxo.txid, utxo.vout);
-        if (ckbRgbppAssets && ckbRgbppAssets.length > 0) {
+        const hasRgbppAssets = await this.cache.optionalCacheHasRgbppAssets({
+          key: `${utxo.txid}:${utxo.vout}`,
+          getter: async () => {
+            const ckbRgbppAssets = await this.service.getRgbppAssetsByBtcUtxo(utxo.txid, utxo.vout);
+            return Array.isArray(ckbRgbppAssets) && ckbRgbppAssets.length > 0;
+          },
+        });
+        if (hasRgbppAssets) {
           continue;
         }
       }

packages/btc/src/transaction/build.ts

@@ -145,6 +145,10 @@ export class TxBuilder {
     const originalInputs = clone(this.inputs);
     const originalOutputs = clone(this.outputs);
 
+    // Create a cache key to enable the internal caching, prevent querying the Utxo[] too often
+    // TODO: consider provide an option to disable the cache
+    const internalCacheKey = `${Date.now()}`;
+
     // Fill a default recommended fee rate if props.feeRate is not provided
     let defaultFeeRate: number | undefined;
     if (!feeRate && !this.feeRate) {
@@ -177,6 +181,7 @@ export class TxBuilder {
           amount: returnAmount,
           fromAddress: address,
           fromPublicKey: publicKey,
+          internalCacheKey,
         });
       } else {
         // If the inputs have insufficient satoshi, a satoshi collection is required.
@@ -189,6 +194,7 @@ export class TxBuilder {
           targetAmount,
           changeAddress,
           deductFromOutputs,
+          internalCacheKey,
         });
       }
 
@@ -203,6 +209,9 @@ export class TxBuilder {
       }
     }
 
+    // Clear cache for the Utxo[] list
+    this.source.cache.cleanUtxos(internalCacheKey);
+
     return {
       fee: currentFee,
       feeRate: currentFeeRate,
@@ -216,6 +225,7 @@ export class TxBuilder {
     changeAddress?: string;
     injectCollected?: boolean;
     deductFromOutputs?: boolean;
+    internalCacheKey?: string;
   }) {
     if (!isSupportedFromAddress(props.address)) {
       throw TxBuildError.withComment(ErrorCodes.UNSUPPORTED_ADDRESS_TYPE, props.address);
@@ -231,12 +241,18 @@ export class TxBuilder {
     /**
      * Collect from the "from" address via DataSource.
      * Will update the value of inputs/collected/changeAmount.
+     *
+     * The API has two layers of data caching:
+     * - noAssetsApiCache: BtcAssetsApi cache, can be disabled if the set to true
+     * - internalCacheKey: Internal cache, enabled if the key is provided
      */
     const _collect = async (_targetAmount: number) => {
       const { utxos, satoshi } = await this.source.collectSatoshi({
         address: props.address,
         targetAmount: _targetAmount,
         allowInsufficient: true,
+        noAssetsApiCache: true,
+        internalCacheKey: props.internalCacheKey,
         minUtxoSatoshi: this.minUtxoSatoshi,
         onlyNonRgbppUtxos: this.onlyNonRgbppUtxos,
         onlyConfirmedUtxos: this.onlyConfirmedUtxos,
@@ -356,8 +372,14 @@ export class TxBuilder {
     };
   }
 
-  async injectChange(props: { amount: number; address: string; fromAddress: string; fromPublicKey?: string }) {
-    const { address, fromAddress, fromPublicKey, amount } = props;
+  async injectChange(props: {
+    amount: number;
+    address: string;
+    fromAddress: string;
+    fromPublicKey?: string;
+    internalCacheKey?: string;
+  }) {
+    const { address, fromAddress, fromPublicKey, amount, internalCacheKey } = props;
 
     // If any (output.fixed != true) is found in the outputs (search in ASC order),
     // return the change value to the first matched output.
@@ -389,6 +411,7 @@ export class TxBuilder {
         changeAddress: address,
         injectCollected: true,
         deductFromOutputs: false,
+        internalCacheKey,
       });
       if (collected < amount) {
         throw TxBuildError.withComment(ErrorCodes.INSUFFICIENT_UTXO, `expected: ${amount}, actual: ${collected}`);

packages/service/README.md

@@ -223,6 +223,7 @@ interface BtcApiBlockTransactionIds {
 
 interface BtcApiBalanceParams {
   min_satoshi?: number;
+  no_cache?: boolean;
 }
 
 interface BtcApiBalance {
@@ -236,6 +237,7 @@ interface BtcApiBalance {
 interface BtcApiUtxoParams {
   only_confirmed?: boolean;
   min_satoshi?: number;
+  no_cache?: boolean;
 }
 
 interface BtcApiUtxo {
@@ -329,6 +331,7 @@ interface RgbppApiTransactionState {
 
 interface RgbppApiAssetsByAddressParams {
   type_script?: string;
+  no_cache?: boolean;
 }
 
 interface RgbppApiSpvProof {