Merge pull request #92 from mempool/bisq

Bisq support

backend/mempool-config.sample.json

@@ -13,6 +13,8 @@
   "INITIAL_BLOCK_AMOUNT": 8,
   "TX_PER_SECOND_SPAN_SECONDS": 150,
   "ELECTRS_API_URL": "https://www.blockstream.info/testnet/api",
+  "BISQ_ENABLED": false,
+  "BSQ_BLOCKS_DATA_PATH": "/bisq/data",
   "SSL": false,
   "SSL_CERT_FILE_PATH": "/etc/letsencrypt/live/mysite/fullchain.pem",
   "SSL_KEY_FILE_PATH": "/etc/letsencrypt/live/mysite/privkey.pem"

backend/src/api/bisq.ts

@@ -0,0 +1,228 @@
+const config = require('../../mempool-config.json');
+import * as fs from 'fs';
+import * as request from 'request';
+import { BisqBlocks, BisqBlock, BisqTransaction, BisqStats, BisqTrade } from '../interfaces';
+import { Common } from './common';
+
+class Bisq {
+  private latestBlockHeight = 0;
+  private blocks: BisqBlock[] = [];
+  private transactions: BisqTransaction[] = [];
+  private transactionIndex: { [txId: string]: BisqTransaction } = {};
+  private blockIndex: { [hash: string]: BisqBlock } = {};
+  private addressIndex: { [address: string]: BisqTransaction[] } = {};
+  private stats: BisqStats = {
+    minted: 0,
+    burnt: 0,
+    addresses: 0,
+    unspent_txos: 0,
+    spent_txos: 0,
+  };
+  private price: number = 0;
+  private priceUpdateCallbackFunction: ((price: number) => void) | undefined;
+  private subdirectoryWatcher: fs.FSWatcher | undefined;
+
+  constructor() {}
+
+  startBisqService(): void {
+    this.loadBisqDumpFile();
+    setInterval(this.updatePrice.bind(this), 1000 * 60 * 60);
+    this.updatePrice();
+    this.startTopLevelDirectoryWatcher();
+    this.restartSubDirectoryWatcher();
+  }
+
+  getTransaction(txId: string): BisqTransaction | undefined {
+    return this.transactionIndex[txId];
+  }
+
+  getTransactions(start: number, length: number): [BisqTransaction[], number] {
+    return [this.transactions.slice(start, length + start), this.transactions.length];
+  }
+
+  getBlock(hash: string): BisqBlock | undefined {
+    return this.blockIndex[hash];
+  }
+
+  getAddress(hash: string): BisqTransaction[] {
+    return this.addressIndex[hash];
+  }
+
+  getBlocks(start: number, length: number): [BisqBlock[], number] {
+    return [this.blocks.slice(start, length + start), this.blocks.length];
+  }
+
+  getStats(): BisqStats {
+    return this.stats;
+  }
+
+  setPriceCallbackFunction(fn: (price: number) => void) {
+    this.priceUpdateCallbackFunction = fn;
+  }
+
+  getLatestBlockHeight(): number {
+    return this.latestBlockHeight;
+  }
+
+  private startTopLevelDirectoryWatcher() {
+    let fsWait: NodeJS.Timeout | null = null;
+    fs.watch(config.BSQ_BLOCKS_DATA_PATH, () => {
+      if (fsWait) {
+        clearTimeout(fsWait);
+      }
+      fsWait = setTimeout(() => {
+        console.log(`Change detected in the top level Bisq data folder. Resetting inner watcher.`);
+        this.restartSubDirectoryWatcher();
+      }, 15000);
+    });
+  }
+
+  private restartSubDirectoryWatcher() {
+    if (this.subdirectoryWatcher) {
+      this.subdirectoryWatcher.close();
+    }
+
+    let fsWait: NodeJS.Timeout | null = null;
+    this.subdirectoryWatcher = fs.watch(config.BSQ_BLOCKS_DATA_PATH + '/all', () => {
+      if (fsWait) {
+        clearTimeout(fsWait);
+      }
+      fsWait = setTimeout(() => {
+        console.log(`Change detected in the Bisq data folder.`);
+        this.loadBisqDumpFile();
+      }, 2000);
+    });
+  }
+
+  private updatePrice() {
+    request('https://markets.bisq.network/api/trades/?market=bsq_btc', { json: true }, (err, res, trades: BisqTrade[]) => {
+      if (err) { return console.log(err); }
+
+      const prices: number[] = [];
+      trades.forEach((trade) => {
+        prices.push(parseFloat(trade.price) * 100000000);
+      });
+      prices.sort((a, b) => a - b);
+      this.price = Common.median(prices);
+      if (this.priceUpdateCallbackFunction) {
+        this.priceUpdateCallbackFunction(this.price);
+      }
+    });
+  }
+
+  private async loadBisqDumpFile(): Promise<void> {
+    try {
+      const data = await this.loadData();
+      await this.loadBisqBlocksDump(data);
+      this.buildIndex();
+      this.calculateStats();
+    } catch (e) {
+      console.log('loadBisqDumpFile() error.', e.message);
+    }
+  }
+
+  private buildIndex() {
+    const start = new Date().getTime();
+    this.transactions = [];
+    this.transactionIndex = {};
+    this.addressIndex = {};
+
+    this.blocks.forEach((block) => {
+      /* Build block index */
+      if (!this.blockIndex[block.hash]) {
+        this.blockIndex[block.hash] = block;
+      }
+
+      /* Build transactions index */
+      block.txs.forEach((tx) => {
+        this.transactions.push(tx);
+        this.transactionIndex[tx.id] = tx;
+      });
+    });
+
+    /* Build address index */
+    this.transactions.forEach((tx) => {
+      tx.inputs.forEach((input) => {
+        if (!this.addressIndex[input.address]) {
+          this.addressIndex[input.address] = [];
+        }
+        if (this.addressIndex[input.address].indexOf(tx) === -1) {
+          this.addressIndex[input.address].push(tx);
+        }
+      });
+      tx.outputs.forEach((output) => {
+        if (!this.addressIndex[output.address]) {
+          this.addressIndex[output.address] = [];
+        }
+        if (this.addressIndex[output.address].indexOf(tx) === -1) {
+          this.addressIndex[output.address].push(tx);
+        }
+      });
+    });
+
+    const time = new Date().getTime() - start;
+    console.log('Bisq data index rebuilt in ' + time + ' ms');
+  }
+
+  private calculateStats() {
+    let minted = 0;
+    let burned = 0;
+    let unspent = 0;
+    let spent = 0;
+
+    this.transactions.forEach((tx) => {
+      tx.outputs.forEach((output) => {
+        if (output.opReturn) {
+          return;
+        }
+        if (output.txOutputType === 'GENESIS_OUTPUT' || output.txOutputType === 'ISSUANCE_CANDIDATE_OUTPUT' && output.isVerified) {
+          minted += output.bsqAmount;
+        }
+        if (output.isUnspent) {
+          unspent++;
+        } else {
+          spent++;
+        }
+      });
+      burned += tx['burntFee'];
+    });
+
+    this.stats = {
+      addresses: Object.keys(this.addressIndex).length,
+      minted: minted,
+      burnt: burned,
+      spent_txos: spent,
+      unspent_txos: unspent,
+    };
+  }
+
+  private async loadBisqBlocksDump(cacheData: string): Promise<void> {
+    const start = new Date().getTime();
+    if (cacheData && cacheData.length !== 0) {
+      console.log('Loading Bisq data from dump...');
+      const data: BisqBlocks = JSON.parse(cacheData);
+      if (data.blocks && data.blocks.length !== this.blocks.length) {
+        this.blocks = data.blocks.filter((block) => block.txs.length > 0);
+        this.blocks.reverse();
+        this.latestBlockHeight = data.chainHeight;
+        const time = new Date().getTime() - start;
+        console.log('Bisq dump loaded in ' + time + ' ms');
+      } else {
+        throw new Error(`Bisq dump didn't contain any blocks`);
+      }
+    }
+  }
+
+  private loadData(): Promise<string> {
+    return new Promise((resolve, reject) => {
+      fs.readFile(config.BSQ_BLOCKS_DATA_PATH + '/all/blocks.json', 'utf8', (err, data) => {
+        if (err) {
+          reject(err);
+        }
+        resolve(data);
+      });
+    });
+  }
+}
+
+export default new Bisq();

backend/src/api/blocks.ts

@@ -73,10 +73,10 @@ class Blocks {
         console.log(`${found} of ${txIds.length} found in mempool. ${notFound} not found.`);
 
         block.reward = transactions[0].vout.reduce((acc, curr) => acc + curr.value, 0);
+        block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]);
         transactions.sort((a, b) => b.feePerVsize - a.feePerVsize);
         block.medianFee = transactions.length > 1 ? Common.median(transactions.map((tx) => tx.feePerVsize)) : 0;
         block.feeRange = transactions.length > 1 ? Common.getFeesInRange(transactions, 8, 1) : [0, 0];
-        block.coinbaseTx = this.stripCoinbaseTransaction(transactions[0]);
 
         this.blocks.push(block);
         if (this.blocks.length > config.KEEP_BLOCK_AMOUNT) {

backend/src/api/mempool.ts

@@ -35,6 +35,9 @@ class Mempool {
 
   public setMempool(mempoolData: { [txId: string]: TransactionExtended }) {
     this.mempoolCache = mempoolData;
+    if (this.mempoolChangedCallback) {
+      this.mempoolChangedCallback(this.mempoolCache, [], []);
+    }
   }
 
   public async updateMemPoolInfo() {

backend/src/api/websocket-handler.ts

@@ -12,13 +12,18 @@ import { Common } from './common';
 class WebsocketHandler {
   private wss: WebSocket.Server | undefined;
   private nativeAssetId = '6f0279e9ed041c3d710a9f57d0c02928416460c4b722ae3457a11eec381c526d';
+  private extraInitProperties = {};
 
   constructor() { }
 
   setWebsocketServer(wss: WebSocket.Server) {
     this.wss = wss;
   }
 
+  setExtraInitProperties(property: string, value: any) {
+    this.extraInitProperties[property] = value;
+  }
+
   setupConnectionHandling() {
     if (!this.wss) {
       throw new Error('WebSocket.Server is not set');
@@ -84,6 +89,7 @@ class WebsocketHandler {
               'mempool-blocks': mempoolBlocks.getMempoolBlocks(),
               'git-commit': backendInfo.gitCommitHash,
               'hostname': backendInfo.hostname,
+              ...this.extraInitProperties
             }));
           }
 

backend/src/index.ts

@@ -14,6 +14,7 @@ import diskCache from './api/disk-cache';
 import statistics from './api/statistics';
 import websocketHandler from './api/websocket-handler';
 import fiatConversion from './api/fiat-conversion';
+import bisq from './api/bisq';
 
 class Server {
   wss: WebSocket.Server;
@@ -50,6 +51,11 @@ class Server {
     fiatConversion.startService();
     diskCache.loadMempoolCache();
 
+    if (config.BISQ_ENABLED) {
+      bisq.startBisqService();
+      bisq.setPriceCallbackFunction((price) => websocketHandler.setExtraInitProperties('bsq-price', price));
+    }
+
     this.server.listen(config.HTTP_PORT, () => {
       console.log(`Server started on port ${config.HTTP_PORT}`);
     });
@@ -84,6 +90,18 @@ class Server {
       .get(config.API_ENDPOINT + 'statistics/1y', routes.get1YStatistics.bind(routes))
       .get(config.API_ENDPOINT + 'backend-info', routes.getBackendInfo)
     ;
+
+    if (config.BISQ_ENABLED) {
+      this.app
+        .get(config.API_ENDPOINT + 'bisq/stats', routes.getBisqStats)
+        .get(config.API_ENDPOINT + 'bisq/tx/:txId', routes.getBisqTransaction)
+        .get(config.API_ENDPOINT + 'bisq/block/:hash', routes.getBisqBlock)
+        .get(config.API_ENDPOINT + 'bisq/blocks/tip/height', routes.getBisqTip)
+        .get(config.API_ENDPOINT + 'bisq/blocks/:index/:length', routes.getBisqBlocks)
+        .get(config.API_ENDPOINT + 'bisq/address/:address', routes.getBisqAddress)
+        .get(config.API_ENDPOINT + 'bisq/txs/:index/:length', routes.getBisqTransactions)
+      ;
+    }
   }
 }