Hive-engine light node

app.js

@@ -11,6 +11,7 @@ const replay = require('./plugins/Replay');
 const p2p = require('./plugins/P2P');
 
 const conf = require('./config');
+const { Database } = require('./libs/Database');
 
 const logger = createLogger({
   format: format.combine(
@@ -131,20 +132,6 @@ const unloadPlugin = async (plugin) => {
   return res;
 };
 
-// start streaming the Hive blockchain and produce the sidechain blocks accordingly
-const start = async (requestedPlugins) => {
-  let res = await loadPlugin(blockchain, requestedPlugins);
-  if (res && res.payload === null) {
-    res = await loadPlugin(streamer, requestedPlugins);
-    if (res && res.payload === null) {
-      res = await loadPlugin(p2p, requestedPlugins);
-      if (res && res.payload === null) {
-        res = await loadPlugin(jsonRPCServer, requestedPlugins);
-      }
-    }
-  }
-};
-
 const stop = async () => {
   logger.info('Stopping node...');
   await unloadPlugin(jsonRPCServer);
@@ -177,6 +164,58 @@ const stopApp = async (signal = 0) => {
   process.kill(process.pid, signal);
 };
 
+// graceful app closing
+let shuttingDown = false;
+
+const gracefulShutdown = () => {
+  if (shuttingDown === false) {
+    shuttingDown = true;
+    stopApp('SIGINT');
+  }
+};
+
+const initLightNode = async () => {
+  const {
+    databaseURL,
+    databaseName,
+    lightNode,
+    blocksToKeep,
+  } = conf;
+  const database = new Database();
+  await database.init(databaseURL, databaseName, lightNode, blocksToKeep);
+
+  if (!lightNode) {
+    // check if was previously a light node
+    const wasLightNode = await database.wasLightNodeBefore();
+    if (wasLightNode) {
+      console.log('Can\'t switch from a node, which was previously a light node, to a full node. Please restore your database from a full node dump.');
+      await gracefulShutdown();
+      process.exit();
+    }
+    return;
+  }
+  console.log('Initializing light node - this may take a while..');
+
+  // cleanup old blocks / transactions for light nodes
+  await database.cleanupLightNode();
+};
+
+// start streaming the Hive blockchain and produce the sidechain blocks accordingly
+const start = async (requestedPlugins) => {
+  await initLightNode();
+
+  let res = await loadPlugin(blockchain, requestedPlugins);
+  if (res && res.payload === null) {
+    res = await loadPlugin(streamer, requestedPlugins);
+    if (res && res.payload === null) {
+      res = await loadPlugin(p2p, requestedPlugins);
+      if (res && res.payload === null) {
+        res = await loadPlugin(jsonRPCServer, requestedPlugins);
+      }
+    }
+  }
+};
+
 // replay the sidechain from a blocks log file
 const replayBlocksLog = async () => {
   let res = await loadPlugin(blockchain);
@@ -202,16 +241,6 @@ if (program.replay !== undefined) {
   start(requestedPlugins);
 }
 
-// graceful app closing
-let shuttingDown = false;
-
-const gracefulShutdown = () => {
-  if (shuttingDown === false) {
-    shuttingDown = true;
-    stopApp('SIGINT');
-  }
-};
-
 process.on('SIGTERM', () => {
   gracefulShutdown();
 });

config.json

@@ -16,5 +16,8 @@
     "startHiveBlock": 41967000,
     "genesisHiveBlock": 41967000,
     "witnessEnabled": true,
-    "defaultLogLevel": "warn"
+    "defaultLogLevel": "warn",
+    "lightNode": false,
+    "blocksToKeep": 864000,
+    "domain" : ""
 }

find_divergent_block.js

@@ -90,45 +90,65 @@ async function findDivergentBlock() {
   const {
     databaseURL,
     databaseName,
+    lightNode,
   } = conf;
   const database = new Database();
   await database.init(databaseURL, databaseName);
   const chain = database.database.collection('chain');
 
   let block = (await chain.find().sort({ _id: -1 }).limit(1).toArray())[0];
-  let low = 0;
-  let high = block._id;
-  const headBlock = high;
   let mainBlock;
-  while (high - low >= 1) {
-    console.log(`low ${low} high ${high}`);
-    const check = Math.floor((low + high) / 2);
-    mainBlock = await getBlock(check);
+  if (lightNode) {
+    let retries = 0;
+    while (!mainBlock && retries < 10) {
+      await new Promise(r => setTimeout(r, 1000)); // sleep 1 second
+      mainBlock = await getBlock(block._id);
+      retries += 1;
+    }
+
     if (!mainBlock) {
-      break;
+      console.log(`failed to fetch block ${block._id} from ${node}`);
+    } else if (mainBlock.hash === block.hash) {
+      console.log('ok');
+    } else {
+      console.log(`head block divergent from ${node}`);
+      console.log(`hash should be ${mainBlock.hash} is ${block.hash}`);
+    }
+  } else {
+    let low = 0;
+    let high = block._id;
+    const headBlock = high;
+    while (high - low >= 1) {
+      console.log(`low ${low} high ${high}`);
+      const check = Math.floor((low + high) / 2);
+      mainBlock = await getBlock(check);
+      if (!mainBlock) {
+        break;
+      }
+      block = await chain.findOne({ _id: check });
+      // Different comparison modes, uncomment desired comparison.
+      if (mainBlock.hash !== block.hash) {
+        // if (mainBlock.refHiveBlockNumber !== block.refHiveBlockNumber) {
+        // if (!compareBlocks(mainBlock, block)) {
+        high = check;
+      } else {
+        low = check + 1;
+      }
     }
-    block = await chain.findOne({ _id: check });
-    // Different comparison modes, uncomment desired comparison.
-    if (mainBlock.hash !== block.hash) {
-    // if (mainBlock.refHiveBlockNumber !== block.refHiveBlockNumber) {
-    // if (!compareBlocks(mainBlock, block)) {
-      high = check;
+    mainBlock = await getBlock(high);
+    block = await chain.findOne({ _id: high });
+
+    if (high === headBlock && high - low <= 0) {
+      console.log('ok');
+    } else if (high !== low) {
+      console.log('not caught up or error fetching block');
     } else {
-      low = check + 1;
+      console.log('### high block');
+      printBlockDiff(block, mainBlock);
+      console.log(`divergent block id at ${high}`);
     }
   }
-  mainBlock = await getBlock(high);
-  block = await chain.findOne({ _id: high });
 
-  if (high === headBlock && high - low <= 0) {
-    console.log('ok');
-  } else if (high !== low) {
-    console.log('not caught up or error fetching block');
-  } else {
-    console.log('### high block');
-    printBlockDiff(block, mainBlock);
-    console.log(`divergent block id at ${high}`);
-  }
   database.close();
 }
 

libs/Block.js

@@ -104,8 +104,20 @@ class Block {
     }
   }
 
+  /**
+   * Try cleaning up blocks after every 100 blocks if node is a light node.
+   * @param database
+   * @returns {Promise<void>}
+   */
+  async cleanupLightNode(database) {
+    if (this.blockNumber % 100 === 0) {
+      await database.cleanupLightNode();
+    }
+  }
+
   // produce the block (deploy a smart contract or execute a smart contract)
   async produceBlock(database, jsVMTimeout, mainBlock) {
+    await this.cleanupLightNode(database);
     await this.blockAdjustments(database);
 
     const nbTransactions = this.transactions.length;

libs/Database.js

@@ -71,6 +71,8 @@ class Database {
     this.session = null;
     this.contractCache = {};
     this.objectCache = {};
+    this.lightNode = false;
+    this.blocksToKeep = 864000; // this only applies if lightNode is true
   }
 
   startSession() {
@@ -132,10 +134,12 @@ class Database {
     });
   }
 
-  async init(databaseURL, databaseName) {
+  async init(databaseURL, databaseName, lightNode = false, blocksToKeep = 864000) {
     // init the database
     this.client = await MongoClient.connect(databaseURL, { useNewUrlParser: true, useUnifiedTopology: true });
     this.database = await this.client.db(databaseName);
+    this.lightNode = lightNode;
+    this.blocksToKeep = blocksToKeep; // this only applies if lightNode is true
     // await database.dropDatabase();
     // return
     // get the chain collection and init the chain if not done yet
@@ -1015,6 +1019,56 @@ class Database {
     await this.updateTableHash(table.split('_')[0], table.split('_')[1]);
     await tableInDb.deleteOne({ _id: record._id }, { session: this.session }); // eslint-disable-line no-underscore-dangle
   }
+
+  /**
+   * Used by light nodes to cleanup (unneeded) blocks / transactions already verified
+   * by witnesses <= lastVerifiedBlockNumber - blocksToKeep
+   * @returns {Promise<void>}
+   */
+  async cleanupLightNode() {
+    if (!this.lightNode) {
+      return;
+    }
+    const params = await this.findOne({ contract: 'witnesses', table: 'params', query: {} });
+    if (params && params.lastVerifiedBlockNumber) {
+      console.log(`cleaning up light node blocks and transactions`);
+      const cleanupUntilBlock = params.lastVerifiedBlockNumber - 1 - this.blocksToKeep;
+      await this.cleanupBlocks(cleanupUntilBlock);
+      await this.cleanupTransactions(cleanupUntilBlock);
+    }
+  }
+
+  /**
+   * Used by light nodes to cleanup (unneeded) blocks already verified
+   * by witnesses <= lastVerifiedBlockNumber - blocksToKeep
+   * @param cleanupUntilBlock cleanup blocks with a smaller blockNumber
+   * @returns {Promise<void>}
+   */
+  async cleanupBlocks(cleanupUntilBlock) {
+    // block 0 is specifically excluded, as the genesis block is also kept by light nodes, due to the condition in
+    // createGenesisBlock in Blockchain.js
+    await this.chain.deleteMany({ $and: [{ _id: { $gt: 0 } }, { _id: { $lte: cleanupUntilBlock } }] }, { session: this.session });
+  }
+
+  /**
+   * Used by light nodes to cleanup (unneeded) transactions
+   * @param cleanupUntilBlock cleanup transactions with a smaller blockNumber
+   * @returns {Promise<void>}
+   */
+  async cleanupTransactions(cleanupUntilBlock) {
+    await this.database.collection('transactions').deleteMany({ blockNumber: { $lte: cleanupUntilBlock } }, { session: this.session });
+  }
+
+  /**
+   * Checks if a node was a light node previously and returns true in case it was. Light nodes
+   * drop block data after a configured number of blocksToKeep, which means that block 1 is not stored
+   * by light nodes, otherwise it would be a full node.
+   * @returns {Promise<boolean>}
+   */
+  async wasLightNodeBefore() {
+    const block = await this.getBlockInfo(1);
+    return !block;
+  }
 }
 
 module.exports.Database = Database;

plugins/Blockchain.js

@@ -166,13 +166,15 @@ const init = async (conf, callback) => {
   const {
     databaseURL,
     databaseName,
+    lightNode,
+    blocksToKeep,
   } = conf;
   javascriptVMTimeout = conf.javascriptVMTimeout; // eslint-disable-line prefer-destructuring
   enableHashVerification = conf.enableHashVerification; // eslint-disable-line prefer-destructuring
   log.setDefaultLevel(conf.defaultLogLevel ? conf.defaultLogLevel : 'warn');
 
   database = new Database();
-  await database.init(databaseURL, databaseName);
+  await database.init(databaseURL, databaseName, lightNode, blocksToKeep);
 
   await createGenesisBlock(conf);