Nipopow bootstrapping from multiple peers

src/main/resources/application.conf

@@ -34,12 +34,18 @@ ergo {
         p2pUtxoSnapshots = 2
     }
 
+    # settings related to headers-chain bootstrapping via NiPoPoWs
+    nipopow {
+        # Download PoPoW proof on node bootstrap
+        nipopowBootstrap = false
 
-    # Download PoPoW proof on node bootstrap
-    popowBootstrap = false
+        # how many different proofs we are downloading from other peers
+        # and comparing with each other, before choosing best one
+        p2pNipopows = 1
 
-    # Minimal suffix size for PoPoW proof (may be pre-defined constant or settings parameter)
-    popowSuffix = 10
+        # Minimal suffix size for NiPoPoW proof (may be pre-defined constant or settings parameter)
+        nipopowSuffix = 10
+    }
 
     # Is the node is doing mining
     mining = false
@@ -246,6 +252,18 @@ ergo {
       version2ActivationDifficultyHex = "6f98d5000000"
     }
 
+    # UTXO set snapshot to be taken every this number of blocks.
+    # The value MUST be divisible by voting epoch length (chainSettings.voting.votingLength),
+    # so after snapshot the state is corresponding to a moment before applying first block of a voting epoch,
+    # and then the first block sets current validation parameters.
+    #
+    # So for the Ergo mainnet the value should be divisible by 1024 (for testnet, 128, any number divisible by
+    # 1024 is divisible by 128 also.
+    #
+    # Not consensus-critical setting, but located here, in chain settings section, as all the nodes in the network
+    # should have the same value.
+    makeSnapshotEvery = 52224
+
     # Defines an id of the genesis block. Other genesis blocks will be considered invalid.
     # genesisId = "ab19bb59871e86507defb9a7769841b1130aad4d8c1ea8b0e01e0dee9e97a27e"
   }

src/main/resources/mainnet.conf

@@ -91,6 +91,20 @@ ergo {
         # in order to start downloading it
         p2pUtxoSnapshots = 2
     }
+
+    # settings related to headers-chain bootstrapping via NiPoPoWs
+    nipopow {
+        # Download PoPoW proof on node bootstrap
+        nipopowBootstrap = false
+
+        # how many different proofs we are downloading from other peers
+        # and comparing with each other, before choosing best one
+        p2pNipopows = 2
+
+        # Minimal suffix size for NiPoPoW proof (may be pre-defined constant or settings parameter)
+        nipopowSuffix = 10
+    }
+
   }
 
 }

src/main/scala/org/ergoplatform/local/NipopowVerificationResult.scala

@@ -6,15 +6,25 @@ package org.ergoplatform.local
   */
 sealed trait NipopowProofVerificationResult
 
+/**
+  * Basic interface for valid proof verification results
+  */
+sealed trait CorrectNipopowProofVerificationResult extends NipopowProofVerificationResult {
+  /**
+    * Total number of known valid proofs verified, including this one
+    */
+  val totalProofsProcessed: Int
+}
+
 /**
   * Presented nipopow proof is better than known one
   */
-case object BetterChain extends NipopowProofVerificationResult
+case class BetterChain(override val totalProofsProcessed: Int) extends CorrectNipopowProofVerificationResult
 
 /**
   * Presented nipopow proof is no better than known one
   */
-case object NoBetterChain extends NipopowProofVerificationResult
+case class NoBetterChain(override val totalProofsProcessed: Int) extends CorrectNipopowProofVerificationResult
 
 /**
   * Presented nipopow proof is not valid

src/main/scala/org/ergoplatform/local/NipopowVerifier.scala

@@ -16,6 +16,7 @@ import scorex.util.ModifierId
 class NipopowVerifier(genesisIdOpt: Option[ModifierId]) {
 
   private var bestProofOpt: Option[NipopowProof] = None
+  private var proofsProcessed = 0
 
   def bestChain: Seq[Header] = bestProofOpt.synchronized {
     bestProofOpt.map(_.headersChain).getOrElse(Seq())
@@ -32,14 +33,21 @@ class NipopowVerifier(genesisIdOpt: Option[ModifierId]) {
         case Some(bestProof) =>
           if (newProof.isBetterThan(bestProof)) {
             bestProofOpt = Some(newProof)
-            BetterChain
+            proofsProcessed += 1
+            BetterChain(proofsProcessed)
           } else {
-            NoBetterChain
+            if (newProof.isValid) {
+              proofsProcessed += 1
+              NoBetterChain(proofsProcessed)
+            } else {
+              ValidationError
+            }
           }
         case None =>
           if (newProof.isValid) {
             bestProofOpt = Some(newProof)
-            BetterChain
+            proofsProcessed += 1
+            BetterChain(proofsProcessed)
           } else {
             ValidationError
           }

src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala

@@ -174,6 +174,11 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     */
   private val availableManifests = mutable.Map[ModifierId, (Height, Seq[ConnectedPeer])]()
 
+  /**
+    * Peers provided nipopow poofs
+    */
+  private val nipopowProviders = mutable.Set[ConnectedPeer]()
+
   /**
     * How many peers should have a utxo set snapshot to start downloading it
     */
@@ -335,7 +340,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     *
     */
   protected def sendSync(history: ErgoHistory): Unit = {
-    if (history.bestHeaderOpt.isEmpty && settings.nodeSettings.popowBootstrap) {
+    if (history.bestHeaderOpt.isEmpty && settings.nodeSettings.nipopowSettings.nipopowBootstrap) {
       // if no any header applied yet, and boostrapping via nipopows is ordered,
       // ask for nipopow proofs instead of sending sync signal
       requireNipopowProof(history)
@@ -918,7 +923,8 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
           log.warn(s"Double manifest declaration for $manifestId from $remote")
         }
       } else {
-        log.error(s"Got wrong manifest id $encodedManifestId from $remote")
+        log.error(s"Got wrong manifest id $encodedManifestId from $remote for height $height, " +
+                  s"their id: ${Algos.encode(manifestId)}, our id ${ownId.map(Algos.encode)}")
       }
     }
     checkUtxoSetManifests(hr) // check if we got enough manifests for the height to download manifests and chunks
@@ -1022,8 +1028,13 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     val m = hr.P2PNipopowProofM
     val k = hr.P2PNipopowProofK
     val msg = Message(GetNipopowProofSpec, Right(NipopowProofData(m, k, None)), None)
-    val peers = NipopowSupportFilter.filter(syncTracker.knownPeers()).toSeq
-    networkControllerRef ! SendToNetwork(msg, SendToPeers(peers))
+    val knownPeers = syncTracker.knownPeers()
+    val peers = NipopowSupportFilter.filter(knownPeers).filter(cp => !nipopowProviders.contains(cp)).toSeq
+    if (peers.nonEmpty) {
+      networkControllerRef ! SendToNetwork(msg, SendToPeers(peers))
+    } else {
+      log.warn("No peers to ask NiPoPoWs from")
+    }
   }
 
   /**
@@ -1049,13 +1060,18 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef,
     */
   private def processNipopowProof(proofBytes: Array[Byte], hr: ErgoHistory, peer: ConnectedPeer): Unit = {
     if (hr.bestHeaderOpt.isEmpty) {
-      hr.nipopowSerializer.parseBytesTry(proofBytes) match {
-        case Success(proof) if proof.isValid =>
-          log.info(s"Got valid nipopow proof, size: ${proofBytes.length}")
-          viewHolderRef ! InitHistoryFromNipopow(proof)
-        case _ =>
-          log.warn(s"Peer $peer sent wrong nipopow")
-          penalizeMisbehavingPeer(peer)
+      if(!nipopowProviders.contains(peer)) {
+        nipopowProviders += peer
+        hr.nipopowSerializer.parseBytesTry(proofBytes) match {
+          case Success(proof) if proof.isValid =>
+            log.info(s"Got valid nipopow proof, size: ${proofBytes.length}")
+            viewHolderRef ! ProcessNipopow(proof)
+          case _ =>
+            log.warn(s"Peer $peer sent wrong nipopow")
+            penalizeMisbehavingPeer(peer)
+        }
+      } else {
+        log.info(s"Received Nipopow proof from $peer again")
       }
     } else {
       log.warn("Got nipopow proof, but it is already applied")
@@ -1721,10 +1737,11 @@ object ErgoNodeViewSynchronizer {
     case class InitStateFromSnapshot(blockHeight: Height, blockId: ModifierId)
 
     /**
-      * Signal for a central node view holder component to initialize headers chain from NiPoPoW proof
+      * Command for a central node view holder component to process NiPoPoW proof,
+      * and possibly initialize headers chain from a best NiPoPoW proof known, when enough proofs collected
       * @param nipopowProof - proof to initialize history from
       */
-    case class InitHistoryFromNipopow(nipopowProof: NipopowProof)
+    case class ProcessNipopow(nipopowProof: NipopowProof)
   }
 
 }

src/main/scala/org/ergoplatform/network/ModePeerFeature.scala

@@ -15,12 +15,12 @@ import scorex.util.serialization.{Reader, Writer}
   *
   * @param stateType - information on whether UTXO set is store (so state type is UTXO/Digest)
   * @param verifyingTransactions - whether the peer is verifying transactions
-  * @param popowSuffix - whether the peer has has bootstrapped via PoPoW suffix, and its length
+  * @param nipopowSuffix - whether the peer has has bootstrapped via Nipopows, and length of proof suffix
   * @param blocksToKeep - how many last full blocks the peer is storing
   */
 case class ModePeerFeature(stateType: StateType,
                            verifyingTransactions: Boolean,
-                           popowSuffix: Option[Int],
+                           nipopowSuffix: Option[Int],
                            blocksToKeep: Int) extends PeerFeature {
   override type M = ModePeerFeature
 
@@ -34,7 +34,11 @@ object ModePeerFeature {
   import io.circe.syntax._
 
   def apply(nodeSettings: NodeConfigurationSettings): ModePeerFeature = {
-    val popowSuffix = if (nodeSettings.popowBootstrap) Some(nodeSettings.popowSuffix) else None
+    val popowSuffix = if (nodeSettings.nipopowSettings.nipopowBootstrap) {
+      Some(nodeSettings.nipopowSettings.nipopowSuffix)
+    } else {
+      None
+    }
 
     new ModePeerFeature(
       nodeSettings.stateType,
@@ -72,7 +76,7 @@ object ModeFeatureSerializer extends ErgoSerializer[ModePeerFeature] {
   override def serialize(mf: ModePeerFeature, w: Writer): Unit = {
     w.put(mf.stateType.stateTypeCode)
     w.put(booleanToByte(mf.verifyingTransactions))
-    w.putOption(mf.popowSuffix)(_.putInt(_))
+    w.putOption(mf.nipopowSuffix)(_.putInt(_))
     w.putInt(mf.blocksToKeep) // todo: put -2 if bootstrapped via utxo set snapshot? https://github.com/ergoplatform/ergo/issues/2014
   }
 

src/main/scala/org/ergoplatform/network/VersionBasedPeerFilteringRule.scala

@@ -125,7 +125,7 @@ object NipopowSupportFilter extends PeerFilteringRule {
   override def condition(peer: ConnectedPeer): Boolean = {
     val version = peer.peerInfo.map(_.peerSpec.protocolVersion).getOrElse(Version.initial)
 
-    peer.mode.flatMap(_.popowSuffix).isEmpty &&
+    peer.mode.flatMap(_.nipopowSuffix).isEmpty &&
       version.compare(Version.NipopowActivationVersion) >= 0
   }
 

src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala

@@ -296,11 +296,13 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti
       } else {
         log.warn("InitStateFromSnapshot arrived when state already initialized")
       }
-    // Initialize headers-chain via NiPoPoW-proof
-    case InitHistoryFromNipopow(proof) =>
+    // Process NiPoPoW proof, initialize history if enough proofs collected
+    case ProcessNipopow(proof) =>
       if (history().isEmpty) {
         history().applyPopowProof(proof)
-        updateNodeView(updatedHistory = Some(history()))
+        if (!history().isEmpty) {
+          updateNodeView(updatedHistory = Some(history()))
+        }
       }
   }
 

src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala

@@ -314,6 +314,23 @@ object ErgoHistory extends ScorexLogging {
 
     repairIfNeeded(history)
 
+    // temporary hack which is injecting nipopow proof to the database to make it possible to bootstrap with
+    // nipopows + utxo set snapshot soon after 5.0.13 release
+    // todo: remove after height 1,096,693 on the mainnet
+    val bestHeaderHeight = history.headersHeight
+    if (bestHeaderHeight > 1054000 && bestHeaderHeight < 1096693) { //history.readPopowProofBytesFromDb().isDefined){
+      history.popowProofBytes(
+        history.P2PNipopowProofM,
+        history.P2PNipopowProofK,
+        Some(ModifierId @@ "25a11667e38e62412522c062d90b073afd9ed9551080ff4e0a67d1757ce18b98")) match {
+        case Success(proofBytes) =>
+          log.info("Writing nipopow proof bytes for height 1,044,469")
+          db.insert(Array(history.NipopowSnapshotHeightKey -> proofBytes), Array.empty[BlockSection])
+        case Failure(e) =>
+          log.warn("Can't dump NiPoPoW proof bytes for height 1,044,469", e)
+      }
+    }
+
     log.info("History database read")
     if(ergoSettings.nodeSettings.extraIndex) // start extra indexer, if enabled
       context.system.eventStream.publish(StartExtraIndexer(history))

src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala

@@ -8,7 +8,7 @@ import org.ergoplatform.nodeView.history.ErgoHistory.Height
 import org.ergoplatform.nodeView.history.extra.ExtraIndex
 import org.ergoplatform.nodeView.history.storage._
 import org.ergoplatform.nodeView.history.storage.modifierprocessors._
-import org.ergoplatform.settings.ErgoSettings
+import org.ergoplatform.settings.{ErgoSettings, NipopowSettings}
 import scorex.core.NodeViewComponent
 import scorex.core.consensus.{ContainsModifiers, Equal, Fork, ModifierSemanticValidity, Older, PeerChainStatus, Unknown, Younger}
 import scorex.core.utils.ScorexEncoding
@@ -36,6 +36,8 @@ trait ErgoHistoryReader
 
   protected val settings: ErgoSettings
 
+  override protected def nipopowSettings: NipopowSettings = settings.nodeSettings.nipopowSettings
+
   private val Valid = 1.toByte
   private val Invalid = 0.toByte
 

src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala

@@ -11,7 +11,7 @@ import org.ergoplatform.modifiers.history.header.Header
 import org.ergoplatform.nodeView.history.ErgoHistory
 import org.ergoplatform.nodeView.history.ErgoHistory.{Difficulty, GenesisHeight, Height}
 import org.ergoplatform.nodeView.history.storage.HistoryStorage
-import org.ergoplatform.settings.Constants.{HashLength, MakeSnapshotEvery}
+import org.ergoplatform.settings.Constants.HashLength
 import org.ergoplatform.settings.ValidationRules._
 import org.ergoplatform.settings._
 import scorex.core.consensus.ProgressInfo
@@ -165,13 +165,14 @@ trait HeadersProcessor extends ToDownloadProcessor with PopowProcessor with Scor
     // e.g. on a height h , where h % MakeSnapshotEvery == MakeSnapshotEvery - 1
     // and NiPoPoW proof is taken Constants.LastHeadersInContext before to have Constants.LastHeadersInContext
     // (actually, Constants.LastHeadersInContext + 1 even) consecutive headers before first full block to be validated
-    val timeToTakeNipopowProof: Boolean = if (settings.nodeSettings.popowBootstrap) {
+    val timeToTakeNipopowProof: Boolean = if (settings.nodeSettings.nipopowSettings.nipopowBootstrap) {
       // currently, the node is not taking NiPoPoW proof if was bootstrapped via a NiPoPoW itself,
       // as no extension sections (with interlinks) available for headers downloaded via nipopows
       false
     } else  {
-     this.isHeadersChainSynced &&
-        h.height % MakeSnapshotEvery == MakeSnapshotEvery - 1 - Constants.LastHeadersInContext
+      val makeSnapshotEvery = chainSettings.makeSnapshotEvery
+      this.isHeadersChainSynced &&
+        h.height % makeSnapshotEvery == makeSnapshotEvery - 1 - Constants.LastHeadersInContext
     }
 
     val nipopowsRow: Seq[(ByteArrayWrapper, Array[Byte])] = if (timeToTakeNipopowProof) {

src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/PopowProcessor.scala

@@ -1,13 +1,13 @@
 package org.ergoplatform.nodeView.history.storage.modifierprocessors
 
-import org.ergoplatform.local.{BetterChain, NipopowProofVerificationResult, NipopowVerifier}
+import org.ergoplatform.local.{CorrectNipopowProofVerificationResult, NipopowProofVerificationResult, NipopowVerifier}
 import org.ergoplatform.modifiers.BlockSection
 import org.ergoplatform.modifiers.history.extension.Extension
 import org.ergoplatform.modifiers.history.header.Header
 import org.ergoplatform.modifiers.history.popow.{NipopowAlgos, NipopowProof, NipopowProofSerializer, PoPowHeader, PoPowParams}
 import org.ergoplatform.nodeView.history.ErgoHistory.GenesisHeight
 import org.ergoplatform.nodeView.history.ErgoHistoryReader
-import org.ergoplatform.settings.ChainSettings
+import org.ergoplatform.settings.{ChainSettings, NipopowSettings}
 import org.ergoplatform.settings.Constants.HashLength
 import scorex.core.consensus.ProgressInfo
 import scorex.db.ByteArrayWrapper
@@ -31,6 +31,8 @@ trait PopowProcessor extends BasicReaders with ScorexLogging {
     */
   protected def chainSettings: ChainSettings
 
+  protected def nipopowSettings: NipopowSettings
+
   private lazy val nipopowAlgos: NipopowAlgos = new NipopowAlgos(chainSettings)
 
   /**
@@ -135,15 +137,19 @@ trait PopowProcessor extends BasicReaders with ScorexLogging {
     */
   def applyPopowProof(proof: NipopowProof): Unit = {
     nipopowVerifier.process(proof) match {
-      case BetterChain =>
-        val headersToApply = nipopowVerifier.bestChain.sortBy(_.height) // sorting could be an overkill, but anyway
-        headersToApply.foreach { h =>
-          if (!historyReader.contains(h.id)) {
-            process(h, nipopowMode = true)
+      case res: CorrectNipopowProofVerificationResult =>
+        if (res.totalProofsProcessed >= nipopowSettings.p2pNipopows) {
+          val headersToApply = nipopowVerifier.bestChain.sortBy(_.height) // sorting could be an overkill, but anyway
+          headersToApply.foreach { h =>
+            if (!historyReader.contains(h.id)) {
+              process(h, nipopowMode = true)
+            }
           }
+          nipopowVerifier.reset()
+          log.info(s"Nipopow proof applied, best header now is ${historyReader.bestHeaderOpt}")
+        } else {
+          log.info(s"Processed ${res.totalProofsProcessed} NiPoPoW proofs, quorum needed: ${nipopowSettings.p2pNipopows}")
         }
-        nipopowVerifier.reset()
-        log.info(s"Nipopow proof applied, best header now is ${historyReader.bestHeaderOpt}")
       case r: NipopowProofVerificationResult =>
         log.warn(s"NiPoPoW proof is no better or invalid ($r): $proof")
     }