Merge pull request #1990 from ergoplatform/v5.0.12

Candidate for 5.0.12 release

avldb/src/main/scala/scorex/core/serialization/ManifestSerializer.scala

@@ -1,6 +1,5 @@
 package scorex.core.serialization
 
-import com.google.common.primitives.Ints
 import scorex.crypto.authds.avltree.batch.Constants.DigestType
 import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, ProxyInternalNode}
 import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage}
@@ -12,30 +11,37 @@ import scorex.util.serialization.{Reader, Writer}
 class ManifestSerializer(manifestDepth: Byte) extends ErgoSerializer[BatchAVLProverManifest[DigestType]] {
   private val nodeSerializer = VersionedLDBAVLStorage.noStoreSerializer
 
-  override def serialize(manifest: BatchAVLProverManifest[DigestType], w: Writer): Unit = {
-    val height = manifest.rootHeight
-    w.putBytes(Ints.toByteArray(height))
+  /**
+    * Serialize manifest provided as top subtree and height separately. Used in tests.
+    */
+  def serialize(rootNode: ProverNodes[DigestType], rootNodeHeight: Byte, w: Writer): Unit = {
+    w.put(rootNodeHeight)
     w.put(manifestDepth)
 
     def loop(node: ProverNodes[DigestType], level: Int): Unit = {
       nodeSerializer.serialize(node, w)
       node match {
-        case _: ProverLeaf[DigestType] =>
         case n: ProxyInternalNode[DigestType] if n.isEmpty =>
         case i: InternalProverNode[DigestType] if level < manifestDepth =>
           loop(i.left, level + 1)
           loop(i.right, level + 1)
+        case _: InternalProverNode[DigestType] | _: ProverLeaf[DigestType] =>
       }
     }
 
-    loop(manifest.root, level = 1)
+    loop(rootNode, level = 1)
+  }
+
+  override def serialize(manifest: BatchAVLProverManifest[DigestType], w: Writer): Unit = {
+    serialize(manifest.root, manifest.rootHeight.toByte, w)
   }
 
   override def parse(r: Reader): BatchAVLProverManifest[DigestType] = {
-    val rootHeight = Ints.fromByteArray(r.getBytes(4))
+    val rootHeight = r.getByte()
     val manifestDepth = r.getByte()
 
-    require(manifestDepth == this.manifestDepth, "Wrong manifest depth")
+    require(manifestDepth == this.manifestDepth,
+            s"Wrong manifest depth, found: $manifestDepth, expected: ${this.manifestDepth}")
 
     def loop(level: Int): ProverNodes[DigestType] = {
       val node = nodeSerializer.parse(r)

avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala

@@ -1,7 +1,6 @@
 package scorex.crypto.authds.avltree.batch
 
 import com.google.common.primitives.Ints
-import scorex.core.serialization.ManifestSerializer
 import scorex.crypto.authds.avltree.batch.Constants.{DigestType, HashFnType, hashFn}
 import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.{topNodeHashKey, topNodeHeightKey}
 import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree, ProxyInternalNode}
@@ -95,9 +94,10 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore)
     *
     * @param dumpStorage   - non-versioned storage to dump tree to
     * @param manifestDepth - depth of manifest tree
+    * @param expectedRootHash - expected UTXO set authenticating tree root hash
     * @return - hash of root node of tree, or failure if an error (e.g. in database) happened
     */
-  def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Int): Try[Array[Byte]] = {
+  def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Byte, expectedRootHash: Array[Byte]): Try[Array[Byte]] = {
     store.processSnapshot { dbReader =>
 
       def subtreeLoop(label: DigestType, builder: mutable.ArrayBuilder[Byte]): Unit = {
@@ -136,12 +136,14 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore)
       }
 
       val rootNodeLabel = dbReader.get(topNodeHashKey)
-      val rootNodeHeight = Ints.fromByteArray(dbReader.get(topNodeHeightKey))
+      val rootNodeHeight = Ints.fromByteArray(dbReader.get(topNodeHeightKey)).toByte
+
+      require(rootNodeLabel.sameElements(expectedRootHash), "Root node hash changed")
 
       val manifestBuilder = mutable.ArrayBuilder.make[Byte]()
       manifestBuilder.sizeHint(200000)
-      manifestBuilder ++= Ints.toByteArray(rootNodeHeight)
-      manifestBuilder += ManifestSerializer.MainnetManifestDepth
+      manifestBuilder += rootNodeHeight
+      manifestBuilder += manifestDepth
 
       manifestLoop(rootNodeLabel, level = 1, manifestBuilder)
       val manifestBytes = manifestBuilder.result()
@@ -238,4 +240,11 @@ object VersionedLDBAVLStorage {
     ADDigest @@ (rootNodeLabel :+ rootNodeHeight.toByte)
   }
 
+  /**
+    * splits 33-byte digest into 32-bytes hash and 1-byte tree height
+    */
+  def splitDigest(digest: ADDigest): (Array[Byte], Byte) = {
+    digest.dropRight(1) -> digest.last
+  }
+
 }

avldb/src/main/scala/scorex/db/LDBKVStore.scala

@@ -13,14 +13,23 @@ import spire.syntax.all.cfor
   * Both keys and values are var-sized byte arrays.
   */
 class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging {
+  /** Immutable empty array can be shared to avoid allocations. */
+  private val emptyArrayOfByteArray = Array.empty[Array[Byte]]
 
-  def update(toInsert: Array[(K, V)], toRemove: Array[K]): Try[Unit] = {
+  /**
+    * Update this database atomically with a batch of insertion and removal operations
+    *
+    * @param toInsertKeys - keys of key-value pairs to insert into database
+    * @param toInsertValues - values of key-value pairs to insert into database
+    * @param toRemove - keys of key-value pairs to remove from the database
+    * @return - error if it happens, or success status
+    */
+  def update(toInsertKeys: Array[K], toInsertValues: Array[V], toRemove: Array[K]): Try[Unit] = {
     val batch = db.createWriteBatch()
-    val insertLen = toInsert.length
-    val removeLen = toRemove.length
     try {
-      cfor(0)(_ < insertLen, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)}
-      cfor(0)(_ < removeLen, _ + 1) { i => batch.delete(toRemove(i))}
+      require(toInsertKeys.length == toInsertValues.length)
+      cfor(0)(_ < toInsertKeys.length, _ + 1) { i => batch.put(toInsertKeys(i), toInsertValues(i))}
+      cfor(0)(_ < toRemove.length, _ + 1) { i => batch.delete(toRemove(i))}
       db.write(batch)
       Success(())
     } catch {
@@ -45,9 +54,19 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging
     }
   }
 
-  def insert(values: Array[(K, V)]): Try[Unit] = update(values, Array.empty)
+  /**
+    * `update` variant where we only insert values into this database
+    */
+  def insert(keys: Array[K], values: Array[V]): Try[Unit] = {
+    update(keys, values, emptyArrayOfByteArray)
+  }
 
-  def remove(keys: Array[K]): Try[Unit] = update(Array.empty, keys)
+  /**
+    * `update` variant where we only remove values from this database
+    */
+  def remove(keys: Array[K]): Try[Unit] = {
+    update(emptyArrayOfByteArray, emptyArrayOfByteArray, keys)
+  }
 
   /**
     * Get last key within some range (inclusive) by used comparator.

avldb/src/main/scala/scorex/db/LDBVersionedStore.scala

@@ -422,8 +422,11 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int)
     */
   def processSnapshot[T](logic: SnapshotReadInterface => T): Try[T] = {
     val ro = new ReadOptions()
-    ro.snapshot(db.getSnapshot)
     try {
+      lock.writeLock().lock()
+      ro.snapshot(db.getSnapshot)
+      lock.writeLock().unlock()
+
       object readInterface extends SnapshotReadInterface {
         def get(key: Array[Byte]): Array[Byte] = db.get(key, ro)
       }

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala

@@ -17,7 +17,6 @@ class LDBVersionedStoreSpecification extends AnyPropSpec
 
   override protected val KL = 32
   override protected val VL = 8
-  override protected val LL = 32
 
   val storeTest: LDBVersionedStore => Unit = { store =>
     var version = store.lastVersionID

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala

@@ -6,26 +6,29 @@ import org.scalatest.Assertion
 import org.scalatest.matchers.should.Matchers
 import org.scalatest.propspec.AnyPropSpec
 import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks
+import scorex.core.serialization.{ManifestSerializer, SubtreeSerializer}
 import scorex.crypto.authds.avltree.batch.helpers.TestHelper
 import scorex.crypto.authds.{ADDigest, ADKey, ADValue, SerializedAdProof}
 import scorex.util.encode.Base16
 import scorex.crypto.hash.{Blake2b256, Digest32}
-import scorex.db.LDBVersionedStore
+import scorex.db.{LDBFactory, LDBVersionedStore}
+import scorex.util.ByteArrayBuilder
+import scorex.util.serialization.VLQByteBufferWriter
 import scorex.utils.{Random => RandomBytes}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
 import scala.concurrent.{Await, Future}
 import scala.util.{Success, Try}
 
-class VersionedLDBAVLStorageSpecification extends AnyPropSpec
-  with ScalaCheckPropertyChecks
-  with Matchers
-  with TestHelper {
+class VersionedLDBAVLStorageSpecification
+  extends AnyPropSpec
+    with ScalaCheckPropertyChecks
+    with Matchers
+    with TestHelper {
 
   override protected val KL = 32
   override protected val VL = 8
-  override protected val LL = 32
 
   def kvGen: Gen[(ADKey, ADValue)] = for {
     key <- Gen.listOfN(KL, Arbitrary.arbitrary[Byte]).map(_.toArray) suchThat
@@ -340,4 +343,31 @@ class VersionedLDBAVLStorageSpecification extends AnyPropSpec
     testAddInfoSaving(createVersionedStore _)
   }
 
+  property("dumping snapshot") {
+    val manifestDepth: Byte = 6
+    val manifestSerializer = new ManifestSerializer(manifestDepth)
+    val prover = createPersistentProver()
+    blockchainWorkflowTest(prover)
+
+    val storage = prover.storage.asInstanceOf[VersionedLDBAVLStorage]
+    val store = LDBFactory.createKvDb(getRandomTempDir.getAbsolutePath)
+
+    val rootNodeLabel = storage.dumpSnapshot(store, manifestDepth, prover.digest.dropRight(1)).get
+    rootNodeLabel.sameElements(prover.digest.dropRight(1)) shouldBe true
+    val manifestBytes = store.get(rootNodeLabel).get
+    val manifest = manifestSerializer.parseBytesTry(manifestBytes).get
+
+    val writer = new VLQByteBufferWriter(new ByteArrayBuilder())
+    manifestSerializer.serialize(prover.prover().topNode, prover.prover().rootNodeHeight.toByte, writer)
+    val altManifestBytes = writer.result().toBytes
+
+    manifestBytes.sameElements(altManifestBytes) shouldBe true
+
+    val subtreeIds = manifest.subtreesIds
+    subtreeIds.foreach { sid =>
+      val chunkBytes = store.get(sid).get
+      SubtreeSerializer.parseBytesTry(chunkBytes).get.id.sameElements(sid) shouldBe true
+    }
+  }
+
 }

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala

@@ -27,7 +27,6 @@ object WithLDB extends VersionedLDBAVLStorageStatefulCommands with TestHelper {
 
   override protected val KL = 32
   override protected val VL = 8
-  override protected val LL = 32
 
   override protected def createStatefulProver: PersistentBatchAVLProver[Digest32, HF] = {
     createPersistentProver(keepVersions)

avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala

@@ -19,7 +19,6 @@ trait TestHelper extends FileHelper {
 
   protected val KL: Int
   protected val VL: Int
-  protected val LL: Int
 
   implicit val hf: HF = Blake2b256
 
@@ -28,7 +27,8 @@ trait TestHelper extends FileHelper {
     new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions)
   }
 
-  def createVersionedStorage(store: LDBVersionedStore): STORAGE = new VersionedLDBAVLStorage(store)
+  def createVersionedStorage(store: LDBVersionedStore): STORAGE =
+    new VersionedLDBAVLStorage(store)
 
   def createPersistentProver(storage: STORAGE): PERSISTENT_PROVER = {
     val prover = new BatchAVLProver[D, HF](KL, Some(VL))

benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala

@@ -39,7 +39,7 @@ object LDBStoreBench
   val txsWithDbGen: Gen[(Seq[BlockTransactions], LDBKVStore)] = txsGen.map { bts =>
     val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray
     val db = storeLDB()
-    toInsert.grouped(5).foreach(db.insert(_).get)
+    toInsert.grouped(5).foreach(kv => db.insert(kv.map(_._1), kv.map(_._2)).get)
     bts -> storeLDB
   }
 
@@ -55,7 +55,7 @@ object LDBStoreBench
   private def benchWriteLDB(bts: Seq[BlockTransactions]): Unit = {
     val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray
     val db = storeLDB()
-    toInsert.grouped(5).foreach(db.insert(_).get)
+    toInsert.grouped(5).foreach(kv => db.insert(kv.map(_._1), kv.map(_._2)).get)
   }
 
   private def benchReadLDB(bts: Seq[BlockTransactions], db: LDBKVStore): Unit = {

benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala

@@ -22,7 +22,7 @@ object UtxoStateBenchmark extends HistoryTestHelpers with NVBenchmark {
     val transactionsQty = blocks.flatMap(_.transactions).size
 
     def bench(mods: Seq[BlockSection]): Long = {
-      val state = ErgoState.generateGenesisUtxoState(createTempDir, StateConstants(realNetworkSetting))._1
+      val state = ErgoState.generateGenesisUtxoState(createTempDir, realNetworkSetting)._1
       Utils.time {
         mods.foldLeft(state) { case (st, mod) =>
           st.applyModifier(mod, None)(_ => ()).get

build.sbt

@@ -37,7 +37,7 @@ val circeVersion = "0.13.0"
 val akkaVersion = "2.6.10"
 val akkaHttpVersion = "10.2.4"
 
-val sigmaStateVersion = "5.0.7"
+val sigmaStateVersion = "5.0.8"
 
 // for testing current sigmastate build (see sigmastate-ergo-it jenkins job)
 val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion)

ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala

@@ -1,13 +1,14 @@
 package org.ergoplatform.contracts
 
 import org.ergoplatform.ErgoBox.{R2, STokensRegType}
-import org.ergoplatform.ErgoScriptPredef.{boxCreationHeight, expectedMinerOutScriptBytesVal}
-import org.ergoplatform.{Height, MinerPubkey, Outputs, Self}
+import org.ergoplatform.ErgoTreePredef.{boxCreationHeight, expectedMinerOutScriptBytesVal}
+import org.ergoplatform.mining.emission.EmissionRules.CoinsInOneErgo
 import org.ergoplatform.settings.MonetarySettings
-import sigmastate.{AND, EQ, GE, GT, LE, Minus, OR, SBox, SCollection, STuple}
+import org.ergoplatform.{Height, MinerPubkey, Outputs, Self}
 import sigmastate.Values.{ByteArrayConstant, ErgoTree, IntConstant, LongConstant, SigmaPropValue, Value}
-import sigmastate.utxo.{ByIndex, ExtractAmount, ExtractRegisterAs, ExtractScriptBytes, OptionGet, SelectField, SizeOf}
-import org.ergoplatform.mining.emission.EmissionRules.CoinsInOneErgo
+import sigmastate.utxo._
+import sigmastate._
+import special.collection.Coll
 
 /**
   * Container for re-emission related contracts. Contains re-emission contract and pay-to-reemission contract.
@@ -22,7 +23,7 @@ trait ReemissionContracts {
   /**
     * @return - ID of NFT token associated with re-emission contract
     */
-  def reemissionNftIdBytes: Array[Byte]
+  def reemissionNftIdBytes: Coll[Byte]
 
   /**
     * @return - height when reemission starts