diff --git a/avldb/src/main/scala/scorex/core/serialization/ManifestSerializer.scala b/avldb/src/main/scala/scorex/core/serialization/ManifestSerializer.scala index 2c4bac6397..70bced053d 100644 --- a/avldb/src/main/scala/scorex/core/serialization/ManifestSerializer.scala +++ b/avldb/src/main/scala/scorex/core/serialization/ManifestSerializer.scala @@ -1,6 +1,5 @@ package scorex.core.serialization -import com.google.common.primitives.Ints import scorex.crypto.authds.avltree.batch.Constants.DigestType import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, ProxyInternalNode} import scorex.crypto.authds.avltree.batch.{InternalProverNode, ProverLeaf, ProverNodes, VersionedLDBAVLStorage} @@ -12,30 +11,37 @@ import scorex.util.serialization.{Reader, Writer} class ManifestSerializer(manifestDepth: Byte) extends ErgoSerializer[BatchAVLProverManifest[DigestType]] { private val nodeSerializer = VersionedLDBAVLStorage.noStoreSerializer - override def serialize(manifest: BatchAVLProverManifest[DigestType], w: Writer): Unit = { - val height = manifest.rootHeight - w.putBytes(Ints.toByteArray(height)) + /** + * Serialize manifest provided as top subtree and height separately. Used in tests. + */ + def serialize(rootNode: ProverNodes[DigestType], rootNodeHeight: Byte, w: Writer): Unit = { + w.put(rootNodeHeight) w.put(manifestDepth) def loop(node: ProverNodes[DigestType], level: Int): Unit = { nodeSerializer.serialize(node, w) node match { - case _: ProverLeaf[DigestType] => case n: ProxyInternalNode[DigestType] if n.isEmpty => case i: InternalProverNode[DigestType] if level < manifestDepth => loop(i.left, level + 1) loop(i.right, level + 1) + case _: InternalProverNode[DigestType] | _: ProverLeaf[DigestType] => } } - loop(manifest.root, level = 1) + loop(rootNode, level = 1) + } + + override def serialize(manifest: BatchAVLProverManifest[DigestType], w: Writer): Unit = { + serialize(manifest.root, manifest.rootHeight.toByte, w) } override def parse(r: Reader): BatchAVLProverManifest[DigestType] = { - val rootHeight = Ints.fromByteArray(r.getBytes(4)) + val rootHeight = r.getByte() val manifestDepth = r.getByte() - require(manifestDepth == this.manifestDepth, "Wrong manifest depth") + require(manifestDepth == this.manifestDepth, + s"Wrong manifest depth, found: $manifestDepth, expected: ${this.manifestDepth}") def loop(level: Int): ProverNodes[DigestType] = { val node = nodeSerializer.parse(r) diff --git a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala index 76c7fef556..b7e5eb3512 100644 --- a/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala +++ b/avldb/src/main/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorage.scala @@ -1,7 +1,6 @@ package scorex.crypto.authds.avltree.batch import com.google.common.primitives.Ints -import scorex.core.serialization.ManifestSerializer import scorex.crypto.authds.avltree.batch.Constants.{DigestType, HashFnType, hashFn} import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.{topNodeHashKey, topNodeHeightKey} import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree, ProxyInternalNode} @@ -95,9 +94,10 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore) * * @param dumpStorage - non-versioned storage to dump tree to * @param manifestDepth - depth of manifest tree + * @param expectedRootHash - expected UTXO set authenticating tree root hash * @return - hash of root node of tree, or failure if an error (e.g. in database) happened */ - def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Int): Try[Array[Byte]] = { + def dumpSnapshot(dumpStorage: LDBKVStore, manifestDepth: Byte, expectedRootHash: Array[Byte]): Try[Array[Byte]] = { store.processSnapshot { dbReader => def subtreeLoop(label: DigestType, builder: mutable.ArrayBuilder[Byte]): Unit = { @@ -136,12 +136,14 @@ class VersionedLDBAVLStorage(store: LDBVersionedStore) } val rootNodeLabel = dbReader.get(topNodeHashKey) - val rootNodeHeight = Ints.fromByteArray(dbReader.get(topNodeHeightKey)) + val rootNodeHeight = Ints.fromByteArray(dbReader.get(topNodeHeightKey)).toByte + + require(rootNodeLabel.sameElements(expectedRootHash), "Root node hash changed") val manifestBuilder = mutable.ArrayBuilder.make[Byte]() manifestBuilder.sizeHint(200000) - manifestBuilder ++= Ints.toByteArray(rootNodeHeight) - manifestBuilder += ManifestSerializer.MainnetManifestDepth + manifestBuilder += rootNodeHeight + manifestBuilder += manifestDepth manifestLoop(rootNodeLabel, level = 1, manifestBuilder) val manifestBytes = manifestBuilder.result() @@ -238,4 +240,11 @@ object VersionedLDBAVLStorage { ADDigest @@ (rootNodeLabel :+ rootNodeHeight.toByte) } + /** + * splits 33-byte digest into 32-bytes hash and 1-byte tree height + */ + def splitDigest(digest: ADDigest): (Array[Byte], Byte) = { + digest.dropRight(1) -> digest.last + } + } diff --git a/avldb/src/main/scala/scorex/db/LDBKVStore.scala b/avldb/src/main/scala/scorex/db/LDBKVStore.scala index ffd53609db..035e43510a 100644 --- a/avldb/src/main/scala/scorex/db/LDBKVStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBKVStore.scala @@ -13,14 +13,23 @@ import spire.syntax.all.cfor * Both keys and values are var-sized byte arrays. */ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging { + /** Immutable empty array can be shared to avoid allocations. */ + private val emptyArrayOfByteArray = Array.empty[Array[Byte]] - def update(toInsert: Array[(K, V)], toRemove: Array[K]): Try[Unit] = { + /** + * Update this database atomically with a batch of insertion and removal operations + * + * @param toInsertKeys - keys of key-value pairs to insert into database + * @param toInsertValues - values of key-value pairs to insert into database + * @param toRemove - keys of key-value pairs to remove from the database + * @return - error if it happens, or success status + */ + def update(toInsertKeys: Array[K], toInsertValues: Array[V], toRemove: Array[K]): Try[Unit] = { val batch = db.createWriteBatch() - val insertLen = toInsert.length - val removeLen = toRemove.length try { - cfor(0)(_ < insertLen, _ + 1) { i => batch.put(toInsert(i)._1, toInsert(i)._2)} - cfor(0)(_ < removeLen, _ + 1) { i => batch.delete(toRemove(i))} + require(toInsertKeys.length == toInsertValues.length) + cfor(0)(_ < toInsertKeys.length, _ + 1) { i => batch.put(toInsertKeys(i), toInsertValues(i))} + cfor(0)(_ < toRemove.length, _ + 1) { i => batch.delete(toRemove(i))} db.write(batch) Success(()) } catch { @@ -45,9 +54,19 @@ class LDBKVStore(protected val db: DB) extends KVStoreReader with ScorexLogging } } - def insert(values: Array[(K, V)]): Try[Unit] = update(values, Array.empty) + /** + * `update` variant where we only insert values into this database + */ + def insert(keys: Array[K], values: Array[V]): Try[Unit] = { + update(keys, values, emptyArrayOfByteArray) + } - def remove(keys: Array[K]): Try[Unit] = update(Array.empty, keys) + /** + * `update` variant where we only remove values from this database + */ + def remove(keys: Array[K]): Try[Unit] = { + update(emptyArrayOfByteArray, emptyArrayOfByteArray, keys) + } /** * Get last key within some range (inclusive) by used comparator. diff --git a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala index ff57dad668..51340723f9 100644 --- a/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala +++ b/avldb/src/main/scala/scorex/db/LDBVersionedStore.scala @@ -422,8 +422,11 @@ class LDBVersionedStore(protected val dir: File, val initialKeepVersions: Int) */ def processSnapshot[T](logic: SnapshotReadInterface => T): Try[T] = { val ro = new ReadOptions() - ro.snapshot(db.getSnapshot) try { + lock.writeLock().lock() + ro.snapshot(db.getSnapshot) + lock.writeLock().unlock() + object readInterface extends SnapshotReadInterface { def get(key: Array[Byte]): Array[Byte] = db.get(key, ro) } diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala index f3236cfff3..c1fe98d5df 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/LDBVersionedStoreSpecification.scala @@ -17,7 +17,6 @@ class LDBVersionedStoreSpecification extends AnyPropSpec override protected val KL = 32 override protected val VL = 8 - override protected val LL = 32 val storeTest: LDBVersionedStore => Unit = { store => var version = store.lastVersionID diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala index 443d42fd26..1c72d26b84 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageSpecification.scala @@ -6,11 +6,14 @@ import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks +import scorex.core.serialization.{ManifestSerializer, SubtreeSerializer} import scorex.crypto.authds.avltree.batch.helpers.TestHelper import scorex.crypto.authds.{ADDigest, ADKey, ADValue, SerializedAdProof} import scorex.util.encode.Base16 import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.LDBVersionedStore +import scorex.db.{LDBFactory, LDBVersionedStore} +import scorex.util.ByteArrayBuilder +import scorex.util.serialization.VLQByteBufferWriter import scorex.utils.{Random => RandomBytes} import scala.concurrent.ExecutionContext.Implicits.global @@ -18,14 +21,14 @@ import scala.concurrent.duration._ import scala.concurrent.{Await, Future} import scala.util.{Success, Try} -class VersionedLDBAVLStorageSpecification extends AnyPropSpec - with ScalaCheckPropertyChecks - with Matchers - with TestHelper { +class VersionedLDBAVLStorageSpecification + extends AnyPropSpec + with ScalaCheckPropertyChecks + with Matchers + with TestHelper { override protected val KL = 32 override protected val VL = 8 - override protected val LL = 32 def kvGen: Gen[(ADKey, ADValue)] = for { key <- Gen.listOfN(KL, Arbitrary.arbitrary[Byte]).map(_.toArray) suchThat @@ -340,4 +343,31 @@ class VersionedLDBAVLStorageSpecification extends AnyPropSpec testAddInfoSaving(createVersionedStore _) } + property("dumping snapshot") { + val manifestDepth: Byte = 6 + val manifestSerializer = new ManifestSerializer(manifestDepth) + val prover = createPersistentProver() + blockchainWorkflowTest(prover) + + val storage = prover.storage.asInstanceOf[VersionedLDBAVLStorage] + val store = LDBFactory.createKvDb(getRandomTempDir.getAbsolutePath) + + val rootNodeLabel = storage.dumpSnapshot(store, manifestDepth, prover.digest.dropRight(1)).get + rootNodeLabel.sameElements(prover.digest.dropRight(1)) shouldBe true + val manifestBytes = store.get(rootNodeLabel).get + val manifest = manifestSerializer.parseBytesTry(manifestBytes).get + + val writer = new VLQByteBufferWriter(new ByteArrayBuilder()) + manifestSerializer.serialize(prover.prover().topNode, prover.prover().rootNodeHeight.toByte, writer) + val altManifestBytes = writer.result().toBytes + + manifestBytes.sameElements(altManifestBytes) shouldBe true + + val subtreeIds = manifest.subtreesIds + subtreeIds.foreach { sid => + val chunkBytes = store.get(sid).get + SubtreeSerializer.parseBytesTry(chunkBytes).get.id.sameElements(sid) shouldBe true + } + } + } diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala index a7b6288edd..3d393582c6 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/VersionedLDBAVLStorageStatefulSpecification.scala @@ -27,7 +27,6 @@ object WithLDB extends VersionedLDBAVLStorageStatefulCommands with TestHelper { override protected val KL = 32 override protected val VL = 8 - override protected val LL = 32 override protected def createStatefulProver: PersistentBatchAVLProver[Digest32, HF] = { createPersistentProver(keepVersions) diff --git a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala index ad8e259f7b..0b2c4d79a9 100644 --- a/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala +++ b/avldb/src/test/scala/scorex/crypto/authds/avltree/batch/helpers/TestHelper.scala @@ -19,7 +19,6 @@ trait TestHelper extends FileHelper { protected val KL: Int protected val VL: Int - protected val LL: Int implicit val hf: HF = Blake2b256 @@ -28,7 +27,8 @@ trait TestHelper extends FileHelper { new LDBVersionedStore(dir, initialKeepVersions = initialKeepVersions) } - def createVersionedStorage(store: LDBVersionedStore): STORAGE = new VersionedLDBAVLStorage(store) + def createVersionedStorage(store: LDBVersionedStore): STORAGE = + new VersionedLDBAVLStorage(store) def createPersistentProver(storage: STORAGE): PERSISTENT_PROVER = { val prover = new BatchAVLProver[D, HF](KL, Some(VL)) diff --git a/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala b/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala index 652bb40982..cb21d133ab 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/db/LDBStoreBench.scala @@ -39,7 +39,7 @@ object LDBStoreBench val txsWithDbGen: Gen[(Seq[BlockTransactions], LDBKVStore)] = txsGen.map { bts => val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray val db = storeLDB() - toInsert.grouped(5).foreach(db.insert(_).get) + toInsert.grouped(5).foreach(kv => db.insert(kv.map(_._1), kv.map(_._2)).get) bts -> storeLDB } @@ -55,7 +55,7 @@ object LDBStoreBench private def benchWriteLDB(bts: Seq[BlockTransactions]): Unit = { val toInsert = bts.map(bt => idToBytes(bt.headerId) -> bt.bytes).toArray val db = storeLDB() - toInsert.grouped(5).foreach(db.insert(_).get) + toInsert.grouped(5).foreach(kv => db.insert(kv.map(_._1), kv.map(_._2)).get) } private def benchReadLDB(bts: Seq[BlockTransactions], db: LDBKVStore): Unit = { diff --git a/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala b/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala index b8d8ffa094..5a257b0420 100644 --- a/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala +++ b/benchmarks/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateBenchmark.scala @@ -22,7 +22,7 @@ object UtxoStateBenchmark extends HistoryTestHelpers with NVBenchmark { val transactionsQty = blocks.flatMap(_.transactions).size def bench(mods: Seq[BlockSection]): Long = { - val state = ErgoState.generateGenesisUtxoState(createTempDir, StateConstants(realNetworkSetting))._1 + val state = ErgoState.generateGenesisUtxoState(createTempDir, realNetworkSetting)._1 Utils.time { mods.foldLeft(state) { case (st, mod) => st.applyModifier(mod, None)(_ => ()).get diff --git a/build.sbt b/build.sbt index 70fd1c86a1..e880b0fa32 100644 --- a/build.sbt +++ b/build.sbt @@ -37,7 +37,7 @@ val circeVersion = "0.13.0" val akkaVersion = "2.6.10" val akkaHttpVersion = "10.2.4" -val sigmaStateVersion = "5.0.7" +val sigmaStateVersion = "5.0.8" // for testing current sigmastate build (see sigmastate-ergo-it jenkins job) val effectiveSigmaStateVersion = Option(System.getenv().get("SIGMASTATE_VERSION")).getOrElse(sigmaStateVersion) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala b/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala index 63070a1e9a..55a70b4016 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/contracts/ReemissionContracts.scala @@ -1,13 +1,14 @@ package org.ergoplatform.contracts import org.ergoplatform.ErgoBox.{R2, STokensRegType} -import org.ergoplatform.ErgoScriptPredef.{boxCreationHeight, expectedMinerOutScriptBytesVal} -import org.ergoplatform.{Height, MinerPubkey, Outputs, Self} +import org.ergoplatform.ErgoTreePredef.{boxCreationHeight, expectedMinerOutScriptBytesVal} +import org.ergoplatform.mining.emission.EmissionRules.CoinsInOneErgo import org.ergoplatform.settings.MonetarySettings -import sigmastate.{AND, EQ, GE, GT, LE, Minus, OR, SBox, SCollection, STuple} +import org.ergoplatform.{Height, MinerPubkey, Outputs, Self} import sigmastate.Values.{ByteArrayConstant, ErgoTree, IntConstant, LongConstant, SigmaPropValue, Value} -import sigmastate.utxo.{ByIndex, ExtractAmount, ExtractRegisterAs, ExtractScriptBytes, OptionGet, SelectField, SizeOf} -import org.ergoplatform.mining.emission.EmissionRules.CoinsInOneErgo +import sigmastate.utxo._ +import sigmastate._ +import special.collection.Coll /** * Container for re-emission related contracts. Contains re-emission contract and pay-to-reemission contract. @@ -22,7 +23,7 @@ trait ReemissionContracts { /** * @return - ID of NFT token associated with re-emission contract */ - def reemissionNftIdBytes: Array[Byte] + def reemissionNftIdBytes: Coll[Byte] /** * @return - height when reemission starts diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/utils/ArithUtils.scala b/ergo-wallet/src/main/scala/org/ergoplatform/utils/ArithUtils.scala deleted file mode 100644 index 61a222a177..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/utils/ArithUtils.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.ergoplatform.utils - -object ArithUtils { - - /** - * Add longs, returning Long.Max value if there was any long overflow - */ - @inline def addExact(a: Long, b: Long): Long = { - val sum = a + b - if (((a ^ sum) & (b ^ sum)) < 0) Long.MaxValue else sum - } - - def addExact(a: Long, b: Long, c: Long*): Long = c.foldLeft(addExact(a,b))((x, y) => addExact(x, y)) - - /** - * Multiply longs, returning Long.Max value if there was any long overflow - */ - @inline def multiplyExact(e1: Long, e2: Long): Long = { - try { - java7.compat.Math.multiplyExact(e1, e2) - } catch { - case _: Throwable => Long.MaxValue - } - } - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/AssetUtils.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/AssetUtils.scala deleted file mode 100644 index eaa435f647..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/AssetUtils.scala +++ /dev/null @@ -1,55 +0,0 @@ -package org.ergoplatform.wallet - -import scorex.util.ModifierId - -import scala.collection.mutable - -object AssetUtils { - - @inline - def mergeAssetsMut(into: mutable.Map[ModifierId, Long], from: TokensMap*): Unit = { - from.foreach(_.foreach { - case (id, amount) => - into.put(id, java7.compat.Math.addExact(into.getOrElse(id, 0L), amount)) - }) - } - - @inline - def mergeAssets(initialMap: TokensMap, maps: TokensMap*): TokensMap = { - maps.foldLeft(initialMap) { case (to, map) => - map.foldLeft(to) { case (acc, (id, amount)) => - acc.updated(id, java7.compat.Math.addExact(acc.getOrElse(id, 0L), amount)) - } - } - } - - @inline - def subtractAssets(initialMap: TokensMap, subtractor: TokensMap*): TokensMap = { - subtractor.foldLeft(initialMap){ case (from, mapToSubtract) => - mapToSubtract.foldLeft(from) { case (acc, (id, amount)) => - val currAmt = acc.getOrElse(id, - throw new IllegalArgumentException(s"Cannot subtract $amount of token $id: token not found in $acc")) - require(amount >= 0, s"Cannot subtract negative amount from token $id: $amount") - require(currAmt >= amount, s"Not enough amount of token $id -> $currAmt to subtract $amount") - if (currAmt == amount) { - acc - id - } else { - acc.updated(id, currAmt - amount) - } - } - } - } - - @inline - def subtractAssetsMut(from: mutable.Map[ModifierId, Long], subtractor: TokensMap): Unit = { - subtractor.foreach { case (id, subtractAmt) => - val fromAmt = from(id) - if (fromAmt == subtractAmt) { - from.remove(id) - } else { - from.put(id, fromAmt - subtractAmt) - } - } - } - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/Constants.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/Constants.scala index 781db18ef7..5e3aab6ff3 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/Constants.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/Constants.scala @@ -1,22 +1,11 @@ package org.ergoplatform.wallet -import org.ergoplatform.wallet.secrets.DerivationPath import supertagged.TaggedType object Constants { object ScanId extends TaggedType[Short] type ScanId = ScanId.Type - /** part of the protocol, do not change */ - val SecretKeyLength = 32 - - /** part of the protocol, do not change */ - val ModifierIdLength = 32 - - val Encoding = "UTF-8" - - val BitcoinSeed: Array[Byte] = "Bitcoin seed".getBytes(Encoding) - // Identifiers for system applications of Ergo node // Ids below 9 are reserved. Ids from 11 (inclusive) are for user scans @@ -26,33 +15,4 @@ object Constants { /** Scan which is checking mining rewards */ val MiningScanId: ScanId = ScanId @@ 9.toShort - - /** - * [See EIP-3 https://github.com/ergoplatform/eips/blob/master/eip-0003.md ] - * - * For coin type, we suggest consider "ergo" word in ASCII and calculate coin_type number as - * - * 101 + 114 + 103 + 111 = 429 - * - * Following this idea we should use next scheme - * - * m / 44' / 429' / account' / change / address_index - */ - val CoinType = 429 - - /** - * There needs to be reasonable amount of tokens in a box due to a byte size limit for ergo box - * */ - val MaxAssetsPerBox = 100 - - /** - * Pre - EIP3 derivation path - */ - val preEip3DerivationPath: DerivationPath = DerivationPath.fromEncoded("m/1").get - - /** - * Post - EIP3 derivation path - */ - val eip3DerivationPath: DerivationPath = DerivationPath.fromEncoded("m/44'/429'/0'/0/0").get - } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/BoxSelector.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/BoxSelector.scala index 6faf194ca3..f945b3e4bf 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/BoxSelector.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/BoxSelector.scala @@ -1,9 +1,9 @@ package org.ergoplatform.wallet.boxes -import org.ergoplatform.{ErgoBoxAssets, ErgoBoxAssetsHolder} import org.ergoplatform.SigmaConstants.MaxBoxSize -import org.ergoplatform.wallet.TokensMap +import org.ergoplatform.sdk.wallet.TokensMap import org.ergoplatform.wallet.boxes.BoxSelector.{BoxSelectionError, BoxSelectionResult} +import org.ergoplatform.{ErgoBoxAssets, ErgoBoxAssetsHolder} import scorex.util.ScorexLogging diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala index 534f55dc49..8de1ec7d49 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelector.scala @@ -1,14 +1,14 @@ package org.ergoplatform.wallet.boxes -import org.ergoplatform.contracts.ReemissionContracts +import org.ergoplatform.sdk.wallet.{AssetUtils, TokensMap} +import org.ergoplatform.sdk.wallet.Constants.MaxAssetsPerBox +import org.ergoplatform.wallet.boxes.BoxSelector.BoxSelectionError +import org.ergoplatform.wallet.transactions.TransactionBuilder._ +import org.ergoplatform.{ErgoBoxAssets, ErgoBoxAssetsHolder} import scorex.util.ModifierId -import org.ergoplatform.{ErgoBoxAssets, ErgoBoxAssetsHolder, ErgoBoxCandidate} -import org.ergoplatform.wallet.Constants.MaxAssetsPerBox -import org.ergoplatform.wallet.{AssetUtils, TokensMap} + import scala.annotation.tailrec import scala.collection.mutable -import org.ergoplatform.wallet.transactions.TransactionBuilder._ -import org.ergoplatform.wallet.boxes.BoxSelector.BoxSelectionError /** * Default implementation of the box selector. It simply picks boxes till sum of their monetary values @@ -19,9 +19,8 @@ import org.ergoplatform.wallet.boxes.BoxSelector.BoxSelectionError */ class DefaultBoxSelector(override val reemissionDataOpt: Option[ReemissionData]) extends BoxSelector { - import DefaultBoxSelector._ import BoxSelector._ - import scorex.util.idToBytes + import DefaultBoxSelector._ // helper function which returns count of assets in `initialMap` not fully spent in `subtractor` private def diffCount(initialMap: mutable.Map[ModifierId, Long], subtractor: TokensMap): Int = { diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala index e7e55d5288..8c2fdf5384 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ErgoBoxAssetExtractor.scala @@ -1,8 +1,8 @@ package org.ergoplatform.wallet.boxes -import org.ergoplatform.ErgoBoxCandidate -import sigmastate.eval.Extensions._ import java7.compat.Math +import org.ergoplatform.ErgoBoxCandidate +import special.collection.Extensions._ import scala.collection.compat.immutable.ArraySeq import scala.collection.mutable @@ -31,7 +31,7 @@ object ErgoBoxAssetExtractor { ) box.additionalTokens.foreach { case (assetId, amount) => - val aiWrapped = ArraySeq.unsafeWrapArray(assetId) + val aiWrapped = ArraySeq.unsafeWrapArray(assetId.toArray) val total = map.getOrElse(aiWrapped, 0L) map.put(aiWrapped, Math.addExact(total, amount)) } diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ReplaceCompactCollectBoxSelector.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ReplaceCompactCollectBoxSelector.scala index 820f97ecf2..0e32ebcecf 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ReplaceCompactCollectBoxSelector.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/ReplaceCompactCollectBoxSelector.scala @@ -1,11 +1,10 @@ package org.ergoplatform.wallet.boxes -import org.ergoplatform.wallet.boxes.BoxSelector.BoxSelectionResult -import org.ergoplatform.wallet.boxes.BoxSelector.BoxSelectionError import org.ergoplatform.ErgoBoxAssets -import org.ergoplatform.wallet.{AssetUtils, TokensMap} -import scorex.util.ModifierId +import org.ergoplatform.sdk.wallet.{AssetUtils, TokensMap} +import org.ergoplatform.wallet.boxes.BoxSelector.{BoxSelectionError, BoxSelectionResult} import org.ergoplatform.wallet.transactions.TransactionBuilder._ +import scorex.util.ModifierId import scala.annotation.tailrec import scala.collection.mutable diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala index e6b3f91096..2cfec5ad29 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/boxes/TrackedBox.scala @@ -1,12 +1,14 @@ package org.ergoplatform.wallet.boxes +import org.ergoplatform.sdk.wallet +import org.ergoplatform.sdk.wallet.TokensMap +import org.ergoplatform.wallet.Constants import org.ergoplatform.wallet.Constants.ScanId -import org.ergoplatform.wallet.{Constants, TokensMap} import org.ergoplatform.wallet.serialization.ErgoWalletSerializer -import org.ergoplatform.{ErgoBox, ErgoLikeTransaction} +import org.ergoplatform.{ErgoBox, ErgoBoxAssets, ErgoLikeTransaction} import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, bytesToId, idToBytes} -import org.ergoplatform.ErgoBoxAssets +import special.collection.Extensions._ /** * A box tracked by a wallet that contains Ergo box itself as well as @@ -67,7 +69,7 @@ case class TrackedBox(creationTxId: ModifierId, def isSpent: Boolean = spendingHeightOpt.isDefined lazy val tokens: TokensMap = box.additionalTokens.toArray.map { - case (id, amt) => bytesToId(id) -> amt + case (id, amt) => id.toModifierId -> amt }.toMap override def equals(obj: Any): Boolean = obj match { @@ -121,10 +123,10 @@ object TrackedBoxSerializer extends ErgoWalletSerializer[TrackedBox] { } override def parse(r: Reader): TrackedBox = { - val creationTxId = bytesToId(r.getBytes(Constants.ModifierIdLength)) + val creationTxId = bytesToId(r.getBytes(wallet.Constants.ModifierIdLength)) val creationOutIndex = r.getShort() val inclusionHeightOpt = r.getOption(r.getInt()) - val spendingTxIdOpt = r.getOption(r.getBytes(Constants.ModifierIdLength)).map(bytesToId) + val spendingTxIdOpt = r.getOption(r.getBytes(wallet.Constants.ModifierIdLength)).map(bytesToId) val spendingHeightOpt = r.getOption(r.getInt()) val appsCount = r.getShort() diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/AES.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/AES.scala index e5984dd1a8..45a448c105 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/AES.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/AES.scala @@ -1,9 +1,9 @@ package org.ergoplatform.wallet.crypto +import org.ergoplatform.sdk.wallet.settings.EncryptionSettings + import javax.crypto.spec.{GCMParameterSpec, PBEKeySpec, SecretKeySpec} import javax.crypto.{Cipher, SecretKeyFactory} -import org.ergoplatform.wallet.settings.EncryptionSettings - import scala.util.Try object AES { diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala index ef0c4a0d91..03f144ea2d 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/crypto/ErgoSignature.scala @@ -3,7 +3,7 @@ package org.ergoplatform.wallet.crypto import org.bouncycastle.util.BigIntegers import scorex.crypto.hash.Blake2b256 import scorex.util.encode.Base16 -import sigmastate.interpreter.CryptoConstants +import sigmastate.basics.CryptoConstants import sigmastate.serialization.GroupElementSerializer import scala.annotation.tailrec diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala index 449ff6cb9a..fbed24ecfb 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoInterpreter.scala @@ -1,8 +1,8 @@ package org.ergoplatform.wallet.interpreter import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeParameters import org.ergoplatform.wallet.protocol.Constants -import org.ergoplatform.wallet.protocol.context.ErgoLikeParameters import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoLikeContext, ErgoLikeInterpreter} import scorex.crypto.authds.ADDigest import scorex.util.ScorexLogging diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala index 65539bb07a..4d1d62ed6f 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreter.scala @@ -1,23 +1,21 @@ package org.ergoplatform.wallet.interpreter -import java.util - import org.ergoplatform._ -import org.ergoplatform.utils.ArithUtils.{addExact, multiplyExact} -import org.ergoplatform.validation.SigmaValidationSettings +import org.ergoplatform.sdk.utils.ArithUtils.{addExact, multiplyExact} +import org.ergoplatform.sdk.wallet.protocol.context.{ErgoLikeParameters, ErgoLikeStateContext} +import org.ergoplatform.sdk.wallet.secrets.{ExtendedPublicKey, ExtendedSecretKey, SecretKey} +import org.ergoplatform.validation.{SigmaValidationSettings, ValidationRules} +import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor +import scorex.crypto.authds.ADDigest +import scorex.util.encode.Base16 import sigmastate.AvlTreeData import sigmastate.Values.SigmaBoolean -import sigmastate.interpreter.{ContextExtension, ProverInterpreter} -import org.ergoplatform.validation.ValidationRules -import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor -import org.ergoplatform.wallet.protocol.context.{ErgoLikeParameters, ErgoLikeStateContext} -import org.ergoplatform.wallet.secrets.SecretKey import sigmastate.basics.SigmaProtocolPrivateInput -import org.ergoplatform.wallet.secrets.{ExtendedPublicKey, ExtendedSecretKey} -import scorex.util.encode.Base16 +import sigmastate.interpreter.{ContextExtension, ProverInterpreter} import special.collection.Coll import special.sigma.{Header, PreHeader} +import java.util import scala.util.{Failure, Success, Try} /** diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala index 88db0570c4..46871b6bdb 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/mnemonic/Mnemonic.scala @@ -2,12 +2,11 @@ package org.ergoplatform.wallet.mnemonic import java.text.Normalizer.Form.NFKD import java.text.Normalizer.normalize - import javax.crypto.SecretKeyFactory import javax.crypto.spec.PBEKeySpec -import org.ergoplatform.wallet.Constants import org.ergoplatform.wallet.interface4j.SecretString import scodec.bits.BitVector +import sigmastate.crypto.CryptoFacade import scala.util.{Failure, Try} @@ -73,7 +72,7 @@ object Mnemonic { val spec = new PBEKeySpec( normalizedMnemonic, - normalizedPass.getBytes(Constants.Encoding), + normalizedPass.getBytes(CryptoFacade.Encoding), Pbkdf2Iterations, Pbkdf2KeyLength ) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/package.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/package.scala deleted file mode 100644 index 7c7e9fcd82..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/package.scala +++ /dev/null @@ -1,7 +0,0 @@ -package org.ergoplatform - -import scorex.util.ModifierId - -package object wallet { - type TokensMap = Map[ModifierId, Long] -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala deleted file mode 100644 index 3f5f893ca7..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeParameters.scala +++ /dev/null @@ -1,63 +0,0 @@ -package org.ergoplatform.wallet.protocol.context - -/** - * Blockchain parameters readjustable via miners voting and voting-related data. - * All these fields are included into extension section of a first block of a voting epoch. - */ -trait ErgoLikeParameters { - - /** - * @return cost of storing 1 byte in UTXO for four years, in nanoErgs - */ - def storageFeeFactor: Int - - /** - * @return cost of a transaction output, in computation unit - */ - def minValuePerByte: Int - - /** - * @return max block size, in bytes - */ - def maxBlockSize: Int - - /** - * @return cost of a token contained in a transaction, in computation unit - */ - def tokenAccessCost: Int - - /** - * @return cost of a transaction input, in computation unit - */ - def inputCost: Int - - /** - * @return cost of a transaction data input, in computation unit - */ - def dataInputCost: Int - - /** - * @return cost of a transaction output, in computation unit - */ - def outputCost: Int - - /** - * @return computation units limit per block - */ - def maxBlockCost: Int - - /** - * @return height when voting for a soft-fork had been started - */ - def softForkStartingHeight: Option[Int] - - /** - * @return votes for soft-fork collected in previous epochs - */ - def softForkVotesCollected: Option[Int] - - /** - * @return Protocol version - */ - def blockVersion: Byte -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeStateContext.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeStateContext.scala deleted file mode 100644 index 9bf5829be9..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/ErgoLikeStateContext.scala +++ /dev/null @@ -1,26 +0,0 @@ -package org.ergoplatform.wallet.protocol.context - -import scorex.crypto.authds.ADDigest -import special.collection.Coll - -/** - * Blockchain context used in transaction validation. - */ -trait ErgoLikeStateContext { - - /** - * @return fixed number (10 in Ergo) of last block headers - */ - def sigmaLastHeaders: Coll[special.sigma.Header] - - // todo remove from ErgoLikeContext and from ErgoStateContext - /** - * @return UTXO set digest from a last header (of sigmaLastHeaders) - */ - def previousStateDigest: ADDigest - - /** - * @return returns pre-header (header without certain fields) of the current block - */ - def sigmaPreHeader: special.sigma.PreHeader -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/TransactionContext.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/TransactionContext.scala deleted file mode 100644 index b1539400bb..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/protocol/context/TransactionContext.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.ergoplatform.wallet.protocol.context - -import org.ergoplatform.{ErgoBox, ErgoLikeTransactionTemplate, UnsignedInput} - -/** - * Part of the execution context in regards with spending transaction - * - * @param boxesToSpend - inputs of the transaction - * @param dataBoxes - data (read-only) inputs of the transaction - * @param spendingTransaction - spending transaction - */ - -// TODO: it seems spendingTransaction is needed only to get output candidates in ErgoLikeContext. -// After ErgoLikeContext refactoring in sigma, this class can be simplified. -case class TransactionContext(boxesToSpend: IndexedSeq[ErgoBox], - dataBoxes: IndexedSeq[ErgoBox], - spendingTransaction: ErgoLikeTransactionTemplate[_ <: UnsignedInput]) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/DerivationPath.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/DerivationPath.scala deleted file mode 100644 index 37425bd833..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/DerivationPath.scala +++ /dev/null @@ -1,150 +0,0 @@ -package org.ergoplatform.wallet.secrets - -import org.ergoplatform.wallet.Constants -import org.ergoplatform.wallet.serialization.ErgoWalletSerializer -import scorex.util.serialization.{Reader, Writer} - -import scala.util.{Failure, Success, Try} - -/** - * HD key derivation path (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) - */ -final case class DerivationPath(decodedPath: Seq[Int], publicBranch: Boolean) { - - import DerivationPath._ - - def depth: Int = decodedPath.length - - /** - * @return last element of the derivation path, e.g. 2 for m/1/2 - */ - def index: Int = decodedPath.last - - def isMaster: Boolean = depth == 1 - - /** Encode this DerivationPath as a parsable string. */ - def encoded: String = { - val masterPrefix = if (publicBranch) s"$PublicBranchMasterId/" else s"$PrivateBranchMasterId/" - val tailPath = decodedPath.tail - .map(x => if (Index.isHardened(x)) s"${x - Index.HardRangeStart}'" else x.toString) - .mkString("/") - masterPrefix + tailPath - } - - def extended(idx: Int): DerivationPath = DerivationPath(decodedPath :+ idx, publicBranch) - - /** - * @return path with last element of the derivation path being increased, e.g. m/1/2 -> m/1/3 - */ - def increased: DerivationPath = DerivationPath(decodedPath.dropRight(1) :+ (index + 1), publicBranch) - - /** - * Convert the derivation path to public branch. See BIP-32 for details. - * @return derivation path from the public branch - */ - def toPublicBranch: DerivationPath = this.copy(publicBranch = true) - - /** - * Convert the derivation path to private branch. See BIP-32 for details. - * @return derivation path from the private branch - */ - def toPrivateBranch: DerivationPath = this.copy(publicBranch = false) - - /** - * @return whether derivation path corresponds to EIP-3 - */ - def isEip3: Boolean = { - decodedPath.tail.startsWith(Constants.eip3DerivationPath.decodedPath.tail.take(3)) - } - - override def toString: String = encoded - - def bytes: Array[Byte] = DerivationPathSerializer.toBytes(this) -} - -object DerivationPath { - - val PublicBranchMasterId = "M" - val PrivateBranchMasterId = "m" - - val MasterPath: DerivationPath = DerivationPath(List(0), publicBranch = false) - - def fromEncoded(path: String): Try[DerivationPath] = { - val split = path.split("/") - if (!split.headOption.exists(Seq(PublicBranchMasterId, PrivateBranchMasterId).contains)) { - Failure(new Exception("Wrong path format")) - } else { - val pathTry = split.tail.foldLeft(Try(List(0))) { case (accTry, sym) => - accTry.flatMap { acc => - Try(if (sym.endsWith("'")) Index.hardIndex(sym.dropRight(1).toInt) else sym.toInt) - .map(acc :+ _) - } - } - val isPublicBranch = split.head == PublicBranchMasterId - pathTry.map(DerivationPath(_, isPublicBranch)) - } - } - - /** - * Finds next available path index for a new key. - * @param secrets - secrets previously generated - * @param usePreEip3Derivation - whether to use pre-EIP3 derivation or not - */ - def nextPath(secrets: IndexedSeq[ExtendedSecretKey], - usePreEip3Derivation: Boolean): Try[DerivationPath] = { - - def pathSequence(secret: ExtendedSecretKey): Seq[Int] = secret.path.decodedPath.tail - - @scala.annotation.tailrec - def nextPath(accPath: List[Int], remaining: Seq[Seq[Int]]): Try[DerivationPath] = { - if (!remaining.forall(_.isEmpty)) { - val maxChildIdx = remaining.flatMap(_.headOption).max - if (!Index.isHardened(maxChildIdx)) { - Success(DerivationPath(0 +: (accPath :+ maxChildIdx + 1), publicBranch = false)) - } else { - nextPath(accPath :+ maxChildIdx, remaining.map(_.drop(1))) - } - } else { - val exc = new Exception("Out of non-hardened index space. Try to derive hardened key specifying path manually") - Failure(exc) - } - } - - if (secrets.isEmpty || (secrets.size == 1 && secrets.head.path.isMaster)) { - // If pre-EIP3 generation, the first key generated after master's would be m/1, otherwise m/44'/429'/0'/0/0 - val path = if(usePreEip3Derivation) { - Constants.preEip3DerivationPath - } else { - Constants.eip3DerivationPath - } - Success(path) - } else { - // If last secret corresponds to EIP-3 path, do EIP-3 derivation, otherwise, old derivation - // For EIP-3 derivation, we increase last segment, m/44'/429'/0'/0/0 -> m/44'/429'/0'/0/1 and so on - // For old derivation, we increase last non-hardened segment, m/1/1 -> m/2 - if (secrets.last.path.isEip3) { - Success(secrets.last.path.increased) - } else { - nextPath(List.empty, secrets.map(pathSequence)) - } - } - } - -} - -object DerivationPathSerializer extends ErgoWalletSerializer[DerivationPath] { - - override def serialize(obj: DerivationPath, w: Writer): Unit = { - w.put(if (obj.publicBranch) 0x01 else 0x00) - w.putInt(obj.depth) - obj.decodedPath.foreach(i => w.putBytes(Index.serializeIndex(i))) - } - - override def parse(r: Reader): DerivationPath = { - val publicBranch = if (r.getByte() == 0x01) true else false - val depth = r.getInt() - val path = (0 until depth).map(_ => Index.parseIndex(r.getBytes(4))) - DerivationPath(path, publicBranch) - } - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala index 54b0ebca7a..4a0e23ff7e 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/EncryptedSecret.scala @@ -3,7 +3,7 @@ package org.ergoplatform.wallet.secrets import io.circe.syntax._ import cats.syntax.either._ // don't remove, it is needed for scala 2.11 import io.circe.{Decoder, Encoder, HCursor, Json} -import org.ergoplatform.wallet.settings.EncryptionSettings +import org.ergoplatform.sdk.wallet.settings.EncryptionSettings import scorex.util.encode.Base16 /** diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala deleted file mode 100644 index fd11e311c2..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedKey.scala +++ /dev/null @@ -1,43 +0,0 @@ -package org.ergoplatform.wallet.secrets - -/** Description from https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki : - * - * We extend both private and public keys first with an extra 256 bits of entropy. - * This extension, called the chain code, is identical for corresponding private and public keys - * and consists of 32 bytes. - * We represent an extended private key as (k, c), with k the normal private key, - * and c the chain code. An extended public key is represented as (K, c), - * with K = point(k) and c the chain code. - * - * Each extended key has 2^31 normal child keys, and 2^31 hardened child keys. - * Each of these child keys has an index. The normal child keys use indices 0 through 2^31-1. - * The hardened child keys use indices 2^31 through `2^32-1`. - */ -trait ExtendedKey[T <: ExtendedKey[T]] { - - val path: DerivationPath - - /** Returns subtype reference. - */ - def selfReflection: T - - /** Given a parent extended key and an index `idx`, it is possible to compute the corresponding - * child extended key. The algorithm to do so depends on whether the child is a hardened key - * or not (or, equivalently, whether `idx ≥ 2^31`), and whether we're talking about private or - * public keys. - * - * @see implementations in derived classes - */ - def child(idx: Int): T - - def derive(upPath: DerivationPath): T = { - require( - upPath.depth >= path.depth && - upPath.decodedPath.take(path.depth).zip(path.decodedPath).forall { case (i1, i2) => i1 == i2 } && - upPath.publicBranch == path.publicBranch, - s"Incompatible paths: $upPath, $path" - ) - upPath.decodedPath.drop(path.depth).foldLeft(selfReflection)((parent, i) => parent.child(i)) - } - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala deleted file mode 100644 index 4fe572243c..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKey.scala +++ /dev/null @@ -1,98 +0,0 @@ -package org.ergoplatform.wallet.secrets - -import java.util -import org.bouncycastle.util.BigIntegers -import org.ergoplatform.wallet.Constants -import org.ergoplatform.wallet.crypto.HmacSHA512 -import org.ergoplatform.wallet.serialization.ErgoWalletSerializer -import scorex.util.serialization.{Reader, Writer} -import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog} -import sigmastate.crypto.CryptoFacade -import sigmastate.interpreter.CryptoConstants - -import scala.annotation.tailrec - -/** - * Public key, its chain code and path in key tree. - * (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) - */ -final class ExtendedPublicKey(private[secrets] val keyBytes: Array[Byte], - private[secrets] val chainCode: Array[Byte], - val path: DerivationPath) - extends ExtendedKey[ExtendedPublicKey] { - - def selfReflection: ExtendedPublicKey = this - - def key: ProveDlog = ProveDlog( - CryptoConstants.dlogGroup.ctx.decodePoint(keyBytes) - ) - - def child(idx: Int): ExtendedPublicKey = ExtendedPublicKey.deriveChildPublicKey(this, idx) - - override def equals(obj: Any): Boolean = (this eq obj.asInstanceOf[AnyRef]) || (obj match { - case that: ExtendedPublicKey => - util.Arrays.equals(that.keyBytes, this.keyBytes) && - util.Arrays.equals(that.chainCode, this.chainCode) && - that.path == this.path - case _ => false - }) - - override def hashCode(): Int = { - var h = util.Arrays.hashCode(keyBytes) - h = 31 * h + util.Arrays.hashCode(chainCode) - h = 31 * h + path.hashCode() - h - } - - override def toString: String = s"ExtendedPublicKey($path : $key)" -} - -object ExtendedPublicKey { - - @tailrec - def deriveChildPublicKey(parentKey: ExtendedPublicKey, idx: Int): ExtendedPublicKey = { - require(!Index.isHardened(idx), "Hardened public keys derivation is not supported") - val (childKeyProto, childChainCode) = HmacSHA512 - .hash(parentKey.chainCode, parentKey.keyBytes ++ Index.serializeIndex(idx)) - .splitAt(Constants.SecretKeyLength) - val childKeyProtoDecoded = BigIntegers.fromUnsignedByteArray(childKeyProto) - val childKey = CryptoFacade.multiplyPoints( - DLogProverInput(childKeyProtoDecoded).publicImage.value, - parentKey.key.value) - if (childKeyProtoDecoded.compareTo(CryptoConstants.groupOrder) >= 0 || CryptoFacade.isInfinityPoint(childKey)) { - deriveChildPublicKey(parentKey, idx + 1) - } else { - new ExtendedPublicKey( - keyBytes = CryptoFacade.encodePoint(childKey, compressed = true), - chainCode = childChainCode, - path = parentKey.path.extended(idx) - ) - } - } - -} - -object ExtendedPublicKeySerializer extends ErgoWalletSerializer[ExtendedPublicKey] { - - import scorex.util.Extensions._ - - //ASN.1 encoding for secp256k1 points - 1 byte for sign + 32 bytes for x-coordinate of the point - val PublicKeyBytesSize: Int = Constants.SecretKeyLength + 1 - - override def serialize(obj: ExtendedPublicKey, w: Writer): Unit = { - w.putBytes(obj.keyBytes) - w.putBytes(obj.chainCode) - val pathBytes = DerivationPathSerializer.toBytes(obj.path) - w.putUInt(pathBytes.length) - w.putBytes(pathBytes) - } - - override def parse(r: Reader): ExtendedPublicKey = { - val keyBytes = r.getBytes(PublicKeyBytesSize) - val chainCode = r.getBytes(Constants.SecretKeyLength) - val pathLen = r.getUInt().toIntExact - val path = DerivationPathSerializer.parseBytes(r.getBytes(pathLen)) - new ExtendedPublicKey(keyBytes, chainCode, path) - } - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala deleted file mode 100644 index 1ac40239f4..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKey.scala +++ /dev/null @@ -1,131 +0,0 @@ -package org.ergoplatform.wallet.secrets - -import java.math.BigInteger -import java.util -import org.bouncycastle.util.BigIntegers -import org.ergoplatform.wallet.Constants -import org.ergoplatform.wallet.crypto.HmacSHA512 -import org.ergoplatform.wallet.serialization.ErgoWalletSerializer -import scorex.util.serialization.{Reader, Writer} -import sigmastate.basics.DLogProtocol -import sigmastate.basics.DLogProtocol.DLogProverInput -import sigmastate.crypto.CryptoFacade -import sigmastate.interpreter.CryptoConstants - -/** - * Secret, its chain code and path in key tree. - * (see: https://github.com/bitcoin/bips/blob/master/bip-0032.mediawiki) - */ -final class ExtendedSecretKey(private[secrets] val keyBytes: Array[Byte], - private[secrets] val chainCode: Array[Byte], - private[secrets] val usePre1627KeyDerivation: Boolean, - val path: DerivationPath) - extends ExtendedKey[ExtendedSecretKey] with SecretKey { - - def selfReflection: ExtendedSecretKey = this - - override def privateInput: DLogProverInput = DLogProverInput(BigIntegers.fromUnsignedByteArray(keyBytes)) - - def publicImage: DLogProtocol.ProveDlog = privateInput.publicImage - - def child(idx: Int): ExtendedSecretKey = ExtendedSecretKey.deriveChildSecretKey(this, idx) - - /** Returns extended public key corresponding to this secret key. */ - def publicKey: ExtendedPublicKey = - new ExtendedPublicKey( - keyBytes = CryptoFacade.encodePoint(privateInput.publicImage.value, compressed = true), - chainCode = chainCode, - path = path.toPublicBranch) - - def isErased: Boolean = keyBytes.forall(_ == 0x00) - - def zeroSecret(): Unit = util.Arrays.fill(keyBytes, 0: Byte) - - override def equals(obj: Any): Boolean = (this eq obj.asInstanceOf[AnyRef]) || (obj match { - case that: ExtendedSecretKey => - util.Arrays.equals(that.keyBytes, this.keyBytes) && - util.Arrays.equals(that.chainCode, this.chainCode) && - that.path == this.path - case _ => false - }) - - override def hashCode(): Int = { - var h = util.Arrays.hashCode(keyBytes) - h = 31 * h + util.Arrays.hashCode(chainCode) - h = 31 * h + path.hashCode() - h - } - -} - -object ExtendedSecretKey { - - @scala.annotation.tailrec - def deriveChildSecretKey(parentKey: ExtendedSecretKey, idx: Int): ExtendedSecretKey = { - val keyCoded: Array[Byte] = - if (Index.isHardened(idx)) (0x00: Byte) +: parentKey.keyBytes - else CryptoFacade.encodePoint(parentKey.privateInput.publicImage.value, compressed = true) - val (childKeyProto, childChainCode) = HmacSHA512 - .hash(parentKey.chainCode, keyCoded ++ Index.serializeIndex(idx)) - .splitAt(Constants.SecretKeyLength) - val childKeyProtoDecoded = BigIntegers.fromUnsignedByteArray(childKeyProto) - val childKey = childKeyProtoDecoded - .add(BigIntegers.fromUnsignedByteArray(parentKey.keyBytes)) - .mod(CryptoConstants.groupOrder) - if (childKeyProtoDecoded.compareTo(CryptoConstants.groupOrder) >= 0 || childKey.equals(BigInteger.ZERO)) - deriveChildSecretKey(parentKey, idx + 1) - else { - val keyBytes = if (parentKey.usePre1627KeyDerivation) { - // maybe less than 32 bytes if childKey is small enough while BIP32 requires 32 bytes. - // see https://github.com/ergoplatform/ergo/issues/1627 for details - BigIntegers.asUnsignedByteArray(childKey) - } else { - // padded with leading zeroes to 32 bytes - BigIntegers.asUnsignedByteArray(Constants.SecretKeyLength, childKey) - } - new ExtendedSecretKey(keyBytes, childChainCode, parentKey.usePre1627KeyDerivation, parentKey.path.extended(idx)) - } - } - - def deriveChildPublicKey(parentKey: ExtendedSecretKey, idx: Int): ExtendedPublicKey = { - val derivedSecret = deriveChildSecretKey(parentKey, idx) - val derivedPk = CryptoFacade.encodePoint( - derivedSecret.privateInput.publicImage.value, compressed = true) - val derivedPath = derivedSecret.path.copy(publicBranch = true) - new ExtendedPublicKey(derivedPk, derivedSecret.chainCode, derivedPath) - } - - - /** - * Derives master secret key from the seed - * @param seed - seed bytes - * @param usePre1627KeyDerivation - use incorrect(previous) BIP32 derivation, expected to be false for new wallets, and true for old pre-1627 wallets (see https://github.com/ergoplatform/ergo/issues/1627 for details) - */ - def deriveMasterKey(seed: Array[Byte], usePre1627KeyDerivation: Boolean): ExtendedSecretKey = { - val (masterKey, chainCode) = HmacSHA512.hash(Constants.BitcoinSeed, seed).splitAt(Constants.SecretKeyLength) - new ExtendedSecretKey(masterKey, chainCode, usePre1627KeyDerivation, DerivationPath.MasterPath) - } - -} - -object ExtendedSecretKeySerializer extends ErgoWalletSerializer[ExtendedSecretKey] { - - import scorex.util.Extensions._ - - override def serialize(obj: ExtendedSecretKey, w: Writer): Unit = { - w.putBytes(obj.keyBytes) - w.putBytes(obj.chainCode) - val pathBytes = DerivationPathSerializer.toBytes(obj.path) - w.putUInt(pathBytes.length) - w.putBytes(pathBytes) - } - - override def parse(r: Reader): ExtendedSecretKey = { - val keyBytes = r.getBytes(Constants.SecretKeyLength) - val chainCode = r.getBytes(Constants.SecretKeyLength) - val pathLen = r.getUInt().toIntExact - val path = DerivationPathSerializer.parseBytes(r.getBytes(pathLen)) - new ExtendedSecretKey(keyBytes, chainCode, false, path) - } - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/Index.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/Index.scala deleted file mode 100644 index 58d79e700e..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/Index.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.ergoplatform.wallet.secrets - -import scodec.bits.ByteVector - -object Index { - - val HardRangeStart = 0x80000000 - - def hardIndex(i: Int): Int = i | HardRangeStart - - def isHardened(i: Int): Boolean = (i & HardRangeStart) != 0 - - def serializeIndex(i: Int): Array[Byte] = ByteVector.fromInt(i).toArray - - def parseIndex(xs: Array[Byte]): Int = ByteVector(xs).toInt(signed = false) - -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala index a4db16771a..e75c9b951c 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/JsonSecretStorage.scala @@ -1,16 +1,18 @@ package org.ergoplatform.wallet.secrets -import java.io.{File, FileNotFoundException, PrintWriter} -import java.util -import java.util.UUID import io.circe.parser._ import io.circe.syntax._ +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey +import org.ergoplatform.sdk.wallet.settings.EncryptionSettings import org.ergoplatform.wallet.crypto -import org.ergoplatform.wallet.mnemonic.Mnemonic import org.ergoplatform.wallet.interface4j.SecretString -import org.ergoplatform.wallet.settings.{EncryptionSettings, SecretStorageSettings} +import org.ergoplatform.wallet.mnemonic.Mnemonic +import org.ergoplatform.wallet.settings.SecretStorageSettings import scorex.util.encode.Base16 +import java.io.{File, FileNotFoundException, PrintWriter} +import java.util +import java.util.UUID import scala.util.{Failure, Success, Try} /** diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretKey.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretKey.scala deleted file mode 100644 index 19be5b82da..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretKey.scala +++ /dev/null @@ -1,39 +0,0 @@ -package org.ergoplatform.wallet.secrets - -import sigmastate.basics.DLogProtocol.DLogProverInput -import sigmastate.basics.{DiffieHellmanTupleProverInput, SigmaProtocolPrivateInput} - -/** - * Basic trait for secret data, encapsulating a corresponding private inputs for a Sigma protocol. - */ -trait SecretKey { - /** - * Private (secret) input of a sigma protocol - */ - def privateInput: SigmaProtocolPrivateInput[_, _] -} - -/** - * Basic trait for a secret which does not have a derivation scheme. - */ -sealed trait PrimitiveSecretKey extends SecretKey - -object PrimitiveSecretKey { - def apply(sigmaPrivateInput: SigmaProtocolPrivateInput[_, _]): PrimitiveSecretKey = sigmaPrivateInput match { - case dls: DLogProverInput => DlogSecretKey(dls) - case dhts: DiffieHellmanTupleProverInput => DhtSecretKey(dhts) - } -} - -/** - * Secret exponent of a group element, i.e. secret w such as h = g^^w, where g is group generator, h is a public key. - * @param privateInput - secret (in form of a sigma-protocol private input) - */ -case class DlogSecretKey(override val privateInput: DLogProverInput) extends PrimitiveSecretKey - -/** - * Secret exponent of a Diffie-Hellman tuple, i.e. secret w such as u = g^^w and v = h^^w, where g and h are group - * generators, (g,h,u,v) is a public input (public key). - * @param privateInput - secret (in form of a sigma-protocol private input) - */ -case class DhtSecretKey(override val privateInput: DiffieHellmanTupleProverInput) extends PrimitiveSecretKey diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretStorage.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretStorage.scala index 148c1001d0..1f3ee1394e 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretStorage.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/secrets/SecretStorage.scala @@ -1,5 +1,7 @@ package org.ergoplatform.wallet.secrets +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey + import java.io.File import scala.util.Try import org.ergoplatform.wallet.interface4j.SecretString diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/serialization/ErgoWalletSerializer.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/serialization/ErgoWalletSerializer.scala index df5807ce50..bc9bf9071b 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/serialization/ErgoWalletSerializer.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/serialization/ErgoWalletSerializer.scala @@ -1,9 +1,10 @@ package org.ergoplatform.wallet.serialization import java.nio.ByteBuffer - import scorex.util.ByteArrayBuilder import scorex.util.serialization._ +import sigmastate.serialization.{SigmaSerializer, ConstantStore} +import sigmastate.utils.{SigmaByteWriter, SigmaByteReader} import scala.util.Try @@ -23,3 +24,19 @@ trait ErgoWalletSerializer[T] extends Serializer[T, T, Reader, Writer] { def parseBytesTry(bytes: Array[Byte]): Try[T] = Try(parseBytes(bytes)) } + +object ErgoWalletSerializer { + + /** Creates a new serializer which delegates to the given [[SigmaSerializer]]. */ + def fromSigmaSerializer[T](ss: SigmaSerializer[T, T]): ErgoWalletSerializer[T] = new ErgoWalletSerializer[T] { + override def serialize(obj: T, w: Writer): Unit = { + val sw = new SigmaByteWriter(w, None) + ss.serialize(obj, sw) + } + + override def parse(r: Reader): T = { + val sr = new SigmaByteReader(r, new ConstantStore(), resolvePlaceholdersToConstants = false) + ss.parse(sr) + } + } +} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/EncryptionSettings.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/EncryptionSettings.scala deleted file mode 100644 index e8369c275f..0000000000 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/EncryptionSettings.scala +++ /dev/null @@ -1,40 +0,0 @@ -package org.ergoplatform.wallet.settings - -import io.circe.{Json, Encoder, Decoder, HCursor} -import io.circe.syntax._ -import cats.syntax.either._ // don't remove, it is needed for scala 2.11 - -/** - * Encryption parameters - * @param prf - pseudo-random function with output of length `dkLen` (PBKDF2 param) - * @param c - number of PBKDF2 iterations (PBKDF2 param) - * @param dkLen - desired bit-length of the derived key (PBKDF2 param) - */ -final case class EncryptionSettings(prf: String, c: Int, dkLen: Int) - -object EncryptionSettings { - - implicit object EncryptionSettingsEncoder extends Encoder[EncryptionSettings] { - - def apply(s: EncryptionSettings): Json = { - Json.obj( - "prf" -> s.prf.asJson, - "c" -> s.c.asJson, - "dkLen" -> s.dkLen.asJson - ) - } - - } - - implicit object EncryptionSettingsDecoder extends Decoder[EncryptionSettings] { - - def apply(cursor: HCursor): Decoder.Result[EncryptionSettings] = { - for { - prf <- cursor.downField("prf").as[String] - c <- cursor.downField("c").as[Int] - dkLen <- cursor.downField("dkLen").as[Int] - } yield EncryptionSettings(prf, c, dkLen) - } - - } -} diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/SecretStorageSettings.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/SecretStorageSettings.scala index 354b723099..d71c3ee02a 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/SecretStorageSettings.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/settings/SecretStorageSettings.scala @@ -1,3 +1,5 @@ package org.ergoplatform.wallet.settings +import org.ergoplatform.sdk.wallet.settings.EncryptionSettings + final case class SecretStorageSettings(secretDir: String, encryption: EncryptionSettings) diff --git a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala index e8dedb0c9e..6ad66b5423 100644 --- a/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala +++ b/ergo-wallet/src/main/scala/org/ergoplatform/wallet/transactions/TransactionBuilder.scala @@ -1,26 +1,20 @@ package org.ergoplatform.wallet.transactions -import org.ergoplatform.ErgoBox -import org.ergoplatform.DataInput -import org.ergoplatform.ErgoBoxCandidate -import org.ergoplatform.ErgoAddress -import org.ergoplatform.ErgoScriptPredef -import org.ergoplatform.UnsignedErgoLikeTransaction -import org.ergoplatform.UnsignedInput -import sigmastate.eval.Extensions._ - -import scala.util.Try -import scorex.util.{ModifierId, bytesToId, idToBytes} -import special.collection.Coll -import sigmastate.eval._ import org.ergoplatform.ErgoBox.TokenId -import scorex.crypto.hash.Digest32 -import org.ergoplatform.wallet.{AssetUtils, TokensMap} -import org.ergoplatform.wallet.boxes.BoxSelector -import org.ergoplatform.wallet.boxes.DefaultBoxSelector +import org.ergoplatform._ +import org.ergoplatform.sdk.wallet.{AssetUtils, TokensMap} +import org.ergoplatform.wallet.boxes.{BoxSelector, DefaultBoxSelector} import scorex.crypto.authds.ADKey import scorex.util.encode.Base16 +import scorex.util.{ModifierId, bytesToId} +import sigmastate.eval.Extensions._ +import sigmastate.eval._ +import sigmastate.utils.Extensions._ +import special.collection.Coll +import special.collection.Extensions._ + import scala.collection.JavaConverters._ +import scala.util.Try object TransactionBuilder { @@ -52,7 +46,7 @@ object TransactionBuilder { currentHeight: Int): UnsignedErgoLikeTransaction = { val feeBox = new ErgoBoxCandidate( feeAmt, - ErgoScriptPredef.feeProposition(), + ErgoTreePredef.feeProposition(), currentHeight, Seq.empty[(ErgoBox.TokenId, Long)].toColl, Map.empty @@ -123,7 +117,7 @@ object TransactionBuilder { ) val fee = new ErgoBoxCandidate( feeAmt, - ErgoScriptPredef.feeProposition(), + ErgoTreePredef.feeProposition(), currentHeight, Seq.empty[(ErgoBox.TokenId, Long)].toColl, Map.empty @@ -164,10 +158,10 @@ object TransactionBuilder { } def collTokensToMap(tokens: Coll[(TokenId, Long)]): TokensMap = - tokens.toArray.map(t => bytesToId(t._1) -> t._2).toMap + tokens.toArray.map(t => t._1.toModifierId -> t._2).toMap def tokensMapToColl(tokens: TokensMap): Coll[(TokenId, Long)] = - tokens.toSeq.map {t => (Digest32 @@ idToBytes(t._1)) -> t._2}.toArray.toColl + tokens.toArray.map {t => t._1.toTokenId -> t._2}.toColl private def validateStatelessChecks(inputs: IndexedSeq[ErgoBox], dataInputs: IndexedSeq[DataInput], outputCandidates: Seq[ErgoBoxCandidate]): Unit = { @@ -258,7 +252,7 @@ object TransactionBuilder { val actualFee = if (changeGoesToFee) fee + changeAmt else fee new ErgoBoxCandidate( actualFee, - ErgoScriptPredef.feeProposition(minerRewardDelay), + ErgoTreePredef.feeProposition(minerRewardDelay), currentHeight ) } diff --git a/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java b/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java index 2daa664862..35944bbf3b 100644 --- a/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java +++ b/ergo-wallet/src/test/java/org/ergoplatform/wallet/AddressGenerationDemo.java @@ -2,13 +2,15 @@ import org.ergoplatform.ErgoAddressEncoder; import org.ergoplatform.P2PKAddress; -import org.ergoplatform.wallet.mnemonic.Mnemonic; +import org.ergoplatform.sdk.wallet.secrets.DerivationPath; +import org.ergoplatform.sdk.wallet.secrets.ExtendedPublicKey; +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey; import org.ergoplatform.wallet.interface4j.SecretString; -import org.ergoplatform.wallet.secrets.DerivationPath; -import org.ergoplatform.wallet.secrets.ExtendedPublicKey; -import org.ergoplatform.wallet.secrets.ExtendedSecretKey; +import org.ergoplatform.wallet.mnemonic.Mnemonic; import scala.Option; +import static org.ergoplatform.sdk.wallet.Constants.eip3DerivationPath; + /** * This runnable example is showing how to derive change address and other addresses according to EIP-3 */ @@ -40,7 +42,7 @@ public static void main(String[] args) { ExtendedSecretKey rootSecret = masterSecretFromSeed(seed); // Let's use "m/44'/429'/0'/0/0" path for change (this path is from EIP-3 which is BIP-44 for Ergo) - DerivationPath changePath = Constants.eip3DerivationPath(); + DerivationPath changePath = eip3DerivationPath(); ExtendedSecretKey changeSecretKey = deriveSecretKey(rootSecret, changePath); ExtendedPublicKey changePubkey = changeSecretKey.publicKey(); P2PKAddress changeAddress = P2PKAddress.apply(changePubkey.key(), addressEncoder); diff --git a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java index a8f38ba40f..11ac65f6b0 100644 --- a/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java +++ b/ergo-wallet/src/test/java/org/ergoplatform/wallet/CreateTransactionDemo.java @@ -2,8 +2,8 @@ import io.circe.Json; import org.ergoplatform.*; +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey; import org.ergoplatform.wallet.interface4j.crypto.ErgoUnsafeProver; -import org.ergoplatform.wallet.secrets.ExtendedSecretKey; import org.ergoplatform.wallet.serialization.JsonCodecsWrapper; import org.ergoplatform.wallet.transactions.TransactionBuilder; import org.ergoplatform.wallet.transactions.TransactionBuilder.Payment; diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/AssetUtilsSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/AssetUtilsSpec.scala index 4ec358b8e1..4a8cec8b1b 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/AssetUtilsSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/AssetUtilsSpec.scala @@ -1,11 +1,12 @@ package org.ergoplatform.wallet +import org.ergoplatform.sdk.wallet.{AssetUtils, TokensMap} import org.ergoplatform.wallet.utils.WalletTestHelpers import org.scalatest.matchers.should.Matchers import org.scalatest.prop.TableDrivenPropertyChecks class AssetUtilsSpec extends WalletTestHelpers with Matchers with TableDrivenPropertyChecks { - import AssetUtils._ + import org.ergoplatform.sdk.wallet.AssetUtils._ val tid1 = stringToId("t1") val tid2 = stringToId("t2") diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala index aee68d6250..311275b06f 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/boxes/DefaultBoxSelectorSpec.scala @@ -1,19 +1,22 @@ package org.ergoplatform.wallet.boxes import org.ergoplatform.ErgoBox.TokenId -import org.ergoplatform.SigmaConstants.MaxBoxSize -import org.ergoplatform.wallet.Constants.{MaxAssetsPerBox, PaymentsScanId} import org.ergoplatform.ErgoLikeTransaction -import scorex.crypto.hash.{Blake2b256, Digest32} -import sigmastate.Values -import sigmastate.Values.SigmaPropValue -import sigmastate.helpers.TestingHelpers._ -import scorex.util.{ModifierId, bytesToId, idToBytes} +import org.ergoplatform.SigmaConstants.MaxBoxSize +import org.ergoplatform.sdk.wallet.Constants.MaxAssetsPerBox +import org.ergoplatform.wallet.Constants.PaymentsScanId +import org.ergoplatform.wallet.boxes.DefaultBoxSelector.{NotEnoughErgsError, NotEnoughTokensError} import org.scalatest.EitherValues -import org.ergoplatform.wallet.boxes.DefaultBoxSelector.NotEnoughErgsError -import org.ergoplatform.wallet.boxes.DefaultBoxSelector.NotEnoughTokensError import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec +import scorex.crypto.hash.Blake2b256 +import scorex.util.bytesToId +import sigmastate.Values +import sigmastate.Values.SigmaPropValue +import sigmastate.eval.Extensions._ +import sigmastate.helpers.TestingHelpers._ +import sigmastate.utils.Extensions._ +import special.collection.Extensions._ import scala.util.Random @@ -28,7 +31,7 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues private val StartHeight: Int = 0 private def genTokens(count: Int) = { - (0 until count).map { i => Digest32 @@ idToBytes(bytesToId(Blake2b256(i.toString))) -> i.toLong } + (0 until count).map { i => Blake2b256(i.toString).toTokenId -> i.toLong } } private val selector = new DefaultBoxSelector(None) @@ -98,9 +101,9 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues val assetId2 = bytesToId(Blake2b256("world")) val parentTx = ErgoLikeTransaction(IndexedSeq(), IndexedSeq()) - val box1 = testBox(1 * MinBoxValue, TrueLeaf, StartHeight, Seq(Digest32 @@ idToBytes(assetId1) -> 1)) - val box2 = testBox(10 * MinBoxValue, TrueLeaf, StartHeight, Seq(Digest32 @@ idToBytes(assetId2) -> 10)) - val box3 = testBox(100 * MinBoxValue, TrueLeaf, StartHeight, Seq(Digest32 @@ idToBytes(assetId1) -> 100)) + val box1 = testBox(1 * MinBoxValue, TrueLeaf, StartHeight, Seq(assetId1.toTokenId -> 1)) + val box2 = testBox(10 * MinBoxValue, TrueLeaf, StartHeight, Seq(assetId2.toTokenId -> 10)) + val box3 = testBox(100 * MinBoxValue, TrueLeaf, StartHeight, Seq(assetId1.toTokenId -> 100)) val uBox1 = TrackedBox(parentTx, 0, Some(100), box1, Set(PaymentsScanId)) val uBox2 = TrackedBox(parentTx, 1, None, box2, Set(PaymentsScanId)) @@ -146,17 +149,20 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues val assetId7 = bytesToId(Blake2b256("7")) val assetId8 = bytesToId(Blake2b256("8")) - val box1 = testBox(1 * MinBoxValue, TrueLeaf, StartHeight, - Seq(Digest32 @@ idToBytes(assetId1) -> 1, Digest32 @@ idToBytes(assetId2) -> 1, - Digest32 @@ idToBytes(assetId3) -> 1, Digest32 @@ idToBytes(assetId4) -> 1)) + val box1 = testBox( + 1 * MinBoxValue, TrueLeaf, StartHeight, + Seq(assetId1.toTokenId -> 1, assetId2.toTokenId -> 1, + assetId3.toTokenId -> 1, assetId4.toTokenId -> 1)) - val box2 = testBox(10 * MinBoxValue, TrueLeaf, StartHeight, - Seq(Digest32 @@ idToBytes(assetId5) -> 10, Digest32 @@ idToBytes(assetId6) -> 10, - Digest32 @@ idToBytes(assetId7) -> 10, Digest32 @@ idToBytes(assetId8) -> 10)) + val box2 = testBox( + 10 * MinBoxValue, TrueLeaf, StartHeight, + Seq(assetId5.toTokenId -> 10, assetId6.toTokenId -> 10, + assetId7.toTokenId -> 10, assetId8.toTokenId -> 10)) - val box3 = testBox(100 * MinBoxValue, TrueLeaf, StartHeight, - Seq(Digest32 @@ idToBytes(assetId3) -> 100, Digest32 @@ idToBytes(assetId4) -> 100, - Digest32 @@ idToBytes(assetId5) -> 100, Digest32 @@ idToBytes(assetId6) -> 100)) + val box3 = testBox( + 100 * MinBoxValue, TrueLeaf, StartHeight, + Seq(assetId3.toTokenId -> 100, assetId4.toTokenId -> 100, + assetId5.toTokenId -> 100, assetId6.toTokenId -> 100)) val uBox1 = TrackedBox(parentTx, 0, Some(100), box1, Set(PaymentsScanId)) val uBox2 = TrackedBox(parentTx, 1, None, box2, Set(PaymentsScanId)) @@ -193,7 +199,7 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues property("Size of a box with MaxAssetsPerBox tokens should not cross MaxBoxSize") { val tokens = (0 until MaxAssetsPerBox).map { _ => - (Digest32 @@ scorex.util.Random.randomBytes(TokenId.size), Random.nextInt(100000000).toLong) + (scorex.util.Random.randomBytes(TokenId.size).toTokenId, Random.nextInt(100000000).toLong) } val box = testBox(1 * MinBoxValue, TrueLeaf, StartHeight, tokens) assert(box.bytes.length <= MaxBoxSize.value) @@ -222,7 +228,7 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues val tokenData = genTokens(3).last tokenData._2 shouldBe 2 - val tokenId = ModifierId @@@ bytesToId(tokenData._1) + val tokenId = tokenData._1.toModifierId val ergValue = 10 * MinBoxValue @@ -254,7 +260,8 @@ class DefaultBoxSelectorSpec extends AnyPropSpec with Matchers with EitherValues val ts = genTokens(2) val reemissionNftId = ts(0)._1 val reemissionTokenId = ts(1)._1 - val selector = new DefaultBoxSelector(Some(ReemissionData(bytesToId(reemissionNftId), bytesToId(reemissionTokenId)))) + val selector = new DefaultBoxSelector( + Some(ReemissionData(reemissionNftId.toModifierId, reemissionTokenId.toModifierId))) val fullValue = 2000000000L val reemissionAmt = fullValue / 2 diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala index 123b70695e..0d96c4c8cc 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoProvingInterpreterSpec.scala @@ -1,15 +1,14 @@ package org.ergoplatform.wallet.interpreter -import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, UnsignedErgoLikeTransaction, UnsignedInput} +import org.ergoplatform.sdk.wallet.secrets.{DlogSecretKey, ExtendedSecretKey} import org.ergoplatform.wallet.crypto.ErgoSignature -import org.ergoplatform.wallet.secrets.{DlogSecretKey, ExtendedSecretKey} import org.ergoplatform.wallet.utils.Generators +import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, UnsignedErgoLikeTransaction, UnsignedInput} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -import scorex.util.ModifierId +import scorex.util.{ModifierId, Random} import scorex.util.encode.Base16 -import scorex.util.Random import sigmastate.CTHRESHOLD import sigmastate.Values.{GroupElementConstant, SigmaBoolean} import sigmastate.interpreter.{ContextExtension, HintsBag} diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala index e19690d8cb..c49445f83a 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/ErgoUnsafeProverSpec.scala @@ -1,7 +1,7 @@ package org.ergoplatform.wallet.interpreter +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey import org.ergoplatform.wallet.crypto.ErgoSignature -import org.ergoplatform.wallet.secrets.ExtendedSecretKey import org.ergoplatform.wallet.utils.Generators import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala index b1095e160e..d4af16e52d 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/interpreter/InterpreterSpecCommon.scala @@ -1,10 +1,10 @@ package org.ergoplatform.wallet.interpreter -import org.ergoplatform.wallet.protocol.context.{ErgoLikeParameters, ErgoLikeStateContext} +import org.ergoplatform.sdk.wallet.protocol.context.{ErgoLikeParameters, ErgoLikeStateContext} import scorex.crypto.authds.ADDigest import scorex.util.encode.Base16 +import sigmastate.basics.CryptoConstants import sigmastate.eval.{CGroupElement, CPreHeader, Colls} -import sigmastate.interpreter.CryptoConstants import special.collection.Coll import special.sigma.{Header, PreHeader} @@ -39,9 +39,9 @@ trait InterpreterSpecCommon { override def sigmaLastHeaders: Coll[Header] = Colls.emptyColl - override def previousStateDigest: ADDigest = Base16.decode("a5df145d41ab15a01e0cd3ffbab046f0d029e5412293072ad0f5827428589b9302") - .map(ADDigest @@ _) - .getOrElse(throw new Error(s"Failed to parse genesisStateDigest")) + override def previousStateDigest: ADDigest = + ADDigest @@ Base16.decode("a5df145d41ab15a01e0cd3ffbab046f0d029e5412293072ad0f5827428589b9302") + .getOrElse(throw new Error(s"Failed to parse genesisStateDigest")) override def sigmaPreHeader: PreHeader = CPreHeader( version = 0, diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala index 0a598b32a4..5b944630e7 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/DerivationPathSpec.scala @@ -1,10 +1,11 @@ package org.ergoplatform.wallet.secrets -import org.ergoplatform.wallet.Constants -import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress} -import org.ergoplatform.wallet.mnemonic.Mnemonic +import org.ergoplatform.sdk.wallet +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.ergoplatform.wallet.interface4j.SecretString +import org.ergoplatform.wallet.mnemonic.Mnemonic import org.ergoplatform.wallet.utils.Generators +import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress} import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks @@ -94,7 +95,7 @@ class DerivationPathSpec property("equality of old derivation") { // Check that hardcoded path from old codebase corresponds to the new string form (Constants.usePreEip3Derivation) - DerivationPath(Array(0, 1), publicBranch = false) shouldBe Constants.preEip3DerivationPath + DerivationPath(Array(0, 1), publicBranch = false) shouldBe wallet.Constants.preEip3DerivationPath } property("master key derivation") { @@ -120,12 +121,12 @@ class DerivationPathSpec } property("isEip3 correctly distinguishing") { - Constants.eip3DerivationPath.isEip3 shouldBe true - Constants.eip3DerivationPath.toPublicBranch.isEip3 shouldBe true + wallet.Constants.eip3DerivationPath.isEip3 shouldBe true + wallet.Constants.eip3DerivationPath.toPublicBranch.isEip3 shouldBe true DerivationPath.fromEncoded("m/44'/429'/0'/0/1").get.isEip3 shouldBe true DerivationPath.fromEncoded("M/44'/429'/0'/0/1").get.isEip3 shouldBe true DerivationPath.fromEncoded("m/44'/429'/0'/1/1").get.isEip3 shouldBe true - Constants.preEip3DerivationPath.isEip3 shouldBe false + wallet.Constants.preEip3DerivationPath.isEip3 shouldBe false DerivationPath.fromEncoded("m/44'/429'/1'/0/1").get.isEip3 shouldBe false } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala index 2cd869c66e..dedfdc1c9d 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedPublicKeySpec.scala @@ -1,5 +1,6 @@ package org.ergoplatform.wallet.secrets +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey import org.ergoplatform.wallet.utils.Generators import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala index f3ff7f401a..d59ff6c93a 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/ExtendedSecretKeySpec.scala @@ -1,14 +1,14 @@ package org.ergoplatform.wallet.secrets -import org.ergoplatform.wallet.mnemonic.Mnemonic +import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress} +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey, Index} import org.ergoplatform.wallet.interface4j.SecretString +import org.ergoplatform.wallet.mnemonic.Mnemonic import org.scalatest.Assertion import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks import scorex.util.encode.Base58 -import org.ergoplatform.P2PKAddress -import org.ergoplatform.ErgoAddressEncoder class ExtendedSecretKeySpec extends AnyPropSpec diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala index 50e473e284..39317270a6 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/secrets/JsonSecretStorageSpec.scala @@ -1,13 +1,14 @@ package org.ergoplatform.wallet.secrets +import org.ergoplatform.sdk.wallet.settings.EncryptionSettings +import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.settings.SecretStorageSettings -import org.ergoplatform.wallet.utils.{Generators, FileUtils} +import org.ergoplatform.wallet.utils.{FileUtils, Generators} +import org.scalacheck.Arbitrary import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks -import org.ergoplatform.wallet.interface4j.SecretString -import org.scalacheck.Arbitrary -import org.ergoplatform.wallet.settings.EncryptionSettings + import java.io.{File, PrintWriter} import java.util.UUID diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/serialization/SerializationSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/serialization/SerializationSpec.scala index b35bf51993..e49573dff0 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/serialization/SerializationSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/serialization/SerializationSpec.scala @@ -1,7 +1,7 @@ package org.ergoplatform.wallet.serialization +import org.ergoplatform.sdk.wallet.secrets.{DerivationPathSerializer, ExtendedPublicKeySerializer, ExtendedSecretKeySerializer} import org.ergoplatform.wallet.boxes.TrackedBoxSerializer -import org.ergoplatform.wallet.secrets.{ExtendedPublicKeySerializer, ExtendedSecretKeySerializer, DerivationPathSerializer} import org.ergoplatform.wallet.utils.Generators import org.scalacheck.Gen import org.scalatest.Assertion @@ -26,7 +26,7 @@ class SerializationSpec } property("DerivationPath serialization") { - checkSerializationRoundtrip(derivationPathGen, DerivationPathSerializer) + checkSerializationRoundtrip(derivationPathGen, ErgoWalletSerializer.fromSigmaSerializer(DerivationPathSerializer)) } property("TrackedBox serialization") { @@ -34,11 +34,11 @@ class SerializationSpec } property("ExtendedSecretKey serialization") { - checkSerializationRoundtrip(extendedSecretGen, ExtendedSecretKeySerializer) + checkSerializationRoundtrip(extendedSecretGen, ErgoWalletSerializer.fromSigmaSerializer(ExtendedSecretKeySerializer)) } property("ExtendedPublicKey serialization") { - checkSerializationRoundtrip(extendedPubKeyGen, ExtendedPublicKeySerializer) + checkSerializationRoundtrip(extendedPubKeyGen, ErgoWalletSerializer.fromSigmaSerializer(ExtendedPublicKeySerializer)) } } diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala index 9dfaad23a0..f3bb1b4cbc 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/transactions/TransactionBuilderSpec.scala @@ -1,24 +1,23 @@ package org.ergoplatform.wallet.transactions +import org.ergoplatform.ErgoBox.TokenId +import org.ergoplatform._ +import org.ergoplatform.sdk.wallet.TokensMap +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey +import org.ergoplatform.wallet.boxes.BoxSelector +import org.ergoplatform.wallet.interface4j.SecretString +import org.ergoplatform.wallet.mnemonic.Mnemonic +import org.ergoplatform.wallet.utils.WalletTestHelpers +import org.scalatest.matchers.should.Matchers import sigmastate.Values import sigmastate.Values.SigmaPropValue -import sigmastate.eval._ import sigmastate.eval.Extensions._ +import sigmastate.eval._ import sigmastate.helpers.TestingHelpers._ +import sigmastate.utils.Extensions._ import sigmastate.utils.Helpers._ -import org.ergoplatform._ -import org.ergoplatform.ErgoBox.TokenId -import org.ergoplatform.wallet.TokensMap import scala.util.{Success, Try} -import scorex.crypto.hash.Digest32 -import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.interface4j.SecretString -import org.ergoplatform.wallet.secrets.ExtendedSecretKey -import org.ergoplatform.wallet.boxes.BoxSelector -import org.ergoplatform.wallet.utils.WalletTestHelpers -import org.scalatest.matchers.should.Matchers -import scorex.util.idToBytes class TransactionBuilderSpec extends WalletTestHelpers with Matchers { import TransactionBuilder.buildUnsignedTx @@ -74,7 +73,7 @@ class TransactionBuilderSpec extends WalletTestHelpers with Matchers { property("token minting") { val inputBox = box(minBoxValue * 2) - val tokenId = Digest32 @@@ inputBox.id + val tokenId = inputBox.id.toTokenId val outBox = boxCandidate(minBoxValue, Seq(tokenId -> 100L)) val res = transaction(inputBox, outBox) @@ -86,8 +85,8 @@ class TransactionBuilderSpec extends WalletTestHelpers with Matchers { } property("token burning") { - val inputBox = box(minBoxValue * 3, Seq(Digest32 @@ idToBytes(tid1) -> 1000L, Digest32 @@ idToBytes(tid2) -> 2000L)) - val tokenId = Digest32 @@@ inputBox.id + val inputBox = box(minBoxValue * 3, Seq(tid1.toTokenId -> 1000L, tid2.toTokenId -> 2000L)) + val tokenId = inputBox.id.toTokenId val outBox = boxCandidate(minBoxValue, Seq(tokenId -> 100L)) val res = transaction(inputBox, outBox, burnTokens = Map(tid1 -> 400L, tid2 -> 800L)) @@ -105,7 +104,7 @@ class TransactionBuilderSpec extends WalletTestHelpers with Matchers { property("no fees") { val inputBox = box(minBoxValue) - val tokenId = Digest32 @@@ inputBox.id + val tokenId = inputBox.id.toTokenId val outBox = boxCandidate(minBoxValue, Seq(tokenId -> 100L)) val res = transaction(inputBox, outBox, fee = None) @@ -117,7 +116,7 @@ class TransactionBuilderSpec extends WalletTestHelpers with Matchers { property("change goes to fee, but no outFee box") { val inputBox = box(minBoxValue + minBoxValue / 2) - val tokenId = Digest32 @@@ inputBox.id + val tokenId = inputBox.id.toTokenId val outBox = boxCandidate(minBoxValue, Seq(tokenId -> 100L)) val res = transaction(inputBox, outBox, fee = None) diff --git a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala index 0a515eb024..fc8d6d3ab3 100644 --- a/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala +++ b/ergo-wallet/src/test/scala/org/ergoplatform/wallet/utils/Generators.scala @@ -1,31 +1,24 @@ package org.ergoplatform.wallet.utils -import org.ergoplatform.ErgoBox.{BoxId, NonMandatoryRegisterId} +import org.ergoplatform.ErgoBox.{BoxId, NonMandatoryRegisterId, TokenId} +import org.ergoplatform.sdk.wallet.secrets._ +import org.ergoplatform.sdk.wallet.settings.EncryptionSettings import org.ergoplatform.wallet.Constants +import org.ergoplatform.wallet.Constants.{PaymentsScanId, ScanId} import org.ergoplatform.wallet.boxes.TrackedBox import org.ergoplatform.wallet.mnemonic.{Mnemonic, WordList} -import org.ergoplatform.wallet.secrets.ExtendedPublicKey -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedSecretKey, Index, SecretKey} -import org.ergoplatform.wallet.settings.EncryptionSettings +import org.ergoplatform._ import org.scalacheck.Arbitrary.arbByte import org.scalacheck.{Arbitrary, Gen} import scorex.crypto.authds.ADKey +import scorex.util._ import sigmastate.Values.{ByteArrayConstant, CollectionConstant, ErgoTree, EvaluatedValue, FalseLeaf, TrueLeaf} import sigmastate.basics.DLogProtocol.ProveDlog -import sigmastate.{SByte, SType} -import org.ergoplatform.wallet.Constants.{ScanId, PaymentsScanId} -import scorex.util._ -import org.ergoplatform.ErgoBox -import org.ergoplatform.ErgoBoxCandidate -import org.ergoplatform.ErgoScriptPredef -import org.ergoplatform.UnsignedErgoLikeTransaction -import org.ergoplatform.UnsignedInput +import sigmastate.crypto.CryptoFacade.SecretKeyLength import sigmastate.eval.Extensions._ -import scorex.util.{ModifierId, bytesToId} import sigmastate.eval._ import sigmastate.helpers.TestingHelpers._ -import org.ergoplatform.ErgoBox.TokenId -import scorex.crypto.hash.Digest32 +import sigmastate.{SByte, SType} trait Generators { @@ -68,16 +61,16 @@ trait Generators { def genExactSizeBytes(size: Int): Gen[Array[Byte]] = genLimitedSizedBytes(size, size) val boxIdGen: Gen[BoxId] = { - val x = ADKey @@ genExactSizeBytes(Constants.ModifierIdLength) + val x = ADKey @@ genExactSizeBytes(sdk.wallet.Constants.ModifierIdLength) x } - val modIdGen: Gen[ModifierId] = genExactSizeBytes(Constants.ModifierIdLength).map(bytesToId) + val modIdGen: Gen[ModifierId] = genExactSizeBytes(sdk.wallet.Constants.ModifierIdLength).map(bytesToId) val assetGen: Gen[(TokenId, Long)] = for { id <- boxIdGen amt <- Gen.oneOf(1, 500, 20000, 10000000, Long.MaxValue) - } yield Digest32 @@@ id -> amt + } yield id.toTokenId -> amt def additionalTokensGen(cnt: Int): Gen[Seq[(TokenId, Long)]] = Gen.listOfN(cnt, assetGen) @@ -120,7 +113,7 @@ trait Generators { heightGen: Gen[Int] = heightGen): Gen[ErgoBox] = for { h <- heightGen prop <- propGen - transactionId: Array[Byte] <- genExactSizeBytes(Constants.ModifierIdLength) + transactionId: Array[Byte] <- genExactSizeBytes(sdk.wallet.Constants.ModifierIdLength) boxId: Short <- boxIndexGen ar <- additionalRegistersGen tokens <- tokensGen @@ -143,7 +136,7 @@ trait Generators { } yield DerivationPath(0 +: indices, isPublic) def extendedSecretGen: Gen[ExtendedSecretKey] = for { - seed <- Gen.const(Constants.SecretKeyLength).map(scorex.utils.Random.randomBytes) + seed <- Gen.const(SecretKeyLength).map(scorex.utils.Random.randomBytes) } yield ExtendedSecretKey.deriveMasterKey(seed, usePre1627KeyDerivation = false) def extendedPubKeyGen: Gen[ExtendedPublicKey] = extendedSecretGen.map(_.publicKey) @@ -189,7 +182,7 @@ trait Generators { h <- Gen.posNum[Int] out = new ErgoBoxCandidate( value, - ErgoScriptPredef.feeProposition(), + ErgoTreePredef.feeProposition(), h, Seq.empty[(ErgoBox.TokenId, Long)].toColl, Map.empty diff --git a/papers/emission.md b/papers/emission.md index 0450143f7e..cbff204297 100644 --- a/papers/emission.md +++ b/papers/emission.md @@ -97,12 +97,6 @@ second one is to pay mining fee supposedly (its value can be 0.01 ERG at most) val correctNftId = EQ(firstTokenId, ByteArrayConstant(reemissionNftId)) - // output of the reemission contract - val reemissionOut = ByIndex(Outputs, IntConstant(0)) - - // output to pay miner - val minerOut = ByIndex(Outputs, IntConstant(1)) - // miner's output must have script which is time-locking reward for miner's pubkey // box height must be the same as block height val correctMinerOutput = AND( diff --git a/papers/utxo.md b/papers/utxo.md index dc8cbbc6cc..d61085d057 100644 --- a/papers/utxo.md +++ b/papers/utxo.md @@ -23,9 +23,9 @@ UTXO set is authenticated via AVL+ tree. Design principles for tree construction [https://eprint.iacr.org/2016/994.pdf](https://eprint.iacr.org/2016/994.pdf), the implementation of the tree is available in [the Scrypto framework](https://github.com/input-output-hk/scrypto). -Time is broken into epochs, 1 epoch = 51,200 blocks (~72 days). +Time is broken into epochs, 1 epoch = 52,224 blocks (~72.5 days). Snapshot is taken after last block of an epoch, namely, after processing a block with -height *h % 51200 == 51199*. +height *h % 52224 == 52,223*. Chunk format ------------ diff --git a/src/main/resources/api/openapi.yaml b/src/main/resources/api/openapi.yaml index 54cb577468..6fd8a18919 100644 --- a/src/main/resources/api/openapi.yaml +++ b/src/main/resources/api/openapi.yaml @@ -1,7 +1,7 @@ openapi: "3.0.2" info: - version: "5.0.11" + version: "5.0.12" title: Ergo Node API description: API docs for Ergo Node. Models are shared between all Ergo products contact: @@ -96,6 +96,17 @@ components: bytes: $ref: '#/components/schemas/HexString' + SnapshotsInfo: + type: object + required: + - availableManifests + properties: + availableManifests: + description: Map of available manifests height -> manifestId + type: array + items: + type: object + ErgoTransactionOutput: type: object required: @@ -5077,6 +5088,28 @@ paths: schema: $ref: '#/components/schemas/ApiError' + /utxo/getSnapshotsInfo: + get: + summary: Get information about locally stored UTXO snapshots + operationId: getSnapshotsInfo + tags: + - utxo + responses: + '200': + description: A list of saved snapshots + content: + application/json: + schema: + $ref: '#/components/schemas/SnapshotsInfo' + default: + description: Error + content: + application/json: + schema: + $ref: '#/components/schemas/ApiError' + + + /utxo/genesis: get: summary: Get genesis boxes (boxes existed before the very first block) diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index fb508cee55..deaaa32dab 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -21,8 +21,19 @@ ergo { # otherwise, it could be hard to find proofs around the peers blocksToKeep = -1 - # Download and apply UTXO set snapshot and full-blocks after that - utxoBootstrap = false + utxo { + # Download and apply UTXO set snapshot and full-blocks after that + # Done in 5.0.12, but better to bootstrap since version 5.0.13 + utxoBootstrap = false + + # how many utxo set snapshots to store, 0 means that they are not stored at all + storingUtxoSnapshots = 0 + + # how many utxo set snapshots for a height with same id we need to find in p2p network + # in order to start downloading it + p2pUtxoSnapshots = 2 + } + # Download PoPoW proof on node bootstrap PoPoWBootstrap = false @@ -103,7 +114,7 @@ ergo { # } # # Before the height given (including it) validation of scripts is missed. - # This improving perfomance and memory usage during initial bootstrapping. + # This improving performance and memory usage during initial bootstrapping. # The node still applying transactions to UTXO set and so checks UTXO set digests for each block. # Block at checkpoint height is to be checked against expected one. checkpoint = null @@ -405,7 +416,7 @@ scorex { nodeName = "ergo-node" # Network protocol version to be sent in handshakes - appVersion = 5.0.11 + appVersion = 5.0.12 # Network agent name. May contain information about client code # stack, starting from core code-base up to the end graphical interface. diff --git a/src/main/resources/mainnet.conf b/src/main/resources/mainnet.conf index 960185d294..42683d7314 100644 --- a/src/main/resources/mainnet.conf +++ b/src/main/resources/mainnet.conf @@ -63,8 +63,8 @@ ergo { # # To validate all the scripts for all the blocks, set checkpoint = null. checkpoint = { - height = 856800 - blockId = "13335f1ec71a0b5511f30f03b283ca8c3549803c9b84a3a92106245b6f6282fb" + height = 1020454 + blockId = "7829c6513c5b319b86e87253ed29d51fed61597c65e32f5197434461ccc4c905" } # List with hex-encoded identifiers of transactions banned from getting into memory pool @@ -72,6 +72,18 @@ ergo { # maximum cost of transaction for it to be propagated maxTransactionCost = 4900000 + + utxo { + # Download and apply UTXO set snapshot and full-blocks after that + utxoBootstrap = false + + # how many utxo set snapshots to store, 0 means that they are not stored at all + storingUtxoSnapshots = 2 + + # how many utxo set snapshots for a height with same id we need to find in p2p network + # in order to start downloading it + p2pUtxoSnapshots = 2 + } } } diff --git a/src/main/scala/org/ergoplatform/ErgoApp.scala b/src/main/scala/org/ergoplatform/ErgoApp.scala index f7a905d0ba..f83eba0a56 100644 --- a/src/main/scala/org/ergoplatform/ErgoApp.scala +++ b/src/main/scala/org/ergoplatform/ErgoApp.scala @@ -59,28 +59,35 @@ class ErgoApp(args: Args) extends ScorexLogging { else None upnpGateway.foreach(_.addPort(scorexSettings.network.bindAddress.getPort)) - //an address to send to peers - private val externalSocketAddress: Option[InetSocketAddress] = - scorexSettings.network.declaredAddress orElse { - upnpGateway.map(u => - new InetSocketAddress(u.externalAddress, scorexSettings.network.bindAddress.getPort) - ) + // own address to send to peers + private val externalSocketAddress: Option[InetSocketAddress] = { + scorexSettings.network.declaredAddress orElse { + upnpGateway.map(u => + new InetSocketAddress(u.externalAddress, scorexSettings.network.bindAddress.getPort) + ) + } } - private val basicSpecs = { + // descriptors of p2p networking protocol messages + private val p2pMessageSpecifications = { Seq( GetPeersSpec, new PeersSpec(scorexSettings.network.maxPeerSpecObjects), + ErgoSyncInfoMessageSpec, InvSpec, RequestModifierSpec, - ModifiersSpec + ModifiersSpec, + GetSnapshotsInfoSpec, + SnapshotsInfoSpec, + GetManifestSpec, + ManifestSpec, + GetUtxoSnapshotChunkSpec, + UtxoSnapshotChunkSpec ) } - private val additionalMessageSpecs: Seq[MessageSpec[_]] = Seq(ErgoSyncInfoMessageSpec) - private val scorexContext = ScorexContext( - messageSpecs = basicSpecs ++ additionalMessageSpecs, + messageSpecs = p2pMessageSpecifications, upnpGateway = upnpGateway, externalNodeAddress = externalSocketAddress ) @@ -129,7 +136,14 @@ class ErgoApp(args: Args) extends ScorexLogging { InvSpec.messageCode -> ergoNodeViewSynchronizerRef, RequestModifierSpec.messageCode -> ergoNodeViewSynchronizerRef, ModifiersSpec.messageCode -> ergoNodeViewSynchronizerRef, - ErgoSyncInfoMessageSpec.messageCode -> ergoNodeViewSynchronizerRef + ErgoSyncInfoMessageSpec.messageCode -> ergoNodeViewSynchronizerRef, + // utxo set snapshot exchange related messages + GetSnapshotsInfoSpec.messageCode -> ergoNodeViewSynchronizerRef, + SnapshotsInfoSpec.messageCode -> ergoNodeViewSynchronizerRef, + GetManifestSpec.messageCode -> ergoNodeViewSynchronizerRef, + ManifestSpec.messageCode -> ergoNodeViewSynchronizerRef, + GetUtxoSnapshotChunkSpec.messageCode-> ergoNodeViewSynchronizerRef, + UtxoSnapshotChunkSpec.messageCode -> ergoNodeViewSynchronizerRef ) // Launching PeerSynchronizer actor which is then registering itself at network controller if (ergoSettings.scorexSettings.network.peerDiscovery) { diff --git a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala index 0649507c17..013a747a76 100644 --- a/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala +++ b/src/main/scala/org/ergoplatform/http/api/ApiCodecs.scala @@ -1,42 +1,42 @@ package org.ergoplatform.http.api -import java.math.BigInteger import io.circe._ +import io.circe.syntax._ import org.bouncycastle.util.BigIntegers -import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, ErgoLikeContext, ErgoLikeTransaction, JsonCodecs, UnsignedErgoLikeTransaction} -import org.ergoplatform.http.api.ApiEncoderOption.Detalization import org.ergoplatform.ErgoBox.RegisterId +import org.ergoplatform._ +import org.ergoplatform.http.api.ApiEncoderOption.Detalization +import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} import org.ergoplatform.mining.{groupElemFromBytes, groupElemToBytes} import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty -import org.ergoplatform.settings.ErgoAlgos +import org.ergoplatform.nodeView.history.extra.ExtraIndexer.getAddress +import org.ergoplatform.nodeView.history.extra.{BalanceInfo, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} import org.ergoplatform.nodeView.wallet.persistence.WalletDigest import org.ergoplatform.nodeView.wallet.requests.{ExternalSecret, GenerateCommitmentsRequest, TransactionSigningRequest} -import org.ergoplatform.settings.Algos +import org.ergoplatform.sdk.wallet.secrets.{DhtSecretKey, DlogSecretKey} +import org.ergoplatform.settings.{Algos, ErgoAlgos} import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.TrackedBox +import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.TransactionHintsBag -import org.ergoplatform.wallet.secrets.{DhtSecretKey, DlogSecretKey} import scorex.core.validation.ValidationResult +import scorex.crypto.authds.merkle.MerkleProof +import scorex.crypto.authds.{LeafData, Side} +import scorex.crypto.hash.Digest import scorex.util.encode.Base16 -import sigmastate.{CAND, COR, CTHRESHOLD, NodePosition, SigSerializer, TrivialProp} import sigmastate.Values.SigmaBoolean +import sigmastate._ +import sigmastate.basics.CryptoConstants.EcPointType import sigmastate.basics.DLogProtocol.{DLogProverInput, FirstDLogProverMessage, ProveDlog} import sigmastate.basics.VerifierMessage.Challenge import sigmastate.basics._ import sigmastate.interpreter._ -import sigmastate.interpreter.CryptoConstants.EcPointType -import io.circe.syntax._ -import org.ergoplatform.http.api.requests.{CryptoResult, ExecuteRequest, HintExtractionRequest} -import org.ergoplatform.nodeView.history.extra.ExtraIndexer.getAddress -import org.ergoplatform.nodeView.history.extra.{BalanceInfo, IndexedErgoBox, IndexedErgoTransaction, IndexedToken} -import org.ergoplatform.wallet.interface4j.SecretString -import scorex.crypto.authds.{LeafData, Side} -import scorex.crypto.authds.merkle.MerkleProof -import scorex.crypto.hash.Digest import sigmastate.serialization.OpCodes import special.sigma.AnyValue - +import org.ergoplatform.nodeView.state.SnapshotsInfo +import org.ergoplatform.nodeView.state.UtxoState.ManifestId +import java.math.BigInteger import scala.util.{Failure, Success, Try} @@ -405,6 +405,20 @@ trait ApiCodecs extends JsonCodecs { } yield TransactionHintsBag(secretHints.mapValues(HintsBag.apply), publicHints.mapValues(HintsBag.apply)) } + implicit val SnapshotInfoEncoder: Encoder[SnapshotsInfo] = { si => + Json.obj( + "availableManifests" -> si.availableManifests.map { case (height, manifest) => + height -> manifest + }.asJson + ) + } + + implicit val SnapshotInfoDecoder: Decoder[SnapshotsInfo] = { cursor => + for { + availableManifests <- Decoder.decodeMap[Int, ManifestId].tryDecode(cursor.downField("availableManifests")) + } yield new SnapshotsInfo(availableManifests) + } + implicit val transactionSigningRequestEncoder: Encoder[TransactionSigningRequest] = { tsr => Json.obj( "tx" -> tsr.unsignedTx.asJson, diff --git a/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala index 0922a83a8f..0a3f370868 100644 --- a/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/EmissionApiRoute.scala @@ -2,13 +2,13 @@ package org.ergoplatform.http.api import akka.actor.ActorRefFactory import akka.http.scaladsl.server.Route +import io.circe.syntax._ import io.circe.{Encoder, Json} -import org.ergoplatform.{ErgoAddressEncoder, ErgoScriptPredef, Pay2SAddress} import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.settings.{ErgoSettings, ReemissionSettings} +import org.ergoplatform.{ErgoAddressEncoder, ErgoTreePredef, Pay2SAddress} import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings -import io.circe.syntax._ case class EmissionApiRoute(ergoSettings: ErgoSettings) (implicit val context: ActorRefFactory) extends ErgoBaseApiRoute { @@ -43,7 +43,7 @@ case class EmissionApiRoute(ergoSettings: ErgoSettings) ApiResponse( Json.obj( - "emission" -> Pay2SAddress(ErgoScriptPredef.emissionBoxProp(ms)).toString().asJson, + "emission" -> Pay2SAddress(ErgoTreePredef.emissionBoxProp(ms)).toString().asJson, "reemission" -> Pay2SAddress(reemissionSettings.reemissionRules.reemissionBoxProp(ms)).toString().asJson, "pay2Reemission" -> Pay2SAddress(reemissionSettings.reemissionRules.payToReemission).toString().asJson ) diff --git a/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala index 62f862bab2..03e7d543a2 100644 --- a/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/MiningApiRoute.scala @@ -10,7 +10,7 @@ import org.ergoplatform.mining.{AutolykosSolution, CandidateGenerator, ErgoMiner import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.nodeView.wallet.ErgoAddressJsonEncoder import org.ergoplatform.settings.ErgoSettings -import org.ergoplatform.{ErgoAddress, ErgoScriptPredef, Pay2SAddress} +import org.ergoplatform.{ErgoAddress, ErgoTreePredef, Pay2SAddress} import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings import sigmastate.basics.DLogProtocol.ProveDlog @@ -68,7 +68,7 @@ case class MiningApiRoute(miner: ActorRef, miner.askWithStatus(ErgoMiner.ReadMinerPk) .mapTo[ProveDlog] .map { pk => - val script = ErgoScriptPredef.rewardOutputScript(ergoSettings.chainSettings.monetary.minerRewardDelay, pk) + val script = ErgoTreePredef.rewardOutputScript(ergoSettings.chainSettings.monetary.minerRewardDelay, pk) Pay2SAddress(script)(ergoSettings.addressEncoder) } diff --git a/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala index 6bbaaa4666..3f5679c499 100644 --- a/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/TransactionsApiRoute.scala @@ -17,10 +17,10 @@ import scorex.core.api.http.ApiError.BadRequest import scorex.core.api.http.{ApiError, ApiResponse} import scorex.core.settings.RESTApiSettings import scorex.crypto.authds.ADKey -import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 import sigmastate.SType import sigmastate.Values.EvaluatedValue +import sigmastate.eval.Extensions.ArrayByteOps import scala.concurrent.Future import scala.util.{Failure, Success} @@ -48,9 +48,9 @@ case class TransactionsApiRoute(readersHolder: ActorRef, val tokenId: Directive1[TokenId] = pathPrefix(Segment).flatMap(handleTokenId) private def handleTokenId(value: String): Directive1[TokenId] = { - Digest32 @@ Algos.decode(value) match { + Algos.decode(value) match { case Success(tokenId) => - provide(tokenId) + provide(tokenId.toTokenId) case _ => reject(ValidationRejection(s"tokenId $value is invalid, it should be 64 chars long hex string")) } @@ -244,7 +244,8 @@ case class TransactionsApiRoute(readersHolder: ActorRef, def getUnconfirmedOutputByTokenIdR: Route = (pathPrefix("unconfirmed" / "outputs" / "byTokenId") & get & tokenId) { tokenId => ApiResponse( - getMemPool.map(_.getAll.flatMap(_.transaction.outputs.filter(_.additionalTokens.exists(_._1.sameElements(tokenId))))) + getMemPool.map(_.getAll.flatMap(unconfirmed => + unconfirmed.transaction.outputs.filter(_.additionalTokens.exists(_._1 == tokenId)))) ) } diff --git a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala index 8d9c813fe2..129d0bf809 100644 --- a/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala +++ b/src/main/scala/org/ergoplatform/http/api/UtxoApiRoute.scala @@ -6,7 +6,7 @@ import akka.pattern.ask import org.ergoplatform.ErgoBox import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader -import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} +import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoSetSnapshotPersistence, UtxoStateReader} import org.ergoplatform.wallet.boxes.ErgoBoxSerializer import scorex.core.api.http.ApiResponse import scorex.core.settings.RESTApiSettings @@ -24,7 +24,7 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS (readersHolder ? GetReaders).mapTo[Readers].map(rs => (rs.s, rs.m)) override val route: Route = pathPrefix("utxo") { - byId ~ serializedById ~ genesis ~ withPoolById ~ withPoolSerializedById ~ getBoxesBinaryProof + byId ~ serializedById ~ genesis ~ withPoolById ~ withPoolSerializedById ~ getBoxesBinaryProof ~ getSnapshotsInfo } def withPoolById: Route = (get & path("withPool" / "byId" / Segment)) { id => @@ -82,4 +82,17 @@ case class UtxoApiRoute(readersHolder: ActorRef, override val settings: RESTApiS case _ => None }) } + + /** + * Handler for /utxo/getSnapshotsInfo API call which is providing list of + * UTXO set snapshots stored locally + */ + def getSnapshotsInfo: Route = (get & path("getSnapshotsInfo")) { + ApiResponse(getState.map { + case usr: UtxoSetSnapshotPersistence => + Some(usr.getSnapshotInfo()) + case _ => None + }) + } + } diff --git a/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala b/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala index fd9b853a9c..858bc51d7b 100644 --- a/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala +++ b/src/main/scala/org/ergoplatform/mining/AutolykosPowScheme.scala @@ -7,11 +7,11 @@ import org.ergoplatform.mining.difficulty.DifficultySerializer import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.extension.ExtensionCandidate +import org.ergoplatform.modifiers.history.header.Header.{Timestamp, Version} import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer, HeaderWithoutPow} import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.mempool.TransactionMembershipProof -import org.ergoplatform.modifiers.history.header.Header.{Timestamp, Version} import scorex.crypto.authds.{ADDigest, SerializedAdProof} import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.util.{ModifierId, ScorexLogging} @@ -19,7 +19,6 @@ import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.crypto.CryptoFacade import scala.annotation.tailrec -import scala.math.BigInt import scala.util.Try /** @@ -399,7 +398,7 @@ class AutolykosPowScheme(val k: Int, val n: Int) extends ScorexLogging { val proofs = if (mandatoryTxIds.nonEmpty) { // constructs fake block transactions section (BlockTransactions instance) to get proofs from it - val fakeHeaderId = scorex.util.bytesToId(Array.fill(org.ergoplatform.wallet.Constants.ModifierIdLength)(0: Byte)) + val fakeHeaderId = scorex.util.bytesToId(Array.fill(org.ergoplatform.sdk.wallet.Constants.ModifierIdLength)(0: Byte)) val bt = BlockTransactions(fakeHeaderId, blockCandidate.version, blockCandidate.transactions) val ps = mandatoryTxIds.flatMap { txId => bt.proofFor(txId).map(mp => TransactionMembershipProof(txId, mp)) } Some(ProofOfUpcomingTransactions(headerCandidate, ps)) diff --git a/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala b/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala index 25d0e40498..13917b06a7 100644 --- a/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala +++ b/src/main/scala/org/ergoplatform/mining/AutolykosSolution.scala @@ -4,12 +4,12 @@ import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.bouncycastle.util.BigIntegers import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.settings.Algos import org.ergoplatform.modifiers.history.header.Header.Version +import org.ergoplatform.settings.Algos import scorex.core.serialization.ErgoSerializer import scorex.util.serialization.{Reader, Writer} -import sigmastate.interpreter.CryptoConstants -import sigmastate.interpreter.CryptoConstants.EcPointType +import sigmastate.basics.CryptoConstants +import sigmastate.basics.CryptoConstants.EcPointType /** * Solution for an Autolykos PoW puzzle. @@ -90,7 +90,7 @@ class AutolykosV1SolutionSerializer extends ErgoSerializer[AutolykosSolution] { */ class AutolykosV2SolutionSerializer extends ErgoSerializer[AutolykosSolution] { - import AutolykosSolution.{wForV2, dForV2} + import AutolykosSolution.{dForV2, wForV2} override def serialize(obj: AutolykosSolution, w: Writer): Unit = { w.putBytes(groupElemToBytes(obj.pk)) diff --git a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala index d6b3dfb7b3..91fc5c3ad4 100644 --- a/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala +++ b/src/main/scala/org/ergoplatform/mining/CandidateGenerator.scala @@ -12,18 +12,17 @@ import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.header.{Header, HeaderWithoutPow} import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages -import ReceivableMessages.{ChangedHistory, ChangedMempool, ChangedState, FullBlockApplied, NodeViewChange} +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ +import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{EliminateTransactions, LocallyGeneratedModifier} import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.history.ErgoHistory.Height import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.nodeView.state.{ErgoState, ErgoStateContext, StateType, UtxoStateReader} import org.ergoplatform.settings.{ErgoSettings, ErgoValidationSettingsUpdate, Parameters} -import org.ergoplatform.wallet.Constants.MaxAssetsPerBox +import org.ergoplatform.sdk.wallet.Constants.MaxAssetsPerBox import org.ergoplatform.wallet.interpreter.ErgoInterpreter -import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, Input} -import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{EliminateTransactions, LocallyGeneratedModifier} +import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input} import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 import scorex.util.{ModifierId, ScorexLogging} @@ -666,11 +665,11 @@ object CandidateGenerator extends ScorexLogging { val reemissionRules = reemissionSettings.reemissionRules val eip27ActivationHeight = reemissionSettings.activationHeight - val reemissionTokenId = Digest32 @@ reemissionSettings.reemissionTokenIdBytes + val reemissionTokenId = Digest32Coll @@ reemissionSettings.reemissionTokenIdBytes val nextHeight = currentHeight + 1 val minerProp = - ErgoScriptPredef.rewardOutputScript(emission.settings.minerRewardDelay, minerPk) + ErgoTreePredef.rewardOutputScript(emission.settings.minerRewardDelay, minerPk) val emissionTxOpt: Option[ErgoTransaction] = emissionBoxOpt.map { emissionBox => val prop = emissionBox.ergoTree diff --git a/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala b/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala index 861dbbb39a..f318a4f426 100644 --- a/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala +++ b/src/main/scala/org/ergoplatform/mining/DefaultFakePowScheme.scala @@ -3,9 +3,8 @@ package org.ergoplatform.mining import org.ergoplatform.modifiers.history.header.Header import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 -import sigmastate.interpreter.CryptoConstants.EcPointType +import sigmastate.basics.CryptoConstants.EcPointType -import scala.math.BigInt import scala.util.{Random, Success, Try} /** diff --git a/src/main/scala/org/ergoplatform/mining/mining.scala b/src/main/scala/org/ergoplatform/mining/mining.scala index 9b3a7fb3d3..3fc4be9f3e 100644 --- a/src/main/scala/org/ergoplatform/mining/mining.scala +++ b/src/main/scala/org/ergoplatform/mining/mining.scala @@ -2,10 +2,9 @@ package org.ergoplatform import org.bouncycastle.util.BigIntegers import scorex.crypto.hash.Blake2b256 -import sigmastate.basics.BcDlogGroup +import sigmastate.basics.CryptoConstants.EcPointType +import sigmastate.basics.{BcDlogGroup, CryptoConstants} import sigmastate.basics.DLogProtocol.DLogProverInput -import sigmastate.interpreter.CryptoConstants -import sigmastate.interpreter.CryptoConstants.EcPointType import sigmastate.serialization.{GroupElementSerializer, SigmaSerializer} package object mining { diff --git a/src/main/scala/org/ergoplatform/modifiers/BlockSection.scala b/src/main/scala/org/ergoplatform/modifiers/BlockSection.scala index 3a65907c5e..fea7be63e3 100644 --- a/src/main/scala/org/ergoplatform/modifiers/BlockSection.scala +++ b/src/main/scala/org/ergoplatform/modifiers/BlockSection.scala @@ -22,4 +22,6 @@ object BlockSection { case other => throw new Exception(s"Unknown block section type: $other") } + /** Immutable empty array can be shared to avoid allocations. */ + val emptyArray: Array[BlockSection] = Array.empty[BlockSection] } diff --git a/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala b/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala index 8e258d96d6..3b6ce1b50d 100644 --- a/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala +++ b/src/main/scala/org/ergoplatform/modifiers/NetworkObjectTypeId.scala @@ -10,7 +10,7 @@ sealed trait NetworkObjectTypeId { /** * 1-byte ID of network object type */ - val value: NetworkObjectTypeId.Value + def value: NetworkObjectTypeId.Value } object NetworkObjectTypeId { @@ -19,26 +19,45 @@ object NetworkObjectTypeId { @inline def fromByte(value: Byte): Value = Value @@ value + + /** + * Threshold for block section type ids + * Block section could have ids >= this threshold only + * Other p2p network objects have type id below the threshold + */ + val BlockSectionThreshold: Value = Value @@ 50.toByte + + /** + * Whether network object type corresponding to block sections, returns true if so + */ + def isBlockSection(typeId: Value): Boolean = { + typeId >= BlockSectionThreshold + } + } /** - * Unconfirmed transactions sent outside blocks + * Block section to be sent over the wire (header, transactions section, extension, UTXO set transformation proofs) */ -object TransactionTypeId extends NetworkObjectTypeId { - override val value: Value = fromByte(2) -} +sealed trait BlockSectionTypeId extends NetworkObjectTypeId + +/** + * Non-block network objects: unconfirmed transactions, utxo set snapshot related data, nipopow related data etc + */ +sealed trait AuxiliaryTypeId extends NetworkObjectTypeId + /** * Block header, section of a block PoW is done on top of. This section is committing to other sections */ -object HeaderTypeId extends NetworkObjectTypeId { +object HeaderTypeId extends BlockSectionTypeId { override val value: Value = fromByte(101) } /** * Block transactions sections. Contains all the transactions for a block. */ -object BlockTransactionsTypeId extends NetworkObjectTypeId { +object BlockTransactionsTypeId extends BlockSectionTypeId { override val value: Value = fromByte(102) } @@ -46,7 +65,7 @@ object BlockTransactionsTypeId extends NetworkObjectTypeId { * Block section which contains proofs of correctness for UTXO set transformations. * The section contains proofs for all the transformations (i.e. for all the block transactions) */ -object ProofsTypeId extends NetworkObjectTypeId { +object ProofsTypeId extends BlockSectionTypeId { override val value: Value = fromByte(104) } @@ -56,15 +75,43 @@ object ProofsTypeId extends NetworkObjectTypeId { * Interlinks vector (for nipopow proofs) written there, as well as current network parameters * (at the beginning of voting epoch), but miners can also put arbitrary data there. */ -object ExtensionTypeId extends NetworkObjectTypeId { +object ExtensionTypeId extends BlockSectionTypeId { override val value: Value = fromByte(108) } +/** + * Unconfirmed transactions sent outside blocks + */ +object TransactionTypeId extends AuxiliaryTypeId { + override val value: Value = fromByte(2) +} + /** * Virtual object which is not being sent over the wire rather, constructed locally from different sections * got over the wire (header, transactions, extension in the "utxo" mode, those three sections plus proofs in * the "digest" mode). */ -object FullBlockTypeId extends NetworkObjectTypeId { +object FullBlockTypeId extends AuxiliaryTypeId { override val value: Value = fromByte(-127) } + +/** + * Not a block section, but a chunk of UTXO set + */ +object UtxoSnapshotChunkTypeId extends AuxiliaryTypeId { + override val value: Value = fromByte(-126) +} + +/** + * Not a block section, but registry of UTXO set snapshots available + */ +object SnapshotsInfoTypeId extends AuxiliaryTypeId { + override val value: Value = fromByte(-125) +} + +/** + * Not a block section, but manifest of a UTXO set snapshot + */ +object ManifestTypeId extends AuxiliaryTypeId { + override val value: Value = fromByte(-124) +} diff --git a/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala b/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala index 6208a182f1..8387a14291 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/PreHeader.scala @@ -6,9 +6,9 @@ import org.ergoplatform.modifiers.history.header.Header._ import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.settings.Constants import scorex.util._ +import sigmastate.basics.CryptoConstants.EcPointType import sigmastate.eval.CGroupElement import sigmastate.eval.Extensions._ -import sigmastate.interpreter.CryptoConstants.EcPointType /** * Only header fields that can be predicted by a miner @@ -63,7 +63,7 @@ object PreHeader { } /** - * fake pre-header, which is used in ErgoStateContext if last headers are empty + * Fake pre-header, which is used in ErgoStateContext if last headers are empty * because ErgoStateContext needs a PreHeader and it's not optional. * See ErgoStateContext.currentHeight returns a height from the passed PreHeader */ diff --git a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala index 13b5d0112b..2d82a70f74 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/header/Header.scala @@ -3,8 +3,8 @@ package org.ergoplatform.modifiers.history.header import io.circe.syntax._ import io.circe.{Decoder, Encoder, HCursor} import org.ergoplatform.http.api.ApiCodecs -import org.ergoplatform.mining.difficulty.DifficultySerializer import org.ergoplatform.mining.AutolykosSolution +import org.ergoplatform.mining.difficulty.DifficultySerializer import org.ergoplatform.modifiers.history.extension.Extension import org.ergoplatform.modifiers.history.{ADProofs, BlockTransactions, PreHeader} import org.ergoplatform.modifiers.{BlockSection, HeaderTypeId, NetworkObjectTypeId, NonHeaderBlockSection} @@ -16,9 +16,9 @@ import scorex.core.serialization.ErgoSerializer import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.util._ +import sigmastate.basics.CryptoConstants.EcPointType import sigmastate.eval.Extensions._ import sigmastate.eval.{CAvlTree, CBigInt, CGroupElement, CHeader} -import sigmastate.interpreter.CryptoConstants.EcPointType import scala.concurrent.duration.FiniteDuration diff --git a/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala b/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala index c45fcdbfda..95aad0a2c9 100644 --- a/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala +++ b/src/main/scala/org/ergoplatform/modifiers/history/popow/PoPowHeader.scala @@ -137,7 +137,7 @@ object PoPowHeader { * Binary serializer for PoPowHeader */ object PoPowHeaderSerializer extends ErgoSerializer[PoPowHeader] { - import org.ergoplatform.wallet.Constants.ModifierIdLength + import org.ergoplatform.sdk.wallet.Constants.ModifierIdLength implicit val hf: HF = Algos.hash val merkleProofSerializer = new BatchMerkleProofSerializer[Digest32, HF] diff --git a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala index 80125bdaff..55fa39b005 100644 --- a/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala +++ b/src/main/scala/org/ergoplatform/modifiers/mempool/ErgoTransaction.scala @@ -6,18 +6,19 @@ import org.ergoplatform.SigmaConstants.{MaxBoxSize, MaxPropositionBytes} import org.ergoplatform._ import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.mining.emission.EmissionRules -import org.ergoplatform.modifiers.{ErgoNodeViewModifier, NetworkObjectTypeId, TransactionTypeId} import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.ErgoTransaction.unresolvedIndices +import org.ergoplatform.modifiers.{ErgoNodeViewModifier, NetworkObjectTypeId, TransactionTypeId} import org.ergoplatform.nodeView.ErgoContext import org.ergoplatform.nodeView.state.ErgoStateContext +import org.ergoplatform.sdk.utils.ArithUtils.{addExact, multiplyExact} +import org.ergoplatform.sdk.wallet.protocol.context.TransactionContext import org.ergoplatform.settings.ValidationRules._ import org.ergoplatform.settings.{Algos, ErgoValidationSettings} -import org.ergoplatform.utils.ArithUtils._ import org.ergoplatform.utils.BoxUtils import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor import org.ergoplatform.wallet.interpreter.ErgoInterpreter -import org.ergoplatform.wallet.protocol.context.{InputContext, TransactionContext} +import org.ergoplatform.wallet.protocol.context.InputContext import org.ergoplatform.wallet.serialization.JsonCodecsWrapper import scorex.core.EphemerealNodeViewModifier import scorex.core.serialization.ErgoSerializer @@ -273,11 +274,11 @@ case class ErgoTransaction(override val inputs: IndexedSeq[Input], val firstEmissionBoxTokenId = emissionOut.additionalTokens.apply(0)._1 val secondEmissionBoxTokenId = emissionOut.additionalTokens.apply(1)._1 require( - firstEmissionBoxTokenId.sameElements(emissionNftIdBytes), + firstEmissionBoxTokenId == emissionNftIdBytes, "No emission box NFT in the emission box" ) require( - secondEmissionBoxTokenId.sameElements(reemissionTokenIdBytes), + secondEmissionBoxTokenId == reemissionTokenIdBytes, "No re-emission token in the emission box" ) diff --git a/src/main/scala/org/ergoplatform/network/ElementPartitioner.scala b/src/main/scala/org/ergoplatform/network/ElementPartitioner.scala index de0ad54be2..939c45f71d 100644 --- a/src/main/scala/org/ergoplatform/network/ElementPartitioner.scala +++ b/src/main/scala/org/ergoplatform/network/ElementPartitioner.scala @@ -7,48 +7,41 @@ object ElementPartitioner { /** * Evenly distributes elements under unique bucket-type keys given min/max limits + * * @param buckets to distribute elements into - * @param maxElements maximum elements to fetch * @param minElementsPerBucket minimum elements to distribute per bucket - * @param maxElementsPerBucket maximum elements to distribute per bucket - * @param fetchMaxElems function that returns elements by type, given a limit (it depends on interpretation of the provider) + * @param fetchedElems elements to distribute over the buckets * @return elements evenly grouped under unique bucket-type keys */ - def distribute[B, T, I]( - buckets: Iterable[B], - maxElements: Int, - minElementsPerBucket: Int, - maxElementsPerBucket: Int - )(fetchMaxElems: Int => Map[T, Seq[I]]): Map[(B, T), Seq[I]] = { - - if (buckets.isEmpty) { + def distribute[B, T, I](buckets: Iterable[B], + minElementsPerBucket: Int, + fetchedElems: Map[T, Seq[I]]): Map[(B, T), Seq[I]] = { + val bucketsCount = buckets.size + if (bucketsCount == 0) { Map.empty } else { - val bucketsCount = buckets.size - val maxElementsToFetch = Math.min(maxElements, bucketsCount * maxElementsPerBucket) - if (maxElementsToFetch <= 0) { - Map.empty - } else { - fetchMaxElems(maxElementsToFetch).foldLeft(Map.empty[(B, T), Seq[I]]) { - case (acc, (elemType, elements)) => - val elementsSize = elements.size - if (elementsSize < 1) { - acc - } else { - val lessBuckets = - if (elementsSize / bucketsCount < minElementsPerBucket) { - buckets.take(Math.max(elementsSize / minElementsPerBucket, 1)) // there must always be at least one bucket - } else buckets - // now let's distribute elements evenly into buckets - val (quot, rem) = - (elementsSize / lessBuckets.size, elementsSize % lessBuckets.size) - val (smaller, bigger) = elements.splitAt(elementsSize - rem * (quot + 1)) - acc ++ lessBuckets - .zip((smaller.grouped(quot) ++ bigger.grouped(quot + 1)).toSeq) - .map { case (p, elems) => (p, elemType) -> elems } - } - } + fetchedElems.foldLeft(Map.empty[(B, T), Seq[I]]) { + case (acc, (elemType, elements)) => + val elementsSize = elements.size + if (elementsSize < 1) { + acc + } else { + val lessBuckets = + if (elementsSize / bucketsCount < minElementsPerBucket) { + buckets.take(Math.max(elementsSize / minElementsPerBucket, 1)) // there must always be at least one bucket + } else { + buckets + } + // now let's distribute elements evenly into buckets + val (quot, rem) = + (elementsSize / lessBuckets.size, elementsSize % lessBuckets.size) + val (smaller, bigger) = elements.splitAt(elementsSize - rem * (quot + 1)) + acc ++ lessBuckets + .zip((smaller.grouped(quot) ++ bigger.grouped(quot + 1)).toSeq) + .map { case (p, elems) => (p, elemType) -> elems } + } } } } + } diff --git a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala index 2eb565492c..ede79994ba 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoNodeViewSynchronizer.scala @@ -4,27 +4,29 @@ import akka.actor.SupervisorStrategy.{Restart, Stop} import akka.actor.{Actor, ActorInitializationException, ActorKilledException, ActorRef, ActorRefFactory, DeathPactException, OneForOneStrategy, Props} import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer, UnconfirmedTransaction} -import org.ergoplatform.modifiers.{BlockSection, NetworkObjectTypeId} +import org.ergoplatform.modifiers.{BlockSection, ManifestTypeId, NetworkObjectTypeId, SnapshotsInfoTypeId, UtxoSnapshotChunkTypeId} import org.ergoplatform.nodeView.history.{ErgoSyncInfoV1, ErgoSyncInfoV2} import org.ergoplatform.nodeView.history._ import ErgoNodeViewSynchronizer.{CheckModifiersToDownload, IncomingTxInfo, TransactionProcessingCacheRecord} import org.ergoplatform.nodeView.ErgoNodeViewHolder.BlockAppliedTransactions import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoSyncInfo, ErgoSyncInfoMessageSpec} import org.ergoplatform.nodeView.mempool.{ErgoMemPool, ErgoMemPoolReader} -import org.ergoplatform.settings.{Constants, ErgoSettings} +import org.ergoplatform.settings.{Algos, Constants, ErgoSettings} import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages._ +import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{ChainIsHealthy, ChainIsStuck, GetNodeViewChanges, IsChainHealthy, ModifiersFromRemote} import org.ergoplatform.nodeView.ErgoNodeViewHolder._ import scorex.core.consensus.{Equal, Fork, Nonsense, Older, Unknown, Younger} import scorex.core.network.ModifiersStatus.Requested import scorex.core.{NodeViewModifier, idsToString} import scorex.core.network.NetworkController.ReceivableMessages.{PenalizePeer, SendToNetwork} import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ -import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoStateReader} +import org.ergoplatform.nodeView.state.{ErgoStateReader, SnapshotsInfo, UtxoSetSnapshotPersistence, UtxoStateReader} +import scorex.core.network.message._ import org.ergoplatform.nodeView.wallet.ErgoWalletReader import scorex.core.network.message.{InvSpec, MessageSpec, ModifiersSpec, RequestModifierSpec} import scorex.core.network._ +import scorex.core.network.{ConnectedPeer, ModifiersStatus, SendToPeer, SendToPeers} import scorex.core.network.message.{InvData, Message, ModifiersData} -import scorex.core.serialization.ErgoSerializer import scorex.core.settings.NetworkSettings import scorex.core.utils.ScorexEncoding import scorex.core.validation.MalformedModifierError @@ -33,12 +35,16 @@ import scorex.core.network.DeliveryTracker import scorex.core.network.peer.PenaltyType import scorex.core.transaction.state.TransactionValidation.TooHighCostError import scorex.core.app.Version - +import scorex.crypto.hash.Digest32 +import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId} +import org.ergoplatform.ErgoLikeContext.Height +import scorex.core.serialization.{ErgoSerializer, ManifestSerializer, SubtreeSerializer} +import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage.splitDigest import scala.annotation.tailrec import scala.collection.mutable import scala.concurrent.ExecutionContext import scala.concurrent.duration._ -import scala.util.{Failure, Success} +import scala.util.{Failure, Random, Success} /** * Contains most top-level logic for p2p networking, communicates with lower-level p2p code and other parts of the @@ -49,10 +55,11 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, syncInfoSpec: ErgoSyncInfoMessageSpec.type, settings: ErgoSettings, syncTracker: ErgoSyncTracker, - deliveryTracker: DeliveryTracker - )(implicit ex: ExecutionContext) + deliveryTracker: DeliveryTracker)(implicit ex: ExecutionContext) extends Actor with Synchronizer with ScorexLogging with ScorexEncoding { + type EncodedManifestId = ModifierId + override val supervisorStrategy: OneForOneStrategy = OneForOneStrategy( maxNrOfRetries = 10, withinTimeRange = 1.minute) { @@ -158,6 +165,18 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, */ private var modifiersCacheSize: Int = 0 + /** + * UTXO set snapshot manifests found in the p2p are stored in this table. The table is cleared when a manifest + * is found which available for downloading from at least min number of peers required (the min is provided in + * ergo.node.utxo.p2pUtxoSnapshots setting) + */ + private val availableManifests = mutable.Map[ModifierId, (Height, Seq[ConnectedPeer])]() + + /** + * How many peers should have a utxo set snapshot to start downloading it + */ + private lazy val MinSnapshots = settings.nodeSettings.utxoSettings.p2pUtxoSnapshots + /** * To be called when the node is synced and new block arrives, to reset transactions cost counter */ @@ -229,7 +248,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, */ override def preStart(): Unit = { // subscribe for history and mempool changes - viewHolderRef ! GetNodeViewChanges(history = true, state = false, vault = false, mempool = true) + viewHolderRef ! GetNodeViewChanges(history = true, state = true, vault = false, mempool = true) val toDownloadCheckInterval = networkSettings.syncInterval @@ -240,6 +259,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, // subscribe for all the node view holder events involving modifiers and transactions context.system.eventStream.subscribe(self, classOf[ChangedHistory]) context.system.eventStream.subscribe(self, classOf[ChangedMempool]) + context.system.eventStream.subscribe(self, classOf[ChangedState]) context.system.eventStream.subscribe(self, classOf[ModificationOutcome]) context.system.eventStream.subscribe(self, classOf[DownloadRequest]) context.system.eventStream.subscribe(self, classOf[BlockAppliedTransactions]) @@ -518,18 +538,18 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * A helper method to ask for block section from given peer * * @param modifierTypeId - block section type id - * @param modifierIds - ids of block section to download - * @param peer - peer to download from - * @param checksDone - how many times the block section was requested before - * (non-zero if we're re-requesting the block section, in this case, there should be only - * one id to request in `modifierIds` + * @param modifierIds - ids of block section to download + * @param peer - peer to download from + * @param checksDone - how many times the block section was requested before + * (non-zero if we're re-requesting the block section, in this case, there should be only + * one id to request in `modifierIds` */ def requestBlockSection(modifierTypeId: NetworkObjectTypeId.Value, modifierIds: Seq[ModifierId], peer: ConnectedPeer, checksDone: Int = 0): Unit = { log.debug(s"Requesting block sections of type $modifierTypeId : $modifierIds") - if(checksDone > 0 && modifierIds.length > 1) { + if (checksDone > 0 && modifierIds.length > 1) { log.warn(s"Incorrect state, checksDone > 0 && modifierIds.length > 1 , for $modifierIds of type $modifierTypeId") } val msg = Message(RequestModifierSpec, Right(InvData(modifierTypeId, modifierIds)), None) @@ -543,15 +563,49 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } } - def onDownloadRequest(historyReader: ErgoHistory): Receive = { + /* + * Private helper methods to request UTXO set snapshots metadata and related data (manifests, chunks) from peers + */ + + private def requestSnapshotsInfo(): Unit = { + // ask all the peers supporting UTXO set snapshots for snapshots they have + val msg = Message(GetSnapshotsInfoSpec, Right(()), None) + val peers = UtxoSetNetworkingFilter.filter(syncTracker.knownPeers()).toSeq + val peersCount = peers.size + if (peersCount >= MinSnapshots) { + networkControllerRef ! SendToNetwork(msg, SendToPeers(peers)) + } else { + log.warn(s"Less UTXO-snapshot supporting peers found than required mininum ($peersCount < $MinSnapshots)") + } + } + + private def requestManifest(manifestId: ManifestId, peer: ConnectedPeer): Unit = { + deliveryTracker.setRequested(ManifestTypeId.value, ModifierId @@ Algos.encode(manifestId), peer) { deliveryCheck => + context.system.scheduler.scheduleOnce(deliveryTimeout, self, deliveryCheck) + } + val msg = Message(GetManifestSpec, Right(manifestId), None) + networkControllerRef ! SendToNetwork(msg, SendToPeer(peer)) + } + + private def requestUtxoSetChunk(subtreeId: SubtreeId, peer: ConnectedPeer): Unit = { + // as we download multiple chunks in parallel and they can be quite large, timeout increased + val chunkDeliveryTimeout = 4 * deliveryTimeout + deliveryTracker.setRequested(UtxoSnapshotChunkTypeId.value, ModifierId @@ Algos.encode(subtreeId), peer) { deliveryCheck => + context.system.scheduler.scheduleOnce(chunkDeliveryTimeout, self, deliveryCheck) + } + val msg = Message(GetUtxoSnapshotChunkSpec, Right(subtreeId), None) + networkControllerRef ! SendToNetwork(msg, SendToPeer(peer)) + } + + private def onDownloadRequest(historyReader: ErgoHistory): Receive = { case DownloadRequest(modifiersToFetch: Map[NetworkObjectTypeId.Value, Seq[ModifierId]]) => log.debug(s"Downloading via DownloadRequest: $modifiersToFetch") - if(modifiersToFetch.nonEmpty) { + if (modifiersToFetch.nonEmpty) { requestDownload( maxModifiers = deliveryTracker.modifiersToDownload, minModifiersPerBucket, maxModifiersPerBucket - )(getPeersForDownloadingBlocks) { howManyPerType => + )(getPeersForDownloadingBlocks) { _ => // leave block section ids only not touched before modifiersToFetch.flatMap { case (tid, mids) => val updMids = mids.filter { mid => @@ -571,32 +625,48 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * Modifier download method that is given min/max constraints for modifiers to download from peers. * It sends requests for modifiers to given peers in optimally sized batches. * - * @param maxModifiers maximum modifiers to download + * @param maxModifiers maximum modifiers to download * @param minModifiersPerBucket minimum modifiers to download per bucket * @param maxModifiersPerBucket maximum modifiers to download per bucket - * @param getPeersOpt optionally get peers to download from, all peers have the same PeerSyncState - * @param fetchMax function that fetches modifiers, it is passed how many of them tops + * @param getPeersOpt optionally get peers to download from, all peers have the same PeerSyncState + * @param fetchMax function that fetches modifiers, it is passed how many of them tops */ protected def requestDownload(maxModifiers: Int, minModifiersPerBucket: Int, maxModifiersPerBucket: Int) (getPeersOpt: => Option[Iterable[ConnectedPeer]]) - (fetchMax: Int => Map[NetworkObjectTypeId.Value, Seq[ModifierId]]): Unit = - getPeersOpt - .foreach { peers => - val modifiersByBucket = ElementPartitioner.distribute(peers, maxModifiers, minModifiersPerBucket, maxModifiersPerBucket)(fetchMax) - // collect and log useful downloading progress information, don't worry it does not run frequently - modifiersByBucket.headOption.foreach { _ => - modifiersByBucket - .groupBy(_._1._2) - .mapValues(_.map(_._2.size)) - .map { case (modType, batchSizes) => - s"Downloading from peers : type[$modType] of ${batchSizes.size} batches each of ~ size: ${batchSizes.take(2).max}" - }.foreach(log.info(_)) + (fetchMax: Int => Map[NetworkObjectTypeId.Value, Seq[ModifierId]]): Unit = { + getPeersOpt match { + case Some(peers) if peers.nonEmpty => + val peersCount = peers.size + val maxElementsToFetch = Math.min(maxModifiers, peersCount * maxModifiersPerBucket) + val fetched = if (maxElementsToFetch <= 0) { + Map.empty + } else { + fetchMax(maxElementsToFetch) } - // bucket represents a peer and a modifierType as we cannot send mixed types to a peer - modifiersByBucket.foreach { case ((peer, modifierTypeId), modifierIds) => - requestBlockSection(modifierTypeId, modifierIds, peer) + if (fetched.size == 1 && fetched.head._1 == SnapshotsInfoTypeId.value) { + // special case when underlying logic in `fetchMax` is providing a request + // to start downloading UTXO set snapshots + requestSnapshotsInfo() + } else { + val modifiersByBucket = ElementPartitioner.distribute(peers, minModifiersPerBucket, fetched) + // collect and log useful downloading progress information, don't worry it does not run frequently + modifiersByBucket.headOption.foreach { _ => + modifiersByBucket + .groupBy(_._1._2) + .mapValues(_.map(_._2.size)) + .map { case (modType, batchSizes) => + s"Downloading from peers : type[$modType] of ${batchSizes.size} batches each of ~ size: ${batchSizes.take(2).max}" + }.foreach(log.info(_)) + } + // bucket represents a peer and a modifierType as we cannot send mixed types to a peer + modifiersByBucket.foreach { case ((peer, modifierTypeId), modifierIds) => + requestBlockSection(modifierTypeId, modifierIds, peer) + } } - } + case _ => + log.warn("No peers available in requestDownload") + } + } private def transactionsFromRemote(requestedModifiers: Map[ModifierId, Array[Byte]], mp: ErgoMemPool, @@ -763,6 +833,180 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, modifiersByStatus.getOrElse(Requested, Map.empty) } + /* + * Private helper methods to send UTXO set snapshots related network messages + */ + + private def sendSnapshotsInfo(usr: UtxoSetSnapshotPersistence, peer: ConnectedPeer): Unit = { + val snapshotsInfo = usr.getSnapshotInfo() + log.debug(s"Sending snapshots info with ${snapshotsInfo.availableManifests.size} snapshots to $peer") + val msg = Message(SnapshotsInfoSpec, Right(snapshotsInfo), None) + networkControllerRef ! SendToNetwork(msg, SendToPeer(peer)) + } + + private def sendManifest(id: ManifestId, usr: UtxoSetSnapshotPersistence, peer: ConnectedPeer): Unit = { + usr.getManifestBytes(id) match { + case Some(manifestBytes) => { + val msg = Message(ManifestSpec, Right(manifestBytes), None) + networkControllerRef ! SendToNetwork(msg, SendToPeer(peer)) + } + case _ => log.warn(s"No manifest ${Algos.encode(id)} available") + } + } + + private def sendUtxoSnapshotChunk(subtreeId: SubtreeId, usr: UtxoSetSnapshotPersistence, peer: ConnectedPeer): Unit = { + usr.getUtxoSnapshotChunkBytes(subtreeId) match { + case Some(snapChunk) => { + log.debug(s"Sending utxo snapshot chunk (${Algos.encode(subtreeId)}) to $peer") + val msg = Message(UtxoSnapshotChunkSpec, Right(snapChunk), None) + networkControllerRef ! SendToNetwork(msg, SendToPeer(peer)) + } + case _ => log.warn(s"No chunk ${Algos.encode(subtreeId)} available") + } + } + + private def requestMoreChunksIfNeeded(hr: ErgoHistory): Unit = { + // we request more chunks if currently less than `ChunksInParallelMin` chunks are being downloading + // we request up to `ChunksInParallelMin`, so in extreme case may download almost 2 * `ChunksInParallelMin` + // we download new chunks from random peers, `ChunksPerPeer` chunks from a random peer (but we may ask the same + // peer twice as choice is truly random) + val ChunksInParallelMin = 16 + val ChunksPerPeer = 4 + hr.utxoSetSnapshotDownloadPlan() match { + case Some(downloadPlan) => + + if (downloadPlan.downloadingChunks < ChunksInParallelMin) { + (1 to ChunksPerPeer).foreach { _ => + val toRequest = hr.getChunkIdsToDownload(howMany = ChunksInParallelMin / ChunksPerPeer) + hr.randomPeerToDownloadChunks() match { + case Some(remote) => toRequest.foreach(subtreeId => requestUtxoSetChunk(subtreeId, remote)) + case None => + log.warn(s"No peers to download chunks from") + } + } + } + case None => + log.warn("No download plan found in requestMoreChunksIfNeeded") + } + } + + /** + * Process information about snapshots got from another peer + */ + private def processSnapshotsInfo(hr: ErgoHistory, + snapshotsInfo: SnapshotsInfo, + remote: ConnectedPeer): Unit = { + snapshotsInfo.availableManifests.foreach { case (height, manifestId: ManifestId) => + val encodedManifestId = ModifierId @@ Algos.encode(manifestId) + val ownId = hr.bestHeaderAtHeight(height).map(_.stateRoot).map(stateDigest => splitDigest(stateDigest)._1) + if (ownId.getOrElse(Array.emptyByteArray).sameElements(manifestId)) { + log.debug(s"Discovered manifest $encodedManifestId for height $height from $remote") + // add manifest to available manifests dictionary if it is not written there yet + val existingOffers = availableManifests.getOrElse(encodedManifestId, (height -> Seq.empty)) + if (!existingOffers._2.contains(remote)) { + log.info(s"Found new manifest ${Algos.encode(manifestId)} for height $height at $remote") + availableManifests.put(encodedManifestId, height -> (existingOffers._2 :+ remote)) + } else { + log.warn(s"Double manifest declaration for $manifestId from $remote") + } + } else { + log.error(s"Got wrong manifest id $encodedManifestId from $remote") + } + } + checkUtxoSetManifests(hr) // check if we got enough manifests for the height to download manifests and chunks + } + + // process serialized manifest got from another peer + private def processManifest(hr: ErgoHistory, manifestBytes: Array[Byte], remote: ConnectedPeer): Unit = { + ManifestSerializer.defaultSerializer.parseBytesTry(manifestBytes) match { + case Success(manifest) => + val manifestId = ModifierId @@ Algos.encode(manifest.id) + log.info(s"Got manifest $manifestId from $remote") + deliveryTracker.getRequestedInfo(ManifestTypeId.value, manifestId) match { + case Some(ri) if ri.peer == remote => + deliveryTracker.setUnknown(manifestId, ManifestTypeId.value) + val manifestRecordOpt = availableManifests.get(manifestId) + manifestRecordOpt match { + case Some(manifestRecord) => + val height = manifestRecord._1 + val peersToDownload = manifestRecord._2 + + // check if manifest is valid against root hash and height of AVL+ tree found in a header + val manifestVerified = hr + .bestHeaderAtHeight(height) + .map(_.stateRoot) + .map(splitDigest) + .exists { case (expRoot, expHeight) => manifest.verify(Digest32 @@ expRoot, expHeight) } + + if(manifestVerified) { + log.info(s"Going to download chunks for manifest ${Algos.encode(manifest.id)} at height $height from $peersToDownload") + hr.registerManifestToDownload(manifest, height, peersToDownload) + availableManifests.clear() + requestMoreChunksIfNeeded(hr) + } else { + log.error(s"Got invalid manifest from $remote") + penalizeMaliciousPeer(remote) + } + case None => + log.error(s"No height found for manifest ${Algos.encode(manifest.id)}") + } + case _ => + log.info(s"Penalizing spamming peer $remote sent non-asked manifest $manifestId") + penalizeSpammingPeer(remote) + } + case Failure(e) => + log.info(s"Cant' restore manifest (got from $remote) from bytes ", e) + penalizeMisbehavingPeer(remote) + } + } + + // process utxo set snapshot chunk got from another peer + private def processUtxoSnapshotChunk(serializedChunk: Array[Byte], hr: ErgoHistory, remote: ConnectedPeer): Unit = { + SubtreeSerializer.parseBytesTry(serializedChunk) match { + case Success(subtree) => + val chunkId = ModifierId @@ Algos.encode(subtree.id) + deliveryTracker.getRequestedInfo(UtxoSnapshotChunkTypeId.value, chunkId) match { + case Some(_) => + log.debug(s"Got utxo snapshot chunk, id: $chunkId, size: ${serializedChunk.length}") + deliveryTracker.setUnknown(chunkId, UtxoSnapshotChunkTypeId.value) + hr.registerDownloadedChunk(subtree.id, serializedChunk) + + hr.utxoSetSnapshotDownloadPlan() match { + case Some(downloadPlan) => + if (downloadPlan.fullyDownloaded) { + log.info("All the UTXO set snapshot chunks downloaded") + // if all the chunks of snapshot are downloaded, initialize UTXO set state with it + if (!hr.isUtxoSnapshotApplied) { + val h = downloadPlan.snapshotHeight + hr.bestHeaderIdAtHeight(h) match { + case Some(blockId) => + viewHolderRef ! InitStateFromSnapshot(h, blockId) + case None => + // shouldn't happen in principle + log.error("No best header found when all chunks are downloaded. Please contact developers.") + } + } else { + log.warn("UTXO set snapshot already applied, double application attemt") + } + } else { + // if not all the chunks are downloaded, request more if needed + requestMoreChunksIfNeeded(hr) + } + case None => + log.warn(s"No download plan found when processing UTXO set snapshot chunk $chunkId") + } + + case None => + log.info(s"Penalizing spamming peer $remote sent non-asked UTXO set snapshot chunk $chunkId") + penalizeSpammingPeer(remote) + } + + case Failure(e) => + log.info(s"Cant' restore snapshot chunk (got from $remote) from bytes ", e) + penalizeMisbehavingPeer(remote) + } + } + /** * Object ids coming from other node. * Filter out modifier ids that are already in process (requested, received or applied), @@ -913,7 +1157,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, * wait for delivery until the number of checks exceeds the maximum if the peer sent `Inv` for this modifier * re-request modifier from a different random peer, if our node does not know a peer who have it */ - protected def checkDelivery: Receive = { + protected def checkDelivery(hr: ErgoHistory): Receive = { case CheckDelivery(peer, modifierTypeId, modifierId) => if (deliveryTracker.status(modifierId, modifierTypeId, Seq.empty) == ModifiersStatus.Requested) { // If transaction not delivered on time, we just forget about it. @@ -922,11 +1166,11 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, deliveryTracker.clearStatusForModifier(modifierId, modifierTypeId, ModifiersStatus.Requested) } else { // A block section is not delivered on time. - log.info(s"Peer ${peer.toString} has not delivered modifier " + + log.info(s"Peer ${peer.toString} has not delivered network object " + s"$modifierTypeId : ${encoder.encodeId(modifierId)} on time") - // Number of block section delivery checks increased or initialized, - // except the case where we can have issues with connectivity, + // Number of delivery checks for a block section, utxo set snapshot chunk or manifest + // increased or initialized, except the case where we can have issues with connectivity, // which is currently defined by comparing request time with time the // node got last modifier (in future we may consider more precise method) val checksDone = deliveryTracker.getRequestedInfo(modifierTypeId, modifierId) match { @@ -940,17 +1184,28 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, val maxDeliveryChecks = networkSettings.maxDeliveryChecks if (checksDone < maxDeliveryChecks) { - // randomly choose a peer for another download attempt - val newPeerCandidates: Seq[ConnectedPeer] = if (modifierTypeId == Header.modifierTypeId) { - getPeersForDownloadingHeaders(peer).toSeq + if (modifierTypeId == UtxoSnapshotChunkTypeId.value) { + // randomly choosing a peer to download UTXO set snapshot chunk + val newPeerOpt = hr.randomPeerToDownloadChunks() + log.info(s"Rescheduling request for UTXO set chunk $modifierId , new peer $newPeerOpt") + deliveryTracker.setUnknown(modifierId, modifierTypeId) + newPeerOpt match { + case Some(newPeer) => requestUtxoSetChunk(Digest32 @@ Algos.decode(modifierId).get, newPeer) + case None => log.warn(s"No peer found to download UTXO set chunk $modifierId") + } } else { - getPeersForDownloadingBlocks.map(_.toSeq).getOrElse(Seq(peer)) + // randomly choose a peer for another block sections download attempt + val newPeerCandidates: Seq[ConnectedPeer] = if (modifierTypeId == Header.modifierTypeId) { + getPeersForDownloadingHeaders(peer).toSeq + } else { + getPeersForDownloadingBlocks.map(_.toSeq).getOrElse(Seq(peer)) + } + val newPeerIndex = scala.util.Random.nextInt(newPeerCandidates.size) + val newPeer = newPeerCandidates(newPeerIndex) + log.info(s"Rescheduling request for $modifierId , new peer $newPeer") + deliveryTracker.setUnknown(modifierId, modifierTypeId) + requestBlockSection(modifierTypeId, Seq(modifierId), newPeer, checksDone) } - val newPeerIndex = scala.util.Random.nextInt(newPeerCandidates.size) - val newPeer = newPeerCandidates(newPeerIndex) - log.info(s"Rescheduling request for $modifierId , new peer $newPeer") - deliveryTracker.setUnknown(modifierId, modifierTypeId) - requestBlockSection(modifierTypeId, Seq(modifierId), newPeer, checksDone) } else { log.error(s"Exceeded max delivery attempts($maxDeliveryChecks) limit for $modifierId") if (modifierTypeId == Header.modifierTypeId) { @@ -1015,8 +1270,31 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, } } + // check if we have enough UTXO set snapshots for some height + // if so, request manifest from a random peer announced it + private def checkUtxoSetManifests(historyReader: ErgoHistory): Unit = { + + if (settings.nodeSettings.utxoSettings.utxoBootstrap && + historyReader.fullBlockHeight == 0 && + availableManifests.nonEmpty && + historyReader.utxoSetSnapshotDownloadPlan().isEmpty) { + val res = availableManifests.filter { case (_, (_, peers)) => peers.length >= MinSnapshots } + if (res.nonEmpty) { + val(encModifierId, (height, peers)) = res.maxBy(_._2._1) + log.info(s"Downloading manifest for height $height from ${peers.size} peers") + val manifestId = Digest32 @@ Algos.decode(encModifierId).get + val randomPeer = peers(Random.nextInt(peers.length)) + requestManifest(manifestId, randomPeer) + } else { + log.info("No manifests to download found ") + } + } + } + + private def viewHolderEvents(historyReader: ErgoHistory, mempoolReader: ErgoMemPool, + utxoStateReaderOpt: Option[UtxoStateReader], blockAppliedTxsCache: FixedSizeApproximateCacheQueue): Receive = { // Requests BlockSections with `Unknown` status that are defined by block headers but not downloaded yet. // Trying to keep size of requested queue equals to `desiredSizeOfExpectingQueue`. @@ -1029,8 +1307,7 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, minModifiersPerBucket, maxModifiersPerBucket )(getPeersForDownloadingBlocks) { howManyPerType => - val tip = historyReader.estimatedTip() - historyReader.nextModifiersToDownload(howManyPerType, tip, downloadRequired(historyReader)) + historyReader.nextModifiersToDownload(howManyPerType, downloadRequired(historyReader)) } } @@ -1090,10 +1367,17 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, deliveryTracker.setInvalid(modId, modTypeId).foreach(penalizeMisbehavingPeer) case ChangedHistory(newHistoryReader: ErgoHistory) => - context.become(initialized(newHistoryReader, mempoolReader, blockAppliedTxsCache)) + context.become(initialized(newHistoryReader, mempoolReader, utxoStateReaderOpt, blockAppliedTxsCache)) case ChangedMempool(newMempoolReader: ErgoMemPool) => - context.become(initialized(historyReader, newMempoolReader, blockAppliedTxsCache)) + context.become(initialized(historyReader, newMempoolReader, utxoStateReaderOpt, blockAppliedTxsCache)) + + case ChangedState(reader: ErgoStateReader) => + reader match { + case utxoStateReader: UtxoStateReader => + context.become(initialized(historyReader, mempoolReader, Some(utxoStateReader), blockAppliedTxsCache)) + case _ => + } case BlockSectionsProcessingCacheUpdate(headersCacheSize, blockSectionsCacheSize, cleared) => val HeadersCacheSizeToDownloadMore = 3184 @@ -1118,20 +1402,25 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, case BlockAppliedTransactions(transactionIds: Seq[ModifierId]) => // We collect applied TXs to history in order to avoid banning peers that sent these afterwards logger.debug("Caching applied transactions") - context.become(initialized(historyReader, mempoolReader, blockAppliedTxsCache.putAll(transactionIds))) + context.become(initialized(historyReader, mempoolReader, utxoStateReaderOpt, blockAppliedTxsCache.putAll(transactionIds))) case ChainIsHealthy => // good news logger.debug("Chain is good") case ChainIsStuck(error) => - log.warn(s"Chain is stuck! $error\nDelivery tracker State:\n$deliveryTracker\nSync tracker state:\n$syncTracker") - deliveryTracker.reset() + if (historyReader.fullBlockHeight > 0) { + log.warn(s"Chain is stuck! $error\nDelivery tracker State:\n$deliveryTracker\nSync tracker state:\n$syncTracker") + deliveryTracker.reset() + } else { + log.debug("Got ChainIsStuck signal when no full-blocks applied yet") + } } - /** get handlers of messages coming from peers */ + /** handlers of messages coming from peers */ private def msgHandlers(hr: ErgoHistory, mp: ErgoMemPool, + usrOpt: Option[UtxoStateReader], blockAppliedTxsCache: FixedSizeApproximateCacheQueue ): PartialFunction[(MessageSpec[_], _, ConnectedPeer), Unit] = { case (_: ErgoSyncInfoMessageSpec.type @unchecked, data: ErgoSyncInfo @unchecked, remote) => @@ -1142,41 +1431,79 @@ class ErgoNodeViewSynchronizer(networkControllerRef: ActorRef, modifiersReq(hr, mp, data, remote) case (_: ModifiersSpec.type, data: ModifiersData, remote) => modifiersFromRemote(hr, mp, data, remote, blockAppliedTxsCache) + case (spec: MessageSpec[_], _, remote) if spec.messageCode == GetSnapshotsInfoSpec.messageCode => + usrOpt match { + case Some(usr) => sendSnapshotsInfo(usr, remote) + case None => log.warn(s"Asked for snapshot when UTXO set is not supported, remote: $remote") + } + case (spec: MessageSpec[_], data: SnapshotsInfo, remote) if spec.messageCode == SnapshotsInfoSpec.messageCode => + processSnapshotsInfo(hr, data, remote) + case (_: GetManifestSpec.type, id: Array[Byte], remote) => + usrOpt match { + case Some(usr) => sendManifest(Digest32 @@ id, usr, remote) + case None => log.warn(s"Asked for snapshot when UTXO set is not supported, remote: $remote") + } + case (_: ManifestSpec.type, manifestBytes: Array[Byte], remote) => + processManifest(hr, manifestBytes, remote) + case (_: GetUtxoSnapshotChunkSpec.type, subtreeId: Array[Byte], remote) => + usrOpt match { + case Some(usr) => sendUtxoSnapshotChunk(Digest32 @@ subtreeId, usr, remote) + case None => log.warn(s"Asked for snapshot when UTXO set is not supported, remote: $remote") + } + case (_: UtxoSnapshotChunkSpec.type, serializedChunk: Array[Byte], remote) => + usrOpt match { + case Some(_) => processUtxoSnapshotChunk(serializedChunk, hr, remote) + case None => log.warn(s"Asked for snapshot when UTXO set is not supported, remote: $remote") + } + } - def initialized(hr: ErgoHistory, mp: ErgoMemPool, blockAppliedTxsCache: FixedSizeApproximateCacheQueue): PartialFunction[Any, Unit] = { - processDataFromPeer(msgHandlers(hr, mp, blockAppliedTxsCache)) orElse + def initialized(hr: ErgoHistory, + mp: ErgoMemPool, + usr: Option[UtxoStateReader], + blockAppliedTxsCache: FixedSizeApproximateCacheQueue): PartialFunction[Any, Unit] = { + processDataFromPeer(msgHandlers(hr, mp, usr, blockAppliedTxsCache)) orElse onDownloadRequest(hr) orElse sendLocalSyncInfo(hr) orElse - viewHolderEvents(hr, mp, blockAppliedTxsCache) orElse + viewHolderEvents(hr, mp, usr, blockAppliedTxsCache) orElse peerManagerEvents orElse - checkDelivery orElse { + checkDelivery(hr) orElse { case a: Any => log.error("Strange input: " + a) } } /** Wait until both historyReader and mempoolReader instances are received so actor can be operational */ - def initializing(hr: Option[ErgoHistory], mp: Option[ErgoMemPool], blockAppliedTxsCache: FixedSizeApproximateCacheQueue): PartialFunction[Any, Unit] = { + def initializing(hr: Option[ErgoHistory], + mp: Option[ErgoMemPool], + usr: Option[UtxoStateReader], + blockAppliedTxsCache: FixedSizeApproximateCacheQueue): PartialFunction[Any, Unit] = { case ChangedHistory(historyReader: ErgoHistory) => mp match { case Some(mempoolReader) => - context.become(initialized(historyReader, mempoolReader, blockAppliedTxsCache)) + context.become(initialized(historyReader, mempoolReader, usr, blockAppliedTxsCache)) case _ => - context.become(initializing(Option(historyReader), mp, blockAppliedTxsCache)) + context.become(initializing(Option(historyReader), mp, usr, blockAppliedTxsCache)) } case ChangedMempool(mempoolReader: ErgoMemPool) => hr match { case Some(historyReader) => - context.become(initialized(historyReader, mempoolReader, blockAppliedTxsCache)) + context.become(initialized(historyReader, mempoolReader, usr, blockAppliedTxsCache)) case _ => - context.become(initializing(hr, Option(mempoolReader), blockAppliedTxsCache)) + context.become(initializing(hr, Option(mempoolReader), usr, blockAppliedTxsCache)) + } + case ChangedState(reader: ErgoStateReader) => + reader match { + case utxoStateReader: UtxoStateReader => + context.become(initializing(hr, mp, Some(utxoStateReader), blockAppliedTxsCache)) + case _ => + context.become(initializing(hr, mp, None, blockAppliedTxsCache)) } case msg => // Actor not initialized yet, scheduling message until it is context.system.scheduler.scheduleOnce(1.second, self, msg) } - override def receive: Receive = initializing(None, None, FixedSizeApproximateCacheQueue.empty(cacheQueueSize = 5)) + override def receive: Receive = initializing(None, None, None, FixedSizeApproximateCacheQueue.empty(cacheQueueSize = 5)) } @@ -1324,6 +1651,15 @@ object ErgoNodeViewSynchronizer { * @param mempool - mempool to check */ case class RecheckMempool(state: UtxoStateReader, mempool: ErgoMemPoolReader) + + /** + * Signal for a central node view holder component to initialize UTXO state from UTXO set snapshot + * stored in the local database + * + * @param blockHeight - height of a block corresponding to the UTXO set snapshot + * @param blockId - id of a block corresponding to the UTXO set snapshot + */ + case class InitStateFromSnapshot(blockHeight: Height, blockId: ModifierId) } } diff --git a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala index 02332cb46d..3c406f9034 100644 --- a/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala +++ b/src/main/scala/org/ergoplatform/network/ErgoSyncTracker.scala @@ -10,7 +10,7 @@ import scorex.util.ScorexLogging import scala.collection.mutable import scala.concurrent.duration._ -import scorex.core.utils.MapPimp +import scorex.core.utils.MapPimpMutable /** * Data structures and methods to keep status of peers, find ones with expired status to send sync message etc @@ -166,6 +166,11 @@ final case class ErgoSyncTracker(networkSettings: NetworkSettings) extends Score } } + /** + * @return all the peers ever sent sync message to the node and still connected + */ + def knownPeers(): Iterable[ConnectedPeer] = statuses.keys + /** * Return the peers to which this node should send a sync signal, including: * outdated peers, if any, otherwise, all the peers with unknown status plus a random peer with diff --git a/src/main/scala/org/ergoplatform/network/PeerFilteringRule.scala b/src/main/scala/org/ergoplatform/network/PeerFilteringRule.scala index 09af67319d..419fd966a0 100644 --- a/src/main/scala/org/ergoplatform/network/PeerFilteringRule.scala +++ b/src/main/scala/org/ergoplatform/network/PeerFilteringRule.scala @@ -85,3 +85,16 @@ object SyncV2Filter extends PeerFilteringRule { } } + +/** + * Filter used to differentiate peers supporting UTXO state snapshots, so possibly + * storing and serving them, from peers do not supporting UTXO set snapshots related networking protocol + */ +object UtxoSetNetworkingFilter extends PeerFilteringRule { + val UtxoSnapsnotActivationVersion = Version(5, 0, 12) + + def condition(version: Version): Boolean = { + // If neighbour version is >= `UtxoSnapsnotActivationVersion`, the neighbour supports utxo snapshots exchange + version.compare(UtxoSnapsnotActivationVersion) >= 0 + } +} diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoContext.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoContext.scala index 37a5820c5a..132501df4b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoContext.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoContext.scala @@ -1,9 +1,10 @@ package org.ergoplatform.nodeView +import org.ergoplatform.ErgoLikeContext import org.ergoplatform.nodeView.state.ErgoStateContext +import org.ergoplatform.sdk.wallet.protocol.context.TransactionContext import org.ergoplatform.wallet.interpreter.ErgoInterpreter -import org.ergoplatform.wallet.protocol.context.{InputContext, TransactionContext} -import org.ergoplatform.ErgoLikeContext +import org.ergoplatform.wallet.protocol.context.InputContext /** * Context to be used during transaction verification diff --git a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala index 5e13ab0d61..7978c3ed31 100644 --- a/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/ErgoNodeViewHolder.scala @@ -277,6 +277,28 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti processingOutcome } + /** + * signal to pull Utxo set snapshot from database and recreate UTXO set from it + */ + def processStateSnapshot: Receive = { + case InitStateFromSnapshot(height, blockId) => + if (!history().isUtxoSnapshotApplied) { + val store = minimalState().store + history().createPersistentProver(store, history(), height, blockId) match { + case Success(pp) => + log.info(s"Restoring state from prover with digest ${pp.digest} reconstructed for height $height") + history().onUtxoSnapshotApplied(height) + val newState = new UtxoState(pp, version = VersionTag @@@ blockId, store, settings) + updateNodeView(updatedState = Some(newState.asInstanceOf[State])) + case Failure(t) => + log.error("UTXO set snapshot application failed: ", t) + } + } else { + log.warn("InitStateFromSnapshot arrived when state already initialized") + } + + } + /** * Process new modifiers from remote. * Put all candidates to modifiersCache and then try to apply as much modifiers from cache as possible. @@ -406,8 +428,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val history = ErgoHistory.readOrGenerate(settings) log.info("History database read") val memPool = ErgoMemPool.empty(settings) - val constants = StateConstants(settings) - restoreConsistentState(ErgoState.readOrGenerate(settings, constants).asInstanceOf[State], history) match { + restoreConsistentState(ErgoState.readOrGenerate(settings).asInstanceOf[State], history) match { case Success(state) => log.info(s"State database read, state synchronized") val wallet = ErgoWallet.readOrGenerate( @@ -533,8 +554,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val dir = stateDir(settings) deleteRecursive(dir) - val constants = StateConstants(settings) - ErgoState.readOrGenerate(settings, constants) + ErgoState.readOrGenerate(settings) .asInstanceOf[State] .ensuring( state => java.util.Arrays.equals(state.rootDigest, settings.chainSettings.genesisStateDigest), @@ -580,7 +600,6 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti * Recovers digest state from history. */ private def recoverDigestState(bestFullBlock: ErgoFullBlock, history: ErgoHistory): Try[DigestState] = { - val constants = StateConstants(settings) val votingLength = settings.chainSettings.voting.votingLength val bestHeight = bestFullBlock.header.height val newEpochHeadersQty = bestHeight % votingLength // how many blocks current epoch lasts @@ -592,12 +611,11 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti val recoveredStateTry = firstExtensionOpt .fold[Try[ErgoStateContext]](Failure(new Exception("Could not find extension to recover from")) - )(ext => ErgoStateContext.recover(constants.genesisStateDigest, ext, lastHeaders)(settings)) + )(ext => ErgoStateContext.recover(settings.chainSettings.genesisStateDigest, ext, lastHeaders)(settings)) .flatMap { ctx => val recoverVersion = idToVersion(lastHeaders.last.id) val recoverRoot = bestFullBlock.header.stateRoot - val parameters = ctx.currentParameters - DigestState.recover(recoverVersion, recoverRoot, ctx, stateDir(settings), constants, parameters) + DigestState.recover(recoverVersion, recoverRoot, ctx, stateDir(settings), settings) } recoveredStateTry match { @@ -609,7 +627,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti case Failure(exception) => // recover using whole headers chain log.warn(s"Failed to recover state from current epoch, using whole chain: ${exception.getMessage}") val wholeChain = history.headerChainBack(Int.MaxValue, bestFullBlock.header, _.isGenesis).headers - val genesisState = DigestState.create(None, None, stateDir(settings), constants) + val genesisState = DigestState.create(None, None, stateDir(settings), settings) wholeChain.foldLeft[Try[DigestState]](Success(genesisState)) { case (acc, m) => acc.flatMap(_.applyModifier(m, history.estimatedTip())(lm => self ! lm)) } @@ -673,6 +691,7 @@ abstract class ErgoNodeViewHolder[State <: ErgoState[State]](settings: ErgoSetti transactionsProcessing orElse getCurrentInfo orElse getNodeViewChanges orElse + processStateSnapshot orElse handleHealthCheck orElse { case a: Any => log.error("Strange input: " + a) } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala index f214fbf743..1d1b47fe09 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistory.scala @@ -102,14 +102,14 @@ trait ErgoHistory if (nonMarkedIds.nonEmpty) { historyStorage.insert( nonMarkedIds.map(id => validityKey(id) -> Array(1.toByte)), - Array.empty[BlockSection]).map(_ => this) + BlockSection.emptyArray).map(_ => this) } else { Success(this) } case _ => historyStorage.insert( Array(validityKey(modifier.id) -> Array(1.toByte)), - Array.empty[BlockSection]).map(_ => this) + BlockSection.emptyArray).map(_ => this) } } @@ -136,7 +136,7 @@ trait ErgoHistory (bestHeaderIsInvalidated, bestFullIsInvalidated) match { case (false, false) => // Modifiers from best header and best full chain are not involved, no rollback and links change required - historyStorage.insert(validityRow, Array.empty[BlockSection]).map { _ => + historyStorage.insert(validityRow, BlockSection.emptyArray).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } case _ => @@ -147,7 +147,7 @@ trait ErgoHistory //Only headers chain involved historyStorage.insert( newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toArray, - Array.empty[BlockSection] + BlockSection.emptyArray ).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } @@ -175,7 +175,7 @@ trait ErgoHistory val changedLinks = validHeadersChain.lastOption.map(b => BestFullBlockKey -> idToBytes(b.id)) ++ newBestHeaderOpt.map(h => BestHeaderKey -> idToBytes(h.id)).toSeq val toInsert = validityRow ++ changedLinks ++ chainStatusRow - historyStorage.insert(toInsert, Array.empty[BlockSection]).map { _ => + historyStorage.insert(toInsert, BlockSection.emptyArray).map { _ => val toRemove = if (genesisInvalidated) invalidatedChain else invalidatedChain.tail this -> ProgressInfo(Some(branchPointHeader.id), toRemove, validChain, Seq.empty) } @@ -184,7 +184,7 @@ trait ErgoHistory case None => //No headers become invalid. Just mark this modifier as invalid log.warn(s"Modifier ${modifier.encodedId} of type ${modifier.modifierTypeId} is missing corresponding header") - historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), Array.empty[BlockSection]).map { _ => + historyStorage.insert(Array(validityKey(modifier.id) -> Array(0.toByte)), BlockSection.emptyArray).map { _ => this -> ProgressInfo[BlockSection](None, Seq.empty, Seq.empty, Seq.empty) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala index 47b5dc6c43..43f1a9c013 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/ErgoHistoryReader.scala @@ -618,7 +618,7 @@ trait ErgoHistoryReader */ def estimatedTip(): Option[Height] = { Try { //error may happen if history not initialized - if(isHeadersChainSynced) { + if (isHeadersChainSynced) { Some(headersHeight) } else { None diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala index 8f94495d73..a6002c1b0c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/BalanceInfo.scala @@ -8,7 +8,7 @@ import scorex.core.serialization.ErgoSerializer import scorex.util.{ModifierId, ScorexLogging, bytesToId} import scorex.util.serialization.{Reader, Writer} import spire.implicits.cfor - +import special.collection.Extensions._ import scala.collection.mutable import scala.collection.mutable.ArrayBuffer @@ -56,7 +56,7 @@ case class BalanceInfo() extends ScorexLogging { def add(box: ErgoBox): Unit = { nanoErgs += box.value cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - val id: ModifierId = bytesToId(box.additionalTokens(i)._1) + val id: ModifierId = box.additionalTokens(i)._1.toModifierId index(id) match { case Some(n) => tokens(n) = Tuple2(id, tokens(n)._2 + box.additionalTokens(i)._2) case None => tokens += Tuple2(id, box.additionalTokens(i)._2) @@ -72,11 +72,11 @@ case class BalanceInfo() extends ScorexLogging { def subtract(box: ErgoBox)(implicit ae: ErgoAddressEncoder): Unit = { nanoErgs = math.max(nanoErgs - box.value, 0) cfor(0)(_ < box.additionalTokens.length, _ + 1) { i => - val id: ModifierId = bytesToId(box.additionalTokens(i)._1) + val id: ModifierId = box.additionalTokens(i)._1.toModifierId index(id) match { case Some(n) => val newVal: Long = tokens(n)._2 - box.additionalTokens(i)._2 - if(newVal == 0) + if (newVal == 0) tokens.remove(n) else tokens(n) = (id, newVal) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala index 7cb23966a9..e7547aa3b8 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexer.scala @@ -208,8 +208,9 @@ trait ExtraIndexerBase extends ScorexLogging { // check if box is creating new tokens, if yes record them cfor(0)(_ < box.additionalTokens.length, _ + 1) { j => - if (!tokens.exists(x => java.util.Arrays.equals(x._1, box.additionalTokens(j)._1))) + if (!tokens.exists(x => x._1 == box.additionalTokens(j)._1)) { general += IndexedToken.fromBox(box, j) + } } globalBoxIndex += 1 diff --git a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala index 31c946fd82..19c6d18d8e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/extra/IndexedToken.scala @@ -6,10 +6,11 @@ import org.ergoplatform.nodeView.history.extra.ExtraIndexer.{ExtraIndexTypeId, f import org.ergoplatform.nodeView.history.extra.IndexedTokenSerializer.{ByteColl, uniqueId} import org.ergoplatform.settings.Algos import scorex.core.serialization.ErgoSerializer -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ByteArrayOps, ModifierId, bytesToId} import scorex.util.serialization.{Reader, Writer} import sigmastate.Values.CollectionConstant import sigmastate.SByte +import special.collection.Extensions._ /** * Index of a token containing creation information. @@ -123,8 +124,8 @@ object IndexedToken { case None => 0 } - IndexedToken(bytesToId(box.additionalTokens(tokenIndex)._1), - bytesToId(box.id), + IndexedToken(box.additionalTokens(tokenIndex)._1.toModifierId, + box.id.toModifierId, box.additionalTokens(tokenIndex)._2, name, description, diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala index 65cba01452..3385322366 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/HistoryStorage.scala @@ -113,20 +113,36 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e } } - def get(id: ModifierId): Option[Array[Byte]] = objectsStore.get(idToBytes(id)).orElse(extraStore.get(idToBytes(id))) + /** + * @return object with `id` if it is in the objects database + */ + def get(id: ModifierId): Option[Array[Byte]] = { + val idBytes = idToBytes(id) + objectsStore.get(idBytes).orElse(extraStore.get(idBytes)) + } + def get(id: Array[Byte]): Option[Array[Byte]] = objectsStore.get(id).orElse(extraStore.get(id)) + /** + * @return if object with `id` is in the objects database + */ + def contains(id: Array[Byte]): Boolean = get(id).isDefined def contains(id: ModifierId): Boolean = get(id).isDefined def insert(indexesToInsert: Array[(ByteArrayWrapper, Array[Byte])], objectsToInsert: Array[BlockSection]): Try[Unit] = { objectsStore.insert( - objectsToInsert.map(mod => (mod.serializedId, HistoryModifierSerializer.toBytes(mod))) + objectsToInsert.map(mod => mod.serializedId), + objectsToInsert.map(mod => HistoryModifierSerializer.toBytes(mod)) ).flatMap { _ => cfor(0)(_ < objectsToInsert.length, _ + 1) { i => cacheModifier(objectsToInsert(i))} if (indexesToInsert.nonEmpty) { - indexStore.insert(indexesToInsert.map { case (k, v) => k.data -> v }).map { _ => - cfor(0)(_ < indexesToInsert.length, _ + 1) { i => indexCache.put(indexesToInsert(i)._1, indexesToInsert(i)._2)} - () + indexStore.insert( + indexesToInsert.map(_._1.data), + indexesToInsert.map(_._2) + ).map { _ => + cfor(0)(_ < indexesToInsert.length, _ + 1) { i => + indexCache.put(indexesToInsert(i)._1, indexesToInsert(i)._2) + } } } else Success(()) } @@ -134,7 +150,10 @@ class HistoryStorage private(indexStore: LDBKVStore, objectsStore: LDBKVStore, e def insertExtra(indexesToInsert: Array[(Array[Byte], Array[Byte])], objectsToInsert: Array[ExtraIndex]): Unit = { - extraStore.insert(objectsToInsert.map(mod => (mod.serializedId, ExtraIndexSerializer.toBytes(mod)))) + extraStore.insert( + objectsToInsert.map(mod => (mod.serializedId)), + objectsToInsert.map(mod => ExtraIndexSerializer.toBytes(mod)) + ) cfor(0)(_ < objectsToInsert.length, _ + 1) { i => val ei = objectsToInsert(i); extraCache.put(ei.id, ei)} cfor(0)(_ < indexesToInsert.length, _ + 1) { i => extraStore.insert(indexesToInsert(i)._1, indexesToInsert(i)._2)} } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala index b86cd81f3d..714dbabce4 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockProcessor.scala @@ -58,8 +58,8 @@ trait FullBlockProcessor extends HeadersProcessor { nonBestBlock private def isValidFirstFullBlock(header: Header): Boolean = { - pruningProcessor.isHeadersChainSynced && - header.height == pruningProcessor.minimalFullBlockHeight && + isHeadersChainSynced && + header.height == minimalFullBlockHeight && bestFullBlockIdOpt.isEmpty } @@ -105,7 +105,7 @@ trait FullBlockProcessor extends HeadersProcessor { if (nonBestChainsCache.nonEmpty) nonBestChainsCache = nonBestChainsCache.dropUntil(minForkRootHeight) if (nodeSettings.isFullBlocksPruned) { - val lastKept = pruningProcessor.updateBestFullBlock(fullBlock.header) + val lastKept = updateBestFullBlock(fullBlock.header) val bestHeight: Int = newBestBlockHeader.height val diff = bestHeight - prevBest.header.height pruneBlockDataAt(((lastKept - diff) until lastKept).filter(_ >= 0)) diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockPruningProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockPruningProcessor.scala index aee18fd00d..d77b9e0def 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockPruningProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockPruningProcessor.scala @@ -2,18 +2,22 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.nodeView.history.ErgoHistory -import org.ergoplatform.settings.{ChainSettings, NodeConfigurationSettings} +import org.ergoplatform.settings.ErgoSettings /** * A class that keeps and calculates minimal height for full blocks starting from which we need to download these full * blocks from the network and keep them in our history. */ -class FullBlockPruningProcessor(nodeConfig: NodeConfigurationSettings, chainSettings: ChainSettings) { +trait FullBlockPruningProcessor extends MinimalFullBlockHeightFunctions { - @volatile private[history] var isHeadersChainSyncedVar: Boolean = false - @volatile private[history] var minimalFullBlockHeightVar: Int = ErgoHistory.GenesisHeight + protected def settings: ErgoSettings + + private def nodeConfig = settings.nodeSettings + private def chainSettings = settings.chainSettings + + private def VotingEpochLength = chainSettings.voting.votingLength - private val VotingEpochLength = chainSettings.voting.votingLength + @volatile private[history] var isHeadersChainSyncedVar: Boolean = false private def extensionWithParametersHeight(height: Int): Int = { require(height >= VotingEpochLength) @@ -28,7 +32,7 @@ class FullBlockPruningProcessor(nodeConfig: NodeConfigurationSettings, chainSett /** Start height to download full blocks from */ - def minimalFullBlockHeight: Int = minimalFullBlockHeightVar + def minimalFullBlockHeight: Int = readMinimalFullBlockHeight() /** Check if headers chain is synchronized with the network and modifier is not too old */ @@ -42,15 +46,16 @@ class FullBlockPruningProcessor(nodeConfig: NodeConfigurationSettings, chainSett * @return minimal height to process best full block */ def updateBestFullBlock(header: Header): Int = { - minimalFullBlockHeightVar = if (!nodeConfig.isFullBlocksPruned) { - ErgoHistory.GenesisHeight // keep all blocks in history - } else if (!isHeadersChainSynced && !nodeConfig.stateType.requireProofs) { - // just synced with the headers chain - determine first full block to apply - //TODO start with the height of UTXO snapshot applied. For now we start from genesis until this is implemented - ErgoHistory.GenesisHeight + val minimalFullBlockHeight = if (nodeConfig.blocksToKeep < 0) { + if (nodeConfig.utxoSettings.utxoBootstrap) { + // we have constant min full block height corresponding to first block after utxo set snapshot + readMinimalFullBlockHeight() + } else { + ErgoHistory.GenesisHeight // keep all blocks in history as no pruning set + } } else { // Start from config.blocksToKeep blocks back - val h = Math.max(minimalFullBlockHeight, header.height - nodeConfig.blocksToKeep + 1) + val h = Math.max(readMinimalFullBlockHeight(), header.height - nodeConfig.blocksToKeep + 1) // ... but not later than the beginning of a voting epoch if (h > VotingEpochLength) { Math.min(h, extensionWithParametersHeight(h)) @@ -59,7 +64,8 @@ class FullBlockPruningProcessor(nodeConfig: NodeConfigurationSettings, chainSett } } if (!isHeadersChainSynced) isHeadersChainSyncedVar = true - minimalFullBlockHeightVar + writeMinimalFullBlockHeight(minimalFullBlockHeight) + minimalFullBlockHeight } } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala index 05b344be92..a702b88d5d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/FullBlockSectionProcessor.scala @@ -99,7 +99,7 @@ trait FullBlockSectionProcessor extends BlockSectionProcessor with FullBlockProc .validate(bsCorrespondsToHeader, header.isCorrespondingModifier(m), InvalidModifier(s"header=${header.encodedId}, id=${m.encodedId}", m.id, m.modifierTypeId)) .validateSemantics(bsHeaderValid, isSemanticallyValid(header.id), InvalidModifier(s"header=${header.encodedId}, id=${m.encodedId}", m.id, m.modifierTypeId)) .validate(bsHeadersChainSynced, isHeadersChainSynced, InvalidModifier(header.id, m.id, m.modifierTypeId)) - .validate(bsTooOld, isHistoryADProof(m, header) || pruningProcessor.shouldDownloadBlockAtHeight(header.height), + .validate(bsTooOld, isHistoryADProof(m, header) || shouldDownloadBlockAtHeight(header.height), InvalidModifier(s"header=${header.encodedId}, id=${m.encodedId}", m.id, m.modifierTypeId)) .result } diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala index 5216f06662..97dfd8fb4c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/HeadersProcessor.scala @@ -10,7 +10,7 @@ import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.nodeView.history.ErgoHistory -import org.ergoplatform.nodeView.history.ErgoHistory.{Difficulty, GenesisHeight} +import org.ergoplatform.nodeView.history.ErgoHistory.{Difficulty, GenesisHeight, Height} import org.ergoplatform.nodeView.history.storage.HistoryStorage import org.ergoplatform.settings.Constants.HashLength import org.ergoplatform.settings.ValidationRules._ @@ -21,6 +21,7 @@ import scorex.core.utils.ScorexEncoding import scorex.core.validation.{InvalidModifier, ModifierValidator, ValidationResult, ValidationState} import scorex.db.ByteArrayWrapper import scorex.util._ +import scorex.util.encode.Base16 import scala.annotation.tailrec import scala.collection.mutable.ArrayBuffer @@ -31,6 +32,25 @@ import scala.util.{Failure, Success, Try} */ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with ScorexEncoding { + /** + * Key for database record storing ID of best block header + */ + protected val BestHeaderKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(Header.modifierTypeId)) + + /** + * Key for database record storing ID of best full block + */ + protected val BestFullBlockKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(-1)) + + /** + * Key for database record storing height of first full block stored + */ + protected val MinFullBlockHeightKey = { + // hash of "minfullheight" UTF-8 string + ByteArrayWrapper(Base16.decode("4987eb6a8fecbed88a6f733f456cdf4e334b944f4436be4cab50cacb442e15e6").get) + } + + protected val historyStorage: HistoryStorage protected val settings: ErgoSettings @@ -56,6 +76,16 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score protected[history] def validityKey(id: ModifierId): ByteArrayWrapper = ByteArrayWrapper(Algos.hash("validity".getBytes(ErgoHistory.CharsetName) ++ idToBytes(id))) + override def writeMinimalFullBlockHeight(height: Height): Unit = { + historyStorage.insert( + indexesToInsert = Array(MinFullBlockHeightKey -> Ints.toByteArray(height)), + objectsToInsert = BlockSection.emptyArray) + } + + override def readMinimalFullBlockHeight(): Height = { + historyStorage.getIndex(MinFullBlockHeightKey).map(Ints.fromByteArray).getOrElse(ErgoHistory.GenesisHeight) + } + def bestHeaderIdOpt: Option[ModifierId] = historyStorage.getIndex(BestHeaderKey).map(bytesToId) /** @@ -158,10 +188,6 @@ trait HeadersProcessor extends ToDownloadProcessor with ScorexLogging with Score */ protected def validate(header: Header): Try[Unit] = HeaderValidator.validate(header).toTry - protected val BestHeaderKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(Header.modifierTypeId)) - - protected val BestFullBlockKey: ByteArrayWrapper = ByteArrayWrapper(Array.fill(HashLength)(-1)) - /** * @param id - header id * @return score of header with such id if is in History diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/MinimalFullBlockHeightFunctions.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/MinimalFullBlockHeightFunctions.scala new file mode 100644 index 0000000000..fc53aca3c0 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/MinimalFullBlockHeightFunctions.scala @@ -0,0 +1,24 @@ +package org.ergoplatform.nodeView.history.storage.modifierprocessors + +import org.ergoplatform.nodeView.history.ErgoHistory.Height + +/** + * Interface to methods providing updating and reading height of first full block stored in local database + */ +trait MinimalFullBlockHeightFunctions { + + /** + * @return minimal height to applu full blocks from. Its value depends on node settings, + * if bootstrapping with UTXO set snapshot is used, the value is being set to a first block after the snapshot, + * for other modes, if blockToKeep > 0, the value is being set to a first block of blockchain suffix + * after headers downloaded, and the value is updated when new blocks added to the chain. If blockToKeep == 0, + * min full block height is set to genesis block height (1). + */ + def readMinimalFullBlockHeight(): Height + + /** + * Update min full block height, see `readMinimalFullBlockHeight` scaladoc on how it should be done + */ + def writeMinimalFullBlockHeight(height: Height): Unit + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala index 508e203c18..de74ea9af2 100644 --- a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/ToDownloadProcessor.scala @@ -1,6 +1,7 @@ package org.ergoplatform.nodeView.history.storage.modifierprocessors -import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId} +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.modifiers.{ErgoFullBlock, NetworkObjectTypeId, SnapshotsInfoTypeId} import org.ergoplatform.modifiers.history._ import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.settings.{ChainSettings, ErgoSettings, NodeConfigurationSettings} @@ -11,8 +12,10 @@ import scala.annotation.tailrec /** * Trait that calculates next modifiers we should download to synchronize our full chain with headers chain */ -trait ToDownloadProcessor extends BasicReaders with ScorexLogging { - import ToDownloadProcessor._ +trait ToDownloadProcessor + extends FullBlockPruningProcessor with UtxoSetSnapshotProcessor with BasicReaders with ScorexLogging { + + import scorex.core.utils.MapPimp protected val settings: ErgoSettings @@ -20,9 +23,6 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { // than headerChainDiff blocks on average from future private lazy val headerChainDiff = settings.nodeSettings.headerChainDiff - protected[history] lazy val pruningProcessor: FullBlockPruningProcessor = - new FullBlockPruningProcessor(nodeSettings, chainSettings) - protected def nodeSettings: NodeConfigurationSettings = settings.nodeSettings protected def chainSettings: ChainSettings = settings.chainSettings @@ -31,29 +31,28 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { def isInBestChain(id: ModifierId): Boolean - /** Returns true if we estimate that our chain is synced with the network. Start downloading full blocks after that + /** + * @return estimated height of a best chain found in the network */ - def isHeadersChainSynced: Boolean = pruningProcessor.isHeadersChainSynced + def estimatedTip(): Option[Height] /** - * Get modifier ids to download to synchronize full blocks + * Get network object ids to download to synchronize full blocks or start UTXO set snapshot downlood * @param howManyPerType how many ModifierIds per ModifierTypeId to fetch * @param condition only ModifierIds which pass filter are included into results * @return next max howManyPerType ModifierIds by ModifierTypeId to download filtered by condition */ def nextModifiersToDownload(howManyPerType: Int, - estimatedTip: Option[Int], condition: (NetworkObjectTypeId.Value, ModifierId) => Boolean): Map[NetworkObjectTypeId.Value, Seq[ModifierId]] = { val FullBlocksToDownloadAhead = 192 // how many full blocks to download forwards during active sync - - def farAwayFromBeingSynced(fb: ErgoFullBlock) = fb.height < (estimatedTip.getOrElse(0) - 128) + def farAwayFromBeingSynced(fb: ErgoFullBlock): Boolean = fb.height < (estimatedTip().getOrElse(0) - 128) @tailrec def continuation(height: Int, acc: Map[NetworkObjectTypeId.Value, Vector[ModifierId]], - maxHeight: Int = Int.MaxValue): Map[NetworkObjectTypeId.Value, Vector[ModifierId]] = { + maxHeight: Int): Map[NetworkObjectTypeId.Value, Vector[ModifierId]] = { if (height > maxHeight) { acc } else { @@ -86,10 +85,17 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { // when blockchain is about to be synced, // download children blocks of last 100 full blocks applied to the best chain, to get block sections from forks val minHeight = Math.max(1, fb.header.height - 100) - continuation(minHeight, Map.empty) - case _ => + continuation(minHeight, Map.empty, maxHeight = Int.MaxValue) + case None if (nodeSettings.utxoSettings.utxoBootstrap && !isUtxoSnapshotApplied) => + // if bootstrapping with UTXO set snapshot is chosen, and no snapshot applied yet, ask peers for snapshots + if (utxoSetSnapshotDownloadPlan().isEmpty) { + Map(SnapshotsInfoTypeId.value -> Seq.empty) + } else { + Map.empty + } + case None => // if headers-chain is synced and no full blocks applied yet, find full block height to go from - continuation(pruningProcessor.minimalFullBlockHeight, Map.empty) + continuation(minimalFullBlockHeight, Map.empty, maxHeight = Int.MaxValue) } } @@ -100,12 +106,12 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { if (!nodeSettings.verifyTransactions) { // A regime that do not download and verify transaction Nil - } else if (pruningProcessor.shouldDownloadBlockAtHeight(header.height)) { + } else if (shouldDownloadBlockAtHeight(header.height)) { // Already synced and header is not too far back. Download required modifiers. requiredModifiersForHeader(header) } else if (!isHeadersChainSynced && header.isNew(chainSettings.blockInterval * headerChainDiff)) { // Headers chain is synced after this header. Start downloading full blocks - pruningProcessor.updateBestFullBlock(header) + updateBestFullBlock(header) log.info(s"Headers chain is likely synced after header ${header.encodedId} at height ${header.height}") Nil } else { @@ -127,15 +133,3 @@ trait ToDownloadProcessor extends BasicReaders with ScorexLogging { } } - -object ToDownloadProcessor { - implicit class MapPimp[K, V](underlying: Map[K, V]) { - /** - * One liner for updating a Map with the possibility to handle case of missing Key - * @param k map key - * @param f function that is passed Option depending on Key being present or missing, returning new Value - * @return new Map with value updated under given key - */ - def adjust(k: K)(f: Option[V] => V): Map[K, V] = underlying.updated(k, f(underlying.get(k))) - } -} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotDownloadPlan.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotDownloadPlan.scala new file mode 100644 index 0000000000..0de26e2545 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotDownloadPlan.scala @@ -0,0 +1,77 @@ +package org.ergoplatform.nodeView.history.storage.modifierprocessors + +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.nodeView.state.UtxoState.SubtreeId +import scorex.core.network.ConnectedPeer +import scorex.crypto.authds.avltree.batch.serialization.BatchAVLProverManifest +import scorex.crypto.hash.Digest32 + + +/** + * Entity which stores information about state of UTXO set snapshots downloading + * + * @param createdTime - time when snapshot was created + * @param latestUpdateTime - latest time when anything was updated for the state of UTXO set snapshot + * @param snapshotHeight - height of a block UTXO set snapshot is corresponding to (UTXO set if after the block applied) + * @param utxoSetRootHash - root hash of AVL+ tree which is authenticating UTXO set snapshot + * @param utxoSetTreeHeight - tree height of AVL+ tree which is authenticating UTXO set snapshot + * @param expectedChunkIds - ids of UTXO set snapshot chunks to be downloaded + * @param downloadedChunkIds - shapshot chunks already downloaded, in form of boolean map over + * `expectedChunkIds` (true = downloaded) + * @param downloadingChunks - number of UTXO set shapshot chunks the node is currently downloading + * @param peersToDownload - peers UTXO set snapshot chunks can be downloaded from + */ +case class UtxoSetSnapshotDownloadPlan(createdTime: Long, + latestUpdateTime: Long, + snapshotHeight: Height, + utxoSetRootHash: Digest32, + utxoSetTreeHeight: Byte, + expectedChunkIds: IndexedSeq[SubtreeId], + downloadedChunkIds: IndexedSeq[Boolean], + downloadingChunks: Int, + peersToDownload: Seq[ConnectedPeer]) { + + def id: Digest32 = utxoSetRootHash + + /** + * @return how many chunks to download + */ + def totalChunks: Int = expectedChunkIds.size + + /** + * @return whether UTXO set snapshot fully downloaded + */ + def fullyDownloaded: Boolean = { + (expectedChunkIds.size == downloadedChunkIds.size) && + downloadingChunks == 0 && + downloadedChunkIds.forall(_ == true) + } + +} + +object UtxoSetSnapshotDownloadPlan { + + /** + * Create UTXO set snapshot download plan from manifest, height of a block corresponding to UTXO set + * manifest represents, and peers to download UTXO set snapshot from + */ + def fromManifest(manifest: BatchAVLProverManifest[Digest32], + blockHeight: Height, + peersToDownload: Seq[ConnectedPeer]): UtxoSetSnapshotDownloadPlan = { + val subtrees = manifest.subtreesIds + val now = System.currentTimeMillis() + + // it is safe to call .toByte below, as the whole tree has height <= 127, and manifest even less + UtxoSetSnapshotDownloadPlan( + createdTime = now, + latestUpdateTime = now, + snapshotHeight = blockHeight, + utxoSetRootHash = manifest.id, + utxoSetTreeHeight = manifest.rootHeight.toByte, + expectedChunkIds = subtrees.toIndexedSeq, + downloadedChunkIds = IndexedSeq.empty, + downloadingChunks = 0, + peersToDownload = peersToDownload) + } + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala new file mode 100644 index 0000000000..66f614da64 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/history/storage/modifierprocessors/UtxoSetSnapshotProcessor.scala @@ -0,0 +1,238 @@ +package org.ergoplatform.nodeView.history.storage.modifierprocessors + +import com.google.common.primitives.Ints +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} +import org.ergoplatform.nodeView.history.storage.HistoryStorage +import org.ergoplatform.nodeView.state.{ErgoStateReader, UtxoState} +import org.ergoplatform.nodeView.state.UtxoState.SubtreeId +import org.ergoplatform.settings.{Algos, ErgoAlgos, ErgoSettings} +import scorex.core.VersionTag +import scorex.core.network.ConnectedPeer +import scorex.core.serialization.SubtreeSerializer +import scorex.crypto.authds.avltree.batch.serialization.{BatchAVLProverManifest, BatchAVLProverSubtree} +import scorex.crypto.hash.{Blake2b256, Digest32} +import scorex.db.LDBVersionedStore +import scorex.util.{ModifierId, ScorexLogging} +import spire.syntax.all.cfor + +import scala.util.{Failure, Random, Success, Try} +import scorex.crypto.authds.avltree.batch.{BatchAVLProver, PersistentBatchAVLProver, VersionedLDBAVLStorage} + +/** + * Parts of history processing and storage corresponding to UTXO set snapshot processing and storage + * + * Stores UTXO set snapshots manifests and chunks for incomplete snapshots. + */ +trait UtxoSetSnapshotProcessor extends MinimalFullBlockHeightFunctions with ScorexLogging { + + import org.ergoplatform.settings.ErgoAlgos.HF + + // node config to read history-related settings here and in descendants + protected val settings: ErgoSettings + + // database to read history-related objects here and in descendants + protected val historyStorage: HistoryStorage + + private val downloadedChunksPrefix = Blake2b256.hash("downloaded chunk").drop(4) + + private var _manifest: Option[BatchAVLProverManifest[Digest32]] = None + + private var _cachedDownloadPlan: Option[UtxoSetSnapshotDownloadPlan] = None + + /** + * @return if UTXO set snapshot was applied during this session (stored in memory only). + * This flag is needed to prevent double application of UTXO set snapshot. + * After first full-block block application not needed anymore. + */ + def isUtxoSnapshotApplied: Boolean = { + readMinimalFullBlockHeight() > ErgoHistory.GenesisHeight + } + + /** + * Writes that UTXO set snapshot applied at height `height`. Starts full blocks applications since the next block + * after. + */ + def onUtxoSnapshotApplied(height: Height): Unit = { + val utxoPhaseTime = { + _cachedDownloadPlan.map(_.latestUpdateTime).getOrElse(0L) - _cachedDownloadPlan.map(_.createdTime).getOrElse(0L) + } + log.info(s"UTXO set downloading and application time: ${utxoPhaseTime / 1000.0} s.") + // remove downloaded utxo set snapshots chunks + val ts0 = System.currentTimeMillis() + _cachedDownloadPlan.foreach { plan => + val chunkIdsToRemove = downloadedChunkIdsIterator(plan.totalChunks) + .map(chunkId => ModifierId @@ Algos.encode(chunkId)) + .toArray + historyStorage.remove(Array.empty, chunkIdsToRemove) + } + _manifest = None + _cachedDownloadPlan = None + val ts = System.currentTimeMillis() + log.info(s"Imported UTXO set snapshots chunks removed in ${ts - ts0} ms") + + // set height of first full block to be downloaded + writeMinimalFullBlockHeight(height + 1) + } + + private def updateUtxoSetSnashotDownloadPlan(plan: UtxoSetSnapshotDownloadPlan): Unit = { + _cachedDownloadPlan = Some(plan) + } + + /** + * Register manifest as one to be downloaded and create download plan from it + * @param manifest - manifest corresponding to UTXO set snapshot to be downloaded + * @param blockHeight - height of a block corresponding to the manifest + * @param peersToDownload - peers to download chunks related to manifest from + * @return download plan + */ + def registerManifestToDownload(manifest: BatchAVLProverManifest[Digest32], + blockHeight: Height, + peersToDownload: Seq[ConnectedPeer]): UtxoSetSnapshotDownloadPlan = { + val plan = UtxoSetSnapshotDownloadPlan.fromManifest(manifest, blockHeight, peersToDownload) + _manifest = Some(manifest) + updateUtxoSetSnashotDownloadPlan(plan) + plan + } + + /** + * @return UTXO set snapshot download plan, if available + */ + def utxoSetSnapshotDownloadPlan(): Option[UtxoSetSnapshotDownloadPlan] = { + _cachedDownloadPlan match { + case s@Some(_) => s + case None => None + } + } + + /** + * @return random peer from which UTXO snapshot chunks can be requested + */ + def randomPeerToDownloadChunks(): Option[ConnectedPeer] = { + val peers = _cachedDownloadPlan.map(_.peersToDownload).getOrElse(Seq.empty) + if (peers.nonEmpty) { + Some(peers(Random.nextInt(peers.size))) + } else { + None + } + } + + /** + * @return up to `howMany` ids of UTXO set snapshot chunks to download + */ + def getChunkIdsToDownload(howMany: Int): Seq[SubtreeId] = { + utxoSetSnapshotDownloadPlan() match { + case Some(plan) => + val expected = plan.expectedChunkIds + val downloadIndex = plan.downloadedChunkIds.size + val toDownload = if (expected.size > downloadIndex) { + expected.slice(downloadIndex, downloadIndex + howMany) + } else { + IndexedSeq.empty + } + log.info(s"Downloaded or waiting ${plan.downloadedChunkIds.size} chunks out of ${expected.size}, downloading ${toDownload.size} more") + val newDownloaded = plan.downloadedChunkIds ++ toDownload.map(_ => false) + val newDownloading = plan.downloadingChunks + toDownload.size + val updPlan = plan.copy( + latestUpdateTime = System.currentTimeMillis(), + downloadedChunkIds = newDownloaded, + downloadingChunks = newDownloading + ) + _cachedDownloadPlan = Some(updPlan) + toDownload + + case None => + log.warn(s"No download plan is found when requested to propose $howMany chunks to download") + Seq.empty + } + } + + /** + * Write serialized UTXO set snapshot chunk to the database + */ + def registerDownloadedChunk(chunkId: Array[Byte], chunkSerialized: Array[Byte]): Unit = { + utxoSetSnapshotDownloadPlan() match { + case Some(plan) => + cfor(0)(_ < plan.downloadedChunkIds.size, _ + 1) { idx => + if (!plan.downloadedChunkIds(idx) && plan.expectedChunkIds(idx).sameElements(chunkId)) { + historyStorage.insert(chunkIdFromIndex(idx), chunkSerialized) + val updDownloaded = plan.downloadedChunkIds.updated(idx, true) + val updDownloading = plan.downloadingChunks - 1 + val updPlan = plan.copy(latestUpdateTime = System.currentTimeMillis(), downloadedChunkIds = updDownloaded, downloadingChunks = updDownloading) + updateUtxoSetSnashotDownloadPlan(updPlan) + return + } + } + case None => + log.warn(s"Chunk ${Algos.encode(chunkId)} downloaded but no download plan found") + } + } + + private def chunkIdFromIndex(index: Int): Array[Byte] = { + val idxBytes = Ints.toByteArray(index) + downloadedChunksPrefix ++ idxBytes + } + + private def downloadedChunkIdsIterator(totalChunks: Int): Iterator[Array[Byte]] = { + Iterator.range(0, totalChunks).map(chunkIdFromIndex) + } + + /** + * @return iterator for chunks downloaded. Reads them from database one-by-one when requested. + */ + def downloadedChunksIterator(): Iterator[BatchAVLProverSubtree[Digest32]] = { + utxoSetSnapshotDownloadPlan() match { + case Some(plan) => + downloadedChunkIdsIterator(plan.totalChunks).flatMap { chunkId => + historyStorage + .get(chunkId) + .flatMap(bs => SubtreeSerializer.parseBytesTry(bs).toOption) + } + case None => + log.error("No download plan found in downloadedChunksIterator") + Iterator.empty + } + } + + /** + * Create disk-persistent authenticated AVL+ tree prover + * @param stateStore - disk database where AVL+ tree will be after restoration + * @param historyReader - history readed to get headers to restore state context + * @param height - height for which prover will be created (prover state will correspond to a + * moment after application of a block at this height) + * @param blockId - id of a block corresponding to the tree (tree is on top of a state after the block) + * @return prover with initialized tree database + */ + def createPersistentProver(stateStore: LDBVersionedStore, + historyReader: ErgoHistoryReader, + height: Height, + blockId: ModifierId): Try[PersistentBatchAVLProver[Digest32, HF]] = { + _manifest match { + case Some(manifest) => + log.info("Starting UTXO set snapshot transfer into state database") + ErgoStateReader.reconstructStateContextBeforeEpoch(historyReader, height, settings) match { + case Success(esc) => + val metadata = UtxoState.metadata(VersionTag @@@ blockId, VersionedLDBAVLStorage.digest(manifest.id, manifest.rootHeight), None, esc) + VersionedLDBAVLStorage.recreate(manifest, downloadedChunksIterator(), additionalData = metadata.toIterator, stateStore).flatMap { + ldbStorage => + log.info("Finished UTXO set snapshot transfer into state database") + ldbStorage.restorePrunedProver().map { + prunedAvlProver => + new PersistentBatchAVLProver[Digest32, HF] { + override var avlProver: BatchAVLProver[Digest32, ErgoAlgos.HF] = prunedAvlProver + override val storage: VersionedLDBAVLStorage = ldbStorage + } + } + } + case Failure(e) => + log.warn("Can't reconstruct state context in createPersistentProver ", e) + Failure(e) + } + case None => + val msg = "No manifest available in createPersistentProver" + log.error(msg) + Failure(new Exception(msg)) + } + } + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala index d4e1f27595..0711bc1a3c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/DigestState.scala @@ -28,7 +28,7 @@ import scala.util.{Failure, Success, Try} class DigestState protected(override val version: VersionTag, override val rootDigest: ADDigest, override val store: LDBVersionedStore, - ergoSettings: ErgoSettings) + override val ergoSettings: ErgoSettings) extends ErgoState[DigestState] with ScorexLogging with ScorexEncoding { @@ -36,8 +36,6 @@ class DigestState protected(override val version: VersionTag, store.lastVersionID .foreach(id => require(version == bytesToVersion(id), "version should always be equal to store.lastVersionID")) - override val constants: StateConstants = StateConstants(ergoSettings) - private lazy val nodeSettings = ergoSettings.nodeSettings private[state] def validateTransactions(transactions: Seq[ErgoTransaction], @@ -163,46 +161,48 @@ object DigestState extends ScorexLogging with ScorexEncoding { rootHash: ADDigest, stateContext: ErgoStateContext, dir: File, - constants: StateConstants, - parameters: Parameters): Try[DigestState] = { - val store = new LDBVersionedStore(dir, initialKeepVersions = constants.keepVersions) + settings: ErgoSettings): Try[DigestState] = { + val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) val toUpdate = DigestState.metadata(version, rootHash, stateContext) store.update(scorex.core.versionToBytes(version), Seq.empty, toUpdate).map { _ => - new DigestState(version, rootHash, store, constants.settings) + new DigestState(version, rootHash, store, settings) } } + /** + * Read digest state from disk, or generate it from genesis data if nothing on the disk + */ def create(versionOpt: Option[VersionTag], rootHashOpt: Option[ADDigest], dir: File, - constants: StateConstants): DigestState = { - val store = new LDBVersionedStore(dir, initialKeepVersions = constants.keepVersions) + settings: ErgoSettings): DigestState = { + val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) Try { - val context = ErgoStateReader.storageStateContext(store, constants) + val context = ErgoStateReader.storageStateContext(store, settings) (versionOpt, rootHashOpt) match { case (Some(version), Some(rootHash)) => val state = if (store.lastVersionID.map(w => bytesToVersion(w)).contains(version)) { - new DigestState(version, rootHash, store, constants.settings) + new DigestState(version, rootHash, store, settings) } else { val inVersion = store.lastVersionID.map(w => bytesToVersion(w)).getOrElse(version) - new DigestState(inVersion, rootHash, store, constants.settings) + new DigestState(inVersion, rootHash, store, settings) .update(version, rootHash, context).get //sync store } state.ensuring(bytesToVersion(store.lastVersionID.get) == version) case (None, None) if store.lastVersionID.isEmpty => - ErgoState.generateGenesisDigestState(dir, constants.settings) + ErgoState.generateGenesisDigestState(dir, settings) case _ => val version = store.lastVersionID.get val rootHash = store.get(version).get - new DigestState(bytesToVersion(version), ADDigest @@ rootHash, store, constants.settings) + new DigestState(bytesToVersion(version), ADDigest @@ rootHash, store, settings) } } match { case Success(state) => state case Failure(e) => store.close() log.warn(s"Failed to create state with ${versionOpt.map(encoder.encode)} and ${rootHashOpt.map(encoder.encode)}", e) - ErgoState.generateGenesisDigestState(dir, constants.settings) + ErgoState.generateGenesisDigestState(dir, settings) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala index cfc95bba4b..dc3644897f 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoState.scala @@ -230,7 +230,7 @@ object ErgoState extends ScorexLogging { val protection = AtLeast(IntConstant(2), pks) val protectionBytes = ValueSerializer.serialize(protection) val value = emission.foundersCoinsTotal - EmissionRules.CoinsInOneErgo - val prop = ErgoScriptPredef.foundationScript(settings.monetary) + val prop = ErgoTreePredef.foundationScript(settings.monetary) createGenesisBox(value, prop, Seq.empty, Map(R4 -> ByteArrayConstant(protectionBytes))) } @@ -255,44 +255,53 @@ object ErgoState extends ScorexLogging { } /** - * All boxes of genesis state. - * Emission box is always the first. + * Genesis state boxes generator. + * Genesis state is corresponding to the state before the very first block processed. + * For Ergo mainnet, contains emission contract box, proof-of-no--premine box, and treasury contract box */ def genesisBoxes(chainSettings: ChainSettings): Seq[ErgoBox] = { Seq(genesisEmissionBox(chainSettings), noPremineBox(chainSettings), genesisFoundersBox(chainSettings)) } - def generateGenesisUtxoState(stateDir: File, - constants: StateConstants): (UtxoState, BoxHolder) = { + /** + * Generate genesis full (UTXO-set) state by inserting genesis boxes into empty UTXO set. + * Assign `genesisStateDigest` from config as its version. + */ + def generateGenesisUtxoState(stateDir: File, settings: ErgoSettings): (UtxoState, BoxHolder) = { log.info("Generating genesis UTXO state") - val boxes = genesisBoxes(constants.settings.chainSettings) + val boxes = genesisBoxes(settings.chainSettings) val bh = BoxHolder(boxes) - UtxoState.fromBoxHolder(bh, boxes.headOption, stateDir, constants, LaunchParameters).ensuring(us => { + UtxoState.fromBoxHolder(bh, boxes.headOption, stateDir, settings, LaunchParameters).ensuring(us => { log.info(s"Genesis UTXO state generated with hex digest ${Base16.encode(us.rootDigest)}") - java.util.Arrays.equals(us.rootDigest, constants.settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion + java.util.Arrays.equals(us.rootDigest, settings.chainSettings.genesisStateDigest) && us.version == genesisStateVersion }) -> bh } + /** + * Generate genesis digest state similarly to `generateGenesisUtxoState`, but without really storing boxes + */ def generateGenesisDigestState(stateDir: File, settings: ErgoSettings): DigestState = { - DigestState.create(Some(genesisStateVersion), Some(settings.chainSettings.genesisStateDigest), - stateDir, StateConstants(settings)) + DigestState.create(Some(genesisStateVersion), Some(settings.chainSettings.genesisStateDigest), stateDir, settings) } val preGenesisStateDigest: ADDigest = ADDigest @@ Array.fill(32)(0: Byte) lazy val genesisStateVersion: VersionTag = idToVersion(Header.GenesisParentId) - def readOrGenerate(settings: ErgoSettings, - constants: StateConstants): ErgoState[_] = { + /** + * Read from disk or generate genesis UTXO-set or digest based state + * @param settings - config used to find state database or extract genesis boxes data + */ + def readOrGenerate(settings: ErgoSettings): ErgoState[_] = { val dir = stateDir(settings) dir.mkdirs() settings.nodeSettings.stateType match { - case StateType.Digest => DigestState.create(None, None, dir, constants) - case StateType.Utxo if dir.listFiles().nonEmpty => UtxoState.create(dir, constants) - case _ => ErgoState.generateGenesisUtxoState(dir, constants)._1 + case StateType.Digest => DigestState.create(None, None, dir, settings) + case StateType.Utxo if dir.listFiles().nonEmpty => UtxoState.create(dir, settings) + case _ => ErgoState.generateGenesisUtxoState(dir, settings)._1 } } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala index 37415bf9ec..eabfafa19d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateContext.scala @@ -8,17 +8,17 @@ import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.history.storage.modifierprocessors.ExtensionValidator +import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeStateContext import org.ergoplatform.settings.ValidationRules._ import org.ergoplatform.settings._ -import org.ergoplatform.wallet.protocol.context.ErgoLikeStateContext import scorex.core.serialization.{BytesSerializable, ErgoSerializer} import scorex.core.utils.ScorexEncoding import scorex.core.validation.{InvalidModifier, ModifierValidator, ValidationState} import scorex.crypto.authds.ADDigest import scorex.util.ScorexLogging import scorex.util.serialization.{Reader, Writer} +import sigmastate.basics.CryptoConstants.EcPointType import sigmastate.eval.SigmaDsl -import sigmastate.interpreter.CryptoConstants.EcPointType import special.collection.Coll import scala.util.{Failure, Success, Try} @@ -310,10 +310,6 @@ object ErgoStateContext { */ val eip27Vote: Byte = 8 - def empty(constants: StateConstants, parameters: Parameters): ErgoStateContext = { - empty(constants.settings.chainSettings.genesisStateDigest, constants.settings, parameters) - } - def empty(settings: ErgoSettings, parameters: Parameters): ErgoStateContext = { empty(settings.chainSettings.genesisStateDigest, settings, parameters) } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala index 9479fe4cf5..18947880c5 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/ErgoStateReader.scala @@ -1,13 +1,17 @@ package org.ergoplatform.nodeView.state import org.ergoplatform.ErgoBox -import org.ergoplatform.settings.{Algos, LaunchParameters, Parameters, VotingSettings} +import org.ergoplatform.nodeView.history.ErgoHistory.Height +import org.ergoplatform.nodeView.history.ErgoHistoryReader +import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, LaunchParameters, Parameters} import scorex.core.{NodeViewComponent, VersionTag} import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.db.LDBVersionedStore import scorex.util.ScorexLogging +import scala.util.{Failure, Success, Try} + /** * State-related data and functions related to any state implementation ("utxo" or "digest") which are * not modifying the state (so only reading it) @@ -26,27 +30,30 @@ trait ErgoStateReader extends NodeViewComponent with ScorexLogging { def version: VersionTag val store: LDBVersionedStore - val constants: StateConstants - - private lazy val chainSettings = constants.settings.chainSettings - protected lazy val votingSettings: VotingSettings = chainSettings.voting + protected def ergoSettings: ErgoSettings /** * If the state is in its genesis version (before genesis block) */ def isGenesis: Boolean = { - rootDigest.sameElements(constants.settings.chainSettings.genesisStateDigest) + rootDigest.sameElements(ergoSettings.chainSettings.genesisStateDigest) } - def stateContext: ErgoStateContext = ErgoStateReader.storageStateContext(store, constants) + /** + * Blockchain-derived context used in scripts validation. It changes from block to block. + */ + def stateContext: ErgoStateContext = ErgoStateReader.storageStateContext(store, ergoSettings) /** * @return current network parameters used in transaction and block validation (block cost and size limits etc) */ def parameters: Parameters = stateContext.currentParameters - def genesisBoxes: Seq[ErgoBox] = ErgoState.genesisBoxes(chainSettings) + /** + * Genesis state boxes, see `ErgoState.genesisBoxes` for details + */ + def genesisBoxes: Seq[ErgoBox] = ErgoState.genesisBoxes(ergoSettings.chainSettings) } @@ -54,13 +61,49 @@ object ErgoStateReader extends ScorexLogging { val ContextKey: Digest32 = Algos.hash("current state context") - def storageStateContext(store: LDBVersionedStore, constants: StateConstants): ErgoStateContext = { + /** + * Read blockchain-related state context from `store` database + */ + def storageStateContext(store: LDBVersionedStore, settings: ErgoSettings): ErgoStateContext = { store.get(ErgoStateReader.ContextKey) - .flatMap(b => ErgoStateContextSerializer(constants.settings).parseBytesTry(b).toOption) + .flatMap(b => ErgoStateContextSerializer(settings).parseBytesTry(b).toOption) .getOrElse { log.warn("Can't read blockchain parameters from database") - ErgoStateContext.empty(constants, LaunchParameters) + ErgoStateContext.empty(settings, LaunchParameters) + } + } + + /** + * Method to reconstruct state context (used in scripts execution) corresponding to last block of a voting epoch, + * except of voting-defined blockchain parameters. Basically, this method is setting proper last headers. + * Then the first block of a new epoch will set the parameters. + * @param historyReader - history reader to get heights from + * @param height - height for which state context will be reconstructed + * @param settings - chain and node settings + */ + def reconstructStateContextBeforeEpoch(historyReader: ErgoHistoryReader, + height: Height, + settings: ErgoSettings): Try[ErgoStateContext] = { + val epochLength = settings.chainSettings.voting.votingLength + if (height % epochLength != epochLength - 1) { + Failure(new Exception(s"Wrong height provided in reconstructStateContextBeforeEpoch, height $height, epoch length $epochLength")) + } else { + val lastHeaders = height.until(height - Constants.LastHeadersInContext, -1).flatMap { h => + historyReader.bestHeaderAtHeight(h) + } + if (lastHeaders.size != Constants.LastHeadersInContext) { + Failure(new Exception(s"Only ${lastHeaders.size} headers found in reconstructStateContextBeforeEpoch")) + } else { + val empty = ErgoStateContext.empty(settings, LaunchParameters) + val esc = new ErgoStateContext( lastHeaders, + None, + empty.genesisStateDigest, + empty.currentParameters, + empty.validationSettings, + empty.votingData)(settings) + Success(esc) } + } } } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala new file mode 100644 index 0000000000..5170247a7c --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsDb.scala @@ -0,0 +1,156 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId} +import org.ergoplatform.settings.{Algos, ErgoSettings} +import scorex.core.serialization.ManifestSerializer +import scorex.crypto.authds.avltree.batch.VersionedLDBAVLStorage +import scorex.crypto.hash.Digest32 +import scorex.db.{LDBFactory, LDBKVStore} +import scorex.util.ScorexLogging +import scorex.util.encode.Base16 + +import scala.collection.mutable +import scala.collection.mutable.ArrayBuffer +import scala.util.{Failure, Success, Try} + +/** + * Interface for a (non-versioned) database storing UTXO set snapshots and metadata about them + */ +class SnapshotsDb(store: LDBKVStore) extends ScorexLogging { + + private val snapshotInfoKey: Array[Byte] = Array.fill(32)(0: Byte) + + // helper method to write information about store UTXO set snapshots into the database + /// private[nodeView] as used in some tests + private[nodeView] def writeSnapshotsInfo(snapshotsInfo: SnapshotsInfo): Try[Unit] = { + store.insert(snapshotInfoKey, SnapshotsInfoSerializer.toBytes(snapshotsInfo)) + } + + // helper method to read information about store UTXO set snapshots from the database + /// private[nodeView] as used in some tests + private[nodeView] def readSnapshotsInfo: SnapshotsInfo = { + store.get(snapshotInfoKey).map(SnapshotsInfoSerializer.parseBytes).getOrElse(SnapshotsInfo.empty) + } + + /** + * Remove old snapshots in the database and metadata records about them, leaving only `toStore` most recent snapshots + */ + def pruneSnapshots(toStore: Int): Unit = { + log.info("Starting snapshots pruning") + + // sort manifests by height to prune oldest ones after + val manifests = readSnapshotsInfo.availableManifests.toSeq.sortBy(_._1) + val manifestSerializer = ManifestSerializer.defaultSerializer + + if (manifests.size > toStore) { + + val lastManifestBytesOpt = store.get(manifests.last._2) + val lastManifestSubtrees = + lastManifestBytesOpt + .flatMap(bs => manifestSerializer.parseBytesTry(bs).toOption) + .map(_.subtreesIds) + .getOrElse(ArrayBuffer.empty) + .map(Algos.encode) + .toSet + + val toPrune = manifests.dropRight(toStore) + val toLeave = manifests.takeRight(toStore) + + toPrune.foreach { case (h, manifestId) => + val pt0 = System.currentTimeMillis() + val keysToRemove: Array[Array[Byte]] = store.get(manifestId) match { + case Some(manifestBytes) => + manifestSerializer.parseBytesTry(manifestBytes) match { + case Success(m) => + val keys = mutable.ArrayBuilder.make[Array[Byte]]() + val subtrees = m.subtreesIds + keys.sizeHint(subtrees.size + 1) + keys += manifestId + // filter out subtrees which are the same in the latest version of tree snapshot + subtrees.foreach { subtreeId => + if (!lastManifestSubtrees.contains(Algos.encode(subtreeId))) { + keys += subtreeId + } + } + keys.result() + case Failure(e) => + log.error(s"Can't parse manifest ${Base16.encode(manifestId)} :", e) + Array.empty + } + case None => + log.error(s"Manifest ${Base16.encode(manifestId)} not found when should be pruned") + Array.empty + } + store.remove(keysToRemove) + val pt = System.currentTimeMillis() + log.info(s"Pruning snapshot at height $h done in ${pt - pt0} ms.") + } + + val updInfo = new SnapshotsInfo(toLeave.toMap) + writeSnapshotsInfo(updInfo) + + log.info("Snapshots pruning finished") + } else { + log.debug("No snapshots to prune") + } + } + + /** + * Lazily read current UTXO set snapshot from versioned AVL+ tree database and store it in this snapshots database + * + * @param pullFrom - versioned AVL+ tree database to pull snapshot from + * @param height - height of a block snapshot is corresponding to + * @param expectedRootHash - expected tree root hash in `pullFrom` database + * @return - id of the snapshot (root hash of its authenticating AVL+ tree), + * or error happened during read-write process + */ + def writeSnapshot(pullFrom: VersionedLDBAVLStorage, + height: Height, + expectedRootHash: Array[Byte], + manifestDepth: Byte = ManifestSerializer.MainnetManifestDepth): Try[Array[Byte]] = { + pullFrom.dumpSnapshot(store, manifestDepth, expectedRootHash).map { manifestId => + val si = readSnapshotsInfo.withNewManifest(height, Digest32 @@ manifestId) + writeSnapshotsInfo(si) + manifestId + } + } + + /** + * Read manifest bytes without deserializing it. Useful when manifest is to be sent over the wir + * + * @param id - manifest id + */ + def readManifestBytes(id: ManifestId): Option[Array[Byte]] = { + store.get(id) + } + + /** + * Read subtree bytes without deserializing it. Useful when subtree is to be sent over the wir + * + * @param id - subtree id + */ + def readSubtreeBytes(id: SubtreeId): Option[Array[Byte]] = { + store.get(id) + } + +} + +object SnapshotsDb { + + // internal method to open or init snapshots database in given folder + // private[nodeView] to use it in tests also + private[nodeView] def create(dir: String): SnapshotsDb = { + val store = LDBFactory.createKvDb(dir) + new SnapshotsDb(store) + } + + /** + * Read or create snapshots database in a folder defined by provided settings + */ + def create(ergoSettings: ErgoSettings): SnapshotsDb = { + val dir = s"${ergoSettings.directory}/snapshots" + create(dir) + } + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala index 6a57e76c50..575e5bdf7a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/SnapshotsInfo.scala @@ -2,6 +2,10 @@ package org.ergoplatform.nodeView.state import org.ergoplatform.ErgoLikeContext.Height import org.ergoplatform.nodeView.state.UtxoState.ManifestId +import org.ergoplatform.settings.Constants +import scorex.core.serialization.ErgoSerializer +import scorex.crypto.hash.Digest32 +import scorex.util.serialization.{Reader, Writer} /** * Container for UTXO set snapshots the node holds @@ -31,6 +35,27 @@ object SnapshotsInfo { } +object SnapshotsInfoSerializer extends ErgoSerializer[SnapshotsInfo] { + + override def serialize(snapshotsInfo: SnapshotsInfo, w: Writer): Unit = { + w.putUInt(snapshotsInfo.availableManifests.size) + snapshotsInfo.availableManifests.foreach { case (height, manifestId) => + w.putUInt(height) + w.putBytes(manifestId) + } + } + + override def parse(r: Reader): SnapshotsInfo = { + val manifestsCount = r.getUInt().toInt // we read from trusted source, no need for extra checks + val manifests = (1 to manifestsCount).map { _ => + val h = r.getUInt().toInt + val manifestId = Digest32 @@ r.getBytes(Constants.HashLength) + h -> manifestId + }.toMap + new SnapshotsInfo(manifests) + } + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/state/StateConstants.scala b/src/main/scala/org/ergoplatform/nodeView/state/StateConstants.scala deleted file mode 100644 index ba606c5efb..0000000000 --- a/src/main/scala/org/ergoplatform/nodeView/state/StateConstants.scala +++ /dev/null @@ -1,17 +0,0 @@ -package org.ergoplatform.nodeView.state - -import org.ergoplatform.settings.{ErgoSettings, VotingSettings} -import scorex.crypto.authds.ADDigest - -/** - * Constants that do not change when state version changes - * - * @param settings - node settings - */ -case class StateConstants(settings: ErgoSettings) { - - lazy val keepVersions: Int = settings.nodeSettings.keepVersions - lazy val votingSettings: VotingSettings = settings.chainSettings.voting - - lazy val genesisStateDigest: ADDigest = settings.chainSettings.genesisStateDigest -} diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala new file mode 100644 index 0000000000..7194904cc8 --- /dev/null +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoSetSnapshotPersistence.scala @@ -0,0 +1,98 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId} +import org.ergoplatform.settings.Algos.HF +import scorex.crypto.authds.avltree.batch.{PersistentBatchAVLProver, VersionedLDBAVLStorage} +import scorex.crypto.hash.Digest32 +import scorex.util.ScorexLogging +import org.ergoplatform.settings.Constants.MakeSnapshotEvery +import org.ergoplatform.settings.ErgoSettings +import scorex.core.serialization.ManifestSerializer + +import scala.concurrent.Future +import scala.util.Try + +/** + * Functions needed for storing UTXO set snapshots and working with them + */ +trait UtxoSetSnapshotPersistence extends ScorexLogging { + + protected def ergoSettings: ErgoSettings + protected def persistentProver: PersistentBatchAVLProver[Digest32, HF] + + private[nodeView] val snapshotsDb = SnapshotsDb.create(ergoSettings) + + // Dump current UTXO set snapshot to persistent snapshots database + // private[nodeView] as used in tests also + private[nodeView] def dumpSnapshot(height: Height, + expectedRootHash: Array[Byte], + manifestDepth: Byte = ManifestSerializer.MainnetManifestDepth): Try[Array[Byte]] = { + val storage = persistentProver.storage.asInstanceOf[VersionedLDBAVLStorage] + snapshotsDb.writeSnapshot(storage, height, expectedRootHash, manifestDepth) + } + + /** + * Check if it is time to store UTXO set snapshot, if so, store it asynchronously, + * to avoid locking the thread currently working with UTXO set, and then prune snapshots which become + * obsolete (also in a separate thread). + * + * Thus this method is doing everything which is needed to store snapshot when needed. Client logic then + * may simply call it on every block applied to the state. + * + * @param height - height of a block just processed (so we work with UTXO set after block with this height applied) + * @param estimatedTip - estimated height of best blockchain in the network + */ + protected def saveSnapshotIfNeeded(height: Height, estimatedTip: Option[Height]): Unit = { + def timeToTakeSnapshot(height: Int): Boolean = { + height % MakeSnapshotEvery == MakeSnapshotEvery - 1 + } + log.info(s"checking snapshot for $height, simple check: " + timeToTakeSnapshot(height)) + if (ergoSettings.nodeSettings.areSnapshotsStored && + timeToTakeSnapshot(height) && + estimatedTip.nonEmpty && + estimatedTip.get - height <= MakeSnapshotEvery) { + + val ms0 = System.currentTimeMillis() + + // drop tree height byte from digest to get current root hash of AVL+ tree + // we then pass the hash and check in another thread whether it is the same after taking db snapshot + // there is small probability of failure, but that is ok, and better than any locking likely + val expectedRootHash = persistentProver.digest.dropRight(1) + Future { + log.info("Started work within future") + val ft0 = System.currentTimeMillis() + dumpSnapshot(height, expectedRootHash) + snapshotsDb.pruneSnapshots(ergoSettings.nodeSettings.utxoSettings.storingUtxoSnapshots) + val ft = System.currentTimeMillis() + log.info("Work within future finished in: " + (ft - ft0) + " ms.") + }(scala.concurrent.ExecutionContext.Implicits.global) + val ms = System.currentTimeMillis() + log.info("Main thread time to dump utxo set snapshot: " + (ms - ms0) + " ms.") + } + } + + /** + * @return list of stored UTXO set snapshots + */ + def getSnapshotInfo(): SnapshotsInfo = { + snapshotsDb.readSnapshotsInfo + } + + /** + * Read snapshot manifest bytes from database without decoding. Used to serve clients over the wire. + * @param id - manifest id + */ + def getManifestBytes(id: ManifestId): Option[Array[Byte]] = { + snapshotsDb.readManifestBytes(id) + } + + /** + * Read snapshot chunk (subtree) bytes from database without decoding. Used to serve clients over the wire. + * @param id - subtree id + */ + def getUtxoSnapshotChunkBytes(id: SubtreeId): Option[Array[Byte]] = { + snapshotsDb.readSubtreeBytes(id) + } + +} diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala index 5ea575903e..96a2ce3817 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoState.scala @@ -9,7 +9,7 @@ import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.settings.Algos.HF import org.ergoplatform.settings.ValidationRules.{fbDigestIncorrect, fbOperationFailed} -import org.ergoplatform.settings.{Algos, Parameters} +import org.ergoplatform.settings.{Algos, ErgoSettings, Parameters} import org.ergoplatform.utils.LoggingUtil import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedModifier import scorex.core._ @@ -26,17 +26,17 @@ import scorex.util.ModifierId import scala.util.{Failure, Success, Try} /** - * Utxo set implementation + * Utxo set based state implementation * - * @param persistentProver - persistent prover that build authenticated AVL+ tree on top of utxo set + * @param persistentProver - persistent prover that builds authenticated AVL+ tree on top of utxo set * @param store - storage of persistentProver that also keeps metadata * @param version - current state version - * @param constants - constants, that do not change with state version changes + * @param ergoSettings - protocol and client config to to get state-related settings from */ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32, HF], override val version: VersionTag, override val store: LDBVersionedStore, - override val constants: StateConstants) + override protected val ergoSettings: ErgoSettings) extends ErgoState[UtxoState] with TransactionValidation with UtxoStateReader @@ -55,7 +55,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 case Some(hash) => val rootHash: ADDigest = ADDigest @@ hash val rollbackResult = p.rollback(rootHash).map { _ => - new UtxoState(p, version, store, constants) + new UtxoState(p, version, store, ergoSettings) } rollbackResult case None => @@ -114,12 +114,14 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 (generate: LocallyGeneratedModifier => Unit): Try[UtxoState] = mod match { case fb: ErgoFullBlock => + val keepVersions = ergoSettings.nodeSettings.keepVersions + // avoid storing versioned information in the database when block being processed is behind // blockchain tip by `keepVersions` blocks at least // we store `keepVersions` diffs in the database if chain tip is not known yet - if (fb.height >= estimatedTip.getOrElse(0) - constants.keepVersions) { - if (store.getKeepVersions < constants.keepVersions) { - store.setKeepVersions(constants.keepVersions) + if (fb.height >= estimatedTip.getOrElse(0) - keepVersions) { + if (store.getKeepVersions < keepVersions) { + store.setKeepVersions(keepVersions) } } else { if (store.getKeepVersions > 0) { @@ -200,7 +202,8 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 log.info(s"Valid modifier with header ${fb.header.encodedId} and emission box " + s"${emissionBox.map(e => Algos.encode(e.id))} applied to UtxoState at height ${fb.header.height}") - new UtxoState(persistentProver, idToVersion(fb.id), store, constants) + saveSnapshotIfNeeded(fb.height, estimatedTip) + new UtxoState(persistentProver, idToVersion(fb.id), store, ergoSettings) } } stateTry.recoverWith[UtxoState] { case e => @@ -210,7 +213,6 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 .ensuring(java.util.Arrays.equals(persistentProver.digest, inRoot)) Failure(e) } - } case h: Header => @@ -218,7 +220,7 @@ class UtxoState(override val persistentProver: PersistentBatchAVLProver[Digest32 //todo: update state context with headers (when snapshot downloading is done), so //todo: application of the first full block after the snapshot should have correct state context //todo: (in particular, "lastHeaders" field of it) - Success(new UtxoState(persistentProver, idToVersion(h.id), this.store, constants)) + Success(new UtxoState(persistentProver, idToVersion(h.id), this.store, ergoSettings)) case a: Any => log.error(s"Unhandled unknown modifier: $a") @@ -260,11 +262,21 @@ object UtxoState { private lazy val bestVersionKey = Algos.hash("best state version") val EmissionBoxIdKey: Digest32 = Algos.hash("emission box id key") - // block-specific metadata to write into database (in addition to AVL+ tree) - private def metadata(modId: VersionTag, - stateRoot: ADDigest, - currentEmissionBoxOpt: Option[ErgoBox], - context: ErgoStateContext): Seq[(Array[Byte], Array[Byte])] = { + + /** + * Block-specific metadata to write into database (in addition to AVL+ tree) + * + * @param modId - ID of a block (header) corresponding to UTXO set + * @param stateRoot - UTXO set digest (hash and tree height) AFTER applying block `modId` + * @param currentEmissionBoxOpt - current unspent emission script box + * @param context - current state context used in input scripts validation (for the next block) + * + * @return binary-serialized metadata + */ + def metadata(modId: VersionTag, + stateRoot: ADDigest, + currentEmissionBoxOpt: Option[ErgoBox], + context: ErgoStateContext): Seq[(Array[Byte], Array[Byte])] = { val modIdBytes = versionToBytes(modId) val idStateDigestIdxElem: (Array[Byte], Array[Byte]) = modIdBytes -> stateRoot val stateDigestIdIdxElem = Algos.hash(stateRoot) -> modIdBytes @@ -275,8 +287,11 @@ object UtxoState { Array(idStateDigestIdxElem, stateDigestIdIdxElem, bestVersion, eb, cb) } - def create(dir: File, constants: StateConstants): UtxoState = { - val store = new LDBVersionedStore(dir, initialKeepVersions = constants.keepVersions) + /** + * @return UTXO set based state on top of existing database, or genesis state if the database is empty + */ + def create(dir: File, settings: ErgoSettings): UtxoState = { + val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) val version = store.get(bestVersionKey).map(w => bytesToVersion(w)) .getOrElse(ErgoState.genesisStateVersion) val persistentProver: PersistentBatchAVLProver[Digest32, HF] = { @@ -284,7 +299,7 @@ object UtxoState { val storage = new VersionedLDBAVLStorage(store) PersistentBatchAVLProver.create(bp, storage).get } - new UtxoState(persistentProver, version, store, constants) + new UtxoState(persistentProver, version, store, settings) } /** @@ -294,16 +309,16 @@ object UtxoState { def fromBoxHolder(bh: BoxHolder, currentEmissionBoxOpt: Option[ErgoBox], dir: File, - constants: StateConstants, + settings: ErgoSettings, parameters: Parameters): UtxoState = { val p = new BatchAVLProver[Digest32, HF](keyLength = 32, valueLengthOpt = None) bh.sortedBoxes.foreach { b => p.performOneOperation(Insert(b.id, ADValue @@ b.bytes)).ensuring(_.isSuccess) } - val store = new LDBVersionedStore(dir, initialKeepVersions = constants.keepVersions) + val store = new LDBVersionedStore(dir, initialKeepVersions = settings.nodeSettings.keepVersions) - val defaultStateContext = ErgoStateContext.empty(constants, parameters) + val defaultStateContext = ErgoStateContext.empty(settings, parameters) val storage = new VersionedLDBAVLStorage(store) val persistentProver = PersistentBatchAVLProver.create( p, @@ -312,7 +327,7 @@ object UtxoState { paranoidChecks = true ).get - new UtxoState(persistentProver, ErgoState.genesisStateVersion, store, constants) + new UtxoState(persistentProver, ErgoState.genesisStateVersion, store, settings) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala index 7bac961b68..b856643302 100644 --- a/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/state/UtxoStateReader.scala @@ -5,7 +5,7 @@ import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader -import org.ergoplatform.settings.Algos +import org.ergoplatform.settings.{Algos, ErgoSettings} import org.ergoplatform.settings.Algos.HF import org.ergoplatform.wallet.boxes.ErgoBoxSerializer import org.ergoplatform.wallet.interpreter.ErgoInterpreter @@ -18,11 +18,16 @@ import scorex.crypto.hash.Digest32 import scala.util.{Failure, Success, Try} -trait UtxoStateReader extends ErgoStateReader with TransactionValidation { +/** + * State reader (i.e. state functions not modifying underlying data) with specialization towards UTXO set as a + * state representation (so functions to generate UTXO set modifiction proofs, do stateful transaction validation, + * get UTXOs are there + */ +trait UtxoStateReader extends ErgoStateReader with UtxoSetSnapshotPersistence with TransactionValidation { protected implicit val hf: HF = Algos.hash - val constants: StateConstants + protected def ergoSettings: ErgoSettings /** * Versioned database where UTXO set and its authenticating AVL+ tree are stored @@ -84,13 +89,14 @@ trait UtxoStateReader extends ErgoStateReader with TransactionValidation { */ protected[state] def extractEmissionBox(fb: ErgoFullBlock): Option[ErgoBox] = { def hasEmissionBox(tx: ErgoTransaction): Boolean = - if(fb.height > constants.settings.chainSettings.reemission.activationHeight) { + if(fb.height > ergoSettings.chainSettings.reemission.activationHeight) { // after EIP-27 we search for emission box NFT for efficiency's sake + val emissionNftId = ergoSettings.chainSettings.reemission.emissionNftIdBytes + val outTokens = tx.outputs.head.additionalTokens tx.outputs.size == 2 && - !tx.outputs.head.additionalTokens.isEmpty && - java.util.Arrays.equals(tx.outputs.head.additionalTokens(0)._1, constants.settings.chainSettings.reemission.emissionNftIdBytes) + !outTokens.isEmpty && outTokens(0)._1 == emissionNftId } else { - tx.outputs.head.ergoTree == constants.settings.chainSettings.monetary.emissionBoxProposition + tx.outputs.head.ergoTree == ergoSettings.chainSettings.monetary.emissionBoxProposition } def fullSearch(fb: ErgoFullBlock): Option[ErgoBox] = { @@ -169,7 +175,7 @@ trait UtxoStateReader extends ErgoStateReader with TransactionValidation { * Useful when checking mempool transactions. */ def withUnconfirmedTransactions(unconfirmedTxs: Seq[UnconfirmedTransaction]): UtxoState = { - new UtxoState(persistentProver, version, store, constants) { + new UtxoState(persistentProver, version, store, ergoSettings) { lazy val createdBoxes: Seq[ErgoBox] = unconfirmedTxs.map(_.transaction).flatMap(_.outputs) override def boxById(id: ADKey): Option[ErgoBox] = { @@ -183,7 +189,7 @@ trait UtxoStateReader extends ErgoStateReader with TransactionValidation { * Useful when checking mempool transactions. */ def withTransactions(transactions: Seq[ErgoTransaction]): UtxoState = { - new UtxoState(persistentProver, version, store, constants) { + new UtxoState(persistentProver, version, store, ergoSettings) { lazy val createdBoxes: Seq[ErgoBox] = transactions.flatMap(_.outputs) override def boxById(id: ADKey): Option[ErgoBox] = { diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/BalancesSnapshot.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/BalancesSnapshot.scala index d51b36e807..96a6814898 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/BalancesSnapshot.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/BalancesSnapshot.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.wallet import org.ergoplatform.nodeView.history.ErgoHistory.Height -import org.ergoplatform.wallet.TokensMap +import org.ergoplatform.sdk.wallet.TokensMap final case class BalancesSnapshot(height: Height, balance: Long, assetBalances: TokensMap) diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala index 18b422095a..0d9732964d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletActor.scala @@ -6,6 +6,7 @@ import akka.pattern.StatusReply import org.ergoplatform.ErgoBox._ import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{ChangedMempool, ChangedState} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoHistoryReader} import org.ergoplatform.nodeView.mempool.ErgoMemPoolReader import org.ergoplatform.nodeView.state.ErgoStateReader @@ -13,15 +14,14 @@ import org.ergoplatform.nodeView.wallet.ErgoWalletService.DeriveNextKeyResult import org.ergoplatform.nodeView.wallet.models.CollectedBoxes import org.ergoplatform.nodeView.wallet.requests.{ExternalSecret, TransactionGenerationRequest} import org.ergoplatform.nodeView.wallet.scanning.{Scan, ScanRequest} +import org.ergoplatform.sdk.wallet.secrets.DerivationPath import org.ergoplatform.settings._ -import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.{BoxSelector, ChainStatus} +import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.TransactionHintsBag -import org.ergoplatform.{ErgoAddressEncoder, ErgoApp, ErgoBox, GlobalConstants, P2PKAddress} +import org.ergoplatform._ import scorex.core.VersionTag -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.{ChangedMempool, ChangedState} -import org.ergoplatform.wallet.secrets.DerivationPath import scorex.core.utils.ScorexEncoding import scorex.util.{ModifierId, ScorexLogging} import sigmastate.Values.SigmaBoolean diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala index 7e10887d71..dab82fe87b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletReader.scala @@ -1,28 +1,28 @@ package org.ergoplatform.nodeView.wallet -import java.util.concurrent.TimeUnit import akka.actor.ActorRef import akka.pattern.ask import akka.util.Timeout import org.ergoplatform.ErgoBox.BoxId -import org.ergoplatform.{ErgoBox, P2PKAddress} import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} import org.ergoplatform.nodeView.wallet.ErgoWalletActor._ import org.ergoplatform.nodeView.wallet.ErgoWalletService.DeriveNextKeyResult import org.ergoplatform.nodeView.wallet.persistence.WalletDigest -import org.ergoplatform.nodeView.wallet.scanning.ScanRequest import org.ergoplatform.nodeView.wallet.requests.{BoxesRequest, ExternalSecret, TransactionGenerationRequest} -import org.ergoplatform.wallet.interface4j.SecretString +import org.ergoplatform.nodeView.wallet.scanning.ScanRequest +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedPublicKey} +import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.ChainStatus import org.ergoplatform.wallet.boxes.ChainStatus.{OffChain, OnChain} -import org.ergoplatform.wallet.Constants.ScanId +import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.TransactionHintsBag -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedPublicKey} +import org.ergoplatform.{ErgoBox, P2PKAddress} import scorex.core.NodeViewComponent import scorex.util.ModifierId import sigmastate.Values.SigmaBoolean import sigmastate.basics.DLogProtocol.DLogProverInput +import java.util.concurrent.TimeUnit import scala.concurrent.Future import scala.util.Try diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala index 530ef59fbd..34c82fcded 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletService.scala @@ -11,19 +11,21 @@ import org.ergoplatform.nodeView.wallet.models.{ChangeBox, CollectedBoxes} import org.ergoplatform.nodeView.wallet.persistence.{WalletRegistry, WalletStorage} import org.ergoplatform.nodeView.wallet.requests.{ExternalSecret, TransactionGenerationRequest} import org.ergoplatform.nodeView.wallet.scanning.{Scan, ScanRequest} +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.ergoplatform.settings.{ErgoSettings, Parameters} import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.{BoxSelector, ErgoBoxSerializer, TrackedBox} import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.{ErgoProvingInterpreter, TransactionHintsBag} import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedSecretKey, JsonSecretStorage} +import org.ergoplatform.wallet.secrets.JsonSecretStorage import org.ergoplatform.wallet.settings.SecretStorageSettings import org.ergoplatform.wallet.utils.FileUtils import scorex.util.encode.Base16 -import scorex.util.{ModifierId, bytesToId} +import scorex.util.{ModifierId} import sigmastate.Values.SigmaBoolean import sigmastate.basics.DLogProtocol.DLogProverInput +import special.collection.Extensions.CollBytesOps import java.io.FileNotFoundException import scala.collection.compat.immutable.ArraySeq @@ -447,8 +449,12 @@ class ErgoWalletServiceImpl(override val ergoSettings: ErgoSettings) extends Erg registry.getTx(txId) .map(tx => AugWalletTransaction(tx, fullHeight - tx.inclusionHeight)) - override def collectBoxes(state: ErgoWalletState, boxSelector: BoxSelector, targetBalance: Long, targetAssets: Map[ErgoBox.TokenId, Long]): Try[CollectedBoxes] = { - val assetsMap = targetAssets.map(t => bytesToId(t._1) -> t._2) + override def collectBoxes( + state: ErgoWalletState, + boxSelector: BoxSelector, + targetBalance: Long, + targetAssets: Map[ErgoBox.TokenId, Long]): Try[CollectedBoxes] = { + val assetsMap = targetAssets.map(t => t._1.toModifierId -> t._2) val inputBoxes = state.getBoxesToSpend boxSelector .select(inputBoxes.iterator, state.walletFilter, targetBalance, assetsMap) diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala index cef8c8e37b..669f69da39 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSupport.scala @@ -1,31 +1,30 @@ package org.ergoplatform.nodeView.wallet -import cats.implicits._ import cats.Traverse +import cats.implicits._ import org.ergoplatform.ErgoBox.{BoxId, R4, R5, R6} -import org.ergoplatform.{DataInput, ErgoAddress, ErgoBox, ErgoBoxAssets, ErgoBoxCandidate, P2PKAddress, UnsignedInput} +import org.ergoplatform._ import org.ergoplatform.modifiers.mempool.UnsignedErgoTransaction import org.ergoplatform.nodeView.wallet.ErgoWalletService.DeriveNextKeyResult import org.ergoplatform.nodeView.wallet.persistence.WalletStorage -import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.nodeView.wallet.requests._ -import org.ergoplatform.settings.Parameters +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedPublicKey, ExtendedSecretKey} +import org.ergoplatform.sdk.wallet.{AssetUtils, TokensMap} +import org.ergoplatform.settings.{ErgoSettings, Parameters} import org.ergoplatform.utils.BoxUtils -import org.ergoplatform.wallet.{AssetUtils, Constants, TokensMap} -import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.Constants.PaymentsScanId import org.ergoplatform.wallet.boxes.BoxSelector.BoxSelectionResult import org.ergoplatform.wallet.boxes.{BoxSelector, TrackedBox} +import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.ErgoProvingInterpreter import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedPublicKey, ExtendedSecretKey} import org.ergoplatform.wallet.transactions.TransactionBuilder -import scorex.crypto.hash.Digest32 -import scorex.util.{ScorexLogging, idToBytes} +import scorex.util.ScorexLogging import sigmastate.Values.ByteArrayConstant import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.eval.Extensions._ import sigmastate.eval._ +import sigmastate.utils.Extensions._ import scala.util.{Failure, Success, Try} @@ -44,7 +43,7 @@ trait ErgoWalletSupport extends ScorexLogging { protected def addSecretToStorage(state: ErgoWalletState, secret: ExtendedSecretKey): Try[ErgoWalletState] = state.walletVars.withExtendedKey(secret).flatMap { newWalletVars => - state.storage.addPublicKeys(secret.publicKey).flatMap { _ => + state.storage.addPublicKey(secret.publicKey).flatMap { _ => newWalletVars.stateCacheOpt.get.withNewPubkey(secret.publicKey).map { updCache => state.copy(walletVars = newWalletVars.copy(stateCacheProvided = Some(updCache))(newWalletVars.settings)) } @@ -94,7 +93,7 @@ trait ErgoWalletSupport extends ScorexLogging { path => masterKey.derive(path) } val oldPubKeys = oldDerivedSecrets.map(_.publicKey) - oldPubKeys.foreach(storage.addPublicKeys(_).get) + oldPubKeys.foreach(storage.addPublicKey(_).get) storage.removePaths().get } } @@ -135,7 +134,7 @@ trait ErgoWalletSupport extends ScorexLogging { if (usePreEip3Derivation) { // If usePreEip3Derivation flag is set in the wallet settings, the first key is the master key val masterPubKey = masterKey.publicKey - state.storage.addPublicKeys(masterPubKey).map { _ => + state.storage.addPublicKey(masterPubKey).map { _ => log.info("Wallet unlock finished using usePreEip3Derivation") updatePublicKeys(state, masterKey, Vector(masterPubKey)) } @@ -150,7 +149,7 @@ trait ErgoWalletSupport extends ScorexLogging { deriveNextKeyForMasterKey(sp, masterKey, usePreEip3Derivation).flatMap { case (derivationResult, newState) => derivationResult.result.flatMap { case (_, _, firstSk) => val firstPk = firstSk.publicKey - newState.storage.addPublicKeys(firstPk).flatMap { _ => + newState.storage.addPublicKey(firstPk).flatMap { _ => newState.storage.updateChangeAddress(P2PKAddress(firstPk.key)(addressEncoder)).map { _ => log.info("Wallet unlock finished") updatePublicKeys(newState, masterKey, Vector(firstPk)) @@ -161,7 +160,7 @@ trait ErgoWalletSupport extends ScorexLogging { } } else { if (pubKeys.size == 1 && - pubKeys.head.path == Constants.eip3DerivationPath.toPublicBranch && + pubKeys.head.path == sdk.wallet.Constants.eip3DerivationPath.toPublicBranch && state.storage.readChangeAddress.isEmpty) { val changeAddress = P2PKAddress(pubKeys.head.key)(addressEncoder) log.info(s"Update change address to $changeAddress") @@ -169,7 +168,7 @@ trait ErgoWalletSupport extends ScorexLogging { } // Add master key's public key to the storage to track payments to it when the wallet is locked if (!state.storage.containsPublicKey(masterKey.path.toPublicBranch)) { - state.storage.addPublicKeys(masterKey.publicKey) + state.storage.addPublicKey(masterKey.publicKey) } log.info("Wallet unlock finished using existing keys in storage") Try(updatePublicKeys(state, masterKey, pubKeys)) @@ -212,7 +211,7 @@ trait ErgoWalletSupport extends ScorexLogging { def minimalErgoAmount: Long = BoxUtils.minimalErgoAmountSimulated( lockWithAddress.script, - Colls.fromItems((Digest32 @@@ assetId) -> amount), + Colls.fromItems(assetId.toTokenId -> amount), nonMandatoryRegisters, parameters ) @@ -221,7 +220,7 @@ trait ErgoWalletSupport extends ScorexLogging { valueOpt.getOrElse(minimalErgoAmount), lockWithAddress.script, fullHeight, - Colls.fromItems((Digest32 @@@ assetId) -> amount), + Colls.fromItems(assetId.toTokenId -> amount), nonMandatoryRegisters ) } @@ -248,17 +247,14 @@ trait ErgoWalletSupport extends ScorexLogging { val p2r = ergoSettings.chainSettings.reemission.reemissionRules.payToReemission new ErgoBoxCandidate(eip27OutputAssets.value, p2r, walletHeight) } - } ++ selectionResult.changeBoxes.map { changeBoxAssets => - changeBoxAssets match { - case candidate: ErgoBoxCandidate => - candidate - case changeBox: ErgoBoxAssets => - // todo: is this extra check needed ? - val reemissionTokenId = ergoSettings.chainSettings.reemission.reemissionTokenId - val assets = changeBox.tokens.filterKeys(_ != reemissionTokenId).map(t => Digest32 @@ idToBytes(t._1) -> t._2).toIndexedSeq - - new ErgoBoxCandidate(changeBox.value, changeAddressOpt.get, walletHeight, assets.toColl) - } + } ++ selectionResult.changeBoxes.map { + case candidate: ErgoBoxCandidate => + candidate + case changeBox: ErgoBoxAssets => + // todo: is this extra check needed ? + val reemissionTokenId = ergoSettings.chainSettings.reemission.reemissionTokenId + val assets = changeBox.tokens.filterKeys(_ != reemissionTokenId).map(t => t._1.toTokenId -> t._2).toIndexedSeq + new ErgoBoxCandidate(changeBox.value, changeAddressOpt.get, walletHeight, assets.toColl) } val inputBoxes = selectionResult.inputBoxes.toIndexedSeq new UnsignedErgoTransaction( diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/IdUtils.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/IdUtils.scala index f7d61ec97b..a3040f805c 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/IdUtils.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/IdUtils.scala @@ -3,8 +3,9 @@ package org.ergoplatform.nodeView.wallet import org.ergoplatform.ErgoBox.{BoxId, TokenId} import org.ergoplatform.settings.Algos import scorex.crypto.authds.ADKey -import scorex.crypto.hash.Digest32 import scorex.util.ModifierId +import sigmastate.eval.Digest32Coll +import sigmastate.eval.Extensions.ArrayOps import supertagged.TaggedType object IdUtils { @@ -22,7 +23,7 @@ object IdUtils { def encodedTokenId(id: TokenId): EncodedTokenId = ModifierId @@ Algos.encode(id) - def decodedTokenId(id: EncodedTokenId): TokenId = Digest32 @@ Algos.decode(id) - .getOrElse(throw new Error("Failed to decode token id")) + def decodedTokenId(id: EncodedTokenId): TokenId = + Digest32Coll @@ (Algos.decode(id).getOrElse(throw new Error("Failed to decode token id"))).toColl } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/WalletCache.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/WalletCache.scala index 9fd46cc0ca..4ead415b3b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/WalletCache.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/WalletCache.scala @@ -1,9 +1,9 @@ package org.ergoplatform.nodeView.wallet import com.google.common.hash.{BloomFilter, Funnels} -import org.ergoplatform.{ErgoAddressEncoder, ErgoScriptPredef, P2PKAddress} +import org.ergoplatform.sdk.wallet.secrets.ExtendedPublicKey import org.ergoplatform.settings.ErgoSettings -import org.ergoplatform.wallet.secrets.ExtendedPublicKey +import org.ergoplatform.{ErgoAddressEncoder, ErgoTreePredef, P2PKAddress} import sigmastate.Values import sigmastate.eval._ @@ -49,7 +49,7 @@ object WalletCache { def miningScripts(trackedPubKeys: Seq[ExtendedPublicKey], settings: ErgoSettings): Seq[Values.ErgoTree] = { trackedPubKeys.map { pk => - ErgoScriptPredef.rewardOutputScript(settings.miningRewardDelay, pk.key) + ErgoTreePredef.rewardOutputScript(settings.miningRewardDelay, pk.key) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/WalletVars.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/WalletVars.scala index 9c5055e99f..a6c3a0303a 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/WalletVars.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/WalletVars.scala @@ -1,13 +1,13 @@ package org.ergoplatform.nodeView.wallet import com.google.common.hash.BloomFilter -import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress} import org.ergoplatform.nodeView.wallet.persistence.WalletStorage import org.ergoplatform.nodeView.wallet.scanning.Scan +import org.ergoplatform.sdk.wallet.secrets.{ExtendedPublicKey, ExtendedSecretKey} import org.ergoplatform.settings.{ErgoSettings, Parameters} import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.interpreter.ErgoProvingInterpreter -import org.ergoplatform.wallet.secrets.{ExtendedPublicKey, ExtendedSecretKey} +import org.ergoplatform.{ErgoAddressEncoder, P2PKAddress} import scorex.util.ScorexLogging import scala.util.Try diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/Balance.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/Balance.scala index f6ca24f847..773cf2fd35 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/Balance.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/Balance.scala @@ -10,5 +10,5 @@ case class Balance(id: EncodedBoxId, object Balance { def apply(tb: TrackedBox): Balance = Balance(encodedBoxId(tb.box.id), tb.box.value, - tb.box.additionalTokens.toArray.map(x => (bytesToId(x._1), x._2)).toMap) + tb.box.additionalTokens.toArray.map(x => (bytesToId(x._1.toArray), x._2)).toMap) } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletDigest.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletDigest.scala index 91b2af73cc..9223a4b91e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletDigest.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletDigest.scala @@ -4,10 +4,11 @@ import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.wallet.IdUtils._ import org.ergoplatform.settings.Constants import scorex.core.serialization.ErgoSerializer -import scorex.crypto.hash.Digest32 +import scorex.util.Extensions._ import scorex.util.serialization.{Reader, Writer} +import sigmastate.eval.Extensions.ArrayByteOps + import scala.collection.mutable -import scorex.util.Extensions._ /** * Holds aggregate wallet data (including off-chain) with no need fo re-processing it on each request. @@ -35,7 +36,7 @@ object WalletDigestSerializer extends ErgoSerializer[WalletDigest] { w.putUInt(obj.walletAssetBalances.size) obj.walletAssetBalances.foreach { case (id, amt) => - w.putBytes(decodedTokenId(id)) + w.putBytes(decodedTokenId(id).toArray) w.putULong(amt) } } @@ -48,7 +49,7 @@ object WalletDigestSerializer extends ErgoSerializer[WalletDigest] { val walletAssetBalances = mutable.LinkedHashMap.empty[EncodedTokenId, Long] (0 until walletAssetBalancesSize).foreach { _ => - val kv = encodedTokenId(Digest32 @@ r.getBytes(Constants.ModifierIdSize)) -> r.getULong() + val kv = encodedTokenId(r.getBytes(Constants.ModifierIdSize).toTokenId) -> r.getULong() walletAssetBalances += kv } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala index cc6fd08562..1dc3753b5d 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistry.scala @@ -1,27 +1,27 @@ package org.ergoplatform.nodeView.wallet.persistence -import java.io.File +import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.BoxId +import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform.modifiers.history.header.PreGenesisHeader import org.ergoplatform.nodeView.wallet.IdUtils.{EncodedTokenId, encodedTokenId} +import org.ergoplatform.nodeView.wallet.WalletScanLogic.ScanResults import org.ergoplatform.nodeView.wallet.{WalletTransaction, WalletTransactionSerializer} +import org.ergoplatform.sdk.wallet.AssetUtils import org.ergoplatform.settings.{Algos, ErgoSettings, WalletSettings} -import org.ergoplatform.wallet.{AssetUtils, Constants} +import org.ergoplatform.wallet.Constants +import org.ergoplatform.wallet.Constants.{PaymentsScanId, ScanId} import org.ergoplatform.wallet.boxes.{TrackedBox, TrackedBoxSerializer} -import scorex.core.VersionTag -import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} -import Constants.{PaymentsScanId, ScanId} -import org.ergoplatform.ErgoBox -import org.ergoplatform.ErgoLikeContext.Height -import org.ergoplatform.modifiers.history.header.PreGenesisHeader -import scorex.db.LDBVersionedStore - -import scala.util.{Failure, Success, Try} -import org.ergoplatform.nodeView.wallet.WalletScanLogic.ScanResults import org.ergoplatform.wallet.transactions.TransactionBuilder +import scorex.core.VersionTag import scorex.crypto.authds.ADKey +import scorex.db.LDBVersionedStore import scorex.util.encode.Base16 +import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} +import java.io.File import scala.collection.mutable +import scala.util.{Failure, Success, Try} /** * Provides an access to version-sensitive wallet-specific indexes: @@ -88,7 +88,10 @@ class WalletRegistry(private val store: LDBVersionedStore)(ws: WalletSettings) e * @return sequences of all the unspent boxes from the database */ def allUnspentBoxes(): Seq[TrackedBox] = { - store.getRange(firstUnspentBoxKey, lastUnspentBoxKey).flatMap(pair => getBox(ADKey @@ pair._2)) + store.getRange(firstUnspentBoxKey, lastUnspentBoxKey) + .flatMap { case (_, boxId) => + getBox(ADKey @@ boxId) + } } /** diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala index 6789272124..617755f474 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorage.scala @@ -1,18 +1,18 @@ package org.ergoplatform.nodeView.wallet.persistence import com.google.common.primitives.{Ints, Shorts} +import org.ergoplatform.P2PKAddress import org.ergoplatform.nodeView.state.{ErgoStateContext, ErgoStateContextSerializer} import org.ergoplatform.nodeView.wallet.scanning.{Scan, ScanRequest, ScanSerializer} +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, DerivationPathSerializer, ExtendedPublicKey, ExtendedPublicKeySerializer} import org.ergoplatform.settings.{Constants, ErgoSettings, Parameters} -import org.ergoplatform.wallet.secrets.{DerivationPath, DerivationPathSerializer, ExtendedPublicKey, ExtendedPublicKeySerializer} -import org.ergoplatform.P2PKAddress -import scorex.crypto.hash.Blake2b256 import org.ergoplatform.wallet.Constants.{PaymentsScanId, ScanId} +import scorex.crypto.hash.Blake2b256 import scorex.db.{LDBFactory, LDBKVStore} - -import java.io.File import scorex.util.ScorexLogging +import sigmastate.serialization.SigmaSerializer +import java.io.File import scala.util.{Failure, Success, Try} /** @@ -40,7 +40,8 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco val qty = Ints.fromByteArray(r.take(4)) (0 until qty).foldLeft((Seq.empty[DerivationPath], r.drop(4))) { case ((acc, bytes), _) => val length = Ints.fromByteArray(bytes.take(4)) - val pathTry = DerivationPathSerializer.parseBytesTry(bytes.slice(4, 4 + length)) + val r = SigmaSerializer.startReader(bytes.slice(4, 4 + length)) + val pathTry = DerivationPathSerializer.parseTry(r) val newAcc = pathTry.map(acc :+ _).getOrElse(acc) val bytesTail = bytes.drop(4 + length) newAcc -> bytesTail @@ -55,14 +56,10 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco /** * Store wallet-related public key in the database * - * @param publicKeys - public key to store + * @param publicKey - public key to store */ - def addPublicKeys(publicKeys: ExtendedPublicKey*): Try[Unit] = { - store.insert { - publicKeys.map { publicKey => - pubKeyPrefixKey(publicKey) -> ExtendedPublicKeySerializer.toBytes(publicKey) - }.toArray - } + def addPublicKey(publicKey: ExtendedPublicKey): Try[Unit] = { + store.insert(pubKeyPrefixKey(publicKey), ExtendedPublicKeySerializer.toBytes(publicKey)) } /** @@ -71,8 +68,9 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco def getPublicKey(path: DerivationPath): Option[ExtendedPublicKey] = { store .get(pubKeyPrefixKey(path)) - .flatMap{bytes => - ExtendedPublicKeySerializer.parseBytesTry(bytes) match { + .flatMap { bytes => + val r = SigmaSerializer.startReader(bytes) + ExtendedPublicKeySerializer.parseTry(r) match { case Success(key) => Some(key) case Failure(t) => @@ -92,7 +90,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco */ def readAllKeys(): Seq[ExtendedPublicKey] = { store.getRange(FirstPublicKeyId, LastPublicKeyId).map { case (_, v) => - ExtendedPublicKeySerializer.parseBytes(v) + ExtendedPublicKeySerializer.fromBytes(v) } } @@ -104,7 +102,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco */ def updateStateContext(ctx: ErgoStateContext): Try[Unit] = { cachedStateContext = Some(ctx) - store.insert(Array(StateContextKey -> ctx.bytes)) + store.insert(StateContextKey, ctx.bytes) } /** @@ -126,7 +124,7 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco */ def updateChangeAddress(address: P2PKAddress): Try[Unit] = { val bytes = settings.chainSettings.addressEncoder.toString(address).getBytes(Constants.StringEncoding) - store.insert(Array(ChangeAddressKey -> bytes)) + store.insert(ChangeAddressKey, bytes) } /** @@ -149,10 +147,10 @@ final class WalletStorage(store: LDBKVStore, settings: ErgoSettings) extends Sco def addScan(scanReq: ScanRequest): Try[Scan] = { val id = ScanId @@ (lastUsedScanId + 1).toShort scanReq.toScan(id).flatMap { app => - store.insert(Array( - scanPrefixKey(id) -> ScanSerializer.toBytes(app), - lastUsedScanIdKey -> Shorts.toByteArray(id) - )).map(_ => app) + store.insert( + Array(scanPrefixKey(id), lastUsedScanIdKey), + Array(ScanSerializer.toBytes(app), Shorts.toByteArray(id)) + ).map(_ => app) } } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/BoxesRequest.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/BoxesRequest.scala index 64e8d43041..7f4818f040 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/BoxesRequest.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/BoxesRequest.scala @@ -3,8 +3,8 @@ package org.ergoplatform.nodeView.wallet.requests import io.circe.generic.decoding.DerivedDecoder.deriveDecoder import io.circe.{Decoder, KeyDecoder} import org.ergoplatform.ErgoBox -import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 +import sigmastate.eval.Extensions.ArrayByteOps /** * A request for boxes with given balance and assets @@ -14,7 +14,7 @@ case class BoxesRequest(targetBalance: Long, targetAssets: Map[ErgoBox.TokenId, object BoxesRequest { implicit val keyDecoder: KeyDecoder[ErgoBox.TokenId] = - KeyDecoder.instance(s => Base16.decode(s).toOption.map(Digest32 @@ _)) + KeyDecoder.instance(s => Base16.decode(s).toOption.map(_.toTokenId)) implicit val decoder: Decoder[BoxesRequest] = cursor => diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/ExternalSecret.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/ExternalSecret.scala index 466628e054..b242d8c38b 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/ExternalSecret.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/ExternalSecret.scala @@ -1,6 +1,6 @@ package org.ergoplatform.nodeView.wallet.requests -import org.ergoplatform.wallet.secrets.PrimitiveSecretKey +import org.ergoplatform.sdk.wallet.secrets.PrimitiveSecretKey /** * Externally provided secret (to be used once for a transaction to sign) diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/GenerateCommitmentsRequest.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/GenerateCommitmentsRequest.scala index 46be4b1dda..66d2d36d7e 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/GenerateCommitmentsRequest.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/GenerateCommitmentsRequest.scala @@ -1,7 +1,7 @@ package org.ergoplatform.nodeView.wallet.requests import org.ergoplatform.modifiers.mempool.UnsignedErgoTransaction -import org.ergoplatform.wallet.secrets.{DhtSecretKey, DlogSecretKey} +import org.ergoplatform.sdk.wallet.secrets.{DhtSecretKey, DlogSecretKey} /** * A request to generate commitments for unsigned transaction, useful for multi-party signing. diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/RequestsHolder.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/RequestsHolder.scala index ee6593c61b..39890f4562 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/RequestsHolder.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/RequestsHolder.scala @@ -5,7 +5,7 @@ import io.circe.{Decoder, Encoder, HCursor, Json} import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.nodeView.wallet.ErgoAddressJsonEncoder import org.ergoplatform.settings.ErgoSettings -import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoScriptPredef, Pay2SAddress} +import org.ergoplatform.{ErgoAddress, ErgoAddressEncoder, ErgoTreePredef, Pay2SAddress} case class RequestsHolder(requests: Seq[TransactionGenerationRequest], @@ -17,7 +17,7 @@ case class RequestsHolder(requests: Seq[TransactionGenerationRequest], // Add separate payment request with fee. def withFee(): Seq[TransactionGenerationRequest] = { - val address = Pay2SAddress(ErgoScriptPredef.feeProposition(minerRewardDelay)) + val address = Pay2SAddress(ErgoTreePredef.feeProposition(minerRewardDelay)) val feeRequests = feeOpt .map(PaymentRequest(address, _, assets = Seq.empty, registers = Map.empty)) .toSeq diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/TransactionSigningRequest.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/TransactionSigningRequest.scala index c5bd413819..5b3b0c4d90 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/requests/TransactionSigningRequest.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/requests/TransactionSigningRequest.scala @@ -1,8 +1,8 @@ package org.ergoplatform.nodeView.wallet.requests import org.ergoplatform.modifiers.mempool.UnsignedErgoTransaction +import org.ergoplatform.sdk.wallet.secrets.{DhtSecretKey, DlogSecretKey} import org.ergoplatform.wallet.interpreter.TransactionHintsBag -import org.ergoplatform.wallet.secrets.{DhtSecretKey, DlogSecretKey} /** * A request to sign a transaction diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala index bd73242ec3..402475a8aa 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicate.scala @@ -1,9 +1,9 @@ package org.ergoplatform.nodeView.wallet.scanning import org.ergoplatform.ErgoBox -import scorex.util.encode.Base16 import sigmastate.Values.EvaluatedValue import sigmastate.{SType, Values} +import special.collection.Extensions._ /** * Basic interface for box scanning predicate functionality @@ -120,17 +120,10 @@ case class EqualsScanningPredicate(regId: ErgoBox.RegisterId, value: EvaluatedVa */ case class ContainsAssetPredicate(assetId: ErgoBox.TokenId) extends ScanningPredicate { override def filter(box: ErgoBox): Boolean = { - box.additionalTokens.exists(_._1.sameElements(assetId)) + box.additionalTokens.exists(_._1 == assetId) } - override def equals(obj: Any): Boolean = obj match { - case other: ContainsAssetPredicate => other.assetId.sameElements(assetId) - case _ => false - } - - override def hashCode(): Int = assetId.toSeq.hashCode() - - override def toString: String = s"ContainsAssetPredicate(${Base16.encode(assetId)})" + override def toString: String = s"ContainsAssetPredicate(${assetId.toHex})" } diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala index 1a29164c9c..e8305fb331 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecs.scala @@ -1,14 +1,13 @@ package org.ergoplatform.nodeView.wallet.scanning -import io.circe.Json +import io.circe.syntax._ +import io.circe.{Decoder, Encoder, Json} import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.RegisterId -import scorex.util.encode.Base16 -import io.circe.{Decoder, Encoder} -import io.circe.syntax._ import org.ergoplatform.http.api.ApiCodecs import sigmastate.SType import sigmastate.Values.EvaluatedValue +import special.collection.Extensions._ object ScanningPredicateJsonCodecs extends ApiCodecs { @@ -19,7 +18,7 @@ object ScanningPredicateJsonCodecs extends ApiCodecs { case ep: EqualsScanningPredicate => Json.obj("predicate" -> "equals".asJson, "register" -> ep.regId.asJson, "value" -> ep.value.asJson) case cap: ContainsAssetPredicate => - Json.obj("predicate" -> "containsAsset".asJson, "assetId" -> Base16.encode(cap.assetId).asJson) + Json.obj("predicate" -> "containsAsset".asJson, "assetId" -> cap.assetId.toHex.asJson) case and: AndScanningPredicate => Json.obj("predicate" -> "and".asJson, "args" -> and.subPredicates.asJson) case or: OrScanningPredicate => diff --git a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSerializer.scala b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSerializer.scala index a8cb34ccbf..2c70437053 100644 --- a/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSerializer.scala +++ b/src/main/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSerializer.scala @@ -3,12 +3,12 @@ package org.ergoplatform.nodeView.wallet.scanning import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.RegisterId import scorex.core.serialization.ErgoSerializer -import scorex.crypto.hash.Digest32 +import scorex.util.Extensions._ import scorex.util.serialization.{Reader, Writer} import sigmastate.SType import sigmastate.Values.EvaluatedValue +import sigmastate.eval.Extensions.ArrayByteOps import sigmastate.serialization.ValueSerializer -import scorex.util.Extensions._ object ScanningPredicateSerializer extends ErgoSerializer[ScanningPredicate] { @@ -34,7 +34,7 @@ object ScanningPredicateSerializer extends ErgoSerializer[ScanningPredicate] { w.putBytes(valueBytes) case a: ContainsAssetPredicate => w.put(ContainsAssetPrefix) - w.putBytes(a.assetId) + w.putBytes(a.assetId.toArray) case AndScanningPredicate(subPredicates@_*) => w.put(AndPrefix) w.putInt(subPredicates.length) @@ -72,7 +72,7 @@ object ScanningPredicateSerializer extends ErgoSerializer[ScanningPredicate] { ContainsScanningPredicate(reg, bs) case b: Byte if b == ContainsAssetPrefix => val bs = r.getBytes(32) - ContainsAssetPredicate(Digest32 @@ bs) + ContainsAssetPredicate(bs.toTokenId) case b: Byte if b == AndPrefix => AndScanningPredicate(parseArgs(r) :_*) case b: Byte if b == OrPrefix => diff --git a/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala b/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala index 201b350ac2..7dc87a66b0 100644 --- a/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala +++ b/src/main/scala/org/ergoplatform/reemission/ReemissionRules.scala @@ -9,6 +9,7 @@ import sigmastate.SBoolean import sigmastate.Values.Value import sigmastate.eval.CompiletimeIRContext import sigmastate.lang.{CompilerSettings, SigmaCompiler, TransformingSigmaBuilder} +import special.collection.Coll import scala.util.Try @@ -18,7 +19,7 @@ import scala.util.Try */ class ReemissionRules(reemissionSettings: ReemissionSettings) extends ReemissionContracts { - override val reemissionNftIdBytes: Array[Byte] = reemissionSettings.reemissionNftIdBytes + override val reemissionNftIdBytes: Coll[Byte] = reemissionSettings.reemissionNftIdBytes override val reemissionStartHeight: Height = reemissionSettings.reemissionStartHeight /** diff --git a/src/main/scala/org/ergoplatform/settings/Constants.scala b/src/main/scala/org/ergoplatform/settings/Constants.scala index ac4f323137..6a47a36088 100644 --- a/src/main/scala/org/ergoplatform/settings/Constants.scala +++ b/src/main/scala/org/ergoplatform/settings/Constants.scala @@ -7,15 +7,18 @@ import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionSeriali import org.ergoplatform.modifiers.history.header.{Header, HeaderSerializer} import org.ergoplatform.modifiers.mempool.{ErgoTransaction, ErgoTransactionSerializer} import org.ergoplatform.nodeView.history.ErgoHistory.Difficulty -import scorex.core.serialization.ErgoSerializer import scorex.core.NodeViewModifier +import scorex.core.serialization.ErgoSerializer import scorex.crypto.authds.avltree.batch.AvlTreeParameters import sigmastate.Values import sigmastate.Values.ErgoTree object Constants { - val HashLength: Int = 32 + /** + * Length of hash function output used around the Ergo code + */ + val HashLength: Int = scorex.crypto.authds.avltree.batch.Constants.HashLength val CoinsInOneErgo: Long = 1000000000 @@ -70,6 +73,17 @@ object Constants { // Maximum extension size during bytes parsing val MaxExtensionSizeMax: Int = 1024 * 1024 + /** + * UTXO set snapshot to be taken every this number of blocks. + * The value MUST be divisible by voting epoch length (chainSettings.voting.votingLength), + * so after snapshot the state is corresponding to a moment before applying first block of a voting epoch, + * and then the first block sets current validation parameters. + * + * So for the Ergo mainnet the value should be divisible by 1024 (for testnet, 128, any number divisible by + * 1024 is divisible by 128 also. + */ + val MakeSnapshotEvery = 52224 + /** * AVL+ tree node parameters. The tree is used to authenticate UTXO set. * Keys and hashes are 256-bits long, values are boxes, so value size is dynamic. diff --git a/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala b/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala index 8a1730375c..a90cc88b3d 100644 --- a/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/ErgoSettings.scala @@ -211,6 +211,10 @@ object ErgoSettings extends ScorexLogging } else if (settings.scorexSettings.restApi.publicUrl.exists(invalidRestApiUrl)) { failWithError(s"scorex.restApi.publicUrl should not contain query, path or fragment and should not " + s"be local or loopback address : ${settings.scorexSettings.restApi.publicUrl.get}") + } else if (settings.nodeSettings.utxoSettings.p2pUtxoSnapshots <= 0) { + failWithError(s"p2pUtxoSnapshots <= 0, must be 1 at least") + } else if (settings.nodeSettings.extraIndex && settings.nodeSettings.isFullBlocksPruned) { + failWithError(s"Extra indexes could be enabled only if there is no blockchain pruning") } else { settings } diff --git a/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala b/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala index 0234eec6b4..72e47a5a05 100644 --- a/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/NodeConfigurationSettings.scala @@ -20,6 +20,24 @@ trait CheckpointingSettingsReader extends ModifierIdReader { } } +/** + * Settings related to bootstrapping with UTXO set snapshots. See ergo.node.utxo section for settings description. + */ +case class UtxoSettings(utxoBootstrap: Boolean, storingUtxoSnapshots: Int, p2pUtxoSnapshots: Int) + +/** + * Custom settings reader for `UtxoSettings` + */ +trait UtxoSettingsReader { + implicit val utxoSettingsReader: ValueReader[UtxoSettings] = { (cfg, path) => + UtxoSettings( + cfg.as[Boolean](s"$path.utxoBootstrap"), + cfg.as[Int](s"$path.storingUtxoSnapshots"), + cfg.as[Int](s"$path.p2pUtxoSnapshots") + ) + } +} + /** * Configuration file for Ergo node regime * @@ -28,6 +46,7 @@ trait CheckpointingSettingsReader extends ModifierIdReader { case class NodeConfigurationSettings(stateType: StateType, verifyTransactions: Boolean, blocksToKeep: Int, + utxoSettings: UtxoSettings, poPoWBootstrap: Boolean, minimalSuffix: Int, mining: Boolean, @@ -54,10 +73,16 @@ case class NodeConfigurationSettings(stateType: StateType, * Whether the node keeping all the full blocks of the blockchain or not. * @return true if the blockchain is pruned, false if not */ - val isFullBlocksPruned: Boolean = blocksToKeep >= 0 + val isFullBlocksPruned: Boolean = blocksToKeep >= 0 || utxoSettings.utxoBootstrap + + val areSnapshotsStored = utxoSettings.storingUtxoSnapshots > 0 } -trait NodeConfigurationReaders extends StateTypeReaders with CheckpointingSettingsReader with ModifierIdReader { +/** + * Custom config reader for ergo.node settings section + */ +trait NodeConfigurationReaders extends StateTypeReaders with CheckpointingSettingsReader + with UtxoSettingsReader with ModifierIdReader { implicit val nodeConfigurationReader: ValueReader[NodeConfigurationSettings] = { (cfg, path) => val stateTypeKey = s"$path.stateType" @@ -66,6 +91,7 @@ trait NodeConfigurationReaders extends StateTypeReaders with CheckpointingSettin stateType, cfg.as[Boolean](s"$path.verifyTransactions"), cfg.as[Int](s"$path.blocksToKeep"), + cfg.as[UtxoSettings](s"$path.utxo"), cfg.as[Boolean](s"$path.PoPoWBootstrap"), cfg.as[Int](s"$path.minimalSuffix"), cfg.as[Boolean](s"$path.mining"), diff --git a/src/main/scala/org/ergoplatform/settings/Parameters.scala b/src/main/scala/org/ergoplatform/settings/Parameters.scala index 90adba7592..f2f896335c 100644 --- a/src/main/scala/org/ergoplatform/settings/Parameters.scala +++ b/src/main/scala/org/ergoplatform/settings/Parameters.scala @@ -11,8 +11,8 @@ import scorex.util.Extensions._ import scala.util.Try import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.modifiers.history.extension.{Extension, ExtensionCandidate} -import org.ergoplatform.wallet.protocol.context.ErgoLikeParameters import Extension.SystemParametersPrefix +import org.ergoplatform.sdk.wallet.protocol.context.ErgoLikeParameters /** * System parameters which could be readjusted via collective miners decision. diff --git a/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala b/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala index 41f90aeaea..1acc476597 100644 --- a/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala +++ b/src/main/scala/org/ergoplatform/settings/ReemissionSettings.scala @@ -5,7 +5,8 @@ import org.ergoplatform.reemission.ReemissionRules import org.ergoplatform.wallet.boxes.ErgoBoxSerializer import scorex.util.ModifierId import scorex.util.encode.Base16 - +import sigmastate.utils.Extensions.ModifierIdOps +import special.collection.Coll /** * Configuration section for re-emission (EIP27) parameters * @@ -19,9 +20,9 @@ case class ReemissionSettings(checkReemissionRules: Boolean, reemissionStartHeight: Int, injectionBoxBytesEncoded: String) { - val emissionNftIdBytes: Array[Byte] = Algos.decode(emissionNftId).get - val reemissionNftIdBytes: Array[Byte] = Algos.decode(reemissionNftId).get - val reemissionTokenIdBytes: Array[Byte] = Algos.decode(reemissionTokenId).get + val emissionNftIdBytes: Coll[Byte] = emissionNftId.toColl + val reemissionNftIdBytes: Coll[Byte] = reemissionNftId.toColl + val reemissionTokenIdBytes: Coll[Byte] = reemissionTokenId.toColl lazy val InjectionBoxBytes: Array[Byte] = Base16.decode(injectionBoxBytesEncoded).get lazy val injectionBox: ErgoBox = ErgoBoxSerializer.parseBytes(InjectionBoxBytes) diff --git a/src/main/scala/scorex/core/core.scala b/src/main/scala/scorex/core/core.scala index b71bae65b2..1bc9628cfd 100644 --- a/src/main/scala/scorex/core/core.scala +++ b/src/main/scala/scorex/core/core.scala @@ -29,7 +29,7 @@ package object core { def idToBytes: util.ModifierId => Array[Byte] = scorex.util.idToBytes - def bytesToVersion(bytes: Array[Byte]): VersionTag = VersionTag @@ Base16.encode(bytes) + def bytesToVersion(bytes: Array[Byte]): VersionTag = VersionTag @@@ Base16.encode(bytes) def versionToBytes(id: VersionTag): Array[Byte] = Base16.decode(id).get diff --git a/src/main/scala/scorex/core/network/DeliveryTracker.scala b/src/main/scala/scorex/core/network/DeliveryTracker.scala index 43ecf16889..3e8d37a578 100644 --- a/src/main/scala/scorex/core/network/DeliveryTracker.scala +++ b/src/main/scala/scorex/core/network/DeliveryTracker.scala @@ -84,8 +84,10 @@ class DeliveryTracker(cacheSettings: NetworkCacheSettings, requested.foldLeft(0) { case (sum, (modTypeId, _)) if modTypeId == Header.modifierTypeId => sum - case (sum, (_, mid)) => + case (sum, (modTypeId, mid)) if NetworkObjectTypeId.isBlockSection(modTypeId) => sum + mid.size + case (sum, _) => + sum } Math.max(0, desiredSizeOfExpectingModifierQueue - nonHeaderModifiersCount) } diff --git a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala index 5e3b913708..7d237107d7 100644 --- a/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala +++ b/src/main/scala/scorex/core/network/message/BasicMessagesRepo.scala @@ -1,16 +1,18 @@ package scorex.core.network.message - import org.ergoplatform.modifiers.NetworkObjectTypeId +import org.ergoplatform.nodeView.state.SnapshotsInfo +import org.ergoplatform.nodeView.state.UtxoState.{ManifestId, SubtreeId} import scorex.core.consensus.SyncInfo import scorex.core.network._ import scorex.core.network.message.Message.MessageCode import scorex.core.serialization.ErgoSerializer import scorex.core.NodeViewModifier +import scorex.crypto.hash.Digest32 import scorex.util.Extensions._ import scorex.util.serialization.{Reader, Writer} import scorex.util.{ModifierId, ScorexLogging, bytesToId, idToBytes} - +import org.ergoplatform.sdk.wallet.Constants.ModifierIdLength import scala.collection.immutable /** @@ -248,3 +250,147 @@ object HandshakeSerializer extends MessageSpecV1[Handshake] { } } + + +/** + * The `GetSnapshotsInfo` message requests an `SnapshotsInfo` message from the receiving node + */ +object GetSnapshotsInfoSpec extends MessageSpecV1[Unit] { + private val SizeLimit = 100 + + override val messageCode: MessageCode = 76: Byte + + override val messageName: String = "GetSnapshotsInfo" + + override def serialize(obj: Unit, w: Writer): Unit = { + } + + override def parse(r: Reader): Unit = { + require(r.remaining < SizeLimit, "Too big GetSnapshotsInfo message") + } +} + +/** + * The `SnapshotsInfo` message is a reply to a `GetSnapshotsInfo` message. + * It contains information about UTXO set snapshots stored locally. + */ +object SnapshotsInfoSpec extends MessageSpecV1[SnapshotsInfo] { + private val SizeLimit = 20000 + + override val messageCode: MessageCode = 77: Byte + + override val messageName: String = "SnapshotsInfo" + + override def serialize(si: SnapshotsInfo, w: Writer): Unit = { + w.putUInt(si.availableManifests.size) + for ((height, manifest) <- si.availableManifests) { + w.putInt(height) + w.putBytes(manifest) + } + } + + override def parse(r: Reader): SnapshotsInfo = { + require(r.remaining <= SizeLimit, s"Too big SnapshotsInfo message: ${r.remaining} bytes found, $SizeLimit max expected.") + + val length = r.getUInt().toIntExact + val manifests = (0 until length).map { _ => + val height = r.getInt() + val manifest = Digest32 @@ r.getBytes(ModifierIdLength) + height -> manifest + }.toMap + new SnapshotsInfo(manifests) + } + +} + +/** + * The `GetManifest` sends manifest (BatchAVLProverManifest) identifier + */ +object GetManifestSpec extends MessageSpecV1[ManifestId] { + private val SizeLimit = 100 + + override val messageCode: MessageCode = 78: Byte + override val messageName: String = "GetManifest" + + override def serialize(id: ManifestId, w: Writer): Unit = { + w.putBytes(id) + } + + override def parse(r: Reader): ManifestId = { + require(r.remaining < SizeLimit, "Too big GetManifest message") + Digest32 @@ r.getBytes(ModifierIdLength) + } + +} + +/** + * The `Manifest` message is a reply to a `GetManifest` message. + * It contains serialized manifest, top subtree of a tree authenticating UTXO set snapshot + */ +object ManifestSpec extends MessageSpecV1[Array[Byte]] { + private val SizeLimit = 4000000 + + override val messageCode: MessageCode = 79: Byte + + override val messageName: String = "Manifest" + + override def serialize(manifestBytes: Array[Byte], w: Writer): Unit = { + w.putUInt(manifestBytes.length) + w.putBytes(manifestBytes) + } + + override def parse(r: Reader): Array[Byte] = { + require(r.remaining <= SizeLimit, s"Too big Manifest message.") + + val length = r.getUInt().toIntExact + r.getBytes(length) + } + +} + +/** + * The `GetUtxoSnapshotChunk` sends send utxo subtree (BatchAVLProverSubtree) identifier + */ +object GetUtxoSnapshotChunkSpec extends MessageSpecV1[SubtreeId] { + private val SizeLimit = 100 + + override val messageCode: MessageCode = 80: Byte + + override val messageName: String = "GetUtxoSnapshotChunk" + + override def serialize(id: SubtreeId, w: Writer): Unit = { + w.putBytes(id) + } + + override def parse(r: Reader): SubtreeId = { + require(r.remaining < SizeLimit, "Too big GetUtxoSnapshotChunk message") + Digest32 @@ r.getBytes(ModifierIdLength) + } + +} + +/** + * The `UtxoSnapshotChunk` message is a reply to a `GetUtxoSnapshotChunk` message. + */ +object UtxoSnapshotChunkSpec extends MessageSpecV1[Array[Byte]] { + private val SizeLimit = 4000000 + + override val messageCode: MessageCode = 81: Byte + + override val messageName: String = "UtxoSnapshotChunk" + + override def serialize(subtree: Array[Byte], w: Writer): Unit = { + w.putUInt(subtree.length) + w.putBytes(subtree) + } + + override def parse(r: Reader): Array[Byte] = { + require(r.remaining <= SizeLimit, s"Too big UtxoSnapshotChunk message.") + + val length = r.getUInt().toIntExact + r.getBytes(length) + } + +} + + diff --git a/src/main/scala/scorex/core/network/peer/LocalAddressPeerFeature.scala b/src/main/scala/scorex/core/network/peer/LocalAddressPeerFeature.scala index 002252285d..6ef6a2f5f5 100644 --- a/src/main/scala/scorex/core/network/peer/LocalAddressPeerFeature.scala +++ b/src/main/scala/scorex/core/network/peer/LocalAddressPeerFeature.scala @@ -1,12 +1,11 @@ package scorex.core.network.peer import java.net.{InetAddress, InetSocketAddress} - import org.ergoplatform.settings.PeerFeatureDescriptors import scorex.core.network.PeerFeature import scorex.core.network.PeerFeature.Id -import scorex.util.serialization._ import scorex.core.serialization.ErgoSerializer +import scorex.util.serialization._ import scorex.util.Extensions._ /** diff --git a/src/main/scala/scorex/core/network/peer/PeerDatabase.scala b/src/main/scala/scorex/core/network/peer/PeerDatabase.scala index 7abd3b848d..bf9f518bb5 100644 --- a/src/main/scala/scorex/core/network/peer/PeerDatabase.scala +++ b/src/main/scala/scorex/core/network/peer/PeerDatabase.scala @@ -76,7 +76,7 @@ final class PeerDatabase(settings: ErgoSettings) extends ScorexLogging { peerInfo.peerSpec.address.foreach { address => log.debug(s"Updating peer info for $address") peers += address -> peerInfo - persistentStore.insert(Array((serialize(address), PeerInfoSerializer.toBytes(peerInfo)))) + persistentStore.insert(serialize(address), PeerInfoSerializer.toBytes(peerInfo)) } } } diff --git a/src/main/scala/scorex/core/network/peer/SessionIdPeerFeature.scala b/src/main/scala/scorex/core/network/peer/SessionIdPeerFeature.scala index 7f50844811..386f999a5f 100644 --- a/src/main/scala/scorex/core/network/peer/SessionIdPeerFeature.scala +++ b/src/main/scala/scorex/core/network/peer/SessionIdPeerFeature.scala @@ -4,8 +4,8 @@ import org.ergoplatform.settings.PeerFeatureDescriptors import scorex.core.network.PeerFeature import scorex.core.network.PeerFeature.Id import scorex.core.network.message.Message -import scorex.util.serialization._ import scorex.core.serialization.ErgoSerializer +import scorex.util.serialization._ /** * This peer feature allows to more reliably detect connections to self node and connections from other networks diff --git a/src/main/scala/scorex/core/serialization/BytesSerializable.scala b/src/main/scala/scorex/core/serialization/BytesSerializable.scala index 5958c5bf98..27f8f6a0d9 100644 --- a/src/main/scala/scorex/core/serialization/BytesSerializable.scala +++ b/src/main/scala/scorex/core/serialization/BytesSerializable.scala @@ -13,5 +13,5 @@ trait BytesSerializable extends Serializable { * Serializer which can convert self to bytes */ def serializer: ErgoSerializer[M] - + } diff --git a/src/main/scala/scorex/core/utils/utils.scala b/src/main/scala/scorex/core/utils/utils.scala index a3532d56fd..93eaf37b0f 100644 --- a/src/main/scala/scorex/core/utils/utils.scala +++ b/src/main/scala/scorex/core/utils/utils.scala @@ -74,8 +74,17 @@ package object utils { result } + implicit class MapPimp[K, V](underlying: Map[K, V]) { + /** + * One liner for updating a Map with the possibility to handle case of missing Key + * @param k map key + * @param f function that is passed Option depending on Key being present or missing, returning new Value + * @return new Map with value updated under given key + */ + def adjust(k: K)(f: Option[V] => V): Map[K, V] = underlying.updated(k, f(underlying.get(k))) + } - implicit class MapPimp[K, V](underlying: mutable.Map[K, V]) { + implicit class MapPimpMutable[K, V](underlying: mutable.Map[K, V]) { /** * One liner for updating a Map with the possibility to handle case of missing Key * @param k map key @@ -97,4 +106,5 @@ package object utils { case Some(v) => underlying.put(k, v) } } + } diff --git a/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala b/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala index c201ebd15d..9159876876 100644 --- a/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala +++ b/src/test/scala/org/ergoplatform/db/KvStoreReaderSpec.scala @@ -11,10 +11,10 @@ class KvStoreReaderSpec extends AnyPropSpec with Matchers with DBSpec { val keyEnd = byteString("Z") store.getRange(keyStart, keyEnd).length shouldBe 0 - store.insert(Array(keyStart -> keyStart)).get + store.insert(keyStart, keyStart).get store.getRange(keyStart, keyEnd).length shouldBe 1 - store.insert(Array(keyEnd -> keyEnd)).get + store.insert(keyEnd, keyEnd).get store.getRange(keyStart, keyEnd).length shouldBe 2 // keys before the range diff --git a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala b/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala index cf675c916d..cd32b21c29 100644 --- a/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala +++ b/src/test/scala/org/ergoplatform/db/LDBKVStoreSpec.scala @@ -10,14 +10,14 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valueA = (byteString("A"), byteString("1")) val valueB = (byteString("B"), byteString("2")) - store.update(toInsert = Array(valueA, valueB), toRemove = Array.empty).get + store.update(Array(valueA._1, valueB._1), Array(valueA._2, valueB._2), toRemove = Array.empty).get store.get(valueA._1).toBs shouldBe Some(valueA._2).toBs store.get(valueB._1).toBs shouldBe Some(valueB._2).toBs store.getAll.toSeq.toBs shouldBe Seq(valueA, valueB).toBs - store.update(toInsert = Array.empty, toRemove = Array(valueA._1)).get + store.update(Array.empty, Array.empty, toRemove = Array(valueA._1)).get store.get(valueA._1) shouldBe None } } @@ -28,11 +28,11 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valA = byteString("1") val valB = byteString("2") - store.insert(Array(key -> valA)).get + store.insert(key, valA).get store.get(key).toBs shouldBe Some(valA).toBs - store.insert(Array(key -> valB)).get + store.insert(key, valB).get store.get(key).toBs shouldBe Some(valB).toBs @@ -49,7 +49,8 @@ class LDBKVStoreSpec extends AnyPropSpec with Matchers with DBSpec { val valueE = (byteString("E"), byteString("3")) val valueF = (byteString("F"), byteString("4")) - store.insert(Array(valueA, valueB, valueC, valueD, valueE, valueF)).get + val values = Array(valueA, valueB, valueC, valueD, valueE, valueF) + store.insert(values.map(_._1), values.map(_._2)).get store.lastKeyInRange(valueA._1, valueC._1).get.toSeq shouldBe valueC._1.toSeq store.lastKeyInRange(valueD._1, valueF._1).get.toSeq shouldBe valueF._1.toSeq diff --git a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala index 4b5e50c6b9..ed0e328ed4 100644 --- a/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/BlocksApiRouteSpec.scala @@ -33,7 +33,7 @@ class BlocksApiRouteSpec extends AnyFlatSpec } it should "post block correctly" in { - val (st, bh) = createUtxoState(parameters) + val (st, bh) = createUtxoState(settings) val block: ErgoFullBlock = validFullBlock(parentOpt = None, st, bh) val blockJson: UniversalEntity = HttpEntity(block.asJson.toString).withContentType(ContentTypes.`application/json`) Post(prefix, blockJson) ~> route ~> check { diff --git a/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala index 0ea07612d7..dee8e45a3f 100644 --- a/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/MiningApiRouteSpec.scala @@ -11,7 +11,7 @@ import org.ergoplatform.mining.AutolykosSolution import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.utils.Stubs import org.ergoplatform.utils.generators.ErgoGenerators -import org.ergoplatform.{Pay2SAddress, ErgoScriptPredef} +import org.ergoplatform.{ErgoTreePredef, Pay2SAddress} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -48,7 +48,7 @@ class MiningApiRouteSpec it should "display miner pk" in { Get(prefix + "/rewardAddress") ~> route ~> check { status shouldBe StatusCodes.OK - val script = ErgoScriptPredef.rewardOutputScript(settings.chainSettings.monetary.minerRewardDelay, pk) + val script = ErgoTreePredef.rewardOutputScript(settings.chainSettings.monetary.minerRewardDelay, pk) val addressStr = Pay2SAddress(script)(settings.addressEncoder).toString() responseAs[Json].hcursor.downField("rewardAddress").as[String] shouldEqual Right(addressStr) } diff --git a/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala b/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala index ab267a4c33..c13bd6cb0c 100644 --- a/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala +++ b/src/test/scala/org/ergoplatform/http/routes/TransactionApiRouteSpec.scala @@ -17,12 +17,12 @@ import org.ergoplatform.{DataInput, ErgoBox, ErgoBoxCandidate, Input} import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers import scorex.core.settings.RESTApiSettings -import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 import sigmastate.SType import sigmastate.Values.{ByteArrayConstant, EvaluatedValue} import sigmastate.eval.Extensions._ import sigmastate.eval._ +import special.collection.Extensions._ import java.net.InetSocketAddress import scala.concurrent.duration._ @@ -44,7 +44,7 @@ class TransactionApiRouteSpec extends AnyFlatSpec val dataInput = DataInput(input.boxId) val absentModifierId = "0000000000000000000000000000000000000000000000000000000000000000" - val tokens = List[(TokenId, Long)](Digest32 @@@ inputBox.id -> 10) + val tokens = List[(TokenId, Long)](inputBox.id.toTokenId -> 10) val registers = Map( ErgoBox.R4 -> ByteArrayConstant("name".getBytes("UTF-8")), @@ -229,7 +229,7 @@ class TransactionApiRouteSpec extends AnyFlatSpec } it should "return unconfirmed output by tokenId from mempool" in { - val searchedToken = Base16.encode(tokens.head._1) + val searchedToken = tokens.head._1.toHex Get(prefix + s"/unconfirmed/outputs/byTokenId/$searchedToken") ~> chainedRoute ~> check { status shouldBe StatusCodes.OK val actualOutputIds = responseAs[List[Json]].map(_.hcursor.downField("boxId").as[String].right.get) diff --git a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala index 09974da381..5096dec09d 100644 --- a/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala +++ b/src/test/scala/org/ergoplatform/local/MempoolAuditorSpec.scala @@ -41,10 +41,10 @@ class MempoolAuditorSpec extends AnyFlatSpec with NodeViewTestOps with ErgoTestH val testProbe = new TestProbe(actorSystem) actorSystem.eventStream.subscribe(testProbe.ref, newTx) - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settingsToTest) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = - WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis) { mod => + WrappedUtxoState(us, bh, settingsToTest, parameters).applyModifier(genesis) { mod => nodeViewHolderRef ! mod } .get @@ -95,7 +95,7 @@ class MempoolAuditorSpec extends AnyFlatSpec with NodeViewTestOps with ErgoTestH it should "rebroadcast transactions correctly" in { - val (us0, bh0) = createUtxoState(parameters) + val (us0, bh0) = createUtxoState(settingsToTest) val (txs0, bh1) = validTransactionsFromBoxHolder(bh0) val b1 = validFullBlock(None, us0, txs0) diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala index a8c6db6a3f..66aa6a8c05 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorPropSpec.scala @@ -1,6 +1,6 @@ package org.ergoplatform.mining -import org.ergoplatform.ErgoScriptPredef +import org.ergoplatform.ErgoTreePredef import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.state.ErgoStateContext import org.ergoplatform.settings.MonetarySettings @@ -16,7 +16,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { val delta: Int = settings.chainSettings.monetary.minerRewardDelay private def expectedRewardOutputScriptBytes(pk: ProveDlog): Array[Byte] = - ErgoScriptPredef.rewardOutputScript(delta, pk).bytes + ErgoTreePredef.rewardOutputScript(delta, pk).bytes implicit private val verifier: ErgoInterpreter = ErgoInterpreter(parameters) @@ -51,7 +51,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { } property("collect reward from emission box only") { - val us = createUtxoState(parameters)._1 + val us = createUtxoState(settings)._1 us.emissionBoxOpt should not be None val expectedReward = emission.minersRewardAtHeight(us.stateContext.currentHeight) @@ -195,7 +195,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { val delta = 1 val inputsNum = 2 - val feeProposition = ErgoScriptPredef.feeProposition(delta) + val feeProposition = ErgoTreePredef.feeProposition(delta) val bh = boxesHolderGen.sample.get var us = createUtxoState(bh, parameters) @@ -240,7 +240,7 @@ class CandidateGeneratorPropSpec extends ErgoPropertyTest { } property("collect reward from both emission box and fees") { - val (us, _) = createUtxoState(parameters) + val (us, _) = createUtxoState(settings) us.emissionBoxOpt should not be None val expectedReward = emission.minersRewardAtHeight(us.stateContext.currentHeight) diff --git a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala index 0bc8866630..0fddaf5750 100644 --- a/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/CandidateGeneratorSpec.scala @@ -9,16 +9,16 @@ import org.ergoplatform.mining.CandidateGenerator.{Candidate, GenerateCandidate} import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.state.StateType import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.utils.ErgoTestHelpers -import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, Input} +import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input} import org.scalatest.concurrent.Eventually import org.scalatest.flatspec.AnyFlatSpec -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied import sigmastate.basics.DLogProtocol import sigmastate.basics.DLogProtocol.DLogProverInput @@ -201,7 +201,7 @@ class CandidateGeneratorSpec extends AnyFlatSpec with ErgoTestHelpers with Event DLogProverInput(BigIntegers.fromUnsignedByteArray("test".getBytes())).publicImage val newlyMinedBlock = readers.h.bestFullBlockOpt.get val rewardBox: ErgoBox = newlyMinedBlock.transactions.last.outputs.last - rewardBox.propositionBytes shouldBe ErgoScriptPredef + rewardBox.propositionBytes shouldBe ErgoTreePredef .rewardOutputScript(emission.settings.minerRewardDelay, defaultMinerPk) .bytes val input = Input(rewardBox.id, emptyProverResult) diff --git a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala index d17b3da442..16e83f5d00 100644 --- a/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala +++ b/src/test/scala/org/ergoplatform/mining/ErgoMinerSpec.scala @@ -10,6 +10,8 @@ import org.ergoplatform.mining.ErgoMiner.StartMining import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction, UnsignedErgoTransaction} +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied +import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction import org.ergoplatform.nodeView.ErgoReadersHolder.{GetReaders, Readers} import org.ergoplatform.nodeView.history.ErgoHistoryReader import org.ergoplatform.nodeView.state._ @@ -18,12 +20,10 @@ import org.ergoplatform.nodeView.{ErgoNodeViewRef, ErgoReadersHolderRef} import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.utils.ErgoTestHelpers import org.ergoplatform.utils.generators.ValidBlocksGenerators -import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, Input} +import org.ergoplatform.wallet.interpreter.ErgoInterpreter +import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, ErgoTreePredef, Input} import org.scalatest.concurrent.Eventually import org.scalatest.flatspec.AnyFlatSpec -import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.LocallyGeneratedTransaction -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages.FullBlockApplied -import org.ergoplatform.wallet.interpreter.ErgoInterpreter import sigmastate.SigmaAnd import sigmastate.Values.{ErgoTree, SigmaPropConstant} import sigmastate.basics.DLogProtocol @@ -85,7 +85,7 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen testProbe.expectMsgClass(newBlockDelay, newBlockSignal) val boxToSpend: ErgoBox = r.h.bestFullBlockOpt.get.transactions.last.outputs.last - boxToSpend.propositionBytes shouldBe ErgoScriptPredef.rewardOutputScript(emission.settings.minerRewardDelay, defaultMinerPk).bytes + boxToSpend.propositionBytes shouldBe ErgoTreePredef.rewardOutputScript(emission.settings.minerRewardDelay, defaultMinerPk).bytes val input = Input(boxToSpend.id, emptyProverResult) @@ -241,7 +241,7 @@ class ErgoMinerSpec extends AnyFlatSpec with ErgoTestHelpers with ValidBlocksGen val prop2: DLogProtocol.ProveDlog = DLogProverInput(BigIntegers.fromUnsignedByteArray("test2".getBytes())).publicImage val boxToDoubleSpend: ErgoBox = r.h.bestFullBlockOpt.get.transactions.last.outputs.last - boxToDoubleSpend.propositionBytes shouldBe ErgoScriptPredef.rewardOutputScript(emission.settings.minerRewardDelay, defaultMinerPk).bytes + boxToDoubleSpend.propositionBytes shouldBe ErgoTreePredef.rewardOutputScript(emission.settings.minerRewardDelay, defaultMinerPk).bytes val input = Input(boxToDoubleSpend.id, emptyProverResult) diff --git a/src/test/scala/org/ergoplatform/mining/difficulty/DifficultySerializerSerializerSpecification.scala b/src/test/scala/org/ergoplatform/mining/difficulty/DifficultySerializerSpecification.scala similarity index 100% rename from src/test/scala/org/ergoplatform/mining/difficulty/DifficultySerializerSerializerSpecification.scala rename to src/test/scala/org/ergoplatform/mining/difficulty/DifficultySerializerSpecification.scala diff --git a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala index b3f9ead789..ee110f90fc 100644 --- a/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala +++ b/src/test/scala/org/ergoplatform/modifiers/mempool/ErgoTransactionSpec.scala @@ -4,28 +4,29 @@ import io.circe.syntax._ import org.ergoplatform.ErgoBox._ import org.ergoplatform.nodeView.ErgoContext import org.ergoplatform.nodeView.state.{ErgoStateContext, VotingData} +import org.ergoplatform.sdk.wallet.protocol.context.TransactionContext import org.ergoplatform.settings.Parameters.MaxBlockCostIncrease import org.ergoplatform.settings.ValidationRules.{bsBlockTransactionsCost, txAssetsInOneBox} import org.ergoplatform.settings._ import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestConstants} import org.ergoplatform.wallet.boxes.ErgoBoxAssetExtractor import org.ergoplatform.wallet.interpreter.{ErgoInterpreter, TransactionHintsBag} -import org.ergoplatform.wallet.protocol.context.{InputContext, TransactionContext} +import org.ergoplatform.wallet.protocol.context.InputContext import org.ergoplatform.{ErgoBox, ErgoBoxCandidate, Input} import org.scalacheck.Gen import scalan.util.BenchmarkUtil import scorex.crypto.authds.ADKey -import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.db.ByteArrayWrapper -import scorex.util.{ModifierId, bytesToId} +import scorex.crypto.hash.Blake2b256 import scorex.util.encode.Base16 +import scorex.util.{ModifierId, bytesToId} import sigmastate.AND import sigmastate.Values.{ByteArrayConstant, ByteConstant, IntConstant, LongArrayConstant, SigmaPropConstant, TrueLeaf} +import sigmastate.basics.CryptoConstants import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.eval._ -import sigmastate.interpreter.{ContextExtension, CryptoConstants, ProverResult} import sigmastate.helpers.TestingHelpers._ - +import sigmastate.interpreter.{ContextExtension, ProverResult} +import sigmastate.eval.Extensions._ import scala.util.{Random, Try} class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { @@ -40,8 +41,8 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { if (modified) { (seq :+ ebc) -> true } else { - if (ebc.additionalTokens.nonEmpty && ebc.additionalTokens.exists(t => !java.util.Arrays.equals(t._1, from.head.id))) { - (seq :+ modifyAsset(ebc, deltaFn, Digest32 @@@ from.head.id)) -> true + if (ebc.additionalTokens.nonEmpty && ebc.additionalTokens.exists(t => !java.util.Arrays.equals(t._1.toArray, from.head.id))) { + (seq :+ modifyAsset(ebc, deltaFn, from.head.id.toTokenId)) -> true } else { (seq :+ ebc) -> false } @@ -62,10 +63,10 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { private def modifyAsset(boxCandidate: ErgoBoxCandidate, deltaFn: Long => Long, idToskip: TokenId): ErgoBoxCandidate = { - val assetId = boxCandidate.additionalTokens.find(t => !java.util.Arrays.equals(t._1, idToskip)).get._1 + val assetId = boxCandidate.additionalTokens.find(t => t._1 != idToskip).get._1 val tokens = boxCandidate.additionalTokens.map { case (id, amount) => - if (java.util.Arrays.equals(id, assetId)) assetId -> deltaFn(amount) else assetId -> amount + if (id == assetId) assetId -> deltaFn(amount) else assetId -> amount } new ErgoBoxCandidate( @@ -205,7 +206,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { property("impossible to overflow an asset value") { val gen = validErgoTransactionGenTemplate(minAssets = 1, maxAssets = 1, maxInputs = 16, propositionGen = trueLeafGen) forAll(gen) { case (from, tx) => - val tokenOpt = tx.outputCandidates.flatMap(_.additionalTokens.toArray).map(t => ByteArrayWrapper.apply(t._1) -> t._2) + val tokenOpt = tx.outputCandidates.flatMap(_.additionalTokens.toArray) .groupBy(_._1).find(_._2.size >= 2) whenever(tokenOpt.nonEmpty) { @@ -215,7 +216,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { var modified = false val updCandidates = tx.outputCandidates.map { c => val updTokens = c.additionalTokens.map { case (id, amount) => - if (!modified && ByteArrayWrapper(id) == tokenId) { + if (!modified && id == tokenId) { modified = true id -> ((Long.MaxValue - tokenAmount) + amount + 1) } else { @@ -251,7 +252,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { // already existing token from one of the inputs val existingToken = from.flatMap(_.additionalTokens.toArray).toSet.head // completely new token - val randomToken = (Digest32 @@ scorex.util.Random.randomBytes(), Random.nextInt(100000000).toLong) + val randomToken = (scorex.util.Random.randomBytes().toTokenId, Random.nextInt(100000000).toLong) val in0 = from.last // new token added to the last input @@ -296,7 +297,7 @@ class ErgoTransactionSpec extends ErgoPropertyTest with ErgoTestConstants { property("spam simulation (transaction validation cost with too many tokens exceeds block limit)") { val bxsQty = 392 // with greater value test is failing with collection size exception val (inputs, tx) = validErgoTransactionGenTemplate(1, 1,16).sample.get // it takes too long to test with `forAll` - val tokens = (0 until 255).map(_ => (Digest32 @@ scorex.util.Random.randomBytes(), Random.nextLong)) + val tokens = (0 until 255).map(_ => (scorex.util.Random.randomBytes().toTokenId, Random.nextLong)) val (in, out) = { val in0 = inputs.head val out0 = tx.outputs.head diff --git a/src/test/scala/org/ergoplatform/network/ElementPartitionerSpecification.scala b/src/test/scala/org/ergoplatform/network/ElementPartitionerSpecification.scala index 89cfab2531..3104f18265 100644 --- a/src/test/scala/org/ergoplatform/network/ElementPartitionerSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/ElementPartitionerSpecification.scala @@ -5,17 +5,33 @@ import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec import org.scalatestplus.scalacheck.ScalaCheckPropertyChecks + class ElementPartitionerSpecification extends AnyPropSpec with ScalaCheckPropertyChecks with Matchers { + def distribute[B, T, I](buckets: Iterable[B], + maxElements: Int, + minElementsPerBucket: Int, + maxElementsPerBucket: Int + )(fetchMax: Int => Map[T, Seq[I]]): Map[(B, T), Seq[I]] = { + val peersCount = buckets.size + val maxElementsToFetch = Math.min(maxElements, peersCount * maxElementsPerBucket) + val fetched = if (maxElementsToFetch <= 0) { + Map.empty[T, Seq[I]] + } else { + fetchMax(maxElementsToFetch) + } + ElementPartitioner.distribute(buckets, minElementsPerBucket, fetched) + } + property("elements should be evenly distributed in buckets limited by bucket size") { forAll(Gen.nonEmptyListOf(Gen.alphaNumChar), Gen.nonEmptyListOf(Gen.alphaNumChar)) { case (buckets, elements) => val (elemsType_1, elemsType_2) = elements.splitAt(elements.size / 2) val elemsByBucket = - ElementPartitioner.distribute(buckets, Integer.MAX_VALUE, 1, 5) { count => + distribute(buckets, Integer.MAX_VALUE, 1, 5) { count => count shouldBe buckets.size * 5 Map("A" -> elemsType_1.take(count), "B" -> elemsType_2.take(count)) } @@ -30,7 +46,7 @@ class ElementPartitionerSpecification forAll(Gen.nonEmptyListOf(Gen.alphaNumChar), Gen.nonEmptyListOf(Gen.alphaNumChar)) { case (buckets, elements) => val (elemsType_1, elemsType_2) = elements.splitAt(elements.size / 2) - val elemsByBucket = ElementPartitioner.distribute(buckets, 5, 1, 100) { count => + val elemsByBucket = distribute(buckets, 5, 1, 100) { count => count shouldBe 5 Map("A" -> elemsType_1.take(count), "B" -> elemsType_2.take(count)) } @@ -44,7 +60,7 @@ class ElementPartitionerSpecification forAll(Gen.listOf(Gen.alphaNumChar), Gen.listOf(Gen.alphaNumChar)) { case (buckets, elements) => val elemsByBucket = - ElementPartitioner.distribute(buckets, Integer.MAX_VALUE, 5, 10) { count => + distribute(buckets, Integer.MAX_VALUE, 5, 10) { count => assert(count <= 10 * buckets.size) Map("A" -> elements.take(count)) } @@ -58,34 +74,29 @@ class ElementPartitionerSpecification } property("empty buckets or elements cannot be partitioned") { - ElementPartitioner - .distribute(List.empty, Integer.MAX_VALUE, 1, 5)(_ => Map("A" -> List(1))) + distribute(List.empty, Integer.MAX_VALUE, 1, 5)(_ => Map("A" -> List(1))) .size shouldBe 0 - ElementPartitioner - .distribute(List(1), Integer.MAX_VALUE, 1, 5)(_ => Map.empty[String, Seq[Int]]) + distribute(List(1), Integer.MAX_VALUE, 1, 5)(_ => Map.empty[String, Seq[Int]]) .size shouldBe 0 - ElementPartitioner - .distribute(List.empty, Integer.MAX_VALUE, 1, 5)(_ => Map.empty[String, Seq[Int]]) + distribute(List.empty, Integer.MAX_VALUE, 1, 5)(_ => Map.empty[String, Seq[Int]]) .size shouldBe 0 } property("0 or negative count of elements to fetch cannot be partitioned") { - ElementPartitioner - .distribute(List.empty, -1, 1, 5)(_ => Map("A" -> List(1))) + distribute(List.empty, -1, 1, 5)(_ => Map("A" -> List(1))) .size shouldBe 0 - ElementPartitioner - .distribute(List.empty, 5, 1, 0)(_ => Map("A" -> List(1))) + distribute(List.empty, 5, 1, 0)(_ => Map("A" -> List(1))) .size shouldBe 0 } property("less or equal elements than buckets should return one element per bucket") { - ElementPartitioner.distribute(List(1, 2, 3), Integer.MAX_VALUE, 1, 5) { _ => + distribute(List(1, 2, 3), Integer.MAX_VALUE, 1, 5) { _ => Map("A" -> List(1)) } shouldBe Map((1, "A") -> List(1)) } property("elements should be distributed into bucket-types") { - ElementPartitioner.distribute(List(1, 2), Integer.MAX_VALUE, 1, 1) { _ => + distribute(List(1, 2), Integer.MAX_VALUE, 1, 1) { _ => Map("A" -> List(1, 2), "B" -> List(1, 2), "C" -> List(1, 2)) } shouldBe Map( (2, "B") -> List(2), @@ -101,7 +112,7 @@ class ElementPartitionerSpecification "minElementsPerBucket constraint should not be used if there is less elements available" ) { val elems = - ElementPartitioner.distribute(List(1), Integer.MAX_VALUE, 100, 1) { _ => + distribute(List(1), Integer.MAX_VALUE, 100, 1) { _ => Map("A" -> List(1)) } elems.size shouldBe 1 diff --git a/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala b/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala index bb80727f8e..1196482d4c 100644 --- a/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/HeaderSerializationSpecification.scala @@ -1,7 +1,5 @@ package org.ergoplatform.network -import java.nio.ByteBuffer - import org.ergoplatform.mining.difficulty.DifficultySerializer import org.ergoplatform.mining.{AutolykosSolution, groupElemFromBytes} import org.ergoplatform.modifiers.history.header.Header @@ -11,7 +9,9 @@ import scorex.crypto.authds.ADDigest import scorex.crypto.hash.{Blake2b256, Digest32} import scorex.util.ModifierId import scorex.util.encode.Base16 -import sigmastate.interpreter.CryptoConstants.EcPointType +import sigmastate.basics.CryptoConstants.EcPointType + +import java.nio.ByteBuffer class HeaderSerializationSpecification extends ErgoPropertyTest with DecodingUtils { diff --git a/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala b/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala index 7ff9ed7883..2309316ad9 100644 --- a/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala +++ b/src/test/scala/org/ergoplatform/network/PeerFilteringRuleSpecification.scala @@ -44,4 +44,18 @@ class PeerFilteringRuleSpecification extends ErgoPropertyTest { Seq(outPeer0, outPeer1, outPeer2) } + property("utxo set snapshot filter") { + val peer0 = peerWithVersion(Version(4, 0, 17)) + val peer1 = peerWithVersion(Version(4, 0, 18)) + val peer2 = peerWithVersion(Version(4, 0, 16)) + val peer3 = peerWithVersion(Version(4, 0, 19)) + val peer4 = peerWithVersion(Version(5, 0, 0)) + val peer5 = peerWithVersion(Version(5, 0, 5)) + val peer6 = peerWithVersion(Version(5, 0, 15)) + val peer7 = peerWithVersion(Version(5, 0, 25)) + + UtxoSetNetworkingFilter.filter(Seq(peer0, peer1, peer2, peer3, peer4, peer5, peer6, peer7)) shouldBe + Seq(peer6, peer7) + } + } diff --git a/src/test/scala/scorex/testkit/properties/NodeViewSynchronizerTests.scala b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala similarity index 66% rename from src/test/scala/scorex/testkit/properties/NodeViewSynchronizerTests.scala rename to src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala index a2806ed420..c867070220 100644 --- a/src/test/scala/scorex/testkit/properties/NodeViewSynchronizerTests.scala +++ b/src/test/scala/org/ergoplatform/nodeView/NodeViewSynchronizerTests.scala @@ -1,46 +1,55 @@ -package scorex.testkit.properties +package org.ergoplatform.nodeView -import akka.actor._ +import akka.actor.{ActorRef, ActorSystem} import akka.testkit.TestProbe import org.ergoplatform.modifiers.BlockSection import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} +import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ +import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{GetNodeViewChanges, ModifiersFromRemote} import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoSyncInfo, ErgoSyncInfoMessageSpec} import org.ergoplatform.nodeView.mempool.ErgoMemPool +import org.ergoplatform.nodeView.state.UtxoState.ManifestId +import org.ergoplatform.nodeView.state._ +import org.ergoplatform.settings.Algos +import org.ergoplatform.wallet.utils.TestFileUtils import org.scalacheck.Gen import org.scalatest.matchers.should.Matchers import org.scalatest.propspec.AnyPropSpec -import org.ergoplatform.nodeView.ErgoNodeViewHolder.ReceivableMessages.{GetNodeViewChanges, ModifiersFromRemote} import scorex.core.consensus.SyncInfo +import scorex.core.network.ConnectedPeer import scorex.core.network.NetworkController.ReceivableMessages.{PenalizePeer, SendToNetwork} -import org.ergoplatform.network.ErgoNodeViewSynchronizer.ReceivableMessages._ -import org.ergoplatform.nodeView.state.ErgoState -import scorex.core.network._ import scorex.core.network.message._ import scorex.core.network.peer.PenaltyType -import scorex.core.serialization.{BytesSerializable, ErgoSerializer} +import scorex.core.serialization.{BytesSerializable, ErgoSerializer, ManifestSerializer} +import scorex.crypto.hash.Digest32 import scorex.testkit.generators.{SyntacticallyTargetedModifierProducer, TotallyValidModifierProducer} import scorex.testkit.utils.AkkaFixture import scorex.util.ScorexLogging -import scorex.util.serialization._ +import scorex.util.serialization.{Reader, Writer} import scala.concurrent.Await -import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ import scala.language.postfixOps +import scala.util.Random @SuppressWarnings(Array("org.wartremover.warts.IsInstanceOf")) trait NodeViewSynchronizerTests[ST <: ErgoState[ST]] extends AnyPropSpec with Matchers with ScorexLogging with SyntacticallyTargetedModifierProducer - with TotallyValidModifierProducer[ST] { + with TotallyValidModifierProducer[ST] + with TestFileUtils { + + implicit val ec: scala.concurrent.ExecutionContext = scala.concurrent.ExecutionContext.global val historyGen: Gen[ErgoHistory] val memPool: ErgoMemPool + val stateGen: Gen[ST] + def nodeViewSynchronizer(implicit system: ActorSystem): - (ActorRef, ErgoSyncInfo, BlockSection, ErgoTransaction, ConnectedPeer, TestProbe, TestProbe, TestProbe, TestProbe, ErgoSerializer[BlockSection]) + (ActorRef, ErgoSyncInfo, BlockSection, ErgoTransaction, ConnectedPeer, TestProbe, TestProbe, TestProbe, TestProbe, ErgoSerializer[BlockSection]) class SynchronizerFixture extends AkkaFixture { @SuppressWarnings(Array("org.wartremover.warts.PublicInference")) @@ -111,7 +120,7 @@ trait NodeViewSynchronizerTests[ST <: ErgoState[ST]] extends AnyPropSpec withFixture { ctx => import ctx._ - val dummySyncInfoMessageSpec = new SyncInfoMessageSpec[SyncInfo](serializer = new ErgoSerializer[SyncInfo]{ + val dummySyncInfoMessageSpec = new SyncInfoMessageSpec[SyncInfo](serializer = new ErgoSerializer[SyncInfo] { override def parse(r: Reader): SyncInfo = { throw new Exception() } @@ -225,4 +234,102 @@ trait NodeViewSynchronizerTests[ST <: ErgoState[ST]] extends AnyPropSpec } + property("NodeViewSynchronizer: GetSnapshotInfo") { + withFixture { ctx => + import ctx._ + + val s = stateGen.sample.get + + if (s.isInstanceOf[UtxoStateReader]) { + // To initialize utxoStateReaderOpt in ErgoNodeView Synchronizer + node ! ChangedState(s) + + // First, store snapshots info in DB + val m = (0 until 100).map { _ => + Random.nextInt(1000000) -> (Digest32 @@ Algos.decode(mod.id).get) + }.toMap + val si = new SnapshotsInfo(m) + val db = SnapshotsDb.create(createTempDir.getPath) + db.writeSnapshotsInfo(si) + + // Then send message to request it + node ! Message[Unit](GetSnapshotsInfoSpec, Left(Array.empty[Byte]), Option(peer)) + ncProbe.fishForMessage(5 seconds) { + case stn: SendToNetwork if stn.message.spec.isInstanceOf[SnapshotsInfoSpec.type] => true + case _: Any => false + } + } else { + log.info("Snapshots not supported by digest-state") + } + } + } + + property("NodeViewSynchronizer: GetManifest") { + withFixture { ctx => + import ctx._ + + val s = stateGen.sample.get + + s match { + case usr: UtxoState => { + // To initialize utxoStateReaderOpt in ErgoNodeView Synchronizer + node ! ChangedState(s) + + // Generate some snapshot + val height = 1 + usr.applyModifier(mod, Some(height))(_ => ()) + + val manifestId = usr.dumpSnapshot(height, usr.rootDigest.dropRight(1)).get + + // Then send message to request it + node ! Message[ManifestId](GetManifestSpec, Left(manifestId), Option(peer)) + ncProbe.fishForMessage(5 seconds) { + case stn: SendToNetwork if stn.message.spec.isInstanceOf[ManifestSpec.type] => true + case _: Any => false + } + } + case _ => + log.info("Snapshots not supported by digest-state") + } + } + } + + property("NodeViewSynchronizer: GetSnapshotChunk") { + withFixture { ctx => + import ctx._ + + val s = stateGen.sample.get + + s match { + case usr: UtxoState => { + // To initialize utxoStateReaderOpt in ErgoNodeView Synchronizer + node ! ChangedState(s) + + // Generate some snapshot + + val height = 1 + + usr.applyModifier(mod, Some(height))(_ => ()) + + val manifestDepth = 2.toByte + val serializer = new ManifestSerializer(manifestDepth) + usr.dumpSnapshot(height, usr.rootDigest.dropRight(1), manifestDepth) + val manifestId = usr.snapshotsDb.readSnapshotsInfo.availableManifests.apply(height) + val manifestBytes = usr.snapshotsDb.readManifestBytes(manifestId).get + val manifest = serializer.parseBytes(manifestBytes) + val subtreeIds = manifest.subtreesIds + + // Then send message to request it + node ! Message[ManifestId](GetUtxoSnapshotChunkSpec, Left(subtreeIds.last), Option(peer)) + ncProbe.fishForMessage(5 seconds) { + case stn: SendToNetwork if stn.message.spec.isInstanceOf[UtxoSnapshotChunkSpec.type] => true + case _: Any => false + } + } + case _ => + log.info("Snapshots not supported by digest-state") + } + } + } + } diff --git a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala index a059e416e9..40feb58cef 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/BlockSectionValidationSpecification.scala @@ -72,10 +72,10 @@ class BlockSectionValidationSpecification extends HistoryTestHelpers { // should not be able to apply when blocks at this height are already pruned history.applicableTry(section) shouldBe 'success - history.pruningProcessor.minimalFullBlockHeightVar = history.bestHeaderOpt.get.height + 1 - history.pruningProcessor.isHeadersChainSyncedVar = true + history.writeMinimalFullBlockHeight(history.bestHeaderOpt.get.height + 1) + history.isHeadersChainSyncedVar = true history.applicableTry(section) shouldBe 'failure - history.pruningProcessor.minimalFullBlockHeightVar = ErgoHistory.GenesisHeight + history.writeMinimalFullBlockHeight(ErgoHistory.GenesisHeight) // should not be able to apply if corresponding header is marked as invalid history.applicableTry(section) shouldBe 'success diff --git a/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala new file mode 100644 index 0000000000..3903251054 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/history/UtxoSetSnapshotProcessorSpecification.scala @@ -0,0 +1,94 @@ +package org.ergoplatform.nodeView.history + +import org.ergoplatform.nodeView.history.storage.HistoryStorage +import org.ergoplatform.nodeView.history.storage.modifierprocessors.UtxoSetSnapshotProcessor +import org.ergoplatform.nodeView.state.{StateType, UtxoState} +import org.ergoplatform.settings.{Algos, ErgoSettings} +import org.ergoplatform.utils.HistoryTestHelpers +import scorex.core.VersionTag +import scorex.core.serialization.{ManifestSerializer, SubtreeSerializer} +import scorex.db.LDBVersionedStore +import scorex.util.ModifierId + +import scala.util.Random + +class UtxoSetSnapshotProcessorSpecification extends HistoryTestHelpers { + + private val s = settings + + val epochLength = 20 + + val utxoSetSnapshotProcessor = new UtxoSetSnapshotProcessor { + var minimalFullBlockHeightVar = ErgoHistory.GenesisHeight + override protected val settings: ErgoSettings = s.copy(chainSettings = + s.chainSettings.copy(voting = s.chainSettings.voting.copy(votingLength = epochLength))) + override protected val historyStorage: HistoryStorage = HistoryStorage(settings) + override def readMinimalFullBlockHeight() = minimalFullBlockHeightVar + override def writeMinimalFullBlockHeight(height: Int): Unit = { + minimalFullBlockHeightVar = height + } + } + + var history = generateHistory( + verifyTransactions = true, + StateType.Utxo, + PoPoWBootstrap = false, + blocksToKeep = -1, + epochLength = epochLength, + useLastEpochs = 2, + initialDiffOpt = None) + + val chain = genHeaderChain(epochLength + 1, history, diffBitsOpt = None, useRealTs = false) + history = applyHeaderChain(history, chain) + + property("registerManifestToDownload + getUtxoSetSnapshotDownloadPlan + getChunkIdsToDownload") { + val bh = boxesHolderGenOfSize(32 * 1024).sample.get + val us = createUtxoState(bh, parameters) + + val snapshotHeight = epochLength - 1 + val serializer = ManifestSerializer.defaultSerializer + + us.dumpSnapshot(snapshotHeight, us.rootDigest.dropRight(1)) + val manifestId = us.snapshotsDb.readSnapshotsInfo.availableManifests.apply(snapshotHeight) + val manifestBytes = us.snapshotsDb.readManifestBytes(manifestId).get + val manifest = serializer.parseBytes(manifestBytes) + val subtreeIds = manifest.subtreesIds + val subtreeIdsEncoded = subtreeIds.map(id => ModifierId @@ Algos.encode(id)) + + subtreeIds.foreach {sid => + val subtreeBytes = us.snapshotsDb.readSubtreeBytes(sid).get + val subtree = SubtreeSerializer.parseBytes(subtreeBytes) + subtree.verify(sid) shouldBe true + } + + val blockId = ModifierId @@ Algos.encode(Array.fill(32)(Random.nextInt(100).toByte)) + utxoSetSnapshotProcessor.registerManifestToDownload(manifest, snapshotHeight, Seq.empty) + val dp = utxoSetSnapshotProcessor.utxoSetSnapshotDownloadPlan().get + dp.snapshotHeight shouldBe snapshotHeight + val expected = dp.expectedChunkIds.map(id => ModifierId @@ Algos.encode(id)) + expected shouldBe subtreeIdsEncoded + val toDownload = utxoSetSnapshotProcessor.getChunkIdsToDownload(expected.size).map(id => ModifierId @@ Algos.encode(id)) + toDownload shouldBe expected + + subtreeIds.foreach { subtreeId => + val subtreeBytes = us.snapshotsDb.readSubtreeBytes(subtreeId).get + utxoSetSnapshotProcessor.registerDownloadedChunk(subtreeId, subtreeBytes) + } + val s = utxoSetSnapshotProcessor.downloadedChunksIterator().map(s => ModifierId @@ Algos.encode(s.id)).toSeq + s shouldBe subtreeIdsEncoded + + val dir = createTempDir + val store = new LDBVersionedStore(dir, initialKeepVersions = 100) + val restoredProver = utxoSetSnapshotProcessor.createPersistentProver(store, history, snapshotHeight, blockId).get + bh.sortedBoxes.foreach { box => + restoredProver.unauthenticatedLookup(box.id).isDefined shouldBe true + } + restoredProver.checkTree(postProof = false) + val restoredState = new UtxoState(restoredProver, version = VersionTag @@@ blockId, store, settings) + restoredState.stateContext.currentHeight shouldBe (epochLength - 1) + bh.sortedBoxes.foreach { box => + restoredState.boxById(box.id).isDefined shouldBe true + } + } + +} diff --git a/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala index 8c489cf3eb..0240d00afb 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/VerifyADHistorySpecification.scala @@ -22,8 +22,8 @@ class VerifyADHistorySpecification extends HistoryTestHelpers with NoShrink { minFullHeight: Option[Int] = Some(ErgoHistory.GenesisHeight)): (ErgoHistory, Seq[ErgoFullBlock]) = { val inHistory = generateHistory(verifyTransactions = true, StateType.Digest, PoPoWBootstrap = false, BlocksToKeep) minFullHeight.foreach { h => - inHistory.pruningProcessor.minimalFullBlockHeightVar = h - inHistory.pruningProcessor.isHeadersChainSyncedVar = true + inHistory.writeMinimalFullBlockHeight(h) + inHistory.isHeadersChainSyncedVar = true } if (blocksNum > 0) { @@ -87,7 +87,7 @@ class VerifyADHistorySpecification extends HistoryTestHelpers with NoShrink { history.bestFullBlockOpt shouldBe None val fullBlocksToApply = chain.tail - history.pruningProcessor.updateBestFullBlock(fullBlocksToApply(BlocksToKeep - 1).header) + history.updateBestFullBlock(fullBlocksToApply(BlocksToKeep - 1).header) history.applicable(chain.head.blockTransactions) shouldBe false @@ -200,7 +200,7 @@ class VerifyADHistorySpecification extends HistoryTestHelpers with NoShrink { history = applyHeaderChain(history, HeaderChain(chain.map(_.header))) history.bestHeaderOpt.value shouldBe chain.last.header history.bestFullBlockOpt shouldBe None - history.pruningProcessor.updateBestFullBlock(chain.last.header) + history.updateBestFullBlock(chain.last.header) val fullBlocksToApply = chain.takeRight(BlocksToKeep) diff --git a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala index 30908ceb1a..cb024dd9bb 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/VerifyNonADHistorySpecification.scala @@ -11,7 +11,7 @@ import org.ergoplatform.utils.HistoryTestHelpers import scorex.core.consensus.ProgressInfo class VerifyNonADHistorySpecification extends HistoryTestHelpers { - import org.ergoplatform.nodeView.history.storage.modifierprocessors.ToDownloadProcessor._ + import scorex.core.utils.MapPimp private def genHistory() = generateHistory(verifyTransactions = true, StateType.Utxo, PoPoWBootstrap = false, BlocksToKeep) @@ -19,8 +19,8 @@ class VerifyNonADHistorySpecification extends HistoryTestHelpers { property("block sections application in incorrect order") { var history = genHistory() val chain = genChain(6, history) - if (!history.pruningProcessor.isHeadersChainSynced) { - history.pruningProcessor.updateBestFullBlock(chain.last.header) + if (!history.isHeadersChainSynced) { + history.updateBestFullBlock(chain.last.header) } history = applyHeaderChain(history, HeaderChain(chain.map(_.header))) chain.foreach(fb => history.append(fb.extension).get) @@ -95,8 +95,8 @@ class VerifyNonADHistorySpecification extends HistoryTestHelpers { history.bestHeaderOpt.value shouldBe chain.last.header history.bestFullBlockOpt shouldBe None - if (!history.pruningProcessor.isHeadersChainSynced) { - history.pruningProcessor.updateBestFullBlock(chain.last.header) + if (!history.isHeadersChainSynced) { + history.updateBestFullBlock(chain.last.header) } // Until UTXO snapshot synchronization is implemented, we should always start to apply full blocks from genesis @@ -121,13 +121,13 @@ class VerifyNonADHistorySpecification extends HistoryTestHelpers { newAcc.adjust(mType)(_.fold(Seq(mId))(_ :+ mId)) } - history.nextModifiersToDownload(1, None, (_, id) => !history.contains(id)) + history.nextModifiersToDownload(1, (_, id) => !history.contains(id)) .map(id => (id._1, id._2.map(Algos.encode))) shouldEqual missedBS.mapValues(_.take(1)).view.force - history.nextModifiersToDownload(2 * (BlocksToKeep - 1), None, (_, id) => !history.contains(id)) + history.nextModifiersToDownload(2 * (BlocksToKeep - 1), (_, id) => !history.contains(id)) .map(id => (id._1, id._2.map(Algos.encode))) shouldEqual missedBS - history.nextModifiersToDownload(2, None, (_, id) => !history.contains(id) && (id != missedChain.head.blockTransactions.id)) + history.nextModifiersToDownload(2, (_, id) => !history.contains(id) && (id != missedChain.head.blockTransactions.id)) .map(id => (id._1, id._2.map(Algos.encode))) shouldEqual missedBS.mapValues(_.take(2).filter( _ != missedChain.head.blockTransactions.id)).view.force } @@ -156,7 +156,7 @@ class VerifyNonADHistorySpecification extends HistoryTestHelpers { } property("append header to genesis - 2") { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val block = validFullBlock(None, us, bh) diff --git a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala index d836c47696..16ca339dee 100644 --- a/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/history/extra/ExtraIndexerSpecification.scala @@ -1,8 +1,8 @@ package org.ergoplatform.nodeView.history.extra import org.ergoplatform.ErgoBox.TokenId -import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, ErgoBoxCandidate, ErgoScriptPredef, P2PKAddress, UnsignedInput} import org.ergoplatform.ErgoLikeContext.Height +import org.ergoplatform._ import org.ergoplatform.mining.difficulty.DifficultySerializer import org.ergoplatform.mining.{AutolykosPowScheme, CandidateBlock, CandidateGenerator} import org.ergoplatform.modifiers.ErgoFullBlock @@ -13,13 +13,13 @@ import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransact import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.history.extra.IndexedErgoAddressSerializer.{boxSegmentId, hashErgoTree, txSegmentId} import org.ergoplatform.nodeView.mempool.ErgoMemPool.SortingOption -import org.ergoplatform.nodeView.state.{ErgoState, ErgoStateContext, StateConstants, StateType, UtxoState, UtxoStateReader} -import org.ergoplatform.settings.{ErgoSettings, NetworkType, NodeConfigurationSettings} +import org.ergoplatform.nodeView.state._ +import org.ergoplatform.settings.{ErgoSettings, NetworkType, NodeConfigurationSettings, UtxoSettings} import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestHelpers, HistoryTestHelpers} -import scorex.crypto.hash.Digest32 import scorex.util.{ModifierId, bytesToId} import sigmastate.Values import sigmastate.basics.DLogProtocol.ProveDlog +import sigmastate.eval.Extensions._ import sigmastate.eval._ import special.collection.Coll import spire.implicits.cfor @@ -38,7 +38,7 @@ class ExtraIndexerSpecification extends ErgoPropertyTest with ExtraIndexerBase w override protected implicit val addressEncoder: ErgoAddressEncoder = initSettings.chainSettings.addressEncoder val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true, - -1, poPoWBootstrap = false, ChainGenerator.minimalSuffix, mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false, + -1, UtxoSettings(false, 0, 2), poPoWBootstrap = false, ChainGenerator.minimalSuffix, mining = false, ChainGenerator.txCostLimit, ChainGenerator.txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20, 1000000, 100, adProofsSuffixLength = 112 * 1024, extraIndex = false) @@ -239,7 +239,7 @@ object ChainGenerator extends ErgoTestHelpers { val MaxTxsPerBlock: Int = 10 val minerPk: ProveDlog = defaultProver.hdKeys.head.publicImage val selfAddressScript: Values.ErgoTree = P2PKAddress(minerPk).script - val minerProp: Values.ErgoTree = ErgoScriptPredef.rewardOutputScript(RewardDelay, minerPk) + val minerProp: Values.ErgoTree = ErgoTreePredef.rewardOutputScript(RewardDelay, minerPk) val votingEpochLength: Height = votingSettings.votingLength val protocolVersion: Byte = initSettings.chainSettings.protocolVersion val minimalSuffix = 2 @@ -251,7 +251,7 @@ object ChainGenerator extends ErgoTestHelpers { def generate(length: Int, dir: File)(history: ErgoHistory): Unit = { val stateDir = new File(s"${dir.getAbsolutePath}/state") stateDir.mkdirs() - val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(initSettings)) + val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, initSettings) System.out.println(s"Going to generate a chain at ${dir.getAbsolutePath} starting from ${history.bestFullBlockOpt}") startTime = System.currentTimeMillis() - (blockInterval * (length - 1)).toMillis val chain = loop(state, None, None, Seq())(history) @@ -298,7 +298,7 @@ object ChainGenerator extends ErgoTestHelpers { val tokens: ArrayBuffer[(TokenId, Long)] = ArrayBuffer.empty[(TokenId, Long)] inOpt match { case Some(input) if cond => - tokens += Tuple2(Digest32 @@@ input.id, math.abs(Random.nextInt())) + tokens += Tuple2(input.id.toTokenId, math.abs(Random.nextInt())) case Some(tokenBox) if !cond => tokenBox.additionalTokens.toArray.foreach(tokens += _) case _ => diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala index c68d279f50..d08408e7cf 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ErgoMemPoolSpec.scala @@ -19,9 +19,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec with ScalaCheckPropertyChecks { it should "accept valid transaction" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get val txs = validTransactionsFromUtxoState(wus) val pool0 = ErgoMemPool.empty(settings) val poolAfter = txs.foldLeft(pool0) { case (pool, tx) => @@ -42,9 +42,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec it should "respect given sorting order" in { implicit val ms = settings.chainSettings.monetary - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get val inputBox = wus.takeBoxes(1).head val feeOut = new ErgoBoxCandidate(inputBox.value, feeProp, creationHeight = 0) val tx = ErgoTransaction( @@ -77,9 +77,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "decline already contained transaction" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get val txs = validTransactionsFromUtxoState(wus) var pool = ErgoMemPool.empty(settings) txs.foreach { tx => @@ -93,9 +93,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec it should "reject double-spending transaction if it is paying no more than one already sitting in the pool" in { forAll(smallPositiveInt, smallPositiveInt) { case (n1, n2) => whenever(n1 != n2) { - val (us, bh) = createUtxoState(extendedParameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, extendedParameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, extendedParameters).applyModifier(genesis)(_ => ()).get val feeProp = settings.chainSettings.monetary.feeProposition val inputBox = wus.takeBoxes(100).collectFirst{ @@ -141,7 +141,7 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "decline transactions invalidated earlier" in { - val us = createUtxoState(parameters)._1 + val us = createUtxoState(settings)._1 var pool = ErgoMemPool.empty(settings) forAll(invalidBlockTransactionsGen) { blockTransactions => val unconfirmedTxs = blockTransactions.txs.map(tx => UnconfirmedTransaction(tx, None)) @@ -152,9 +152,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "decline transactions not meeting min fee" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get val txs = validTransactionsFromUtxoState(wus) val unconfirmedTxs = txs.map(tx => UnconfirmedTransaction(tx, None)) @@ -176,7 +176,7 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "invalidate or reject invalid transaction" in { - val us = createUtxoState(parameters)._1 + val us = createUtxoState(settings)._1 val pool = ErgoMemPool.empty(settings) forAll(invalidBlockTransactionsGen) { blockTransactions => blockTransactions.txs.forall{tx => @@ -214,9 +214,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "Accept output of pooled transactions" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get val txs = validTransactionsFromUtxoState(wus).map(tx => UnconfirmedTransaction(tx, None)) var pool = ErgoMemPool.empty(settings) txs.foreach { tx => @@ -234,9 +234,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "consider families for replacement policy" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get var txs = validTransactionsFromUtxoState(wus).map(tx => UnconfirmedTransaction(tx, None)) val family_depth = 10 val limitedPoolSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(mempoolCapacity = (family_depth + 1) * txs.size)) @@ -269,9 +269,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "correctly remove transaction from pool and rebuild families" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get var txs = validTransactionsFromUtxoState(wus).map(tx => UnconfirmedTransaction(tx, None)) var allTxs = txs val family_depth = 10 @@ -302,9 +302,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec it should "return results take / getAll / getAllPrioritized sorted by priority" in { val feeProp = settings.chainSettings.monetary.feeProposition - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get var txs = validTransactionsFromUtxoState(wus).map(tx => UnconfirmedTransaction(tx, None)) val family_depth = 10 val limitedPoolSettings = settings.copy(nodeSettings = settings.nodeSettings.copy(mempoolCapacity = (family_depth + 1) * txs.size)) @@ -344,9 +344,9 @@ class ErgoMemPoolSpec extends AnyFlatSpec } it should "add removed transaction to mempool statistics" in { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(None, us, bh) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wus = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get var txs = validTransactionsFromUtxoState(wus).map(tx => UnconfirmedTransaction(tx, None)) var allTxs = txs val family_depth = 10 diff --git a/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala b/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala index 96c8aad776..ce643b96ef 100644 --- a/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/mempool/ScriptsSpec.scala @@ -1,18 +1,18 @@ package org.ergoplatform.nodeView.mempool import org.ergoplatform.ErgoAddressEncoder.TestnetNetworkPrefix -import org.ergoplatform.ErgoScriptPredef.boxCreationHeight -import org.ergoplatform.{ErgoBox, ErgoScriptPredef, Height, Self} +import org.ergoplatform.ErgoTreePredef.boxCreationHeight import org.ergoplatform.nodeView.state.{BoxHolder, ErgoState, UtxoState} import org.ergoplatform.settings.Algos import org.ergoplatform.utils.{ErgoPropertyTest, RandomWrapper} +import org.ergoplatform.{ErgoBox, ErgoTreePredef, Height, Self} import scorex.crypto.authds.avltree.batch.Remove -import sigmastate._ import sigmastate.Values._ -import sigmastate.lang.Terms._ +import sigmastate._ +import sigmastate.basics.CryptoConstants.dlogGroup import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.eval.{CompiletimeIRContext, IRContext} -import sigmastate.interpreter.CryptoConstants.dlogGroup +import sigmastate.lang.Terms._ import sigmastate.lang.{CompilerSettings, SigmaCompiler, TransformingSigmaBuilder} import scala.util.Try @@ -55,7 +55,7 @@ class ScriptsSpec extends ErgoPropertyTest { delta shouldBe -1000 applyBlockSpendingScript(GE(Height, Plus(boxCreationHeight(Self), IntConstant(delta))).toSigmaProp) shouldBe 'success - applyBlockSpendingScript(ErgoScriptPredef.rewardOutputScript(delta, defaultMinerPk)) shouldBe 'success + applyBlockSpendingScript(ErgoTreePredef.rewardOutputScript(delta, defaultMinerPk)) shouldBe 'success // applyBlockSpendingScript(ErgoScriptPredef.feeProposition(delta)) shouldBe 'success } @@ -67,7 +67,7 @@ class ScriptsSpec extends ErgoPropertyTest { private def applyBlockSpendingScript(script: ErgoTree): Try[UtxoState] = { val scriptBox = ergoBoxGen(script, heightGen = 0).sample.get val bh = BoxHolder(Seq(fixedBox, scriptBox)) - val us = UtxoState.fromBoxHolder(bh, None, createTempDir, stateConstants, parameters) + val us = UtxoState.fromBoxHolder(bh, None, createTempDir, settings, parameters) bh.boxes.map(b => us.boxById(b._2.id) shouldBe Some(b._2)) val tx = validTransactionsFromBoxHolder(bh, new RandomWrapper(Some(1)), 201)._1 tx.size shouldBe 1 diff --git a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala index 4d670365d1..7cf71def1a 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/DigestStateSpecification.scala @@ -21,19 +21,19 @@ class DigestStateSpecification extends ErgoPropertyTest { val fb = validFullBlock(parentOpt = None, us, bh) val dir2 = createTempDir - val ds = DigestState.create(Some(us.version), Some(us.rootDigest), dir2, stateConstants) + val ds = DigestState.create(Some(us.version), Some(us.rootDigest), dir2, settings) ds.applyModifier(fb, None)(_ => ()) shouldBe 'success ds.close() - val state = DigestState.create(None, None, dir2, stateConstants) + val state = DigestState.create(None, None, dir2, settings) state.version shouldEqual fb.header.id state.rootDigest shouldEqual fb.header.stateRoot } } property("validate() - valid block") { - var (us, bh) = createUtxoState(parameters) - var ds = createDigestState(us.version, us.rootDigest, parameters) + var (us, bh) = createUtxoState(settings) + var ds = createDigestState(us.version, us.rootDigest) var parentOpt: Option[ErgoFullBlock] = None forAll { seed: Int => @@ -48,7 +48,7 @@ class DigestStateSpecification extends ErgoPropertyTest { property("validate() - invalid block") { forAll(invalidErgoFullBlockGen) { b => - val state = createDigestState(emptyVersion, emptyAdDigest, parameters) + val state = createDigestState(emptyVersion, emptyAdDigest) state.validate(b).isFailure shouldBe true } } @@ -61,14 +61,14 @@ class DigestStateSpecification extends ErgoPropertyTest { val block = validFullBlock(parentOpt = None, us, bh) block.blockTransactions.transactions.exists(_.dataInputs.nonEmpty) shouldBe true - val ds = createDigestState(us.version, us.rootDigest, parameters) + val ds = createDigestState(us.version, us.rootDigest) ds.applyModifier(block, None)(_ => ()) shouldBe 'success } } property("applyModifier() - invalid block") { forAll(invalidErgoFullBlockGen) { b => - val state = createDigestState(emptyVersion, emptyAdDigest, parameters) + val state = createDigestState(emptyVersion, emptyAdDigest) state.applyModifier(b, None)(_ => ()).isFailure shouldBe true } } @@ -80,7 +80,7 @@ class DigestStateSpecification extends ErgoPropertyTest { val block = validFullBlock(parentOpt = None, us, bh) - val ds = createDigestState(us.version, us.rootDigest, parameters) + val ds = createDigestState(us.version, us.rootDigest) ds.rollbackVersions.size shouldEqual 1 @@ -104,7 +104,7 @@ class DigestStateSpecification extends ErgoPropertyTest { property("validateTransactions() - dataInputs") { forAll(boxesHolderGen) { bh => val us = createUtxoState(bh, parameters) - val ds = createDigestState(us.version, us.rootDigest, parameters) + val ds = createDigestState(us.version, us.rootDigest) // generate 2 independent transactions spending state boxes only val headTx = validTransactionsFromBoxes(1, bh.boxes.take(10).values.toSeq, new RandomWrapper())._1.head diff --git a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala index 60d49ff68a..059b4e0d1c 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/ErgoStateSpecification.scala @@ -20,7 +20,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { property("applyModifier() - double spending") { forAll(boxesHolderGen, Gen.choose(1: Byte, 2: Byte)) { case (bh, version) => val us = createUtxoState(bh, parameters) - val ds = createDigestState(bytesToVersion(Array.fill(32)(100: Byte)), us.rootDigest, parameters) + val ds = createDigestState(bytesToVersion(Array.fill(32)(100: Byte)), us.rootDigest) val validBlock = validFullBlock(None, us, bh) val dsTxs = validBlock.transactions ++ validBlock.transactions @@ -51,8 +51,8 @@ class ErgoStateSpecification extends ErgoPropertyTest { s1.genesisStateDigest shouldBe s2.genesisStateDigest } - var (us, bh) = createUtxoState(parameters) - var ds = createDigestState(us.version, us.rootDigest, parameters) + var (us, bh) = createUtxoState(settings) + var ds = createDigestState(us.version, us.rootDigest) var lastBlocks: Seq[ErgoFullBlock] = Seq() forAll { seed: Int => val blBh = validFullBlockWithBoxHolder(lastBlocks.headOption, us, bh, new RandomWrapper(Some(seed))) @@ -68,13 +68,13 @@ class ErgoStateSpecification extends ErgoPropertyTest { property("generateGenesisUtxoState & generateGenesisDigestState are compliant") { val settings = ErgoSettings.read(Args.empty) val dir = createTempDir - val rootHash = createUtxoState(parameters)._1.rootDigest + val rootHash = createUtxoState(settings)._1.rootDigest val expectedRootHash = ErgoState.generateGenesisDigestState(dir, settings).rootDigest rootHash shouldBe expectedRootHash } property("ErgoState.boxChanges() should generate operations in the same order") { - var (us, bh) = createUtxoState(parameters) + var (us, bh) = createUtxoState(settings) var parentOpt: Option[ErgoFullBlock] = None forAll { seed: Int => @@ -92,7 +92,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { } property("ErgoState.boxChanges() double spend attempt") { - val (_, bh) = createUtxoState(parameters) + val (_, bh) = createUtxoState(settings) val emissionBox = genesisBoxes.head forAll { seed: Int => @@ -116,7 +116,7 @@ class ErgoStateSpecification extends ErgoPropertyTest { } property("ErgoState.stateChanges()") { - val bh = createUtxoState(parameters)._2 + val bh = createUtxoState(settings)._2 val emissionBox = genesisBoxes.head forAll { seed: Int => diff --git a/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala new file mode 100644 index 0000000000..f273749ca7 --- /dev/null +++ b/src/test/scala/org/ergoplatform/nodeView/state/SnapshotsDbSpecification.scala @@ -0,0 +1,51 @@ +package org.ergoplatform.nodeView.state + +import org.ergoplatform.utils.ErgoPropertyTest +import org.scalacheck.Gen +import scorex.crypto.hash.Digest32 +import scorex.util.{ModifierId, bytesToId, idToBytes} + +import scala.util.Random + +class SnapshotsDbSpecification extends ErgoPropertyTest { + + def seededDatabase(manifestIds: Seq[ModifierId]): (SnapshotsInfo, SnapshotsDb) = { + val m = manifestIds.map { mid => + Random.nextInt(1000000) -> (Digest32 @@ idToBytes(mid)) + }.toMap + val si = new SnapshotsInfo(m) + val dir = createTempDir.getAbsolutePath + val db = SnapshotsDb.create(dir) + db.writeSnapshotsInfo(si) + si -> db + } + + property("snapshotsInfo round-trip") { + forAll(Gen.nonEmptyListOf(modifierIdGen)) { manifestIds => + val (si, db) = seededDatabase(manifestIds) + val read = db.readSnapshotsInfo.availableManifests.mapValues(bs => bytesToId(bs)) + val siTocompare = si.availableManifests.mapValues(bs => bytesToId(bs)) + read shouldBe siTocompare + } + } + + property("pruneSnapshots choosing snapshots correctly") { + forAll(Gen.nonEmptyListOf(modifierIdGen)) { manifestIds => + val (si, db) = seededDatabase(manifestIds) + + val toStore = Random.nextInt(manifestIds.size + 3) + + db.pruneSnapshots(toStore) + + val after = db.readSnapshotsInfo + + if (toStore >= manifestIds.size) { + after.availableManifests.size == manifestIds.size + } else { + val storedKeys = si.availableManifests.keySet.toSeq.sorted.takeRight(toStore) + val stored = si.availableManifests.filterKeys(h => storedKeys.contains(h)) + after.availableManifests.mapValues(bytesToId) shouldBe stored.mapValues(bytesToId) + } + } + } +} diff --git a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala index 205caa634a..a0cd911b10 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/UtxoStateSpecification.scala @@ -35,7 +35,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera private val emptyModifierId: ModifierId = bytesToId(Array.fill(32)(0.toByte)) property("Founders box workflow") { - var (us, bh) = createUtxoState(parameters) + var (us, bh) = createUtxoState(settings) var foundersBox = genesisBoxes.last var lastBlock = validFullBlock(parentOpt = None, us, bh) us = us.applyModifier(lastBlock, None)(_ => ()).get @@ -60,7 +60,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera } property("Founders should be able to spend genesis founders box") { - var (us, bh) = createUtxoState(parameters) + var (us, bh) = createUtxoState(settings) val foundersBox = genesisBoxes.last var height: Int = ErgoHistory.GenesisHeight @@ -111,7 +111,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera } property("Correct genesis state") { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val boxes = bh.boxes.values.toList boxes.size shouldBe 3 @@ -135,7 +135,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera } property("extractEmissionBox() should extract correct box") { - var (us, bh) = createUtxoState(parameters) + var (us, bh) = createUtxoState(settings) us.emissionBoxOpt should not be None var lastBlockOpt: Option[ErgoFullBlock] = None forAll { seed: Int => @@ -158,7 +158,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera } property("proofsForTransactions") { - var (us: UtxoState, bh) = createUtxoState(parameters) + var (us: UtxoState, bh) = createUtxoState(settings) var height: Int = ErgoHistory.GenesisHeight forAll(defaultHeaderGen) { header => val t = validTransactionsFromBoxHolder(bh, new RandomWrapper(Some(height))) @@ -435,13 +435,13 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera property("applyModifier() - invalid block") { forAll(invalidErgoFullBlockGen) { b => - val state = createUtxoState(parameters)._1 + val state = createUtxoState(settings)._1 state.applyModifier(b, None)(_ => ()).isFailure shouldBe true } } property("applyModifier() - valid full block after invalid one") { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val validBlock = validFullBlock(parentOpt = None, us, bh) //Different state @@ -460,20 +460,20 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera property("2 forks switching") { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val genesis = validFullBlock(parentOpt = None, us, bh) - val wusAfterGenesis = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wusAfterGenesis = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get val chain1block1 = validFullBlock(Some(genesis), wusAfterGenesis) val wusChain1Block1 = wusAfterGenesis.applyModifier(chain1block1)(_ => ()).get val chain1block2 = validFullBlock(Some(chain1block1), wusChain1Block1) - val (us2, bh2) = createUtxoState(parameters) - val wus2AfterGenesis = WrappedUtxoState(us2, bh2, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val (us2, bh2) = createUtxoState(settings) + val wus2AfterGenesis = WrappedUtxoState(us2, bh2, settings, parameters).applyModifier(genesis)(_ => ()).get val chain2block1 = validFullBlock(Some(genesis), wus2AfterGenesis) val wusChain2Block1 = wus2AfterGenesis.applyModifier(chain2block1)(_ => ()).get val chain2block2 = validFullBlock(Some(chain2block1), wusChain2Block1) - var (state, _) = createUtxoState(parameters) + var (state, _) = createUtxoState(settings) state = state.applyModifier(genesis, None)(_ => ()).get state = state.applyModifier(chain1block1, None)(_ => ()).get @@ -494,7 +494,7 @@ class UtxoStateSpecification extends ErgoPropertyTest with ErgoTransactionGenera val us = createUtxoState(bh, parameters) bh.sortedBoxes.foreach(box => us.boxById(box.id) should not be None) val genesis = validFullBlock(parentOpt = None, us, bh) - val wusAfterGenesis = WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis)(_ => ()).get + val wusAfterGenesis = WrappedUtxoState(us, bh, settings, parameters).applyModifier(genesis)(_ => ()).get wusAfterGenesis.rootDigest shouldEqual genesis.header.stateRoot val (finalState: WrappedUtxoState, chain: Seq[ErgoFullBlock]) = (0 until depth) diff --git a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala index fbb2cf883b..d747a4b5be 100644 --- a/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala +++ b/src/test/scala/org/ergoplatform/nodeView/state/wrapped/WrappedUtxoState.scala @@ -22,8 +22,8 @@ class WrappedUtxoState(prover: PersistentBatchAVLProver[Digest32, HF], override val version: VersionTag, store: LDBVersionedStore, val versionedBoxHolder: VersionedInMemoryBoxHolder, - constants: StateConstants) - extends UtxoState(prover, version, store, constants) { + settings: ErgoSettings) + extends UtxoState(prover, version, store, settings) { def size: Int = versionedBoxHolder.size @@ -32,7 +32,7 @@ class WrappedUtxoState(prover: PersistentBatchAVLProver[Digest32, HF], override def rollbackTo(version: VersionTag): Try[WrappedUtxoState] = super.rollbackTo(version) match { case Success(us) => val updHolder = versionedBoxHolder.rollback(us.version) - Success(new WrappedUtxoState(us.persistentProver, version, us.store, updHolder, constants)) + Success(new WrappedUtxoState(us.persistentProver, version, us.store, updHolder, settings)) case Failure(e) => Failure(e) } @@ -49,10 +49,10 @@ class WrappedUtxoState(prover: PersistentBatchAVLProver[Digest32, HF], us.version, changes.toRemove.map(_.key).map(ByteArrayWrapper.apply), changes.toAppend.map(_.value).map(ErgoBoxSerializer.parseBytes)) - Success(new WrappedUtxoState(us.persistentProver, idToVersion(mod.id), us.store, updHolder, constants)) + Success(new WrappedUtxoState(us.persistentProver, idToVersion(mod.id), us.store, updHolder, settings)) case _ => val updHolder = versionedBoxHolder.applyChanges(us.version, Seq(), Seq()) - Success(new WrappedUtxoState(us.persistentProver, idToVersion(mod.id), us.store, updHolder, constants)) + Success(new WrappedUtxoState(us.persistentProver, idToVersion(mod.id), us.store, updHolder, settings)) } case Failure(e) => Failure(e) } @@ -65,13 +65,12 @@ object WrappedUtxoState { nodeViewHolderRef: Option[ActorRef], parameters: Parameters, settings: ErgoSettings): WrappedUtxoState = { - val constants = StateConstants(settings) - val emissionBox = ErgoState.genesisBoxes(constants.settings.chainSettings).headOption - val us = UtxoState.fromBoxHolder(boxHolder, emissionBox, dir, constants, parameters) - WrappedUtxoState(us, boxHolder, constants, parameters) + val emissionBox = ErgoState.genesisBoxes(settings.chainSettings).headOption + val us = UtxoState.fromBoxHolder(boxHolder, emissionBox, dir, settings, parameters) + WrappedUtxoState(us, boxHolder, settings, parameters) } - def apply(us: UtxoState, boxHolder: BoxHolder, constants: StateConstants, parameters: Parameters): WrappedUtxoState = { + def apply(us: UtxoState, boxHolder: BoxHolder, settings: ErgoSettings, parameters: Parameters): WrappedUtxoState = { val boxes = boxHolder.boxes val version = us.version @@ -81,6 +80,6 @@ object WrappedUtxoState { Map(version -> (Seq() -> boxHolder.sortedBoxes.toSeq)) ) - new WrappedUtxoState(us.persistentProver, ErgoState.genesisStateVersion, us.store, vbh, constants) + new WrappedUtxoState(us.persistentProver, ErgoState.genesisStateVersion, us.store, vbh, settings) } } diff --git a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala index f3b89434a5..83e9243ad9 100644 --- a/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/viewholder/ErgoNodeViewHolderSpec.scala @@ -22,7 +22,7 @@ import scorex.util.{ModifierId, bytesToId} class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers with NodeViewTestOps with NoShrink { private val t0 = TestCase("check chain is healthy") { fixture => - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(settings) val block = validFullBlock(None, us, bh) val history = generateHistory(true, StateType.Utxo, false, 2) @@ -51,7 +51,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t3 = TestCase("apply valid block header") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val block = validFullBlock(None, us, bh) getBestHeaderOpt shouldBe None @@ -71,7 +71,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t3a = TestCase("do not apply block headers in invalid order") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val parentBlock = validFullBlock(None, us, bh) val block = validFullBlock(Some(parentBlock), us, bh) @@ -96,7 +96,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t4 = TestCase("apply valid block as genesis") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) subscribeEvents(classOf[SyntacticallySuccessfulModifier]) @@ -116,10 +116,10 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t5 = TestCase("apply full blocks after genesis") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = - WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis) { mod => + WrappedUtxoState(us, bh, fixture.settings, parameters).applyModifier(genesis) { mod => nodeViewHolderRef ! mod }.get applyBlock(genesis) shouldBe 'success @@ -138,7 +138,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t6 = TestCase("add transaction to memory pool") { fixture => import fixture._ if (stateType == Utxo) { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) applyBlock(genesis) shouldBe 'success @@ -155,10 +155,10 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t7 = TestCase("apply statefully invalid full block") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = - WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis) { mod => + WrappedUtxoState(us, bh, fixture.settings, parameters).applyModifier(genesis) { mod => nodeViewHolderRef ! mod }.get // TODO looks like another bug is still present here, see https://github.com/ergoplatform/ergo/issues/309 @@ -210,10 +210,10 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t8 = TestCase("switching for a better chain") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = - WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis) { mod => + WrappedUtxoState(us, bh, fixture.settings, parameters).applyModifier(genesis) { mod => nodeViewHolderRef ! mod }.get @@ -247,7 +247,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t9 = TestCase("UTXO state should generate adProofs and put them in history") { fixture => import fixture._ if (stateType == StateType.Utxo) { - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) nodeViewHolderRef ! LocallyGeneratedModifier(genesis.header) @@ -261,10 +261,10 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t10 = TestCase("NodeViewHolder start from inconsistent state") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = - WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis) { mod => + WrappedUtxoState(us, bh, fixture.settings, parameters).applyModifier(genesis) { mod => nodeViewHolderRef ! mod }.get applyBlock(genesis) shouldBe 'success @@ -284,10 +284,10 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t11 = TestCase("apply payload in incorrect order (excluding extension)") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val genesis = validFullBlock(parentOpt = None, us, bh) val wusAfterGenesis = - WrappedUtxoState(us, bh, stateConstants, parameters).applyModifier(genesis) { mod => + WrappedUtxoState(us, bh, fixture.settings, parameters).applyModifier(genesis) { mod => nodeViewHolderRef ! mod }.get @@ -314,7 +314,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t12 = TestCase("Do not apply txs with wrong header id") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val block = validFullBlock(None, us, bh) getBestHeaderOpt shouldBe None getHistoryHeight shouldBe ErgoHistory.EmptyHistoryHeight @@ -367,7 +367,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t13 = TestCase("Do not apply wrong adProofs") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val block = validFullBlock(None, us, bh) getBestHeaderOpt shouldBe None @@ -400,7 +400,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi "it's not equal to genesisId from config") { fixture => import fixture._ updateConfig(genesisIdConfig(modifierIdGen.sample)) - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val block = validFullBlock(None, us, bh) getBestHeaderOpt shouldBe None @@ -419,7 +419,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t15 = TestCase("apply genesis block header if it's equal to genesisId from config") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val block = validFullBlock(None, us, bh) updateConfig(genesisIdConfig(Some(block.header.id))) @@ -439,8 +439,8 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t16 = TestCase("apply forks that include genesis block") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) - val wusGenesis = WrappedUtxoState(us, bh, stateConstants, parameters) + val (us, bh) = createUtxoState(fixture.settings) + val wusGenesis = WrappedUtxoState(us, bh, fixture.settings, parameters) val chain1block1 = validFullBlock(parentOpt = None, us, bh) @@ -469,7 +469,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t17 = TestCase("apply invalid genesis header") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val header = validFullBlock(None, us, bh).header.copy(parentId = bytesToId(Array.fill(32)(9: Byte))) getBestHeaderOpt shouldBe None @@ -488,7 +488,7 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t18 = TestCase("apply syntactically invalid genesis block") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) + val (us, bh) = createUtxoState(fixture.settings) val validBlock = validFullBlock(parentOpt = None, us, bh) val invalidBlock = validBlock.copy(header = validBlock.header.copy(parentId = bytesToId(Array.fill(32)(9: Byte)))) @@ -501,8 +501,8 @@ class ErgoNodeViewHolderSpec extends ErgoPropertyTest with HistoryTestHelpers wi private val t19 = TestCase("apply semantically invalid genesis block") { fixture => import fixture._ - val (us, bh) = createUtxoState(parameters) - val wusGenesis = WrappedUtxoState(us, bh, stateConstants, parameters) + val (us, bh) = createUtxoState(fixture.settings) + val wusGenesis = WrappedUtxoState(us, bh, fixture.settings, parameters) val invalidBlock = generateInvalidFullBlock(None, wusGenesis) diff --git a/src/test/scala/org/ergoplatform/nodeView/viewholder/PrunedNodeViewHolderSpec.scala b/src/test/scala/org/ergoplatform/nodeView/viewholder/PrunedNodeViewHolderSpec.scala index 7087481567..0dd39c88c8 100644 --- a/src/test/scala/org/ergoplatform/nodeView/viewholder/PrunedNodeViewHolderSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/viewholder/PrunedNodeViewHolderSpec.scala @@ -48,8 +48,8 @@ class PrunedNodeViewHolderSpec extends ErgoPropertyTest with NodeViewTestOps wit private def testCode(fixture: NodeViewFixture, toSkip: Int, totalBlocks: Int = 20) = { import fixture._ - val (us, bh) = createUtxoState(stateConstants, parameters) - val wus = WrappedUtxoState(us, bh, stateConstants, parameters) + val (us, bh) = createUtxoState(fixture.settings) + val wus = WrappedUtxoState(us, bh, fixture.settings, parameters) val fullChain = genFullChain(wus, totalBlocks, nodeViewHolderRef) diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala index 4ed82f40ac..c492591ac0 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletServiceSpec.scala @@ -9,6 +9,7 @@ import org.ergoplatform.nodeView.wallet.WalletScanLogic.ScanResults import org.ergoplatform.nodeView.wallet.persistence.{OffChainRegistry, WalletRegistry, WalletStorage} import org.ergoplatform.nodeView.wallet.requests.{AssetIssueRequest, PaymentRequest} import org.ergoplatform.nodeView.wallet.scanning.{EqualsScanningPredicate, ScanRequest, ScanWalletInteraction} +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.ergoplatform.settings.ErgoSettings import org.ergoplatform.utils.fixtures.WalletFixture import org.ergoplatform.utils.generators.ErgoTransactionGenerators @@ -19,13 +20,12 @@ import org.ergoplatform.wallet.boxes.{ErgoBoxSerializer, ReplaceCompactCollectBo import org.ergoplatform.wallet.crypto.ErgoSignature import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.scalacheck.Gen import org.scalatest.BeforeAndAfterAll import scorex.db.{LDBKVStore, LDBVersionedStore} import scorex.util.encode.Base16 -import sigmastate.ErgoTreeBenchmarks.traversableColl import sigmastate.Values.{ByteArrayConstant, EvaluatedValue} +import sigmastate.eval.Extensions.ArrayOps import sigmastate.helpers.TestingHelpers.testBox import sigmastate.{SType, Values} @@ -149,7 +149,7 @@ class ErgoWalletServiceSpec ErgoBox.R5 -> ByteArrayConstant("test-description".getBytes("UTF-8")), ErgoBox.R6 -> ByteArrayConstant("4".getBytes("UTF-8")), ) - validCandidate.additionalTokens.toMap shouldBe Map(ergoBox.id -> 1) + validCandidate.additionalTokens.toArray.toMap shouldBe Map(ergoBox.id.toColl -> 1) validCandidate.creationHeight shouldBe startHeight validCandidate.ergoTree shouldBe pks.head.script } diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala index 9c223aa364..9659837cb8 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/ErgoWalletSpec.scala @@ -6,23 +6,21 @@ import org.ergoplatform.nodeView.state.{ErgoStateContext, VotingData} import org.ergoplatform.nodeView.wallet.IdUtils._ import org.ergoplatform.nodeView.wallet.persistence.{WalletDigest, WalletDigestSerializer} import org.ergoplatform.nodeView.wallet.requests.{AssetIssueRequest, BurnTokensRequest, ExternalSecret, PaymentRequest} +import org.ergoplatform.sdk.wallet.secrets.PrimitiveSecretKey import org.ergoplatform.settings.{Algos, Constants} import org.ergoplatform.utils._ import org.ergoplatform.utils.fixtures.WalletFixture -import org.ergoplatform.wallet.interpreter.{ErgoInterpreter, TransactionHintsBag} -import scorex.util.encode.Base16 -import sigmastate.eval._ -import sigmastate.eval.Extensions._ - import org.ergoplatform.wallet.boxes.BoxSelector.MinBoxValue import org.ergoplatform.wallet.boxes.ErgoBoxSerializer -import org.ergoplatform.wallet.secrets.PrimitiveSecretKey +import org.ergoplatform.wallet.interpreter.{ErgoInterpreter, TransactionHintsBag} import org.scalacheck.Gen import org.scalatest.concurrent.Eventually -import scorex.crypto.hash.Digest32 import scorex.util.ModifierId -import sigmastate.{CAND, CTHRESHOLD} +import scorex.util.encode.Base16 import sigmastate.basics.DLogProtocol.DLogProverInput +import sigmastate.eval.Extensions._ +import sigmastate.eval._ +import sigmastate.{CAND, CTHRESHOLD} import scala.concurrent.duration._ @@ -242,7 +240,7 @@ class ErgoWalletSpec extends ErgoPropertyTest with WalletTestOps with Eventually property("whitelist set, preserve tokens from auto-burn") { val inputs = { val x = IndexedSeq(new Input(genesisEmissionBox.id, emptyProverResult)) - Seq(encodedTokenId(Digest32 @@@ x.head.boxId)) + Seq(encodedTokenId(x.head.boxId.toTokenId)) } implicit val ww: WalletFixture = new WalletFixture(settings diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala index 251de1eb57..09639adbb7 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletRegistryBenchmark.scala @@ -1,15 +1,15 @@ package org.ergoplatform.nodeView.wallet.persistence -import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} import org.ergoplatform.ErgoBox.{AdditionalRegisters, TokenId} import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.nodeView.wallet.WalletScanLogic.ScanResults import org.ergoplatform.nodeView.wallet.{WalletTransaction, WalletVars} +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.ergoplatform.utils.ErgoTestConstants import org.ergoplatform.wallet.Constants import org.ergoplatform.wallet.boxes.TrackedBox import org.ergoplatform.wallet.interpreter.ErgoProvingInterpreter -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedSecretKey} +import org.ergoplatform.{ErgoAddressEncoder, ErgoBox, Input} import scorex.util.ModifierId import scorex.util.encode.Base16 import sigmastate.Values.ErgoTree @@ -42,7 +42,7 @@ object WalletRegistryBenchmark extends App with ErgoTestConstants { val derivedSecrets = (1 to 15000).map { i => val k = rootSecret.derive(DerivationPath.fromEncoded(s"m/44'/429'/0'/0/$i").get) - storage.addPublicKeys(k.publicKey).get + storage.addPublicKey(k.publicKey).get k } diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala index a78cb1dd4d..c0a1826bab 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/persistence/WalletStorageSpec.scala @@ -4,8 +4,8 @@ import com.google.common.primitives.Ints import org.ergoplatform.db.DBSpec import org.ergoplatform.nodeView.wallet.persistence.WalletStorage.SecretPathsKey import org.ergoplatform.nodeView.wallet.scanning.{ScanRequest, ScanWalletInteraction} +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, DerivationPathSerializer} import org.ergoplatform.utils.generators.WalletGenerators -import org.ergoplatform.wallet.secrets.{DerivationPathSerializer, DerivationPath} import org.scalacheck.Gen import org.scalatest.flatspec.AnyFlatSpec import org.scalatest.matchers.should.Matchers @@ -27,7 +27,7 @@ class WalletStorageSpec val bytes = DerivationPathSerializer.toBytes(path) acc ++ Ints.toByteArray(bytes.length) ++ bytes } - store.insert(Array(SecretPathsKey -> toInsert)).get + store.insert(SecretPathsKey, toInsert).get } forAll(Gen.nonEmptyListOf(derivationPathGen)) { paths => @@ -43,7 +43,7 @@ class WalletStorageSpec forAll(extendedPubKeyListGen) { pubKeys => withStore { store => val storage = new WalletStorage(store, settings) - pubKeys.foreach(storage.addPublicKeys(_).get) + pubKeys.foreach(storage.addPublicKey(_).get) val keysRead = storage.readAllKeys() keysRead.length shouldBe pubKeys.length keysRead should contain theSameElementsAs pubKeys.toSet diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecsSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecsSpecification.scala index 54991244fb..4d015ac027 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecsSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateJsonCodecsSpecification.scala @@ -4,28 +4,28 @@ import io.circe.parser._ import org.ergoplatform.ErgoBox import org.ergoplatform.utils.ErgoPropertyTest import org.ergoplatform.utils.generators.WalletGenerators -import scorex.crypto.hash.Digest32 import scorex.util.encode.Base16 import sigmastate.Values.ByteArrayConstant +import sigmastate.eval.Extensions.ArrayByteOps import scala.language.implicitConversions class ScanningPredicateJsonCodecsSpecification extends ErgoPropertyTest with WalletGenerators { - import ScanningPredicateJsonCodecs.{scanningPredicateEncoder, scanningPredicateDecoder} + import ScanningPredicateJsonCodecs.{scanningPredicateDecoder, scanningPredicateEncoder} private implicit def bacFromBytes(bs: Array[Byte]) = ByteArrayConstant(bs) private val complexOr = OrScanningPredicate( ContainsScanningPredicate(ErgoBox.R1, ByteArrayConstant(Array.fill(32)(1: Byte))), EqualsScanningPredicate(ErgoBox.R4, ByteArrayConstant(Array.fill(32)(0: Byte))), - ContainsAssetPredicate(Digest32 @@ Array.fill(32)(0: Byte)) + ContainsAssetPredicate(Array.fill(32)(0: Byte).toTokenId) ) private val complexAnd = AndScanningPredicate( ContainsScanningPredicate(ErgoBox.R1, ByteArrayConstant(Array.fill(32)(1: Byte))), EqualsScanningPredicate(ErgoBox.R4, ByteArrayConstant(Array.fill(32)(1: Byte))), - ContainsAssetPredicate(Digest32 @@ Array.fill(32)(1: Byte)) + ContainsAssetPredicate(Array.fill(32)(1: Byte).toTokenId) ) property("json roundtrip for generated predicate") { @@ -57,7 +57,7 @@ class ScanningPredicateJsonCodecsSpecification extends ErgoPropertyTest with Wal val bs = Base16.decode("02dada811a888cd0dc7a0a41739a3ad9b0f427741fe6ca19700cf1a51200c96bf7").get scanningPredicateDecoder.decodeJson(j).toTry.get == AndScanningPredicate( ContainsScanningPredicate(ErgoBox.R1, bs), - ContainsAssetPredicate(Digest32 @@ bs) + ContainsAssetPredicate(bs.toTokenId) ) } @@ -71,7 +71,7 @@ class ScanningPredicateJsonCodecsSpecification extends ErgoPropertyTest with Wal val bs = Base16.decode("02dada811a888cd0dc7a0a41739a3ad9b0f427741fe6ca19700cf1a51200c96bf7").get scanningPredicateDecoder.decodeJson(j).toTry.get == AndScanningPredicate( ContainsScanningPredicate(ErgoBox.R4, bs), - ContainsAssetPredicate(Digest32 @@ bs) + ContainsAssetPredicate(bs.toTokenId) ) } diff --git a/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSpecification.scala b/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSpecification.scala index 6138f216ad..dcfc049dbe 100644 --- a/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSpecification.scala +++ b/src/test/scala/org/ergoplatform/nodeView/wallet/scanning/ScanningPredicateSpecification.scala @@ -1,17 +1,17 @@ package org.ergoplatform.nodeView.wallet.scanning -import org.ergoplatform.{ErgoScriptPredef, P2PKAddress} +import io.circe.parser._ import org.ergoplatform.ErgoBox.R1 import org.ergoplatform.utils.ErgoPropertyTest import org.ergoplatform.utils.generators.ErgoTransactionGenerators -import scorex.crypto.hash.Digest32 +import org.ergoplatform.wallet.serialization.JsonCodecsWrapper +import org.ergoplatform.{ErgoTreePredef, P2PKAddress} import sigmastate.Values.ByteArrayConstant +import sigmastate.eval.Extensions.ArrayByteOps import sigmastate.helpers.TestingHelpers._ -import scala.util.Random import scala.language.implicitConversions -import io.circe.parser._ -import org.ergoplatform.wallet.serialization.JsonCodecsWrapper +import scala.util.Random class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransactionGenerators { @@ -19,7 +19,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti private implicit def bacFromBytes(bs: Array[Byte]) = ByteArrayConstant(bs) - //helper function to change random byte + /** Helper function to create a new array with changed random byte. */ private def mutateRandomByte(source: Array[Byte]): Array[Byte] = { val sourceModified = source.clone() val idx = Random.nextInt(sourceModified.length) @@ -45,7 +45,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti property("equals - miner prop") { forAll(proveDlogGen) { pk => - val minerProp = ErgoScriptPredef.rewardOutputScript(testDelay, pk) + val minerProp = ErgoTreePredef.rewardOutputScript(testDelay, pk) val mpBytes = minerProp.bytes //look for exact miner script bytes @@ -77,7 +77,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti property("contains - miner prop") { forAll(ergoAddressGen) { p2pkAddress => //look for exact p2pk script bytes - val minerProp = ErgoScriptPredef.rewardOutputScript(testDelay, p2pkAddress.asInstanceOf[P2PKAddress].pubkey) + val minerProp = ErgoTreePredef.rewardOutputScript(testDelay, p2pkAddress.asInstanceOf[P2PKAddress].pubkey) val box = testBox(value = 1, minerProp, creationHeight = 0) val pkBytes = p2pkAddress.asInstanceOf[P2PKAddress].pubkeyBytes @@ -101,7 +101,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti val emptyBox = testBox(value = 1, pk, creationHeight = 0) ContainsAssetPredicate(tokenId).filter(emptyBox) shouldBe false - ContainsAssetPredicate(Digest32 @@ mutateRandomByte(tokenId)).filter(box) shouldBe false + ContainsAssetPredicate(mutateRandomByte(tokenId.toArray).toTokenId).filter(box) shouldBe false } } } @@ -121,7 +121,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti ).filter(box) shouldBe true AndScanningPredicate( - ContainsAssetPredicate(Digest32 @@ mutateRandomByte(tokenId)), + ContainsAssetPredicate(mutateRandomByte(tokenId.toArray).toTokenId), ContainsScanningPredicate(R1, p2pk.contentBytes) ).filter(box) shouldBe false @@ -148,7 +148,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti ).filter(box) shouldBe true OrScanningPredicate( - ContainsAssetPredicate(Digest32 @@ mutateRandomByte(tokenId)), + ContainsAssetPredicate(mutateRandomByte(tokenId.toArray).toTokenId), ContainsScanningPredicate(R1, p2pk.contentBytes) ).filter(box) shouldBe true @@ -158,7 +158,7 @@ class ScanningPredicateSpecification extends ErgoPropertyTest with ErgoTransacti ).filter(box) shouldBe true OrScanningPredicate( - ContainsAssetPredicate(Digest32 @@ mutateRandomByte(tokenId)), + ContainsAssetPredicate(mutateRandomByte(tokenId.toArray).toTokenId), ContainsScanningPredicate(R1, mutateRandomByte(p2pk.contentBytes)) ).filter(box) shouldBe false } diff --git a/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala b/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala index 060e79c9ad..2d5eca8bf7 100644 --- a/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala +++ b/src/test/scala/org/ergoplatform/reemission/ReemissionRulesSpec.scala @@ -1,13 +1,13 @@ package org.ergoplatform.reemission +import org.ergoplatform._ import org.ergoplatform.settings.{MonetarySettings, ReemissionSettings} import org.ergoplatform.utils.{ErgoPropertyTest, ErgoTestConstants} -import org.ergoplatform._ -import scorex.crypto.hash.{Blake2b256, Digest32} +import scorex.crypto.hash.Blake2b256 import scorex.util.ModifierId import sigmastate.AvlTreeData import sigmastate.TrivialProp.TrueProp -import sigmastate.eval.{Colls, Digest32RType} +import sigmastate.eval.{Colls, Digest32Coll} import sigmastate.helpers.TestingHelpers.testBox import sigmastate.helpers.{ContextEnrichingTestProvingInterpreter, ErgoLikeContextTesting, ErgoLikeTestInterpreter} import sigmastate.interpreter.Interpreter.emptyEnv @@ -30,7 +30,7 @@ class ReemissionRulesSpec extends ErgoPropertyTest with ErgoTestConstants { private val rr = new ReemissionRules(rs) - private val reemissionBoxAssets = Colls.fromItems((Digest32 @@ rs.reemissionNftIdBytes) -> 1L) + private val reemissionBoxAssets = Colls.fromItems((Digest32Coll @@ rs.reemissionNftIdBytes) -> 1L) private val fakeMessage = Blake2b256("Hello World") @@ -67,7 +67,7 @@ class ReemissionRulesSpec extends ErgoPropertyTest with ErgoTestConstants { property("reemissionBoxProp - spending path") { val minerPk = prover.dlogSecrets.head.publicImage val pkBytes = minerPk.pkBytes - val minerProp = ErgoScriptPredef.rewardOutputScript(ms.minerRewardDelay, minerPk) + val minerProp = ErgoTreePredef.rewardOutputScript(ms.minerRewardDelay, minerPk) val currentHeight = rs.reemissionStartHeight val nextHeight = currentHeight + 1 @@ -122,7 +122,7 @@ class ReemissionRulesSpec extends ErgoPropertyTest with ErgoTestConstants { checkRewardsTx(nextHeight, pkBytes6, inputBoxes, spendingTransaction, false) // we modify reward delay here, not PK - val minerProp7 = ErgoScriptPredef.rewardOutputScript(ms.minerRewardDelay - 1, minerPk) + val minerProp7 = ErgoTreePredef.rewardOutputScript(ms.minerRewardDelay - 1, minerPk) val minerBox7 = new ErgoBoxCandidate(reemissionReward, minerProp7, nextHeight) val spendingTransaction7 = ErgoLikeTransaction(inputs, IndexedSeq(newReemissionBox, minerBox7)) checkRewardsTx(nextHeight, pkBytes, inputBoxes, spendingTransaction7, false) @@ -185,9 +185,11 @@ class ReemissionRulesSpec extends ErgoPropertyTest with ErgoTestConstants { // pay-2-reemission box can be spent only with a box with reemission NFT as input #0 val reemissionBoxAssets6 = Colls.fromItems( - (Digest32 @@ rs.reemissionNftIdBytes.reverse) -> 1L + (Digest32Coll @@ rs.reemissionNftIdBytes.reverse) -> 1L ) - val newReemissionBox6 = new ErgoBoxCandidate(reemissionBox.value + mergedValue - feeValue, prop, currentHeight, reemissionBoxAssets6) + val newReemissionBox6 = new ErgoBoxCandidate( + reemissionBox.value + mergedValue - feeValue, + prop, currentHeight, reemissionBoxAssets6) val spendingTransaction6 = ErgoLikeTransaction(inputs, IndexedSeq(newReemissionBox6, feeBox)) val ctx = ErgoLikeContextTesting( diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala index e4d9de0836..d1a7518e16 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanity.scala @@ -7,6 +7,7 @@ import org.ergoplatform.modifiers.history.BlockTransactions import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnconfirmedTransaction} import org.ergoplatform.modifiers.{BlockSection, ErgoFullBlock} import org.ergoplatform.network.{ErgoNodeViewSynchronizer, ErgoSyncTracker} +import org.ergoplatform.nodeView.NodeViewSynchronizerTests import org.ergoplatform.nodeView.history.{ErgoHistory, ErgoSyncInfo, ErgoSyncInfoMessageSpec} import org.ergoplatform.nodeView.mempool.ErgoMemPool import org.ergoplatform.nodeView.state.{DigestState, ErgoState, UtxoState} diff --git a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala index 4b8b3685ac..74955421b2 100644 --- a/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala +++ b/src/test/scala/org/ergoplatform/sanity/ErgoSanityDigest.scala @@ -28,7 +28,7 @@ class ErgoSanityDigest extends ErgoSanity[DIGEST_ST] { override val stateGen: Gen[WrappedDigestState] = { boxesHolderGen.map(WrappedUtxoState(_, createTempDir, None, parameters, settings)).map { wus => - val digestState = DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, stateConstants) + val digestState = DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, settings) new WrappedDigestState(digestState, wus, settings) } } diff --git a/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala b/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala index 48e9cb2732..571586ca26 100644 --- a/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala +++ b/src/test/scala/org/ergoplatform/serialization/JsonSerializationSpec.scala @@ -4,19 +4,19 @@ import io.circe.syntax._ import io.circe.{ACursor, Decoder, Encoder, Json} import org.ergoplatform.ErgoBox import org.ergoplatform.ErgoBox.NonMandatoryRegisterId +import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.http.api.ApiEncoderOption.HideDetails.implicitValue import org.ergoplatform.http.api.ApiEncoderOption.{Detalization, ShowDetails} -import org.ergoplatform.http.api.ApiCodecs import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.popow.NipopowProof import org.ergoplatform.modifiers.mempool.UnsignedErgoTransaction import org.ergoplatform.nodeView.wallet.requests._ +import org.ergoplatform.sdk.wallet.secrets.{DhtSecretKey, DlogSecretKey} import org.ergoplatform.settings.{Algos, ErgoSettings} import org.ergoplatform.utils.ErgoPropertyTest import org.ergoplatform.utils.generators.WalletGenerators import org.ergoplatform.wallet.Constants.ScanId import org.ergoplatform.wallet.boxes.TrackedBox -import org.ergoplatform.wallet.secrets.{DhtSecretKey, DlogSecretKey} import org.scalatest.Inspectors import sigmastate.SType import sigmastate.Values.{ErgoTree, EvaluatedValue} @@ -28,7 +28,7 @@ class JsonSerializationSpec extends ErgoPropertyTest with WalletGenerators with property("ErgoFullBlock should be encoded into JSON and decoded back correctly") { - val (st, bh) = createUtxoState(parameters) + val (st, bh) = createUtxoState(settings) val block: ErgoFullBlock = validFullBlock(parentOpt = None, st, bh) val blockJson: Json = block.asJson diff --git a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala index 89d394d17b..c81ac939a4 100644 --- a/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala +++ b/src/test/scala/org/ergoplatform/settings/ErgoSettingsSpecification.scala @@ -24,6 +24,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { StateType.Utxo, verifyTransactions = true, 1000, + utxoSettings = UtxoSettings(false, 0, 2), poPoWBootstrap = false, 10, mining = true, @@ -73,6 +74,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { StateType.Utxo, verifyTransactions = true, 12, + utxoSettings = UtxoSettings(false, 0, 2), poPoWBootstrap = false, 10, mining = true, @@ -115,6 +117,7 @@ class ErgoSettingsSpecification extends ErgoPropertyTest { StateType.Utxo, verifyTransactions = true, 13, + utxoSettings = UtxoSettings(false, 0, 2), poPoWBootstrap = false, 10, mining = true, diff --git a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala index c234dab7e8..aaaf6755bb 100644 --- a/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala +++ b/src/test/scala/org/ergoplatform/tools/ChainGenerator.scala @@ -44,7 +44,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val prover = defaultProver val minerPk = prover.hdKeys.head.publicImage val selfAddressScript = P2PKAddress(minerPk).script - val minerProp = ErgoScriptPredef.rewardOutputScript(RewardDelay, minerPk) + val minerProp = ErgoTreePredef.rewardOutputScript(RewardDelay, minerPk) val pow = new AutolykosPowScheme(powScheme.k, powScheme.n) val blockInterval = 2.minute @@ -59,7 +59,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val txCostLimit = initSettings.nodeSettings.maxTransactionCost val txSizeLimit = initSettings.nodeSettings.maxTransactionSize val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(StateType.Utxo, verifyTransactions = true, - -1, poPoWBootstrap = false, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, + -1, UtxoSettings(false, 0, 2), poPoWBootstrap = false, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 20, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false) @@ -78,7 +78,7 @@ object ChainGenerator extends App with ErgoTestHelpers { val history = ErgoHistory.readOrGenerate(fullHistorySettings)(null) HistoryTestHelpers.allowToApplyOldBlocks(history) - val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, StateConstants(fullHistorySettings)) + val (state, _) = ErgoState.generateGenesisUtxoState(stateDir, fullHistorySettings) log.info(s"Going to generate a chain at ${dir.getAbsoluteFile} starting from ${history.bestFullBlockOpt}") val chain = loop(state, None, None, Seq()) diff --git a/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala b/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala index 70b6bc8e9c..c09e9c6bd9 100644 --- a/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala +++ b/src/test/scala/org/ergoplatform/tools/FeeSimulator.scala @@ -1,15 +1,15 @@ package org.ergoplatform.tools import org.ergoplatform.modifiers.mempool.ErgoTransaction +import org.ergoplatform.settings.Constants._ +import org.ergoplatform.settings.LaunchParameters._ import org.ergoplatform.{ErgoBoxCandidate, Input} import scorex.crypto.authds.ADKey -import scorex.crypto.hash.Digest32 import scorex.utils.Random import sigmastate.basics.DLogProtocol.DLogProverInput -import sigmastate.interpreter.{ProverResult, ContextExtension} +import sigmastate.eval.Extensions.ArrayByteOps import sigmastate.eval._ -import org.ergoplatform.settings.LaunchParameters._ -import org.ergoplatform.settings.Constants._ +import sigmastate.interpreter.{ContextExtension, ProverResult} object FeeSimulator extends App { @@ -23,7 +23,9 @@ object FeeSimulator extends App { val input = Input(ADKey @@ Random.randomBytes(32), ProverResult(Random.randomBytes(65), ContextExtension(Map()))) val creationHeight: Int = 100000 - val box1 = new ErgoBoxCandidate(scala.util.Random.nextLong(), k1, creationHeight, Colls.fromItems((Digest32 @@ Random.randomBytes(32)) -> scala.util.Random.nextLong())) + val box1 = new ErgoBoxCandidate( + scala.util.Random.nextLong(), k1, creationHeight, + Colls.fromItems(Random.randomBytes(32).toTokenId -> scala.util.Random.nextLong())) val box2 = new ErgoBoxCandidate(scala.util.Random.nextLong(), k2, creationHeight) val simpleTx = ErgoTransaction(IndexedSeq(input, input), IndexedSeq(box1, box2)) diff --git a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala index 707035bf79..6fe0c8f310 100644 --- a/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala +++ b/src/test/scala/org/ergoplatform/utils/ErgoTestConstants.scala @@ -6,7 +6,8 @@ import org.ergoplatform.mining.emission.EmissionRules import org.ergoplatform.mining.{AutolykosPowScheme, DefaultFakePowScheme} import org.ergoplatform.modifiers.history.extension.ExtensionCandidate import org.ergoplatform.modifiers.history.popow.NipopowAlgos -import org.ergoplatform.nodeView.state.{ErgoState, ErgoStateContext, StateConstants, StateType, UpcomingStateContext} +import org.ergoplatform.nodeView.state._ +import org.ergoplatform.sdk.wallet.secrets.ExtendedSecretKey import org.ergoplatform.settings.Constants.HashLength import org.ergoplatform.settings.Parameters.{MaxBlockCostIncrease, MinValuePerByteIncrease} import org.ergoplatform.settings.ValidationRules._ @@ -14,16 +15,15 @@ import org.ergoplatform.settings._ import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.{ErgoInterpreter, ErgoProvingInterpreter} import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.secrets.ExtendedSecretKey -import org.ergoplatform.{DataInput, ErgoBox, ErgoScriptPredef} +import org.ergoplatform.{DataInput, ErgoBox, ErgoTreePredef} import scorex.core.app.Version import scorex.core.network.PeerSpec import scorex.crypto.authds.ADDigest import scorex.crypto.hash.Digest32 import scorex.util.ScorexLogging import sigmastate.Values.ErgoTree +import sigmastate.basics.CryptoConstants.EcPointType import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog} -import sigmastate.interpreter.CryptoConstants.EcPointType import sigmastate.interpreter.{ContextExtension, ProverResult} import scala.concurrent.duration._ @@ -58,9 +58,8 @@ trait ErgoTestConstants extends ScorexLogging { val emission: EmissionRules = settings.chainSettings.emissionRules val coinsTotal: Long = emission.coinsTotal - val stateConstants: StateConstants = StateConstants(settings) val genesisStateDigest: ADDigest = settings.chainSettings.genesisStateDigest - val feeProp: ErgoTree = ErgoScriptPredef.feeProposition(emission.settings.minerRewardDelay) + val feeProp: ErgoTree = ErgoTreePredef.feeProposition(emission.settings.minerRewardDelay) val emptyProverResult: ProverResult = ProverResult(Array.emptyByteArray, ContextExtension.empty) lazy val defaultSeed: Array[Byte] = Mnemonic.toSeed(settings.walletSettings.testMnemonic.fold[SecretString](SecretString.empty())(SecretString.create(_))) diff --git a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala index 5937979e48..517563d142 100644 --- a/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala +++ b/src/test/scala/org/ergoplatform/utils/HistoryTestHelpers.scala @@ -46,7 +46,7 @@ trait HistoryTestHelpers extends ErgoPropertyTest { val txCostLimit = initSettings.nodeSettings.maxTransactionCost val txSizeLimit = initSettings.nodeSettings.maxTransactionSize val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(stateType, verifyTransactions, blocksToKeep, - PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, + UtxoSettings(false, 0, 2), poPoWBootstrap = PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second, miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false diff --git a/src/test/scala/org/ergoplatform/utils/Stubs.scala b/src/test/scala/org/ergoplatform/utils/Stubs.scala index 66ffffeea1..17735eec72 100644 --- a/src/test/scala/org/ergoplatform/utils/Stubs.scala +++ b/src/test/scala/org/ergoplatform/utils/Stubs.scala @@ -22,15 +22,15 @@ import org.ergoplatform.nodeView.wallet._ import org.ergoplatform.nodeView.wallet.persistence.WalletDigest import org.ergoplatform.nodeView.wallet.scanning.Scan import org.ergoplatform.sanity.ErgoSanity.HT +import org.ergoplatform.sdk.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.ergoplatform.settings.Constants.HashLength import org.ergoplatform.settings._ import org.ergoplatform.utils.generators.{ChainGenerator, ErgoGenerators, ErgoTransactionGenerators} -import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.Constants.{PaymentsScanId, ScanId} import org.ergoplatform.wallet.boxes.{ChainStatus, TrackedBox} +import org.ergoplatform.wallet.interface4j.SecretString import org.ergoplatform.wallet.interpreter.ErgoProvingInterpreter import org.ergoplatform.wallet.mnemonic.Mnemonic -import org.ergoplatform.wallet.secrets.{DerivationPath, ExtendedSecretKey} import org.ergoplatform.wallet.utils.TestFileUtils import org.scalacheck.Gen import scorex.core.app.Version @@ -58,7 +58,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val digestState: DigestState = { boxesHolderGen.map(WrappedUtxoState(_, createTempDir, None, parameters, settings)).map { wus => - DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, stateConstants) + DigestState.create(Some(wus.version), Some(wus.rootDigest), createTempDir, settings) } }.sample.value @@ -257,7 +257,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with sender() ! Success(tx) case SignTransaction(tx, secrets, hints, boxesToSpendOpt, dataBoxesOpt) => - val sc = ErgoStateContext.empty(stateConstants, parameters) + val sc = ErgoStateContext.empty(settings, parameters) sender() ! ergoWalletService.signTransaction(Some(prover), tx, secrets, hints, boxesToSpendOpt, dataBoxesOpt, parameters, sc) { boxId => utxoState.versionedBoxHolder.get(ByteArrayWrapper(boxId)) } @@ -370,7 +370,7 @@ trait Stubs extends ErgoGenerators with ErgoTestHelpers with ChainGenerator with val txCostLimit = initSettings.nodeSettings.maxTransactionCost val txSizeLimit = initSettings.nodeSettings.maxTransactionSize val nodeSettings: NodeConfigurationSettings = NodeConfigurationSettings(stateType, verifyTransactions, blocksToKeep, - PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, + UtxoSettings(false, 0, 2), poPoWBootstrap = PoPoWBootstrap, minimalSuffix, mining = false, txCostLimit, txSizeLimit, useExternalMiner = false, internalMinersCount = 1, internalMinerPollingInterval = 1.second,miningPubKeyHex = None, offlineGeneration = false, 200, 5.minutes, 100000, 1.minute, mempoolSorting = SortingOption.FeePerByte, rebroadcastCount = 200, 1000000, 100, adProofsSuffixLength = 112*1024, extraIndex = false diff --git a/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala b/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala index ac6df7067d..b76ac7485b 100644 --- a/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala +++ b/src/test/scala/org/ergoplatform/utils/WalletTestOps.scala @@ -10,21 +10,22 @@ import org.ergoplatform.nodeView.state.{ErgoState, UtxoState} import org.ergoplatform.nodeView.wallet.ErgoWallet import org.ergoplatform.nodeView.wallet.IdUtils._ import org.ergoplatform.nodeView.wallet.persistence.WalletDigest +import org.ergoplatform.sdk.wallet.TokensMap import org.ergoplatform.settings.Constants import org.ergoplatform.utils.fixtures.WalletFixture -import org.ergoplatform.wallet.TokensMap import scorex.crypto.authds.ADKey -import scorex.crypto.hash.{Blake2b256, Digest32} -import scorex.util.{ModifierId, bytesToId} +import scorex.crypto.hash.Blake2b256 +import scorex.util.ModifierId import sigmastate.Values.ErgoTree import sigmastate.basics.DLogProtocol.ProveDlog import sigmastate.eval.Extensions._ import sigmastate.eval._ import sigmastate.interpreter.ProverResult +import special.collection.Extensions._ trait WalletTestOps extends NodeViewBaseOps { - def newAssetIdStub: TokenId = Blake2b256.hash("new_asset") + def newAssetIdStub: TokenId = Blake2b256.hash("new_asset").toTokenId def withFixture[T](test: WalletFixture => T): T = new WalletFixture(settings, parameters, getCurrentView(_).vault).apply(test) @@ -51,10 +52,10 @@ trait WalletTestOps extends NodeViewBaseOps { tx.outputs.filter(_.propositionBytes.containsSlice(org.ergoplatform.mining.groupElemToBytes(pk.value))) def assetAmount(boxes: Seq[ErgoBoxCandidate]): Seq[(ModifierId, Long)] = - assetsByTokenId(boxes).map { case (tokenId, sum) => (bytesToId(tokenId), sum) }.toArray[(ModifierId, Long)] + assetsByTokenId(boxes).map { case (tokenId, sum) => (tokenId.toModifierId, sum) }.toArray[(ModifierId, Long)] def toAssetMap(assetSeq: Seq[(TokenId, Long)]): TokensMap = - assetSeq.map { case (tokenId, sum) => (bytesToId(tokenId), sum) }.toMap + assetSeq.map { case (tokenId, sum) => (tokenId.toModifierId, sum) }.toMap def assetsByTokenId(boxes: Seq[ErgoBoxCandidate]): Map[TokenId, Long] = { boxes @@ -78,7 +79,7 @@ trait WalletTestOps extends NodeViewBaseOps { def makeGenesisTxWithAsset(publicKey: ProveDlog, issueAsset: Boolean): ErgoTransaction = { val inputs = IndexedSeq(new Input(genesisEmissionBox.id, emptyProverResult)) val assets: Seq[(TokenId, Long)] = if (issueAsset) { - Seq((Digest32 @@@ inputs.head.boxId) -> 1L) + Seq(inputs.head.boxId.toTokenId -> 1L) } else { Seq.empty } @@ -125,10 +126,10 @@ trait WalletTestOps extends NodeViewBaseOps { } private def replaceNewAssetStub(assets: Seq[(TokenId, Long)], inputs: Seq[Input]): Seq[(TokenId, Long)] = { - def isNewAsset(tokenId: TokenId, value: Long): Boolean = java.util.Arrays.equals(tokenId, newAssetIdStub) + def isNewAsset(tokenId: TokenId, value: Long): Boolean = tokenId == newAssetIdStub val (newAsset, spentAssets) = assets.partition((isNewAsset _).tupled) - newAsset.map(Digest32 @@@ inputs.head.boxId -> _._2) ++ spentAssets + newAsset.map(inputs.head.boxId.toTokenId -> _._2) ++ spentAssets } def randomNewAsset: Seq[(TokenId, Long)] = Seq(newAssetIdStub -> randomLong()) diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala index f24d6e3718..2053f11e0f 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoGenerators.scala @@ -23,10 +23,10 @@ import scorex.crypto.authds.{ADDigest, SerializedAdProof} import scorex.crypto.hash.Digest32 import scorex.testkit.generators.CoreGenerators import sigmastate.Values.ErgoTree +import sigmastate.basics.CryptoConstants.EcPointType import sigmastate.basics.DLogProtocol.{DLogProverInput, ProveDlog} -import sigmastate.basics.{DiffieHellmanTupleProverInput, ProveDHTuple} -import sigmastate.interpreter.CryptoConstants.EcPointType -import sigmastate.interpreter.{CryptoConstants, ProverResult} +import sigmastate.basics.{CryptoConstants, DiffieHellmanTupleProverInput, ProveDHTuple} +import sigmastate.interpreter.ProverResult import scala.util.Random diff --git a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala index 6edb2f67d0..7630d2cdbe 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ErgoTransactionGenerators.scala @@ -3,24 +3,24 @@ package org.ergoplatform.utils.generators import org.ergoplatform.ErgoBox.TokenId import org.ergoplatform.modifiers.ErgoFullBlock import org.ergoplatform.modifiers.history.BlockTransactions +import org.ergoplatform.modifiers.history.header.Header import org.ergoplatform.modifiers.mempool.{ErgoTransaction, UnsignedErgoTransaction} import org.ergoplatform.nodeView.history.ErgoHistory import org.ergoplatform.nodeView.state.wrapped.WrappedUtxoState import org.ergoplatform.nodeView.state.{BoxHolder, ErgoStateContext, VotingData} import org.ergoplatform.nodeView.wallet.requests.{ExternalSecret, TransactionSigningRequest} import org.ergoplatform.nodeView.wallet.{AugWalletTransaction, WalletTransaction} +import org.ergoplatform.sdk.wallet.secrets.{DhtSecretKey, DlogSecretKey} import org.ergoplatform.settings.Parameters._ import org.ergoplatform.settings.{Constants, Parameters} import org.ergoplatform.utils.{BoxUtils, RandomLike, RandomWrapper} -import org.ergoplatform.wallet.Constants.{MaxAssetsPerBox, ScanId} -import org.ergoplatform.wallet.secrets.{DhtSecretKey, DlogSecretKey} -import org.ergoplatform.UnsignedInput -import org.ergoplatform.modifiers.history.header.Header +import org.ergoplatform.sdk.wallet.Constants.MaxAssetsPerBox import org.ergoplatform.wallet.interpreter.TransactionHintsBag import org.ergoplatform.wallet.utils.Generators -import org.ergoplatform.{DataInput, ErgoAddress, ErgoAddressEncoder, ErgoBox, ErgoBoxCandidate, Input, P2PKAddress} +import org.ergoplatform._ +import org.ergoplatform.wallet.Constants.ScanId import org.scalacheck.Gen -import scorex.crypto.hash.{Blake2b256, Digest32} +import scorex.crypto.hash.Blake2b256 import scorex.db.ByteArrayWrapper import scorex.util.encode.Base16 import sigmastate.Values.ErgoTree @@ -129,7 +129,7 @@ trait ErgoTransactionGenerators extends ErgoGenerators with Generators { val inputSum = boxesToSpend.map(_.value).reduce(Math.addExact(_, _)) val assetsMap: mutable.Map[ByteArrayWrapper, Long] = mutable.Map(boxesToSpend.flatMap(_.additionalTokens.toArray).map { case (bs, amt) => - ByteArrayWrapper(bs) -> amt + ByteArrayWrapper(bs.toArray) -> amt }: _*) //randomly creating a new asset @@ -195,7 +195,7 @@ trait ErgoTransactionGenerators extends ErgoGenerators with Generators { } val newBoxes = outputAmounts.zip(tokenAmounts.toIndexedSeq).map { case (amt, tokens) => - val normalizedTokens = tokens.toSeq.map(t => (Digest32 @@ t._1.data) -> t._2) + val normalizedTokens = tokens.toSeq.map(t => t._1.data.toTokenId -> t._2) testBox(amt, outputsProposition, 0, normalizedTokens) } val inputs = boxesToSpend.map(b => Input(b.id, emptyProverResult)) @@ -223,7 +223,7 @@ trait ErgoTransactionGenerators extends ErgoGenerators with Generators { def disperseTokens(inputsCount: Int, tokensCount: Byte): Gen[IndexedSeq[Seq[(TokenId, Long)]]] = { val tokensDistribution = mutable.IndexedSeq.fill(inputsCount)(Seq[(TokenId, Long)]()) (1 to tokensCount).foreach { i => - val (id, amt) = Blake2b256(s"$i" + Random.nextString(5)) -> (Random.nextInt(Int.MaxValue).toLong + 100) + val (id, amt) = Blake2b256(s"$i" + Random.nextString(5)).toTokenId -> (Random.nextInt(Int.MaxValue).toLong + 100) val idx = i % tokensDistribution.size val s = tokensDistribution(idx) tokensDistribution(idx) = s :+ (id -> amt) @@ -276,6 +276,9 @@ trait ErgoTransactionGenerators extends ErgoGenerators with Generators { lazy val validErgoTransactionWithAssetsGen: Gen[(IndexedSeq[ErgoBox], ErgoTransaction)] = validErgoTransactionGenTemplate(minAssets = 1) + def boxesHolderGenOfSize(numBoxes:Int): Gen[BoxHolder] = Gen.listOfN(numBoxes, ergoBoxGenForTokens(Seq(), trueLeafGen)) + .map(l => BoxHolder(l)) + lazy val boxesHolderGen: Gen[BoxHolder] = Gen.listOfN(2000, ergoBoxGenForTokens(Seq(), trueLeafGen)) .map(l => BoxHolder(l)) diff --git a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala index 31f60d02cb..b579b50cd7 100644 --- a/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala +++ b/src/test/scala/org/ergoplatform/utils/generators/ValidBlocksGenerators.scala @@ -9,7 +9,7 @@ import org.ergoplatform.modifiers.history.popow.NipopowAlgos import org.ergoplatform.modifiers.mempool.ErgoTransaction import org.ergoplatform.nodeView.state._ import org.ergoplatform.nodeView.state.wrapped.WrappedUtxoState -import org.ergoplatform.settings.{Algos, Constants, Parameters} +import org.ergoplatform.settings.{Algos, Constants, ErgoSettings, Parameters} import org.ergoplatform.utils.{LoggingUtil, RandomLike, RandomWrapper} import org.ergoplatform.wallet.utils.TestFileUtils import org.scalatest.matchers.should.Matchers @@ -26,19 +26,15 @@ import scala.util.{Failure, Random, Success} trait ValidBlocksGenerators extends TestkitHelpers with TestFileUtils with Matchers with ChainGenerator with ErgoTransactionGenerators { - def createUtxoState(parameters: Parameters): (UtxoState, BoxHolder) = { - createUtxoState(StateConstants(settings), parameters) - } - - def createUtxoState(constants: StateConstants, parameters: Parameters): (UtxoState, BoxHolder) = { - ErgoState.generateGenesisUtxoState(createTempDir, constants) + def createUtxoState(settings: ErgoSettings): (UtxoState, BoxHolder) = { + ErgoState.generateGenesisUtxoState(createTempDir, settings) } def createUtxoState(bh: BoxHolder, parameters: Parameters): UtxoState = - UtxoState.fromBoxHolder(bh, None, createTempDir, stateConstants, parameters) + UtxoState.fromBoxHolder(bh, None, createTempDir, settings, parameters) - def createDigestState(version: VersionTag, digest: ADDigest, parameters: Parameters): DigestState = - DigestState.create(Some(version), Some(digest), createTempDir, stateConstants) + def createDigestState(version: VersionTag, digest: ADDigest): DigestState = + DigestState.create(Some(version), Some(digest), createTempDir, settings) def validTransactionsFromBoxHolder(boxHolder: BoxHolder): (Seq[ErgoTransaction], BoxHolder) = validTransactionsFromBoxHolder(boxHolder, new RandomWrapper) @@ -70,7 +66,7 @@ trait ValidBlocksGenerators val currentSize = acc.map(_.size).sum val averageSize = if (currentSize > 0) currentSize / acc.length else 1000 val customTokens = (stateBoxes ++ selfBoxes).flatMap(_.additionalTokens.toArray) - val customTokensNum = customTokens.map(ct => ByteArrayWrapper(ct._1)).toSet.size + val customTokensNum = customTokens.map(ct => ByteArrayWrapper(ct._1.toArray)).toSet.size val issueNew = customTokensNum == 0 stateBoxes.find(isEmissionBox) match {