NODE-2548 Orderly storage of NFTs (#3801)

benchmark/src/test/scala/com/wavesplatform/RollbackBenchmark.scala

@@ -1,7 +1,6 @@
 package com.wavesplatform
 
 import java.io.File
-
 import com.google.common.primitives.Ints
 import com.google.protobuf.ByteString
 import com.wavesplatform.account.{Address, AddressScheme, KeyPair}
@@ -17,6 +16,8 @@ import com.wavesplatform.transaction.assets.IssueTransaction
 import com.wavesplatform.utils.{NTP, ScorexLogging}
 import monix.reactive.Observer
 
+import scala.collection.immutable.VectorMap
+
 object RollbackBenchmark extends ScorexLogging {
   def main(args: Array[String]): Unit = {
     val settings      = Application.loadApplicationConfig(Some(new File(args(0))))
@@ -66,7 +67,7 @@ object RollbackBenchmark extends ScorexLogging {
       )
       .explicitGet()
 
-    val map = assets.map(it => IssuedAsset(it.id()) -> 1L).toMap
+    val map = assets.map(it => IssuedAsset(it.id()) -> 1L).to(VectorMap)
     val portfolios = for {
       address <- addresses
     } yield address -> Portfolio(assets = map)
@@ -85,7 +86,7 @@ object RollbackBenchmark extends ScorexLogging {
       Block
         .buildAndSign(2.toByte, time.getTimestamp(), genesisBlock.id(), 1000, Block.GenesisGenerationSignature, Seq.empty, issuer, Seq.empty, -1)
         .explicitGet()
-    val nextDiff = Diff(portfolios = addresses.map(_ -> Portfolio(1, assets = Map(IssuedAsset(assets.head.id()) -> 1L))).toMap)
+    val nextDiff = Diff(portfolios = addresses.map(_ -> Portfolio(1, assets = VectorMap(IssuedAsset(assets.head.id()) -> 1L))).toMap)
 
     log.info("Appending next block")
     levelDBWriter.append(nextDiff, 0, 0, None, ByteStr.empty, nextBlock)

node-it/src/test/scala/com/wavesplatform/it/api/AsyncHttpApi.scala

@@ -31,18 +31,7 @@ import com.wavesplatform.transaction.lease.{LeaseCancelTransaction, LeaseTransac
 import com.wavesplatform.transaction.smart.{InvokeExpressionTransaction, InvokeScriptTransaction, SetScriptTransaction}
 import com.wavesplatform.transaction.transfer.*
 import com.wavesplatform.transaction.transfer.MassTransferTransaction.{ParsedTransfer, Transfer}
-import com.wavesplatform.transaction.{
-  Asset,
-  CreateAliasTransaction,
-  DataTransaction,
-  Proofs,
-  TxDecimals,
-  TxExchangeAmount,
-  TxExchangePrice,
-  TxNonNegativeAmount,
-  TxPositiveAmount,
-  TxVersion
-}
+import com.wavesplatform.transaction.{Asset, CreateAliasTransaction, DataTransaction, Proofs, TxDecimals, TxExchangeAmount, TxExchangePrice, TxNonNegativeAmount, TxPositiveAmount, TxVersion}
 import org.asynchttpclient.*
 import org.asynchttpclient.Dsl.{delete as _delete, get as _get, post as _post, put as _put}
 import org.asynchttpclient.util.HttpConstants.ResponseStatusCodes.OK_200
@@ -51,6 +40,7 @@ import org.scalatest.{Assertions, matchers}
 import play.api.libs.json.*
 import play.api.libs.json.Json.{stringify, toJson}
 
+import scala.collection.immutable.VectorMap
 import scala.compat.java8.FutureConverters.*
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -939,9 +929,9 @@ object AsyncHttpApi extends Assertions {
         .as[Seq[BalanceHistory]](amountsAsStrings)
     }
 
-    implicit val assetMapReads: Reads[Map[IssuedAsset, Long]] = implicitly[Reads[Map[String, Long]]].map(_.map { case (k, v) =>
+    implicit val assetMapReads: Reads[VectorMap[IssuedAsset, Long]] = implicitly[Reads[Map[String, Long]]].map(_.map { case (k, v) =>
       IssuedAsset(ByteStr.decodeBase58(k).get) -> v
-    })
+    }.to(VectorMap))
     implicit val leaseBalanceFormat: Reads[LeaseBalance] = Json.reads[LeaseBalance]
     implicit val portfolioFormat: Reads[Portfolio]       = Json.reads[Portfolio]
 

node/src/main/scala/com/wavesplatform/Explorer.scala

@@ -3,7 +3,6 @@ package com.wavesplatform
 import java.io.File
 import java.nio.ByteBuffer
 import java.util
-
 import com.google.common.primitives.Longs
 import com.wavesplatform.account.Address
 import com.wavesplatform.api.common.AddressPortfolio
@@ -20,6 +19,7 @@ import com.wavesplatform.utils.ScorexLogging
 import org.iq80.leveldb.DB
 
 import scala.annotation.tailrec
+import scala.collection.immutable.VectorMap
 import scala.collection.mutable
 import scala.jdk.CollectionConverters.*
 
@@ -31,7 +31,7 @@ object Explorer extends ScorexLogging {
     Portfolio(
       blockchain.balance(address),
       blockchain.leaseBalance(address),
-      db.withResource(r => AddressPortfolio.assetBalanceIterator(r, address, Diff.empty, _ => true).toMap)
+      db.withResource(r => AddressPortfolio.assetBalanceIterator(r, address, Diff.empty, _ => true).to(VectorMap))
     )
 
   def main(argsRaw: Array[String]): Unit = {

node/src/main/scala/com/wavesplatform/database/LevelDBWriter.scala

@@ -310,7 +310,7 @@ abstract class LevelDBWriter private[database] (
       balanceThreshold: Int
   ): Unit = {
     val changedAssetBalances = MultimapBuilder.hashKeys().hashSetValues().build[IssuedAsset, java.lang.Long]()
-    val updatedNftLists      = MultimapBuilder.hashKeys().hashSetValues().build[java.lang.Long, IssuedAsset]()
+    val updatedNftLists      = MultimapBuilder.hashKeys().linkedHashSetValues().build[java.lang.Long, IssuedAsset]()
 
     for ((addressId, updatedBalances) <- balances) {
       for ((asset, balance) <- updatedBalances) {
@@ -350,7 +350,7 @@ abstract class LevelDBWriter private[database] (
       val previousNftCount = rw.get(kCount)
       rw.put(kCount, previousNftCount + nftIds.size())
       for ((id, idx) <- nftIds.asScala.zipWithIndex) {
-        rw.put(Keys.nftAt(AddressId(addressId.toLong), idx, id), Some(()))
+        rw.put(Keys.nftAt(AddressId(addressId.toLong), previousNftCount + idx, id), Some(()))
       }
     }
 

node/src/main/scala/com/wavesplatform/state/DiffToStateApplier.scala

@@ -5,6 +5,8 @@ import com.wavesplatform.account.Address
 import com.wavesplatform.transaction.Asset
 import com.wavesplatform.transaction.Asset.Waves
 
+import scala.collection.immutable.VectorMap
+
 /**
   * A set of functions that apply diff
   * to the blockchain and return new
@@ -22,7 +24,7 @@ object DiffToStateApplier {
 
     for ((address, portfolioDiff) <- diff.portfolios) {
       // balances for address
-      val bs = Map.newBuilder[Asset, Long]
+      val bs = VectorMap.newBuilder[Asset, Long]
 
       if (portfolioDiff.balance != 0) {
         bs += Waves -> (blockchain.balance(address, Waves) + portfolioDiff.balance)

node/src/main/scala/com/wavesplatform/state/Portfolio.scala

@@ -1,12 +1,12 @@
 package com.wavesplatform.state
 
-import cats.Monoid
-import cats.implicits.*
 import com.wavesplatform.state.diffs.BlockDiffer.Fraction
 import com.wavesplatform.transaction.Asset
 import com.wavesplatform.transaction.Asset.*
 
-case class Portfolio(balance: Long = 0L, lease: LeaseBalance = LeaseBalance.empty, assets: Map[IssuedAsset, Long] = Map.empty) {
+import scala.collection.immutable.VectorMap
+
+case class Portfolio(balance: Long = 0L, lease: LeaseBalance = LeaseBalance.empty, assets: VectorMap[IssuedAsset, Long] = VectorMap.empty) {
   import Portfolio.*
   lazy val effectiveBalance: Either[String, Long] = safeSum(balance, lease.in, "Effective balance").map(_ - lease.out)
   lazy val spendableBalance: Long                 = balance - lease.out
@@ -33,12 +33,12 @@ object Portfolio {
     try Right(Math.addExact(a, b))
     catch { case _: ArithmeticException => Left(error) }
 
-  def combineAssets(a: Map[IssuedAsset, Long], b: Map[IssuedAsset, Long]): Either[String, Map[IssuedAsset, Long]] = {
+  def combineAssets(a: VectorMap[IssuedAsset, Long], b: VectorMap[IssuedAsset, Long]): Either[String, VectorMap[IssuedAsset, Long]] = {
     if (a.isEmpty) Right(b)
     else if (b.isEmpty) Right(a)
     else
-      b.foldLeft[Either[String, Map[IssuedAsset, Long]]](Right(a)) {
-        case (Right(seed), kv @ (asset, balance)) =>
+      b.foldLeft[Either[String, VectorMap[IssuedAsset, Long]]](Right(a)) {
+        case (Right(seed), (asset, balance)) =>
           seed.get(asset) match {
             case None =>
               Right(seed.updated(asset, balance))
@@ -51,16 +51,25 @@ object Portfolio {
       }
   }
 
+  private def unsafeCombineAssets(a: VectorMap[IssuedAsset, Long], b: VectorMap[IssuedAsset, Long]): VectorMap[IssuedAsset, Long] =
+    if (a.isEmpty) b
+    else if (b.isEmpty) a
+    else
+      b.foldLeft(a) { case (seed, (asset, balance)) =>
+        val newBalance = seed.get(asset).fold(balance)(_ + balance)
+        seed.updated(asset, newBalance)
+      }
+
   def waves(amount: Long): Portfolio = build(Waves, amount)
 
   def build(af: (Asset, Long)): Portfolio = build(af._1, af._2)
 
   def build(a: Asset, amount: Long): Portfolio = a match {
     case Waves              => Portfolio(amount)
-    case t @ IssuedAsset(_) => Portfolio(assets = Map(t -> amount))
+    case t @ IssuedAsset(_) => Portfolio(assets = VectorMap(t -> amount))
   }
 
-  def build(wavesAmount: Long, a: IssuedAsset, amount: Long): Portfolio = Portfolio(wavesAmount, assets = Map(a -> amount))
+  def build(wavesAmount: Long, a: IssuedAsset, amount: Long): Portfolio = Portfolio(wavesAmount, assets = VectorMap(a -> amount))
 
   val empty: Portfolio = Portfolio()
 
@@ -77,10 +86,10 @@ object Portfolio {
     )
 
     def multiply(f: Fraction): Portfolio =
-      Portfolio(f(self.balance), LeaseBalance.empty, self.assets.view.mapValues(f.apply).toMap)
+      Portfolio(f(self.balance), LeaseBalance.empty, self.assets.view.mapValues(f.apply).to(VectorMap))
 
     def minus(other: Portfolio): Portfolio =
-      Portfolio(self.balance - other.balance, LeaseBalance.empty, Monoid.combine(self.assets, other.assets.view.mapValues(-_).to(Map)))
+      Portfolio(self.balance - other.balance, LeaseBalance.empty, unsafeCombineAssets(self.assets, other.assets.view.mapValues(- _).to(VectorMap)))
 
     def negate: Portfolio = Portfolio.empty minus self
 

node/src/main/scala/com/wavesplatform/state/diffs/AssetTransactionsDiff.scala

@@ -16,6 +16,8 @@ import com.wavesplatform.transaction.TxValidationError.GenericError
 import com.wavesplatform.transaction.assets.*
 import com.wavesplatform.transaction.{Asset, ERC20Address}
 
+import scala.collection.immutable.VectorMap
+
 object AssetTransactionsDiff {
   def issue(blockchain: Blockchain)(tx: IssueTransaction): Either[ValidationError, Diff] = { // TODO: unify with InvokeScript action diff?
     def requireValidUtf(): Boolean = {
@@ -45,7 +47,7 @@ object AssetTransactionsDiff {
         .flatTap(checkEstimationOverflow(blockchain, _))
         .map(script =>
           Diff(
-            portfolios = Map(tx.sender.toAddress -> Portfolio.build(-tx.fee.value, asset, tx.quantity.value)),
+            portfolios = VectorMap(tx.sender.toAddress -> Portfolio.build(-tx.fee.value, asset, tx.quantity.value)),
             issuedAssets = Map(asset -> NewAssetInfo(staticInfo, info, volumeInfo)),
             assetScripts = Map(asset -> script.map(AssetScriptInfo.tupled)),
             scriptsRun = DiffsCommon.countScriptRuns(blockchain, tx)

node/src/main/scala/com/wavesplatform/state/diffs/invoke/InvokeDiffsCommon.scala

@@ -39,6 +39,7 @@ import com.wavesplatform.transaction.{Asset, AssetIdLength, ERC20Address, PBSinc
 import com.wavesplatform.utils.*
 import shapeless.Coproduct
 
+import scala.collection.immutable.VectorMap
 import scala.util.{Failure, Right, Success, Try}
 
 object InvokeDiffsCommon {
@@ -468,8 +469,8 @@ object InvokeDiffsCommon {
               TracedResult(
                 Diff
                   .combine(
-                    Map(address     -> Portfolio(assets = Map(a -> amount))),
-                    Map(dAppAddress -> Portfolio(assets = Map(a -> -amount)))
+                    Map(address     -> Portfolio(assets = VectorMap(a -> amount))),
+                    Map(dAppAddress -> Portfolio(assets = VectorMap(a -> -amount)))
                   )
                   .bimap(GenericError(_), p => Diff(portfolios = p))
               ).flatMap(nextDiff =>
@@ -486,8 +487,8 @@ object InvokeDiffsCommon {
                       else
                         nextDiff.withPortfolios(
                           Map(
-                            address     -> Portfolio(assets = Map(a -> amount)),
-                            dAppAddress -> Portfolio(assets = Map(a -> -amount))
+                            address     -> Portfolio(assets = VectorMap(a -> amount)),
+                            dAppAddress -> Portfolio(assets = VectorMap(a -> -amount))
                           )
                         )
                     val pseudoTxRecipient =
@@ -551,7 +552,7 @@ object InvokeDiffsCommon {
           val info       = AssetInfo(ByteString.copyFromUtf8(issue.name), ByteString.copyFromUtf8(issue.description), Height @@ blockchain.height)
           Right(
             Diff(
-              portfolios = Map(pk.toAddress -> Portfolio(assets = Map(asset -> issue.quantity))),
+              portfolios = Map(pk.toAddress -> Portfolio(assets = VectorMap(asset -> issue.quantity))),
               issuedAssets = Map(asset -> NewAssetInfo(staticInfo, info, volumeInfo)),
               assetScripts = Map(asset -> None)
             )

node/src/test/scala/com/wavesplatform/http/AssetsRouteSpec.scala

@@ -22,9 +22,10 @@ import com.wavesplatform.lang.v1.compiler.TestCompiler
 import com.wavesplatform.lang.v1.estimator.ScriptEstimatorV1
 import com.wavesplatform.settings.WavesSettings
 import com.wavesplatform.state.{AssetDescription, AssetScriptInfo, BinaryDataEntry, Height}
-import com.wavesplatform.test.DomainPresets.*
 import com.wavesplatform.test.*
+import com.wavesplatform.test.DomainPresets.*
 import com.wavesplatform.transaction.Asset.IssuedAsset
+import com.wavesplatform.transaction.TxHelpers.*
 import com.wavesplatform.transaction.assets.IssueTransaction
 import com.wavesplatform.transaction.smart.SetScriptTransaction
 import com.wavesplatform.transaction.transfer.*
@@ -33,8 +34,8 @@ import com.wavesplatform.transaction.utils.EthTxGenerator.Arg
 import com.wavesplatform.transaction.{AssetIdLength, GenesisTransaction, Transaction, TxHelpers, TxNonNegativeAmount, TxVersion}
 import com.wavesplatform.utils.Schedulers
 import org.scalatest.concurrent.Eventually
+import play.api.libs.json.*
 import play.api.libs.json.Json.JsValueWrapper
-import play.api.libs.json.{JsArray, JsObject, JsValue, Json, Writes}
 
 import scala.concurrent.duration.*
 
@@ -523,33 +524,60 @@ class AssetsRouteSpec extends RouteSpec("/assets") with Eventually with RestAPIS
     ) ~> route ~> check(checkErrorResponse())
   }
 
-  routePath("/nft/list") in routeTest() { (d, route) =>
-    val issuer = testWallet.generateNewAccount().get
-    val nfts = Seq.tabulate(5) { i =>
-      TxHelpers.issue(issuer, 1, name = s"NFT_0$i", reissuable = false, fee = 0.001.waves)
-    }
-    d.appendBlock(TxHelpers.genesis(issuer.toAddress, 100.waves))
-    val nonNFT = TxHelpers.issue(issuer, 100, 2.toByte)
-    d.appendBlock((nfts :+ nonNFT)*)
-
-    Get(routePath(s"/balance/${issuer.toAddress}/${nonNFT.id()}")) ~> route ~> check {
-      val balance = responseAs[JsObject]
-      (balance \ "address").as[String] shouldEqual issuer.toAddress.toString
-      (balance \ "balance").as[Long] shouldEqual nonNFT.quantity.value
-      (balance \ "assetId").as[String] shouldEqual nonNFT.id().toString
+  routePath("/nft/list") - {
+    "NFTs in 1 block" in {
+      routeTest() { (d, route) =>
+        val issuer = testWallet.generateNewAccount().get
+        val nfts = Seq.tabulate(5) { i =>
+          TxHelpers.issue(issuer, 1, name = s"NFT_0$i", reissuable = false, fee = 0.001.waves)
+        }
+        d.appendBlock(TxHelpers.genesis(issuer.toAddress, 100.waves))
+        val nonNFT = TxHelpers.issue(issuer, 100, 2.toByte)
+        d.appendBlock((nfts :+ nonNFT)*)
+
+        Get(routePath(s"/balance/${issuer.toAddress}/${nonNFT.id()}")) ~> route ~> check {
+          val balance = responseAs[JsObject]
+          (balance \ "address").as[String] shouldEqual issuer.toAddress.toString
+          (balance \ "balance").as[Long] shouldEqual nonNFT.quantity.value
+          (balance \ "assetId").as[String] shouldEqual nonNFT.id().toString
+        }
+
+        Get(routePath(s"/nft/${issuer.toAddress}/limit/6")) ~> route ~> check {
+          status shouldBe StatusCodes.OK
+          val nftList = responseAs[Seq[JsObject]]
+          nftList.size shouldEqual nfts.size
+          nftList.foreach { jso =>
+            val nftId = (jso \ "assetId").as[ByteStr]
+            val nft   = nfts.find(_.id() == nftId).get
+
+            nft.name.toStringUtf8 shouldEqual (jso \ "name").as[String]
+            nft.timestamp shouldEqual (jso \ "issueTimestamp").as[Long]
+            nft.id() shouldEqual (jso \ "originTransactionId").as[ByteStr]
+          }
+        }
+      }
     }
-
-    Get(routePath(s"/nft/${issuer.toAddress}/limit/6")) ~> route ~> check {
-      status shouldBe StatusCodes.OK
-      val nftList = responseAs[Seq[JsObject]]
-      nftList.size shouldEqual nfts.size
-      nftList.foreach { jso =>
-        val nftId = (jso \ "assetId").as[ByteStr]
-        val nft   = nfts.find(_.id() == nftId).get
-
-        nft.name.toStringUtf8 shouldEqual (jso \ "name").as[String]
-        nft.timestamp shouldEqual (jso \ "issueTimestamp").as[Long]
-        nft.id() shouldEqual (jso \ "originTransactionId").as[ByteStr]
+    "NFTs in multiple blocks" in {
+      routeTest() { (d, route) =>
+        d.appendBlock(genesis(secondAddress, 100.waves))
+        val indexes =
+          (1 to 5).flatMap { block =>
+            val txs = (0 to 9).map { count =>
+              val i   = block * 10 + count
+              val tx1 = issue(defaultSigner, 1, name = s"NFT$i", reissuable = false)
+              val tx2 = issue(secondSigner, 1, name = s"NFT$i", reissuable = false)
+              (i, Seq(tx1, tx2))
+            }
+            d.appendBlock(txs.flatMap(_._2): _*)
+            txs.map(_._1)
+          }
+        Seq(defaultAddress, secondAddress).foreach { address =>
+          Get(routePath(s"/nft/$address/limit/50")) ~> route ~> check {
+            val nftList = responseAs[Seq[JsObject]]
+            nftList.size shouldEqual 50
+            nftList.map { jso => (jso \ "name").as[String].drop(3).toInt } shouldBe indexes
+          }
+        }
       }
     }
   }

node/src/test/scala/com/wavesplatform/state/PortfolioTest.scala

@@ -1,12 +1,13 @@
 package com.wavesplatform.state
 
 import java.nio.charset.StandardCharsets
-
 import com.wavesplatform.TestValues
 import com.wavesplatform.common.state.ByteStr
 import com.wavesplatform.test.FunSuite
 import com.wavesplatform.transaction.Asset.IssuedAsset
 
+import scala.collection.immutable.VectorMap
+
 class PortfolioTest extends FunSuite {
   test("pessimistic - should return only withdraws") {
     val fooKey = IssuedAsset(ByteStr("foo".getBytes(StandardCharsets.UTF_8)))
@@ -19,7 +20,7 @@ class PortfolioTest extends FunSuite {
         in = 11,
         out = 12
       ),
-      assets = Map(
+      assets = VectorMap(
         fooKey -> -13,
         barKey -> 14,
         bazKey -> 0