diff --git a/scorex-basics/build.sbt b/scorex-basics/build.sbt index fc3de690..514a1a4d 100644 --- a/scorex-basics/build.sbt +++ b/scorex-basics/build.sbt @@ -5,6 +5,7 @@ libraryDependencies ++= Dependencies.akka ++ Dependencies.spray ++ Dependencies.testKit ++ + Dependencies.db ++ Dependencies.logging ++ Seq( "org.whispersystems" % "curve25519-java" % "+", "commons-net" % "commons-net" % "3.+" diff --git a/scorex-basics/src/main/scala/scorex/block/Block.scala b/scorex-basics/src/main/scala/scorex/block/Block.scala index c3956eac..5c441163 100644 --- a/scorex-basics/src/main/scala/scorex/block/Block.scala +++ b/scorex-basics/src/main/scala/scorex/block/Block.scala @@ -12,22 +12,22 @@ import scorex.utils.ScorexLogging import scala.util.{Failure, Try} /** - * A block is a an atomic piece of data network participates are agreed on. - * - * A block has: - * - transactions data: a sequence of transactions, where a transaction is an atomic state update. - * Some metadata is possible as well(transactions Merkle tree root, state Merkle tree root etc). - * - * - consensus data to check whether block was generated by a right party in a right way. E.g. - * "baseTarget" & "generatorSignature" fields in the Nxt block structure, nonce & difficulty in the - * Bitcoin block structure. - * - * - a signature(s) of a block generator(s) - * - * - additional data: block structure version no, timestamp etc - */ - -trait Block { + * A block is a an atomic piece of data network participates are agreed on. + * + * A block has: + * - transactions data: a sequence of transactions, where a transaction is an atomic state update. + * Some metadata is possible as well(transactions Merkle tree root, state Merkle tree root etc). + * + * - consensus data to check whether block was generated by a right party in a right way. E.g. + * "baseTarget" & "generatorSignature" fields in the Nxt block structure, nonce & difficulty in the + * Bitcoin block structure. + * + * - a signature(s) of a block generator(s) + * + * - additional data: block structure version no, timestamp etc + */ + +trait Block extends ScorexLogging { type ConsensusDataType type TransactionDataType @@ -79,13 +79,21 @@ trait Block { lazy val bytesWithoutSignature = bytes.dropRight(EllipticCurveImpl.SignatureLength) - def isValid = - consensusModule.isValid(this) && + def isValid: Boolean = { + val v = consensusModule.isValid(this) && transactionModule.isValid(this) && transactionModule.history.contains(referenceField.value) && EllipticCurveImpl.verify(signerDataField.value.signature, bytesWithoutSignature, signerDataField.value.generator.publicKey) + if (!v) log.debug( + s"Block checks: ${consensusModule.isValid(this)} && ${transactionModule.isValid(this)} && " + + s"${transactionModule.history.contains(referenceField.value)} && " + + EllipticCurveImpl.verify(signerDataField.value.signature, bytesWithoutSignature, + signerDataField.value.generator.publicKey) + ) + v + } } diff --git a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala index e8a3056d..771e1660 100644 --- a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala +++ b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala @@ -8,22 +8,22 @@ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future -trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[ConsensusBlockData]{ +trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[ConsensusBlockData] { def isValid[TT](block: Block)(implicit transactionModule: TransactionModule[TT]): Boolean /** - * Fees could go to a single miner(forger) usually, but can go to many parties, e.g. see - * Meni Rosenfeld's Proof-of-Activity proposal http://eprint.iacr.org/2014/452.pdf - */ + * Fees could go to a single miner(forger) usually, but can go to many parties, e.g. see + * Meni Rosenfeld's Proof-of-Activity proposal http://eprint.iacr.org/2014/452.pdf + */ def feesDistribution(block: Block): Map[Account, Long] /** - * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not - * (see e.g. Meni Rosenfeld's Proof-of-Activity paper http://eprint.iacr.org/2014/452.pdf) - * @param block - * @return - */ + * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not + * (see e.g. Meni Rosenfeld's Proof-of-Activity paper http://eprint.iacr.org/2014/452.pdf) + * @param block + * @return + */ def generators(block: Block): Seq[Account] def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt diff --git a/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala b/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala index f2bee338..7d6c1656 100644 --- a/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala +++ b/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala @@ -21,10 +21,12 @@ import java.security.MessageDigest trait CryptographicHash { import CryptographicHash._ - val ValueSize: Int //in bytes + val DigestSize: Int //in bytes def hash(input: Message): Digest + def hash(input: String): Digest = hash(input.getBytes) + def doubleHash(input: Message): Digest = hash(hash(input)) } @@ -37,7 +39,7 @@ object CryptographicHash { * Hashing functions implementation with sha256 impl from Java SDK */ object Sha256 extends CryptographicHash { - override val ValueSize = 32 + override val DigestSize = 32 override def hash(input: Array[Byte]) = MessageDigest.getInstance("SHA-256").digest(input) } \ No newline at end of file diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala new file mode 100644 index 00000000..91bc9e23 --- /dev/null +++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala @@ -0,0 +1,73 @@ +package scorex.crypto.ads.merkle + +import play.api.libs.json._ +import scorex.crypto.CryptographicHash._ +import scorex.crypto.ads.merkle.TreeStorage.Position +import scorex.crypto.{Base58, CryptographicHash, Sha256} + +import scala.annotation.tailrec + +/** + * @param data - data block + * @param merklePath - merkle path, complementary to data block + */ +case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) { + + def check[HashImpl <: CryptographicHash](index: Position, rootHash: Digest) + (hashFunction: HashImpl = Sha256): Boolean = { + + @tailrec + def calculateHash(idx: Position, nodeHash: Digest, path: Seq[Digest]): Digest = { + val hash = if (idx % 2 == 0) + hashFunction.hash(nodeHash ++ path.head) + else + hashFunction.hash(path.head ++ nodeHash) + + if (path.size == 1) + hash + else + calculateHash(idx / 2, hash, path.tail) + } + + if (merklePath.nonEmpty) + calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath) sameElements rootHash + else + false + } +} + +object AuthDataBlock { + + implicit def authDataBlockReads[T](implicit fmt: Reads[T]): Reads[AuthDataBlock[T]] = new Reads[AuthDataBlock[T]] { + def reads(json: JsValue): JsResult[AuthDataBlock[T]] = JsSuccess(AuthDataBlock[T]( + (json \ "data").get match { + case JsString(ts) => + Base58.decode(ts).get.asInstanceOf[T] + case _ => + throw new RuntimeException("Data MUST be a string") + }, + (json \ "merklePath").get match { + case JsArray(ts) => ts.map { t => + t match { + case JsString(digest) => + Base58.decode(digest) + case m => + throw new RuntimeException("MerklePath MUST be array of strings" + m + " given") + } + }.map(_.get) + case m => + throw new RuntimeException("MerklePath MUST be a list " + m + " given") + } + )) + } + + implicit def authDataBlockWrites[T](implicit fmt: Writes[T]): Writes[AuthDataBlock[T]] = new Writes[AuthDataBlock[T]] { + def writes(ts: AuthDataBlock[T]) = JsObject(Seq( + "data" -> JsString(Base58.encode(ts.data.asInstanceOf[Array[Byte]])), + "merklePath" -> JsArray( + ts.merklePath.map(digest => JsString(Base58.encode(digest))) + ) + )) + } +} + diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala index 353b589f..0ddb61ca 100644 --- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala +++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala @@ -1,175 +1,145 @@ package scorex.crypto.ads.merkle +import java.io.{FileOutputStream, RandomAccessFile} +import java.nio.file.{Files, Paths} + import scorex.crypto.CryptographicHash.Digest +import scorex.crypto.ads.merkle.TreeStorage.Position import scorex.crypto.{CryptographicHash, Sha256} +import scorex.utils.ScorexLogging import scala.annotation.tailrec -import scala.math - -/** - * @param data - data block - * @param merklePath - merkle path, complementary to data block - */ -case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) - -//bottom up -trait MerkleTreeI[Block] { - def byIndex(n: Int): Option[AuthDataBlock[Block]] -} +class MerkleTree[H <: CryptographicHash](treeFolder: String, + val nonEmptyBlocks: Position, + blockSize: Int = 1024, + hash: H = Sha256 + ) extends ScorexLogging { -//todo: check/optimize the code + import MerkleTree._ -object MerkleTree { + val level = calculateRequiredLevel(nonEmptyBlocks) - def check[Block, Hash <: CryptographicHash](index: Int, rootHash: Digest, block: AuthDataBlock[Block]) - (hashFunction: Hash = Sha256): Boolean = { + lazy val storage = new TreeStorage(treeFolder + TreeFileName, level) - @tailrec - def calculateHash(i: Int, nodeHash: Digest, path: Seq[Digest]): Digest = { - if (i % 2 == 0) { - val hash = hashFunction.hash(nodeHash ++ path.head) - if (path.size == 1) { - hash - } else { - calculateHash(i / 2, hash, path.tail) - } - } else { - val hash = hashFunction.hash(path.head ++ nodeHash) - if (path.size == 1) { - hash - } else { - calculateHash(i / 2, hash, path.tail) - } - } - } - val calculated = calculateHash(index, hashFunction.hash(block.data.toString.getBytes), block.merklePath) - calculated.mkString == rootHash.mkString - } + val rootHash: Digest = getHash((level, 0)).get - class MerkleTree[Block, Hash <: CryptographicHash](val tree: Tree[Block, Hash], val leaves: Seq[Tree[Block, Hash]]) - extends MerkleTreeI[Block] { + storage.commit() - val Size = leaves.size - lazy val rootNode: Node[Block, Hash] = tree.asInstanceOf[Node[Block, Hash]] - lazy val hash = tree.hash - - def byIndex(index: Int): Option[AuthDataBlock[Block]] = { + def byIndex(index: Position): Option[AuthDataBlock[Block]] = { + if (index < nonEmptyBlocks && index >= 0) { @tailrec - def calculateTreePath(n: Int, node: Node[Block, Hash], levelSize: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = { - val halfLevelSize: Int = levelSize / 2 - if (n < halfLevelSize) { - node.leftChild match { - case nd: Node[Block, Hash] => - calculateTreePath(n, nd, halfLevelSize, node.rightChild.hash +: acc) - case _ => - node.rightChild.hash +: acc + def calculateTreePath(n: Position, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = { + if (currentLevel < level) { + val hashOpt = if (n % 2 == 0) getHash((currentLevel, n + 1)) else getHash((currentLevel, n - 1)) + hashOpt match { + case Some(h) => + calculateTreePath(n / 2, currentLevel + 1, h +: acc) + case None if currentLevel == 0 && index == nonEmptyBlocks - 1 => + calculateTreePath(n / 2, currentLevel + 1, emptyHash +: acc) + case None => + log.error(s"Enable to get hash for lev=$currentLevel, position=$n") + acc.reverse } } else { - node.rightChild match { - case nd: Node[Block, Hash] => - calculateTreePath(n - halfLevelSize, nd, halfLevelSize, node.leftChild.hash +: acc) - case _ => - node.leftChild.hash +: acc - } + acc.reverse } } - leaves.lift(index).flatMap(l => - l match { - case Leaf(data: Block) => - val treePath = calculateTreePath(index, rootNode, Size) - Some(AuthDataBlock(data, treePath)) - case _ => - None - } - ) + val path = Paths.get(treeFolder + "/" + index) + val data: Block = Files.readAllBytes(path) + val treePath = calculateTreePath(index, 0) + Some(AuthDataBlock(data, treePath)) + } else { + None } + } - override def toString: String = { - def printHashes(node: Tree[Any, Hash], prefix: String = ""): List[String] = { - node match { - case Node(leftChild: Tree[Block, Hash], rightChild: Tree[Block, Hash]) => - (prefix + node.hash.mkString) :: printHashes(leftChild, " " + prefix) ++ - printHashes(rightChild, " " + prefix) - case l: Leaf[Block, Hash] => - List(prefix + l.hash.mkString) - case _ => - List() + private lazy val emptyHash = hash.hash("") + + def getHash(key: TreeStorage.Key): Option[Digest] = { + storage.get(key) match { + case None => + if (key._1 > 0) { + val h1 = getHash((key._1 - 1, key._2 * 2)) + val h2 = getHash((key._1 - 1, key._2 * 2 + 1)) + val calculatedHash = (h1, h2) match { + case (Some(hash1), Some(hash2)) => hash.hash(hash1 ++ hash2) + case (Some(h), _) => hash.hash(h ++ emptyHash) + case (_, Some(h)) => hash.hash(emptyHash ++ h) + case _ => emptyHash + } + storage.set(key, calculatedHash) + Some(calculatedHash) + } else { + None } - } - printHashes(tree).mkString("\n") + case digest => + digest } } +} - sealed trait Tree[+Block, Hash <: CryptographicHash] { - val hash: Digest - } - - case class Node[+Block, Hash <: CryptographicHash]( - leftChild: Tree[Block, Hash], - rightChild: Tree[Block, Hash])(hashFunction: Hash) - extends Tree[Block, Hash] { - - override val hash: Digest = hashFunction.hash(leftChild.hash ++ rightChild.hash) - - } - - case class Leaf[+Block, Hash <: CryptographicHash](data: Block)(hashFunction: Hash) - extends Tree[Block, Hash] { - - override val hash: Digest = hashFunction.hash(data.toString.getBytes) - } +object MerkleTree { + type Block = Array[Byte] + + val TreeFileName = "/hashTree" + + def fromFile[H <: CryptographicHash](fileName: String, + treeFolder: String, + blockSize: Int = 1024, + hash: H = Sha256 + ): MerkleTree[H] = { + val byteBuffer = new Array[Byte](blockSize) + + def readLines(bigDataFilePath: String, chunkIndex: Position): Array[Byte] = { + val randomAccessFile = new RandomAccessFile(fileName, "r") + try { + val seek = chunkIndex * blockSize + randomAccessFile.seek(seek) + randomAccessFile.read(byteBuffer) + byteBuffer + } finally { + randomAccessFile.close() + } + } - case class EmptyLeaf[Block <: CryptographicHash]()(hashFunction: Block) extends Tree[Nothing, Block] { - override val hash: Digest = Array.empty[Byte] - } + val nonEmptyBlocks: Position = { + val randomAccessFile = new RandomAccessFile(fileName, "r") + try { + (randomAccessFile.length / blockSize).toInt + } finally { + randomAccessFile.close() + } + } - def create[Block, Hash <: CryptographicHash]( - dataBlocks: Seq[Block], - hashFunction: Hash = Sha256): MerkleTree[Block, Hash] = { - val level = calculateRequiredLevel(dataBlocks.size) + val level = calculateRequiredLevel(nonEmptyBlocks) - val dataLeaves = dataBlocks.map(data => Leaf(data)(hashFunction)) + lazy val storage = new TreeStorage(treeFolder + TreeFileName, level) - val paddingNeeded = math.pow(2, level).toInt - dataBlocks.size - val padding = Seq.fill(paddingNeeded)(EmptyLeaf()(hashFunction)) + def processBlocks(currentBlock: Position = 0): Unit = { + val block: Block = readLines(fileName, currentBlock) + val fos = new FileOutputStream(treeFolder + "/" + currentBlock) + fos.write(block) + fos.close() + storage.set((0, currentBlock), hash.hash(block)) + if (currentBlock < nonEmptyBlocks - 1) { + processBlocks(currentBlock + 1) + } + } - val leaves: Seq[Tree[Block, Hash]] = dataLeaves ++ padding + processBlocks() - new MerkleTree(makeTree(leaves, hashFunction), leaves) - } + storage.commit() + storage.close() - def merge[Block, Hash <: CryptographicHash]( - leftChild: Tree[Block, Hash], - rightChild: Tree[Block, Hash], - hashFunction: Hash): Node[Block, Hash] = { - Node(leftChild, rightChild)(hashFunction) + new MerkleTree(treeFolder, nonEmptyBlocks, blockSize, hash) } private def log2(x: Double): Double = math.log(x) / math.log(2) - private def calculateRequiredLevel(numberOfDataBlocks: Int): Int = { - + def calculateRequiredLevel(numberOfDataBlocks: Position): Int = { math.ceil(log2(numberOfDataBlocks)).toInt } - - @tailrec - private def makeTree[Block, Hash <: CryptographicHash]( - trees: Seq[Tree[Block, Hash]], - hashFunction: Hash): Tree[Block, Hash] = { - def createParent(treePair: Seq[Tree[Block, Hash]]): Node[Block, Hash] = { - val leftChild +: rightChild +: _ = treePair - merge(leftChild, rightChild, hashFunction) - } - - if (trees.isEmpty) { - EmptyLeaf()(hashFunction) - } else if (trees.size == 1) { - trees.head - } else { - makeTree(trees.grouped(2).map(createParent).toSeq, hashFunction) - } - } } \ No newline at end of file diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala new file mode 100644 index 00000000..022d768e --- /dev/null +++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala @@ -0,0 +1,76 @@ +package scorex.crypto.ads.merkle + +import java.io.File + +import org.mapdb.{DBMaker, HTreeMap, Serializer} +import scorex.crypto.CryptographicHash.Digest +import scorex.storage.Storage +import scorex.utils.ScorexLogging + +import scala.util.{Failure, Success, Try} + +class TreeStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Long], Array[Byte]] with ScorexLogging { + + import TreeStorage._ + + private val dbs = + (0 to levels) map { n: Int => + DBMaker.fileDB(new File(fileName + n + ".mapDB")) + .fileMmapEnableIfSupported() + .closeOnJvmShutdown() + .checksumEnable() + .make() + } + + private val maps: Map[Int, HTreeMap[Long, Digest]] = { + val t = (0 to levels) map { n: Int => + val m: HTreeMap[Long, Digest] = dbs(n).hashMapCreate("map_" + n) + .keySerializer(Serializer.LONG) + .valueSerializer(Serializer.BYTE_ARRAY) + .makeOrGet() + n -> m + } + t.toMap + } + + override def set(key: Key, value: Digest): Unit = { + val map = maps(key._1.asInstanceOf[Int]) + Try { + map.put(key._2, value) + }.recoverWith { case t: Throwable => + log.warn("Failed to set key:" + key, t) + Failure(t) + } + } + + override def commit(): Unit = dbs.foreach(_.commit()) + + override def close(): Unit = { + commit() + dbs.foreach(_.close()) + } + + override def get(key: Key): Option[Digest] = { + Try { + maps(key._1).get(key._2) + } match { + case Success(v) => + Option(v) + + case Failure(e) => + if (key._1 == 0) { + log.debug("Enable to load key for level 0: " + key) + } + None + } + } + +} + +object TreeStorage { + type Level = Int + type Position = Long + type Key = (Level, Position) + type Value = Digest + +} diff --git a/scorex-basics/src/main/scala/scorex/settings/Settings.scala b/scorex-basics/src/main/scala/scorex/settings/Settings.scala index 81f17200..80349bd8 100644 --- a/scorex-basics/src/main/scala/scorex/settings/Settings.scala +++ b/scorex-basics/src/main/scala/scorex/settings/Settings.scala @@ -6,17 +6,18 @@ import play.api.libs.json.{JsObject, Json} import scorex.crypto.Base58 import scorex.utils.ScorexLogging +import scala.concurrent.duration._ import scala.util.Try /** - * Settings - */ + * Settings + */ trait Settings extends ScorexLogging { lazy val Port = 9084 - val filename:String + val filename: String lazy val settingsJSON: JsObject = Try { val jsonString = scala.io.Source.fromFile(filename).mkString @@ -53,6 +54,8 @@ trait Settings extends ScorexLogging { lazy val pingInterval = (settingsJSON \ "pinginterval").asOpt[Int].getOrElse(DefaultPingInterval) lazy val offlineGeneration = (settingsJSON \ "offline-generation").asOpt[Boolean].getOrElse(false) lazy val bindAddress = (settingsJSON \ "bindAddress").asOpt[String].getOrElse(DefaultBindAddress) + lazy val blockGenerationDelay: FiniteDuration = (settingsJSON \ "blockGenerationDelay").asOpt[Long] + .map(x => FiniteDuration(x, MILLISECONDS)).getOrElse(DefaultBlockGenerationDelay) lazy val walletDirOpt = (settingsJSON \ "walletdir").asOpt[String] .ensuring(pathOpt => pathOpt.map(directoryEnsuring).getOrElse(true)) @@ -72,4 +75,7 @@ trait Settings extends ScorexLogging { //API private val DefaultRpcPort = 9085 private val DefaultRpcAllowed = "127.0.0.1" + + private val DefaultBlockGenerationDelay: FiniteDuration = 1.second + } \ No newline at end of file diff --git a/scorex-basics/src/main/scala/scorex/storage/Storage.scala b/scorex-basics/src/main/scala/scorex/storage/Storage.scala new file mode 100644 index 00000000..8f133882 --- /dev/null +++ b/scorex-basics/src/main/scala/scorex/storage/Storage.scala @@ -0,0 +1,17 @@ +package scorex.storage + +trait Storage[Key, Value] { + + def set(key: Key, value: Value): Unit + + def get(key: Key): Option[Value] + + def commit(): Unit + + def close(): Unit + + def containsKey(key: Key): Boolean = { + get(key).isDefined + } + +} \ No newline at end of file diff --git a/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala b/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala index 6ee6633b..4ac6e45d 100644 --- a/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala +++ b/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala @@ -3,6 +3,8 @@ package scorex.transaction import scorex.block.Block import scorex.utils.ScorexLogging +import scala.util.Try + trait BlockChain extends History with ScorexLogging { def blockAt(height: Int): Option[Block] @@ -28,6 +30,18 @@ trait BlockChain extends History with ScorexLogging { } } + /** + * Average delay in milliseconds between last $blockNum blocks starting from $block + */ + def averageDelay(block: Block, blockNum: Int): Try[Long] = Try { + val height: Int = heightOf(block).get + val lastBlocks = (0 until blockNum).flatMap(i => blockAt(height - i)).reverse + require(lastBlocks.length == blockNum) + (0 until blockNum - 1).map { i => + lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value + }.sum / (blockNum - 1) + } + def lastSignature(): Block.BlockId = lastBlock.uniqueId def removeAfter(signature: Block.BlockId) = while (!lastSignature().sameElements(signature)) discardBlock() diff --git a/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala b/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala new file mode 100644 index 00000000..79a7204d --- /dev/null +++ b/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala @@ -0,0 +1,46 @@ +package scorex.utils + +import play.api.libs.json._ +import scorex.crypto.Base58 + +import scala.util.{Failure, Success} + +trait JsonSerialization { + + type Bytes = Array[Byte] + + implicit val bytesWrites = new Writes[Bytes] { + def writes(bytes: Bytes) = JsString(Base58.encode(bytes)) + } + + implicit def bytesReads: Reads[Bytes] = new Reads[Bytes] { + def reads(json: JsValue): JsResult[Bytes] = json match { + case JsString(encoded) => + Base58.decode(encoded) match { + case Success(decoded) => + JsSuccess(decoded) + case Failure(e) => + throw new RuntimeException(s"Failed to parse Base58 encoded bytes") + } + case m => + throw new RuntimeException(s"Bigint MUST be represented as string in json $m ${m.getClass} given") + } + } + + + implicit val bigIntWrites = new Writes[BigInt] { + def writes(bitInt: BigInt) = JsString(bitInt.toString) + } + + implicit def bigIntReads: Reads[BigInt] = new Reads[BigInt] { + def reads(json: JsValue): JsResult[BigInt] = json match { + case JsString(bigint) => + JsSuccess(BigInt(bigint)) + case JsNumber(bigint) => + JsSuccess(BigInt(bigint.toString)) + case m => + throw new RuntimeException(s"Bigint MUST be represented as string in json $m ${m.getClass} given") + } + } + +} diff --git a/scorex-basics/src/main/scala/scorex/utils/utils.scala b/scorex-basics/src/main/scala/scorex/utils/utils.scala index 8167718a..6364dd72 100644 --- a/scorex-basics/src/main/scala/scorex/utils/utils.scala +++ b/scorex-basics/src/main/scala/scorex/utils/utils.scala @@ -1,5 +1,7 @@ package scorex +import java.security.SecureRandom + import scala.annotation.tailrec import scala.concurrent.duration._ @@ -19,4 +21,11 @@ package object utils { case util.Failure(e) => throw e } } + + def randomBytes(howMany: Int) = { + val r = new Array[Byte](howMany) + new SecureRandom().nextBytes(r) //overrides r + r + } + } diff --git a/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala b/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala index 7a237a64..1f9a8845 100644 --- a/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala +++ b/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala @@ -10,5 +10,6 @@ class ScorexTestSuite extends Suites ( new unit.Sha256Specification, new props.SigningFunctionsSpecification, - new props.MerkleSpecification + new props.MerkleSpecification, + new props.TreeStorageSpecification ) diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala index adadc99e..a90d1a2a 100644 --- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala +++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala @@ -1,8 +1,10 @@ package scorex.props -import org.scalacheck.{Arbitrary, Gen} -import org.scalatest.{Matchers, PropSpec} +import java.io.{File, FileOutputStream} + +import org.scalacheck.Gen import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks} +import org.scalatest.{Matchers, PropSpec} import scorex.crypto.Sha256 import scorex.crypto.ads.merkle.MerkleTree @@ -10,21 +12,56 @@ import scala.util.Random class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers { + property("value returned from byIndex() is valid for random dataset") { - val dataSetGen = for { - dataSet <- Gen.nonEmptyContainerOf[Array, Array[Byte]](Arbitrary.arbitrary[Array[Byte]]) - } yield dataSet + //fix block numbers for faster tests + for (blocks <- List(7, 8, 9, 128)) { + val smallInteger = Gen.choose(0, blocks - 1) + val (treeDirName: String, _, tempFile: String) = generateFile(blocks) + val tree = MerkleTree.fromFile(tempFile, treeDirName) + forAll(smallInteger) { (index: Int) => + val leafOption = tree.byIndex(index) + leafOption should not be None + val leaf = leafOption.get + val resp = leaf.check(index, tree.rootHash)(Sha256) + resp shouldBe true + } + tree.storage.close() + } + } - forAll(dataSetGen) { dataSet => - if (dataSet.length > 1) { - val index = Random.nextInt(dataSet.length) + property("hash root is the same") { + //fix block numbers for faster tests + for (blocks <- List(7, 8, 9, 128)) { + val (treeDirName: String, _, tempFile: String) = generateFile(blocks, "2") - val tree = MerkleTree.create(dataSet) + val fileTree = MerkleTree.fromFile(tempFile, treeDirName) + val rootHash = fileTree.rootHash - val leaf = tree.byIndex(index).get - MerkleTree.check(index, tree.hash, leaf)(Sha256) - } + fileTree.storage.close() + + val tree = new MerkleTree(treeDirName, fileTree.nonEmptyBlocks) + val newRootHash = tree.rootHash + tree.storage.close() + rootHash shouldBe newRootHash } } + + def generateFile(blocks: Int, subdir: String = "1"): (String, File, String) = { + val treeDirName = "/tmp/scorex/test/" + subdir + "/" + val treeDir = new File(treeDirName) + val tempFile = treeDirName + "/data.file" + + + val data = new Array[Byte](1024 * blocks) + Random.nextBytes(data) + treeDir.mkdirs() + for (file <- treeDir.listFiles) file.delete + + val fos = new FileOutputStream(tempFile) + fos.write(data) + fos.close() + (treeDirName, treeDir, tempFile) + } } \ No newline at end of file diff --git a/scorex-basics/src/test/scala/scorex/props/TreeStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/TreeStorageSpecification.scala new file mode 100644 index 00000000..cf32e171 --- /dev/null +++ b/scorex-basics/src/test/scala/scorex/props/TreeStorageSpecification.scala @@ -0,0 +1,43 @@ +package scorex.props + +import java.io.File + +import org.scalacheck.{Arbitrary, Gen} +import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks} +import org.scalatest.{Matchers, PropSpec} +import scorex.crypto.CryptographicHash.Digest +import scorex.crypto.ads.merkle.TreeStorage.Key +import scorex.crypto.ads.merkle.TreeStorage + + +class TreeStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers { + + val treeDirName = "/tmp/scorex/test/MapDBStorageSpecification/" + val treeDir = new File(treeDirName) + treeDir.mkdirs() + val dbFile = new File(treeDirName + "/db.file") + val maxLevel = 50 + dbFile.delete() + + val keyVal = for { + level: Int <- Gen.choose(1, maxLevel) + key: Long <- Arbitrary.arbitrary[Long] + value <- Arbitrary.arbitrary[String] + } yield ((level, math.abs(key)), value.getBytes) + + + property("set value and get it") { + lazy val storage = new TreeStorage(treeDirName + "/test_db", maxLevel) + + forAll(keyVal) { x => + val key: Key = x._1 + val value: Digest = x._2 + whenever(key._1 >= 0.toLong && key._2 >= 0.toLong) { + storage.set(key, value) + + assert(storage.get(key).get sameElements value) + } + } + storage.close() + } +} \ No newline at end of file diff --git a/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala b/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala index 67857f7f..65ddad26 100644 --- a/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala +++ b/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala @@ -18,9 +18,9 @@ class Sha256Specification extends FunSuite with Matchers { assert(bytes2hex(hash(testBytes)) == "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9") //test samples from a Qeditas unit test - assert(bytes2hex(hash("".getBytes)) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") - assert(bytes2hex(hash("abc".getBytes)) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") - assert(bytes2hex(hash("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq".getBytes)) == + assert(bytes2hex(hash("")) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855") + assert(bytes2hex(hash("abc")) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad") + assert(bytes2hex(hash("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq")) == "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1") } } diff --git a/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala b/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala index 3cb766f2..23df429a 100644 --- a/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala +++ b/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala @@ -79,7 +79,6 @@ class NxtConsensusApiRoute(consensusModule: NxtLikeConsensusModule, blockchain: Json.obj("base-target" -> bt).toString() } } - } @Path("/algo") diff --git a/scorex-perma/src/main/resources/perma.conf b/scorex-perma/src/main/resources/perma.conf new file mode 100644 index 00000000..9bbc1fd1 --- /dev/null +++ b/scorex-perma/src/main/resources/perma.conf @@ -0,0 +1,10 @@ +perma { + segmentSize = 1024 + n = 1024 + l = 1024 + k = 4 + hash = "sha256" + initialTarget = "79980561713257764341044371529078133767288724078815817939205408376510588897" + targetRecalculation = 10 + averageDelay = 2 +} diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala new file mode 100644 index 00000000..2fc339b1 --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala @@ -0,0 +1,81 @@ +package scorex.perma + +import akka.actor.{Actor, ActorRef} +import scorex.crypto.{Sha256, CryptographicHash} +import scorex.perma.actors.MinerSpec._ +import scorex.perma.consensus.Ticket +import scorex.utils._ + +import scala.collection.mutable +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ +import scala.util.Random + + +case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket) + +class BlockchainBuilder(miners: Seq[ActorRef], dealer: ActorRef) extends Actor with ScorexLogging { + + import BlockchainBuilderSpec._ + + var initialized = 0 + + val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte)) + val blockchainLike = mutable.Buffer[BlockHeaderLike]() + + def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty) + + def calcPuz(block: Option[BlockHeaderLike], hash: CryptographicHash = Sha256): Array[Byte] = block match { + case Some(b) => + hash.hash(b.puz ++ b.ticket.s ++ b.ticket.publicKey + ++ b.ticket.proofs.foldLeft(Array.empty: Array[Byte])((b, a) => b ++ a.signature)) + case None => + hash.hash("Scorex perma genesis") + } + + override def receive = { + case s: MinerStatus => + s match { + case Initialized => + initialized = initialized + 1 + if (initialized == miners.length) { + log.info("All miners initialized") + self ! SendWorkToMiners + } + case LoadingData => + sender() ! Initialize(Seq(dealer)) + context.system.scheduler.scheduleOnce(200 millis, sender(), GetStatus) + } + + case SendWorkToMiners => + miners.foreach { minerRef => + if (initialized == miners.length) { + minerRef ! TicketGeneration(difficulty, calcPuz(blockchainLike.lastOption)) + } else { + minerRef ! Initialize(miners) + context.system.scheduler.scheduleOnce(200 millis, minerRef, GetStatus) + } + } + + //miners are honest (lol) in our setting, so no validation here + case WinningTicket(minerPuz, score, ticket) => + val puz = calcPuz(blockchainLike.lastOption) + if (minerPuz sameElements puz) { + val newBlock = BlockHeaderLike(score, puz, ticket) + log.info(s"Block generated: $newBlock, blockchain size: ${blockchainLike.size}") + blockchainLike += newBlock + self ! SendWorkToMiners + } else { + sender() ! TicketGeneration(difficulty, puz) + log.debug("Wrong puz from miner: " + minerPuz.mkString) + } + } +} + +object BlockchainBuilderSpec { + + case object SendWorkToMiners + + case class WinningTicket(puz: Array[Byte], score: BigInt, t: Ticket) + +} \ No newline at end of file diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala deleted file mode 100644 index 5e04dd58..00000000 --- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala +++ /dev/null @@ -1,15 +0,0 @@ -package scorex.perma - - -object Parameters { - - type DataSegment = Array[Byte] - - //few segments to be stored in a block, so segment size shouldn't be big - val segmentSize = 1024 //segment size in bytes - - val n = 1024*4 //how many segments in a dataset in total - val l = 16 //how many segments to store for an each miner - - val k = 4 //number of iterations during scratch-off phase -} diff --git a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala new file mode 100644 index 00000000..42e24d12 --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala @@ -0,0 +1,66 @@ +package scorex.perma.Storage + +import java.io.File + +import org.mapdb.{DBMaker, HTreeMap, Serializer} +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.perma.settings.Constants +import scorex.perma.settings.Constants.DataSegment +import scorex.storage.Storage +import scorex.utils.ScorexLogging + +import scala.util.{Failure, Success, Try} + +class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[DataSegment]] with ScorexLogging { + + + //TODO https://github.com/jankotek/mapdb/issues/634 workaround + private var commitNeeded = false + + private val db = + DBMaker.appendFileDB(new File(fileName + ".mapDB")) + .fileMmapEnableIfSupported() + .closeOnJvmShutdown() + .checksumEnable() + .make() + + private val map: HTreeMap[Long, AuthDataBlock[DataSegment]] = db.hashMapCreate("segments") + .keySerializer(Serializer.LONG) + .makeOrGet() + + override def set(key: Long, value: AuthDataBlock[DataSegment]): Unit = { + Try { + map.put(key, value) + + }.recoverWith { case t: Throwable => + log.warn("Failed to set key:" + key, t) + Failure(t) + } + } + + override def commit(): Unit = if (commitNeeded) { + db.commit() + commitNeeded = false + } + + override def close(): Unit = db.close() + + + override def containsKey(key: Long): Boolean = { + map.containsKey(key) + } + + override def get(key: Long): Option[AuthDataBlock[DataSegment]] = { + Try { + map.get(key) + } match { + case Success(v) => + Option(v) + + case Failure(e) => + log.debug("Enable to load key for level 0: " + key) + None + } + } + +} diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala index 0ddf2808..1735ff3f 100644 --- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala +++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala @@ -1,87 +1,64 @@ package scorex.perma -import akka.actor.{Actor, ActorRef, ActorSystem, Props} -import org.slf4j.LoggerFactory -import scorex.crypto.ads.merkle.MerkleTree -import scorex.perma.BlockchainBuilderSpec.{SendWorkToMiners, WinningTicket} -import scorex.perma.actors.MinerSpec.{Initialize, TicketGeneration} -import scorex.perma.actors.{Miner, Ticket, TrustedDealer} +import java.io.{File, RandomAccessFile} +import java.nio.file.{Files, Paths} +import akka.actor.{ActorSystem, Props} +import akka.util.Timeout +import scorex.crypto.ads.merkle.MerkleTree +import scorex.perma.BlockchainBuilderSpec.SendWorkToMiners +import scorex.perma.Storage.AuthDataStorage +import scorex.perma.actors.MinerSpec.Initialize +import scorex.perma.actors.{Miner, TrustedDealer} +import scorex.perma.settings.{Constants, PermaSettings} +import scorex.settings.Settings import scorex.utils.ScorexLogging -import scala.collection.mutable -import scala.util.Random - - -case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket) - -class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging { - - var puz: Array[Byte] = calcPuz +import scala.concurrent.duration._ - val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte)) - val blockchainLike = mutable.Buffer[BlockHeaderLike]() - - private def calcPuz = 1.to(100).toArray.map(_ => Random.nextInt(256).toByte) - - def difficulty = blockchainLike.lastOption.map(_.difficulty).getOrElse(InitialDifficulty) - - override def receive = { - case SendWorkToMiners => - miners.foreach { minerRef => - minerRef ! TicketGeneration(difficulty, puz) - } - - //miners are honest (lol) in our setting, so no validation here - case WinningTicket(minerPuz, score, ticket) => - if (minerPuz sameElements puz) { - val newBlock = BlockHeaderLike(score, puz, ticket) - log.info(s"Block generated: $newBlock, blockchain size: ${blockchainLike.size}") - blockchainLike += newBlock - puz = calcPuz - self ! SendWorkToMiners - } else { - sender() ! TicketGeneration(difficulty, puz) - log.debug("Wrong puz from miner: " + minerPuz.mkString) - } - } -} - -object BlockchainBuilderSpec { - - case object SendWorkToMiners - - case class WinningTicket(puz: Array[Byte], score: BigInt, t: Ticket) - -} - -object TestApp extends App { +object TestApp extends App with ScorexLogging { val MinersCount = 10 - val log = LoggerFactory.getLogger(this.getClass) + implicit val settings = new Settings with PermaSettings { + val filename = "settings.json" + } - import Parameters._ + val tree = if (Files.exists(Paths.get(settings.treeDir + "/tree0.mapDB"))) { + log.info("Get existing tree") + new MerkleTree(settings.treeDir, Constants.n, Constants.segmentSize, Constants.hash) + } else { + log.info("Generating random data set") + val treeDir = new File(settings.treeDir) + treeDir.mkdirs() + val datasetFile = settings.treeDir + "/data.file" + new RandomAccessFile(datasetFile, "rw").setLength(Constants.n * Constants.segmentSize) + log.info("Calculate tree") + val tree = MerkleTree.fromFile(datasetFile, settings.treeDir, Constants.segmentSize, Constants.hash) + require(tree.nonEmptyBlocks == Constants.n, s"${tree.nonEmptyBlocks} == ${Constants.n}") + tree + } - log.info("Generating random data set") - val rnd = new Random() - val dataSet = (1 to n).map(x => Random.alphanumeric.take(segmentSize).mkString).toArray.map(_.getBytes) + log.info("test tree") + val index = Constants.n - 3 + val leaf = tree.byIndex(index).get + require(leaf.check(index, tree.rootHash)(Constants.hash)) - log.info("Calculate tree") - val tree = MerkleTree.create(dataSet) + log.info("Success: " + tree.rootHash.mkString) log.info("start actor system") protected lazy val actorSystem = ActorSystem("lagonaki") - val dealer = actorSystem.actorOf(Props(classOf[TrustedDealer], dataSet)) - val miners: Seq[ActorRef] = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.hash))) - - miners.foreach(minerRef => minerRef ! Initialize) + val dealer = actorSystem.actorOf(Props(new TrustedDealer(tree))) + val storage = new AuthDataStorage(settings.authDataStorage) + val miners = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], tree.rootHash, storage), s"m-$x")) - Thread.sleep(2000) + implicit val timeout = Timeout(1 minute) - log.info("start BlockchainBuilder") + //Test, that miners download data from each other + miners.head ! Initialize(Seq(dealer)) + Thread.sleep(200) - val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners)) - blockchainBuilder ! BlockchainBuilderSpec.SendWorkToMiners + val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners, dealer), "BlockchainBuilder") + blockchainBuilder ! SendWorkToMiners } diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala index 1fe24753..40ca4a0f 100644 --- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala +++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala @@ -1,53 +1,86 @@ package scorex.perma.actors -import java.security.SecureRandom - import akka.actor.{Actor, ActorLogging, ActorRef} +import akka.util.Timeout import scorex.crypto.CryptographicHash._ import scorex.crypto.SigningFunctions.{PrivateKey, PublicKey, Signature} -import scorex.crypto.ads.merkle.{MerkleTree, AuthDataBlock} import scorex.crypto._ +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.crypto.ads.merkle.TreeStorage.Position import scorex.perma.BlockchainBuilderSpec.WinningTicket -import scorex.perma.Parameters import scorex.perma.actors.MinerSpec._ import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore} - - +import scorex.perma.consensus.{PartialProof, Ticket} +import scorex.perma.settings.Constants +import scorex.perma.settings.Constants.DataSegment +import scorex.storage.Storage +import scorex.utils._ + +import scala.collection.mutable.ListBuffer +import scala.concurrent.ExecutionContext.Implicits.global +import scala.concurrent.duration._ import scala.util.Try -case class PartialProof(signature: Signature, segmentIndex: Int, segment: AuthDataBlock[Parameters.DataSegment]) - -case class Ticket(publicKey: PublicKey, - s: Array[Byte], - proofs: IndexedSeq[PartialProof]) - -class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with ActorLogging { +class Miner(rootHash: Digest)(implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]]) + extends Actor with ActorLogging { import Miner._ private val keyPair = EllipticCurveImpl.createKeyPair(randomBytes(32)) + private implicit val timeout = Timeout(1.minute) + + private val segmentIds: Seq[Long] = 1.to(Constants.l).map { i => + u(keyPair._2, i - 1) + }.toSeq - private var segments: Subset = Map() + //Mutable list of ids, remaining to download + private var segmentToDownload: ListBuffer[Long] = segmentIds.to[ListBuffer] override def receive = { - case Initialize => - log.info("Initialize") + case Initialize(miners) => + log.debug("Initialize") - val segmentIdsToDownload = 1.to(Parameters.l).map { i => - u(keyPair._2, i - 1) - }.toArray + segmentToDownload foreach { s => + if (authDataStorage.containsKey(s)) { + segmentToDownload -= s + } + } + + if (segmentToDownload.nonEmpty) { + miners.foreach(_ ! SegmentsRequest(segmentToDownload)) + } + + case GetStatus => + log.debug("Get status") + if (segmentToDownload.isEmpty) { + sender ! Initialized + } else { + sender ! LoadingData + } - trustedDealerRef ! SegmentsRequest(segmentIdsToDownload) + case SegmentsRequest(ids) => + val segments: Subset = ids.map { x => + x -> authDataStorage.get(x) + }.toMap.collect { + case (key, Some(value)) => key -> value + } + log.info(s"Miner SegmentsRequest for ${ids.length} blocks returns ${segments.size} blocks") + sender ! SegmentsToStore(segments) case SegmentsToStore(sgs) => log.debug("SegmentsToStore({})", sgs) - require(segments.isEmpty) - segments = sgs + sgs.foreach { s => + if (segmentToDownload.contains(s._1) && s._2.check(s._1, rootHash)(Constants.hash)) { + authDataStorage.set(s._1, s._2) + segmentToDownload -= s._1 + } + } + authDataStorage.commit() case TicketGeneration(difficulty, puz) => log.debug("TicketGeneration({})", puz) - val ticket = generate(keyPair, puz, segments) + val ticket = generate(keyPair, puz) val check = validate(keyPair._2, puz, difficulty, ticket, rootHash) val score = ticketScore(ticket) @@ -56,8 +89,7 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act if (check) { sender() ! WinningTicket(puz, score, ticket) } else { - Thread.sleep(100) - self ! TicketGeneration(difficulty, puz) + context.system.scheduler.scheduleOnce(200.millis, self, TicketGeneration(difficulty, puz)) } case TicketValidation(difficulty, puz, t: Ticket) => @@ -73,20 +105,13 @@ object Miner { val NoSig = Array[Byte]() //calculate index of i-th segment - private def u(pubKey: SigningFunctions.PublicKey, i: Int): Int = { + private def u(pubKey: PublicKey, i: Int): Long = { val h = Sha256.hash(pubKey ++ BigInt(i).toByteArray) - BigInt(1, h).mod(Parameters.n).toInt - } - - - //todo: move to utils - def randomBytes(howMany: Int) = { - val r = new Array[Byte](howMany) - new SecureRandom().nextBytes(r) //overrides s - r + BigInt(1, h).mod(Constants.n).toLong } - def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte], segments: Subset): Ticket = { + def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]) + (implicit authDataStorage: Storage[Long, AuthDataBlock[DataSegment]]): Ticket = { val (privateKey, publicKey) = keyPair @@ -94,20 +119,19 @@ object Miner { val s = randomBytes(32) val sig0 = NoSig - val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Parameters.l).toInt) - .ensuring(r => segments.keySet.contains(r)) + val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Constants.l).toInt) - val proofs = 1.to(Parameters.k).foldLeft( + val proofs: IndexedSeq[PartialProof] = 1.to(Constants.k).foldLeft( (r1, sig0, Seq[PartialProof]()) ) { case ((ri, sig_prev, seq), _) => - val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segments(ri).data) + val segment = authDataStorage.get(ri).get + val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segment.data) val sig = EllipticCurveImpl.sign(privateKey, hi) - val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Parameters.l).toInt) - .ensuring(r => segments.keySet.contains(r)) + val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt) - (r_next, sig, seq :+ PartialProof(sig, ri, segments(ri))) - }._3.toIndexedSeq.ensuring(_.size == Parameters.k) + (r_next, sig, seq :+ PartialProof(sig, ri, segment)) + }._3.toIndexedSeq.ensuring(_.size == Constants.k) Ticket(publicKey, s, proofs) } @@ -119,17 +143,17 @@ object Miner { t: Ticket, rootHash: CryptographicHash.Digest): Boolean = Try { val proofs = t.proofs - require(proofs.size == Parameters.k) + require(proofs.size == Constants.k) //Local-POR lottery verification val sigs = NoSig +: proofs.map(_.signature) val ris = proofs.map(_.segmentIndex) - val partialProofsCheck = 1.to(Parameters.k).foldLeft(true) { case (partialResult, i) => + val partialProofsCheck = 1.to(Constants.k).foldLeft(true) { case (partialResult, i) => val segment = proofs(i - 1).segment - MerkleTree.check(ris(i - 1), rootHash, segment)() || { + segment.check(ris(i - 1), rootHash)() || { val hi = Sha256.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data) EllipticCurveImpl.verify(sigs(i), hi, publicKey) } @@ -145,13 +169,21 @@ object Miner { object MinerSpec { - type Index = Int - type Subset = Map[Index, AuthDataBlock[Parameters.DataSegment]] + type Subset = Map[Position, AuthDataBlock[DataSegment]] - case class Initialize() + case class Initialize(miners: Seq[ActorRef]) case class TicketGeneration(difficulty: BigInt, puz: Array[Byte]) case class TicketValidation(difficulty: BigInt, puz: Array[Byte], ticket: Ticket) + case object GetStatus + + sealed trait MinerStatus + + case object Initialized extends MinerStatus + + case object LoadingData extends MinerStatus + + } \ No newline at end of file diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala index 0777ad64..d1d9c84e 100644 --- a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala +++ b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala @@ -1,22 +1,18 @@ package scorex.perma.actors -import akka.actor.{ActorLogging, Actor} +import akka.actor.{Actor, ActorLogging} +import scorex.crypto.CryptographicHash import scorex.crypto.ads.merkle.MerkleTree -import scorex.perma.Parameters -import scorex.perma.Parameters.DataSegment import scorex.perma.actors.MinerSpec.Subset -import scorex.perma.actors.TrustedDealerSpec.{SegmentsToStore, SegmentsRequest} +import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore} +import scorex.perma.settings.Constants -class TrustedDealer(val dataSet: Array[DataSegment]) extends Actor with ActorLogging { - - val tree = MerkleTree.create(dataSet) +class TrustedDealer[H <: CryptographicHash](val tree: MerkleTree[H]) extends Actor with ActorLogging { override def receive = { case SegmentsRequest(segmentIds) => - log.info(s"SegmentsRequest(${segmentIds.mkString(", ")})") - - assert(segmentIds.length == Parameters.l) + log.info(s"Dealer SegmentsRequest for ${segmentIds.length} blocks") val segments: Subset = segmentIds.map { x => x -> tree.byIndex(x) @@ -34,7 +30,8 @@ object TrustedDealerSpec { case object PublishDataset - case class SegmentsRequest(segments: Array[Int]) + case class SegmentsRequest(segments: Seq[Long]) case class SegmentsToStore(segments: Subset) + } diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala new file mode 100644 index 00000000..4d6a96aa --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala @@ -0,0 +1,24 @@ +package scorex.perma.consensus + +import play.api.libs.functional.syntax._ +import play.api.libs.json.{JsPath, Reads, Writes} +import scorex.crypto.SigningFunctions._ +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.perma.settings.Constants._ +import scorex.utils.JsonSerialization + +case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[DataSegment]) + +object PartialProof extends JsonSerialization { + implicit val writes: Writes[PartialProof] = ( + (JsPath \ "signature").write[Bytes] and + (JsPath \ "segmentIndex").write[Long] and + (JsPath \ "segment").write[AuthDataBlock[DataSegment]] + ) (unlift(PartialProof.unapply)) + + implicit val reads: Reads[PartialProof] = ( + (JsPath \ "signature").read[Bytes] and + (JsPath \ "segmentIndex").read[Long] and + (JsPath \ "segment").read[AuthDataBlock[DataSegment]] + ) (PartialProof.apply _) +} diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala index 604c7cc5..ca754c00 100644 --- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala +++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala @@ -1,30 +1,91 @@ package scorex.perma.consensus -import play.api.libs.json.{JsObject, Json} +import com.google.common.primitives.{Bytes, Ints, Longs} +import play.api.libs.json._ import scorex.block.BlockField -import scorex.crypto.Base58 +import scorex.crypto.{Sha256, EllipticCurveImpl} +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.perma.settings.Constants +import scala.annotation.tailrec case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockData) extends BlockField[PermaLikeConsensusBlockData] { + import PermaConsensusBlockField._ - override val name: String = "perma-consensus" + override val name: String = PermaConsensusBlockField.FieldName - override def bytes: Array[Byte] = { - //todo: implement - ??? - } + override def bytes: Array[Byte] = + Bytes.ensureCapacity(Ints.toByteArray(value.target.toByteArray.length), 4, 0) ++ value.target.toByteArray ++ + Bytes.ensureCapacity(value.puz, PuzLength, 0) ++ + Bytes.ensureCapacity(value.ticket.publicKey, PublicKeyLength, 0) ++ + Bytes.ensureCapacity(value.ticket.s, SLength, 0) ++ + Bytes.ensureCapacity(Ints.toByteArray(value.ticket.proofs.length), 4, 0) ++ + value.ticket.proofs.foldLeft(Array.empty: Array[Byte]) { (b, p) => + val proofBytes = + Bytes.ensureCapacity(p.signature, SignatureLength, 0) ++ + Bytes.ensureCapacity(Longs.toByteArray(p.segmentIndex), 8, 0) ++ + Bytes.ensureCapacity(p.segment.data, Constants.segmentSize, 0) ++ + Bytes.ensureCapacity(Ints.toByteArray(p.segment.merklePath.length), 4, 0) ++ + p.segment.merklePath.foldLeft(Array.empty: Array[Byte]) { (acc, d) => + acc ++ d + } + b ++ proofBytes + } - override def json: JsObject = Json.obj(name -> Json.obj( - "difficulty" -> value.difficulty.toString(), - "puz" -> value.puz, - "s" -> value.ticket.s, - "segments" -> value.ticket.proofs.map { proof => - Json.obj( - "data" -> proof.segment.data, - "path" -> Json.arr(proof.segment.merklePath.map(Base58.encode)) - ) + override def json: JsObject = Json.obj(name -> Json.toJson(value)) +} + +object PermaConsensusBlockField { + + val FieldName = "perma-consensus" + val PuzLength = 32 + val PublicKeyLength = EllipticCurveImpl.KeyLength + val SLength = 32 + val HashLength = Sha256.DigestSize + val SignatureLength = EllipticCurveImpl.SignatureLength + + def parse(bytes: Array[Byte]): PermaConsensusBlockField = { + @tailrec + def parseProofs(from: Int, total: Int, current: Int, acc: IndexedSeq[PartialProof]): IndexedSeq[PartialProof] = { + if (current < total) { + val proofsStart = from + val signatureStart = proofsStart + SignatureLength + val dataStart = signatureStart + 8 + val merklePathStart = dataStart + Constants.segmentSize + + val signature = bytes.slice(proofsStart, proofsStart + SignatureLength) + val signatureIndex = Longs.fromByteArray(bytes.slice(signatureStart, signatureStart + 8)) + val blockData = bytes.slice(dataStart, dataStart + Constants.segmentSize) + val merklePathSize = Ints.fromByteArray(bytes.slice(merklePathStart, merklePathStart + 4)) + val merklePath = (0 until merklePathSize).map { i => + bytes.slice(merklePathStart + 4 + i * HashLength, merklePathStart + 4 + (i + 1) * HashLength) + } + parseProofs( + merklePathStart + 4 + merklePathSize * HashLength, + total, + current + 1, + PartialProof(signature, signatureIndex, AuthDataBlock(blockData, merklePath)) +: acc + ) + } else { + acc.reverse + } } - )) + + val targetSize = Ints.fromByteArray(bytes.take(4)) + val targetLength = 4 + targetSize + val proofsSize = Ints.fromByteArray(bytes.slice( + PuzLength + targetLength + PublicKeyLength + SLength, PuzLength + targetLength + PublicKeyLength + SLength + 4)) + + PermaConsensusBlockField(PermaLikeConsensusBlockData( + BigInt(bytes.slice(4, targetLength)), + bytes.slice(targetLength, PuzLength + targetLength), + Ticket( + bytes.slice(PuzLength + targetLength, PuzLength + targetLength + PublicKeyLength), + bytes.slice(PuzLength + targetLength + PublicKeyLength, PuzLength + targetLength + PublicKeyLength + SLength), + parseProofs(PuzLength + targetLength + PublicKeyLength + SLength + 4, proofsSize, 0, IndexedSeq.empty) + ) + )) + } } diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala index 423d3c80..74709ee1 100644 --- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala +++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala @@ -3,27 +3,65 @@ package scorex.perma.consensus import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount} import scorex.block.{Block, BlockField} import scorex.consensus.ConsensusModule -import scorex.perma.actors.Ticket -import scorex.transaction.TransactionModule +import scorex.crypto.CryptographicHash.Digest +import scorex.crypto.EllipticCurveImpl +import scorex.crypto.SigningFunctions._ +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.perma.settings.Constants +import scorex.perma.settings.Constants._ +import scorex.storage.Storage +import scorex.transaction.{BlockChain, TransactionModule} +import scorex.utils._ import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.Future +import scala.util.{Success, Try} /** * Data and functions related to a consensus algo */ +class PermaConsensusModule(rootHash: Array[Byte]) + (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]]) + extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging { -class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData] { + val InitialTarget = Constants.initialTarget + val initialTargetPow: BigInt = log2(InitialTarget) + val TargetRecalculation = Constants.targetRecalculation + val AvgDelay = Constants.averageDelay + val Hash = Constants.hash + val SSize = Hash.DigestSize + require(SSize == PermaConsensusBlockField.SLength) - val MiningReward = 1000000 - val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)) - val GenesisCreator = new PublicKeyAccount(Array()) + val GenesisCreator = new PublicKeyAccount(Array.fill(PermaConsensusBlockField.PublicKeyLength)(0: Byte)) + val Version: Byte = 1 + + implicit val consensusModule: ConsensusModule[PermaLikeConsensusBlockData] = this + + def miningReward(block: Block) = 1000000 private def blockGenerator(block: Block) = block.signerDataField.value.generator def isValid[TT](block: Block)(implicit transactionModule: TransactionModule[TT]): Boolean = { - //TODO - false + val f = block.consensusDataField.asInstanceOf[PermaConsensusBlockField] + val trans = transactionModule.history.asInstanceOf[BlockChain] + trans.parent(block) match { + case Some(parent) => + lazy val publicKey = blockGenerator(block).publicKey + lazy val puzIsValid = f.value.puz sameElements generatePuz(parent) + lazy val targetIsValid = f.value.target == calcTarget(parent) + lazy val ticketIsValid = validate(publicKey, f.value.puz, f.value.target, f.value.ticket, rootHash) + if (puzIsValid && targetIsValid && ticketIsValid) + true + else { + log.debug( + s"Non-valid block: puzIsValid=$puzIsValid, targetIsValid=$targetIsValid && ticketIsValid=$ticketIsValid" + ) + false + } + + case None => + true + } } /** @@ -31,46 +69,152 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData] * Meni Rosenfeld's Proof-of-Activity proposal http://eprint.iacr.org/2014/452.pdf */ def feesDistribution(block: Block): Map[Account, Long] = - Map(blockGenerator(block) -> MiningReward) + Map(blockGenerator(block) -> (miningReward(block) + block.transactions.map(_.fee).sum)) /** * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not * (see e.g. Meni Rosenfeld's Proof-of-Activity paper http://eprint.iacr.org/2014/452.pdf) - * @param block - * @return */ def generators(block: Block): Seq[Account] = Seq(blockGenerator(block)) - def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = BigInt(1) + def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = { + val score = initialTargetPow - + log2(block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].target) + if (score > 0) score else 1 + } def generateNextBlock[TT](account: PrivateKeyAccount) - (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = { - //TODO - Future(None) - } + (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = Try { + + val parent = transactionModule.history.asInstanceOf[BlockChain].lastBlock + val puz = generatePuz(parent) + + val keyPair = (account.privateKey, account.publicKey) + val ticket = generate(keyPair, puz) + val target = calcTarget(parent) + + if (validate(keyPair._2, puz, target, ticket, rootHash)) { + val timestamp = NTP.correctedTime() + val consensusData = PermaLikeConsensusBlockData(target, puz, ticket) + + Future(Some(Block.buildAndSign(Version, + timestamp, + parent.uniqueId, + consensusData, + transactionModule.packUnconfirmed(), + account))) + + } else { + Future(None) + } + }.recoverWith { case t: Throwable => + log.error("Error when creating new block", t) + t.printStackTrace() + Try(Future(None)) + }.getOrElse(Future(None)) override def consensusBlockData(block: Block): PermaLikeConsensusBlockData = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData] - override def parseBlockData(bytes: Array[Byte]): BlockField[PermaLikeConsensusBlockData] = ??? - - /* - PermaConsensusBlockField(new PermaLikeConsensusBlockData{ - ??? - })*/ - - override def genesisData: BlockField[PermaLikeConsensusBlockData] = - PermaConsensusBlockField(new PermaLikeConsensusBlockData { - override val difficulty = InitialDifficulty - override val puz = Array[Byte]() - override val ticket = Ticket(GenesisCreator.publicKey, Array(), IndexedSeq()) - }) + override def parseBlockData(bytes: Array[Byte]): PermaConsensusBlockField = PermaConsensusBlockField.parse(bytes) - /* - PermaConsensusBlockField(new PermaLikeConsensusBlockData { - override val generatorSignature: Array[Byte] = Array.fill(32)(0: Byte) - })*/ + override def genesisData: PermaConsensusBlockField = + PermaConsensusBlockField(PermaLikeConsensusBlockData( + InitialTarget, + Array.fill(PermaConsensusBlockField.PuzLength)(0: Byte), + Ticket(GenesisCreator.publicKey, Array.fill(PermaConsensusBlockField.SLength)(0: Byte), IndexedSeq()) + )) override def formBlockData(data: PermaLikeConsensusBlockData): BlockField[PermaLikeConsensusBlockData] = PermaConsensusBlockField(data) + + def generatePuz(block: Block) = Hash.hash(block.bytes) + + private val NoSig = Array[Byte]() + + //todo: validate r\i + private def validate(publicKey: PublicKey, + puz: Array[Byte], + target: BigInt, + t: Ticket, + rootHash: Digest): Boolean = Try { + val proofs = t.proofs + require(proofs.size == Constants.k) + require(t.s.length == SSize) + + //Local-POR lottery verification + + val sigs = NoSig +: proofs.map(_.signature) + val ris = proofs.map(_.segmentIndex) + + val partialProofsCheck = 1.to(Constants.k).foldLeft(true) { case (partialResult, i) => + val segment = proofs(i - 1).segment + + segment.check(ris(i - 1), rootHash)() || { + val hi = Hash.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data) + EllipticCurveImpl.verify(sigs(i), hi, publicKey) + } + } + partialProofsCheck && (ticketScore(t) < target) + }.getOrElse(false) + + private def ticketScore(t: Ticket): BigInt = if (t.proofs.nonEmpty) { + BigInt(1, Hash.hash(t.proofs.map(_.signature).reduce(_ ++ _))) + } else { + //Genesis block contains empty ticket + 0 + } + + private def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = { + + val (privateKey, publicKey) = keyPair + + //scratch-off for the Local-POR lottery + val s = randomBytes(SSize) + + val sig0 = NoSig + val r1 = u(publicKey, (BigInt(1, Hash.hash(puz ++ publicKey ++ s)) % Constants.l).toInt) + + val proofs: IndexedSeq[PartialProof] = 1.to(Constants.k).foldLeft( + (r1, sig0, Seq[PartialProof]()) + ) { + case ((ri, sig_prev, seq), _) => + val segment = authDataStorage.get(ri).get + val hi = Hash.hash(puz ++ publicKey ++ sig_prev ++ segment.data) + val sig = EllipticCurveImpl.sign(privateKey, hi) + val r_next = u(publicKey, BigInt(1, Hash.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt) + + (r_next, sig, seq :+ PartialProof(sig, ri, segment)) + }._3.toIndexedSeq.ensuring(_.size == Constants.k) + + Ticket(publicKey, s, proofs) + } + + //calculate index of i-th segment + private def u(pubKey: PublicKey, i: Int): Long = { + val h = Hash.hash(pubKey ++ BigInt(i).toByteArray) + BigInt(1, h).mod(Constants.n).toLong + } + + private def calcTarget(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = { + val trans = transactionModule.history.asInstanceOf[BlockChain] + val currentTarget = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].target + Try { + val height = trans.heightOf(block).get + if (height % TargetRecalculation == 0 && height > TargetRecalculation) { + val lastAvgDuration: BigInt = trans.averageDelay(block, TargetRecalculation).get + val newTarget = currentTarget * lastAvgDuration / 1000 / AvgDelay + log.debug(s"Height: $height, target:$newTarget vs $currentTarget, lastAvgDuration:$lastAvgDuration") + newTarget + } else { + currentTarget + } + }.recoverWith { case t: Throwable => + log.error(s"Error when calculating target: ${t.getMessage}") + t.printStackTrace() + Success(currentTarget) + }.getOrElse(currentTarget) + } + + private def log2(i: BigInt): BigInt = BigDecimal(math.log(i.doubleValue()) / math.log(2)).toBigInt() } \ No newline at end of file diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala index a1c50433..8661c1a6 100644 --- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala +++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala @@ -1,11 +1,23 @@ package scorex.perma.consensus -import scorex.perma.actors.Ticket +import play.api.libs.functional.syntax._ +import play.api.libs.json._ +import scorex.utils.JsonSerialization -//case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket) +case class PermaLikeConsensusBlockData(target: BigInt, puz: Array[Byte], ticket: Ticket) -trait PermaLikeConsensusBlockData { - val difficulty: BigInt - val puz: Array[Byte] - val ticket: Ticket -} +object PermaLikeConsensusBlockData extends JsonSerialization { + + implicit val writes: Writes[PermaLikeConsensusBlockData] = ( + (JsPath \ "difficulty").write[BigInt] and + (JsPath \ "puz").write[Bytes] and + (JsPath \ "ticket").write[Ticket] + ) (unlift(PermaLikeConsensusBlockData.unapply)) + + implicit val reads: Reads[PermaLikeConsensusBlockData] = ( + (JsPath \ "difficulty").read[BigInt] and + (JsPath \ "puz").read[Bytes] and + (JsPath \ "ticket").read[Ticket] + ) (PermaLikeConsensusBlockData.apply _) + +} \ No newline at end of file diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala new file mode 100644 index 00000000..a1cbc710 --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala @@ -0,0 +1,26 @@ +package scorex.perma.consensus + +import play.api.libs.functional.syntax._ +import play.api.libs.json.{JsPath, Reads, Writes} +import scorex.crypto.SigningFunctions._ +import scorex.utils.JsonSerialization + +case class Ticket(publicKey: PublicKey, + s: Array[Byte], + proofs: IndexedSeq[PartialProof]) + + +object Ticket extends JsonSerialization { + implicit val writes: Writes[Ticket] = ( + (JsPath \ "publicKey").write[PublicKey] and + (JsPath \ "s").write[Bytes] and + (JsPath \ "proofs").write[IndexedSeq[PartialProof]] + ) (unlift(Ticket.unapply)) + + implicit val reads: Reads[Ticket] = ( + (JsPath \ "publicKey").read[PublicKey] and + (JsPath \ "s").read[Bytes] and + (JsPath \ "proofs").read[IndexedSeq[PartialProof]] + ) (Ticket.apply _) + +} diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala new file mode 100644 index 00000000..093fcc12 --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala @@ -0,0 +1,89 @@ +package scorex.perma.consensus.http + +import javax.ws.rs.Path + +import akka.actor.ActorRefFactory +import com.wordnik.swagger.annotations._ +import play.api.libs.json.Json +import scorex.api.http.{ApiRoute, CommonApiFunctions} +import scorex.crypto.Base58 +import scorex.perma.consensus.PermaConsensusModule +import scorex.transaction.BlockChain +import spray.routing.Route + + +@Api(value = "/consensus", description = "Consensus-related calls") +class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain: BlockChain) + (implicit val context: ActorRefFactory) + extends ApiRoute with CommonApiFunctions { + + override val route: Route = + pathPrefix("consensus") { + algo ~ target ~ targetId ~ puz ~ puzId + } + + @Path("/target") + @ApiOperation(value = "Last target", notes = "Target of a last block", httpMethod = "GET") + def target = { + path("target") { + jsonRoute { + Json.obj("target" -> consensusModule.consensusBlockData(blockchain.lastBlock).target.toString).toString + } + } + } + + @Path("/target/{blockId}") + @ApiOperation(value = "Target of selected block", notes = "Target of a block with specified id", httpMethod = "GET") + @ApiImplicitParams(Array( + new ApiImplicitParam(name = "blockId", value = "Block id ", required = true, dataType = "String", paramType = "path") + )) + def targetId = { + path("target" / Segment) { case encodedSignature => + jsonRoute { + withBlock(blockchain, encodedSignature) { block => + Json.obj( + "target" -> consensusModule.consensusBlockData(block).target.toString + ) + }.toString + } + } + } + + @Path("/puz") + @ApiOperation(value = "Current puzzle", notes = "Current puzzle", httpMethod = "GET") + def puz = { + path("puz") { + jsonRoute { + Json.obj("puz" -> Base58.encode(consensusModule.generatePuz(blockchain.lastBlock))).toString + } + } + } + + @Path("/puz/{blockId}") + @ApiOperation(value = "Puzzle of selected block", notes = "Puzzle of a block with specified id", httpMethod = "GET") + @ApiImplicitParams(Array( + new ApiImplicitParam(name = "blockId", value = "Block id ", required = true, dataType = "String", paramType = "path") + )) + def puzId = { + path("puz" / Segment) { case encodedSignature => + jsonRoute { + withBlock(blockchain, encodedSignature) { block => + Json.obj( + "puz" -> Base58.encode(consensusModule.consensusBlockData(block).puz) + ) + }.toString + } + } + } + + + @Path("/algo") + @ApiOperation(value = "Consensus algo", notes = "Shows which consensus algo being using", httpMethod = "GET") + def algo = { + path("algo") { + jsonRoute { + Json.obj("consensus-algo" -> "perma").toString() + } + } + } +} diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala new file mode 100644 index 00000000..34c98b65 --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala @@ -0,0 +1,36 @@ +package scorex.perma.settings + +import com.typesafe.config.ConfigFactory +import scorex.crypto.Sha256 +import scorex.utils.ScorexLogging + +object Constants extends ScorexLogging { + + private val permaConf = ConfigFactory.load("perma").getConfig("perma") + + type DataSegment = Array[Byte] + + //few segments to be stored in a block, so segment size shouldn't be big + val segmentSize = permaConf.getInt("segmentSize") //segment size in bytes + + val n = permaConf.getLong("n") //how many segments in a dataset in total + + val l = permaConf.getInt("l") //how many segments to store for an each miner + + val k = permaConf.getInt("k") //number of iterations during scratch-off phase + + val initialTarget = BigInt(permaConf.getString("initialTarget")) + + val targetRecalculation = permaConf.getInt("targetRecalculation") //recalculate target every targetRecalculation blocks + + val averageDelay = permaConf.getInt("averageDelay") //average delay between blocks in seconds + + val hash = permaConf.getString("hash") match { + case s: String if s.equalsIgnoreCase("sha256") => + Sha256 + case hashFunction => + log.warn(s"Unknown hash function: $hashFunction. Use Sha256 instead.") + Sha256 + } + +} diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala b/scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala new file mode 100644 index 00000000..709d622c --- /dev/null +++ b/scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala @@ -0,0 +1,15 @@ +package scorex.perma.settings + +import play.api.libs.json.JsObject + +trait PermaSettings { + val settingsJSON: JsObject + + private val DefaultTreeDir = "/tmp/scorex/perma/" + private val DefaultAuthDataStorage = DefaultTreeDir + "authDataStorage.mapDB" + + lazy val treeDir = (settingsJSON \ "perma" \ "treeDir").asOpt[String].getOrElse(DefaultTreeDir) + lazy val authDataStorage = + (settingsJSON \ "perma" \ "authDataStorage").asOpt[String].getOrElse(DefaultAuthDataStorage) + +} diff --git a/scorex-perma/src/test/resources/settings-test.json b/scorex-perma/src/test/resources/settings-test.json new file mode 100644 index 00000000..026772fc --- /dev/null +++ b/scorex-perma/src/test/resources/settings-test.json @@ -0,0 +1,5 @@ +{ + "perma": { + "treeDir": "/tmp/scorex/perma/test/" + } +} \ No newline at end of file diff --git a/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala b/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala new file mode 100644 index 00000000..3abd2f41 --- /dev/null +++ b/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala @@ -0,0 +1,8 @@ +package scorex + +import org.scalatest.Suites + +class PermaTestSuite extends Suites ( + new props.AuthDataStorageSpecification, + new props.PermaConsensusBlockFiendSpecification +) diff --git a/scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala b/scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala new file mode 100644 index 00000000..b32fc3f3 --- /dev/null +++ b/scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala @@ -0,0 +1,36 @@ +package scorex.props + +import java.io.File + +import org.scalacheck.{Arbitrary, Gen} +import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks} +import org.scalatest.{Matchers, PropSpec} +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.perma.Storage.AuthDataStorage + + +class AuthDataStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers { + + val treeDirName = "/tmp/scorex/test/AuthDataStorageSpecification/" + val treeDir = new File(treeDirName) + treeDir.mkdirs() + + val keyVal = for { + key: Long <- Arbitrary.arbitrary[Long] + value <- Arbitrary.arbitrary[String] + } yield (key, AuthDataBlock(value.getBytes, Seq())) + + + property("set value and get it") { + lazy val storage = new AuthDataStorage(treeDirName + "/test_db") + + forAll(keyVal) { x => + val key = x._1 + val value = x._2 + storage.set(key, value) + + assert(storage.get(key).get.data sameElements value.data) + } + storage.close() + } +} \ No newline at end of file diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala new file mode 100644 index 00000000..030684c7 --- /dev/null +++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala @@ -0,0 +1,79 @@ +package scorex.props + +import java.io.File + +import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks} +import org.scalatest.{Matchers, PropSpec} +import scorex.crypto.ads.merkle.AuthDataBlock +import scorex.perma.Storage.AuthDataStorage +import scorex.perma.consensus._ +import scorex.perma.settings.Constants.DataSegment +import scorex.perma.settings.{Constants, PermaSettings} +import scorex.settings.Settings +import scorex.storage.Storage +import scorex.utils._ + +class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers { + + implicit val settings = new Settings with PermaSettings { + val filename = "settings-test.json" + } + new File(settings.treeDir).mkdirs() + implicit lazy val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage) + val consensus = new PermaConsensusModule(randomBytes(32)) + + property("Encode to bytes round-trip") { + forAll { (diff: Long, segmentIndex: Long) => + + val puz = randomBytes(PermaConsensusBlockField.PuzLength) + val pubkey = randomBytes(PermaConsensusBlockField.PublicKeyLength) + val s = randomBytes(PermaConsensusBlockField.SLength) + val signature = randomBytes(PermaConsensusBlockField.SignatureLength) + val signature2 = randomBytes(PermaConsensusBlockField.SignatureLength) + val blockdata = randomBytes(Constants.segmentSize) + val hash1 = randomBytes(PermaConsensusBlockField.HashLength) + val hash2 = randomBytes(PermaConsensusBlockField.HashLength) + + val authDataBlock: AuthDataBlock[DataSegment] = AuthDataBlock(blockdata, Seq(hash1, hash2)) + val initialBlock = PermaConsensusBlockField(PermaLikeConsensusBlockData( + math.abs(diff), + puz, + Ticket(pubkey, s, IndexedSeq( + PartialProof(signature, segmentIndex, authDataBlock), + PartialProof(signature2, segmentIndex, authDataBlock) + )) + )) + val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes) + + checkAll(initialBlock, parsedBlock) + + } + } + + property("Encode to bytes round-trip for genesis") { + val initialBlock = consensus.genesisData + val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes) + checkAll(initialBlock, parsedBlock) + } + + def checkAll(initialBlock: PermaConsensusBlockField, parsedBlock: PermaConsensusBlockField): Unit = { + parsedBlock.value.target shouldBe initialBlock.value.target + assert(parsedBlock.value.puz sameElements initialBlock.value.puz) + assert(parsedBlock.value.ticket.publicKey sameElements initialBlock.value.ticket.publicKey) + assert(parsedBlock.value.ticket.s sameElements initialBlock.value.ticket.s) + parsedBlock.value.ticket.proofs.size shouldBe initialBlock.value.ticket.proofs.size + parsedBlock.value.ticket.proofs.indices.foreach { i => + val parsedProof = parsedBlock.value.ticket.proofs(i) + val initialProof = initialBlock.value.ticket.proofs(i) + assert(parsedProof.signature sameElements initialProof.signature) + parsedProof.segmentIndex shouldBe initialProof.segmentIndex + assert(parsedProof.segment.data sameElements initialProof.segment.data) + parsedProof.segment.merklePath.size shouldBe initialProof.segment.merklePath.size + + if (initialProof.segment.merklePath.nonEmpty) { + assert(parsedProof.segment.merklePath.head sameElements initialProof.segment.merklePath.head) + assert(parsedProof.segment.merklePath.last sameElements initialProof.segment.merklePath.last) + } + } + } +} \ No newline at end of file diff --git a/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala b/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala index 68a7bf99..c2e76ac0 100644 --- a/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala +++ b/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala @@ -16,7 +16,7 @@ case class BlocksApiRoute(blockchain: BlockChain, wallet: Wallet)(implicit val c override lazy val route = pathPrefix("blocks") { - signature ~ first ~ last ~ at ~ height ~ heightEncoded ~ child ~ address + signature ~ first ~ last ~ at ~ height ~ heightEncoded ~ child ~ address ~ delay } @Path("/address/{address}") @@ -49,6 +49,26 @@ case class BlocksApiRoute(blockchain: BlockChain, wallet: Wallet)(implicit val c } } + @Path("/delay/{height}/{blockNum}") + @ApiOperation(value = "Average delay", notes = "Average delay in milliseconds between last $blockNum blocks starting from $height", httpMethod = "GET") + @ApiImplicitParams(Array( + new ApiImplicitParam(name = "height", value = "Height of block", required = true, dataType = "String", paramType = "path"), + new ApiImplicitParam(name = "blockNum", value = "Number of blocks to count delay", required = true, dataType = "String", paramType = "path") + )) + def delay: Route = { + path("delay" / IntNumber / IntNumber) { case (height, count) => + jsonRoute { + blockchain.blockAt(height) match { + case Some(block) => + blockchain.averageDelay(block, count).map(d => Json.obj("delay" -> d)) + .getOrElse(Json.obj("status" -> "error", "details" -> "Internal error")).toString + case None => + Json.obj("status" -> "error", "details" -> "No block for this height").toString() + } + } + } + } + @Path("/height/{signature}") @ApiOperation(value = "Height", notes = "Get height of a block by its Base58-encoded signature", httpMethod = "GET") @ApiImplicitParams(Array( diff --git a/settings.json b/settings.json index c41150cc..a6031540 100644 --- a/settings.json +++ b/settings.json @@ -1,18 +1,19 @@ { - "bindAddress" : "127.0.0.1", - "knownpeers":[ + "bindAddress": "127.0.0.1", + "knownpeers": [ "5.9.86.168" ], - "walletdir" : "/tmp/scorex/wallet", - "walletpassword" : "cookies", - "walletseed" : "FQgbSAm6swGbtqA3NE8PttijPhT4N3Ufh4bHFAkyVnQz", - "datadir" : "/tmp/scorex/data", + "walletdir": "/tmp/scorex/wallet", + "walletpassword": "cookies", + "walletseed": "FQgbSAm6swGbtqA3NE8PttijPhT4N3Ufh4bHFAkyVnQz", + "datadir": "/tmp/scorex/data", "minconnections": 10, - "rpcport" : 9085, - "rpcallowed":[ + "rpcport": 9085, + "rpcallowed": [ "127.0.0.1", "123.123.123.123" ], - "max-rollback" : 100, - "offline-generation" : true + "max-rollback": 100, + "blockGenerationDelay": 0, + "offline-generation": true } \ No newline at end of file diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf index b5626a0e..2ca36001 100644 --- a/src/main/resources/application.conf +++ b/src/main/resources/application.conf @@ -2,5 +2,5 @@ app { product = "Scorex" release = "Lagonaki" version = "1.1.3-SNAPSHOT" - consensusAlgo = "Nxt" + consensusAlgo = "Perma" } diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala index a3d7d3a5..4edc428c 100644 --- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala +++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala @@ -7,6 +7,7 @@ import scorex.block.Block import scorex.lagonaki.network.BlockchainSyncer._ import scorex.lagonaki.network.message.{BlockMessage, GetSignaturesMessage} import scorex.lagonaki.server.LagonakiApplication +import scorex.settings.Settings import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.duration._ @@ -15,9 +16,11 @@ import scala.util.{Failure, Success} case class NewBlock(block: Block, sender: Option[InetSocketAddress]) -class BlockchainSyncer(application: LagonakiApplication, networkController: ActorRef) extends FSM[Status, Unit] { +class BlockchainSyncer(application: LagonakiApplication, networkController: ActorRef, settings: Settings) + extends FSM[Status, Unit] { private val stateTimeout = 1.second + val blockGenerationDelay = settings.blockGenerationDelay startWith(Offline, Unit) @@ -60,10 +63,13 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto scoreOpt, onNone = () => { tryToGenerateABlock() + context.system.scheduler.scheduleOnce(blockGenerationDelay, networkController, GetMaxChainScore) stay() }, onLocal = () => { tryToGenerateABlock() + networkController ! GetMaxChainScore + context.system.scheduler.scheduleOnce(blockGenerationDelay, networkController, GetMaxChainScore) stay() } ) @@ -103,9 +109,10 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto def processNewBlock(block: Block, remoteOpt: Option[InetSocketAddress]) = { val fromStr = remoteOpt.map(_.toString).getOrElse("local") if (block.isValid) { - log.info(s"New block: $block from $fromStr") application.storedState.processBlock(block) application.blockchainImpl.appendBlock(block) + log.info(s"New block: $block from $fromStr. New size: ${application.blockchainImpl.height()}, " + + s"score: ${application.blockchainImpl.score()}, timestamp: ${block.timestampField.value}") block.transactionModule.clearFromUnconfirmed(block.transactionDataField.value) val height = application.blockchainImpl.height() @@ -123,7 +130,7 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto val consModule = application.consensusModule implicit val transModule = application.transactionModule - log.info("Trying to generate a new block") +// log.info("Trying to generate a new block") val accounts = application.wallet.privateKeyAccounts() consModule.generateNextBlocks(accounts)(transModule) onComplete { case Success(blocks: Seq[Block]) => diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala index 9a4b88d2..9f86cb5a 100644 --- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala +++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala @@ -1,29 +1,38 @@ package scorex.lagonaki.server +import java.io.{File, RandomAccessFile} +import java.nio.file.{Files, Paths} + import akka.actor.Props import akka.io.IO import com.typesafe.config.ConfigFactory import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount} import scorex.api.http._ import scorex.app.Application -import scorex.consensus.LagonakiConsensusModule -import scorex.consensus.nxt.api.http.NxtConsensusApiRoute -import scorex.consensus.qora.api.http.QoraConsensusApiRoute -import scorex.lagonaki.api.http.{PeersHttpService, PaymentApiRoute, ScorexApiRoute} import scorex.block.Block import scorex.consensus.nxt.NxtLikeConsensusModule +import scorex.consensus.nxt.api.http.NxtConsensusApiRoute import scorex.consensus.qora.QoraLikeConsensusModule +import scorex.consensus.qora.api.http.QoraConsensusApiRoute +import scorex.crypto.ads.merkle.{AuthDataBlock, MerkleTree} +import scorex.lagonaki.api.http.{PaymentApiRoute, PeersHttpService, ScorexApiRoute} import scorex.lagonaki.network.message._ import scorex.lagonaki.network.{BlockchainSyncer, NetworkController} +import scorex.perma.Storage.AuthDataStorage +import scorex.perma.consensus.PermaConsensusModule +import scorex.perma.consensus.http.PermaConsensusApiRoute +import scorex.perma.settings.Constants +import scorex.perma.settings.Constants._ +import scorex.storage.Storage import scorex.transaction.LagonakiTransaction.ValidationResult import scorex.transaction._ import scorex.transaction.state.database.UnconfirmedTransactionsDatabaseImpl import scorex.transaction.state.wallet.{Payment, Wallet} import scorex.utils.{NTP, ScorexLogging} import spray.can.Http -import scala.reflect.runtime.universe._ import scala.concurrent.ExecutionContext.Implicits.global +import scala.reflect.runtime.universe._ class LagonakiApplication(val settingsFilename: String) extends Application with ScorexLogging { @@ -40,6 +49,40 @@ class LagonakiApplication(val settingsFilename: String) new NxtLikeConsensusModule case s: String if s.equalsIgnoreCase("qora") => new QoraLikeConsensusModule + case s: String if s.equalsIgnoreCase("perma") => + val tree = if (Files.exists(Paths.get(settings.treeDir + "/tree0.mapDB"))) { + log.info("Get existing tree") + new MerkleTree(settings.treeDir, Constants.n, Constants.segmentSize, Constants.hash) + } else { + log.info("Generating random data set") + val treeDir = new File(settings.treeDir) + treeDir.mkdirs() + val datasetFile = settings.treeDir + "/data.file" + new RandomAccessFile(datasetFile, "rw").setLength(Constants.n * Constants.segmentSize) + log.info("Calculate tree") + val tree = MerkleTree.fromFile(datasetFile, settings.treeDir, Constants.segmentSize, Constants.hash) + require(tree.nonEmptyBlocks == Constants.n, s"${tree.nonEmptyBlocks} == ${Constants.n}") + tree + } + + log.info("Test tree") + val index = Constants.n - 3 + val leaf = tree.byIndex(index).get + require(leaf.check(index, tree.rootHash)(Constants.hash)) + + log.info("Put ALL data to local storage") + new File(settings.treeDir).mkdirs() + val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage) + def addBlock(i: Long): Unit = { + authDataStorage.set(i, tree.byIndex(i).get) + if (i > 0) { + addBlock(i - 1) + } + } + addBlock(Constants.n - 1) + + log.info("Create consensus module") + new PermaConsensusModule(tree.rootHash)(authDataStorage) case algo => log.error(s"Unknown consensus algo: $algo. Use NxtLikeConsensusModule instead.") new NxtLikeConsensusModule @@ -48,7 +91,7 @@ class LagonakiApplication(val settingsFilename: String) override implicit val transactionModule: SimpleTransactionModule = new SimpleTransactionModule lazy val networkController = actorSystem.actorOf(Props(classOf[NetworkController], this)) - lazy val blockchainSyncer = actorSystem.actorOf(Props(classOf[BlockchainSyncer], this, networkController)) + lazy val blockchainSyncer = actorSystem.actorOf(Props(classOf[BlockchainSyncer], this, networkController, settings)) private lazy val walletFileOpt = settings.walletDirOpt.map(walletDir => new java.io.File(walletDir, "wallet.s.dat")) implicit lazy val wallet = new Wallet(walletFileOpt, settings.walletPassword, settings.walletSeed.get) @@ -61,6 +104,8 @@ class LagonakiApplication(val settingsFilename: String) new NxtConsensusApiRoute(ncm, blockchainImpl) case qcm: QoraLikeConsensusModule => new QoraConsensusApiRoute(qcm, blockchainImpl) + case pcm: PermaConsensusModule => + new PermaConsensusApiRoute(pcm, blockchainImpl) } override lazy val apiRoutes = Seq( @@ -81,6 +126,7 @@ class LagonakiApplication(val settingsFilename: String) consensusApiRoute match { case nxt: NxtConsensusApiRoute => typeOf[NxtConsensusApiRoute] case qora: QoraConsensusApiRoute => typeOf[QoraConsensusApiRoute] + case pcm: PermaConsensusApiRoute => typeOf[PermaConsensusApiRoute] }, typeOf[WalletApiRoute], typeOf[PaymentApiRoute], diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala b/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala index 22ec7496..db28f34d 100644 --- a/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala +++ b/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala @@ -1,6 +1,7 @@ package scorex.lagonaki.server +import scorex.perma.settings.PermaSettings import scorex.settings.Settings import scorex.transaction.TransactionSettings -class LagonakiSettings(override val filename:String) extends Settings with TransactionSettings \ No newline at end of file +class LagonakiSettings(override val filename: String) extends Settings with TransactionSettings with PermaSettings \ No newline at end of file