From 5d339b7c18eb3ecc8a9cdca5060472a1bc3b2ab0 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Mon, 16 Nov 2015 23:54:00 +0300
Subject: [PATCH 01/66] make merge method private

---
 .../src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala    | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 353b589f..69d38f5a 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -141,7 +141,7 @@ object MerkleTree {
     new MerkleTree(makeTree(leaves, hashFunction), leaves)
   }
 
-  def merge[Block, Hash <: CryptographicHash](
+  private def merge[Block, Hash <: CryptographicHash](
                                                leftChild: Tree[Block, Hash],
                                                rightChild: Tree[Block, Hash],
                                                hashFunction: Hash): Node[Block, Hash] = {

From 357b0fbaabb01c52ba02321aa667ff5c7d356778 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Tue, 17 Nov 2015 09:11:10 +0300
Subject: [PATCH 02/66] headoption, 32 bytes puz

---
 scorex-perma/src/main/scala/scorex/perma/TestApp.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index 0ddf2808..e50f8bb8 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -23,9 +23,9 @@ class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging
   val blockchainLike = mutable.Buffer[BlockHeaderLike]()
 
 
-  private def calcPuz = 1.to(100).toArray.map(_ => Random.nextInt(256).toByte)
+  private def calcPuz = 1.to(32).toArray.map(_ => Random.nextInt(256).toByte)
 
-  def difficulty = blockchainLike.lastOption.map(_.difficulty).getOrElse(InitialDifficulty)
+  def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
 
   override def receive = {
     case SendWorkToMiners =>

From d541cc82a274828d78d9dcc92de00d6ebece681e Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Tue, 17 Nov 2015 09:15:08 +0300
Subject: [PATCH 03/66] BlockchainBuilder.scala

---
 .../scorex/perma/BlockchainBuilder.scala      | 54 +++++++++++++++++++
 .../src/main/scala/scorex/perma/TestApp.scala | 42 ---------------
 2 files changed, 54 insertions(+), 42 deletions(-)
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala

diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
new file mode 100644
index 00000000..c3078a4c
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -0,0 +1,54 @@
+package scorex.perma
+
+import akka.actor.{Actor, ActorRef}
+import scorex.perma.actors.MinerSpec.TicketGeneration
+import scorex.perma.actors.Ticket
+import scorex.utils.ScorexLogging
+
+import scala.collection.mutable
+import scala.util.Random
+
+
+case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
+
+class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging {
+  import BlockchainBuilderSpec._
+
+  var puz: Array[Byte] = calcPuz
+
+  val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte))
+  val blockchainLike = mutable.Buffer[BlockHeaderLike]()
+
+
+  private def calcPuz = 1.to(32).toArray.map(_ => Random.nextInt(256).toByte)
+
+  def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
+
+  override def receive = {
+    case SendWorkToMiners =>
+      miners.foreach { minerRef =>
+        minerRef ! TicketGeneration(difficulty, puz)
+      }
+
+    //miners are honest (lol) in our setting, so no validation here
+    case WinningTicket(minerPuz, score, ticket) =>
+      if (minerPuz sameElements puz) {
+        val newBlock = BlockHeaderLike(score, puz, ticket)
+        log.info(s"Block generated: $newBlock, blockchain size: ${blockchainLike.size}")
+        blockchainLike += newBlock
+        puz = calcPuz
+        self ! SendWorkToMiners
+      } else {
+        sender() ! TicketGeneration(difficulty, puz)
+        log.debug("Wrong puz from miner: " + minerPuz.mkString)
+      }
+  }
+}
+
+object BlockchainBuilderSpec {
+
+  case object SendWorkToMiners
+
+  case class WinningTicket(puz: Array[Byte], score: BigInt, t: Ticket)
+
+}
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index e50f8bb8..15bf2f19 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -13,48 +13,6 @@ import scala.collection.mutable
 import scala.util.Random
 
 
-case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
-
-class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging {
-
-  var puz: Array[Byte] = calcPuz
-
-  val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte))
-  val blockchainLike = mutable.Buffer[BlockHeaderLike]()
-
-
-  private def calcPuz = 1.to(32).toArray.map(_ => Random.nextInt(256).toByte)
-
-  def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
-
-  override def receive = {
-    case SendWorkToMiners =>
-      miners.foreach { minerRef =>
-        minerRef ! TicketGeneration(difficulty, puz)
-      }
-
-    //miners are honest (lol) in our setting, so no validation here
-    case WinningTicket(minerPuz, score, ticket) =>
-      if (minerPuz sameElements puz) {
-        val newBlock = BlockHeaderLike(score, puz, ticket)
-        log.info(s"Block generated: $newBlock, blockchain size: ${blockchainLike.size}")
-        blockchainLike += newBlock
-        puz = calcPuz
-        self ! SendWorkToMiners
-      } else {
-        sender() ! TicketGeneration(difficulty, puz)
-        log.debug("Wrong puz from miner: " + minerPuz.mkString)
-      }
-  }
-}
-
-object BlockchainBuilderSpec {
-
-  case object SendWorkToMiners
-
-  case class WinningTicket(puz: Array[Byte], score: BigInt, t: Ticket)
-
-}
 
 object TestApp extends App {
 

From 9bec7f9c034e929b5e991a5216db1b2446e5531d Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 17 Nov 2015 01:06:06 +0300
Subject: [PATCH 04/66] Merkle reimplementation

---
 scorex-basics/build.sbt                       |   1 +
 .../crypto/ads/merkle/AuthDataBlock.scala     |  43 ++++
 .../crypto/ads/merkle/MapDBStorage.scala      |  42 ++++
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 213 +++++++-----------
 .../scorex/crypto/ads/merkle/Storage.scala    |  23 ++
 .../scorex/props/MerkleSpecification.scala    |  41 ++--
 .../src/main/scala/scorex/perma/TestApp.scala |   8 +-
 .../scala/scorex/perma/actors/Miner.scala     |   2 +-
 .../scorex/perma/actors/TrustedDealer.scala   |  28 +--
 9 files changed, 240 insertions(+), 161 deletions(-)
 create mode 100644 scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
 create mode 100644 scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
 create mode 100644 scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala

diff --git a/scorex-basics/build.sbt b/scorex-basics/build.sbt
index de26b18f..20aeab43 100644
--- a/scorex-basics/build.sbt
+++ b/scorex-basics/build.sbt
@@ -5,6 +5,7 @@ libraryDependencies ++=
     Dependencies.akka ++
     Dependencies.spray ++
     Dependencies.testKit ++
+    Dependencies.db ++
     Dependencies.logging ++ Seq(
       "net.vrallev.ecc" % "ecc-25519-java" % "+",
       "commons-net" % "commons-net" % "3.+"
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
new file mode 100644
index 00000000..21c6a7b2
--- /dev/null
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -0,0 +1,43 @@
+package scorex.crypto.ads.merkle
+
+import scorex.crypto.{Sha256, CryptographicHash}
+import scorex.crypto.CryptographicHash._
+
+import scala.annotation.tailrec
+
+/**
+  * @param data - data block
+  * @param merklePath - merkle path, complementary to data block
+  */
+case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
+
+  def check[Hash <: CryptographicHash](index: Int, rootHash: Digest)
+                                      (hashFunction: Hash = Sha256): Boolean = {
+
+    @tailrec
+    def calculateHash(i: Int, nodeHash: Digest, path: Seq[Digest]): Digest = {
+      if (i % 2 == 0) {
+        val hash = hashFunction.hash(nodeHash ++ path.head)
+        if (path.size == 1) {
+          hash
+        } else {
+          calculateHash(i / 2, hash, path.tail)
+        }
+      } else {
+        val hash = hashFunction.hash(path.head ++ nodeHash)
+        if (path.size == 1) {
+          hash
+        } else {
+          calculateHash(i / 2, hash, path.tail)
+        }
+      }
+    }
+    if(merklePath.nonEmpty) {
+      val calculated = calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath)
+      calculated.mkString == rootHash.mkString
+    } else {
+      true
+    }
+  }
+}
+
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
new file mode 100644
index 00000000..8ccd02c6
--- /dev/null
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -0,0 +1,42 @@
+package scorex.crypto.ads.merkle
+
+import java.io.File
+import java.util.concurrent.ConcurrentNavigableMap
+
+import org.mapdb.DBMaker
+import scorex.crypto.CryptographicHash.Digest
+
+import scala.util.Try
+
+class MapDBStorage(file: File) extends Storage {
+
+  import Storage._
+
+  val db = DBMaker.appendFileDB(file)
+    .fileMmapEnableIfSupported()
+    .closeOnJvmShutdown()
+    .checksumEnable()
+    .make()
+
+  val map: ConcurrentNavigableMap[String, Digest] = db.treeMap("tree")
+
+  override def set(key: Key, value: Digest): Try[Digest] = {
+    Try {
+      map.put(stringKey(key), value)
+    }
+  }
+
+
+  override def commit(): Unit = {
+    db.commit()
+    db.close()
+  }
+
+  override def get(key: Key): Option[Digest] = {
+    Option(map.get(stringKey(key)))
+  }
+
+  private def stringKey(key: Key): String = {
+    key._1 + "_" + key._2
+  }
+}
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 69d38f5a..824706e2 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -1,175 +1,126 @@
 package scorex.crypto.ads.merkle
 
+import java.io.{File, FileInputStream, FileOutputStream}
+import java.nio.file.{Files, Paths}
+
 import scorex.crypto.CryptographicHash.Digest
 import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
-import scala.math
-
-/**
-  * @param data - data block
-  * @param merklePath - merkle path, complementary to data block
-  */
-case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest])
 
-//bottom up
+class MerkleTree[H <: CryptographicHash](treeFolder: String,
+                                         val nonEmptyBlocks: Int,
+                                         blockSize: Int = 1024,
+                                         hash: H = Sha256
+                                        ) {
 
-trait MerkleTreeI[Block] {
-  def byIndex(n: Int): Option[AuthDataBlock[Block]]
-}
+  import MerkleTree._
 
-//todo: check/optimize the code
+  lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
 
-object MerkleTree {
+  val level = calculateRequiredLevel(nonEmptyBlocks)
 
-  def check[Block, Hash <: CryptographicHash](index: Int, rootHash: Digest, block: AuthDataBlock[Block])
-                                             (hashFunction: Hash = Sha256): Boolean = {
+  lazy val rootHash: Digest = getHash((level, 0)).get
 
-    @tailrec
-    def calculateHash(i: Int, nodeHash: Digest, path: Seq[Digest]): Digest = {
-      if (i % 2 == 0) {
-        val hash = hashFunction.hash(nodeHash ++ path.head)
-        if (path.size == 1) {
-          hash
-        } else {
-          calculateHash(i / 2, hash, path.tail)
-        }
-      } else {
-        val hash = hashFunction.hash(path.head ++ nodeHash)
-        if (path.size == 1) {
-          hash
-        } else {
-          calculateHash(i / 2, hash, path.tail)
-        }
-      }
-    }
-    val calculated = calculateHash(index, hashFunction.hash(block.data.toString.getBytes), block.merklePath)
-    calculated.mkString == rootHash.mkString
-  }
-
-  class MerkleTree[Block, Hash <: CryptographicHash](val tree: Tree[Block, Hash], val leaves: Seq[Tree[Block, Hash]])
-    extends MerkleTreeI[Block] {
-
-    val Size = leaves.size
-    lazy val rootNode: Node[Block, Hash] = tree.asInstanceOf[Node[Block, Hash]]
-    lazy val hash = tree.hash
-
-    def byIndex(index: Int): Option[AuthDataBlock[Block]] = {
+  def byIndex(index: Int): Option[AuthDataBlock[Block]] = {
+    if (index < nonEmptyBlocks && index >= 0) {
       @tailrec
-      def calculateTreePath(n: Int, node: Node[Block, Hash], levelSize: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
-        val halfLevelSize: Int = levelSize / 2
-        if (n < halfLevelSize) {
-          node.leftChild match {
-            case nd: Node[Block, Hash] =>
-              calculateTreePath(n, nd, halfLevelSize, node.rightChild.hash +: acc)
-            case _ =>
-              node.rightChild.hash +: acc
+      def calculateTreePath(n: Int, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
+        if (currentLevel < level) {
+          //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
+          if (n % 2 == 0) {
+            getHash((currentLevel, n + 1)) match {
+              case Some(v) =>
+                calculateTreePath(n / 2, currentLevel + 1, v +: acc)
+              case None =>
+                acc.reverse
+            }
+          } else {
+            calculateTreePath(n / 2, currentLevel + 1, getHash((currentLevel, n - 1)).get +: acc)
           }
         } else {
-          node.rightChild match {
-            case nd: Node[Block, Hash] =>
-              calculateTreePath(n - halfLevelSize, nd, halfLevelSize, node.leftChild.hash +: acc)
-            case _ =>
-              node.leftChild.hash +: acc
-          }
+          acc.reverse
         }
       }
 
-      leaves.lift(index).flatMap(l =>
-        l match {
-          case Leaf(data: Block) =>
-            val treePath = calculateTreePath(index, rootNode, Size)
-            Some(AuthDataBlock(data, treePath))
-          case _ =>
-            None
-        }
-      )
+      val path = Paths.get(treeFolder + "/" + index)
+      val data: Block = Files.readAllBytes(path)
+      val treePath = calculateTreePath(index, 0)
+      Some(AuthDataBlock(data, treePath))
+    } else {
+      None
     }
+  }
 
-    override def toString: String = {
-      def printHashes(node: Tree[Any, Hash], prefix: String = ""): List[String] = {
-        node match {
-          case Node(leftChild: Tree[Block, Hash], rightChild: Tree[Block, Hash]) =>
-            (prefix + node.hash.mkString) :: printHashes(leftChild, " " + prefix) ++
-              printHashes(rightChild, " " + prefix)
-          case l: Leaf[Block, Hash] =>
-            List(prefix + l.hash.mkString)
-          case _ =>
-            List()
+  def getHash(key: Storage.Key): Option[Digest] = {
+    storage.get(key) match {
+      case None =>
+        if (key._1 > 0) {
+          val h1 = getHash((key._1 - 1, key._2 * 2))
+          val h2 = getHash((key._1 - 1, key._2 * 2 + 1))
+          (h1, h2) match {
+            case (Some(hash1), Some(hash2)) => Some(hash.hash(hash1 ++ hash2))
+            case (Some(h), _) => h1
+            case (_, Some(h)) => h2
+            case _ => None
+          }
+        } else {
+          None
         }
-      }
-      printHashes(tree).mkString("\n")
+      case digest =>
+        digest
     }
-  }
 
-
-  sealed trait Tree[+Block, Hash <: CryptographicHash] {
-    val hash: Digest
   }
 
-  case class Node[+Block, Hash <: CryptographicHash](
-                                                      leftChild: Tree[Block, Hash],
-                                                      rightChild: Tree[Block, Hash])(hashFunction: Hash)
-    extends Tree[Block, Hash] {
-
-    override val hash: Digest = hashFunction.hash(leftChild.hash ++ rightChild.hash)
 
-  }
+}
 
-  case class Leaf[+Block, Hash <: CryptographicHash](data: Block)(hashFunction: Hash)
-    extends Tree[Block, Hash] {
+object MerkleTree {
+  type Block = Array[Byte]
 
-    override val hash: Digest = hashFunction.hash(data.toString.getBytes)
-  }
 
-  case class EmptyLeaf[Block <: CryptographicHash]()(hashFunction: Block) extends Tree[Nothing, Block] {
-    override val hash: Digest = Array.empty[Byte]
-  }
+  def fromFile[H <: CryptographicHash](file: FileInputStream,
+                                       treeFolder: String,
+                                       blockSize: Int = 1024,
+                                       hash: H = Sha256
+                                      ): MerkleTree[H] = {
 
-  def create[Block, Hash <: CryptographicHash](
-                                                dataBlocks: Seq[Block],
-                                                hashFunction: Hash = Sha256): MerkleTree[Block, Hash] = {
-    val level = calculateRequiredLevel(dataBlocks.size)
+    @tailrec
+    def processFile(file: FileInputStream, blockIndex: Int = 0): Int = {
+      val buf = new Array[Byte](blockSize)
+      val length = file.read(buf)
+      if (length != -1) {
+        file.read(buf, 0, length)
+        processBlock(buf, blockIndex)
+        processFile(file, blockIndex + 1)
+      } else {
+        blockIndex
+      }
+    }
 
-    val dataLeaves = dataBlocks.map(data => Leaf(data)(hashFunction))
+    def processBlock(block: Block, i: Int): Unit = {
+      val fos = new FileOutputStream(treeFolder + "/" + i)
+      fos.write(block)
+      fos.close()
+      storage.set((0, i), hash.hash(block))
+    }
 
-    val paddingNeeded = math.pow(2, level).toInt - dataBlocks.size
-    val padding = Seq.fill(paddingNeeded)(EmptyLeaf()(hashFunction))
+    lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
 
-    val leaves: Seq[Tree[Block, Hash]] = dataLeaves ++ padding
+    val nonEmptyBlocks = processFile(file)
+    storage.commit()
 
-    new MerkleTree(makeTree(leaves, hashFunction), leaves)
-  }
+    new MerkleTree(treeFolder, nonEmptyBlocks, blockSize, hash)
 
-  private def merge[Block, Hash <: CryptographicHash](
-                                               leftChild: Tree[Block, Hash],
-                                               rightChild: Tree[Block, Hash],
-                                               hashFunction: Hash): Node[Block, Hash] = {
-    Node(leftChild, rightChild)(hashFunction)
   }
 
   private def log2(x: Double): Double = math.log(x) / math.log(2)
 
-  private def calculateRequiredLevel(numberOfDataBlocks: Int): Int = {
+  def calculateRequiredLevel(numberOfDataBlocks: Int): Int = {
 
     math.ceil(log2(numberOfDataBlocks)).toInt
   }
 
-  @tailrec
-  private def makeTree[Block, Hash <: CryptographicHash](
-                                                          trees: Seq[Tree[Block, Hash]],
-                                                          hashFunction: Hash): Tree[Block, Hash] = {
-    def createParent(treePair: Seq[Tree[Block, Hash]]): Node[Block, Hash] = {
-      val leftChild +: rightChild +: _ = treePair
-      merge(leftChild, rightChild, hashFunction)
-    }
 
-    if (trees.isEmpty) {
-      EmptyLeaf()(hashFunction)
-    } else if (trees.size == 1) {
-      trees.head
-    } else {
-      makeTree(trees.grouped(2).map(createParent).toSeq, hashFunction)
-    }
-  }
-}
\ No newline at end of file
+}
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
new file mode 100644
index 00000000..547240c1
--- /dev/null
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
@@ -0,0 +1,23 @@
+package scorex.crypto.ads.merkle
+
+import scorex.crypto.CryptographicHash.Digest
+
+import scala.util.Try
+
+trait Storage {
+
+  import Storage._
+
+  def set(key: Key, value: Digest): Try[Digest]
+
+  def get(key: Key): Option[Digest]
+
+  def commit(): Unit
+}
+
+object Storage {
+  type Level = Int
+  type Position = Int
+  type Key = (Level, Position)
+
+}
\ No newline at end of file
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index adadc99e..674af6f5 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -1,9 +1,12 @@
 package scorex.props
 
-import org.scalacheck.{Arbitrary, Gen}
-import org.scalatest.{Matchers, PropSpec}
+import java.io.{File, FileInputStream, FileOutputStream}
+
+import org.scalacheck.{Gen, Arbitrary}
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
+import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.Sha256
+import scorex.crypto.Sha256._
 import scorex.crypto.ads.merkle.MerkleTree
 
 import scala.util.Random
@@ -11,20 +14,32 @@ import scala.util.Random
 class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
   property("value returned from byIndex() is valid for random dataset") {
-    val dataSetGen = for {
-      dataSet <- Gen.nonEmptyContainerOf[Array, Array[Byte]](Arbitrary.arbitrary[Array[Byte]])
-    } yield dataSet
 
-    forAll(dataSetGen) { dataSet =>
-      if (dataSet.length > 1) {
-        val index = Random.nextInt(dataSet.length)
 
-        val tree = MerkleTree.create(dataSet)
 
-        val leaf = tree.byIndex(index).get
-        MerkleTree.check(index, tree.hash, leaf)(Sha256)
-      }
+    val smallInteger = Gen.choose(1, 200)
+
+    forAll(smallInteger) { (blocks: Int) =>
+      val treeDirName = "/tmp/scorex/test/" + Random.alphanumeric(8).toString
+      val treeDir = new File(treeDirName)
+      val tempFile = treeDirName + "/data.file"
+
+
+      val data = new Array[Byte](1024 * blocks)
+      Random.nextBytes(data)
+      treeDir.mkdirs()
+      val fos = new FileOutputStream(tempFile)
+      fos.write(data)
+      fos.close()
+      val file = new FileInputStream(tempFile)
+
+      val tree = MerkleTree.fromFile(file, treeDirName)
+      val index = Random.nextInt(tree.nonEmptyBlocks)
+
+      val leaf = tree.byIndex(index).get
+      leaf.check(index, tree.rootHash)(Sha256)
+      for (file <- treeDir.listFiles) file.delete
     }
-  }
 
+  }
 }
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index 0ddf2808..cfc50173 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -1,5 +1,7 @@
 package scorex.perma
 
+import java.io.FileInputStream
+
 import akka.actor.{Actor, ActorRef, ActorSystem, Props}
 import org.slf4j.LoggerFactory
 import scorex.crypto.ads.merkle.MerkleTree
@@ -69,12 +71,14 @@ object TestApp extends App {
   val dataSet = (1 to n).map(x => Random.alphanumeric.take(segmentSize).mkString).toArray.map(_.getBytes)
 
   log.info("Calculate tree")
-  val tree = MerkleTree.create(dataSet)
+  val file = new FileInputStream("/home/pozharko/Documents/protokolnew-160915.doc")
+  val treeFolder = "/tmp/scorex"
+  val tree = MerkleTree.fromFile(file, treeFolder)
 
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")
   val dealer = actorSystem.actorOf(Props(classOf[TrustedDealer], dataSet))
-  val miners: Seq[ActorRef] = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.hash)))
+  val miners: Seq[ActorRef] = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.rootHash)))
 
   miners.foreach(minerRef => minerRef ! Initialize)
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index ca629ecd..9acbb4bc 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -129,7 +129,7 @@ object Miner {
     val partialProofsCheck = 1.to(Parameters.k).foldLeft(true) { case (partialResult, i) =>
       val segment = proofs(i - 1).segment
 
-      MerkleTree.check(ris(i - 1), rootHash, segment)() || {
+      segment.check(ris(i - 1), rootHash)() || {
         val hi = Sha256.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data)
         SigningFunctionsImpl.verify(sigs(i), hi, publicKey)
       }
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
index 0777ad64..3657f39d 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
@@ -9,21 +9,21 @@ import scorex.perma.actors.TrustedDealerSpec.{SegmentsToStore, SegmentsRequest}
 
 
 class TrustedDealer(val dataSet: Array[DataSegment]) extends Actor with ActorLogging {
-
-  val tree = MerkleTree.create(dataSet)
-
+//
+//  val tree = MerkleTree.create(dataSet)
+//
   override def receive = {
-    case SegmentsRequest(segmentIds) =>
-      log.info(s"SegmentsRequest(${segmentIds.mkString(", ")})")
-
-      assert(segmentIds.length == Parameters.l)
-
-      val segments: Subset = segmentIds.map { x =>
-        x -> tree.byIndex(x)
-      }.toMap.collect {
-        case (key, Some(value)) => key -> value
-      }
-      sender ! SegmentsToStore(segments)
+//    case SegmentsRequest(segmentIds) =>
+//      log.info(s"SegmentsRequest(${segmentIds.mkString(", ")})")
+//
+//      assert(segmentIds.length == Parameters.l)
+//
+//      val segments: Subset = segmentIds.map { x =>
+//        x -> tree.byIndex(x)
+//      }.toMap.collect {
+//        case (key, Some(value)) => key -> value
+//      }
+//      sender ! SegmentsToStore(segments)
     case m =>
       log.warning("Unknown message: {}", m)
   }

From 157e3129aae95aedc16b94d4d07a5e1e01cf96c0 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 18 Nov 2015 16:03:16 +0300
Subject: [PATCH 05/66] Fix exceptions on JVM shutdown

---
 .../main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala  | 6 +++++-
 .../main/scala/scorex/crypto/ads/merkle/MerkleTree.scala    | 2 +-
 .../src/main/scala/scorex/crypto/ads/merkle/Storage.scala   | 2 ++
 .../src/test/scala/scorex/props/MerkleSpecification.scala   | 1 +
 4 files changed, 9 insertions(+), 2 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
index 8ccd02c6..db8a0b69 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -12,7 +12,7 @@ class MapDBStorage(file: File) extends Storage {
 
   import Storage._
 
-  val db = DBMaker.appendFileDB(file)
+  val db = DBMaker.fileDB(file)
     .fileMmapEnableIfSupported()
     .closeOnJvmShutdown()
     .checksumEnable()
@@ -32,6 +32,10 @@ class MapDBStorage(file: File) extends Storage {
     db.close()
   }
 
+  override def close(): Unit = {
+    db.close()
+  }
+
   override def get(key: Key): Option[Digest] = {
     Option(map.get(stringKey(key)))
   }
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 824706e2..8339bc55 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -27,7 +27,6 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
       @tailrec
       def calculateTreePath(n: Int, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
         if (currentLevel < level) {
-          //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
           if (n % 2 == 0) {
             getHash((currentLevel, n + 1)) match {
               case Some(v) =>
@@ -36,6 +35,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
                 acc.reverse
             }
           } else {
+            //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
             calculateTreePath(n / 2, currentLevel + 1, getHash((currentLevel, n - 1)).get +: acc)
           }
         } else {
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
index 547240c1..a4d0ca64 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
@@ -13,6 +13,8 @@ trait Storage {
   def get(key: Key): Option[Digest]
 
   def commit(): Unit
+
+  def close(): Unit
 }
 
 object Storage {
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index 674af6f5..ca24841b 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -38,6 +38,7 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
 
       val leaf = tree.byIndex(index).get
       leaf.check(index, tree.rootHash)(Sha256)
+      tree.storage.close()
       for (file <- treeDir.listFiles) file.delete
     }
 

From 0cac5910a00c4f5062153c63cde72c0a01888218 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 18 Nov 2015 16:08:18 +0300
Subject: [PATCH 06/66] Reformat code

---
 .../src/test/scala/scorex/props/MerkleSpecification.scala     | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index ca24841b..1a4002dc 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -2,11 +2,10 @@ package scorex.props
 
 import java.io.{File, FileInputStream, FileOutputStream}
 
-import org.scalacheck.{Gen, Arbitrary}
+import org.scalacheck.Gen
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.Sha256
-import scorex.crypto.Sha256._
 import scorex.crypto.ads.merkle.MerkleTree
 
 import scala.util.Random
@@ -16,7 +15,6 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
   property("value returned from byIndex() is valid for random dataset") {
 
 
-
     val smallInteger = Gen.choose(1, 200)
 
     forAll(smallInteger) { (blocks: Int) =>

From db851bf07e3a6c3b906ac3c63c0a995acf6a46d6 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 18 Nov 2015 17:20:28 +0300
Subject: [PATCH 07/66] More merkle test

---
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 18 +++---
 .../scorex/props/MerkleSpecification.scala    | 56 +++++++++++++------
 2 files changed, 46 insertions(+), 28 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 8339bc55..688fd0d1 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -27,15 +27,10 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
       @tailrec
       def calculateTreePath(n: Int, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
         if (currentLevel < level) {
+          //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
           if (n % 2 == 0) {
-            getHash((currentLevel, n + 1)) match {
-              case Some(v) =>
-                calculateTreePath(n / 2, currentLevel + 1, v +: acc)
-              case None =>
-                acc.reverse
-            }
+            calculateTreePath(n / 2, currentLevel + 1, getHash((currentLevel, n + 1)).get +: acc)
           } else {
-            //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
             calculateTreePath(n / 2, currentLevel + 1, getHash((currentLevel, n - 1)).get +: acc)
           }
         } else {
@@ -52,6 +47,8 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
     }
   }
 
+  private lazy val emptyHash = hash.hash("".getBytes)
+
   def getHash(key: Storage.Key): Option[Digest] = {
     storage.get(key) match {
       case None =>
@@ -60,9 +57,9 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
           val h2 = getHash((key._1 - 1, key._2 * 2 + 1))
           (h1, h2) match {
             case (Some(hash1), Some(hash2)) => Some(hash.hash(hash1 ++ hash2))
-            case (Some(h), _) => h1
-            case (_, Some(h)) => h2
-            case _ => None
+            case (Some(h), _) => Some(hash.hash(h ++ emptyHash))
+            case (_, Some(h)) => Some(hash.hash(emptyHash ++ h))
+            case _ => Some(emptyHash)
           }
         } else {
           None
@@ -70,7 +67,6 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
       case digest =>
         digest
     }
-
   }
 
 
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index 1a4002dc..3621723c 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -11,34 +11,56 @@ import scorex.crypto.ads.merkle.MerkleTree
 import scala.util.Random
 
 class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
+  val smallInteger = Gen.choose(100, 500)
 
-  property("value returned from byIndex() is valid for random dataset") {
-
-
-    val smallInteger = Gen.choose(1, 200)
 
+  property("value returned from byIndex() is valid for random dataset") {
     forAll(smallInteger) { (blocks: Int) =>
-      val treeDirName = "/tmp/scorex/test/" + Random.alphanumeric(8).toString
-      val treeDir = new File(treeDirName)
-      val tempFile = treeDirName + "/data.file"
-
-
-      val data = new Array[Byte](1024 * blocks)
-      Random.nextBytes(data)
-      treeDir.mkdirs()
-      val fos = new FileOutputStream(tempFile)
-      fos.write(data)
-      fos.close()
+      val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks)
       val file = new FileInputStream(tempFile)
 
       val tree = MerkleTree.fromFile(file, treeDirName)
       val index = Random.nextInt(tree.nonEmptyBlocks)
 
       val leaf = tree.byIndex(index).get
-      leaf.check(index, tree.rootHash)(Sha256)
+      val resp = leaf.check(index, tree.rootHash)(Sha256)
+      tree.storage.close()
+      resp shouldBe true
+    }
+  }
+
+  property("hash root is the same") {
+    forAll(smallInteger) { (blocks: Int) =>
+      val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks, "2")
+      val file = new FileInputStream(tempFile)
+
+      val fileTree = MerkleTree.fromFile(file, treeDirName)
+      val rootHash = fileTree.rootHash
+
+      fileTree.storage.close()
+
+      val tree = new MerkleTree(treeDirName, fileTree.nonEmptyBlocks)
+      val newRootHash = tree.rootHash
       tree.storage.close()
-      for (file <- treeDir.listFiles) file.delete
+      rootHash shouldBe newRootHash
     }
+  }
+
+
+  def generateFile(blocks: Int, subdir: String = "1"): (String, File, String) = {
+    val treeDirName = "/tmp/scorex/test/" + subdir + "/"
+    val treeDir = new File(treeDirName)
+    val tempFile = treeDirName + "/data.file"
+
+
+    val data = new Array[Byte](1024 * blocks)
+    Random.nextBytes(data)
+    treeDir.mkdirs()
+    for (file <- treeDir.listFiles) file.delete
 
+    val fos = new FileOutputStream(tempFile)
+    fos.write(data)
+    fos.close()
+    (treeDirName, treeDir, tempFile)
   }
 }
\ No newline at end of file

From e4f8db7ffe9c887cb5040407c3420c9283c59c90 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 18 Nov 2015 19:52:55 +0300
Subject: [PATCH 08/66] Fix testApp

---
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 48 ++++++++++++-------
 .../scorex/props/MerkleSpecification.scala    |  6 +--
 .../src/main/scala/scorex/perma/TestApp.scala | 33 ++++++-------
 .../scorex/perma/actors/TrustedDealer.scala   | 30 ++++++------
 4 files changed, 62 insertions(+), 55 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 688fd0d1..5601cada 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -1,6 +1,6 @@
 package scorex.crypto.ads.merkle
 
-import java.io.{File, FileInputStream, FileOutputStream}
+import java.io.{File, FileOutputStream, RandomAccessFile}
 import java.nio.file.{Files, Paths}
 
 import scorex.crypto.CryptographicHash.Digest
@@ -76,24 +76,12 @@ object MerkleTree {
   type Block = Array[Byte]
 
 
-  def fromFile[H <: CryptographicHash](file: FileInputStream,
+  def fromFile[H <: CryptographicHash](fileName: String,
                                        treeFolder: String,
                                        blockSize: Int = 1024,
                                        hash: H = Sha256
                                       ): MerkleTree[H] = {
-
-    @tailrec
-    def processFile(file: FileInputStream, blockIndex: Int = 0): Int = {
-      val buf = new Array[Byte](blockSize)
-      val length = file.read(buf)
-      if (length != -1) {
-        file.read(buf, 0, length)
-        processBlock(buf, blockIndex)
-        processFile(file, blockIndex + 1)
-      } else {
-        blockIndex
-      }
-    }
+    lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
 
     def processBlock(block: Block, i: Int): Unit = {
       val fos = new FileOutputStream(treeFolder + "/" + i)
@@ -102,9 +90,35 @@ object MerkleTree {
       storage.set((0, i), hash.hash(block))
     }
 
-    lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
+    val byteBuffer = new Array[Byte](blockSize)
+
+    def readLines(bigDataFilePath: String, chunkIndex: Int): Array[Byte] = {
+      val randomAccessFile = new RandomAccessFile(fileName, "r")
+      try {
+        val seek = chunkIndex * blockSize
+        randomAccessFile.seek(seek)
+        randomAccessFile.read(byteBuffer)
+        byteBuffer
+      } finally {
+        randomAccessFile.close()
+      }
+    }
+
+    val nonEmptyBlocks = {
+      val randomAccessFile = new RandomAccessFile(fileName, "r")
+      try {
+        (randomAccessFile.length / blockSize).toInt
+      } finally {
+        randomAccessFile.close()
+      }
+    }
+
+    for (i <- 0 to (nonEmptyBlocks - 1)) {
+      val block = readLines(fileName, i)
+      processBlock(block, i)
+    }
+
 
-    val nonEmptyBlocks = processFile(file)
     storage.commit()
 
     new MerkleTree(treeFolder, nonEmptyBlocks, blockSize, hash)
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index 3621723c..4bf473fc 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -17,9 +17,8 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
   property("value returned from byIndex() is valid for random dataset") {
     forAll(smallInteger) { (blocks: Int) =>
       val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks)
-      val file = new FileInputStream(tempFile)
 
-      val tree = MerkleTree.fromFile(file, treeDirName)
+      val tree = MerkleTree.fromFile(tempFile, treeDirName)
       val index = Random.nextInt(tree.nonEmptyBlocks)
 
       val leaf = tree.byIndex(index).get
@@ -32,9 +31,8 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
   property("hash root is the same") {
     forAll(smallInteger) { (blocks: Int) =>
       val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks, "2")
-      val file = new FileInputStream(tempFile)
 
-      val fileTree = MerkleTree.fromFile(file, treeDirName)
+      val fileTree = MerkleTree.fromFile(tempFile, treeDirName)
       val rootHash = fileTree.rootHash
 
       fileTree.storage.close()
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index f0d30af5..c0c8c617 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -1,19 +1,12 @@
 package scorex.perma
 
-import java.io.FileInputStream
+import java.io.{File, RandomAccessFile}
 
-import akka.actor.{Actor, ActorRef, ActorSystem, Props}
+import akka.actor.{ActorRef, ActorSystem, Props}
 import org.slf4j.LoggerFactory
 import scorex.crypto.ads.merkle.MerkleTree
-import scorex.perma.BlockchainBuilderSpec.{SendWorkToMiners, WinningTicket}
-import scorex.perma.actors.MinerSpec.{Initialize, TicketGeneration}
-import scorex.perma.actors.{Miner, Ticket, TrustedDealer}
-
-import scorex.utils.ScorexLogging
-
-import scala.collection.mutable
-import scala.util.Random
-
+import scorex.perma.actors.MinerSpec.Initialize
+import scorex.perma.actors.{Miner, TrustedDealer}
 
 
 object TestApp extends App {
@@ -22,20 +15,22 @@ object TestApp extends App {
 
   val log = LoggerFactory.getLogger(this.getClass)
 
-  import Parameters._
-
   log.info("Generating random data set")
-  val rnd = new Random()
-  val dataSet = (1 to n).map(x => Random.alphanumeric.take(segmentSize).mkString).toArray.map(_.getBytes)
+  val treeDirName = "/tmp/scorex/testApp/"
+  val treeDir = new File(treeDirName)
+  val datasetFile = treeDirName + "/data.file"
+
+  treeDir.mkdirs()
+  val f = new RandomAccessFile(datasetFile, "rw")
+  f.setLength(Parameters.n * Parameters.segmentSize)
 
   log.info("Calculate tree")
-  val file = new FileInputStream("/home/pozharko/Documents/protokolnew-160915.doc")
-  val treeFolder = "/tmp/scorex"
-  val tree = MerkleTree.fromFile(file, treeFolder)
+  val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize)
+  assert(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
 
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")
-  val dealer = actorSystem.actorOf(Props(classOf[TrustedDealer], dataSet))
+  val dealer = actorSystem.actorOf(Props(new TrustedDealer(tree)))
   val miners: Seq[ActorRef] = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.rootHash)))
 
   miners.foreach(minerRef => minerRef ! Initialize)
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
index 3657f39d..5cf74268 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
@@ -1,6 +1,7 @@
 package scorex.perma.actors
 
 import akka.actor.{ActorLogging, Actor}
+import scorex.crypto.CryptographicHash
 import scorex.crypto.ads.merkle.MerkleTree
 import scorex.perma.Parameters
 import scorex.perma.Parameters.DataSegment
@@ -8,22 +9,20 @@ import scorex.perma.actors.MinerSpec.Subset
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsToStore, SegmentsRequest}
 
 
-class TrustedDealer(val dataSet: Array[DataSegment]) extends Actor with ActorLogging {
-//
-//  val tree = MerkleTree.create(dataSet)
-//
+class TrustedDealer[H <: CryptographicHash](val tree: MerkleTree[H]) extends Actor with ActorLogging {
+
   override def receive = {
-//    case SegmentsRequest(segmentIds) =>
-//      log.info(s"SegmentsRequest(${segmentIds.mkString(", ")})")
-//
-//      assert(segmentIds.length == Parameters.l)
-//
-//      val segments: Subset = segmentIds.map { x =>
-//        x -> tree.byIndex(x)
-//      }.toMap.collect {
-//        case (key, Some(value)) => key -> value
-//      }
-//      sender ! SegmentsToStore(segments)
+    case SegmentsRequest(segmentIds) =>
+      log.info(s"SegmentsRequest(${segmentIds.mkString(", ")})")
+
+      assert(segmentIds.length == Parameters.l)
+
+      val segments: Subset = segmentIds.map { x =>
+        x -> tree.byIndex(x)
+      }.toMap.collect {
+        case (key, Some(value)) => key -> value
+      }
+      sender ! SegmentsToStore(segments)
     case m =>
       log.warning("Unknown message: {}", m)
   }
@@ -37,4 +36,5 @@ object TrustedDealerSpec {
   case class SegmentsRequest(segments: Array[Int])
 
   case class SegmentsToStore(segments: Subset)
+
 }

From 01314d201b976473eca75c0d1b357c94bb7923f7 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 18 Nov 2015 20:49:28 +0300
Subject: [PATCH 09/66] Start work only when miners are ready

---
 .../scorex/perma/BlockchainBuilder.scala      | 18 +++++++++++--
 .../src/main/scala/scorex/perma/TestApp.scala | 19 +++++++-------
 .../scala/scorex/perma/actors/Miner.scala     | 26 ++++++++++++++-----
 .../scorex/perma/actors/TrustedDealer.scala   |  5 ++--
 4 files changed, 48 insertions(+), 20 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
index c3078a4c..6fed5e0e 100644
--- a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -1,7 +1,7 @@
 package scorex.perma
 
 import akka.actor.{Actor, ActorRef}
-import scorex.perma.actors.MinerSpec.TicketGeneration
+import scorex.perma.actors.MinerSpec.{Initialize, Initialized, TicketGeneration}
 import scorex.perma.actors.Ticket
 import scorex.utils.ScorexLogging
 
@@ -12,9 +12,11 @@ import scala.util.Random
 case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
 
 class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging {
+
   import BlockchainBuilderSpec._
 
   var puz: Array[Byte] = calcPuz
+  var initialized = 0
 
   val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte))
   val blockchainLike = mutable.Buffer[BlockHeaderLike]()
@@ -25,9 +27,21 @@ class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging
   def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
 
   override def receive = {
+    case Initialized =>
+      initialized = initialized + 1
+      if (initialized == miners.length) {
+        log.info("start BlockchainBuilder")
+        Thread.sleep(2000)
+        self ! SendWorkToMiners
+      }
+
     case SendWorkToMiners =>
       miners.foreach { minerRef =>
-        minerRef ! TicketGeneration(difficulty, puz)
+        if (initialized == miners.length) {
+          minerRef ! TicketGeneration(difficulty, puz)
+        } else {
+          minerRef ! Initialize
+        }
       }
 
     //miners are honest (lol) in our setting, so no validation here
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index c0c8c617..d6083688 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -2,12 +2,15 @@ package scorex.perma
 
 import java.io.{File, RandomAccessFile}
 
-import akka.actor.{ActorRef, ActorSystem, Props}
+import akka.actor.{ActorSystem, Props}
+import akka.util.Timeout
 import org.slf4j.LoggerFactory
 import scorex.crypto.ads.merkle.MerkleTree
-import scorex.perma.actors.MinerSpec.Initialize
+import scorex.perma.BlockchainBuilderSpec.SendWorkToMiners
 import scorex.perma.actors.{Miner, TrustedDealer}
 
+import scala.concurrent.duration._
+
 
 object TestApp extends App {
 
@@ -26,19 +29,17 @@ object TestApp extends App {
 
   log.info("Calculate tree")
   val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize)
-  assert(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
+  require(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
 
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")
   val dealer = actorSystem.actorOf(Props(new TrustedDealer(tree)))
-  val miners: Seq[ActorRef] = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.rootHash)))
+  val miners = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.rootHash), s"m-$x"))
 
-  miners.foreach(minerRef => minerRef ! Initialize)
+  implicit val timeout = Timeout(1 minute)
 
-  Thread.sleep(2000)
+  val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners), "BlockchainBuilder")
+  blockchainBuilder ! SendWorkToMiners
 
-  log.info("start BlockchainBuilder")
 
-  val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners))
-  blockchainBuilder ! BlockchainBuilderSpec.SendWorkToMiners
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 4ade44db..124d4649 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -3,17 +3,20 @@ package scorex.perma.actors
 import java.security.SecureRandom
 
 import akka.actor.{Actor, ActorLogging, ActorRef}
+import akka.pattern.ask
+import akka.util.Timeout
 import scorex.crypto.CryptographicHash._
 import scorex.crypto.SigningFunctions.{PrivateKey, PublicKey, Signature}
-import scorex.crypto.ads.merkle.{MerkleTree, AuthDataBlock}
 import scorex.crypto._
+import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.Parameters
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
 
-
-import scala.util.Try
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
+import scala.util.{Success, Try}
 
 case class PartialProof(signature: Signature, segmentIndex: Int, segment: AuthDataBlock[Parameters.DataSegment])
 
@@ -26,19 +29,28 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act
   import Miner._
 
   private val keyPair = EllipticCurveImpl.createKeyPair(randomBytes(32))
+  private implicit val timeout = Timeout(1 minute)
 
   private var segments: Subset = Map()
 
   override def receive = {
 
     case Initialize =>
-      log.info("Initialize")
+      log.debug("Initialize")
 
       val segmentIdsToDownload = 1.to(Parameters.l).map { i =>
         u(keyPair._2, i - 1)
       }.toArray
 
-      trustedDealerRef ! SegmentsRequest(segmentIdsToDownload)
+      val s = sender()
+
+      trustedDealerRef ? SegmentsRequest(segmentIdsToDownload) onComplete {
+        case Success(m) =>
+          self ! m
+          s ! Initialized
+        case _ =>
+          context.stop(self)
+      }
 
     case SegmentsToStore(sgs) =>
       log.debug("SegmentsToStore({})", sgs)
@@ -148,7 +160,9 @@ object MinerSpec {
   type Index = Int
   type Subset = Map[Index, AuthDataBlock[Parameters.DataSegment]]
 
-  case class Initialize()
+  case object Initialize
+
+  case object Initialized
 
   case class TicketGeneration(difficulty: BigInt, puz: Array[Byte])
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
index 5cf74268..a7361a55 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
@@ -1,12 +1,11 @@
 package scorex.perma.actors
 
-import akka.actor.{ActorLogging, Actor}
+import akka.actor.{Actor, ActorLogging}
 import scorex.crypto.CryptographicHash
 import scorex.crypto.ads.merkle.MerkleTree
 import scorex.perma.Parameters
-import scorex.perma.Parameters.DataSegment
 import scorex.perma.actors.MinerSpec.Subset
-import scorex.perma.actors.TrustedDealerSpec.{SegmentsToStore, SegmentsRequest}
+import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
 
 
 class TrustedDealer[H <: CryptographicHash](val tree: MerkleTree[H]) extends Actor with ActorLogging {

From 8ca0fb5baf9c857a1bfa303a276471740da2f143 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 18 Nov 2015 23:55:19 +0300
Subject: [PATCH 10/66] Save calculated hash values

---
 .../scorex/crypto/ads/merkle/MapDBStorage.scala |  2 +-
 .../scorex/crypto/ads/merkle/MerkleTree.scala   | 17 +++++++++++------
 2 files changed, 12 insertions(+), 7 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
index db8a0b69..80641d3c 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -29,10 +29,10 @@ class MapDBStorage(file: File) extends Storage {
 
   override def commit(): Unit = {
     db.commit()
-    db.close()
   }
 
   override def close(): Unit = {
+    commit()
     db.close()
   }
 
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 5601cada..421b03f3 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -20,7 +20,9 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   val level = calculateRequiredLevel(nonEmptyBlocks)
 
-  lazy val rootHash: Digest = getHash((level, 0)).get
+  val rootHash: Digest = getHash((level, 0)).get
+
+  storage.commit()
 
   def byIndex(index: Int): Option[AuthDataBlock[Block]] = {
     if (index < nonEmptyBlocks && index >= 0) {
@@ -55,12 +57,14 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
         if (key._1 > 0) {
           val h1 = getHash((key._1 - 1, key._2 * 2))
           val h2 = getHash((key._1 - 1, key._2 * 2 + 1))
-          (h1, h2) match {
-            case (Some(hash1), Some(hash2)) => Some(hash.hash(hash1 ++ hash2))
-            case (Some(h), _) => Some(hash.hash(h ++ emptyHash))
-            case (_, Some(h)) => Some(hash.hash(emptyHash ++ h))
-            case _ => Some(emptyHash)
+          val calculatedHash = (h1, h2) match {
+            case (Some(hash1), Some(hash2)) => hash.hash(hash1 ++ hash2)
+            case (Some(h), _) => hash.hash(h ++ emptyHash)
+            case (_, Some(h)) => hash.hash(emptyHash ++ h)
+            case _ => emptyHash
           }
+          storage.set(key, calculatedHash)
+          Some(calculatedHash)
         } else {
           None
         }
@@ -120,6 +124,7 @@ object MerkleTree {
 
 
     storage.commit()
+    storage.close()
 
     new MerkleTree(treeFolder, nonEmptyBlocks, blockSize, hash)
 

From ecb49e5652f0b242276a5e68c618648d32399f78 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 19 Nov 2015 00:29:41 +0300
Subject: [PATCH 11/66] Don't recreate existing Tree

---
 .../main/scala/scorex/perma/Parameters.scala  |  4 +--
 .../src/main/scala/scorex/perma/TestApp.scala | 26 +++++++++++++------
 2 files changed, 20 insertions(+), 10 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
index 5e04dd58..ff31023f 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
@@ -8,8 +8,8 @@ object Parameters {
   //few segments to be stored in a block, so segment size shouldn't be big
   val segmentSize = 1024 //segment size in bytes
 
-  val n = 1024*4 //how many segments in a dataset in total
-  val l = 16 //how many segments to store for an each miner
+  val n = 1024*1024 //how many segments in a dataset in total
+  val l = 1024 //how many segments to store for an each miner
 
   val k = 4 //number of iterations during scratch-off phase
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index d6083688..e78765a5 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -1,10 +1,12 @@
 package scorex.perma
 
 import java.io.{File, RandomAccessFile}
+import java.nio.file.{Files, Paths}
 
 import akka.actor.{ActorSystem, Props}
 import akka.util.Timeout
 import org.slf4j.LoggerFactory
+import scorex.crypto.Sha256
 import scorex.crypto.ads.merkle.MerkleTree
 import scorex.perma.BlockchainBuilderSpec.SendWorkToMiners
 import scorex.perma.actors.{Miner, TrustedDealer}
@@ -20,16 +22,24 @@ object TestApp extends App {
 
   log.info("Generating random data set")
   val treeDirName = "/tmp/scorex/testApp/"
-  val treeDir = new File(treeDirName)
-  val datasetFile = treeDirName + "/data.file"
-
-  treeDir.mkdirs()
-  val f = new RandomAccessFile(datasetFile, "rw")
-  f.setLength(Parameters.n * Parameters.segmentSize)
 
   log.info("Calculate tree")
-  val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize)
-  require(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
+  val tree = if (Files.exists(Paths.get(treeDirName + "/tree.mapDB"))) {
+    new MerkleTree(treeDirName, Parameters.n)
+  } else {
+    val treeDir = new File(treeDirName)
+    treeDir.mkdirs()
+    val datasetFile = treeDirName + "/data.file"
+    new RandomAccessFile(datasetFile, "rw").setLength(Parameters.n * Parameters.segmentSize)
+    val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize)
+    require(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
+    tree
+  }
+
+  log.info("test tree")
+  val index = Parameters.n - 3
+  val leaf = tree.byIndex(index).get
+  require(leaf.check(index, tree.rootHash)(Sha256))
 
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")

From eef33d5c2451684fac0b7be4473d327911dd7406 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 19 Nov 2015 10:57:14 +0300
Subject: [PATCH 12/66] TestApp refactoring

---
 .../src/main/scala/scorex/perma/BlockchainBuilder.scala      | 1 -
 scorex-perma/src/main/scala/scorex/perma/TestApp.scala       | 5 +++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
index 6fed5e0e..b09c35a8 100644
--- a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -31,7 +31,6 @@ class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging
       initialized = initialized + 1
       if (initialized == miners.length) {
         log.info("start BlockchainBuilder")
-        Thread.sleep(2000)
         self ! SendWorkToMiners
       }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index e78765a5..cfeabf8f 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -20,17 +20,18 @@ object TestApp extends App {
 
   val log = LoggerFactory.getLogger(this.getClass)
 
-  log.info("Generating random data set")
   val treeDirName = "/tmp/scorex/testApp/"
 
-  log.info("Calculate tree")
   val tree = if (Files.exists(Paths.get(treeDirName + "/tree.mapDB"))) {
+    log.info("Get existing tree")
     new MerkleTree(treeDirName, Parameters.n)
   } else {
+    log.info("Generating random data set")
     val treeDir = new File(treeDirName)
     treeDir.mkdirs()
     val datasetFile = treeDirName + "/data.file"
     new RandomAccessFile(datasetFile, "rw").setLength(Parameters.n * Parameters.segmentSize)
+    log.info("Calculate tree")
     val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize)
     require(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
     tree

From 2cd93adcb6fc8e66ad1ef4c488854733dc481a32 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 19 Nov 2015 11:51:04 +0300
Subject: [PATCH 13/66] MapDBStorage refactoring

---
 .../crypto/ads/merkle/MapDBStorage.scala      | 40 +++++++++++------
 .../scorex/crypto/ads/merkle/Storage.scala    |  4 +-
 .../props/MapDBStorageSpecification.scala     | 45 +++++++++++++++++++
 .../main/scala/scorex/perma/Parameters.scala  |  2 +-
 4 files changed, 75 insertions(+), 16 deletions(-)
 create mode 100644 scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
index 80641d3c..5fdde10b 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -1,29 +1,38 @@
 package scorex.crypto.ads.merkle
 
 import java.io.File
-import java.util.concurrent.ConcurrentNavigableMap
 
-import org.mapdb.DBMaker
+import org.mapdb.{DBMaker, HTreeMap, Serializer}
+import org.slf4j.LoggerFactory
 import scorex.crypto.CryptographicHash.Digest
 
-import scala.util.Try
+import scala.util.{Success, Failure, Try}
 
 class MapDBStorage(file: File) extends Storage {
 
   import Storage._
 
-  val db = DBMaker.fileDB(file)
+  private val log = LoggerFactory.getLogger(this.getClass)
+
+  private val db = DBMaker.fileDB(file)
     .fileMmapEnableIfSupported()
     .closeOnJvmShutdown()
     .checksumEnable()
     .make()
 
-  val map: ConcurrentNavigableMap[String, Digest] = db.treeMap("tree")
+  def mapsStream(n: Int): Stream[HTreeMap[Long, Digest]] = Stream.cons(
+    db.hashMapCreate("map_" + n)
+      .keySerializer(Serializer.LONG)
+      .valueSerializer(Serializer.BYTE_ARRAY)
+      .makeOrGet(),
+    mapsStream(n + 1)
+  )
 
-  override def set(key: Key, value: Digest): Try[Digest] = {
-    Try {
-      map.put(stringKey(key), value)
-    }
+  private val maps = mapsStream(0)
+
+
+  override def set(key: Key, value: Digest): Unit = {
+    maps(key._1.asInstanceOf[Int]).put(key._2, value)
   }
 
 
@@ -37,10 +46,15 @@ class MapDBStorage(file: File) extends Storage {
   }
 
   override def get(key: Key): Option[Digest] = {
-    Option(map.get(stringKey(key)))
+    Try {
+      maps(key._1).get(key._2)
+    } match {
+      case Failure(e) =>
+        log.debug("Enable to load key: " + key)
+        None
+      case Success(v) =>
+        Option(v)
+    }
   }
 
-  private def stringKey(key: Key): String = {
-    key._1 + "_" + key._2
-  }
 }
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
index a4d0ca64..b9c419c9 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
@@ -8,7 +8,7 @@ trait Storage {
 
   import Storage._
 
-  def set(key: Key, value: Digest): Try[Digest]
+  def set(key: Key, value: Digest): Unit
 
   def get(key: Key): Option[Digest]
 
@@ -19,7 +19,7 @@ trait Storage {
 
 object Storage {
   type Level = Int
-  type Position = Int
+  type Position = Long
   type Key = (Level, Position)
 
 }
\ No newline at end of file
diff --git a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
new file mode 100644
index 00000000..9ccbe6d2
--- /dev/null
+++ b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
@@ -0,0 +1,45 @@
+package scorex.props
+
+import java.io.{File, FileOutputStream}
+
+import org.scalacheck.{Arbitrary, Gen}
+import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
+import org.scalatest.{Matchers, PropSpec}
+import scorex.crypto.CryptographicHash.Digest
+import scorex.crypto.Sha256
+import scorex.crypto.ads.merkle.Storage.Key
+import scorex.crypto.ads.merkle.{MapDBStorage, Storage, MerkleTree}
+
+import scala.util.Random
+
+class MapDBStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
+
+  val treeDirName = "/tmp/scorex/test/MapDBStorageSpecification/"
+  val treeDir = new File(treeDirName)
+  treeDir.mkdirs()
+  val dbFile = new File(treeDirName + "/db.file")
+  dbFile.delete()
+
+  val keyVal = for {
+    level: Int <- Gen.choose(1, 50)
+    key: Long <- Arbitrary.arbitrary[Long]
+    value <- Arbitrary.arbitrary[String]
+  } yield ((level, key), value.getBytes)
+
+
+  property("set value and get it") {
+    lazy val storage: Storage = new MapDBStorage(dbFile)
+
+    forAll(keyVal) { x =>
+      val key: Key = x._1
+      val value: Digest = x._2
+      whenever(key._1 >= 0.toLong && key._2 >= 0.toLong) {
+        storage.set(key, value)
+
+        assert(storage.get(key).get sameElements value)
+      }
+    }
+    storage.close()
+  }
+
+}
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
index ff31023f..fd34892d 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
@@ -8,7 +8,7 @@ object Parameters {
   //few segments to be stored in a block, so segment size shouldn't be big
   val segmentSize = 1024 //segment size in bytes
 
-  val n = 1024*1024 //how many segments in a dataset in total
+  val n = 1024*256 //how many segments in a dataset in total
   val l = 1024 //how many segments to store for an each miner
 
   val k = 4 //number of iterations during scratch-off phase

From f9622a90babe8d3494241cf8006c2ea156c8c08e Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 19 Nov 2015 12:23:04 +0300
Subject: [PATCH 14/66] merkle tree index is Long

---
 .../crypto/ads/merkle/AuthDataBlock.scala     |  7 ++--
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 33 ++++++++++---------
 .../scorex/props/MerkleSpecification.scala    |  2 +-
 3 files changed, 22 insertions(+), 20 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index 21c6a7b2..d680af39 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -1,5 +1,6 @@
 package scorex.crypto.ads.merkle
 
+import scorex.crypto.ads.merkle.Storage.Position
 import scorex.crypto.{Sha256, CryptographicHash}
 import scorex.crypto.CryptographicHash._
 
@@ -11,11 +12,11 @@ import scala.annotation.tailrec
   */
 case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
 
-  def check[Hash <: CryptographicHash](index: Int, rootHash: Digest)
+  def check[Hash <: CryptographicHash](index: Position, rootHash: Digest)
                                       (hashFunction: Hash = Sha256): Boolean = {
 
     @tailrec
-    def calculateHash(i: Int, nodeHash: Digest, path: Seq[Digest]): Digest = {
+    def calculateHash(i: Position, nodeHash: Digest, path: Seq[Digest]): Digest = {
       if (i % 2 == 0) {
         val hash = hashFunction.hash(nodeHash ++ path.head)
         if (path.size == 1) {
@@ -32,7 +33,7 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
         }
       }
     }
-    if(merklePath.nonEmpty) {
+    if (merklePath.nonEmpty) {
       val calculated = calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath)
       calculated.mkString == rootHash.mkString
     } else {
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 421b03f3..582d37a6 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -4,12 +4,13 @@ import java.io.{File, FileOutputStream, RandomAccessFile}
 import java.nio.file.{Files, Paths}
 
 import scorex.crypto.CryptographicHash.Digest
+import scorex.crypto.ads.merkle.Storage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
 
 class MerkleTree[H <: CryptographicHash](treeFolder: String,
-                                         val nonEmptyBlocks: Int,
+                                         val nonEmptyBlocks: Position,
                                          blockSize: Int = 1024,
                                          hash: H = Sha256
                                         ) {
@@ -24,10 +25,10 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   storage.commit()
 
-  def byIndex(index: Int): Option[AuthDataBlock[Block]] = {
+  def byIndex(index: Position): Option[AuthDataBlock[Block]] = {
     if (index < nonEmptyBlocks && index >= 0) {
       @tailrec
-      def calculateTreePath(n: Int, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
+      def calculateTreePath(n: Position, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
         if (currentLevel < level) {
           //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
           if (n % 2 == 0) {
@@ -87,16 +88,9 @@ object MerkleTree {
                                       ): MerkleTree[H] = {
     lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
 
-    def processBlock(block: Block, i: Int): Unit = {
-      val fos = new FileOutputStream(treeFolder + "/" + i)
-      fos.write(block)
-      fos.close()
-      storage.set((0, i), hash.hash(block))
-    }
-
     val byteBuffer = new Array[Byte](blockSize)
 
-    def readLines(bigDataFilePath: String, chunkIndex: Int): Array[Byte] = {
+    def readLines(bigDataFilePath: String, chunkIndex: Position): Array[Byte] = {
       val randomAccessFile = new RandomAccessFile(fileName, "r")
       try {
         val seek = chunkIndex * blockSize
@@ -108,7 +102,7 @@ object MerkleTree {
       }
     }
 
-    val nonEmptyBlocks = {
+    val nonEmptyBlocks: Position = {
       val randomAccessFile = new RandomAccessFile(fileName, "r")
       try {
         (randomAccessFile.length / blockSize).toInt
@@ -117,11 +111,18 @@ object MerkleTree {
       }
     }
 
-    for (i <- 0 to (nonEmptyBlocks - 1)) {
-      val block = readLines(fileName, i)
-      processBlock(block, i)
+    def processBlocks(currentBlock: Position = 0): Unit = {
+      val block: Block = readLines(fileName, currentBlock)
+      val fos = new FileOutputStream(treeFolder + "/" + currentBlock)
+      fos.write(block)
+      fos.close()
+      storage.set((0, currentBlock), hash.hash(block))
+      if (currentBlock < nonEmptyBlocks) {
+        processBlocks(currentBlock + 1)
+      }
     }
 
+    processBlocks()
 
     storage.commit()
     storage.close()
@@ -132,7 +133,7 @@ object MerkleTree {
 
   private def log2(x: Double): Double = math.log(x) / math.log(2)
 
-  def calculateRequiredLevel(numberOfDataBlocks: Int): Int = {
+  def calculateRequiredLevel(numberOfDataBlocks: Position): Int = {
 
     math.ceil(log2(numberOfDataBlocks)).toInt
   }
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index 4bf473fc..4b6f4caf 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -19,7 +19,7 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
       val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks)
 
       val tree = MerkleTree.fromFile(tempFile, treeDirName)
-      val index = Random.nextInt(tree.nonEmptyBlocks)
+      val index = Random.nextInt(tree.nonEmptyBlocks.asInstanceOf[Int])
 
       val leaf = tree.byIndex(index).get
       val resp = leaf.check(index, tree.rootHash)(Sha256)

From 1a472e88ed31cf67425635195f18b9a8c2c287a9 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 19 Nov 2015 13:38:38 +0300
Subject: [PATCH 15/66] Put different levels to different files

---
 .../crypto/ads/merkle/MapDBStorage.scala      | 53 +++++++++++--------
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 10 ++--
 .../props/MapDBStorageSpecification.scala     |  7 +--
 .../main/scala/scorex/perma/Parameters.scala  |  2 +-
 .../src/main/scala/scorex/perma/TestApp.scala |  4 +-
 5 files changed, 46 insertions(+), 30 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
index 5fdde10b..e74f4f4f 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -6,43 +6,52 @@ import org.mapdb.{DBMaker, HTreeMap, Serializer}
 import org.slf4j.LoggerFactory
 import scorex.crypto.CryptographicHash.Digest
 
-import scala.util.{Success, Failure, Try}
+import scala.util.{Failure, Success, Try}
 
-class MapDBStorage(file: File) extends Storage {
+class MapDBStorage(fileName: String, levels: Int) extends Storage {
 
   import Storage._
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
-  private val db = DBMaker.fileDB(file)
-    .fileMmapEnableIfSupported()
-    .closeOnJvmShutdown()
-    .checksumEnable()
-    .make()
-
-  def mapsStream(n: Int): Stream[HTreeMap[Long, Digest]] = Stream.cons(
-    db.hashMapCreate("map_" + n)
-      .keySerializer(Serializer.LONG)
-      .valueSerializer(Serializer.BYTE_ARRAY)
-      .makeOrGet(),
-    mapsStream(n + 1)
-  )
-
-  private val maps = mapsStream(0)
+  private val dbs =
+    (0 to levels) map { n: Int =>
+      DBMaker.fileDB(new File(fileName + n + ".mapDB"))
+        .fileMmapEnableIfSupported()
+        .closeOnJvmShutdown()
+        .checksumEnable()
+        .make()
+    }
 
+  private val maps: Map[Int, HTreeMap[Long, Digest]] = {
+    val t = (0 to levels) map { n: Int =>
+      val m: HTreeMap[Long, Digest] = dbs(n).hashMapCreate("map_" + n)
+        .keySerializer(Serializer.LONG)
+        .valueSerializer(Serializer.BYTE_ARRAY)
+        .makeOrGet()
+      n -> m
+    }
+    t.toMap
+  }
 
   override def set(key: Key, value: Digest): Unit = {
-    maps(key._1.asInstanceOf[Int]).put(key._2, value)
+    val map = maps(key._1.asInstanceOf[Int])
+    val t = Try {
+      map.put(key._2, value)
+    }
+    if (t.isFailure) {
+      log.warn("Failed to set key:" + key)
+    }
   }
 
 
   override def commit(): Unit = {
-    db.commit()
+    dbs.map(_.commit())
   }
 
   override def close(): Unit = {
     commit()
-    db.close()
+    dbs.map(_.close())
   }
 
   override def get(key: Key): Option[Digest] = {
@@ -50,7 +59,9 @@ class MapDBStorage(file: File) extends Storage {
       maps(key._1).get(key._2)
     } match {
       case Failure(e) =>
-        log.debug("Enable to load key: " + key)
+        if (key._1 == 0) {
+          log.debug("Enable to load key for level 0: " + key)
+        }
         None
       case Success(v) =>
         Option(v)
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 582d37a6..506e1d8a 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -17,10 +17,10 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   import MerkleTree._
 
-  lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
-
   val level = calculateRequiredLevel(nonEmptyBlocks)
 
+  lazy val storage: Storage = new MapDBStorage(treeFolder + "/tree", level)
+
   val rootHash: Digest = getHash((level, 0)).get
 
   storage.commit()
@@ -86,8 +86,6 @@ object MerkleTree {
                                        blockSize: Int = 1024,
                                        hash: H = Sha256
                                       ): MerkleTree[H] = {
-    lazy val storage: Storage = new MapDBStorage(new File(treeFolder + "/tree.mapDB"))
-
     val byteBuffer = new Array[Byte](blockSize)
 
     def readLines(bigDataFilePath: String, chunkIndex: Position): Array[Byte] = {
@@ -111,6 +109,10 @@ object MerkleTree {
       }
     }
 
+    val level = calculateRequiredLevel(nonEmptyBlocks)
+
+    lazy val storage: Storage = new MapDBStorage(treeFolder + "/tree", level)
+
     def processBlocks(currentBlock: Position = 0): Unit = {
       val block: Block = readLines(fileName, currentBlock)
       val fos = new FileOutputStream(treeFolder + "/" + currentBlock)
diff --git a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
index 9ccbe6d2..891516de 100644
--- a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
@@ -18,17 +18,18 @@ class MapDBStorageSpecification extends PropSpec with PropertyChecks with Genera
   val treeDir = new File(treeDirName)
   treeDir.mkdirs()
   val dbFile = new File(treeDirName + "/db.file")
+  val maxLevel = 50
   dbFile.delete()
 
   val keyVal = for {
-    level: Int <- Gen.choose(1, 50)
+    level: Int <- Gen.choose(1, maxLevel)
     key: Long <- Arbitrary.arbitrary[Long]
     value <- Arbitrary.arbitrary[String]
-  } yield ((level, key), value.getBytes)
+  } yield ((level, math.abs(key)), value.getBytes)
 
 
   property("set value and get it") {
-    lazy val storage: Storage = new MapDBStorage(dbFile)
+    lazy val storage: Storage = new MapDBStorage(treeDirName + "/test_db", maxLevel)
 
     forAll(keyVal) { x =>
       val key: Key = x._1
diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
index fd34892d..4194ebbf 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
@@ -8,7 +8,7 @@ object Parameters {
   //few segments to be stored in a block, so segment size shouldn't be big
   val segmentSize = 1024 //segment size in bytes
 
-  val n = 1024*256 //how many segments in a dataset in total
+  val n = 1024*64 //how many segments in a dataset in total
   val l = 1024 //how many segments to store for an each miner
 
   val k = 4 //number of iterations during scratch-off phase
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index cfeabf8f..8718620e 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -22,7 +22,7 @@ object TestApp extends App {
 
   val treeDirName = "/tmp/scorex/testApp/"
 
-  val tree = if (Files.exists(Paths.get(treeDirName + "/tree.mapDB"))) {
+  val tree = if (Files.exists(Paths.get(treeDirName + "/tree0.mapDB"))) {
     log.info("Get existing tree")
     new MerkleTree(treeDirName, Parameters.n)
   } else {
@@ -42,6 +42,8 @@ object TestApp extends App {
   val leaf = tree.byIndex(index).get
   require(leaf.check(index, tree.rootHash)(Sha256))
 
+  log.info("Success: " + tree.rootHash.mkString)
+
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")
   val dealer = actorSystem.actorOf(Props(new TrustedDealer(tree)))

From af46a9745ba1a3666f639125a7d331bd8e093edd Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Thu, 19 Nov 2015 17:19:34 +0300
Subject: [PATCH 16/66] unused imports, formatting, styling

---
 .../crypto/ads/merkle/MapDBStorage.scala      | 20 +++++++++----------
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 12 +++--------
 .../scorex/crypto/ads/merkle/Storage.scala    |  3 ---
 .../props/MapDBStorageSpecification.scala     |  7 ++-----
 .../scorex/props/MerkleSpecification.scala    |  3 ++-
 .../scorex/perma/BlockchainBuilder.scala      |  1 -
 .../main/scala/scorex/perma/Parameters.scala  |  1 -
 .../src/main/scala/scorex/perma/TestApp.scala |  2 --
 8 files changed, 16 insertions(+), 33 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
index e74f4f4f..fe682844 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -36,35 +36,33 @@ class MapDBStorage(fileName: String, levels: Int) extends Storage {
 
   override def set(key: Key, value: Digest): Unit = {
     val map = maps(key._1.asInstanceOf[Int])
-    val t = Try {
+    Try {
       map.put(key._2, value)
-    }
-    if (t.isFailure) {
-      log.warn("Failed to set key:" + key)
+    }.recoverWith{case t: Throwable =>
+      log.warn("Failed to set key:" + key, t)
+      Failure(t)
     }
   }
 
-
-  override def commit(): Unit = {
-    dbs.map(_.commit())
-  }
+  override def commit(): Unit = dbs.foreach(_.commit())
 
   override def close(): Unit = {
     commit()
-    dbs.map(_.close())
+    dbs.foreach(_.close())
   }
 
   override def get(key: Key): Option[Digest] = {
     Try {
       maps(key._1).get(key._2)
     } match {
+      case Success(v) =>
+        Option(v)
+
       case Failure(e) =>
         if (key._1 == 0) {
           log.debug("Enable to load key for level 0: " + key)
         }
         None
-      case Success(v) =>
-        Option(v)
     }
   }
 
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 506e1d8a..81605e9c 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -1,6 +1,6 @@
 package scorex.crypto.ads.merkle
 
-import java.io.{File, FileOutputStream, RandomAccessFile}
+import java.io.{FileOutputStream, RandomAccessFile}
 import java.nio.file.{Files, Paths}
 
 import scorex.crypto.CryptographicHash.Digest
@@ -73,14 +73,12 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
         digest
     }
   }
-
-
 }
 
+
 object MerkleTree {
   type Block = Array[Byte]
 
-
   def fromFile[H <: CryptographicHash](fileName: String,
                                        treeFolder: String,
                                        blockSize: Int = 1024,
@@ -130,15 +128,11 @@ object MerkleTree {
     storage.close()
 
     new MerkleTree(treeFolder, nonEmptyBlocks, blockSize, hash)
-
   }
 
   private def log2(x: Double): Double = math.log(x) / math.log(2)
 
   def calculateRequiredLevel(numberOfDataBlocks: Position): Int = {
-
     math.ceil(log2(numberOfDataBlocks)).toInt
   }
-
-
-}
+}
\ No newline at end of file
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
index b9c419c9..ed0d476e 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
@@ -2,8 +2,6 @@ package scorex.crypto.ads.merkle
 
 import scorex.crypto.CryptographicHash.Digest
 
-import scala.util.Try
-
 trait Storage {
 
   import Storage._
@@ -21,5 +19,4 @@ object Storage {
   type Level = Int
   type Position = Long
   type Key = (Level, Position)
-
 }
\ No newline at end of file
diff --git a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
index 891516de..7f7b2cf6 100644
--- a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
@@ -1,16 +1,14 @@
 package scorex.props
 
-import java.io.{File, FileOutputStream}
+import java.io.File
 
 import org.scalacheck.{Arbitrary, Gen}
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.CryptographicHash.Digest
-import scorex.crypto.Sha256
 import scorex.crypto.ads.merkle.Storage.Key
-import scorex.crypto.ads.merkle.{MapDBStorage, Storage, MerkleTree}
+import scorex.crypto.ads.merkle.{MapDBStorage, Storage}
 
-import scala.util.Random
 
 class MapDBStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
@@ -42,5 +40,4 @@ class MapDBStorageSpecification extends PropSpec with PropertyChecks with Genera
     }
     storage.close()
   }
-
 }
\ No newline at end of file
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index 4b6f4caf..a06c9e28 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -1,6 +1,6 @@
 package scorex.props
 
-import java.io.{File, FileInputStream, FileOutputStream}
+import java.io.{File, FileOutputStream}
 
 import org.scalacheck.Gen
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
@@ -11,6 +11,7 @@ import scorex.crypto.ads.merkle.MerkleTree
 import scala.util.Random
 
 class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
+
   val smallInteger = Gen.choose(100, 500)
 
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
index b09c35a8..5eb2e00d 100644
--- a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -21,7 +21,6 @@ class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging
   val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte))
   val blockchainLike = mutable.Buffer[BlockHeaderLike]()
 
-
   private def calcPuz = 1.to(32).toArray.map(_ => Random.nextInt(256).toByte)
 
   def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
index 4194ebbf..abaaac0b 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
@@ -1,6 +1,5 @@
 package scorex.perma
 
-
 object Parameters {
 
   type DataSegment = Array[Byte]
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index 8718620e..fe2bae64 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -53,6 +53,4 @@ object TestApp extends App {
 
   val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners), "BlockchainBuilder")
   blockchainBuilder ! SendWorkToMiners
-
-
 }

From bf0a83da2f7de45cc1d47464098c2fd13b158863 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Thu, 19 Nov 2015 17:34:27 +0300
Subject: [PATCH 17/66] if-else braces

---
 .../scorex/crypto/ads/merkle/AuthDataBlock.scala   | 14 +++++---------
 1 file changed, 5 insertions(+), 9 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index d680af39..f6c510fd 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -19,26 +19,22 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
     def calculateHash(i: Position, nodeHash: Digest, path: Seq[Digest]): Digest = {
       if (i % 2 == 0) {
         val hash = hashFunction.hash(nodeHash ++ path.head)
-        if (path.size == 1) {
+        if (path.size == 1)
           hash
-        } else {
+        else
           calculateHash(i / 2, hash, path.tail)
-        }
       } else {
         val hash = hashFunction.hash(path.head ++ nodeHash)
-        if (path.size == 1) {
+        if (path.size == 1)
           hash
-        } else {
+        else
           calculateHash(i / 2, hash, path.tail)
-        }
       }
     }
     if (merklePath.nonEmpty) {
       val calculated = calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath)
       calculated.mkString == rootHash.mkString
-    } else {
-      true
-    }
+    } else true
   }
 }
 

From 6be7e07b5108a4e0ba6226c2eb8cf461ce459b94 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Thu, 19 Nov 2015 17:47:08 +0300
Subject: [PATCH 18/66] code simplification

---
 .../crypto/ads/merkle/AuthDataBlock.scala     | 30 ++++++++-----------
 1 file changed, 13 insertions(+), 17 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index f6c510fd..54011e86 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -1,8 +1,8 @@
 package scorex.crypto.ads.merkle
 
-import scorex.crypto.ads.merkle.Storage.Position
-import scorex.crypto.{Sha256, CryptographicHash}
 import scorex.crypto.CryptographicHash._
+import scorex.crypto.ads.merkle.Storage.Position
+import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
 
@@ -17,24 +17,20 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
 
     @tailrec
     def calculateHash(i: Position, nodeHash: Digest, path: Seq[Digest]): Digest = {
-      if (i % 2 == 0) {
-        val hash = hashFunction.hash(nodeHash ++ path.head)
-        if (path.size == 1)
-          hash
-        else
-          calculateHash(i / 2, hash, path.tail)
-      } else {
-        val hash = hashFunction.hash(path.head ++ nodeHash)
-        if (path.size == 1)
-          hash
-        else
-          calculateHash(i / 2, hash, path.tail)
-      }
+      val hash = if (i % 2 == 0)
+        hashFunction.hash(nodeHash ++ path.head)
+      else
+        hashFunction.hash(path.head ++ nodeHash)
+
+      if (path.size == 1)
+        hash
+      else
+        calculateHash(i / 2, hash, path.tail)
     }
+
     if (merklePath.nonEmpty) {
       val calculated = calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath)
       calculated.mkString == rootHash.mkString
     } else true
   }
-}
-
+}
\ No newline at end of file

From 0877cbe488d74a13e60cab5c6958e9fc1a6e3725 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Thu, 19 Nov 2015 17:48:42 +0300
Subject: [PATCH 19/66] not used vals fix

---
 .../src/test/scala/scorex/props/MerkleSpecification.scala     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index a06c9e28..ca500ca0 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -17,7 +17,7 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
 
   property("value returned from byIndex() is valid for random dataset") {
     forAll(smallInteger) { (blocks: Int) =>
-      val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks)
+      val (treeDirName: String, _, tempFile: String) = generateFile(blocks)
 
       val tree = MerkleTree.fromFile(tempFile, treeDirName)
       val index = Random.nextInt(tree.nonEmptyBlocks.asInstanceOf[Int])
@@ -31,7 +31,7 @@ class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDri
 
   property("hash root is the same") {
     forAll(smallInteger) { (blocks: Int) =>
-      val (treeDirName: String, treeDir: File, tempFile: String) = generateFile(blocks, "2")
+      val (treeDirName: String, _, tempFile: String) = generateFile(blocks, "2")
 
       val fileTree = MerkleTree.fromFile(tempFile, treeDirName)
       val rootHash = fileTree.rootHash

From 2a747db21111c7d44c8ca5a6d50c6417890531d2 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Thu, 19 Nov 2015 17:49:27 +0300
Subject: [PATCH 20/66] unused commented code removed

---
 .../scorex/perma/consensus/PermaLikeConsensusBlockData.scala    | 2 --
 1 file changed, 2 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
index a1c50433..1087ed50 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
@@ -2,8 +2,6 @@ package scorex.perma.consensus
 
 import scorex.perma.actors.Ticket
 
-//case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
-
 trait PermaLikeConsensusBlockData {
   val difficulty: BigInt
   val puz: Array[Byte]

From 134d626bcc941eb35973358d4876925408863c8a Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Mon, 23 Nov 2015 10:05:01 +0300
Subject: [PATCH 21/66] MapDBStorage refactoring

---
 .../crypto/ads/merkle/AuthDataBlock.scala     |  4 ++--
 .../crypto/ads/merkle/MapDBStorage.scala      | 14 ++++++++++---
 .../scorex/crypto/ads/merkle/MerkleTree.scala |  8 +++----
 .../scorex/crypto/ads/merkle/Storage.scala    | 16 +++-----------
 .../props/MapDBStorageSpecification.scala     |  4 ++--
 .../scorex/props/MerkleSpecification.scala    |  2 +-
 .../scala/scorex/perma/actors/Miner.scala     | 21 ++++++++++++-------
 7 files changed, 36 insertions(+), 33 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index 54011e86..e6df257b 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -1,7 +1,7 @@
 package scorex.crypto.ads.merkle
 
 import scorex.crypto.CryptographicHash._
-import scorex.crypto.ads.merkle.Storage.Position
+import scorex.crypto.ads.merkle.MapDBStorage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
@@ -33,4 +33,4 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
       calculated.mkString == rootHash.mkString
     } else true
   }
-}
\ No newline at end of file
+}
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
index fe682844..e8daf1a5 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
@@ -8,9 +8,9 @@ import scorex.crypto.CryptographicHash.Digest
 
 import scala.util.{Failure, Success, Try}
 
-class MapDBStorage(fileName: String, levels: Int) extends Storage {
+class MapDBStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Long], Array[Byte]] {
 
-  import Storage._
+  import MapDBStorage._
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
@@ -38,7 +38,7 @@ class MapDBStorage(fileName: String, levels: Int) extends Storage {
     val map = maps(key._1.asInstanceOf[Int])
     Try {
       map.put(key._2, value)
-    }.recoverWith{case t: Throwable =>
+    }.recoverWith { case t: Throwable =>
       log.warn("Failed to set key:" + key, t)
       Failure(t)
     }
@@ -67,3 +67,11 @@ class MapDBStorage(fileName: String, levels: Int) extends Storage {
   }
 
 }
+
+object MapDBStorage {
+  type Level = Int
+  type Position = Long
+  type Key = (Level, Position)
+  type Value = Digest
+
+}
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 81605e9c..1ab15b21 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -4,7 +4,7 @@ import java.io.{FileOutputStream, RandomAccessFile}
 import java.nio.file.{Files, Paths}
 
 import scorex.crypto.CryptographicHash.Digest
-import scorex.crypto.ads.merkle.Storage.Position
+import scorex.crypto.ads.merkle.MapDBStorage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
@@ -19,7 +19,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   val level = calculateRequiredLevel(nonEmptyBlocks)
 
-  lazy val storage: Storage = new MapDBStorage(treeFolder + "/tree", level)
+  lazy val storage = new MapDBStorage(treeFolder + "/tree", level)
 
   val rootHash: Digest = getHash((level, 0)).get
 
@@ -52,7 +52,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   private lazy val emptyHash = hash.hash("".getBytes)
 
-  def getHash(key: Storage.Key): Option[Digest] = {
+  def getHash(key: MapDBStorage.Key): Option[Digest] = {
     storage.get(key) match {
       case None =>
         if (key._1 > 0) {
@@ -109,7 +109,7 @@ object MerkleTree {
 
     val level = calculateRequiredLevel(nonEmptyBlocks)
 
-    lazy val storage: Storage = new MapDBStorage(treeFolder + "/tree", level)
+    lazy val storage = new MapDBStorage(treeFolder + "/tree", level)
 
     def processBlocks(currentBlock: Position = 0): Unit = {
       val block: Block = readLines(fileName, currentBlock)
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
index ed0d476e..2a104ba0 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
@@ -1,22 +1,12 @@
 package scorex.crypto.ads.merkle
 
-import scorex.crypto.CryptographicHash.Digest
+trait Storage[Key, Value] {
 
-trait Storage {
+  def set(key: Key, value: Value): Unit
 
-  import Storage._
-
-  def set(key: Key, value: Digest): Unit
-
-  def get(key: Key): Option[Digest]
+  def get(key: Key): Option[Value]
 
   def commit(): Unit
 
   def close(): Unit
-}
-
-object Storage {
-  type Level = Int
-  type Position = Long
-  type Key = (Level, Position)
 }
\ No newline at end of file
diff --git a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
index 7f7b2cf6..7eebf69c 100644
--- a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
@@ -6,7 +6,7 @@ import org.scalacheck.{Arbitrary, Gen}
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.CryptographicHash.Digest
-import scorex.crypto.ads.merkle.Storage.Key
+import scorex.crypto.ads.merkle.MapDBStorage.Key
 import scorex.crypto.ads.merkle.{MapDBStorage, Storage}
 
 
@@ -27,7 +27,7 @@ class MapDBStorageSpecification extends PropSpec with PropertyChecks with Genera
 
 
   property("set value and get it") {
-    lazy val storage: Storage = new MapDBStorage(treeDirName + "/test_db", maxLevel)
+    lazy val storage = new MapDBStorage(treeDirName + "/test_db", maxLevel)
 
     forAll(keyVal) { x =>
       val key: Key = x._1
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index ca500ca0..f1ec8fd8 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -12,7 +12,7 @@ import scala.util.Random
 
 class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
-  val smallInteger = Gen.choose(100, 500)
+  val smallInteger = Gen.choose(8, 128)
 
 
   property("value returned from byIndex() is valid for random dataset") {
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 124d4649..060f8db2 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -8,7 +8,8 @@ import akka.util.Timeout
 import scorex.crypto.CryptographicHash._
 import scorex.crypto.SigningFunctions.{PrivateKey, PublicKey, Signature}
 import scorex.crypto._
-import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.crypto.ads.merkle.{Storage, AuthDataBlock}
+import scorex.crypto.ads.merkle.MerkleTree.Block
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.Parameters
 import scorex.perma.actors.MinerSpec._
@@ -33,18 +34,19 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act
 
   private var segments: Subset = Map()
 
+  private lazy val segmentIds = 1.to(Parameters.l).map { i =>
+    u(keyPair._2, i - 1)
+  }.toArray
+
   override def receive = {
 
     case Initialize =>
       log.debug("Initialize")
 
-      val segmentIdsToDownload = 1.to(Parameters.l).map { i =>
-        u(keyPair._2, i - 1)
-      }.toArray
-
       val s = sender()
 
-      trustedDealerRef ? SegmentsRequest(segmentIdsToDownload) onComplete {
+      //TODO request other miners first
+      trustedDealerRef ? SegmentsRequest(segmentIds) onComplete {
         case Success(m) =>
           self ! m
           s ! Initialized
@@ -54,7 +56,6 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act
 
     case SegmentsToStore(sgs) =>
       log.debug("SegmentsToStore({})", sgs)
-      require(segments.isEmpty)
       segments = sgs
 
     case TicketGeneration(difficulty, puz) =>
@@ -85,7 +86,7 @@ object Miner {
   val NoSig = Array[Byte]()
 
   //calculate index of i-th segment
-  private def u(pubKey: SigningFunctions.PublicKey, i: Int): Int = {
+  private def u(pubKey: PublicKey, i: Int): Int = {
     val h = Sha256.hash(pubKey ++ BigInt(i).toByteArray)
     BigInt(1, h).mod(Parameters.n).toInt
   }
@@ -98,6 +99,10 @@ object Miner {
     r
   }
 
+//  def segments(index: Index, storage: Storage): AuthDataBlock[Block] = {
+//
+//  }
+
   def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte], segments: Subset): Ticket = {
 
     val (privateKey, publicKey) = keyPair

From ad574b736013f48828edffee2a9de69a860b2442 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Mon, 23 Nov 2015 10:37:32 +0300
Subject: [PATCH 22/66] Rename MapDBStorage

---
 .../scala/scorex/crypto/ads/merkle/AuthDataBlock.scala    | 2 +-
 .../main/scala/scorex/crypto/ads/merkle/MerkleTree.scala  | 8 ++++----
 .../ads/merkle/{MapDBStorage.scala => TreeStorage.scala}  | 7 ++++---
 .../scorex/{crypto/ads/merkle => storage}/Storage.scala   | 2 +-
 .../scala/scorex/props/MapDBStorageSpecification.scala    | 6 +++---
 .../src/main/scala/scorex/perma/actors/Miner.scala        | 2 +-
 6 files changed, 14 insertions(+), 13 deletions(-)
 rename scorex-basics/src/main/scala/scorex/crypto/ads/merkle/{MapDBStorage.scala => TreeStorage.scala} (90%)
 rename scorex-basics/src/main/scala/scorex/{crypto/ads/merkle => storage}/Storage.scala (81%)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index e6df257b..7c8a04e4 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -1,7 +1,7 @@
 package scorex.crypto.ads.merkle
 
 import scorex.crypto.CryptographicHash._
-import scorex.crypto.ads.merkle.MapDBStorage.Position
+import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 1ab15b21..aeb35512 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -4,7 +4,7 @@ import java.io.{FileOutputStream, RandomAccessFile}
 import java.nio.file.{Files, Paths}
 
 import scorex.crypto.CryptographicHash.Digest
-import scorex.crypto.ads.merkle.MapDBStorage.Position
+import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
@@ -19,7 +19,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   val level = calculateRequiredLevel(nonEmptyBlocks)
 
-  lazy val storage = new MapDBStorage(treeFolder + "/tree", level)
+  lazy val storage = new TreeStorage(treeFolder + "/tree", level)
 
   val rootHash: Digest = getHash((level, 0)).get
 
@@ -52,7 +52,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   private lazy val emptyHash = hash.hash("".getBytes)
 
-  def getHash(key: MapDBStorage.Key): Option[Digest] = {
+  def getHash(key: TreeStorage.Key): Option[Digest] = {
     storage.get(key) match {
       case None =>
         if (key._1 > 0) {
@@ -109,7 +109,7 @@ object MerkleTree {
 
     val level = calculateRequiredLevel(nonEmptyBlocks)
 
-    lazy val storage = new MapDBStorage(treeFolder + "/tree", level)
+    lazy val storage = new TreeStorage(treeFolder + "/tree", level)
 
     def processBlocks(currentBlock: Position = 0): Unit = {
       val block: Block = readLines(fileName, currentBlock)
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala
similarity index 90%
rename from scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
rename to scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala
index e8daf1a5..37dd4c14 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MapDBStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala
@@ -5,12 +5,13 @@ import java.io.File
 import org.mapdb.{DBMaker, HTreeMap, Serializer}
 import org.slf4j.LoggerFactory
 import scorex.crypto.CryptographicHash.Digest
+import scorex.storage.Storage
 
 import scala.util.{Failure, Success, Try}
 
-class MapDBStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Long], Array[Byte]] {
+class TreeStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Long], Array[Byte]] {
 
-  import MapDBStorage._
+  import TreeStorage._
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
@@ -68,7 +69,7 @@ class MapDBStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Lo
 
 }
 
-object MapDBStorage {
+object TreeStorage {
   type Level = Int
   type Position = Long
   type Key = (Level, Position)
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala b/scorex-basics/src/main/scala/scorex/storage/Storage.scala
similarity index 81%
rename from scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
rename to scorex-basics/src/main/scala/scorex/storage/Storage.scala
index 2a104ba0..4f2127f3 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/Storage.scala
+++ b/scorex-basics/src/main/scala/scorex/storage/Storage.scala
@@ -1,4 +1,4 @@
-package scorex.crypto.ads.merkle
+package scorex.storage
 
 trait Storage[Key, Value] {
 
diff --git a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
index 7eebf69c..39a3d2ff 100644
--- a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
@@ -6,8 +6,8 @@ import org.scalacheck.{Arbitrary, Gen}
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.CryptographicHash.Digest
-import scorex.crypto.ads.merkle.MapDBStorage.Key
-import scorex.crypto.ads.merkle.{MapDBStorage, Storage}
+import scorex.crypto.ads.merkle.TreeStorage.Key
+import scorex.crypto.ads.merkle.TreeStorage$
 
 
 class MapDBStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
@@ -27,7 +27,7 @@ class MapDBStorageSpecification extends PropSpec with PropertyChecks with Genera
 
 
   property("set value and get it") {
-    lazy val storage = new MapDBStorage(treeDirName + "/test_db", maxLevel)
+    lazy val storage = new TreeStorage(treeDirName + "/test_db", maxLevel)
 
     forAll(keyVal) { x =>
       val key: Key = x._1
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 060f8db2..b2321ec0 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -8,7 +8,7 @@ import akka.util.Timeout
 import scorex.crypto.CryptographicHash._
 import scorex.crypto.SigningFunctions.{PrivateKey, PublicKey, Signature}
 import scorex.crypto._
-import scorex.crypto.ads.merkle.{Storage, AuthDataBlock}
+import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.crypto.ads.merkle.MerkleTree.Block
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.Parameters

From e99b67ce94513c64adb8df7c97148d6393e95964 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Mon, 23 Nov 2015 11:10:52 +0300
Subject: [PATCH 23/66] Miners keep their data in mapDB storage

---
 .../test/scala/scorex/ScorexTestSuite.scala   |  3 +-
 ...n.scala => TreeStorageSpecification.scala} |  4 +-
 .../main/scala/scorex/perma/Parameters.scala  |  2 +-
 .../perma/Storage/AuthDataStorage.scala       | 57 +++++++++++++++++++
 .../scala/scorex/perma/actors/Miner.scala     | 24 ++++----
 .../test/scala/scorex/PermaTestSuite.scala    |  7 +++
 .../props/AuthDataStorageSpecification.scala  | 36 ++++++++++++
 7 files changed, 116 insertions(+), 17 deletions(-)
 rename scorex-basics/src/test/scala/scorex/props/{MapDBStorageSpecification.scala => TreeStorageSpecification.scala} (86%)
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
 create mode 100644 scorex-perma/src/test/scala/scorex/PermaTestSuite.scala
 create mode 100644 scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala

diff --git a/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala b/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala
index 7a237a64..1f9a8845 100644
--- a/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala
+++ b/scorex-basics/src/test/scala/scorex/ScorexTestSuite.scala
@@ -10,5 +10,6 @@ class ScorexTestSuite extends Suites (
   new unit.Sha256Specification,
 
   new props.SigningFunctionsSpecification,
-  new props.MerkleSpecification
+  new props.MerkleSpecification,
+  new props.TreeStorageSpecification
 )
diff --git a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala b/scorex-basics/src/test/scala/scorex/props/TreeStorageSpecification.scala
similarity index 86%
rename from scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
rename to scorex-basics/src/test/scala/scorex/props/TreeStorageSpecification.scala
index 39a3d2ff..cf32e171 100644
--- a/scorex-basics/src/test/scala/scorex/props/MapDBStorageSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/TreeStorageSpecification.scala
@@ -7,10 +7,10 @@ import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.CryptographicHash.Digest
 import scorex.crypto.ads.merkle.TreeStorage.Key
-import scorex.crypto.ads.merkle.TreeStorage$
+import scorex.crypto.ads.merkle.TreeStorage
 
 
-class MapDBStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
+class TreeStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
   val treeDirName = "/tmp/scorex/test/MapDBStorageSpecification/"
   val treeDir = new File(treeDirName)
diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
index abaaac0b..f8240a4d 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
@@ -7,7 +7,7 @@ object Parameters {
   //few segments to be stored in a block, so segment size shouldn't be big
   val segmentSize = 1024 //segment size in bytes
 
-  val n = 1024*64 //how many segments in a dataset in total
+  val n = 1024*4 //how many segments in a dataset in total
   val l = 1024 //how many segments to store for an each miner
 
   val k = 4 //number of iterations during scratch-off phase
diff --git a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
new file mode 100644
index 00000000..84d56f39
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
@@ -0,0 +1,57 @@
+package scorex.perma.Storage
+
+import java.io.File
+
+import org.mapdb.{DBMaker, HTreeMap, Serializer}
+import org.slf4j.LoggerFactory
+import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.perma.Parameters.DataSegment
+import scorex.storage.Storage
+
+import scala.util.{Failure, Success, Try}
+
+class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[DataSegment]] {
+
+  private val log = LoggerFactory.getLogger(this.getClass)
+
+  private val db =
+    DBMaker.appendFileDB(new File(fileName + ".mapDB"))
+      .fileMmapEnableIfSupported()
+      .closeOnJvmShutdown()
+      .checksumEnable()
+      .make()
+
+  private val map: HTreeMap[Long, AuthDataBlock[DataSegment]] = db.hashMapCreate("segments")
+    .keySerializer(Serializer.LONG)
+    .makeOrGet()
+
+  override def set(key: Long, value: AuthDataBlock[DataSegment]): Unit = {
+    Try {
+      map.put(key, value)
+    }.recoverWith { case t: Throwable =>
+      log.warn("Failed to set key:" + key, t)
+      Failure(t)
+    }
+  }
+
+  override def commit(): Unit = db.commit()
+
+  override def close(): Unit = {
+    commit()
+    db.close()
+  }
+
+  override def get(key: Long): Option[AuthDataBlock[DataSegment]] = {
+    Try {
+      map.get(key)
+    } match {
+      case Success(v) =>
+        Option(v)
+
+      case Failure(e) =>
+        log.debug("Enable to load key for level 0: " + key)
+        None
+    }
+  }
+
+}
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index b2321ec0..3717cdf7 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -12,6 +12,7 @@ import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.crypto.ads.merkle.MerkleTree.Block
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.Parameters
+import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
 
@@ -32,8 +33,6 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act
   private val keyPair = EllipticCurveImpl.createKeyPair(randomBytes(32))
   private implicit val timeout = Timeout(1 minute)
 
-  private var segments: Subset = Map()
-
   private lazy val segmentIds = 1.to(Parameters.l).map { i =>
     u(keyPair._2, i - 1)
   }.toArray
@@ -56,11 +55,12 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act
 
     case SegmentsToStore(sgs) =>
       log.debug("SegmentsToStore({})", sgs)
-      segments = sgs
+      sgs.map(s => storage.set(s._1, s._2))
+      storage.commit()
 
     case TicketGeneration(difficulty, puz) =>
       log.debug("TicketGeneration({})", puz)
-      val ticket = generate(keyPair, puz, segments)
+      val ticket = generate(keyPair, puz)
 
       val check = validate(keyPair._2, puz, difficulty, ticket, rootHash)
       val score = ticketScore(ticket)
@@ -99,11 +99,10 @@ object Miner {
     r
   }
 
-//  def segments(index: Index, storage: Storage): AuthDataBlock[Block] = {
-//
-//  }
+  //TODO take file name from config
+  val storage = new AuthDataStorage("/tmp/scorex/testApp/segmentsStorage.mapDB")
 
-  def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte], segments: Subset): Ticket = {
+  def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
 
     val (privateKey, publicKey) = keyPair
 
@@ -112,18 +111,17 @@ object Miner {
 
     val sig0 = NoSig
     val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Parameters.l).toInt)
-      .ensuring(r => segments.keySet.contains(r))
 
-    val proofs = 1.to(Parameters.k).foldLeft(
+    val proofs:IndexedSeq[PartialProof] = 1.to(Parameters.k).foldLeft(
       (r1, sig0, Seq[PartialProof]())
     ) {
       case ((ri, sig_prev, seq), _) =>
-        val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segments(ri).data)
+        val segment = storage.get(ri).get
+        val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
         val sig = EllipticCurveImpl.sign(privateKey, hi)
         val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Parameters.l).toInt)
-          .ensuring(r => segments.keySet.contains(r))
 
-        (r_next, sig, seq :+ PartialProof(sig, ri, segments(ri)))
+        (r_next, sig, seq :+ PartialProof(sig, ri, segment))
     }._3.toIndexedSeq.ensuring(_.size == Parameters.k)
 
     Ticket(publicKey, s, proofs)
diff --git a/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala b/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala
new file mode 100644
index 00000000..c0e71dac
--- /dev/null
+++ b/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala
@@ -0,0 +1,7 @@
+package scorex
+
+import org.scalatest.Suites
+
+class PermaTestSuite extends Suites (
+  new props.AuthDataStorageSpecification
+)
diff --git a/scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala b/scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala
new file mode 100644
index 00000000..b32fc3f3
--- /dev/null
+++ b/scorex-perma/src/test/scala/scorex/props/AuthDataStorageSpecification.scala
@@ -0,0 +1,36 @@
+package scorex.props
+
+import java.io.File
+
+import org.scalacheck.{Arbitrary, Gen}
+import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
+import org.scalatest.{Matchers, PropSpec}
+import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.perma.Storage.AuthDataStorage
+
+
+class AuthDataStorageSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
+
+  val treeDirName = "/tmp/scorex/test/AuthDataStorageSpecification/"
+  val treeDir = new File(treeDirName)
+  treeDir.mkdirs()
+
+  val keyVal = for {
+    key: Long <- Arbitrary.arbitrary[Long]
+    value <- Arbitrary.arbitrary[String]
+  } yield (key, AuthDataBlock(value.getBytes, Seq()))
+
+
+  property("set value and get it") {
+    lazy val storage = new AuthDataStorage(treeDirName + "/test_db")
+
+    forAll(keyVal) { x =>
+      val key = x._1
+      val value = x._2
+      storage.set(key, value)
+
+      assert(storage.get(key).get.data sameElements value.data)
+    }
+    storage.close()
+  }
+}
\ No newline at end of file

From 014b25190db5ccc86e5ba0e52ed2d2a2f9c6384a Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Mon, 23 Nov 2015 16:25:48 +0300
Subject: [PATCH 24/66] Ask segments from other miners first

---
 .../scorex/crypto/ads/merkle/MerkleTree.scala |  2 +-
 .../main/scala/scorex/storage/Storage.scala   |  5 ++
 .../scorex/perma/BlockchainBuilder.scala      | 25 ++++--
 .../main/scala/scorex/perma/Parameters.scala  |  4 +
 .../perma/Storage/AuthDataStorage.scala       |  4 +
 .../src/main/scala/scorex/perma/TestApp.scala | 16 ++--
 .../scala/scorex/perma/actors/Miner.scala     | 87 +++++++++++++------
 .../scorex/perma/actors/TrustedDealer.scala   |  6 +-
 8 files changed, 102 insertions(+), 47 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index aeb35512..03f147e8 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -117,7 +117,7 @@ object MerkleTree {
       fos.write(block)
       fos.close()
       storage.set((0, currentBlock), hash.hash(block))
-      if (currentBlock < nonEmptyBlocks) {
+      if (currentBlock < nonEmptyBlocks - 1) {
         processBlocks(currentBlock + 1)
       }
     }
diff --git a/scorex-basics/src/main/scala/scorex/storage/Storage.scala b/scorex-basics/src/main/scala/scorex/storage/Storage.scala
index 4f2127f3..8f133882 100644
--- a/scorex-basics/src/main/scala/scorex/storage/Storage.scala
+++ b/scorex-basics/src/main/scala/scorex/storage/Storage.scala
@@ -9,4 +9,9 @@ trait Storage[Key, Value] {
   def commit(): Unit
 
   def close(): Unit
+
+  def containsKey(key: Key): Boolean = {
+    get(key).isDefined
+  }
+  
 }
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
index 5eb2e00d..77c619c1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -1,17 +1,19 @@
 package scorex.perma
 
 import akka.actor.{Actor, ActorRef}
-import scorex.perma.actors.MinerSpec.{Initialize, Initialized, TicketGeneration}
+import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.Ticket
 import scorex.utils.ScorexLogging
 
 import scala.collection.mutable
+import scala.concurrent.ExecutionContext.Implicits.global
+import scala.concurrent.duration._
 import scala.util.Random
 
 
 case class BlockHeaderLike(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
 
-class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging {
+class BlockchainBuilder(miners: Seq[ActorRef], dealer: ActorRef) extends Actor with ScorexLogging {
 
   import BlockchainBuilderSpec._
 
@@ -26,11 +28,17 @@ class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging
   def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
 
   override def receive = {
-    case Initialized =>
-      initialized = initialized + 1
-      if (initialized == miners.length) {
-        log.info("start BlockchainBuilder")
-        self ! SendWorkToMiners
+    case s: MinerStatus =>
+      s match {
+        case Initialized =>
+          initialized = initialized + 1
+          if (initialized == miners.length) {
+            log.info("All miners initialized")
+            self ! SendWorkToMiners
+          }
+        case LoadingData =>
+          sender() ! Initialize(Seq(dealer))
+          context.system.scheduler.scheduleOnce(200 millis, sender(), GetStatus)
       }
 
     case SendWorkToMiners =>
@@ -38,7 +46,8 @@ class BlockchainBuilder(miners: Seq[ActorRef]) extends Actor with ScorexLogging
         if (initialized == miners.length) {
           minerRef ! TicketGeneration(difficulty, puz)
         } else {
-          minerRef ! Initialize
+          minerRef ! Initialize(miners)
+          context.system.scheduler.scheduleOnce(200 millis, minerRef, GetStatus)
         }
       }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
index f8240a4d..bc4279f1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
@@ -1,5 +1,7 @@
 package scorex.perma
 
+import scorex.crypto.Sha256
+
 object Parameters {
 
   type DataSegment = Array[Byte]
@@ -11,4 +13,6 @@ object Parameters {
   val l = 1024 //how many segments to store for an each miner
 
   val k = 4 //number of iterations during scratch-off phase
+
+  val hash = Sha256
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
index 84d56f39..f6f7bb75 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
@@ -41,6 +41,10 @@ class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[Data
     db.close()
   }
 
+  override def containsKey(key: Long): Boolean = {
+    map.containsKey(key)
+  }
+
   override def get(key: Long): Option[AuthDataBlock[DataSegment]] = {
     Try {
       map.get(key)
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index fe2bae64..d9f5de81 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -6,9 +6,9 @@ import java.nio.file.{Files, Paths}
 import akka.actor.{ActorSystem, Props}
 import akka.util.Timeout
 import org.slf4j.LoggerFactory
-import scorex.crypto.Sha256
 import scorex.crypto.ads.merkle.MerkleTree
 import scorex.perma.BlockchainBuilderSpec.SendWorkToMiners
+import scorex.perma.actors.MinerSpec.Initialize
 import scorex.perma.actors.{Miner, TrustedDealer}
 
 import scala.concurrent.duration._
@@ -24,7 +24,7 @@ object TestApp extends App {
 
   val tree = if (Files.exists(Paths.get(treeDirName + "/tree0.mapDB"))) {
     log.info("Get existing tree")
-    new MerkleTree(treeDirName, Parameters.n)
+    new MerkleTree(treeDirName, Parameters.n, Parameters.segmentSize, Parameters.hash)
   } else {
     log.info("Generating random data set")
     val treeDir = new File(treeDirName)
@@ -32,7 +32,7 @@ object TestApp extends App {
     val datasetFile = treeDirName + "/data.file"
     new RandomAccessFile(datasetFile, "rw").setLength(Parameters.n * Parameters.segmentSize)
     log.info("Calculate tree")
-    val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize)
+    val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize, Parameters.hash)
     require(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
     tree
   }
@@ -40,17 +40,21 @@ object TestApp extends App {
   log.info("test tree")
   val index = Parameters.n - 3
   val leaf = tree.byIndex(index).get
-  require(leaf.check(index, tree.rootHash)(Sha256))
+  require(leaf.check(index, tree.rootHash)(Parameters.hash))
 
   log.info("Success: " + tree.rootHash.mkString)
 
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")
   val dealer = actorSystem.actorOf(Props(new TrustedDealer(tree)))
-  val miners = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], dealer, tree.rootHash), s"m-$x"))
+  val miners = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], tree.rootHash), s"m-$x"))
 
   implicit val timeout = Timeout(1 minute)
 
-  val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners), "BlockchainBuilder")
+  //Test, that miners download data from each other
+  miners.head ! Initialize(Seq(dealer))
+  Thread.sleep(200)
+
+  val blockchainBuilder = actorSystem.actorOf(Props(classOf[BlockchainBuilder], miners, dealer), "BlockchainBuilder")
   blockchainBuilder ! SendWorkToMiners
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 3717cdf7..57a1356f 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -3,60 +3,85 @@ package scorex.perma.actors
 import java.security.SecureRandom
 
 import akka.actor.{Actor, ActorLogging, ActorRef}
-import akka.pattern.ask
 import akka.util.Timeout
 import scorex.crypto.CryptographicHash._
 import scorex.crypto.SigningFunctions.{PrivateKey, PublicKey, Signature}
 import scorex.crypto._
 import scorex.crypto.ads.merkle.AuthDataBlock
-import scorex.crypto.ads.merkle.MerkleTree.Block
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.Parameters
 import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
 
+import scala.collection.mutable.ListBuffer
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
-import scala.util.{Success, Try}
+import scala.util.Try
 
-case class PartialProof(signature: Signature, segmentIndex: Int, segment: AuthDataBlock[Parameters.DataSegment])
+case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[Parameters.DataSegment])
 
 case class Ticket(publicKey: PublicKey,
                   s: Array[Byte],
                   proofs: IndexedSeq[PartialProof])
 
-class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with ActorLogging {
+class Miner(rootHash: Digest) extends Actor with ActorLogging {
 
   import Miner._
 
   private val keyPair = EllipticCurveImpl.createKeyPair(randomBytes(32))
   private implicit val timeout = Timeout(1 minute)
 
-  private lazy val segmentIds = 1.to(Parameters.l).map { i =>
+  private val segmentIds: Seq[Long] = 1.to(Parameters.l).map { i =>
     u(keyPair._2, i - 1)
-  }.toArray
+  }.toSeq
+
+  //Mutable list of ids, remaining to download
+  private var segmentToDownload: ListBuffer[Long] = segmentIds.to[ListBuffer]
 
   override def receive = {
 
-    case Initialize =>
+    case Initialize(miners) =>
       log.debug("Initialize")
 
       val s = sender()
 
-      //TODO request other miners first
-      trustedDealerRef ? SegmentsRequest(segmentIds) onComplete {
-        case Success(m) =>
-          self ! m
-          s ! Initialized
-        case _ =>
-          context.stop(self)
+      segmentToDownload foreach { s =>
+        if (authDataStorage.containsKey(s)) {
+          segmentToDownload -= s
+        }
+      }
+
+      if (segmentToDownload.nonEmpty) {
+        miners.foreach(_ ! SegmentsRequest(segmentToDownload))
+      }
+
+    case GetStatus =>
+      log.debug("Get status")
+      if (segmentToDownload.isEmpty) {
+        sender ! Initialized
+      } else {
+        sender ! LoadingData
+      }
+
+    case SegmentsRequest(ids) =>
+      val segments: Subset = ids.map { x =>
+        x -> authDataStorage.get(x)
+      }.toMap.collect {
+        case (key, Some(value)) => key -> value
       }
+      log.info(s"Miner SegmentsRequest for ${ids.length} blocks returns ${segments.size} blocks")
+      sender ! SegmentsToStore(segments)
 
     case SegmentsToStore(sgs) =>
       log.debug("SegmentsToStore({})", sgs)
-      sgs.map(s => storage.set(s._1, s._2))
-      storage.commit()
+      sgs.foreach { s =>
+        if (segmentToDownload.contains(s._1) && s._2.check(s._1, rootHash)(Parameters.hash)) {
+          authDataStorage.set(s._1, s._2)
+          segmentToDownload -= s._1
+        }
+      }
+      authDataStorage.commit()
 
     case TicketGeneration(difficulty, puz) =>
       log.debug("TicketGeneration({})", puz)
@@ -69,8 +94,7 @@ class Miner(trustedDealerRef: ActorRef, rootHash: Digest) extends Actor with Act
       if (check) {
         sender() ! WinningTicket(puz, score, ticket)
       } else {
-        Thread.sleep(100)
-        self ! TicketGeneration(difficulty, puz)
+        context.system.scheduler.scheduleOnce(200 millis, self, TicketGeneration(difficulty, puz))
       }
 
     case TicketValidation(difficulty, puz, t: Ticket) =>
@@ -86,9 +110,9 @@ object Miner {
   val NoSig = Array[Byte]()
 
   //calculate index of i-th segment
-  private def u(pubKey: PublicKey, i: Int): Int = {
+  private def u(pubKey: PublicKey, i: Int): Long = {
     val h = Sha256.hash(pubKey ++ BigInt(i).toByteArray)
-    BigInt(1, h).mod(Parameters.n).toInt
+    BigInt(1, h).mod(Parameters.n).toLong
   }
 
 
@@ -100,7 +124,7 @@ object Miner {
   }
 
   //TODO take file name from config
-  val storage = new AuthDataStorage("/tmp/scorex/testApp/segmentsStorage.mapDB")
+  val authDataStorage = new AuthDataStorage("/tmp/scorex/testApp/segmentsStorage.mapDB")
 
   def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
 
@@ -112,11 +136,11 @@ object Miner {
     val sig0 = NoSig
     val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Parameters.l).toInt)
 
-    val proofs:IndexedSeq[PartialProof] = 1.to(Parameters.k).foldLeft(
+    val proofs: IndexedSeq[PartialProof] = 1.to(Parameters.k).foldLeft(
       (r1, sig0, Seq[PartialProof]())
     ) {
       case ((ri, sig_prev, seq), _) =>
-        val segment = storage.get(ri).get
+        val segment = authDataStorage.get(ri).get
         val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
         val sig = EllipticCurveImpl.sign(privateKey, hi)
         val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Parameters.l).toInt)
@@ -160,15 +184,22 @@ object Miner {
 
 object MinerSpec {
 
-  type Index = Int
+  type Index = Long
   type Subset = Map[Index, AuthDataBlock[Parameters.DataSegment]]
 
-  case object Initialize
-
-  case object Initialized
+  case class Initialize(miners: Seq[ActorRef])
 
   case class TicketGeneration(difficulty: BigInt, puz: Array[Byte])
 
   case class TicketValidation(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
 
+  case object GetStatus
+
+  sealed trait MinerStatus
+
+  case object Initialized extends MinerStatus
+
+  case object LoadingData extends MinerStatus
+
+
 }
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
index a7361a55..b59574c7 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
@@ -12,9 +12,7 @@ class TrustedDealer[H <: CryptographicHash](val tree: MerkleTree[H]) extends Act
 
   override def receive = {
     case SegmentsRequest(segmentIds) =>
-      log.info(s"SegmentsRequest(${segmentIds.mkString(", ")})")
-
-      assert(segmentIds.length == Parameters.l)
+      log.info(s"Dealer SegmentsRequest for ${segmentIds.length} blocks")
 
       val segments: Subset = segmentIds.map { x =>
         x -> tree.byIndex(x)
@@ -32,7 +30,7 @@ object TrustedDealerSpec {
 
   case object PublishDataset
 
-  case class SegmentsRequest(segments: Array[Int])
+  case class SegmentsRequest(segments: Seq[Long])
 
   case class SegmentsToStore(segments: Subset)
 

From 975200f847fd1ba622df2c00e6cf9d4f54046b4d Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Mon, 23 Nov 2015 18:55:54 +0300
Subject: [PATCH 25/66] Commit only when need

---
 .../scorex/perma/Storage/AuthDataStorage.scala    | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
index f6f7bb75..ccec95e9 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
@@ -14,6 +14,9 @@ class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[Data
 
   private val log = LoggerFactory.getLogger(this.getClass)
 
+  //TODO https://github.com/jankotek/mapdb/issues/634 workaround
+  private var commitNeeded = false
+
   private val db =
     DBMaker.appendFileDB(new File(fileName + ".mapDB"))
       .fileMmapEnableIfSupported()
@@ -28,19 +31,21 @@ class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[Data
   override def set(key: Long, value: AuthDataBlock[DataSegment]): Unit = {
     Try {
       map.put(key, value)
+
     }.recoverWith { case t: Throwable =>
       log.warn("Failed to set key:" + key, t)
       Failure(t)
     }
   }
 
-  override def commit(): Unit = db.commit()
-
-  override def close(): Unit = {
-    commit()
-    db.close()
+  override def commit(): Unit = if (commitNeeded) {
+    db.commit()
+    commitNeeded = false
   }
 
+  override def close(): Unit = db.close()
+
+
   override def containsKey(key: Long): Boolean = {
     map.containsKey(key)
   }

From de39ba403f3ed93c798f5d1b1d5f6d7180566dd4 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 24 Nov 2015 11:14:56 +0300
Subject: [PATCH 26/66] Move constants to config

---
 .../scorex/crypto/ads/merkle/MerkleTree.scala |  6 ++--
 .../crypto/ads/merkle/TreeStorage.scala       |  6 ++--
 .../src/main/resources/application.conf       |  7 +++++
 .../main/scala/scorex/perma/Parameters.scala  | 18 -----------
 .../perma/Storage/AuthDataStorage.scala       |  8 ++---
 .../src/main/scala/scorex/perma/TestApp.scala | 30 ++++++++++---------
 .../scala/scorex/perma/actors/Miner.scala     | 30 +++++++++----------
 .../scorex/perma/actors/TrustedDealer.scala   |  2 +-
 .../scorex/perma/settings/Constants.scala     | 30 +++++++++++++++++++
 .../scorex/perma/settings/PermaSettings.scala | 15 ++++++++++
 10 files changed, 94 insertions(+), 58 deletions(-)
 create mode 100644 scorex-perma/src/main/resources/application.conf
 delete mode 100644 scorex-perma/src/main/scala/scorex/perma/Parameters.scala
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 03f147e8..4269743a 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -19,7 +19,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 
   val level = calculateRequiredLevel(nonEmptyBlocks)
 
-  lazy val storage = new TreeStorage(treeFolder + "/tree", level)
+  lazy val storage = new TreeStorage(treeFolder + TreeFileName, level)
 
   val rootHash: Digest = getHash((level, 0)).get
 
@@ -79,6 +79,8 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
 object MerkleTree {
   type Block = Array[Byte]
 
+  val TreeFileName = "/hashTree"
+
   def fromFile[H <: CryptographicHash](fileName: String,
                                        treeFolder: String,
                                        blockSize: Int = 1024,
@@ -109,7 +111,7 @@ object MerkleTree {
 
     val level = calculateRequiredLevel(nonEmptyBlocks)
 
-    lazy val storage = new TreeStorage(treeFolder + "/tree", level)
+    lazy val storage = new TreeStorage(treeFolder + TreeFileName, level)
 
     def processBlocks(currentBlock: Position = 0): Unit = {
       val block: Block = readLines(fileName, currentBlock)
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala
index 37dd4c14..022d768e 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/TreeStorage.scala
@@ -3,18 +3,16 @@ package scorex.crypto.ads.merkle
 import java.io.File
 
 import org.mapdb.{DBMaker, HTreeMap, Serializer}
-import org.slf4j.LoggerFactory
 import scorex.crypto.CryptographicHash.Digest
 import scorex.storage.Storage
+import scorex.utils.ScorexLogging
 
 import scala.util.{Failure, Success, Try}
 
-class TreeStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Long], Array[Byte]] {
+class TreeStorage(fileName: String, levels: Int) extends Storage[Tuple2[Int, Long], Array[Byte]] with ScorexLogging {
 
   import TreeStorage._
 
-  private val log = LoggerFactory.getLogger(this.getClass)
-
   private val dbs =
     (0 to levels) map { n: Int =>
       DBMaker.fileDB(new File(fileName + n + ".mapDB"))
diff --git a/scorex-perma/src/main/resources/application.conf b/scorex-perma/src/main/resources/application.conf
new file mode 100644
index 00000000..20ff02c6
--- /dev/null
+++ b/scorex-perma/src/main/resources/application.conf
@@ -0,0 +1,7 @@
+perma {
+  segmentSize = 1024
+  n = 1024
+  l = 1024
+  k = 4
+  hash = "sha256"
+}
diff --git a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala b/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
deleted file mode 100644
index bc4279f1..00000000
--- a/scorex-perma/src/main/scala/scorex/perma/Parameters.scala
+++ /dev/null
@@ -1,18 +0,0 @@
-package scorex.perma
-
-import scorex.crypto.Sha256
-
-object Parameters {
-
-  type DataSegment = Array[Byte]
-
-  //few segments to be stored in a block, so segment size shouldn't be big
-  val segmentSize = 1024 //segment size in bytes
-
-  val n = 1024*4 //how many segments in a dataset in total
-  val l = 1024 //how many segments to store for an each miner
-
-  val k = 4 //number of iterations during scratch-off phase
-
-  val hash = Sha256
-}
diff --git a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
index ccec95e9..42e24d12 100644
--- a/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/Storage/AuthDataStorage.scala
@@ -3,16 +3,16 @@ package scorex.perma.Storage
 import java.io.File
 
 import org.mapdb.{DBMaker, HTreeMap, Serializer}
-import org.slf4j.LoggerFactory
 import scorex.crypto.ads.merkle.AuthDataBlock
-import scorex.perma.Parameters.DataSegment
+import scorex.perma.settings.Constants
+import scorex.perma.settings.Constants.DataSegment
 import scorex.storage.Storage
+import scorex.utils.ScorexLogging
 
 import scala.util.{Failure, Success, Try}
 
-class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[DataSegment]] {
+class AuthDataStorage(fileName: String) extends Storage[Long, AuthDataBlock[DataSegment]] with ScorexLogging {
 
-  private val log = LoggerFactory.getLogger(this.getClass)
 
   //TODO https://github.com/jankotek/mapdb/issues/634 workaround
   private var commitNeeded = false
diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index d9f5de81..d6f19f77 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -5,42 +5,44 @@ import java.nio.file.{Files, Paths}
 
 import akka.actor.{ActorSystem, Props}
 import akka.util.Timeout
-import org.slf4j.LoggerFactory
 import scorex.crypto.ads.merkle.MerkleTree
 import scorex.perma.BlockchainBuilderSpec.SendWorkToMiners
 import scorex.perma.actors.MinerSpec.Initialize
 import scorex.perma.actors.{Miner, TrustedDealer}
+import scorex.perma.settings.{Constants, PermaSettings}
+import scorex.settings.Settings
+import scorex.utils.ScorexLogging
 
 import scala.concurrent.duration._
 
 
-object TestApp extends App {
+object TestApp extends App with ScorexLogging {
 
   val MinersCount = 10
 
-  val log = LoggerFactory.getLogger(this.getClass)
-
-  val treeDirName = "/tmp/scorex/testApp/"
+  implicit val settings = new Settings with PermaSettings {
+    val filename = "settings.json"
+  }
 
-  val tree = if (Files.exists(Paths.get(treeDirName + "/tree0.mapDB"))) {
+  val tree = if (Files.exists(Paths.get(settings.treeDir + "/tree0.mapDB"))) {
     log.info("Get existing tree")
-    new MerkleTree(treeDirName, Parameters.n, Parameters.segmentSize, Parameters.hash)
+    new MerkleTree(settings.treeDir, Constants.n, Constants.segmentSize, Constants.hash)
   } else {
     log.info("Generating random data set")
-    val treeDir = new File(treeDirName)
+    val treeDir = new File(settings.treeDir)
     treeDir.mkdirs()
-    val datasetFile = treeDirName + "/data.file"
-    new RandomAccessFile(datasetFile, "rw").setLength(Parameters.n * Parameters.segmentSize)
+    val datasetFile = settings.treeDir + "/data.file"
+    new RandomAccessFile(datasetFile, "rw").setLength(Constants.n * Constants.segmentSize)
     log.info("Calculate tree")
-    val tree = MerkleTree.fromFile(datasetFile, treeDirName, Parameters.segmentSize, Parameters.hash)
-    require(tree.nonEmptyBlocks == Parameters.n, s"${tree.nonEmptyBlocks} == ${Parameters.n}")
+    val tree = MerkleTree.fromFile(datasetFile, settings.treeDir, Constants.segmentSize, Constants.hash)
+    require(tree.nonEmptyBlocks == Constants.n, s"${tree.nonEmptyBlocks} == ${Constants.n}")
     tree
   }
 
   log.info("test tree")
-  val index = Parameters.n - 3
+  val index = Constants.n - 3
   val leaf = tree.byIndex(index).get
-  require(leaf.check(index, tree.rootHash)(Parameters.hash))
+  require(leaf.check(index, tree.rootHash)(Constants.hash))
 
   log.info("Success: " + tree.rootHash.mkString)
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 57a1356f..0517673b 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -8,18 +8,20 @@ import scorex.crypto.CryptographicHash._
 import scorex.crypto.SigningFunctions.{PrivateKey, PublicKey, Signature}
 import scorex.crypto._
 import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
-import scorex.perma.Parameters
 import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
+import scorex.perma.settings.{PermaSettings, Constants}
+import scorex.perma.settings.Constants.DataSegment
 
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
 import scala.util.Try
 
-case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[Parameters.DataSegment])
+case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[DataSegment])
 
 case class Ticket(publicKey: PublicKey,
                   s: Array[Byte],
@@ -32,7 +34,7 @@ class Miner(rootHash: Digest) extends Actor with ActorLogging {
   private val keyPair = EllipticCurveImpl.createKeyPair(randomBytes(32))
   private implicit val timeout = Timeout(1 minute)
 
-  private val segmentIds: Seq[Long] = 1.to(Parameters.l).map { i =>
+  private val segmentIds: Seq[Long] = 1.to(Constants.l).map { i =>
     u(keyPair._2, i - 1)
   }.toSeq
 
@@ -76,7 +78,7 @@ class Miner(rootHash: Digest) extends Actor with ActorLogging {
     case SegmentsToStore(sgs) =>
       log.debug("SegmentsToStore({})", sgs)
       sgs.foreach { s =>
-        if (segmentToDownload.contains(s._1) && s._2.check(s._1, rootHash)(Parameters.hash)) {
+        if (segmentToDownload.contains(s._1) && s._2.check(s._1, rootHash)(Constants.hash)) {
           authDataStorage.set(s._1, s._2)
           segmentToDownload -= s._1
         }
@@ -112,7 +114,7 @@ object Miner {
   //calculate index of i-th segment
   private def u(pubKey: PublicKey, i: Int): Long = {
     val h = Sha256.hash(pubKey ++ BigInt(i).toByteArray)
-    BigInt(1, h).mod(Parameters.n).toLong
+    BigInt(1, h).mod(Constants.n).toLong
   }
 
 
@@ -123,8 +125,7 @@ object Miner {
     r
   }
 
-  //TODO take file name from config
-  val authDataStorage = new AuthDataStorage("/tmp/scorex/testApp/segmentsStorage.mapDB")
+  def authDataStorage(implicit settings: PermaSettings) = new AuthDataStorage(settings.authDataStorage)
 
   def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
 
@@ -134,19 +135,19 @@ object Miner {
     val s = randomBytes(32)
 
     val sig0 = NoSig
-    val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Parameters.l).toInt)
+    val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
 
-    val proofs: IndexedSeq[PartialProof] = 1.to(Parameters.k).foldLeft(
+    val proofs: IndexedSeq[PartialProof] = 1.to(Constants.k).foldLeft(
       (r1, sig0, Seq[PartialProof]())
     ) {
       case ((ri, sig_prev, seq), _) =>
         val segment = authDataStorage.get(ri).get
         val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
         val sig = EllipticCurveImpl.sign(privateKey, hi)
-        val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Parameters.l).toInt)
+        val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt)
 
         (r_next, sig, seq :+ PartialProof(sig, ri, segment))
-    }._3.toIndexedSeq.ensuring(_.size == Parameters.k)
+    }._3.toIndexedSeq.ensuring(_.size == Constants.k)
 
     Ticket(publicKey, s, proofs)
   }
@@ -158,14 +159,14 @@ object Miner {
                t: Ticket,
                rootHash: CryptographicHash.Digest): Boolean = Try {
     val proofs = t.proofs
-    require(proofs.size == Parameters.k)
+    require(proofs.size == Constants.k)
 
     //Local-POR lottery verification
 
     val sigs = NoSig +: proofs.map(_.signature)
     val ris = proofs.map(_.segmentIndex)
 
-    val partialProofsCheck = 1.to(Parameters.k).foldLeft(true) { case (partialResult, i) =>
+    val partialProofsCheck = 1.to(Constants.k).foldLeft(true) { case (partialResult, i) =>
       val segment = proofs(i - 1).segment
 
       segment.check(ris(i - 1), rootHash)() || {
@@ -184,8 +185,7 @@ object Miner {
 
 object MinerSpec {
 
-  type Index = Long
-  type Subset = Map[Index, AuthDataBlock[Parameters.DataSegment]]
+  type Subset = Map[Position, AuthDataBlock[DataSegment]]
 
   case class Initialize(miners: Seq[ActorRef])
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
index b59574c7..d1d9c84e 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/TrustedDealer.scala
@@ -3,9 +3,9 @@ package scorex.perma.actors
 import akka.actor.{Actor, ActorLogging}
 import scorex.crypto.CryptographicHash
 import scorex.crypto.ads.merkle.MerkleTree
-import scorex.perma.Parameters
 import scorex.perma.actors.MinerSpec.Subset
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
+import scorex.perma.settings.Constants
 
 
 class TrustedDealer[H <: CryptographicHash](val tree: MerkleTree[H]) extends Actor with ActorLogging {
diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
new file mode 100644
index 00000000..0f66db10
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
@@ -0,0 +1,30 @@
+package scorex.perma.settings
+
+import com.typesafe.config.ConfigFactory
+import scorex.crypto.Sha256
+import scorex.utils.ScorexLogging
+
+object Constants extends ScorexLogging {
+
+  private val appConf = ConfigFactory.load().getConfig("perma")
+
+  type DataSegment = Array[Byte]
+
+  //few segments to be stored in a block, so segment size shouldn't be big
+  val segmentSize = appConf.getInt("segmentSize") //segment size in bytes
+
+  val n = appConf.getLong("n") //how many segments in a dataset in total
+
+  val l = appConf.getInt("l") //how many segments to store for an each miner
+
+  val k = appConf.getInt("k") //number of iterations during scratch-off phase
+
+  val hash = appConf.getString("hash") match {
+    case s: String if s.equalsIgnoreCase("sha256") =>
+      Sha256
+    case hashFunction =>
+      log.error(s"Unknown hash function: $hashFunction. Use Sha256 instead.")
+      Sha256
+  }
+
+}
diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala b/scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala
new file mode 100644
index 00000000..709d622c
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/settings/PermaSettings.scala
@@ -0,0 +1,15 @@
+package scorex.perma.settings
+
+import play.api.libs.json.JsObject
+
+trait PermaSettings {
+  val settingsJSON: JsObject
+
+  private val DefaultTreeDir = "/tmp/scorex/perma/"
+  private val DefaultAuthDataStorage = DefaultTreeDir + "authDataStorage.mapDB"
+
+  lazy val treeDir = (settingsJSON \ "perma" \ "treeDir").asOpt[String].getOrElse(DefaultTreeDir)
+  lazy val authDataStorage =
+    (settingsJSON \ "perma" \ "authDataStorage").asOpt[String].getOrElse(DefaultAuthDataStorage)
+
+}

From bd9eb682aced7c89705af5adcec974bd76290412 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 24 Nov 2015 12:39:12 +0300
Subject: [PATCH 27/66] Refactoring

---
 .../src/main/scala/scorex/crypto/CryptographicHash.scala  | 2 ++
 .../scala/scorex/crypto/ads/merkle/AuthDataBlock.scala    | 8 ++++----
 .../src/test/scala/scorex/unit/Sha256Specification.scala  | 6 +++---
 3 files changed, 9 insertions(+), 7 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala b/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala
index f2bee338..fbd456a6 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala
@@ -25,6 +25,8 @@ trait CryptographicHash {
 
   def hash(input: Message): Digest
 
+  def hash(input: String): Digest = hash(input.getBytes)
+
   def doubleHash(input: Message): Digest = hash(hash(input))
 }
 
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index 7c8a04e4..047851a3 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -28,9 +28,9 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
         calculateHash(i / 2, hash, path.tail)
     }
 
-    if (merklePath.nonEmpty) {
-      val calculated = calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath)
-      calculated.mkString == rootHash.mkString
-    } else true
+    if (merklePath.nonEmpty)
+      calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath) sameElements rootHash
+    else
+      true
   }
 }
diff --git a/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala b/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala
index 67857f7f..65ddad26 100644
--- a/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala
+++ b/scorex-basics/src/test/scala/scorex/unit/Sha256Specification.scala
@@ -18,9 +18,9 @@ class Sha256Specification  extends FunSuite with Matchers {
     assert(bytes2hex(hash(testBytes)) == "b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9")
 
     //test samples from a Qeditas unit test
-    assert(bytes2hex(hash("".getBytes)) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
-    assert(bytes2hex(hash("abc".getBytes)) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
-    assert(bytes2hex(hash("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq".getBytes)) ==
+    assert(bytes2hex(hash("")) == "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")
+    assert(bytes2hex(hash("abc")) == "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad")
+    assert(bytes2hex(hash("abcdbcdecdefdefgefghfghighijhijkijkljklmklmnlmnomnopnopq")) ==
       "248d6a61d20638b8e5c026930c3e6039a33ce45964ff2167f6ecedd419db06c1")
   }
 }

From 849c11012d997dda7b6f0430e11f39e642297731 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 24 Nov 2015 12:57:51 +0300
Subject: [PATCH 28/66] fast Merkle tree tests

---
 .../crypto/ads/merkle/AuthDataBlock.scala     |  2 +-
 .../scorex/crypto/ads/merkle/MerkleTree.scala | 19 ++++++++++------
 .../scorex/props/MerkleSpecification.scala    | 22 ++++++++++---------
 3 files changed, 25 insertions(+), 18 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index 047851a3..80b1a545 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -31,6 +31,6 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
     if (merklePath.nonEmpty)
       calculateHash(index, hashFunction.hash(data.asInstanceOf[Message]), merklePath) sameElements rootHash
     else
-      true
+      false
   }
 }
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
index 4269743a..0ddb61ca 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/MerkleTree.scala
@@ -6,6 +6,7 @@ import java.nio.file.{Files, Paths}
 import scorex.crypto.CryptographicHash.Digest
 import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
+import scorex.utils.ScorexLogging
 
 import scala.annotation.tailrec
 
@@ -13,7 +14,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
                                          val nonEmptyBlocks: Position,
                                          blockSize: Int = 1024,
                                          hash: H = Sha256
-                                        ) {
+                                        ) extends ScorexLogging {
 
   import MerkleTree._
 
@@ -30,11 +31,15 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
       @tailrec
       def calculateTreePath(n: Position, currentLevel: Int, acc: Seq[Digest] = Seq()): Seq[Digest] = {
         if (currentLevel < level) {
-          //TODO remove get? it should exists when (index < nonEmptyBlocks && index > 0)
-          if (n % 2 == 0) {
-            calculateTreePath(n / 2, currentLevel + 1, getHash((currentLevel, n + 1)).get +: acc)
-          } else {
-            calculateTreePath(n / 2, currentLevel + 1, getHash((currentLevel, n - 1)).get +: acc)
+          val hashOpt = if (n % 2 == 0) getHash((currentLevel, n + 1)) else getHash((currentLevel, n - 1))
+          hashOpt match {
+            case Some(h) =>
+              calculateTreePath(n / 2, currentLevel + 1, h +: acc)
+            case None if currentLevel == 0 && index == nonEmptyBlocks - 1 =>
+              calculateTreePath(n / 2, currentLevel + 1, emptyHash +: acc)
+            case None =>
+              log.error(s"Enable to get hash for lev=$currentLevel, position=$n")
+              acc.reverse
           }
         } else {
           acc.reverse
@@ -50,7 +55,7 @@ class MerkleTree[H <: CryptographicHash](treeFolder: String,
     }
   }
 
-  private lazy val emptyHash = hash.hash("".getBytes)
+  private lazy val emptyHash = hash.hash("")
 
   def getHash(key: TreeStorage.Key): Option[Digest] = {
     storage.get(key) match {
diff --git a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
index f1ec8fd8..a90d1a2a 100644
--- a/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
+++ b/scorex-basics/src/test/scala/scorex/props/MerkleSpecification.scala
@@ -12,25 +12,27 @@ import scala.util.Random
 
 class MerkleSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
-  val smallInteger = Gen.choose(8, 128)
-
 
   property("value returned from byIndex() is valid for random dataset") {
-    forAll(smallInteger) { (blocks: Int) =>
+    //fix block numbers for faster tests
+    for (blocks <- List(7, 8, 9, 128)) {
+      val smallInteger = Gen.choose(0, blocks - 1)
       val (treeDirName: String, _, tempFile: String) = generateFile(blocks)
-
       val tree = MerkleTree.fromFile(tempFile, treeDirName)
-      val index = Random.nextInt(tree.nonEmptyBlocks.asInstanceOf[Int])
-
-      val leaf = tree.byIndex(index).get
-      val resp = leaf.check(index, tree.rootHash)(Sha256)
+      forAll(smallInteger) { (index: Int) =>
+        val leafOption = tree.byIndex(index)
+        leafOption should not be None
+        val leaf = leafOption.get
+        val resp = leaf.check(index, tree.rootHash)(Sha256)
+        resp shouldBe true
+      }
       tree.storage.close()
-      resp shouldBe true
     }
   }
 
   property("hash root is the same") {
-    forAll(smallInteger) { (blocks: Int) =>
+    //fix block numbers for faster tests
+    for (blocks <- List(7, 8, 9, 128)) {
       val (treeDirName: String, _, tempFile: String) = generateFile(blocks, "2")
 
       val fileTree = MerkleTree.fromFile(tempFile, treeDirName)

From 00f077da345a35efe43a00ca11da562a6e3418e6 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 24 Nov 2015 13:31:47 +0300
Subject: [PATCH 29/66] implicit settings

---
 scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 0517673b..afcce3e4 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -27,7 +27,7 @@ case class Ticket(publicKey: PublicKey,
                   s: Array[Byte],
                   proofs: IndexedSeq[PartialProof])
 
-class Miner(rootHash: Digest) extends Actor with ActorLogging {
+class Miner(rootHash: Digest)(implicit settings: PermaSettings) extends Actor with ActorLogging {
 
   import Miner._
 
@@ -127,7 +127,7 @@ object Miner {
 
   def authDataStorage(implicit settings: PermaSettings) = new AuthDataStorage(settings.authDataStorage)
 
-  def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
+  def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte])(implicit settings: PermaSettings): Ticket = {
 
     val (privateKey, publicKey) = keyPair
 

From 83d1da378a42b7290a934704c99a49958b762dce Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 25 Nov 2015 20:12:17 +0300
Subject: [PATCH 30/66] rename config file

---
 .../main/resources/{application.conf => perma.conf}  |  0
 .../main/scala/scorex/perma/settings/Constants.scala | 12 ++++++------
 2 files changed, 6 insertions(+), 6 deletions(-)
 rename scorex-perma/src/main/resources/{application.conf => perma.conf} (100%)

diff --git a/scorex-perma/src/main/resources/application.conf b/scorex-perma/src/main/resources/perma.conf
similarity index 100%
rename from scorex-perma/src/main/resources/application.conf
rename to scorex-perma/src/main/resources/perma.conf
diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
index 0f66db10..6b27d2af 100644
--- a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
@@ -6,20 +6,20 @@ import scorex.utils.ScorexLogging
 
 object Constants extends ScorexLogging {
 
-  private val appConf = ConfigFactory.load().getConfig("perma")
+  private val permaConf = ConfigFactory.load("perma").getConfig("perma")
 
   type DataSegment = Array[Byte]
 
   //few segments to be stored in a block, so segment size shouldn't be big
-  val segmentSize = appConf.getInt("segmentSize") //segment size in bytes
+  val segmentSize = permaConf.getInt("segmentSize") //segment size in bytes
 
-  val n = appConf.getLong("n") //how many segments in a dataset in total
+  val n = permaConf.getLong("n") //how many segments in a dataset in total
 
-  val l = appConf.getInt("l") //how many segments to store for an each miner
+  val l = permaConf.getInt("l") //how many segments to store for an each miner
 
-  val k = appConf.getInt("k") //number of iterations during scratch-off phase
+  val k = permaConf.getInt("k") //number of iterations during scratch-off phase
 
-  val hash = appConf.getString("hash") match {
+  val hash = permaConf.getString("hash") match {
     case s: String if s.equalsIgnoreCase("sha256") =>
       Sha256
     case hashFunction =>

From 54e389fe4a5bf4d1dc45fe7e571ce0385438bef6 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 25 Nov 2015 20:47:28 +0300
Subject: [PATCH 31/66] pass data storage to miner

---
 .../src/main/scala/scorex/perma/TestApp.scala         |  4 +++-
 .../src/main/scala/scorex/perma/actors/Miner.scala    | 11 ++++++-----
 2 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
index d6f19f77..1735ff3f 100644
--- a/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/TestApp.scala
@@ -7,6 +7,7 @@ import akka.actor.{ActorSystem, Props}
 import akka.util.Timeout
 import scorex.crypto.ads.merkle.MerkleTree
 import scorex.perma.BlockchainBuilderSpec.SendWorkToMiners
+import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.actors.MinerSpec.Initialize
 import scorex.perma.actors.{Miner, TrustedDealer}
 import scorex.perma.settings.{Constants, PermaSettings}
@@ -49,7 +50,8 @@ object TestApp extends App with ScorexLogging {
   log.info("start actor system")
   protected lazy val actorSystem = ActorSystem("lagonaki")
   val dealer = actorSystem.actorOf(Props(new TrustedDealer(tree)))
-  val miners = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], tree.rootHash), s"m-$x"))
+  val storage = new AuthDataStorage(settings.authDataStorage)
+  val miners = (1 to MinersCount).map(x => actorSystem.actorOf(Props(classOf[Miner], tree.rootHash, storage), s"m-$x"))
 
   implicit val timeout = Timeout(1 minute)
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index afcce3e4..700515e1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -13,8 +13,9 @@ import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
-import scorex.perma.settings.{PermaSettings, Constants}
+import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants.DataSegment
+import scorex.storage.Storage
 
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -27,7 +28,8 @@ case class Ticket(publicKey: PublicKey,
                   s: Array[Byte],
                   proofs: IndexedSeq[PartialProof])
 
-class Miner(rootHash: Digest)(implicit settings: PermaSettings) extends Actor with ActorLogging {
+class Miner(rootHash: Digest)(implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
+  extends Actor with ActorLogging {
 
   import Miner._
 
@@ -125,9 +127,8 @@ object Miner {
     r
   }
 
-  def authDataStorage(implicit settings: PermaSettings) = new AuthDataStorage(settings.authDataStorage)
-
-  def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte])(implicit settings: PermaSettings): Ticket = {
+  def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte])
+              (implicit authDataStorage: Storage[Long, AuthDataBlock[DataSegment]]): Ticket = {
 
     val (privateKey, publicKey) = keyPair
 

From 70b5ea78e265fe3610ad56b75736fb1ddef4f7ed Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Wed, 25 Nov 2015 20:53:15 +0300
Subject: [PATCH 32/66] Refactor

---
 .../src/main/scala/scorex/utils/utils.scala    |  9 +++++++++
 .../main/scala/scorex/perma/actors/Miner.scala | 18 +++---------------
 2 files changed, 12 insertions(+), 15 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/utils/utils.scala b/scorex-basics/src/main/scala/scorex/utils/utils.scala
index 8167718a..6364dd72 100644
--- a/scorex-basics/src/main/scala/scorex/utils/utils.scala
+++ b/scorex-basics/src/main/scala/scorex/utils/utils.scala
@@ -1,5 +1,7 @@
 package scorex
 
+import java.security.SecureRandom
+
 import scala.annotation.tailrec
 
 import scala.concurrent.duration._
@@ -19,4 +21,11 @@ package object utils {
       case util.Failure(e) => throw e
     }
   }
+
+  def randomBytes(howMany: Int) = {
+    val r = new Array[Byte](howMany)
+    new SecureRandom().nextBytes(r) //overrides r
+    r
+  }
+
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index 700515e1..e3aa4188 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -1,7 +1,5 @@
 package scorex.perma.actors
 
-import java.security.SecureRandom
-
 import akka.actor.{Actor, ActorLogging, ActorRef}
 import akka.util.Timeout
 import scorex.crypto.CryptographicHash._
@@ -10,12 +8,12 @@ import scorex.crypto._
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
-import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
 import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants.DataSegment
 import scorex.storage.Storage
+import scorex.utils._
 
 import scala.collection.mutable.ListBuffer
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -34,7 +32,7 @@ class Miner(rootHash: Digest)(implicit val authDataStorage: Storage[Long, AuthDa
   import Miner._
 
   private val keyPair = EllipticCurveImpl.createKeyPair(randomBytes(32))
-  private implicit val timeout = Timeout(1 minute)
+  private implicit val timeout = Timeout(1.minute)
 
   private val segmentIds: Seq[Long] = 1.to(Constants.l).map { i =>
     u(keyPair._2, i - 1)
@@ -48,8 +46,6 @@ class Miner(rootHash: Digest)(implicit val authDataStorage: Storage[Long, AuthDa
     case Initialize(miners) =>
       log.debug("Initialize")
 
-      val s = sender()
-
       segmentToDownload foreach { s =>
         if (authDataStorage.containsKey(s)) {
           segmentToDownload -= s
@@ -98,7 +94,7 @@ class Miner(rootHash: Digest)(implicit val authDataStorage: Storage[Long, AuthDa
       if (check) {
         sender() ! WinningTicket(puz, score, ticket)
       } else {
-        context.system.scheduler.scheduleOnce(200 millis, self, TicketGeneration(difficulty, puz))
+        context.system.scheduler.scheduleOnce(200.millis, self, TicketGeneration(difficulty, puz))
       }
 
     case TicketValidation(difficulty, puz, t: Ticket) =>
@@ -119,14 +115,6 @@ object Miner {
     BigInt(1, h).mod(Constants.n).toLong
   }
 
-
-  //todo: move to utils
-  def randomBytes(howMany: Int) = {
-    val r = new Array[Byte](howMany)
-    new SecureRandom().nextBytes(r) //overrides s
-    r
-  }
-
   def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte])
               (implicit authDataStorage: Storage[Long, AuthDataBlock[DataSegment]]): Ticket = {
 

From 45b61fbdd5f674b6bf4dd551b2f61ac2102c8e95 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 01:52:21 +0300
Subject: [PATCH 33/66] Calc puz from last block

---
 .../scala/scorex/perma/BlockchainBuilder.scala | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
index 77c619c1..ac09f790 100644
--- a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -1,9 +1,10 @@
 package scorex.perma
 
 import akka.actor.{Actor, ActorRef}
+import scorex.crypto.{Sha256, CryptographicHash}
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.Ticket
-import scorex.utils.ScorexLogging
+import scorex.utils._
 
 import scala.collection.mutable
 import scala.concurrent.ExecutionContext.Implicits.global
@@ -17,16 +18,21 @@ class BlockchainBuilder(miners: Seq[ActorRef], dealer: ActorRef) extends Actor w
 
   import BlockchainBuilderSpec._
 
-  var puz: Array[Byte] = calcPuz
   var initialized = 0
 
   val InitialDifficulty = BigInt(1, Array.fill(33)(1: Byte))
   val blockchainLike = mutable.Buffer[BlockHeaderLike]()
 
-  private def calcPuz = 1.to(32).toArray.map(_ => Random.nextInt(256).toByte)
-
   def difficulty = blockchainLike.headOption.map(_.difficulty).getOrElse(InitialDifficulty)
 
+  def calcPuz(block: Option[BlockHeaderLike], hash: CryptographicHash = Sha256): Array[Byte] = block match {
+    case Some(b) =>
+      hash.hash(b.puz ++ b.ticket.s ++ b.ticket.publicKey
+        ++ b.ticket.proofs.foldLeft(Array.empty: Array[Byte])((b, a) => b ++ a.signature))
+    case None =>
+      hash.hash("Scorex perma genesis")
+  }
+
   override def receive = {
     case s: MinerStatus =>
       s match {
@@ -44,7 +50,7 @@ class BlockchainBuilder(miners: Seq[ActorRef], dealer: ActorRef) extends Actor w
     case SendWorkToMiners =>
       miners.foreach { minerRef =>
         if (initialized == miners.length) {
-          minerRef ! TicketGeneration(difficulty, puz)
+          minerRef ! TicketGeneration(difficulty, calcPuz(blockchainLike.lastOption))
         } else {
           minerRef ! Initialize(miners)
           context.system.scheduler.scheduleOnce(200 millis, minerRef, GetStatus)
@@ -53,11 +59,11 @@ class BlockchainBuilder(miners: Seq[ActorRef], dealer: ActorRef) extends Actor w
 
     //miners are honest (lol) in our setting, so no validation here
     case WinningTicket(minerPuz, score, ticket) =>
+      val puz = calcPuz(blockchainLike.lastOption)
       if (minerPuz sameElements puz) {
         val newBlock = BlockHeaderLike(score, puz, ticket)
         log.info(s"Block generated: $newBlock, blockchain size: ${blockchainLike.size}")
         blockchainLike += newBlock
-        puz = calcPuz
         self ! SendWorkToMiners
       } else {
         sender() ! TicketGeneration(difficulty, puz)

From 5defa8169125510ed1a72a7bb48aadf4569ec269 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 13:53:43 +0300
Subject: [PATCH 34/66] Perma consensus

---
 .../consensus/PermaConsensusBlockField.scala  |  5 +--
 .../http/PermaConsensusApiRoute.scala         | 33 +++++++++++++++++++
 src/main/resources/application.conf           |  2 +-
 .../lagonaki/server/LagonakiApplication.scala |  9 +++++
 4 files changed, 46 insertions(+), 3 deletions(-)
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 604c7cc5..57f0ac00 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -11,9 +11,10 @@ case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockD
 
   override val name: String = "perma-consensus"
 
+  //TODO hash from bytes?
   override def bytes: Array[Byte] = {
-    //todo: implement
-    ???
+    value.difficulty.toByteArray ++ value.puz ++ value.ticket.s ++ value.ticket.publicKey ++
+      value.ticket.proofs.foldLeft(Array.empty: Array[Byte])((b, a) => b ++ a.signature)
   }
 
   override def json: JsObject = Json.obj(name -> Json.obj(
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
new file mode 100644
index 00000000..e0b60a90
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
@@ -0,0 +1,33 @@
+package scorex.perma.consensus.http
+
+import javax.ws.rs.Path
+
+import akka.actor.ActorRefFactory
+import com.wordnik.swagger.annotations._
+import play.api.libs.json.Json
+import scorex.api.http.{ApiRoute, CommonApiFunctions}
+import scorex.perma.consensus.PermaConsensusModule
+import scorex.transaction.BlockChain
+import spray.routing.Route
+
+
+@Api(value = "/consensus", description = "Consensus-related calls")
+class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain: BlockChain)
+                            (implicit val context: ActorRefFactory)
+  extends ApiRoute with CommonApiFunctions {
+
+  override val route: Route =
+    pathPrefix("consensus") {
+      algo
+    }
+
+  @Path("/algo")
+  @ApiOperation(value = "Consensus algo", notes = "Shows which consensus algo being using", httpMethod = "GET")
+  def algo = {
+    path("algo") {
+      jsonRoute {
+        Json.obj("consensus-algo" -> "perma").toString()
+      }
+    }
+  }
+}
diff --git a/src/main/resources/application.conf b/src/main/resources/application.conf
index b5626a0e..2ca36001 100644
--- a/src/main/resources/application.conf
+++ b/src/main/resources/application.conf
@@ -2,5 +2,5 @@ app {
   product = "Scorex"
   release = "Lagonaki"
   version = "1.1.3-SNAPSHOT"
-  consensusAlgo = "Nxt"
+  consensusAlgo = "Perma"
 }
diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
index 9a4b88d2..3e4b660d 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
@@ -15,6 +15,8 @@ import scorex.consensus.nxt.NxtLikeConsensusModule
 import scorex.consensus.qora.QoraLikeConsensusModule
 import scorex.lagonaki.network.message._
 import scorex.lagonaki.network.{BlockchainSyncer, NetworkController}
+import scorex.perma.consensus.PermaConsensusModule
+import scorex.perma.consensus.http.PermaConsensusApiRoute
 import scorex.transaction.LagonakiTransaction.ValidationResult
 import scorex.transaction._
 import scorex.transaction.state.database.UnconfirmedTransactionsDatabaseImpl
@@ -40,6 +42,8 @@ class LagonakiApplication(val settingsFilename: String)
         new NxtLikeConsensusModule
       case s: String if s.equalsIgnoreCase("qora") =>
         new QoraLikeConsensusModule
+      case s: String if s.equalsIgnoreCase("perma") =>
+        new PermaConsensusModule
       case algo =>
         log.error(s"Unknown consensus algo: $algo. Use NxtLikeConsensusModule instead.")
         new NxtLikeConsensusModule
@@ -61,6 +65,10 @@ class LagonakiApplication(val settingsFilename: String)
       new NxtConsensusApiRoute(ncm, blockchainImpl)
     case qcm: QoraLikeConsensusModule =>
       new QoraConsensusApiRoute(qcm, blockchainImpl)
+    case pcm: PermaConsensusModule =>
+      new PermaConsensusApiRoute(pcm, blockchainImpl)
+
+
   }
 
   override lazy val apiRoutes = Seq(
@@ -81,6 +89,7 @@ class LagonakiApplication(val settingsFilename: String)
     consensusApiRoute match {
       case nxt: NxtConsensusApiRoute => typeOf[NxtConsensusApiRoute]
       case qora: QoraConsensusApiRoute => typeOf[QoraConsensusApiRoute]
+      case pcm: PermaConsensusApiRoute => typeOf[PermaConsensusApiRoute]
     },
     typeOf[WalletApiRoute],
     typeOf[PaymentApiRoute],

From 6d84ee08d2f882175f227abce900b93c907d9fc5 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 17:27:30 +0300
Subject: [PATCH 35/66] Serialization of data blocks

---
 .../crypto/ads/merkle/AuthDataBlock.scala     | 28 ++++++++++++++++
 .../scorex/perma/BlockchainBuilder.scala      |  2 +-
 .../scala/scorex/perma/actors/Miner.scala     |  7 +---
 .../scorex/perma/consensus/PartialProof.scala | 24 ++++++++++++++
 .../consensus/PermaConsensusBlockField.scala  | 33 +++++++------------
 .../consensus/PermaConsensusModule.scala      | 20 ++++++-----
 .../PermaLikeConsensusBlockData.scala         | 24 ++++++++++----
 .../scala/scorex/perma/consensus/Ticket.scala | 25 ++++++++++++++
 8 files changed, 121 insertions(+), 42 deletions(-)
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
 create mode 100644 scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index 80b1a545..e1b23b00 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -1,5 +1,6 @@
 package scorex.crypto.ads.merkle
 
+import play.api.libs.json._
 import scorex.crypto.CryptographicHash._
 import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.crypto.{CryptographicHash, Sha256}
@@ -34,3 +35,30 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
       false
   }
 }
+
+object AuthDataBlock {
+
+  implicit def authDataBlockReads[T](implicit fmt: Reads[T]): Reads[AuthDataBlock[T]] = new Reads[AuthDataBlock[T]] {
+    def reads(json: JsValue): JsResult[AuthDataBlock[T]] = JsSuccess(AuthDataBlock[T](
+      (json \ "data").get match {
+        case JsString(ts) => ts.getBytes.asInstanceOf[T]
+        case _ => throw new RuntimeException("Data MUST be a string")
+      },
+      (json \ "merklePath").get match {
+        case JsArray(ts) => ts.map(t => t.as[String].getBytes)
+        case _ => throw new RuntimeException("MerklePath MUST be a list")
+      }
+    ))
+  }
+
+  implicit def authDataBlockWrites[T](implicit fmt: Writes[T]): Writes[AuthDataBlock[T]] = new Writes[AuthDataBlock[T]] {
+    def writes(ts: AuthDataBlock[T]) = JsObject(Seq(
+      "data" -> JsString(new String(ts.data.asInstanceOf[Array[Byte]])),
+      "merklePath" -> JsArray(
+        ts.merklePath.map(digest => JsString(new String(digest)))
+      )
+    ))
+  }
+
+}
+
diff --git a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
index ac09f790..2fc339b1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/BlockchainBuilder.scala
@@ -3,7 +3,7 @@ package scorex.perma
 import akka.actor.{Actor, ActorRef}
 import scorex.crypto.{Sha256, CryptographicHash}
 import scorex.perma.actors.MinerSpec._
-import scorex.perma.actors.Ticket
+import scorex.perma.consensus.Ticket
 import scorex.utils._
 
 import scala.collection.mutable
diff --git a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
index e3aa4188..40ca4a0f 100644
--- a/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/actors/Miner.scala
@@ -10,6 +10,7 @@ import scorex.crypto.ads.merkle.TreeStorage.Position
 import scorex.perma.BlockchainBuilderSpec.WinningTicket
 import scorex.perma.actors.MinerSpec._
 import scorex.perma.actors.TrustedDealerSpec.{SegmentsRequest, SegmentsToStore}
+import scorex.perma.consensus.{PartialProof, Ticket}
 import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants.DataSegment
 import scorex.storage.Storage
@@ -20,12 +21,6 @@ import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
 import scala.util.Try
 
-case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[DataSegment])
-
-case class Ticket(publicKey: PublicKey,
-                  s: Array[Byte],
-                  proofs: IndexedSeq[PartialProof])
-
 class Miner(rootHash: Digest)(implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends Actor with ActorLogging {
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
new file mode 100644
index 00000000..bbd15055
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
@@ -0,0 +1,24 @@
+package scorex.perma.consensus
+
+import play.api.libs.functional.syntax._
+import play.api.libs.json.{JsPath, Reads, Writes}
+import scorex.crypto.SigningFunctions._
+import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.perma.settings.Constants._
+
+case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[DataSegment])
+
+object PartialProof {
+  implicit val writes: Writes[PartialProof] = (
+    (JsPath \ "signature").write[Signature] and
+      (JsPath \ "segmentIndex").write[Long] and
+      (JsPath \ "segment").write[AuthDataBlock[DataSegment]]
+    ) (unlift(PartialProof.unapply))
+
+  implicit val reads: Reads[PartialProof] = (
+    (JsPath \ "signature").read[Signature] and
+      (JsPath \ "segmentIndex").read[Long] and
+      (JsPath \ "segment").read[AuthDataBlock[DataSegment]]
+    ) (PartialProof.apply _)
+
+}
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 57f0ac00..ee6f2f93 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -1,31 +1,22 @@
 package scorex.perma.consensus
 
-import play.api.libs.json.{JsObject, Json}
+import play.api.libs.json._
 import scorex.block.BlockField
-import scorex.crypto.Base58
 
 
 case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockData)
   extends BlockField[PermaLikeConsensusBlockData] {
 
 
-  override val name: String = "perma-consensus"
-
-  //TODO hash from bytes?
-  override def bytes: Array[Byte] = {
-    value.difficulty.toByteArray ++ value.puz ++ value.ticket.s ++ value.ticket.publicKey ++
-      value.ticket.proofs.foldLeft(Array.empty: Array[Byte])((b, a) => b ++ a.signature)
-  }
-
-  override def json: JsObject = Json.obj(name -> Json.obj(
-    "difficulty" -> value.difficulty.toString(),
-    "puz" -> value.puz,
-    "s" -> value.ticket.s,
-    "segments" -> value.ticket.proofs.map { proof =>
-      Json.obj(
-        "data" -> proof.segment.data,
-        "path" -> Json.arr(proof.segment.merklePath.map(Base58.encode))
-      )
-    }
-  ))
+  override val name: String = PermaConsensusBlockField.fieldName
+
+  override def bytes: Array[Byte] = json.toString().getBytes
+
+  override def json: JsObject = Json.obj(name -> Json.toJson(value))
+}
+
+object PermaConsensusBlockField {
+
+  val fieldName: String = "perma-consensus"
+
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 423d3c80..e4dff0e1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -3,8 +3,8 @@ package scorex.perma.consensus
 import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount}
 import scorex.block.{Block, BlockField}
 import scorex.consensus.ConsensusModule
-import scorex.perma.actors.Ticket
 import scorex.transaction.TransactionModule
+import play.api.libs.json.{JsObject, Json}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -16,7 +16,7 @@ import scala.concurrent.Future
 class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData] {
 
   val MiningReward = 1000000
-  val InitialDifficulty = BigInt(Array.fill(32)(1: Byte))
+  val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)).toLong
   val GenesisCreator = new PublicKeyAccount(Array())
 
   private def blockGenerator(block: Block) = block.signerDataField.value.generator
@@ -52,7 +52,11 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
   override def consensusBlockData(block: Block): PermaLikeConsensusBlockData =
     block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData]
 
-  override def parseBlockData(bytes: Array[Byte]): BlockField[PermaLikeConsensusBlockData] = ???
+  override def parseBlockData(bytes: Array[Byte]): BlockField[PermaLikeConsensusBlockData] = {
+    val json = Json.parse(bytes)
+    ???
+  }
+
 
   /*
     PermaConsensusBlockField(new PermaLikeConsensusBlockData{
@@ -60,11 +64,11 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
     })*/
 
   override def genesisData: BlockField[PermaLikeConsensusBlockData] =
-    PermaConsensusBlockField(new PermaLikeConsensusBlockData {
-      override val difficulty = InitialDifficulty
-      override val puz = Array[Byte]()
-      override val ticket = Ticket(GenesisCreator.publicKey, Array(), IndexedSeq())
-    })
+    PermaConsensusBlockField(PermaLikeConsensusBlockData(
+      InitialDifficulty,
+      Array[Byte](),
+      Ticket(GenesisCreator.publicKey, Array(), IndexedSeq())
+    ))
 
   /*
     PermaConsensusBlockField(new PermaLikeConsensusBlockData {
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
index 1087ed50..158a0fef 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
@@ -1,9 +1,21 @@
 package scorex.perma.consensus
 
-import scorex.perma.actors.Ticket
+import play.api.libs.functional.syntax._
+import play.api.libs.json.{JsPath, Reads, Writes}
 
-trait PermaLikeConsensusBlockData {
-  val difficulty: BigInt
-  val puz: Array[Byte]
-  val ticket: Ticket
-}
+case class PermaLikeConsensusBlockData(difficulty: Long, puz: Array[Byte], ticket: Ticket)
+
+object PermaLikeConsensusBlockData {
+  implicit val writes: Writes[PermaLikeConsensusBlockData] = (
+    (JsPath \ "difficulty").write[Long] and
+      (JsPath \ "puz").write[Array[Byte]] and
+      (JsPath \ "ticket").write[Ticket]
+    ) (unlift(PermaLikeConsensusBlockData.unapply))
+
+  implicit val reads: Reads[PermaLikeConsensusBlockData] = (
+    (JsPath \ "difficulty").read[Long] and
+      (JsPath \ "puz").read[Array[Byte]] and
+      (JsPath \ "ticket").read[Ticket]
+    ) (PermaLikeConsensusBlockData.apply _)
+
+}
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala
new file mode 100644
index 00000000..d7031b7e
--- /dev/null
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala
@@ -0,0 +1,25 @@
+package scorex.perma.consensus
+
+import play.api.libs.functional.syntax._
+import play.api.libs.json.{JsPath, Reads, Writes}
+import scorex.crypto.SigningFunctions._
+
+case class Ticket(publicKey: PublicKey,
+                  s: Array[Byte],
+                  proofs: IndexedSeq[PartialProof])
+
+
+object Ticket {
+  implicit val writes: Writes[Ticket] = (
+    (JsPath \ "publicKey").write[PublicKey] and
+      (JsPath \ "s").write[Array[Byte]] and
+      (JsPath \ "proofs").write[IndexedSeq[PartialProof]]
+    ) (unlift(Ticket.unapply))
+
+  implicit val reads: Reads[Ticket] = (
+    (JsPath \ "publicKey").read[PublicKey] and
+      (JsPath \ "s").read[Array[Byte]] and
+      (JsPath \ "proofs").read[IndexedSeq[PartialProof]]
+    ) (Ticket.apply _)
+
+}

From e85d657b214f3a70fd36f3faacba30a25d23c7de Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 17:47:23 +0300
Subject: [PATCH 36/66] Use serialization

---
 .../crypto/ads/merkle/AuthDataBlock.scala     | 24 ++++++++----
 .../consensus/PermaConsensusBlockField.scala  |  9 +++++
 .../consensus/PermaConsensusModule.scala      | 15 ++------
 .../test/scala/scorex/PermaTestSuite.scala    |  3 +-
 ...ermaConsensusBlockFiendSpecification.scala | 37 +++++++++++++++++++
 5 files changed, 69 insertions(+), 19 deletions(-)
 create mode 100644 scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala

diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index e1b23b00..621791ab 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -3,7 +3,7 @@ package scorex.crypto.ads.merkle
 import play.api.libs.json._
 import scorex.crypto.CryptographicHash._
 import scorex.crypto.ads.merkle.TreeStorage.Position
-import scorex.crypto.{CryptographicHash, Sha256}
+import scorex.crypto.{Base58, CryptographicHash, Sha256}
 
 import scala.annotation.tailrec
 
@@ -41,21 +41,31 @@ object AuthDataBlock {
   implicit def authDataBlockReads[T](implicit fmt: Reads[T]): Reads[AuthDataBlock[T]] = new Reads[AuthDataBlock[T]] {
     def reads(json: JsValue): JsResult[AuthDataBlock[T]] = JsSuccess(AuthDataBlock[T](
       (json \ "data").get match {
-        case JsString(ts) => ts.getBytes.asInstanceOf[T]
-        case _ => throw new RuntimeException("Data MUST be a string")
+        case JsString(ts) =>
+          Base58.decode(ts).get.asInstanceOf[T]
+        case _ =>
+          throw new RuntimeException("Data MUST be a string")
       },
       (json \ "merklePath").get match {
-        case JsArray(ts) => ts.map(t => t.as[String].getBytes)
-        case _ => throw new RuntimeException("MerklePath MUST be a list")
+        case JsArray(ts) => ts.map { t =>
+          t match {
+            case JsString(digest) =>
+              Base58.decode(digest)
+            case m =>
+              throw new RuntimeException("MerklePath MUST be array of strings" + m + " given")
+          }
+        }.map(_.get)
+        case m =>
+          throw new RuntimeException("MerklePath MUST be a list " + m + " given")
       }
     ))
   }
 
   implicit def authDataBlockWrites[T](implicit fmt: Writes[T]): Writes[AuthDataBlock[T]] = new Writes[AuthDataBlock[T]] {
     def writes(ts: AuthDataBlock[T]) = JsObject(Seq(
-      "data" -> JsString(new String(ts.data.asInstanceOf[Array[Byte]])),
+      "data" -> JsString(Base58.encode(ts.data.asInstanceOf[Array[Byte]])),
       "merklePath" -> JsArray(
-        ts.merklePath.map(digest => JsString(new String(digest)))
+        ts.merklePath.map(digest => JsString(Base58.encode(digest)))
       )
     ))
   }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index ee6f2f93..6d9436dc 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -1,5 +1,6 @@
 package scorex.perma.consensus
 
+import com.google.common.primitives.{Longs, Bytes}
 import play.api.libs.json._
 import scorex.block.BlockField
 
@@ -19,4 +20,12 @@ object PermaConsensusBlockField {
 
   val fieldName: String = "perma-consensus"
 
+  def parse(bytes: Array[Byte]): PermaConsensusBlockField = {
+    (Json.parse(bytes) \ fieldName).validate[PermaLikeConsensusBlockData] match {
+      case JsSuccess(block, _) => PermaConsensusBlockField(block)
+      case m => throw new RuntimeException("enable to parse block data")
+    }
+  }
+
+
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index e4dff0e1..1d5088cf 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -1,10 +1,11 @@
 package scorex.perma.consensus
 
+import com.google.common.primitives.Longs
 import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount}
 import scorex.block.{Block, BlockField}
 import scorex.consensus.ConsensusModule
 import scorex.transaction.TransactionModule
-import play.api.libs.json.{JsObject, Json}
+import play.api.libs.json.{JsSuccess, JsResult, JsObject, Json}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -52,16 +53,8 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
   override def consensusBlockData(block: Block): PermaLikeConsensusBlockData =
     block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData]
 
-  override def parseBlockData(bytes: Array[Byte]): BlockField[PermaLikeConsensusBlockData] = {
-    val json = Json.parse(bytes)
-    ???
-  }
-
-
-  /*
-    PermaConsensusBlockField(new PermaLikeConsensusBlockData{
-      ???
-    })*/
+  //TODO return Try
+  override def parseBlockData(bytes: Array[Byte]): PermaConsensusBlockField = PermaConsensusBlockField.parse(bytes)
 
   override def genesisData: BlockField[PermaLikeConsensusBlockData] =
     PermaConsensusBlockField(PermaLikeConsensusBlockData(
diff --git a/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala b/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala
index c0e71dac..3abd2f41 100644
--- a/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala
+++ b/scorex-perma/src/test/scala/scorex/PermaTestSuite.scala
@@ -3,5 +3,6 @@ package scorex
 import org.scalatest.Suites
 
 class PermaTestSuite extends Suites (
-  new props.AuthDataStorageSpecification
+  new props.AuthDataStorageSpecification,
+  new props.PermaConsensusBlockFiendSpecification
 )
diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
new file mode 100644
index 00000000..cb1419eb
--- /dev/null
+++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
@@ -0,0 +1,37 @@
+package scorex.props
+
+import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
+import org.scalatest.{Matchers, PropSpec}
+import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.perma.consensus._
+import scorex.perma.settings.Constants.DataSegment
+
+class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
+
+  val consensus = new PermaConsensusModule
+
+  property("set value and get it") {
+    forAll { (diff: Long, puz: Array[Byte], pubkey: Array[Byte], s: Array[Byte], signature: Array[Byte], segmentIndex: Long) =>
+
+      val blockdata = "nonempty".getBytes ++ puz
+
+      val authDataBlock: AuthDataBlock[DataSegment] = AuthDataBlock(blockdata, Seq(puz, pubkey))
+      val initialBlock = PermaConsensusBlockField(PermaLikeConsensusBlockData(
+        math.abs(diff),
+        puz,
+        Ticket(pubkey, s, IndexedSeq(PartialProof(signature, segmentIndex, authDataBlock)))
+      ))
+      val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes)
+
+      parsedBlock.value.difficulty shouldBe initialBlock.value.difficulty
+      assert(parsedBlock.value.puz sameElements initialBlock.value.puz)
+      assert(parsedBlock.value.ticket.publicKey sameElements initialBlock.value.ticket.publicKey)
+      assert(parsedBlock.value.ticket.s sameElements initialBlock.value.ticket.s)
+      parsedBlock.value.ticket.proofs.size shouldBe initialBlock.value.ticket.proofs.size
+      assert(parsedBlock.value.ticket.proofs.head.signature sameElements initialBlock.value.ticket.proofs.head.signature)
+      assert(parsedBlock.value.ticket.proofs.head.segmentIndex == initialBlock.value.ticket.proofs.head.segmentIndex)
+      assert(parsedBlock.value.ticket.proofs.head.segment.data sameElements initialBlock.value.ticket.proofs.head.segment.data)
+
+    }
+  }
+}
\ No newline at end of file

From c0da60079e5d7292f87fa1d2776382d0fd9ac63a Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 20:12:02 +0300
Subject: [PATCH 37/66] Check genesis block

---
 .../consensus/PermaConsensusModule.scala      |  2 +-
 ...ermaConsensusBlockFiendSpecification.scala | 22 ++++++++++++++-----
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 1d5088cf..75e06578 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -56,7 +56,7 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
   //TODO return Try
   override def parseBlockData(bytes: Array[Byte]): PermaConsensusBlockField = PermaConsensusBlockField.parse(bytes)
 
-  override def genesisData: BlockField[PermaLikeConsensusBlockData] =
+  override def genesisData: PermaConsensusBlockField =
     PermaConsensusBlockField(PermaLikeConsensusBlockData(
       InitialDifficulty,
       Array[Byte](),
diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
index cb1419eb..7f4f9d31 100644
--- a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
+++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
@@ -23,15 +23,25 @@ class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks
       ))
       val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes)
 
-      parsedBlock.value.difficulty shouldBe initialBlock.value.difficulty
-      assert(parsedBlock.value.puz sameElements initialBlock.value.puz)
-      assert(parsedBlock.value.ticket.publicKey sameElements initialBlock.value.ticket.publicKey)
-      assert(parsedBlock.value.ticket.s sameElements initialBlock.value.ticket.s)
-      parsedBlock.value.ticket.proofs.size shouldBe initialBlock.value.ticket.proofs.size
+      checkAll(initialBlock, parsedBlock)
+
+    }
+
+    val initialBlock = consensus.genesisData
+    val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes)
+    checkAll(initialBlock, parsedBlock)
+  }
+
+  def checkAll(initialBlock: PermaConsensusBlockField, parsedBlock: PermaConsensusBlockField): Unit = {
+    parsedBlock.value.difficulty shouldBe initialBlock.value.difficulty
+    assert(parsedBlock.value.puz sameElements initialBlock.value.puz)
+    assert(parsedBlock.value.ticket.publicKey sameElements initialBlock.value.ticket.publicKey)
+    assert(parsedBlock.value.ticket.s sameElements initialBlock.value.ticket.s)
+    parsedBlock.value.ticket.proofs.size shouldBe initialBlock.value.ticket.proofs.size
+    if(parsedBlock.value.ticket.proofs.nonEmpty) {
       assert(parsedBlock.value.ticket.proofs.head.signature sameElements initialBlock.value.ticket.proofs.head.signature)
       assert(parsedBlock.value.ticket.proofs.head.segmentIndex == initialBlock.value.ticket.proofs.head.segmentIndex)
       assert(parsedBlock.value.ticket.proofs.head.segment.data sameElements initialBlock.value.ticket.proofs.head.segment.data)
-
     }
   }
 }
\ No newline at end of file

From a13273c829ab70f8355693ebce995148406f47d9 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 21:07:40 +0300
Subject: [PATCH 38/66] Is valid block

---
 .../consensus/PermaConsensusModule.scala      | 51 +++++++++++++++----
 1 file changed, 41 insertions(+), 10 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 75e06578..147533b4 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -1,14 +1,16 @@
 package scorex.perma.consensus
 
-import com.google.common.primitives.Longs
 import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount}
 import scorex.block.{Block, BlockField}
 import scorex.consensus.ConsensusModule
+import scorex.crypto.SigningFunctions._
+import scorex.crypto.{CryptographicHash, EllipticCurveImpl, Sha256}
+import scorex.perma.settings.Constants
 import scorex.transaction.TransactionModule
-import play.api.libs.json.{JsSuccess, JsResult, JsObject, Json}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
+import scala.util.Try
 
 /**
   * Data and functions related to a consensus algo
@@ -19,12 +21,14 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
   val MiningReward = 1000000
   val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)).toLong
   val GenesisCreator = new PublicKeyAccount(Array())
+  val RootHash: Array[Byte] = "".getBytes
 
   private def blockGenerator(block: Block) = block.signerDataField.value.generator
 
   def isValid[TT](block: Block)(implicit transactionModule: TransactionModule[TT]): Boolean = {
-    //TODO
-    false
+    val f = block.consensusDataField.asInstanceOf[PermaConsensusBlockField]
+    val publicKey = blockGenerator(block).publicKey
+    validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, RootHash)
   }
 
   /**
@@ -42,7 +46,9 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
     */
   def generators(block: Block): Seq[Account] = Seq(blockGenerator(block))
 
-  def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = BigInt(1)
+  def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
+    ticketScore(block.consensusDataField.asInstanceOf[PermaConsensusBlockField].value.ticket)
+  }
 
   def generateNextBlock[TT](account: PrivateKeyAccount)
                            (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = {
@@ -63,11 +69,36 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
       Ticket(GenesisCreator.publicKey, Array(), IndexedSeq())
     ))
 
-  /*
-    PermaConsensusBlockField(new PermaLikeConsensusBlockData {
-      override val generatorSignature: Array[Byte] = Array.fill(32)(0: Byte)
-    })*/
-
   override def formBlockData(data: PermaLikeConsensusBlockData): BlockField[PermaLikeConsensusBlockData] =
     PermaConsensusBlockField(data)
+
+  private val NoSig = Array[Byte]()
+
+  //todo: validate r\i
+  private def validate(publicKey: PublicKey,
+                       puz: Array[Byte],
+                       difficulty: BigInt,
+                       t: Ticket,
+                       rootHash: CryptographicHash.Digest): Boolean = Try {
+    val proofs = t.proofs
+    require(proofs.size == Constants.k)
+
+    //Local-POR lottery verification
+
+    val sigs = NoSig +: proofs.map(_.signature)
+    val ris = proofs.map(_.segmentIndex)
+
+    val partialProofsCheck = 1.to(Constants.k).foldLeft(true) { case (partialResult, i) =>
+      val segment = proofs(i - 1).segment
+
+      segment.check(ris(i - 1), rootHash)() || {
+        val hi = Sha256.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data)
+        EllipticCurveImpl.verify(sigs(i), hi, publicKey)
+      }
+    }
+    partialProofsCheck && (ticketScore(t) < difficulty)
+  }.getOrElse(false)
+
+  private def ticketScore(t: Ticket): BigInt = BigInt(1, Sha256.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
+
 }
\ No newline at end of file

From 6cd81611122bb08cabd9f23ca27c276f7e5699f1 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Thu, 26 Nov 2015 22:53:58 +0300
Subject: [PATCH 39/66] New block generation

---
 .../consensus/PermaConsensusModule.scala      | 93 +++++++++++++++++--
 ...ermaConsensusBlockFiendSpecification.scala |  5 +
 .../lagonaki/server/LagonakiSettings.scala    |  3 +-
 .../scala/scorex/lagonaki/server/Server.scala |  2 +
 4 files changed, 93 insertions(+), 10 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 147533b4..555069d2 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -4,9 +4,14 @@ import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount}
 import scorex.block.{Block, BlockField}
 import scorex.consensus.ConsensusModule
 import scorex.crypto.SigningFunctions._
+import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.crypto.{CryptographicHash, EllipticCurveImpl, Sha256}
-import scorex.perma.settings.Constants
-import scorex.transaction.TransactionModule
+import scorex.perma.Storage.AuthDataStorage
+import scorex.perma.settings.{PermaSettings, Constants}
+import scorex.perma.settings.Constants._
+import scorex.storage.Storage
+import scorex.transaction.{BalanceSheet, BlockChain, TransactionModule}
+import scorex.utils._
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
@@ -15,13 +20,17 @@ import scala.util.Try
 /**
   * Data and functions related to a consensus algo
   */
+class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settings: PermaSettings)
+  extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
-class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData] {
-
-  val MiningReward = 1000000
   val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)).toLong
   val GenesisCreator = new PublicKeyAccount(Array())
+  val Version: Byte = 1
   val RootHash: Array[Byte] = "".getBytes
+  lazy val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
+  implicit val consensusModule: ConsensusModule[PermaLikeConsensusBlockData] = this
+
+  def miningReward(block: Block) = 1000000
 
   private def blockGenerator(block: Block) = block.signerDataField.value.generator
 
@@ -29,6 +38,7 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
     val f = block.consensusDataField.asInstanceOf[PermaConsensusBlockField]
     val publicKey = blockGenerator(block).publicKey
     validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, RootHash)
+    //TODO check puz and previous block
   }
 
   /**
@@ -36,7 +46,7 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
     * Meni Rosenfeld's Proof-of-Activity proposal http://eprint.iacr.org/2014/452.pdf
     */
   def feesDistribution(block: Block): Map[Account, Long] =
-    Map(blockGenerator(block) -> MiningReward)
+    Map(blockGenerator(block) -> (miningReward(block) + block.transactions.map(_.fee).sum))
 
   /**
     * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not
@@ -52,14 +62,40 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
 
   def generateNextBlock[TT](account: PrivateKeyAccount)
                            (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = {
-    //TODO
-    Future(None)
+
+    val lastBlock = transactionModule.history.asInstanceOf[BlockChain].lastBlock
+    val lastBlockKernelData = lastBlock.consensusDataField.asInstanceOf[PermaConsensusBlockField].value
+    val lastBlockTime = lastBlock.timestampField.value
+
+    val eta = (NTP.correctedTime() - lastBlockTime) / 1000
+    val puz = generatePuz(lastBlock)
+
+    log.debug(s"eta $eta, " +
+      s"account:  $account " +
+      s"account balance: ${transactionModule.state.asInstanceOf[BalanceSheet].generationBalance(account)}"
+    )
+
+    val keyPair = (account.privateKey, account.publicKey)
+    val ticket = generate(keyPair, puz)
+    val difficulty = calcDifficulty(lastBlock)
+
+    if (validate(keyPair._2, puz, difficulty, ticket, RootHash)) {
+      val timestamp = NTP.correctedTime()
+      val consensusData = PermaLikeConsensusBlockData(difficulty, puz, ticket)
+
+      Future(Some(Block.buildAndSign(Version,
+        timestamp,
+        lastBlock.uniqueId,
+        consensusData,
+        transactionModule.packUnconfirmed(),
+        account)))
+
+    } else Future(None)
   }
 
   override def consensusBlockData(block: Block): PermaLikeConsensusBlockData =
     block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData]
 
-  //TODO return Try
   override def parseBlockData(bytes: Array[Byte]): PermaConsensusBlockField = PermaConsensusBlockField.parse(bytes)
 
   override def genesisData: PermaConsensusBlockField =
@@ -99,6 +135,45 @@ class PermaConsensusModule extends ConsensusModule[PermaLikeConsensusBlockData]
     partialProofsCheck && (ticketScore(t) < difficulty)
   }.getOrElse(false)
 
+  private def generatePuz(block: Block) = hash.hash(block.bytes)
+
   private def ticketScore(t: Ticket): BigInt = BigInt(1, Sha256.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
 
+  private def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
+
+    val (privateKey, publicKey) = keyPair
+
+    //scratch-off for the Local-POR lottery
+    val s = randomBytes(32)
+
+    val sig0 = NoSig
+    val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
+
+    val proofs: IndexedSeq[PartialProof] = 1.to(Constants.k).foldLeft(
+      (r1, sig0, Seq[PartialProof]())
+    ) {
+      case ((ri, sig_prev, seq), _) =>
+        val segment = authDataStorage.get(ri).get
+        val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
+        val sig = EllipticCurveImpl.sign(privateKey, hi)
+        val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt)
+
+        (r_next, sig, seq :+ PartialProof(sig, ri, segment))
+    }._3.toIndexedSeq.ensuring(_.size == Constants.k)
+
+    Ticket(publicKey, s, proofs)
+  }
+
+  //calculate index of i-th segment
+  private def u(pubKey: PublicKey, i: Int): Long = {
+    val h = Sha256.hash(pubKey ++ BigInt(i).toByteArray)
+    BigInt(1, h).mod(Constants.n).toLong
+  }
+
+  //TODO implement
+  private def calcDifficulty(lastBlock: Block): Long = {
+    InitialDifficulty
+  }
+
+
 }
\ No newline at end of file
diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
index 7f4f9d31..b719fbf5 100644
--- a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
+++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
@@ -5,9 +5,14 @@ import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.consensus._
 import scorex.perma.settings.Constants.DataSegment
+import scorex.perma.settings.PermaSettings
+import scorex.settings.Settings
 
 class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
+  implicit val settings = new Settings with PermaSettings {
+    val filename = "settings-test.json"
+  }
   val consensus = new PermaConsensusModule
 
   property("set value and get it") {
diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala b/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala
index 22ec7496..db28f34d 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiSettings.scala
@@ -1,6 +1,7 @@
 package scorex.lagonaki.server
 
+import scorex.perma.settings.PermaSettings
 import scorex.settings.Settings
 import scorex.transaction.TransactionSettings
 
-class LagonakiSettings(override val filename:String) extends Settings with TransactionSettings
\ No newline at end of file
+class LagonakiSettings(override val filename: String) extends Settings with TransactionSettings with PermaSettings
\ No newline at end of file
diff --git a/src/main/scala/scorex/lagonaki/server/Server.scala b/src/main/scala/scorex/lagonaki/server/Server.scala
index fb6b8bf3..47b87962 100644
--- a/src/main/scala/scorex/lagonaki/server/Server.scala
+++ b/src/main/scala/scorex/lagonaki/server/Server.scala
@@ -45,6 +45,8 @@ object Server extends App with ScorexLogging {
     Thread.sleep(3000)
 
     val genesisBlock = application.blockchainImpl.blockAt(1)
+    println("!!!!!!!!!!!!!!!!!GENESIS")
+    println(genesisBlock)
     val genesisAccs = genesisBlock.get.transactions.flatMap { tx => tx match {
       case gtx: GenesisTransaction =>
         Some(gtx.recipient)

From 99048ece1a54b35ecd9c85cedce57a6b48995077 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Fri, 27 Nov 2015 10:40:23 +0300
Subject: [PATCH 40/66] unused import, formatting

---
 .../scorex/perma/consensus/PermaConsensusBlockField.scala      | 3 ---
 1 file changed, 3 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 6d9436dc..9a7f8576 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -1,6 +1,5 @@
 package scorex.perma.consensus
 
-import com.google.common.primitives.{Longs, Bytes}
 import play.api.libs.json._
 import scorex.block.BlockField
 
@@ -26,6 +25,4 @@ object PermaConsensusBlockField {
       case m => throw new RuntimeException("enable to parse block data")
     }
   }
-
-
 }

From 3ef1904cd5f7461839811a0c5a7dc51e54ec8feb Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 14:05:11 +0300
Subject: [PATCH 41/66] Try from block generation

---
 .../scorex/perma/consensus/PermaConsensusModule.scala  | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 555069d2..0b39d9e6 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -7,8 +7,8 @@ import scorex.crypto.SigningFunctions._
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.crypto.{CryptographicHash, EllipticCurveImpl, Sha256}
 import scorex.perma.Storage.AuthDataStorage
-import scorex.perma.settings.{PermaSettings, Constants}
 import scorex.perma.settings.Constants._
+import scorex.perma.settings.{Constants, PermaSettings}
 import scorex.storage.Storage
 import scorex.transaction.{BalanceSheet, BlockChain, TransactionModule}
 import scorex.utils._
@@ -61,7 +61,7 @@ class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settin
   }
 
   def generateNextBlock[TT](account: PrivateKeyAccount)
-                           (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = {
+                           (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = Try {
 
     val lastBlock = transactionModule.history.asInstanceOf[BlockChain].lastBlock
     val lastBlockKernelData = lastBlock.consensusDataField.asInstanceOf[PermaConsensusBlockField].value
@@ -91,7 +91,11 @@ class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settin
         account)))
 
     } else Future(None)
-  }
+  }.recoverWith { case t: Throwable =>
+    log.error("Error when creating new block", t)
+    t.printStackTrace()
+    Try(Future(None))
+  }.getOrElse(Future(None))
 
   override def consensusBlockData(block: Block): PermaLikeConsensusBlockData =
     block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData]

From bd4b7eb0ea69771b0b81be51220532f164f71cb1 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 14:32:51 +0300
Subject: [PATCH 42/66] Create storage folder

---
 .../perma/consensus/PermaConsensusModule.scala    | 15 ++++++---------
 .../PermaConsensusBlockFiendSpecification.scala   |  5 ++++-
 .../lagonaki/server/LagonakiApplication.scala     | 11 ++++++++++-
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 0b39d9e6..ee8f98ac 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -6,9 +6,8 @@ import scorex.consensus.ConsensusModule
 import scorex.crypto.SigningFunctions._
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.crypto.{CryptographicHash, EllipticCurveImpl, Sha256}
-import scorex.perma.Storage.AuthDataStorage
+import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants._
-import scorex.perma.settings.{Constants, PermaSettings}
 import scorex.storage.Storage
 import scorex.transaction.{BalanceSheet, BlockChain, TransactionModule}
 import scorex.utils._
@@ -20,14 +19,14 @@ import scala.util.Try
 /**
   * Data and functions related to a consensus algo
   */
-class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settings: PermaSettings)
+class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha256)
+                          (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
   val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)).toLong
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
-  val RootHash: Array[Byte] = "".getBytes
-  lazy val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
+
   implicit val consensusModule: ConsensusModule[PermaLikeConsensusBlockData] = this
 
   def miningReward(block: Block) = 1000000
@@ -37,7 +36,7 @@ class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settin
   def isValid[TT](block: Block)(implicit transactionModule: TransactionModule[TT]): Boolean = {
     val f = block.consensusDataField.asInstanceOf[PermaConsensusBlockField]
     val publicKey = blockGenerator(block).publicKey
-    validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, RootHash)
+    validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, rootHash)
     //TODO check puz and previous block
   }
 
@@ -51,8 +50,6 @@ class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settin
   /**
     * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not
     * (see e.g. Meni Rosenfeld's Proof-of-Activity paper http://eprint.iacr.org/2014/452.pdf)
-    * @param block
-    * @return
     */
   def generators(block: Block): Seq[Account] = Seq(blockGenerator(block))
 
@@ -79,7 +76,7 @@ class PermaConsensusModule(hash: CryptographicHash = Sha256)(implicit val settin
     val ticket = generate(keyPair, puz)
     val difficulty = calcDifficulty(lastBlock)
 
-    if (validate(keyPair._2, puz, difficulty, ticket, RootHash)) {
+    if (validate(keyPair._2, puz, difficulty, ticket, rootHash)) {
       val timestamp = NTP.correctedTime()
       val consensusData = PermaLikeConsensusBlockData(difficulty, puz, ticket)
 
diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
index b719fbf5..45ad3957 100644
--- a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
+++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
@@ -3,17 +3,20 @@ package scorex.props
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.consensus._
 import scorex.perma.settings.Constants.DataSegment
 import scorex.perma.settings.PermaSettings
 import scorex.settings.Settings
+import scorex.storage.Storage
 
 class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
   implicit val settings = new Settings with PermaSettings {
     val filename = "settings-test.json"
   }
-  val consensus = new PermaConsensusModule
+  implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
+  val consensus = new PermaConsensusModule("test".getBytes)
 
   property("set value and get it") {
     forAll { (diff: Long, puz: Array[Byte], pubkey: Array[Byte], s: Array[Byte], signature: Array[Byte], segmentIndex: Long) =>
diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
index 3e4b660d..a3e8b9f5 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
@@ -1,5 +1,7 @@
 package scorex.lagonaki.server
 
+import java.io.File
+
 import akka.actor.Props
 import akka.io.IO
 import com.typesafe.config.ConfigFactory
@@ -9,14 +11,18 @@ import scorex.app.Application
 import scorex.consensus.LagonakiConsensusModule
 import scorex.consensus.nxt.api.http.NxtConsensusApiRoute
 import scorex.consensus.qora.api.http.QoraConsensusApiRoute
+import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.lagonaki.api.http.{PeersHttpService, PaymentApiRoute, ScorexApiRoute}
 import scorex.block.Block
 import scorex.consensus.nxt.NxtLikeConsensusModule
 import scorex.consensus.qora.QoraLikeConsensusModule
 import scorex.lagonaki.network.message._
 import scorex.lagonaki.network.{BlockchainSyncer, NetworkController}
+import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.consensus.PermaConsensusModule
 import scorex.perma.consensus.http.PermaConsensusApiRoute
+import scorex.perma.settings.Constants._
+import scorex.storage.Storage
 import scorex.transaction.LagonakiTransaction.ValidationResult
 import scorex.transaction._
 import scorex.transaction.state.database.UnconfirmedTransactionsDatabaseImpl
@@ -43,7 +49,10 @@ class LagonakiApplication(val settingsFilename: String)
       case s: String if s.equalsIgnoreCase("qora") =>
         new QoraLikeConsensusModule
       case s: String if s.equalsIgnoreCase("perma") =>
-        new PermaConsensusModule
+        new File(settings.treeDir).mkdirs()
+        val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
+        val rootHash = "".getBytes
+        new PermaConsensusModule(rootHash)(authDataStorage)
       case algo =>
         log.error(s"Unknown consensus algo: $algo. Use NxtLikeConsensusModule instead.")
         new NxtLikeConsensusModule

From f9d523ee6faf5a99d0fa125a53507ced9ed6c340 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 14:45:18 +0300
Subject: [PATCH 43/66] Hash function from config

---
 .../consensus/PermaConsensusModule.scala      | 20 ++++++++++---------
 .../scorex/perma/settings/Constants.scala     |  2 +-
 2 files changed, 12 insertions(+), 10 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index ee8f98ac..093dd483 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -5,7 +5,8 @@ import scorex.block.{Block, BlockField}
 import scorex.consensus.ConsensusModule
 import scorex.crypto.SigningFunctions._
 import scorex.crypto.ads.merkle.AuthDataBlock
-import scorex.crypto.{CryptographicHash, EllipticCurveImpl, Sha256}
+import scorex.crypto.EllipticCurveImpl
+import scorex.crypto.CryptographicHash.Digest
 import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants._
 import scorex.storage.Storage
@@ -19,13 +20,14 @@ import scala.util.Try
 /**
   * Data and functions related to a consensus algo
   */
-class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha256)
+class PermaConsensusModule(rootHash: Array[Byte])
                           (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
   val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)).toLong
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
+  val hash = Constants.hash
 
   implicit val consensusModule: ConsensusModule[PermaLikeConsensusBlockData] = this
 
@@ -116,7 +118,7 @@ class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha2
                        puz: Array[Byte],
                        difficulty: BigInt,
                        t: Ticket,
-                       rootHash: CryptographicHash.Digest): Boolean = Try {
+                       rootHash: Digest): Boolean = Try {
     val proofs = t.proofs
     require(proofs.size == Constants.k)
 
@@ -129,7 +131,7 @@ class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha2
       val segment = proofs(i - 1).segment
 
       segment.check(ris(i - 1), rootHash)() || {
-        val hi = Sha256.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data)
+        val hi = hash.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data)
         EllipticCurveImpl.verify(sigs(i), hi, publicKey)
       }
     }
@@ -138,7 +140,7 @@ class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha2
 
   private def generatePuz(block: Block) = hash.hash(block.bytes)
 
-  private def ticketScore(t: Ticket): BigInt = BigInt(1, Sha256.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
+  private def ticketScore(t: Ticket): BigInt = BigInt(1, hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
 
   private def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
 
@@ -148,16 +150,16 @@ class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha2
     val s = randomBytes(32)
 
     val sig0 = NoSig
-    val r1 = u(publicKey, (BigInt(1, Sha256.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
+    val r1 = u(publicKey, (BigInt(1, hash.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
 
     val proofs: IndexedSeq[PartialProof] = 1.to(Constants.k).foldLeft(
       (r1, sig0, Seq[PartialProof]())
     ) {
       case ((ri, sig_prev, seq), _) =>
         val segment = authDataStorage.get(ri).get
-        val hi = Sha256.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
+        val hi = hash.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
         val sig = EllipticCurveImpl.sign(privateKey, hi)
-        val r_next = u(publicKey, BigInt(1, Sha256.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt)
+        val r_next = u(publicKey, BigInt(1, hash.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt)
 
         (r_next, sig, seq :+ PartialProof(sig, ri, segment))
     }._3.toIndexedSeq.ensuring(_.size == Constants.k)
@@ -167,7 +169,7 @@ class PermaConsensusModule(rootHash: Array[Byte], hash: CryptographicHash = Sha2
 
   //calculate index of i-th segment
   private def u(pubKey: PublicKey, i: Int): Long = {
-    val h = Sha256.hash(pubKey ++ BigInt(i).toByteArray)
+    val h = hash.hash(pubKey ++ BigInt(i).toByteArray)
     BigInt(1, h).mod(Constants.n).toLong
   }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
index 6b27d2af..ab00e784 100644
--- a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
@@ -23,7 +23,7 @@ object Constants extends ScorexLogging {
     case s: String if s.equalsIgnoreCase("sha256") =>
       Sha256
     case hashFunction =>
-      log.error(s"Unknown hash function: $hashFunction. Use Sha256 instead.")
+      log.warn(s"Unknown hash function: $hashFunction. Use Sha256 instead.")
       Sha256
   }
 

From eab0d9b0e2e1d08e9742f7397c2a26d71f6a3417 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 15:15:56 +0300
Subject: [PATCH 44/66] Test initialization of dataset

---
 .../lagonaki/server/LagonakiApplication.scala | 40 +++++++++++++++++--
 1 file changed, 36 insertions(+), 4 deletions(-)

diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
index a3e8b9f5..c6c40394 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
@@ -1,6 +1,7 @@
 package scorex.lagonaki.server
 
-import java.io.File
+import java.io.{RandomAccessFile, File}
+import java.nio.file.{Paths, Files}
 
 import akka.actor.Props
 import akka.io.IO
@@ -11,7 +12,7 @@ import scorex.app.Application
 import scorex.consensus.LagonakiConsensusModule
 import scorex.consensus.nxt.api.http.NxtConsensusApiRoute
 import scorex.consensus.qora.api.http.QoraConsensusApiRoute
-import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.crypto.ads.merkle.{MerkleTree, AuthDataBlock}
 import scorex.lagonaki.api.http.{PeersHttpService, PaymentApiRoute, ScorexApiRoute}
 import scorex.block.Block
 import scorex.consensus.nxt.NxtLikeConsensusModule
@@ -19,8 +20,10 @@ import scorex.consensus.qora.QoraLikeConsensusModule
 import scorex.lagonaki.network.message._
 import scorex.lagonaki.network.{BlockchainSyncer, NetworkController}
 import scorex.perma.Storage.AuthDataStorage
+import scorex.perma.TestApp._
 import scorex.perma.consensus.PermaConsensusModule
 import scorex.perma.consensus.http.PermaConsensusApiRoute
+import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants._
 import scorex.storage.Storage
 import scorex.transaction.LagonakiTransaction.ValidationResult
@@ -49,10 +52,39 @@ class LagonakiApplication(val settingsFilename: String)
       case s: String if s.equalsIgnoreCase("qora") =>
         new QoraLikeConsensusModule
       case s: String if s.equalsIgnoreCase("perma") =>
+        val tree = if (Files.exists(Paths.get(settings.treeDir + "/tree0.mapDB"))) {
+          log.info("Get existing tree")
+          new MerkleTree(settings.treeDir, Constants.n, Constants.segmentSize, Constants.hash)
+        } else {
+          log.info("Generating random data set")
+          val treeDir = new File(settings.treeDir)
+          treeDir.mkdirs()
+          val datasetFile = settings.treeDir + "/data.file"
+          new RandomAccessFile(datasetFile, "rw").setLength(Constants.n * Constants.segmentSize)
+          log.info("Calculate tree")
+          val tree = MerkleTree.fromFile(datasetFile, settings.treeDir, Constants.segmentSize, Constants.hash)
+          require(tree.nonEmptyBlocks == Constants.n, s"${tree.nonEmptyBlocks} == ${Constants.n}")
+          tree
+        }
+
+        log.info("Test tree")
+        val index = Constants.n - 3
+        val leaf = tree.byIndex(index).get
+        require(leaf.check(index, tree.rootHash)(Constants.hash))
+
+        log.info("Put ALL data to local storage")
         new File(settings.treeDir).mkdirs()
         val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
-        val rootHash = "".getBytes
-        new PermaConsensusModule(rootHash)(authDataStorage)
+        def addBlock(i: Long): Unit = {
+          authDataStorage.set(i, tree.byIndex(i).get)
+          if (i > 0) {
+            addBlock(i - 1)
+          }
+        }
+        addBlock(Constants.n - 1)
+
+        log.info("Create consensus module")
+        new PermaConsensusModule(tree.rootHash)(authDataStorage)
       case algo =>
         log.error(s"Unknown consensus algo: $algo. Use NxtLikeConsensusModule instead.")
         new NxtLikeConsensusModule

From fe24051c8e052f66efaebf6b4c2deb9fda473f89 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 15:26:33 +0300
Subject: [PATCH 45/66] Score for genesis block

---
 .../perma/consensus/PermaConsensusModule.scala      | 13 +++++++++++--
 .../scorex/lagonaki/network/BlockchainSyncer.scala  |  1 +
 2 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 093dd483..58320ad8 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -79,6 +79,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
     val difficulty = calcDifficulty(lastBlock)
 
     if (validate(keyPair._2, puz, difficulty, ticket, rootHash)) {
+      log.info("New valid ticket created. Current score:" + transactionModule.history.asInstanceOf[BlockChain].score())
       val timestamp = NTP.correctedTime()
       val consensusData = PermaLikeConsensusBlockData(difficulty, puz, ticket)
 
@@ -89,7 +90,10 @@ class PermaConsensusModule(rootHash: Array[Byte])
         transactionModule.packUnconfirmed(),
         account)))
 
-    } else Future(None)
+    } else {
+      log.info("Invalid ticket created. Current score:" + transactionModule.history.asInstanceOf[BlockChain].score())
+      Future(None)
+    }
   }.recoverWith { case t: Throwable =>
     log.error("Error when creating new block", t)
     t.printStackTrace()
@@ -140,7 +144,12 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
   private def generatePuz(block: Block) = hash.hash(block.bytes)
 
-  private def ticketScore(t: Ticket): BigInt = BigInt(1, hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
+  private def ticketScore(t: Ticket): BigInt = if(t.proofs.nonEmpty) {
+    BigInt(1, hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
+  } else {
+    //Genesis block contains empty ticket
+    0
+  }
 
   private def generate(keyPair: (PrivateKey, PublicKey), puz: Array[Byte]): Ticket = {
 
diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
index a3d7d3a5..75e13b9d 100644
--- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
+++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
@@ -127,6 +127,7 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
     val accounts = application.wallet.privateKeyAccounts()
     consModule.generateNextBlocks(accounts)(transModule) onComplete {
       case Success(blocks: Seq[Block]) =>
+        log.info(blocks.size + " blocks created")
         if (blocks.nonEmpty) {
           val bestBlock = blocks.maxBy(consModule.blockScore)
           self ! NewBlock(bestBlock, None)

From 93dadf8f89195eed731e59811c697fe76ed6e39e Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 15:53:19 +0300
Subject: [PATCH 46/66] Difficulty should be bigint

---
 .../consensus/PermaConsensusModule.scala      |  6 ++---
 .../PermaLikeConsensusBlockData.scala         | 23 +++++++++++++++----
 .../lagonaki/network/BlockchainSyncer.scala   |  4 ++--
 3 files changed, 23 insertions(+), 10 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 58320ad8..3c2e6252 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -24,7 +24,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
                           (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
-  val InitialDifficulty = BigInt(Array.fill(32)(1: Byte)).toLong
+  val InitialDifficulty:BigInt = BigInt(Array.fill(36)(1: Byte))
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
   val hash = Constants.hash
@@ -79,7 +79,6 @@ class PermaConsensusModule(rootHash: Array[Byte])
     val difficulty = calcDifficulty(lastBlock)
 
     if (validate(keyPair._2, puz, difficulty, ticket, rootHash)) {
-      log.info("New valid ticket created. Current score:" + transactionModule.history.asInstanceOf[BlockChain].score())
       val timestamp = NTP.correctedTime()
       val consensusData = PermaLikeConsensusBlockData(difficulty, puz, ticket)
 
@@ -91,7 +90,6 @@ class PermaConsensusModule(rootHash: Array[Byte])
         account)))
 
     } else {
-      log.info("Invalid ticket created. Current score:" + transactionModule.history.asInstanceOf[BlockChain].score())
       Future(None)
     }
   }.recoverWith { case t: Throwable =>
@@ -183,7 +181,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
   }
 
   //TODO implement
-  private def calcDifficulty(lastBlock: Block): Long = {
+  private def calcDifficulty(lastBlock: Block): BigInt = {
     InitialDifficulty
   }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
index 158a0fef..02fcda4e 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
@@ -1,19 +1,34 @@
 package scorex.perma.consensus
 
 import play.api.libs.functional.syntax._
-import play.api.libs.json.{JsPath, Reads, Writes}
+import play.api.libs.json._
 
-case class PermaLikeConsensusBlockData(difficulty: Long, puz: Array[Byte], ticket: Ticket)
+case class PermaLikeConsensusBlockData(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
 
 object PermaLikeConsensusBlockData {
+  implicit val bigIntWrites = new Writes[BigInt] {
+    def writes(bitInt: BigInt) = JsString(bitInt.toString)
+  }
+
+  implicit def bigIntReads: Reads[BigInt] = new Reads[BigInt] {
+    def reads(json: JsValue): JsResult[BigInt] = json match {
+      case JsString(bigint) =>
+        JsSuccess(BigInt(bigint))
+      case JsNumber(bigint) =>
+        JsSuccess(BigInt(bigint.toString))
+      case m =>
+        throw new RuntimeException(s"Bigint MUST be represented as string in json $m ${m.getClass} given")
+    }
+  }
+
   implicit val writes: Writes[PermaLikeConsensusBlockData] = (
-    (JsPath \ "difficulty").write[Long] and
+    (JsPath \ "difficulty").write[BigInt] and
       (JsPath \ "puz").write[Array[Byte]] and
       (JsPath \ "ticket").write[Ticket]
     ) (unlift(PermaLikeConsensusBlockData.unapply))
 
   implicit val reads: Reads[PermaLikeConsensusBlockData] = (
-    (JsPath \ "difficulty").read[Long] and
+    (JsPath \ "difficulty").read[BigInt] and
       (JsPath \ "puz").read[Array[Byte]] and
       (JsPath \ "ticket").read[Ticket]
     ) (PermaLikeConsensusBlockData.apply _)
diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
index 75e13b9d..5c0610c4 100644
--- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
+++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
@@ -103,9 +103,10 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
   def processNewBlock(block: Block, remoteOpt: Option[InetSocketAddress]) = {
     val fromStr = remoteOpt.map(_.toString).getOrElse("local")
     if (block.isValid) {
-      log.info(s"New block: $block from $fromStr")
       application.storedState.processBlock(block)
       application.blockchainImpl.appendBlock(block)
+      log.info(s"New block: $block from $fromStr. New size: ${application.blockchainImpl.height()}, " +
+        s"score: ${application.blockchainImpl.score()}")
 
       block.transactionModule.clearFromUnconfirmed(block.transactionDataField.value)
       val height = application.blockchainImpl.height()
@@ -127,7 +128,6 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
     val accounts = application.wallet.privateKeyAccounts()
     consModule.generateNextBlocks(accounts)(transModule) onComplete {
       case Success(blocks: Seq[Block]) =>
-        log.info(blocks.size + " blocks created")
         if (blocks.nonEmpty) {
           val bestBlock = blocks.maxBy(consModule.blockScore)
           self ! NewBlock(bestBlock, None)

From 60338b7ede2c3a9ffcc162e7452e64aad43c444c Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 16:11:50 +0300
Subject: [PATCH 47/66] Equals block score for all blocks

---
 .../scala/scorex/perma/consensus/PermaConsensusModule.scala   | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 3c2e6252..9f578bae 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -55,9 +55,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
     */
   def generators(block: Block): Seq[Account] = Seq(blockGenerator(block))
 
-  def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
-    ticketScore(block.consensusDataField.asInstanceOf[PermaConsensusBlockField].value.ticket)
-  }
+  def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = BigInt(1)
 
   def generateNextBlock[TT](account: PrivateKeyAccount)
                            (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = Try {

From 1be31ba0c93f36561a71025d46ff2ef447c51009 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Fri, 27 Nov 2015 16:36:27 +0300
Subject: [PATCH 48/66] Difficulty algo

---
 .../consensus/PermaConsensusModule.scala      | 80 +++++++++++++------
 .../lagonaki/network/BlockchainSyncer.scala   |  7 +-
 .../lagonaki/server/LagonakiApplication.scala | 16 ++--
 .../scala/scorex/lagonaki/server/Server.scala |  2 -
 4 files changed, 67 insertions(+), 38 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 9f578bae..f3df99ef 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -3,18 +3,19 @@ package scorex.perma.consensus
 import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount}
 import scorex.block.{Block, BlockField}
 import scorex.consensus.ConsensusModule
+import scorex.crypto.CryptographicHash.Digest
+import scorex.crypto.EllipticCurveImpl
 import scorex.crypto.SigningFunctions._
 import scorex.crypto.ads.merkle.AuthDataBlock
-import scorex.crypto.EllipticCurveImpl
-import scorex.crypto.CryptographicHash.Digest
 import scorex.perma.settings.Constants
 import scorex.perma.settings.Constants._
 import scorex.storage.Storage
-import scorex.transaction.{BalanceSheet, BlockChain, TransactionModule}
+import scorex.transaction.{BlockChain, TransactionModule}
 import scorex.utils._
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
+import scala.concurrent.duration._
 import scala.util.Try
 
 /**
@@ -24,7 +25,11 @@ class PermaConsensusModule(rootHash: Array[Byte])
                           (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
-  val InitialDifficulty:BigInt = BigInt(Array.fill(36)(1: Byte))
+  val InitialDifficulty: BigInt = BigInt(Array.fill(36)(1: Byte))
+  val initialDifficultyPow: BigInt = log2(InitialDifficulty)
+  val AvgDelay = 2.seconds.toSeconds
+  val DifficultyRecalculation = 30
+
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
   val hash = Constants.hash
@@ -37,9 +42,17 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
   def isValid[TT](block: Block)(implicit transactionModule: TransactionModule[TT]): Boolean = {
     val f = block.consensusDataField.asInstanceOf[PermaConsensusBlockField]
-    val publicKey = blockGenerator(block).publicKey
-    validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, rootHash)
-    //TODO check puz and previous block
+    val trans = transactionModule.history.asInstanceOf[BlockChain]
+    trans.parent(block) match {
+      case Some(parent) =>
+        lazy val publicKey = blockGenerator(block).publicKey
+        lazy val puzIsValid = f.value.puz sameElements generatePuz(parent)
+        lazy val difficultyIsValid = f.value.difficulty == calcDifficulty(parent)
+        lazy val ticketIsValid = validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, rootHash)
+        puzIsValid && difficultyIsValid && ticketIsValid
+      case None =>
+        true
+    }
   }
 
   /**
@@ -55,26 +68,21 @@ class PermaConsensusModule(rootHash: Array[Byte])
     */
   def generators(block: Block): Seq[Account] = Seq(blockGenerator(block))
 
-  def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = BigInt(1)
+  def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
+    val score = initialDifficultyPow -
+      log2(block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].difficulty)
+    if (score > 0) score else 1
+  }
 
   def generateNextBlock[TT](account: PrivateKeyAccount)
                            (implicit transactionModule: TransactionModule[TT]): Future[Option[Block]] = Try {
 
-    val lastBlock = transactionModule.history.asInstanceOf[BlockChain].lastBlock
-    val lastBlockKernelData = lastBlock.consensusDataField.asInstanceOf[PermaConsensusBlockField].value
-    val lastBlockTime = lastBlock.timestampField.value
-
-    val eta = (NTP.correctedTime() - lastBlockTime) / 1000
-    val puz = generatePuz(lastBlock)
-
-    log.debug(s"eta $eta, " +
-      s"account:  $account " +
-      s"account balance: ${transactionModule.state.asInstanceOf[BalanceSheet].generationBalance(account)}"
-    )
+    val parent = transactionModule.history.asInstanceOf[BlockChain].lastBlock
+    val puz = generatePuz(parent)
 
     val keyPair = (account.privateKey, account.publicKey)
     val ticket = generate(keyPair, puz)
-    val difficulty = calcDifficulty(lastBlock)
+    val difficulty = calcDifficulty(parent)
 
     if (validate(keyPair._2, puz, difficulty, ticket, rootHash)) {
       val timestamp = NTP.correctedTime()
@@ -82,7 +90,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
       Future(Some(Block.buildAndSign(Version,
         timestamp,
-        lastBlock.uniqueId,
+        parent.uniqueId,
         consensusData,
         transactionModule.packUnconfirmed(),
         account)))
@@ -140,7 +148,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
   private def generatePuz(block: Block) = hash.hash(block.bytes)
 
-  private def ticketScore(t: Ticket): BigInt = if(t.proofs.nonEmpty) {
+  private def ticketScore(t: Ticket): BigInt = if (t.proofs.nonEmpty) {
     BigInt(1, hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
   } else {
     //Genesis block contains empty ticket
@@ -178,10 +186,30 @@ class PermaConsensusModule(rootHash: Array[Byte])
     BigInt(1, h).mod(Constants.n).toLong
   }
 
-  //TODO implement
-  private def calcDifficulty(lastBlock: Block): BigInt = {
-    InitialDifficulty
+  private def calcDifficulty(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
+    val trans = transactionModule.history.asInstanceOf[BlockChain]
+    lazy val currentDiff = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].difficulty
+    trans.heightOf(block) match {
+      case Some(height) =>
+        if (height < DifficultyRecalculation + 1) {
+          InitialDifficulty
+        } else if (height % DifficultyRecalculation == 0) {
+          val lastBlocks = (0 until DifficultyRecalculation).flatMap(i => trans.blockAt(height - i)).reverse
+          require(lastBlocks.length == DifficultyRecalculation)
+          val lastAvgDuration: Long = (0 until DifficultyRecalculation - 1).map { i =>
+            lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value
+          }.sum / (DifficultyRecalculation - 1)
+          val newDiff = currentDiff * lastAvgDuration / 1000 / AvgDelay
+          log.debug(s"Height: $height, newDiff: $newDiff, currentDiff:$currentDiff, lastAvgDuration:$lastAvgDuration")
+          newDiff
+        } else {
+          currentDiff
+        }
+      case None =>
+        log.warn(s"Enable to get height of block ${block.uniqueId}")
+        currentDiff
+    }
   }
 
-
+  private def log2(i: BigInt): BigInt = BigDecimal(math.log(i.doubleValue()) / math.log(2)).toBigInt()
 }
\ No newline at end of file
diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
index 5c0610c4..cbf19b51 100644
--- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
+++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
@@ -18,6 +18,8 @@ case class NewBlock(block: Block, sender: Option[InetSocketAddress])
 class BlockchainSyncer(application: LagonakiApplication, networkController: ActorRef) extends FSM[Status, Unit] {
 
   private val stateTimeout = 1.second
+  //TODO get from config
+  val blockGenerationDelay = 10.millis
 
   startWith(Offline, Unit)
 
@@ -60,10 +62,13 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
         scoreOpt,
         onNone = () => {
           tryToGenerateABlock()
+          context.system.scheduler.scheduleOnce(blockGenerationDelay, networkController, GetMaxChainScore)
           stay()
         },
         onLocal = () => {
           tryToGenerateABlock()
+          networkController ! GetMaxChainScore
+          context.system.scheduler.scheduleOnce(blockGenerationDelay, networkController, GetMaxChainScore)
           stay()
         }
       )
@@ -106,7 +111,7 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
       application.storedState.processBlock(block)
       application.blockchainImpl.appendBlock(block)
       log.info(s"New block: $block from $fromStr. New size: ${application.blockchainImpl.height()}, " +
-        s"score: ${application.blockchainImpl.score()}")
+        s"score: ${application.blockchainImpl.score()}, timestamp: ${block.timestampField.value}")
 
       block.transactionModule.clearFromUnconfirmed(block.transactionDataField.value)
       val height = application.blockchainImpl.height()
diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
index c6c40394..b503d6dd 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
@@ -1,7 +1,7 @@
 package scorex.lagonaki.server
 
-import java.io.{RandomAccessFile, File}
-import java.nio.file.{Paths, Files}
+import java.io.{File, RandomAccessFile}
+import java.nio.file.{Files, Paths}
 
 import akka.actor.Props
 import akka.io.IO
@@ -9,18 +9,16 @@ import com.typesafe.config.ConfigFactory
 import scorex.account.{Account, PrivateKeyAccount, PublicKeyAccount}
 import scorex.api.http._
 import scorex.app.Application
-import scorex.consensus.LagonakiConsensusModule
-import scorex.consensus.nxt.api.http.NxtConsensusApiRoute
-import scorex.consensus.qora.api.http.QoraConsensusApiRoute
-import scorex.crypto.ads.merkle.{MerkleTree, AuthDataBlock}
-import scorex.lagonaki.api.http.{PeersHttpService, PaymentApiRoute, ScorexApiRoute}
 import scorex.block.Block
 import scorex.consensus.nxt.NxtLikeConsensusModule
+import scorex.consensus.nxt.api.http.NxtConsensusApiRoute
 import scorex.consensus.qora.QoraLikeConsensusModule
+import scorex.consensus.qora.api.http.QoraConsensusApiRoute
+import scorex.crypto.ads.merkle.{AuthDataBlock, MerkleTree}
+import scorex.lagonaki.api.http.{PaymentApiRoute, PeersHttpService, ScorexApiRoute}
 import scorex.lagonaki.network.message._
 import scorex.lagonaki.network.{BlockchainSyncer, NetworkController}
 import scorex.perma.Storage.AuthDataStorage
-import scorex.perma.TestApp._
 import scorex.perma.consensus.PermaConsensusModule
 import scorex.perma.consensus.http.PermaConsensusApiRoute
 import scorex.perma.settings.Constants
@@ -32,9 +30,9 @@ import scorex.transaction.state.database.UnconfirmedTransactionsDatabaseImpl
 import scorex.transaction.state.wallet.{Payment, Wallet}
 import scorex.utils.{NTP, ScorexLogging}
 import spray.can.Http
-import scala.reflect.runtime.universe._
 
 import scala.concurrent.ExecutionContext.Implicits.global
+import scala.reflect.runtime.universe._
 
 class LagonakiApplication(val settingsFilename: String)
   extends Application with ScorexLogging {
diff --git a/src/main/scala/scorex/lagonaki/server/Server.scala b/src/main/scala/scorex/lagonaki/server/Server.scala
index 47b87962..fb6b8bf3 100644
--- a/src/main/scala/scorex/lagonaki/server/Server.scala
+++ b/src/main/scala/scorex/lagonaki/server/Server.scala
@@ -45,8 +45,6 @@ object Server extends App with ScorexLogging {
     Thread.sleep(3000)
 
     val genesisBlock = application.blockchainImpl.blockAt(1)
-    println("!!!!!!!!!!!!!!!!!GENESIS")
-    println(genesisBlock)
     val genesisAccs = genesisBlock.get.transactions.flatMap { tx => tx match {
       case gtx: GenesisTransaction =>
         Some(gtx.recipient)

From 6d5bf0dcd9ec5cceb0bd74409ee449b59aedfcff Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 16:04:21 +0300
Subject: [PATCH 49/66] Rename difficulty => target

---
 .../consensus/PermaConsensusModule.scala      | 56 +++++++++----------
 .../PermaLikeConsensusBlockData.scala         |  2 +-
 ...ermaConsensusBlockFiendSpecification.scala |  2 +-
 .../lagonaki/network/BlockchainSyncer.scala   |  2 +-
 4 files changed, 31 insertions(+), 31 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index f3df99ef..4bfcf18a 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -25,10 +25,10 @@ class PermaConsensusModule(rootHash: Array[Byte])
                           (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
-  val InitialDifficulty: BigInt = BigInt(Array.fill(36)(1: Byte))
-  val initialDifficultyPow: BigInt = log2(InitialDifficulty)
+  val InitialTarget = BigInt("108789072715742611466048832036162432282119640008661078894418176744671493285459")
+  val initialTargetPow: BigInt = log2(InitialTarget)
+  val TargetRecalculation = 10
   val AvgDelay = 2.seconds.toSeconds
-  val DifficultyRecalculation = 30
 
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
@@ -47,9 +47,9 @@ class PermaConsensusModule(rootHash: Array[Byte])
       case Some(parent) =>
         lazy val publicKey = blockGenerator(block).publicKey
         lazy val puzIsValid = f.value.puz sameElements generatePuz(parent)
-        lazy val difficultyIsValid = f.value.difficulty == calcDifficulty(parent)
-        lazy val ticketIsValid = validate(publicKey, f.value.puz, f.value.difficulty, f.value.ticket, rootHash)
-        puzIsValid && difficultyIsValid && ticketIsValid
+        lazy val target = f.value.target == calcTarget(parent)
+        lazy val ticketIsValid = validate(publicKey, f.value.puz, f.value.target, f.value.ticket, rootHash)
+        puzIsValid && target && ticketIsValid
       case None =>
         true
     }
@@ -69,8 +69,8 @@ class PermaConsensusModule(rootHash: Array[Byte])
   def generators(block: Block): Seq[Account] = Seq(blockGenerator(block))
 
   def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
-    val score = initialDifficultyPow -
-      log2(block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].difficulty)
+    val score = initialTargetPow -
+      log2(block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].target)
     if (score > 0) score else 1
   }
 
@@ -82,11 +82,11 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
     val keyPair = (account.privateKey, account.publicKey)
     val ticket = generate(keyPair, puz)
-    val difficulty = calcDifficulty(parent)
+    val target = calcTarget(parent)
 
-    if (validate(keyPair._2, puz, difficulty, ticket, rootHash)) {
+    if (validate(keyPair._2, puz, target, ticket, rootHash)) {
       val timestamp = NTP.correctedTime()
-      val consensusData = PermaLikeConsensusBlockData(difficulty, puz, ticket)
+      val consensusData = PermaLikeConsensusBlockData(target, puz, ticket)
 
       Future(Some(Block.buildAndSign(Version,
         timestamp,
@@ -111,7 +111,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
   override def genesisData: PermaConsensusBlockField =
     PermaConsensusBlockField(PermaLikeConsensusBlockData(
-      InitialDifficulty,
+      InitialTarget,
       Array[Byte](),
       Ticket(GenesisCreator.publicKey, Array(), IndexedSeq())
     ))
@@ -124,7 +124,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
   //todo: validate r\i
   private def validate(publicKey: PublicKey,
                        puz: Array[Byte],
-                       difficulty: BigInt,
+                       target: BigInt,
                        t: Ticket,
                        rootHash: Digest): Boolean = Try {
     val proofs = t.proofs
@@ -143,7 +143,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
         EllipticCurveImpl.verify(sigs(i), hi, publicKey)
       }
     }
-    partialProofsCheck && (ticketScore(t) < difficulty)
+    partialProofsCheck && (ticketScore(t) < target)
   }.getOrElse(false)
 
   private def generatePuz(block: Block) = hash.hash(block.bytes)
@@ -186,28 +186,28 @@ class PermaConsensusModule(rootHash: Array[Byte])
     BigInt(1, h).mod(Constants.n).toLong
   }
 
-  private def calcDifficulty(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
+  private def calcTarget(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
     val trans = transactionModule.history.asInstanceOf[BlockChain]
-    lazy val currentDiff = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].difficulty
+    lazy val currentTarget = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].target
     trans.heightOf(block) match {
       case Some(height) =>
-        if (height < DifficultyRecalculation + 1) {
-          InitialDifficulty
-        } else if (height % DifficultyRecalculation == 0) {
-          val lastBlocks = (0 until DifficultyRecalculation).flatMap(i => trans.blockAt(height - i)).reverse
-          require(lastBlocks.length == DifficultyRecalculation)
-          val lastAvgDuration: Long = (0 until DifficultyRecalculation - 1).map { i =>
+        if (height < TargetRecalculation + 1) {
+          InitialTarget
+        } else if (height % TargetRecalculation == 0) {
+          val lastBlocks = (0 until TargetRecalculation).flatMap(i => trans.blockAt(height - i)).reverse
+          require(lastBlocks.length == TargetRecalculation)
+          val lastAvgDuration: Long = (0 until TargetRecalculation - 1).map { i =>
             lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value
-          }.sum / (DifficultyRecalculation - 1)
-          val newDiff = currentDiff * lastAvgDuration / 1000 / AvgDelay
-          log.debug(s"Height: $height, newDiff: $newDiff, currentDiff:$currentDiff, lastAvgDuration:$lastAvgDuration")
-          newDiff
+          }.sum / (TargetRecalculation - 1)
+          val newTarget = currentTarget * lastAvgDuration / 1000 / AvgDelay
+          log.debug(s"Height: $height, target:$newTarget vs $currentTarget, lastAvgDuration:$lastAvgDuration")
+          newTarget
         } else {
-          currentDiff
+          currentTarget
         }
       case None =>
         log.warn(s"Enable to get height of block ${block.uniqueId}")
-        currentDiff
+        currentTarget
     }
   }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
index 02fcda4e..e8c17444 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
@@ -3,7 +3,7 @@ package scorex.perma.consensus
 import play.api.libs.functional.syntax._
 import play.api.libs.json._
 
-case class PermaLikeConsensusBlockData(difficulty: BigInt, puz: Array[Byte], ticket: Ticket)
+case class PermaLikeConsensusBlockData(target: BigInt, puz: Array[Byte], ticket: Ticket)
 
 object PermaLikeConsensusBlockData {
   implicit val bigIntWrites = new Writes[BigInt] {
diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
index 45ad3957..3a874453 100644
--- a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
+++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
@@ -41,7 +41,7 @@ class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks
   }
 
   def checkAll(initialBlock: PermaConsensusBlockField, parsedBlock: PermaConsensusBlockField): Unit = {
-    parsedBlock.value.difficulty shouldBe initialBlock.value.difficulty
+    parsedBlock.value.target shouldBe initialBlock.value.target
     assert(parsedBlock.value.puz sameElements initialBlock.value.puz)
     assert(parsedBlock.value.ticket.publicKey sameElements initialBlock.value.ticket.publicKey)
     assert(parsedBlock.value.ticket.s sameElements initialBlock.value.ticket.s)
diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
index cbf19b51..7eeda14e 100644
--- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
+++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
@@ -19,7 +19,7 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
 
   private val stateTimeout = 1.second
   //TODO get from config
-  val blockGenerationDelay = 10.millis
+  val blockGenerationDelay = 1.hour
 
   startWith(Offline, Unit)
 

From 18d7e6345e0fa942bde1561cef9551dd2d384450 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 16:18:37 +0300
Subject: [PATCH 50/66] Move constants to config

---
 scorex-perma/src/main/resources/perma.conf    |  3 +++
 .../consensus/PermaConsensusModule.scala      | 22 +++++++++----------
 .../scorex/perma/settings/Constants.scala     |  6 +++++
 3 files changed, 20 insertions(+), 11 deletions(-)

diff --git a/scorex-perma/src/main/resources/perma.conf b/scorex-perma/src/main/resources/perma.conf
index 20ff02c6..cc235965 100644
--- a/scorex-perma/src/main/resources/perma.conf
+++ b/scorex-perma/src/main/resources/perma.conf
@@ -4,4 +4,7 @@ perma {
   l = 1024
   k = 4
   hash = "sha256"
+  initialTarget = "108789072715742611466048832036162432282119640008661078894418176744671493285459"
+  targetRecalculation = 10
+  averageDelay = 2
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 4bfcf18a..8d8ff019 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -25,14 +25,14 @@ class PermaConsensusModule(rootHash: Array[Byte])
                           (implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]])
   extends ConsensusModule[PermaLikeConsensusBlockData] with ScorexLogging {
 
-  val InitialTarget = BigInt("108789072715742611466048832036162432282119640008661078894418176744671493285459")
+  val InitialTarget = Constants.initialTarget
   val initialTargetPow: BigInt = log2(InitialTarget)
-  val TargetRecalculation = 10
-  val AvgDelay = 2.seconds.toSeconds
+  val TargetRecalculation = Constants.targetRecalculation
+  val AvgDelay = Constants.averageDelay
+  val Hash = Constants.hash
 
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
-  val hash = Constants.hash
 
   implicit val consensusModule: ConsensusModule[PermaLikeConsensusBlockData] = this
 
@@ -139,17 +139,17 @@ class PermaConsensusModule(rootHash: Array[Byte])
       val segment = proofs(i - 1).segment
 
       segment.check(ris(i - 1), rootHash)() || {
-        val hi = hash.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data)
+        val hi = Hash.hash(puz ++ publicKey ++ sigs(i - 1) ++ segment.data)
         EllipticCurveImpl.verify(sigs(i), hi, publicKey)
       }
     }
     partialProofsCheck && (ticketScore(t) < target)
   }.getOrElse(false)
 
-  private def generatePuz(block: Block) = hash.hash(block.bytes)
+  private def generatePuz(block: Block) = Hash.hash(block.bytes)
 
   private def ticketScore(t: Ticket): BigInt = if (t.proofs.nonEmpty) {
-    BigInt(1, hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
+    BigInt(1, Hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
   } else {
     //Genesis block contains empty ticket
     0
@@ -163,16 +163,16 @@ class PermaConsensusModule(rootHash: Array[Byte])
     val s = randomBytes(32)
 
     val sig0 = NoSig
-    val r1 = u(publicKey, (BigInt(1, hash.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
+    val r1 = u(publicKey, (BigInt(1, Hash.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
 
     val proofs: IndexedSeq[PartialProof] = 1.to(Constants.k).foldLeft(
       (r1, sig0, Seq[PartialProof]())
     ) {
       case ((ri, sig_prev, seq), _) =>
         val segment = authDataStorage.get(ri).get
-        val hi = hash.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
+        val hi = Hash.hash(puz ++ publicKey ++ sig_prev ++ segment.data)
         val sig = EllipticCurveImpl.sign(privateKey, hi)
-        val r_next = u(publicKey, BigInt(1, hash.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt)
+        val r_next = u(publicKey, BigInt(1, Hash.hash(puz ++ publicKey ++ sig)).mod(Constants.l).toInt)
 
         (r_next, sig, seq :+ PartialProof(sig, ri, segment))
     }._3.toIndexedSeq.ensuring(_.size == Constants.k)
@@ -182,7 +182,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
   //calculate index of i-th segment
   private def u(pubKey: PublicKey, i: Int): Long = {
-    val h = hash.hash(pubKey ++ BigInt(i).toByteArray)
+    val h = Hash.hash(pubKey ++ BigInt(i).toByteArray)
     BigInt(1, h).mod(Constants.n).toLong
   }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
index ab00e784..50f1cae6 100644
--- a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
@@ -19,6 +19,12 @@ object Constants extends ScorexLogging {
 
   val k = permaConf.getInt("k") //number of iterations during scratch-off phase
 
+  val initialTarget = BigInt(permaConf.getString("initialTarget"))
+
+  val targetRecalculation = permaConf.getInt("targetRecalculation")
+
+  val averageDelay = permaConf.getInt("averageDelay")
+
   val hash = permaConf.getString("hash") match {
     case s: String if s.equalsIgnoreCase("sha256") =>
       Sha256

From 7145ff56e117b3d01684775393d67a0cb832bdd9 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 16:28:16 +0300
Subject: [PATCH 51/66] Use nonce size = hash.valueSize

---
 .../scala/scorex/perma/consensus/PermaConsensusModule.scala  | 5 +++--
 .../src/main/scala/scorex/perma/settings/Constants.scala     | 4 ++--
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 8d8ff019..126c2d12 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -15,7 +15,6 @@ import scorex.utils._
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
-import scala.concurrent.duration._
 import scala.util.Try
 
 /**
@@ -30,6 +29,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
   val TargetRecalculation = Constants.targetRecalculation
   val AvgDelay = Constants.averageDelay
   val Hash = Constants.hash
+  val SSize = Hash.ValueSize
 
   val GenesisCreator = new PublicKeyAccount(Array())
   val Version: Byte = 1
@@ -129,6 +129,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
                        rootHash: Digest): Boolean = Try {
     val proofs = t.proofs
     require(proofs.size == Constants.k)
+    require(t.s.length <= SSize)
 
     //Local-POR lottery verification
 
@@ -160,7 +161,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
     val (privateKey, publicKey) = keyPair
 
     //scratch-off for the Local-POR lottery
-    val s = randomBytes(32)
+    val s = randomBytes(SSize)
 
     val sig0 = NoSig
     val r1 = u(publicKey, (BigInt(1, Hash.hash(puz ++ publicKey ++ s)) % Constants.l).toInt)
diff --git a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
index 50f1cae6..34c98b65 100644
--- a/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/settings/Constants.scala
@@ -21,9 +21,9 @@ object Constants extends ScorexLogging {
 
   val initialTarget = BigInt(permaConf.getString("initialTarget"))
 
-  val targetRecalculation = permaConf.getInt("targetRecalculation")
+  val targetRecalculation = permaConf.getInt("targetRecalculation") //recalculate target every targetRecalculation blocks
 
-  val averageDelay = permaConf.getInt("averageDelay")
+  val averageDelay = permaConf.getInt("averageDelay") //average delay between blocks in seconds
 
   val hash = permaConf.getString("hash") match {
     case s: String if s.equalsIgnoreCase("sha256") =>

From f3fd7b45cf293305ef17fe1cbfda6e9bf24833c1 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 17:07:28 +0300
Subject: [PATCH 52/66] more debug

---
 .../src/main/scala/scorex/block/Block.scala   | 44 +++++++++++--------
 .../consensus/PermaConsensusModule.scala      | 12 ++++-
 .../lagonaki/network/BlockchainSyncer.scala   |  1 +
 3 files changed, 37 insertions(+), 20 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/block/Block.scala b/scorex-basics/src/main/scala/scorex/block/Block.scala
index c3956eac..5c441163 100644
--- a/scorex-basics/src/main/scala/scorex/block/Block.scala
+++ b/scorex-basics/src/main/scala/scorex/block/Block.scala
@@ -12,22 +12,22 @@ import scorex.utils.ScorexLogging
 import scala.util.{Failure, Try}
 
 /**
- * A block is a an atomic piece of data network participates are agreed on.
- *
- * A block has:
- * - transactions data: a sequence of transactions, where a transaction is an atomic state update.
- * Some metadata is possible as well(transactions Merkle tree root, state Merkle tree root etc).
- *
- * - consensus data to check whether block was generated by a right party in a right way. E.g.
- * "baseTarget" & "generatorSignature" fields in the Nxt block structure, nonce & difficulty in the
- * Bitcoin block structure.
- *
- * - a signature(s) of a block generator(s)
- *
- * - additional data: block structure version no, timestamp etc
- */
-
-trait Block {
+  * A block is a an atomic piece of data network participates are agreed on.
+  *
+  * A block has:
+  * - transactions data: a sequence of transactions, where a transaction is an atomic state update.
+  * Some metadata is possible as well(transactions Merkle tree root, state Merkle tree root etc).
+  *
+  * - consensus data to check whether block was generated by a right party in a right way. E.g.
+  * "baseTarget" & "generatorSignature" fields in the Nxt block structure, nonce & difficulty in the
+  * Bitcoin block structure.
+  *
+  * - a signature(s) of a block generator(s)
+  *
+  * - additional data: block structure version no, timestamp etc
+  */
+
+trait Block extends ScorexLogging {
   type ConsensusDataType
   type TransactionDataType
 
@@ -79,13 +79,21 @@ trait Block {
 
   lazy val bytesWithoutSignature = bytes.dropRight(EllipticCurveImpl.SignatureLength)
 
-  def isValid =
-    consensusModule.isValid(this) &&
+  def isValid: Boolean = {
+    val v = consensusModule.isValid(this) &&
       transactionModule.isValid(this) &&
       transactionModule.history.contains(referenceField.value) &&
       EllipticCurveImpl.verify(signerDataField.value.signature,
         bytesWithoutSignature,
         signerDataField.value.generator.publicKey)
+    if (!v) log.debug(
+      s"Block checks: ${consensusModule.isValid(this)} && ${transactionModule.isValid(this)} && " +
+        s"${transactionModule.history.contains(referenceField.value)} && " +
+        EllipticCurveImpl.verify(signerDataField.value.signature, bytesWithoutSignature,
+          signerDataField.value.generator.publicKey)
+    )
+    v
+  }
 }
 
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 126c2d12..47c1567a 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -47,9 +47,17 @@ class PermaConsensusModule(rootHash: Array[Byte])
       case Some(parent) =>
         lazy val publicKey = blockGenerator(block).publicKey
         lazy val puzIsValid = f.value.puz sameElements generatePuz(parent)
-        lazy val target = f.value.target == calcTarget(parent)
+        lazy val targetIsValid = f.value.target == calcTarget(parent)
         lazy val ticketIsValid = validate(publicKey, f.value.puz, f.value.target, f.value.ticket, rootHash)
-        puzIsValid && target && ticketIsValid
+        if (puzIsValid && targetIsValid && ticketIsValid)
+          true
+        else {
+          log.debug(
+            s"Non-valid block: puzIsValid=$puzIsValid, targetIsValid=$targetIsValid && ticketIsValid=$ticketIsValid"
+          )
+          false
+        }
+
       case None =>
         true
     }
diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
index 7eeda14e..71cbb1b5 100644
--- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
+++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
@@ -4,6 +4,7 @@ import java.net.InetSocketAddress
 
 import akka.actor.{ActorRef, FSM}
 import scorex.block.Block
+import scorex.crypto.EllipticCurveImpl
 import scorex.lagonaki.network.BlockchainSyncer._
 import scorex.lagonaki.network.message.{BlockMessage, GetSignaturesMessage}
 import scorex.lagonaki.server.LagonakiApplication

From 533c49e7b1e23173902b3d8a56e9cc0e9c5af117 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 18:12:55 +0300
Subject: [PATCH 53/66] Move average delay to parent ConsensusModule trait

---
 .../scorex/consensus/ConsensusModule.scala    | 36 +++++++++++++-----
 .../consensus/PermaConsensusModule.scala      | 37 ++++++++-----------
 2 files changed, 42 insertions(+), 31 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
index e8a3056d..7222e123 100644
--- a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
+++ b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
@@ -2,28 +2,29 @@ package scorex.consensus
 
 import scorex.account.{Account, PrivateKeyAccount}
 import scorex.block.{Block, BlockProcessingModule}
-import scorex.transaction.TransactionModule
+import scorex.transaction.{BlockChain, TransactionModule}
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
+import scala.util.Try
 
 
-trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[ConsensusBlockData]{
+trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[ConsensusBlockData] {
 
   def isValid[TT](block: Block)(implicit transactionModule: TransactionModule[TT]): Boolean
 
   /**
-  * Fees could go to a single miner(forger) usually, but can go to many parties, e.g. see
-  * Meni Rosenfeld's Proof-of-Activity proposal http://eprint.iacr.org/2014/452.pdf
-   */
+    * Fees could go to a single miner(forger) usually, but can go to many parties, e.g. see
+    * Meni Rosenfeld's Proof-of-Activity proposal http://eprint.iacr.org/2014/452.pdf
+    */
   def feesDistribution(block: Block): Map[Account, Long]
 
   /**
-   * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not
-   * (see e.g. Meni Rosenfeld's Proof-of-Activity paper http://eprint.iacr.org/2014/452.pdf)
-   * @param block
-   * @return
-   */
+    * Get block producers(miners/forgers). Usually one miner produces a block, but in some proposals not
+    * (see e.g. Meni Rosenfeld's Proof-of-Activity paper http://eprint.iacr.org/2014/452.pdf)
+    * @param block
+    * @return
+    */
   def generators(block: Block): Seq[Account]
 
   def blockScore(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt
@@ -37,4 +38,19 @@ trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[Consensu
   }
 
   def consensusBlockData(block: Block): ConsensusBlockData
+
+  /**
+    * Average delay between last $blockNum blocks starting from $block
+    */
+  def averageDelay(block: Block, blockNum: Int)
+                  (implicit transactionModule: TransactionModule[_]): Try[Long] = Try {
+    val trans = transactionModule.history.asInstanceOf[BlockChain]
+    val height: Int = trans.heightOf(block).get
+    val lastBlocks = (0 until blockNum).flatMap(i => trans.blockAt(height - i)).reverse
+    require(lastBlocks.length == blockNum)
+    (0 until blockNum - 1).map { i =>
+      lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value
+    }.sum / (blockNum - 1)
+  }
+
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 47c1567a..603fdcfc 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -15,7 +15,7 @@ import scorex.utils._
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
-import scala.util.Try
+import scala.util.{Success, Try}
 
 /**
   * Data and functions related to a consensus algo
@@ -197,27 +197,22 @@ class PermaConsensusModule(rootHash: Array[Byte])
 
   private def calcTarget(block: Block)(implicit transactionModule: TransactionModule[_]): BigInt = {
     val trans = transactionModule.history.asInstanceOf[BlockChain]
-    lazy val currentTarget = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].target
-    trans.heightOf(block) match {
-      case Some(height) =>
-        if (height < TargetRecalculation + 1) {
-          InitialTarget
-        } else if (height % TargetRecalculation == 0) {
-          val lastBlocks = (0 until TargetRecalculation).flatMap(i => trans.blockAt(height - i)).reverse
-          require(lastBlocks.length == TargetRecalculation)
-          val lastAvgDuration: Long = (0 until TargetRecalculation - 1).map { i =>
-            lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value
-          }.sum / (TargetRecalculation - 1)
-          val newTarget = currentTarget * lastAvgDuration / 1000 / AvgDelay
-          log.debug(s"Height: $height, target:$newTarget vs $currentTarget, lastAvgDuration:$lastAvgDuration")
-          newTarget
-        } else {
-          currentTarget
-        }
-      case None =>
-        log.warn(s"Enable to get height of block ${block.uniqueId}")
+    val currentTarget = block.consensusDataField.value.asInstanceOf[PermaLikeConsensusBlockData].target
+    Try {
+      val height = trans.heightOf(block).get
+      if (height % TargetRecalculation == 0 && height > TargetRecalculation) {
+        val lastAvgDuration: BigInt = averageDelay(block, TargetRecalculation).get
+        val newTarget = currentTarget * lastAvgDuration / 1000 / AvgDelay
+        log.debug(s"Height: $height, target:$newTarget vs $currentTarget, lastAvgDuration:$lastAvgDuration")
+        newTarget
+      } else {
         currentTarget
-    }
+      }
+    }.recoverWith { case t: Throwable =>
+      log.error(s"Error when calculating target: ${t.getMessage}")
+      t.printStackTrace()
+      Success(currentTarget)
+    }.getOrElse(currentTarget)
   }
 
   private def log2(i: BigInt): BigInt = BigDecimal(math.log(i.doubleValue()) / math.log(2)).toBigInt()

From 81edc6e956ead3df4820400d7194ed0d1d4b9eb8 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 18:21:42 +0300
Subject: [PATCH 54/66] Some consensus API

---
 .../nxt/api/http/NxtConsensusApiRoute.scala   |  1 -
 .../http/PermaConsensusApiRoute.scala         | 32 ++++++++++++++++++-
 2 files changed, 31 insertions(+), 2 deletions(-)

diff --git a/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala b/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala
index 3cb766f2..23df429a 100644
--- a/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala
+++ b/scorex-consensus/src/main/scala/scorex/consensus/nxt/api/http/NxtConsensusApiRoute.scala
@@ -79,7 +79,6 @@ class NxtConsensusApiRoute(consensusModule: NxtLikeConsensusModule, blockchain:
         Json.obj("base-target" -> bt).toString()
       }
     }
-
   }
 
   @Path("/algo")
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
index e0b60a90..d98e572d 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
@@ -18,9 +18,39 @@ class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain:
 
   override val route: Route =
     pathPrefix("consensus") {
-      algo
+      algo ~ target
     }
 
+  @Path("/target")
+  @ApiOperation(value = "Last target", notes = "Target of a last block", httpMethod = "GET")
+  def target = {
+    path("target") {
+      jsonRoute {
+        val lastBlock = blockchain.lastBlock
+        val bt = consensusModule.consensusBlockData(lastBlock).target
+        Json.obj("target" -> bt.toString).toString
+      }
+    }
+  }
+
+  @Path("/target/{blockId}")
+  @ApiOperation(value = "Target of selected block", notes = "Target of a block with specified id", httpMethod = "GET")
+  @ApiImplicitParams(Array(
+    new ApiImplicitParam(name = "blockId", value = "Block id ", required = true, dataType = "String", paramType = "path")
+  ))
+  def baseTargetId = {
+    path("target" / Segment) { case encodedSignature =>
+      jsonRoute {
+        withBlock(blockchain, encodedSignature) { block =>
+          Json.obj(
+            "target" -> consensusModule.consensusBlockData(block).target.toString
+          )
+        }.toString
+      }
+    }
+  }
+
+
   @Path("/algo")
   @ApiOperation(value = "Consensus algo", notes = "Shows which consensus algo being using", httpMethod = "GET")
   def algo = {

From d95bd35027705f77fc9aead9f97f5f0aa5093fe2 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 19:41:06 +0300
Subject: [PATCH 55/66] Move Json serialization to separate trait

---
 .../scorex/utils/JsonSerialization.scala      | 22 +++++++++++++++++++
 .../PermaLikeConsensusBlockData.scala         | 17 ++------------
 2 files changed, 24 insertions(+), 15 deletions(-)
 create mode 100644 scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala

diff --git a/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala b/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala
new file mode 100644
index 00000000..33f34946
--- /dev/null
+++ b/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala
@@ -0,0 +1,22 @@
+package scorex.utils
+
+import play.api.libs.json._
+
+trait JsonSerialization {
+
+  implicit val bigIntWrites = new Writes[BigInt] {
+    def writes(bitInt: BigInt) = JsString(bitInt.toString)
+  }
+
+  implicit def bigIntReads: Reads[BigInt] = new Reads[BigInt] {
+    def reads(json: JsValue): JsResult[BigInt] = json match {
+      case JsString(bigint) =>
+        JsSuccess(BigInt(bigint))
+      case JsNumber(bigint) =>
+        JsSuccess(BigInt(bigint.toString))
+      case m =>
+        throw new RuntimeException(s"Bigint MUST be represented as string in json $m ${m.getClass} given")
+    }
+  }
+
+}
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
index e8c17444..e6ad8291 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
@@ -2,24 +2,11 @@ package scorex.perma.consensus
 
 import play.api.libs.functional.syntax._
 import play.api.libs.json._
+import scorex.utils.JsonSerialization
 
 case class PermaLikeConsensusBlockData(target: BigInt, puz: Array[Byte], ticket: Ticket)
 
-object PermaLikeConsensusBlockData {
-  implicit val bigIntWrites = new Writes[BigInt] {
-    def writes(bitInt: BigInt) = JsString(bitInt.toString)
-  }
-
-  implicit def bigIntReads: Reads[BigInt] = new Reads[BigInt] {
-    def reads(json: JsValue): JsResult[BigInt] = json match {
-      case JsString(bigint) =>
-        JsSuccess(BigInt(bigint))
-      case JsNumber(bigint) =>
-        JsSuccess(BigInt(bigint.toString))
-      case m =>
-        throw new RuntimeException(s"Bigint MUST be represented as string in json $m ${m.getClass} given")
-    }
-  }
+object PermaLikeConsensusBlockData extends JsonSerialization {
 
   implicit val writes: Writes[PermaLikeConsensusBlockData] = (
     (JsPath \ "difficulty").write[BigInt] and

From ccb28c0356c1b750abc7fc7fde227a12956cea5a Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sat, 28 Nov 2015 19:52:11 +0300
Subject: [PATCH 56/66] Serialize Array[Byte]

---
 .../scorex/utils/JsonSerialization.scala      | 24 +++++++++++++++++++
 .../scorex/perma/consensus/PartialProof.scala |  7 +++---
 .../PermaLikeConsensusBlockData.scala         |  4 ++--
 .../scala/scorex/perma/consensus/Ticket.scala |  7 +++---
 4 files changed, 34 insertions(+), 8 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala b/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala
index 33f34946..79a7204d 100644
--- a/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala
+++ b/scorex-basics/src/main/scala/scorex/utils/JsonSerialization.scala
@@ -1,9 +1,33 @@
 package scorex.utils
 
 import play.api.libs.json._
+import scorex.crypto.Base58
+
+import scala.util.{Failure, Success}
 
 trait JsonSerialization {
 
+  type Bytes = Array[Byte]
+
+  implicit val bytesWrites = new Writes[Bytes] {
+    def writes(bytes: Bytes) = JsString(Base58.encode(bytes))
+  }
+
+  implicit def bytesReads: Reads[Bytes] = new Reads[Bytes] {
+    def reads(json: JsValue): JsResult[Bytes] = json match {
+      case JsString(encoded) =>
+        Base58.decode(encoded) match {
+          case Success(decoded) =>
+            JsSuccess(decoded)
+          case Failure(e) =>
+            throw new RuntimeException(s"Failed to parse Base58 encoded bytes")
+        }
+      case m =>
+        throw new RuntimeException(s"Bigint MUST be represented as string in json $m ${m.getClass} given")
+    }
+  }
+
+
   implicit val bigIntWrites = new Writes[BigInt] {
     def writes(bitInt: BigInt) = JsString(bitInt.toString)
   }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
index bbd15055..fcfde331 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
@@ -5,18 +5,19 @@ import play.api.libs.json.{JsPath, Reads, Writes}
 import scorex.crypto.SigningFunctions._
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.settings.Constants._
+import scorex.utils.JsonSerialization
 
 case class PartialProof(signature: Signature, segmentIndex: Long, segment: AuthDataBlock[DataSegment])
 
-object PartialProof {
+object PartialProof extends JsonSerialization {
   implicit val writes: Writes[PartialProof] = (
-    (JsPath \ "signature").write[Signature] and
+    (JsPath \ "signature").write[Bytes] and
       (JsPath \ "segmentIndex").write[Long] and
       (JsPath \ "segment").write[AuthDataBlock[DataSegment]]
     ) (unlift(PartialProof.unapply))
 
   implicit val reads: Reads[PartialProof] = (
-    (JsPath \ "signature").read[Signature] and
+    (JsPath \ "signature").read[Bytes] and
       (JsPath \ "segmentIndex").read[Long] and
       (JsPath \ "segment").read[AuthDataBlock[DataSegment]]
     ) (PartialProof.apply _)
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
index e6ad8291..8661c1a6 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaLikeConsensusBlockData.scala
@@ -10,13 +10,13 @@ object PermaLikeConsensusBlockData extends JsonSerialization {
 
   implicit val writes: Writes[PermaLikeConsensusBlockData] = (
     (JsPath \ "difficulty").write[BigInt] and
-      (JsPath \ "puz").write[Array[Byte]] and
+      (JsPath \ "puz").write[Bytes] and
       (JsPath \ "ticket").write[Ticket]
     ) (unlift(PermaLikeConsensusBlockData.unapply))
 
   implicit val reads: Reads[PermaLikeConsensusBlockData] = (
     (JsPath \ "difficulty").read[BigInt] and
-      (JsPath \ "puz").read[Array[Byte]] and
+      (JsPath \ "puz").read[Bytes] and
       (JsPath \ "ticket").read[Ticket]
     ) (PermaLikeConsensusBlockData.apply _)
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala
index d7031b7e..a1cbc710 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/Ticket.scala
@@ -3,22 +3,23 @@ package scorex.perma.consensus
 import play.api.libs.functional.syntax._
 import play.api.libs.json.{JsPath, Reads, Writes}
 import scorex.crypto.SigningFunctions._
+import scorex.utils.JsonSerialization
 
 case class Ticket(publicKey: PublicKey,
                   s: Array[Byte],
                   proofs: IndexedSeq[PartialProof])
 
 
-object Ticket {
+object Ticket extends JsonSerialization {
   implicit val writes: Writes[Ticket] = (
     (JsPath \ "publicKey").write[PublicKey] and
-      (JsPath \ "s").write[Array[Byte]] and
+      (JsPath \ "s").write[Bytes] and
       (JsPath \ "proofs").write[IndexedSeq[PartialProof]]
     ) (unlift(Ticket.unapply))
 
   implicit val reads: Reads[Ticket] = (
     (JsPath \ "publicKey").read[PublicKey] and
-      (JsPath \ "s").read[Array[Byte]] and
+      (JsPath \ "s").read[Bytes] and
       (JsPath \ "proofs").read[IndexedSeq[PartialProof]]
     ) (Ticket.apply _)
 

From 221013d8f85d39a43aaa1adcf1bbcde73b980f71 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sun, 29 Nov 2015 13:15:18 +0300
Subject: [PATCH 57/66] Puz to API route

---
 .../consensus/PermaConsensusModule.scala      |  4 +--
 .../http/PermaConsensusApiRoute.scala         | 36 ++++++++++++++++---
 2 files changed, 33 insertions(+), 7 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 603fdcfc..a7927f96 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -127,6 +127,8 @@ class PermaConsensusModule(rootHash: Array[Byte])
   override def formBlockData(data: PermaLikeConsensusBlockData): BlockField[PermaLikeConsensusBlockData] =
     PermaConsensusBlockField(data)
 
+  def generatePuz(block: Block) = Hash.hash(block.bytes)
+
   private val NoSig = Array[Byte]()
 
   //todo: validate r\i
@@ -155,8 +157,6 @@ class PermaConsensusModule(rootHash: Array[Byte])
     partialProofsCheck && (ticketScore(t) < target)
   }.getOrElse(false)
 
-  private def generatePuz(block: Block) = Hash.hash(block.bytes)
-
   private def ticketScore(t: Ticket): BigInt = if (t.proofs.nonEmpty) {
     BigInt(1, Hash.hash(t.proofs.map(_.signature).reduce(_ ++ _)))
   } else {
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
index d98e572d..093fcc12 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/http/PermaConsensusApiRoute.scala
@@ -6,6 +6,7 @@ import akka.actor.ActorRefFactory
 import com.wordnik.swagger.annotations._
 import play.api.libs.json.Json
 import scorex.api.http.{ApiRoute, CommonApiFunctions}
+import scorex.crypto.Base58
 import scorex.perma.consensus.PermaConsensusModule
 import scorex.transaction.BlockChain
 import spray.routing.Route
@@ -18,7 +19,7 @@ class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain:
 
   override val route: Route =
     pathPrefix("consensus") {
-      algo ~ target
+      algo ~ target ~ targetId ~ puz ~ puzId
     }
 
   @Path("/target")
@@ -26,9 +27,7 @@ class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain:
   def target = {
     path("target") {
       jsonRoute {
-        val lastBlock = blockchain.lastBlock
-        val bt = consensusModule.consensusBlockData(lastBlock).target
-        Json.obj("target" -> bt.toString).toString
+        Json.obj("target" -> consensusModule.consensusBlockData(blockchain.lastBlock).target.toString).toString
       }
     }
   }
@@ -38,7 +37,7 @@ class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain:
   @ApiImplicitParams(Array(
     new ApiImplicitParam(name = "blockId", value = "Block id ", required = true, dataType = "String", paramType = "path")
   ))
-  def baseTargetId = {
+  def targetId = {
     path("target" / Segment) { case encodedSignature =>
       jsonRoute {
         withBlock(blockchain, encodedSignature) { block =>
@@ -50,6 +49,33 @@ class PermaConsensusApiRoute(consensusModule: PermaConsensusModule, blockchain:
     }
   }
 
+  @Path("/puz")
+  @ApiOperation(value = "Current puzzle", notes = "Current puzzle", httpMethod = "GET")
+  def puz = {
+    path("puz") {
+      jsonRoute {
+        Json.obj("puz" -> Base58.encode(consensusModule.generatePuz(blockchain.lastBlock))).toString
+      }
+    }
+  }
+
+  @Path("/puz/{blockId}")
+  @ApiOperation(value = "Puzzle of selected block", notes = "Puzzle of a block with specified id", httpMethod = "GET")
+  @ApiImplicitParams(Array(
+    new ApiImplicitParam(name = "blockId", value = "Block id ", required = true, dataType = "String", paramType = "path")
+  ))
+  def puzId = {
+    path("puz" / Segment) { case encodedSignature =>
+      jsonRoute {
+        withBlock(blockchain, encodedSignature) { block =>
+          Json.obj(
+            "puz" -> Base58.encode(consensusModule.consensusBlockData(block).puz)
+          )
+        }.toString
+      }
+    }
+  }
+
 
   @Path("/algo")
   @ApiOperation(value = "Consensus algo", notes = "Shows which consensus algo being using", httpMethod = "GET")

From d2836e156999a4f0181630702131affbbd5c0a79 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sun, 29 Nov 2015 13:34:08 +0300
Subject: [PATCH 58/66] Move average delay to blockchain

---
 .../scorex/consensus/ConsensusModule.scala      | 17 +----------------
 .../scala/scorex/transaction/BlockChain.scala   | 14 ++++++++++++++
 .../perma/consensus/PermaConsensusModule.scala  |  2 +-
 3 files changed, 16 insertions(+), 17 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
index 7222e123..6463d1bf 100644
--- a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
+++ b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
@@ -2,11 +2,10 @@ package scorex.consensus
 
 import scorex.account.{Account, PrivateKeyAccount}
 import scorex.block.{Block, BlockProcessingModule}
-import scorex.transaction.{BlockChain, TransactionModule}
+import scorex.transaction.TransactionModule
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.Future
-import scala.util.Try
 
 
 trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[ConsensusBlockData] {
@@ -39,18 +38,4 @@ trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[Consensu
 
   def consensusBlockData(block: Block): ConsensusBlockData
 
-  /**
-    * Average delay between last $blockNum blocks starting from $block
-    */
-  def averageDelay(block: Block, blockNum: Int)
-                  (implicit transactionModule: TransactionModule[_]): Try[Long] = Try {
-    val trans = transactionModule.history.asInstanceOf[BlockChain]
-    val height: Int = trans.heightOf(block).get
-    val lastBlocks = (0 until blockNum).flatMap(i => trans.blockAt(height - i)).reverse
-    require(lastBlocks.length == blockNum)
-    (0 until blockNum - 1).map { i =>
-      lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value
-    }.sum / (blockNum - 1)
-  }
-
 }
diff --git a/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala b/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala
index 6ee6633b..34a5559a 100644
--- a/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala
+++ b/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala
@@ -3,6 +3,8 @@ package scorex.transaction
 import scorex.block.Block
 import scorex.utils.ScorexLogging
 
+import scala.util.Try
+
 trait BlockChain extends History with ScorexLogging {
 
   def blockAt(height: Int): Option[Block]
@@ -28,6 +30,18 @@ trait BlockChain extends History with ScorexLogging {
     }
   }
 
+  /**
+    * Average delay between last $blockNum blocks starting from $block
+    */
+  def averageDelay(block: Block, blockNum: Int): Try[Long] = Try {
+    val height: Int = heightOf(block).get
+    val lastBlocks = (0 until blockNum).flatMap(i => blockAt(height - i)).reverse
+    require(lastBlocks.length == blockNum)
+    (0 until blockNum - 1).map { i =>
+      lastBlocks(i + 1).timestampField.value - lastBlocks(i).timestampField.value
+    }.sum / (blockNum - 1)
+  }
+
   def lastSignature(): Block.BlockId = lastBlock.uniqueId
 
   def removeAfter(signature: Block.BlockId) = while (!lastSignature().sameElements(signature)) discardBlock()
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index a7927f96..6ec404bf 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -201,7 +201,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
     Try {
       val height = trans.heightOf(block).get
       if (height % TargetRecalculation == 0 && height > TargetRecalculation) {
-        val lastAvgDuration: BigInt = averageDelay(block, TargetRecalculation).get
+        val lastAvgDuration: BigInt = trans.averageDelay(block, TargetRecalculation).get
         val newTarget = currentTarget * lastAvgDuration / 1000 / AvgDelay
         log.debug(s"Height: $height, target:$newTarget vs $currentTarget, lastAvgDuration:$lastAvgDuration")
         newTarget

From 5583e01b0aebb4e5348478d85dee5eae683cc9b4 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sun, 29 Nov 2015 13:57:06 +0300
Subject: [PATCH 59/66] Blocks delay to API

---
 .../scala/scorex/transaction/BlockChain.scala |  2 +-
 .../scorex/api/http/BlocksApiRoute.scala      | 22 ++++++++++++++++++-
 2 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala b/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala
index 34a5559a..4ac6e45d 100644
--- a/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala
+++ b/scorex-basics/src/main/scala/scorex/transaction/BlockChain.scala
@@ -31,7 +31,7 @@ trait BlockChain extends History with ScorexLogging {
   }
 
   /**
-    * Average delay between last $blockNum blocks starting from $block
+    * Average delay in milliseconds between last $blockNum blocks starting from $block
     */
   def averageDelay(block: Block, blockNum: Int): Try[Long] = Try {
     val height: Int = heightOf(block).get
diff --git a/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala b/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala
index 68a7bf99..c2e76ac0 100644
--- a/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala
+++ b/scorex-transaction/src/main/scala/scorex/api/http/BlocksApiRoute.scala
@@ -16,7 +16,7 @@ case class BlocksApiRoute(blockchain: BlockChain, wallet: Wallet)(implicit val c
 
   override lazy val route =
     pathPrefix("blocks") {
-      signature ~ first ~ last ~ at ~ height ~ heightEncoded ~ child ~ address
+      signature ~ first ~ last ~ at ~ height ~ heightEncoded ~ child ~ address ~ delay
     }
 
   @Path("/address/{address}")
@@ -49,6 +49,26 @@ case class BlocksApiRoute(blockchain: BlockChain, wallet: Wallet)(implicit val c
     }
   }
 
+  @Path("/delay/{height}/{blockNum}")
+  @ApiOperation(value = "Average delay", notes = "Average delay in milliseconds between last $blockNum blocks starting from $height", httpMethod = "GET")
+  @ApiImplicitParams(Array(
+    new ApiImplicitParam(name = "height", value = "Height of block", required = true, dataType = "String", paramType = "path"),
+    new ApiImplicitParam(name = "blockNum", value = "Number of blocks to count delay", required = true, dataType = "String", paramType = "path")
+  ))
+  def delay: Route = {
+    path("delay" / IntNumber / IntNumber) { case (height, count) =>
+      jsonRoute {
+        blockchain.blockAt(height) match {
+          case Some(block) =>
+            blockchain.averageDelay(block, count).map(d => Json.obj("delay" -> d))
+              .getOrElse(Json.obj("status" -> "error", "details" -> "Internal error")).toString
+          case None =>
+            Json.obj("status" -> "error", "details" -> "No block for this height").toString()
+        }
+      }
+    }
+  }
+
   @Path("/height/{signature}")
   @ApiOperation(value = "Height", notes = "Get height of a block by its Base58-encoded signature", httpMethod = "GET")
   @ApiImplicitParams(Array(

From 68da42522d882a1139e5990e3b659a60cddc8033 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Sun, 29 Nov 2015 15:37:13 +0300
Subject: [PATCH 60/66] binary serialization

---
 scorex-perma/src/main/resources/perma.conf    |  2 +-
 .../consensus/PermaConsensusBlockField.scala  | 63 +++++++++++++++++--
 .../consensus/PermaConsensusModule.scala      |  9 +--
 .../src/test/resources/settings-test.json     |  5 ++
 ...ermaConsensusBlockFiendSpecification.scala | 48 ++++++++++----
 5 files changed, 104 insertions(+), 23 deletions(-)
 create mode 100644 scorex-perma/src/test/resources/settings-test.json

diff --git a/scorex-perma/src/main/resources/perma.conf b/scorex-perma/src/main/resources/perma.conf
index cc235965..204e3a75 100644
--- a/scorex-perma/src/main/resources/perma.conf
+++ b/scorex-perma/src/main/resources/perma.conf
@@ -4,7 +4,7 @@ perma {
   l = 1024
   k = 4
   hash = "sha256"
-  initialTarget = "108789072715742611466048832036162432282119640008661078894418176744671493285459"
+  initialTarget = "40878907271574261146604883203616243228211964000866107889441817674467149328545"
   targetRecalculation = 10
   averageDelay = 2
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 9a7f8576..bf0d734e 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -1,16 +1,35 @@
 package scorex.perma.consensus
 
+import com.google.common.primitives.{Bytes, Ints, Longs}
 import play.api.libs.json._
 import scorex.block.BlockField
-
+import scorex.crypto.ads.merkle.AuthDataBlock
+import scorex.perma.settings.Constants
 
 case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockData)
   extends BlockField[PermaLikeConsensusBlockData] {
 
+  import PermaConsensusBlockField._
 
   override val name: String = PermaConsensusBlockField.fieldName
 
-  override def bytes: Array[Byte] = json.toString().getBytes
+  override def bytes: Array[Byte] =
+    Bytes.ensureCapacity(Ints.toByteArray(value.target.toByteArray.length), 4, 0) ++ value.target.toByteArray ++
+      Bytes.ensureCapacity(value.puz, PuzLength, 0) ++
+      Bytes.ensureCapacity(value.ticket.publicKey, PublicKeyLength, 0) ++
+      Bytes.ensureCapacity(value.ticket.s, SLength, 0) ++
+      Bytes.ensureCapacity(Ints.toByteArray(value.ticket.proofs.length), 4, 0) ++
+      value.ticket.proofs.foldLeft(Array.empty: Array[Byte]) { (b, p) =>
+        val proofBytes =
+          Bytes.ensureCapacity(p.signature, SignatureLength, 0) ++
+            Bytes.ensureCapacity(Longs.toByteArray(p.segmentIndex), 8, 0) ++
+            Bytes.ensureCapacity(p.segment.data, Constants.segmentSize, 0) ++
+            Bytes.ensureCapacity(Ints.toByteArray(p.segment.merklePath.length), 4, 0) ++
+            p.segment.merklePath.foldLeft(Array.empty: Array[Byte]) { (acc, d) =>
+              acc ++ d
+            }
+        b ++ proofBytes
+      }
 
   override def json: JsObject = Json.obj(name -> Json.toJson(value))
 }
@@ -18,11 +37,43 @@ case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockD
 object PermaConsensusBlockField {
 
   val fieldName: String = "perma-consensus"
+  val PuzLength = 32
+  val PublicKeyLength = 32
+  val SLength = 32
+  val HashLength = 32
+  val DigestLength = 32
+  val SignatureLength = 64
+  //TODO FIX FOR non-first proof
+  val SingleProofSize = 0
 
   def parse(bytes: Array[Byte]): PermaConsensusBlockField = {
-    (Json.parse(bytes) \ fieldName).validate[PermaLikeConsensusBlockData] match {
-      case JsSuccess(block, _) => PermaConsensusBlockField(block)
-      case m => throw new RuntimeException("enable to parse block data")
-    }
+    val targetSize = Ints.fromByteArray(bytes.take(4))
+    val targetLength = 4 + targetSize
+    val proofsSize = Ints.fromByteArray(bytes.slice(
+      PuzLength + targetLength + PublicKeyLength + SLength, PuzLength + targetLength + PublicKeyLength + SLength + 4))
+
+    PermaConsensusBlockField(PermaLikeConsensusBlockData(
+      BigInt(bytes.slice(4, targetLength)),
+      bytes.slice(targetLength, PuzLength + targetLength),
+      Ticket(
+        bytes.slice(PuzLength + targetLength, PuzLength + targetLength + PublicKeyLength),
+        bytes.slice(PuzLength + targetLength + PublicKeyLength, PuzLength + targetLength + PublicKeyLength + SLength),
+        (0 until proofsSize).map { i =>
+          val proofsStart = PuzLength + targetLength + PublicKeyLength + SLength + 4 + i * SingleProofSize
+          val signatureStart = proofsStart + SignatureLength
+          val dataStart = signatureStart + 8
+          val merklePathStart = dataStart + Constants.segmentSize
+
+          val signature = bytes.slice(proofsStart, proofsStart + SignatureLength)
+          val signatureIndex = Longs.fromByteArray(bytes.slice(signatureStart, signatureStart + 8))
+          val blockData = bytes.slice(dataStart, dataStart + Constants.segmentSize)
+          val merklePathSize = Ints.fromByteArray(bytes.slice(merklePathStart, merklePathStart + 4))
+          val merklePath = (0 until merklePathSize).map { i =>
+            bytes.slice(merklePathStart + 4 + i * DigestLength, merklePathStart + 4 + (i + 1) * DigestLength)
+          }
+          PartialProof(signature, signatureIndex, AuthDataBlock(blockData, merklePath))
+        }
+      )
+    ))
   }
 }
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index 6ec404bf..bcabedac 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -30,8 +30,9 @@ class PermaConsensusModule(rootHash: Array[Byte])
   val AvgDelay = Constants.averageDelay
   val Hash = Constants.hash
   val SSize = Hash.ValueSize
+  require(SSize == PermaConsensusBlockField.SLength)
 
-  val GenesisCreator = new PublicKeyAccount(Array())
+  val GenesisCreator = new PublicKeyAccount(Array.fill(PermaConsensusBlockField.PublicKeyLength)(0: Byte))
   val Version: Byte = 1
 
   implicit val consensusModule: ConsensusModule[PermaLikeConsensusBlockData] = this
@@ -120,8 +121,8 @@ class PermaConsensusModule(rootHash: Array[Byte])
   override def genesisData: PermaConsensusBlockField =
     PermaConsensusBlockField(PermaLikeConsensusBlockData(
       InitialTarget,
-      Array[Byte](),
-      Ticket(GenesisCreator.publicKey, Array(), IndexedSeq())
+      Array.fill(PermaConsensusBlockField.PuzLength)(0: Byte),
+      Ticket(GenesisCreator.publicKey, Array.fill(PermaConsensusBlockField.SLength)(0: Byte), IndexedSeq())
     ))
 
   override def formBlockData(data: PermaLikeConsensusBlockData): BlockField[PermaLikeConsensusBlockData] =
@@ -139,7 +140,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
                        rootHash: Digest): Boolean = Try {
     val proofs = t.proofs
     require(proofs.size == Constants.k)
-    require(t.s.length <= SSize)
+    require(t.s.length == SSize)
 
     //Local-POR lottery verification
 
diff --git a/scorex-perma/src/test/resources/settings-test.json b/scorex-perma/src/test/resources/settings-test.json
new file mode 100644
index 00000000..026772fc
--- /dev/null
+++ b/scorex-perma/src/test/resources/settings-test.json
@@ -0,0 +1,5 @@
+{
+  "perma": {
+    "treeDir": "/tmp/scorex/perma/test/"
+  }
+}
\ No newline at end of file
diff --git a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
index 3a874453..030684c7 100644
--- a/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
+++ b/scorex-perma/src/test/scala/scorex/props/PermaConsensusBlockFiendSpecification.scala
@@ -1,40 +1,56 @@
 package scorex.props
 
+import java.io.File
+
 import org.scalatest.prop.{GeneratorDrivenPropertyChecks, PropertyChecks}
 import org.scalatest.{Matchers, PropSpec}
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.Storage.AuthDataStorage
 import scorex.perma.consensus._
 import scorex.perma.settings.Constants.DataSegment
-import scorex.perma.settings.PermaSettings
+import scorex.perma.settings.{Constants, PermaSettings}
 import scorex.settings.Settings
 import scorex.storage.Storage
+import scorex.utils._
 
 class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks with GeneratorDrivenPropertyChecks with Matchers {
 
   implicit val settings = new Settings with PermaSettings {
     val filename = "settings-test.json"
   }
-  implicit val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
-  val consensus = new PermaConsensusModule("test".getBytes)
+  new File(settings.treeDir).mkdirs()
+  implicit lazy val authDataStorage: Storage[Long, AuthDataBlock[DataSegment]] = new AuthDataStorage(settings.authDataStorage)
+  val consensus = new PermaConsensusModule(randomBytes(32))
 
-  property("set value and get it") {
-    forAll { (diff: Long, puz: Array[Byte], pubkey: Array[Byte], s: Array[Byte], signature: Array[Byte], segmentIndex: Long) =>
+  property("Encode to bytes round-trip") {
+    forAll { (diff: Long, segmentIndex: Long) =>
 
-      val blockdata = "nonempty".getBytes ++ puz
+      val puz = randomBytes(PermaConsensusBlockField.PuzLength)
+      val pubkey = randomBytes(PermaConsensusBlockField.PublicKeyLength)
+      val s = randomBytes(PermaConsensusBlockField.SLength)
+      val signature = randomBytes(PermaConsensusBlockField.SignatureLength)
+      val signature2 = randomBytes(PermaConsensusBlockField.SignatureLength)
+      val blockdata = randomBytes(Constants.segmentSize)
+      val hash1 = randomBytes(PermaConsensusBlockField.HashLength)
+      val hash2 = randomBytes(PermaConsensusBlockField.HashLength)
 
-      val authDataBlock: AuthDataBlock[DataSegment] = AuthDataBlock(blockdata, Seq(puz, pubkey))
+      val authDataBlock: AuthDataBlock[DataSegment] = AuthDataBlock(blockdata, Seq(hash1, hash2))
       val initialBlock = PermaConsensusBlockField(PermaLikeConsensusBlockData(
         math.abs(diff),
         puz,
-        Ticket(pubkey, s, IndexedSeq(PartialProof(signature, segmentIndex, authDataBlock)))
+        Ticket(pubkey, s, IndexedSeq(
+          PartialProof(signature, segmentIndex, authDataBlock),
+          PartialProof(signature2, segmentIndex, authDataBlock)
+        ))
       ))
       val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes)
 
       checkAll(initialBlock, parsedBlock)
 
     }
+  }
 
+  property("Encode to bytes round-trip for genesis") {
     val initialBlock = consensus.genesisData
     val parsedBlock = PermaConsensusBlockField.parse(initialBlock.bytes)
     checkAll(initialBlock, parsedBlock)
@@ -46,10 +62,18 @@ class PermaConsensusBlockFiendSpecification extends PropSpec with PropertyChecks
     assert(parsedBlock.value.ticket.publicKey sameElements initialBlock.value.ticket.publicKey)
     assert(parsedBlock.value.ticket.s sameElements initialBlock.value.ticket.s)
     parsedBlock.value.ticket.proofs.size shouldBe initialBlock.value.ticket.proofs.size
-    if(parsedBlock.value.ticket.proofs.nonEmpty) {
-      assert(parsedBlock.value.ticket.proofs.head.signature sameElements initialBlock.value.ticket.proofs.head.signature)
-      assert(parsedBlock.value.ticket.proofs.head.segmentIndex == initialBlock.value.ticket.proofs.head.segmentIndex)
-      assert(parsedBlock.value.ticket.proofs.head.segment.data sameElements initialBlock.value.ticket.proofs.head.segment.data)
+    parsedBlock.value.ticket.proofs.indices.foreach { i =>
+      val parsedProof = parsedBlock.value.ticket.proofs(i)
+      val initialProof = initialBlock.value.ticket.proofs(i)
+      assert(parsedProof.signature sameElements initialProof.signature)
+      parsedProof.segmentIndex shouldBe initialProof.segmentIndex
+      assert(parsedProof.segment.data sameElements initialProof.segment.data)
+      parsedProof.segment.merklePath.size shouldBe initialProof.segment.merklePath.size
+
+      if (initialProof.segment.merklePath.nonEmpty) {
+        assert(parsedProof.segment.merklePath.head sameElements initialProof.segment.merklePath.head)
+        assert(parsedProof.segment.merklePath.last sameElements initialProof.segment.merklePath.last)
+      }
     }
   }
 }
\ No newline at end of file

From 896e8b241181d9aa13a3f36bbf14f98ab1e4fca4 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 1 Dec 2015 00:50:41 +0300
Subject: [PATCH 61/66] Proofs binary parsing

---
 .../consensus/PermaConsensusBlockField.scala  | 49 ++++++++++++-------
 1 file changed, 31 insertions(+), 18 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index bf0d734e..71734ae1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -6,6 +6,8 @@ import scorex.block.BlockField
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.settings.Constants
 
+import scala.annotation.tailrec
+
 case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockData)
   extends BlockField[PermaLikeConsensusBlockData] {
 
@@ -41,38 +43,49 @@ object PermaConsensusBlockField {
   val PublicKeyLength = 32
   val SLength = 32
   val HashLength = 32
-  val DigestLength = 32
   val SignatureLength = 64
-  //TODO FIX FOR non-first proof
-  val SingleProofSize = 0
 
   def parse(bytes: Array[Byte]): PermaConsensusBlockField = {
+    @tailrec
+    def parseProofs(from: Int, total: Int, current: Int, acc: IndexedSeq[PartialProof]): IndexedSeq[PartialProof] = {
+      if (current < total) {
+        val proofsStart = from
+        val signatureStart = proofsStart + SignatureLength
+        val dataStart = signatureStart + 8
+        val merklePathStart = dataStart + Constants.segmentSize
+
+        val signature = bytes.slice(proofsStart, proofsStart + SignatureLength)
+        val signatureIndex = Longs.fromByteArray(bytes.slice(signatureStart, signatureStart + 8))
+        val blockData = bytes.slice(dataStart, dataStart + Constants.segmentSize)
+        val merklePathSize = Ints.fromByteArray(bytes.slice(merklePathStart, merklePathStart + 4))
+        val merklePath = (0 until merklePathSize).map { i =>
+          bytes.slice(merklePathStart + 4 + i * HashLength, merklePathStart + 4 + (i + 1) * HashLength)
+        }
+        parseProofs(
+          merklePathStart + 4 + merklePathSize * HashLength,
+          total,
+          current + 1,
+          PartialProof(signature, signatureIndex, AuthDataBlock(blockData, merklePath)) +: acc
+        )
+      } else {
+        acc.reverse
+      }
+    }
+
+
     val targetSize = Ints.fromByteArray(bytes.take(4))
     val targetLength = 4 + targetSize
     val proofsSize = Ints.fromByteArray(bytes.slice(
       PuzLength + targetLength + PublicKeyLength + SLength, PuzLength + targetLength + PublicKeyLength + SLength + 4))
 
+
     PermaConsensusBlockField(PermaLikeConsensusBlockData(
       BigInt(bytes.slice(4, targetLength)),
       bytes.slice(targetLength, PuzLength + targetLength),
       Ticket(
         bytes.slice(PuzLength + targetLength, PuzLength + targetLength + PublicKeyLength),
         bytes.slice(PuzLength + targetLength + PublicKeyLength, PuzLength + targetLength + PublicKeyLength + SLength),
-        (0 until proofsSize).map { i =>
-          val proofsStart = PuzLength + targetLength + PublicKeyLength + SLength + 4 + i * SingleProofSize
-          val signatureStart = proofsStart + SignatureLength
-          val dataStart = signatureStart + 8
-          val merklePathStart = dataStart + Constants.segmentSize
-
-          val signature = bytes.slice(proofsStart, proofsStart + SignatureLength)
-          val signatureIndex = Longs.fromByteArray(bytes.slice(signatureStart, signatureStart + 8))
-          val blockData = bytes.slice(dataStart, dataStart + Constants.segmentSize)
-          val merklePathSize = Ints.fromByteArray(bytes.slice(merklePathStart, merklePathStart + 4))
-          val merklePath = (0 until merklePathSize).map { i =>
-            bytes.slice(merklePathStart + 4 + i * DigestLength, merklePathStart + 4 + (i + 1) * DigestLength)
-          }
-          PartialProof(signature, signatureIndex, AuthDataBlock(blockData, merklePath))
-        }
+        parseProofs(PuzLength + targetLength + PublicKeyLength + SLength + 4, proofsSize, 0, IndexedSeq.empty)
       )
     ))
   }

From 9222ede356f363494716c778cea7d672cfa683b2 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Tue, 1 Dec 2015 10:36:26 +0300
Subject: [PATCH 62/66] PublicKeyLength = EllipticCurveImpl.KeyLength

---
 .../scorex/perma/consensus/PermaConsensusBlockField.scala      | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 71734ae1..7ccefad7 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -3,6 +3,7 @@ package scorex.perma.consensus
 import com.google.common.primitives.{Bytes, Ints, Longs}
 import play.api.libs.json._
 import scorex.block.BlockField
+import scorex.crypto.EllipticCurveImpl
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.settings.Constants
 
@@ -40,7 +41,7 @@ object PermaConsensusBlockField {
 
   val fieldName: String = "perma-consensus"
   val PuzLength = 32
-  val PublicKeyLength = 32
+  val PublicKeyLength = EllipticCurveImpl.KeyLength
   val SLength = 32
   val HashLength = 32
   val SignatureLength = 64

From a52220597dc105fb4d5bbc87ebf4edb6d527c324 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Tue, 1 Dec 2015 10:46:36 +0300
Subject: [PATCH 63/66] externalizing constants, formatting

---
 .../src/main/scala/scorex/crypto/CryptographicHash.scala  | 4 ++--
 .../scorex/perma/consensus/PermaConsensusBlockField.scala | 8 +++-----
 .../scorex/perma/consensus/PermaConsensusModule.scala     | 2 +-
 3 files changed, 6 insertions(+), 8 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala b/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala
index fbd456a6..7d6c1656 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/CryptographicHash.scala
@@ -21,7 +21,7 @@ import java.security.MessageDigest
 trait CryptographicHash {
   import CryptographicHash._
 
-  val ValueSize: Int //in bytes
+  val DigestSize: Int //in bytes
 
   def hash(input: Message): Digest
 
@@ -39,7 +39,7 @@ object CryptographicHash {
  * Hashing functions implementation with sha256 impl from Java SDK
  */
 object Sha256 extends CryptographicHash {
-  override val ValueSize = 32
+  override val DigestSize = 32
 
   override def hash(input: Array[Byte]) = MessageDigest.getInstance("SHA-256").digest(input)
 }
\ No newline at end of file
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 7ccefad7..76afa973 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -3,7 +3,7 @@ package scorex.perma.consensus
 import com.google.common.primitives.{Bytes, Ints, Longs}
 import play.api.libs.json._
 import scorex.block.BlockField
-import scorex.crypto.EllipticCurveImpl
+import scorex.crypto.{Sha256, EllipticCurveImpl}
 import scorex.crypto.ads.merkle.AuthDataBlock
 import scorex.perma.settings.Constants
 
@@ -43,8 +43,8 @@ object PermaConsensusBlockField {
   val PuzLength = 32
   val PublicKeyLength = EllipticCurveImpl.KeyLength
   val SLength = 32
-  val HashLength = 32
-  val SignatureLength = 64
+  val HashLength = Sha256.DigestSize
+  val SignatureLength = EllipticCurveImpl.SignatureLength
 
   def parse(bytes: Array[Byte]): PermaConsensusBlockField = {
     @tailrec
@@ -73,13 +73,11 @@ object PermaConsensusBlockField {
       }
     }
 
-
     val targetSize = Ints.fromByteArray(bytes.take(4))
     val targetLength = 4 + targetSize
     val proofsSize = Ints.fromByteArray(bytes.slice(
       PuzLength + targetLength + PublicKeyLength + SLength, PuzLength + targetLength + PublicKeyLength + SLength + 4))
 
-
     PermaConsensusBlockField(PermaLikeConsensusBlockData(
       BigInt(bytes.slice(4, targetLength)),
       bytes.slice(targetLength, PuzLength + targetLength),
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
index bcabedac..74709ee1 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusModule.scala
@@ -29,7 +29,7 @@ class PermaConsensusModule(rootHash: Array[Byte])
   val TargetRecalculation = Constants.targetRecalculation
   val AvgDelay = Constants.averageDelay
   val Hash = Constants.hash
-  val SSize = Hash.ValueSize
+  val SSize = Hash.DigestSize
   require(SSize == PermaConsensusBlockField.SLength)
 
   val GenesisCreator = new PublicKeyAccount(Array.fill(PermaConsensusBlockField.PublicKeyLength)(0: Byte))

From 1318c2c9b95148eb8bb282219755f144dcf328c1 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Tue, 1 Dec 2015 10:54:11 +0300
Subject: [PATCH 64/66] FieldName

---
 .../scorex/perma/consensus/PermaConsensusBlockField.scala     | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
index 76afa973..ca754c00 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PermaConsensusBlockField.scala
@@ -14,7 +14,7 @@ case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockD
 
   import PermaConsensusBlockField._
 
-  override val name: String = PermaConsensusBlockField.fieldName
+  override val name: String = PermaConsensusBlockField.FieldName
 
   override def bytes: Array[Byte] =
     Bytes.ensureCapacity(Ints.toByteArray(value.target.toByteArray.length), 4, 0) ++ value.target.toByteArray ++
@@ -39,7 +39,7 @@ case class PermaConsensusBlockField(override val value: PermaLikeConsensusBlockD
 
 object PermaConsensusBlockField {
 
-  val fieldName: String = "perma-consensus"
+  val FieldName = "perma-consensus"
   val PuzLength = 32
   val PublicKeyLength = EllipticCurveImpl.KeyLength
   val SLength = 32

From 566ca177b49f2f4b2253cd8121e57331b5b5cf37 Mon Sep 17 00:00:00 2001
From: catena <catena@protonmail.com>
Date: Tue, 1 Dec 2015 15:54:26 +0300
Subject: [PATCH 65/66] Permanent block generation

---
 .../main/scala/scorex/settings/Settings.scala | 12 ++++++++---
 scorex-perma/src/main/resources/perma.conf    |  2 +-
 settings.json                                 | 21 ++++++++++---------
 .../lagonaki/network/BlockchainSyncer.scala   | 10 ++++-----
 .../lagonaki/server/LagonakiApplication.scala |  2 +-
 5 files changed, 27 insertions(+), 20 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/settings/Settings.scala b/scorex-basics/src/main/scala/scorex/settings/Settings.scala
index 81f17200..80349bd8 100644
--- a/scorex-basics/src/main/scala/scorex/settings/Settings.scala
+++ b/scorex-basics/src/main/scala/scorex/settings/Settings.scala
@@ -6,17 +6,18 @@ import play.api.libs.json.{JsObject, Json}
 import scorex.crypto.Base58
 import scorex.utils.ScorexLogging
 
+import scala.concurrent.duration._
 import scala.util.Try
 
 /**
- * Settings
- */
+  * Settings
+  */
 
 trait Settings extends ScorexLogging {
 
   lazy val Port = 9084
 
-  val filename:String
+  val filename: String
 
   lazy val settingsJSON: JsObject = Try {
     val jsonString = scala.io.Source.fromFile(filename).mkString
@@ -53,6 +54,8 @@ trait Settings extends ScorexLogging {
   lazy val pingInterval = (settingsJSON \ "pinginterval").asOpt[Int].getOrElse(DefaultPingInterval)
   lazy val offlineGeneration = (settingsJSON \ "offline-generation").asOpt[Boolean].getOrElse(false)
   lazy val bindAddress = (settingsJSON \ "bindAddress").asOpt[String].getOrElse(DefaultBindAddress)
+  lazy val blockGenerationDelay: FiniteDuration = (settingsJSON \ "blockGenerationDelay").asOpt[Long]
+    .map(x => FiniteDuration(x, MILLISECONDS)).getOrElse(DefaultBlockGenerationDelay)
 
   lazy val walletDirOpt = (settingsJSON \ "walletdir").asOpt[String]
     .ensuring(pathOpt => pathOpt.map(directoryEnsuring).getOrElse(true))
@@ -72,4 +75,7 @@ trait Settings extends ScorexLogging {
   //API
   private val DefaultRpcPort = 9085
   private val DefaultRpcAllowed = "127.0.0.1"
+
+  private val DefaultBlockGenerationDelay: FiniteDuration = 1.second
+
 }
\ No newline at end of file
diff --git a/scorex-perma/src/main/resources/perma.conf b/scorex-perma/src/main/resources/perma.conf
index 204e3a75..9bbc1fd1 100644
--- a/scorex-perma/src/main/resources/perma.conf
+++ b/scorex-perma/src/main/resources/perma.conf
@@ -4,7 +4,7 @@ perma {
   l = 1024
   k = 4
   hash = "sha256"
-  initialTarget = "40878907271574261146604883203616243228211964000866107889441817674467149328545"
+  initialTarget = "79980561713257764341044371529078133767288724078815817939205408376510588897"
   targetRecalculation = 10
   averageDelay = 2
 }
diff --git a/settings.json b/settings.json
index c41150cc..a6031540 100644
--- a/settings.json
+++ b/settings.json
@@ -1,18 +1,19 @@
 {
-  "bindAddress" : "127.0.0.1",
-  "knownpeers":[
+  "bindAddress": "127.0.0.1",
+  "knownpeers": [
     "5.9.86.168"
   ],
-  "walletdir" : "/tmp/scorex/wallet",
-  "walletpassword" : "cookies",
-  "walletseed" : "FQgbSAm6swGbtqA3NE8PttijPhT4N3Ufh4bHFAkyVnQz",
-  "datadir" : "/tmp/scorex/data",
+  "walletdir": "/tmp/scorex/wallet",
+  "walletpassword": "cookies",
+  "walletseed": "FQgbSAm6swGbtqA3NE8PttijPhT4N3Ufh4bHFAkyVnQz",
+  "datadir": "/tmp/scorex/data",
   "minconnections": 10,
-  "rpcport" : 9085,
-  "rpcallowed":[
+  "rpcport": 9085,
+  "rpcallowed": [
     "127.0.0.1",
     "123.123.123.123"
   ],
-  "max-rollback" : 100,
-  "offline-generation" : true
+  "max-rollback": 100,
+  "blockGenerationDelay": 0,
+  "offline-generation": true
 }
\ No newline at end of file
diff --git a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
index 71cbb1b5..4edc428c 100644
--- a/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
+++ b/src/main/scala/scorex/lagonaki/network/BlockchainSyncer.scala
@@ -4,10 +4,10 @@ import java.net.InetSocketAddress
 
 import akka.actor.{ActorRef, FSM}
 import scorex.block.Block
-import scorex.crypto.EllipticCurveImpl
 import scorex.lagonaki.network.BlockchainSyncer._
 import scorex.lagonaki.network.message.{BlockMessage, GetSignaturesMessage}
 import scorex.lagonaki.server.LagonakiApplication
+import scorex.settings.Settings
 
 import scala.concurrent.ExecutionContext.Implicits.global
 import scala.concurrent.duration._
@@ -16,11 +16,11 @@ import scala.util.{Failure, Success}
 
 case class NewBlock(block: Block, sender: Option[InetSocketAddress])
 
-class BlockchainSyncer(application: LagonakiApplication, networkController: ActorRef) extends FSM[Status, Unit] {
+class BlockchainSyncer(application: LagonakiApplication, networkController: ActorRef, settings: Settings)
+  extends FSM[Status, Unit] {
 
   private val stateTimeout = 1.second
-  //TODO get from config
-  val blockGenerationDelay = 1.hour
+  val blockGenerationDelay = settings.blockGenerationDelay
 
   startWith(Offline, Unit)
 
@@ -130,7 +130,7 @@ class BlockchainSyncer(application: LagonakiApplication, networkController: Acto
     val consModule = application.consensusModule
     implicit val transModule = application.transactionModule
 
-    log.info("Trying to generate a new block")
+//    log.info("Trying to generate a new block")
     val accounts = application.wallet.privateKeyAccounts()
     consModule.generateNextBlocks(accounts)(transModule) onComplete {
       case Success(blocks: Seq[Block]) =>
diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
index b503d6dd..ec199df8 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
@@ -91,7 +91,7 @@ class LagonakiApplication(val settingsFilename: String)
   override implicit val transactionModule: SimpleTransactionModule = new SimpleTransactionModule
 
   lazy val networkController = actorSystem.actorOf(Props(classOf[NetworkController], this))
-  lazy val blockchainSyncer = actorSystem.actorOf(Props(classOf[BlockchainSyncer], this, networkController))
+  lazy val blockchainSyncer = actorSystem.actorOf(Props(classOf[BlockchainSyncer], this, networkController, settings))
 
   private lazy val walletFileOpt = settings.walletDirOpt.map(walletDir => new java.io.File(walletDir, "wallet.s.dat"))
   implicit lazy val wallet = new Wallet(walletFileOpt, settings.walletPassword, settings.walletSeed.get)

From 7dd1fed5c87c0f4145bf31b7a964af42e2d672b2 Mon Sep 17 00:00:00 2001
From: kushti <kushtech@yahoo.com>
Date: Tue, 1 Dec 2015 15:57:03 +0300
Subject: [PATCH 66/66] formatting

---
 .../main/scala/scorex/consensus/ConsensusModule.scala |  1 -
 .../scorex/crypto/ads/merkle/AuthDataBlock.scala      | 11 +++++------
 .../scala/scorex/perma/consensus/PartialProof.scala   |  1 -
 .../scorex/lagonaki/server/LagonakiApplication.scala  |  2 --
 4 files changed, 5 insertions(+), 10 deletions(-)

diff --git a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
index 6463d1bf..771e1660 100644
--- a/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
+++ b/scorex-basics/src/main/scala/scorex/consensus/ConsensusModule.scala
@@ -37,5 +37,4 @@ trait ConsensusModule[ConsensusBlockData] extends BlockProcessingModule[Consensu
   }
 
   def consensusBlockData(block: Block): ConsensusBlockData
-
 }
diff --git a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
index 621791ab..91bc9e23 100644
--- a/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
+++ b/scorex-basics/src/main/scala/scorex/crypto/ads/merkle/AuthDataBlock.scala
@@ -13,12 +13,12 @@ import scala.annotation.tailrec
   */
 case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
 
-  def check[Hash <: CryptographicHash](index: Position, rootHash: Digest)
-                                      (hashFunction: Hash = Sha256): Boolean = {
+  def check[HashImpl <: CryptographicHash](index: Position, rootHash: Digest)
+                                      (hashFunction: HashImpl = Sha256): Boolean = {
 
     @tailrec
-    def calculateHash(i: Position, nodeHash: Digest, path: Seq[Digest]): Digest = {
-      val hash = if (i % 2 == 0)
+    def calculateHash(idx: Position, nodeHash: Digest, path: Seq[Digest]): Digest = {
+      val hash = if (idx % 2 == 0)
         hashFunction.hash(nodeHash ++ path.head)
       else
         hashFunction.hash(path.head ++ nodeHash)
@@ -26,7 +26,7 @@ case class AuthDataBlock[Block](data: Block, merklePath: Seq[Digest]) {
       if (path.size == 1)
         hash
       else
-        calculateHash(i / 2, hash, path.tail)
+        calculateHash(idx / 2, hash, path.tail)
     }
 
     if (merklePath.nonEmpty)
@@ -69,6 +69,5 @@ object AuthDataBlock {
       )
     ))
   }
-
 }
 
diff --git a/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
index fcfde331..4d6a96aa 100644
--- a/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
+++ b/scorex-perma/src/main/scala/scorex/perma/consensus/PartialProof.scala
@@ -21,5 +21,4 @@ object PartialProof extends JsonSerialization {
       (JsPath \ "segmentIndex").read[Long] and
       (JsPath \ "segment").read[AuthDataBlock[DataSegment]]
     ) (PartialProof.apply _)
-
 }
diff --git a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
index ec199df8..9f86cb5a 100644
--- a/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
+++ b/src/main/scala/scorex/lagonaki/server/LagonakiApplication.scala
@@ -106,8 +106,6 @@ class LagonakiApplication(val settingsFilename: String)
       new QoraConsensusApiRoute(qcm, blockchainImpl)
     case pcm: PermaConsensusModule =>
       new PermaConsensusApiRoute(pcm, blockchainImpl)
-
-
   }
 
   override lazy val apiRoutes = Seq(