40 src/main/scala/core/BitcoinDB.scala
View file
@@ -83,14 +83,17 @@ trait BitcoinDB {
}
}


def initializeStatsTables: Unit = {
transactionDBSession{
deleteIfExists(stats, richestAddresses, richestClosures)
deleteIfExists(stats, richestAddresses, richestClosures, balances, closureBalances)

stats.ddl.create
richestAddresses.ddl.create
richestClosures.ddl.create

balances.ddl.create
closureBalances.ddl.create

}
}

@@ -126,30 +129,6 @@ trait BitcoinDB {
}
}

def copyUTXOs = {
lazy val table = LmdbMap.open("utxos")
lazy val outputMap: UTXOs = new UTXOs (table)

// (txhash,index) -> (address,value,blockIn)

import Hash.hashToArray
val values = for (((txhash,index),(address,value,blockIn)) <- outputMap.view) //makes it a lazy collection
yield (hashToArray(txhash),hashToArray(address),index,value,blockIn)

transactionDBSession{
deleteIfExists(utxo)
utxo.ddl.create
values.grouped(100000).foldLeft(0){
case (count,group) =>
println(count + " elements read at " + java.util.Calendar.getInstance().getTime())
val seq = group.toSeq
utxo.insertAll(seq: _*)
count+seq.size
}
}
table.close
}

def updateBalanceTables(changedAddresses: collection.mutable.Map[Hash,Long]) = {
var clock = System.currentTimeMillis
println("DEBUG: Updating balances ...")
@@ -266,9 +245,9 @@ trait BitcoinDB {
val query = """
insert
into stats select
(select max(block_height) from blocks),
(select sum(balance)/100000000 from balances),
(select sum(txs) from blocks),
(select coalesce(max(block_height),0) from blocks),
(select coalesce(sum(balance)/100000000,0) from balances),
(select coalesce(sum(txs),0) from blocks),
(select count(1) from addresses),
(select count(distinct(representant)) from addresses),
(select count(1) from balances),
@@ -285,6 +264,7 @@ trait BitcoinDB {
}
}



def getGini[A <: Table[_] with BalanceField](balanceTable: TableQuery[A]): (Long, Double) = {
println("DEBUG: calculating Gini: " + balanceTable + java.util.Calendar.getInstance().getTime())
@@ -299,7 +279,7 @@ trait BitcoinDB {

val summe = balances.sum
val mainSum = balances.zipWithIndex.map(p => p._1*(p._2+1.0)/n).sum
val gini:Double = 2.0*mainSum/(summe) - (n+1.0)/n
val gini:Double = if (n==0) 0.0 else 2.0*mainSum/(summe) - (n+1.0)/n
println("DONE: gini calculated in " + (System.currentTimeMillis - time)/1000 + "s")
(n, gini)
}
25 src/main/scala/core/BitcoinDRawFileBlockSource.scala
View file
@@ -3,17 +3,30 @@ package core
import org.bitcoinj.core._
import org.bitcoinj.params.MainNetParams
import org.bitcoinj.utils.BlockFileLoader
import org.bitcoinj.store.MemoryBlockStore;
import java.net.InetAddress


import scala.collection.convert.WrapAsScala._

// In java that should be implements libs.BlockSource
trait BitcoinDRawFileBlockSource extends BlockSource
{
def params = MainNetParams.get
override def blockSource: Iterator[(Block,Int)] = {
start

println("starting at " + java.util.Calendar.getInstance().getTime())
val b = for {
block <- asScalaIterator(loader)
storedBlock = blockStore.get(block.getHash)
if storedBlock != null
}
yield
(block,storedBlock.getHeight)

stop

b
}

private lazy val loader = {
val context = new Context(params) // had to put this here because of scala trait initialization madness
new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)}

override def blockSource: Iterator[Block] = asScalaIterator(loader)
}
20 src/main/scala/core/BlockReader.scala
View file
@@ -15,15 +15,15 @@ trait BlockReader extends BlockSource {

def saveTransaction(transaction: Transaction, blockHeight: Int)

def saveBlock(b: Hash, txs: Int, btcs: Long, tstamp: Long): Unit
def saveBlock(b: Hash, txs: Int, btcs: Long, tstamp: Long, height: Int): Unit

def pre: Unit
def useDatabase: Boolean
def post: Unit

var processedBlocks: Vector[Int] = Vector.empty
var savedBlockSet: Set[Hash] = Set.empty
val longestChain: Map[Hash, Int] = getLongestBlockChainHashSet
// val longestChain: Map[Hash, Int] = getLongestBlockChainHashSet
var transactionCounter = 0
var startTime = System.currentTimeMillis

@@ -62,17 +62,17 @@ trait BlockReader extends BlockSource {

def blockFilter(b: Block) = {
val blockHash = Hash(b.getHash.getBytes)
(longestChain contains blockHash) && !(savedBlockSet contains blockHash)
/*(longestChain contains blockHash) &&*/ !(savedBlockSet contains blockHash)
}

def withoutDuplicates(b: Block, t: Transaction): Boolean =
!(Hash(b.getHash.getBytes) == Hash("00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec") &&
Hash(t.getHash.getBytes) == Hash("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599")) &&
!(Hash(b.getHash.getBytes) == Hash("00000000000a4d0a3B83B59A507C6B843DE3DB4E365B141621FB2381A2641B16C4E10C110E1C2EFBD98161ffc163c503763b1f4360639393e0e4c8e300e0caec") &&
!(Hash(b.getHash.getBytes) == Hash("00000000000a4d0a3B83B59A507C6B843DE3DB4E365B141621FB2381A2641B16C4E10C110E1C2EFBD98161ffc163c503763b1f4360639393e0e4c8e300e0caec") && // TODO: this looks wrong
Hash(t.getHash.getBytes) == Hash("d5d27987d2a3dfc724e359870c6644b40e497bdc0589a033220fe15429d88599"))

lazy val filteredBlockSource =
blockSource withFilter blockFilter
blockSource withFilter (p=>blockFilter(p._1))


def transactionsInBlock(b: Block) =
@@ -91,13 +91,13 @@ trait BlockReader extends BlockSource {
yield o.getValue.value).sum


// TODO: replace 0 0 with txs and btcs from the transaction set // TODO: ask Jorge what that means

lazy val transactionSource: Iterator[(Transaction,Int)] =
filteredBlockSource flatMap { b =>
filteredBlockSource flatMap { case (b,n) =>
val blockHash = Hash(b.getHash.getBytes)
saveBlock(blockHash, b.getTransactions.size,getTxValue(b),b.getTimeSeconds)
transactionsInBlock(b) map ((_, longestChain.getOrElse(blockHash, 0)))
}
saveBlock(blockHash, b.getTransactions.size,getTxValue(b),b.getTimeSeconds,n) // TODO: this is ugly!
transactionsInBlock(b) map ((_, n))
}


def getAddressFromOutput(output: TransactionOutput): Option[Array[Byte]] =
68 src/main/scala/core/BlockSource.scala
View file
@@ -1,11 +1,75 @@
package core

import org.bitcoinj.core.{Block, NetworkParameters}
import org.bitcoinj.params.MainNetParams
import org.bitcoinj.utils.BlockFileLoader
import org.bitcoinj.core._
import org.bitcoinj.store.MemoryBlockStore
import org.bitcoinj.store.BlockStore;
import java.net.InetAddress
import util._
import scala.collection.convert.WrapAsScala._
import sys.process._

/**
* Created by yzark on 25.08.14.
*/
trait BlockSource {
def params: NetworkParameters
def blockSource: Iterator[Block]
def params = MainNetParams.get

val numberOfHeaders = 500000

lazy val blockStore = new MemoryBlockStore(params,blockStoreFile){
//println("shit")
// numHeaders=1000000
// randomAccessFile.setLength(getFileSize)
// buffer = randomAccessFile.getChannel.map(java.nio.channels.FileChannel.MapMode.READ_WRITE, 0, getFileSize)

override def getFileSize : Int = {
SPVBlockStore.RECORD_SIZE * numberOfHeaders + SPVBlockStore.FILE_PROLOGUE_BYTES
}
}


lazy val chain = new BlockChain(params, blockStore);
lazy val peerGroup = new PeerGroup(params, chain);
lazy val loader = {
val context = new Context(params) // had to put this here because of scala trait initialization madness
new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
}

lazy val addr = new PeerAddress(InetAddress.getLocalHost(), params.getPort());

def start: Unit = {
Seq("bitcoind","-daemon").run
peerGroup.start();
peerGroup.addAddress(addr);
peerGroup.waitForPeers(1).get();

for (block <- asScalaIterator(loader))
chain add block
waitIfNewBlocks(0)
}

def blockSource: Iterator[(Block,Int)] // block,height


@annotation.tailrec final def waitIfNewBlocks(last: Int): Unit = {

val waitTime = 10000
Thread sleep waitTime
val current = chain.getBestChainHeight

println("waiting " + waitTime + " for bitcoinJ: last state (0 is error): " + last + ". current state:"+ current + " at " + java.util.Calendar.getInstance().getTime())

if (current == 0 || ( /*current < 10000 &&*/ last != current))
waitIfNewBlocks(current)
else
println("done")
}

def stop = {
peerGroup.stopAsync()
}

}
19 src/main/scala/core/FastBlockReader.scala
View file
@@ -10,7 +10,7 @@ import util._
// A FastBlockReader is a BlockReader that uses an UTXO set map
abstract class FastBlockReader extends BlockReader {

lazy val table: mutable.Map[Hash,Hash] = LmdbMap.create("utxos")
lazy val table = LmdbMap.create("utxos")
// (txhash,index) -> (address,value,blockIn)
lazy val outputMap: UTXOs = new UTXOs (table)

@@ -28,7 +28,6 @@ abstract class FastBlockReader extends BlockReader {
def saveTransaction(trans: Transaction, blockHeight: Int) =
{
val transactionHash = Hash(trans.getHash.getBytes)
//println("saving transaction from block " + transactionHash)
val addresses =
for {
input <- inputsInTransaction(trans)
@@ -127,8 +126,7 @@ abstract class FastBlockReader extends BlockReader {
}
}

def saveBlock(b: Hash, txs: Int, btcs: Long, tstamp: Long) = {
val height = longestChain.getOrElse(b,0)
def saveBlock(b: Hash, txs: Int, btcs: Long, tstamp: Long, height:Int) = {
processedBlocks :+= height
insertBlock(b, height, txs, btcs, tstamp)
println("DEBUG: Saving block " + height + " consisting of " + txs + " txs at " + java.util.Calendar.getInstance().getTime() )
@@ -150,7 +148,7 @@ abstract class FastBlockReader extends BlockReader {

def saveUnmatchedInputs: Unit =
{
println(outOfOrderInputMap.size + " unmatched Inputs")
assert(outOfOrderInputMap.size == 0, "unmatched Inputs")
//for (((outpointTransactionHash, outpointIndex), transactionHash) <- outOfOrderInputMap)
// insertInsertIntoList(Some(transactionHash), Some(outpointTransactionHash), None, Some(outpointIndex), None, None)
}
@@ -167,8 +165,15 @@ abstract class FastBlockReader extends BlockReader {
def vectorMovementsConverter[A,B,C,D](v:Vector[(Hash,Hash,Option[Hash],A,B,C,D)]) = v map {
case (a,b,c,d,e,f,g) => (Hash.hashToArray(a),Hash.hashToArray(b),ohc(c),d,e,f,g) }
val convertedVectorMovements = vectorMovementsConverter(vectorMovements)
movements.insertAll(convertedVectorMovements:_*)


try{
movements.insertAll(convertedVectorMovements:_*)
}
catch {
case e: java.sql.BatchUpdateException =>
throw e.getNextException
}


vectorMovements = Vector()
vectorBlocks = Vector()
38 src/main/scala/core/PeerSource.scala
View file
@@ -12,41 +12,35 @@ import scala.collection.convert.WrapAsScala._

// In java that should be implements libs.BlockSource
trait PeerSource extends BlockSource {
def params = MainNetParams.get

// private lazy val loader = {
// val context = new Context(params)
// new BlockFileLoader(params,BlockFileLoader.getReferenceClientBlockFileList)
// }
//lazy val lines = scala.io.Source.fromFile(blockHashListFile).getLines.drop(blockCount) take 100

lazy val blockStore = new MemoryBlockStore(params);
lazy val chain = new BlockChain(params, blockStore);
lazy val peerGroup = new PeerGroup(params, chain);

lazy val addr = new PeerAddress(InetAddress.getLocalHost(), params.getPort());

lazy val lines = scala.io.Source.fromFile(blockHashListFile).getLines.drop(blockCount).take(100)
lazy val begin = blockCount
lazy val end = chain.getBestChainHeight() - 5
lazy val range = (begin until end) take 100
lazy val lines = for (no <- range)
yield (chain.getHeightFuture(no),no)

override def blockSource = {
peerGroup.start();
peerGroup.addAddress(addr);
peerGroup.waitForPeers(1).get();
start

val peer = peerGroup.getConnectedPeers().get(0);
println("reading blocks from " + begin + " until " + end)

for (line <- lines) yield {
val blockHash = Sha256Hash.wrap(line.toLowerCase);


for ((line,no) <- lines.toIterator) yield {
val blockHash = line.get.getHeader.getHash
val future = peer.getBlock(blockHash);
System.out.println("Waiting for node to send us the requested block: " + blockHash + " at " + java.util.Calendar.getInstance().getTime());
val res = future.get();
val block = future.get();
System.out.println("Block received at " + java.util.Calendar.getInstance().getTime())
res
(block,no)
}



}

def stop = {
peerGroup.stopAsync()
}

}
9 src/main/scala/util/package.scala
View file
@@ -23,13 +23,15 @@ package object util extends BitcoinDB
lazy val arrayNull = Hash.zero(1).array.toArray


def getLongestBlockChainHashSet: Map[Hash,Int] =
val blockStoreFile = new java.io.File("/root/Bitcoin-Graph-Explorer/blockchain/spv.blockstore")

/* def getLongestBlockChainHashSet: Map[Hash,Int] =
{
val lines = scala.io.Source.fromFile(blockHashListFile).getLines
val hashes = for (line <- lines) yield Hash(line)
hashes.zipWithIndex.toMap
}

*/


def toArrayBuf[A:IntOrLong](x:A)(implicit f:IntOrLong[A]): ArrayBuffer[Byte] = {
@@ -60,6 +62,9 @@ package object util extends BitcoinDB
}
}

def countLines(fileName: String) =
scala.io.Source.fromFile(fileName).getLines.length


// just experimenting here
//(spent_in_transaction_hash,transaction_hash,address,index,value, height_in, height_out)