Optimized tortoise integration

app/main.go

@@ -281,6 +281,7 @@ func (app *SpacemeshApp) setupGenesis() {
 	}
 
 	app.state.Commit(false)
+	app.mesh.AddLayer(consensus.CreateGenesisLayer())
 }
 
 func (app *SpacemeshApp) setupTestFeatures() {
@@ -292,9 +293,6 @@ func (app *SpacemeshApp) initServices(instanceName string, swarm server.Service,
 
 	//todo: should we add all components to a single struct?
 	lg := log.New("shmekel_"+instanceName, "", "")
-
-	trtl := consensus.NewAlgorithm(50, 100)
-
 	db, err := database.NewLDBDatabase(dbStorepath, 0, 0)
 	if err != nil {
 		return err
@@ -306,8 +304,10 @@ func (app *SpacemeshApp) initServices(instanceName string, swarm server.Service,
 	rng := rand.New(mt19937.New())
 	processor := state.NewTransactionProcessor(rng, st, lg)
 
-	mesh := mesh.NewMesh(db, db, db, &trtl, processor, lg) //todo: what to do with the logger?
-	trtl.RegisterLayerCallback(mesh.LayerCompleteCallback)
+	//trtl := consensus.NewTortoise(50, 100)
+	trtl := consensus.NewAlgorithm(consensus.NewNinjaTortoise(50))
+	mesh := mesh.NewMesh(db, db, db, trtl, processor, lg) //todo: what to do with the logger?
+
 	coinToss := consensus.WeakCoin{}
 	clock := timesync.NewTicker(timesync.RealClock{}, 5*time.Second, time.Now())
 

consensus/algorithem.go

@@ -0,0 +1,44 @@
+package consensus
+
+import (
+	"github.com/spacemeshos/go-spacemesh/log"
+	"github.com/spacemeshos/go-spacemesh/mesh"
+)
+
+type Algorithm struct {
+	Tortoise
+	callback func(mesh.LayerID)
+}
+
+type Tortoise interface {
+	handleIncomingLayer(ll *mesh.Layer)
+}
+
+func NewAlgorithm(trtl Tortoise) *Algorithm {
+	return &Algorithm{Tortoise: trtl}
+}
+
+func (alg *Algorithm) RegisterLayerCallback(callback func(mesh.LayerID)) {
+	alg.callback = callback
+}
+
+func (alg *Algorithm) HandleLateBlock(b *mesh.Block) {
+	log.Info("received block with layer Id %v block id: %v ", b.Layer(), b.ID())
+}
+
+func (alg *Algorithm) HandleIncomingLayer(ll *mesh.Layer) {
+	alg.Tortoise.handleIncomingLayer(ll)
+	alg.callback(ll.Index())
+}
+
+func CreateGenesisLayer() *mesh.Layer {
+	log.Info("Creating genesis")
+	bl := &mesh.Block{
+		Id:         mesh.BlockID(0),
+		LayerIndex: 0,
+		Data:       []byte("genesis"),
+	}
+	l := mesh.NewLayer(Genesis)
+	l.AddBlock(bl)
+	return l
+}

consensus/ninja_tortoise.go

@@ -82,6 +82,27 @@ type ninjaTortoise struct {
 	tPatSupport        map[votingPattern]map[mesh.LayerID]votingPattern //pattern support count
 }
 
+func NewNinjaTortoise(layerSize uint32) *ninjaTortoise {
+	return &ninjaTortoise{
+		Log:                log.New("optimized tortoise ", "", ""),
+		avgLayerSize:       layerSize,
+		pBase:              votingPattern{},
+		blocks:             map[mesh.BlockID]*mesh.Block{},
+		tEffective:         map[mesh.BlockID]votingPattern{},
+		tCorrect:           map[mesh.BlockID]map[mesh.BlockID]vec{},
+		layerBlocks:        map[mesh.LayerID][]mesh.BlockID{},
+		tExplicit:          map[mesh.BlockID]map[mesh.LayerID]votingPattern{},
+		tGood:              map[mesh.LayerID]votingPattern{},
+		tSupport:           map[votingPattern]int{},
+		tPattern:           map[votingPattern]map[mesh.BlockID]struct{}{},
+		tVote:              map[votingPattern]map[mesh.BlockID]vec{},
+		tTally:             map[votingPattern]map[mesh.BlockID]vec{},
+		tComplete:          map[votingPattern]struct{}{},
+		tEffectiveToBlocks: map[votingPattern][]mesh.BlockID{},
+		tPatSupport:        map[votingPattern]map[mesh.LayerID]votingPattern{},
+	}
+}
+
 func (ni *ninjaTortoise) processBlock(b *mesh.Block) {
 
 	ni.Debug("process block: %d layer: %d  ", b.Id, b.Layer())
@@ -344,21 +365,11 @@ func (ni *ninjaTortoise) processBlocks(layer *mesh.Layer) {
 
 }
 
-func (ni *ninjaTortoise) init(genesis *mesh.Layer, l1 *mesh.Layer) {
-	ni.processBlocks(genesis)
+func (ni *ninjaTortoise) handleGenesis(genesis *mesh.Layer) {
 	vp := votingPattern{id: getId(ni.layerBlocks[Genesis]), LayerID: Genesis}
 	ni.pBase = vp
 	ni.tGood[Genesis] = vp
 	ni.tExplicit[genesis.Blocks()[0].ID()] = make(map[mesh.LayerID]votingPattern, K*ni.avgLayerSize)
-	ni.processBlocks(l1)
-	vp1 := votingPattern{id: getId(ni.layerBlocks[Genesis+1]), LayerID: Genesis + 1}
-	ni.tPattern[vp1] = map[mesh.BlockID]struct{}{}
-	for _, b := range ni.layerBlocks[Genesis+1] {
-		ni.tPattern[vp1][b] = struct{}{}
-	}
-	ni.tVote[vp1] = make(map[mesh.BlockID]vec)
-	ni.tVote[vp1][genesis.Blocks()[0].ID()] = Support
-
 }
 
 //todo send map instead of ni
@@ -380,11 +391,16 @@ func initTallyToBase(tally map[votingPattern]map[mesh.BlockID]vec, base votingPa
 	}
 }
 
-func (ni *ninjaTortoise) HandleIncomingLayer(newlyr *mesh.Layer) mesh.LayerID { //i most recent layer
-	ni.Debug("update tables layer %d", newlyr.Index())
-	//initialize these tables //not in article
+func (ni *ninjaTortoise) handleIncomingLayer(newlyr *mesh.Layer) { //i most recent layer
+	ni.Info("update tables layer %d with %d blocks", newlyr.Index(), len(newlyr.Blocks()))
+
 	ni.processBlocks(newlyr)
 
+	if newlyr.Index() == Genesis {
+		ni.handleGenesis(newlyr)
+		return
+	}
+
 	l := ni.findMinimalNewlyGoodLayer(newlyr)
 
 	//from minimal newly good pattern to current layer
@@ -465,5 +481,5 @@ func (ni *ninjaTortoise) HandleIncomingLayer(newlyr *mesh.Layer) mesh.LayerID {
 			}
 		}
 	}
-	return ni.pBase.LayerID
+	return
 }

consensus/ninja_tortoise_test.go

@@ -50,7 +50,7 @@ func TestNinjaTortoise_GlobalOpinion(t *testing.T) {
 func TestForEachInView(t *testing.T) {
 	blocks := make(map[mesh.BlockID]*mesh.Block)
 	alg := NewNinjaTortoise(2)
-	l := createGenesisLayer()
+	l := CreateGenesisLayer()
 	for _, b := range l.Blocks() {
 		blocks[b.ID()] = b
 	}
@@ -88,41 +88,21 @@ func TestForEachInView(t *testing.T) {
 func TestNinjaTortoise_UpdatePatternTally(t *testing.T) {
 }
 
-func NewNinjaTortoise(layerSize uint32) *ninjaTortoise {
-	return &ninjaTortoise{
-		Log:                log.New("optimized tortoise ", "", ""),
-		avgLayerSize:       layerSize,
-		pBase:              votingPattern{},
-		blocks:             map[mesh.BlockID]*mesh.Block{},
-		tEffective:         map[mesh.BlockID]votingPattern{},
-		tCorrect:           map[mesh.BlockID]map[mesh.BlockID]vec{},
-		layerBlocks:        map[mesh.LayerID][]mesh.BlockID{},
-		tExplicit:          map[mesh.BlockID]map[mesh.LayerID]votingPattern{},
-		tGood:              map[mesh.LayerID]votingPattern{},
-		tSupport:           map[votingPattern]int{},
-		tPattern:           map[votingPattern]map[mesh.BlockID]struct{}{},
-		tVote:              map[votingPattern]map[mesh.BlockID]vec{},
-		tTally:             map[votingPattern]map[mesh.BlockID]vec{},
-		tComplete:          map[votingPattern]struct{}{},
-		tEffectiveToBlocks: map[votingPattern][]mesh.BlockID{},
-		tPatSupport:        map[votingPattern]map[mesh.LayerID]votingPattern{},
-	}
-}
-
 //vote explicitly only for previous layer
 //correction vectors have no affect here
 func TestNinjaTortoise_Sanity1(t *testing.T) {
 	layerSize := 30
 	patternSize := layerSize
 	alg := NewNinjaTortoise(uint32(layerSize))
-	l1 := createGenesisLayer()
+	l1 := CreateGenesisLayer()
 	genesisId := l1.Blocks()[0].ID()
+	alg.handleIncomingLayer(l1)
 	l := createLayerWithRandVoting(l1.Index()+1, []*mesh.Layer{l1}, layerSize, 1)
-	alg.init(l1, l)
+	alg.handleIncomingLayer(l)
 	for i := 0; i < 30; i++ {
 		lyr := createLayerWithRandVoting(l.Index()+1, []*mesh.Layer{l}, layerSize, layerSize)
 		start := time.Now()
-		alg.HandleIncomingLayer(lyr)
+		alg.handleIncomingLayer(lyr)
 		alg.Info("Time to process layer: %v ", time.Since(start))
 		l = lyr
 		for b, vec := range alg.tTally[alg.pBase] {
@@ -143,10 +123,11 @@ func TestNinjaTortoise_Sanity2(t *testing.T) {
 	l3 := createMulExplicitLayer(3, map[mesh.LayerID]*mesh.Layer{l2.Index(): l2}, map[mesh.LayerID][]int{l2.Index(): {0}}, 3)
 	l4 := createMulExplicitLayer(4, map[mesh.LayerID]*mesh.Layer{l2.Index(): l2, l3.Index(): l3}, map[mesh.LayerID][]int{l2.Index(): {1, 2}, l3.Index(): {1, 2}}, 4)
 
-	alg.init(l, l1)
-	alg.HandleIncomingLayer(l2)
-	alg.HandleIncomingLayer(l3)
-	alg.HandleIncomingLayer(l4)
+	alg.handleIncomingLayer(l)
+	alg.handleIncomingLayer(l1)
+	alg.handleIncomingLayer(l2)
+	alg.handleIncomingLayer(l3)
+	alg.handleIncomingLayer(l4)
 	for b, vec := range alg.tTally[alg.pBase] {
 		alg.Info("------> tally for block %d according to complete pattern %d are %d", b, alg.pBase, vec)
 	}

consensus/tortoise.go

@@ -16,7 +16,7 @@ type BlockPosition struct {
 	layer      LayerID
 }
 
-type Algorithm struct {
+type tortoise struct {
 	block2Id           map[BlockID]uint32
 	allBlocks          map[BlockID]*TortoiseBlock
 	layerQueue         LayerQueue
@@ -32,9 +32,9 @@ type Algorithm struct {
 	mu                 sync.Mutex
 }
 
-func NewAlgorithm(layerSize uint32, cachedLayers uint32) Algorithm {
+func NewTortoise(layerSize uint32, cachedLayers uint32) *tortoise {
 	totBlocks := layerSize * cachedLayers
-	trtl := Algorithm{
+	trtl := tortoise{
 		block2Id:          make(map[BlockID]uint32),
 		allBlocks:         make(map[BlockID]*TortoiseBlock),
 		layerQueue:        make(LayerQueue, cachedLayers+1),
@@ -48,22 +48,22 @@ func NewAlgorithm(layerSize uint32, cachedLayers uint32) Algorithm {
 		cachedLayers:       cachedLayers,
 		layerReadyCallback: nil,
 	}
-	return trtl
+	return &trtl
 }
 
-func (alg *Algorithm) RegisterLayerCallback(callback func(mesh.LayerID)) {
+func (alg *tortoise) RegisterLayerCallback(callback func(mesh.LayerID)) {
 	alg.layerReadyCallback = callback
 }
 
-func (alg *Algorithm) GlobalVotingAvg() uint64 {
+func (alg *tortoise) GlobalVotingAvg() uint64 {
 	return 100
 }
 
-func (alg *Algorithm) LayerVotingAvg() uint64 {
+func (alg *tortoise) LayerVotingAvg() uint64 {
 	return 30
 }
 
-func (alg *Algorithm) IsTortoiseValid(originBlock *TortoiseBlock, targetBlock BlockID, targetBlockIdx uint64, visibleBlocks bitarray.BitArray) bool {
+func (alg *tortoise) IsTortoiseValid(originBlock *TortoiseBlock, targetBlock BlockID, targetBlockIdx uint64, visibleBlocks bitarray.BitArray) bool {
 	voteFor, voteAgainst := alg.countTotalVotesForBlock(targetBlockIdx, visibleBlocks)
 
 	if voteFor > alg.GlobalVotingAvg() {
@@ -85,18 +85,18 @@ func (alg *Algorithm) IsTortoiseValid(originBlock *TortoiseBlock, targetBlock Bl
 	return originBlock.Coin
 }
 
-func (alg *Algorithm) getLayerById(layerId LayerID) (*Layer, error) {
+func (alg *tortoise) getLayerById(layerId LayerID) (*Layer, error) {
 	if _, ok := alg.layers[layerId]; !ok {
 		return nil, fmt.Errorf("layer Id not found %v", layerId)
 	}
 	return alg.layers[layerId], nil
 }
 
-func (alg *Algorithm) CountVotesInLastLayer(block *TortoiseBlock) (uint64, uint64) {
+func (alg *tortoise) CountVotesInLastLayer(block *TortoiseBlock) (uint64, uint64) {
 	return block.ConVotes, block.ProVotes
 }
 
-func (alg *Algorithm) createBlockVotingMap(origin *TortoiseBlock) (*bitarray.BitArray, *bitarray.BitArray) {
+func (alg *tortoise) createBlockVotingMap(origin *TortoiseBlock) (*bitarray.BitArray, *bitarray.BitArray) {
 	blockMap := bitarray.NewBitArray(uint64(alg.totalBlocks))
 	visibilityMap := bitarray.NewBitArray(uint64(alg.totalBlocks))
 	// Count direct voters
@@ -137,7 +137,7 @@ func (alg *Algorithm) createBlockVotingMap(origin *TortoiseBlock) (*bitarray.Bit
 	return &blockMap, &visibilityMap
 }
 
-func (alg *Algorithm) countTotalVotesForBlock(targetIdx uint64, visibleBlocks bitarray.BitArray) (uint64, uint64) {
+func (alg *tortoise) countTotalVotesForBlock(targetIdx uint64, visibleBlocks bitarray.BitArray) (uint64, uint64) {
 	var posVotes, conVotes uint64 = 0, 0
 	targetLayer := alg.visibilityMap[targetIdx].layer
 	ln := len(alg.block2Id)
@@ -162,14 +162,14 @@ func (alg *Algorithm) countTotalVotesForBlock(targetIdx uint64, visibleBlocks bi
 	return posVotes, conVotes
 }
 
-func (alg *Algorithm) zeroBitColumn(idx uint64) {
+func (alg *tortoise) zeroBitColumn(idx uint64) {
 	for row, bitvec := range alg.posVotes {
 		bitvec.ClearBit(idx)
 		alg.visibilityMap[row].visibility.ClearBit(idx)
 	}
 }
 
-func (alg *Algorithm) recycleLayer(l *Layer) {
+func (alg *tortoise) recycleLayer(l *Layer) {
 	for _, block := range l.blocks {
 		id := alg.block2Id[block.Id]
 		alg.idQueue <- alg.block2Id[block.Id]
@@ -182,7 +182,7 @@ func (alg *Algorithm) recycleLayer(l *Layer) {
 	alg.mu.Unlock()
 }
 
-func (alg *Algorithm) assignIdForBlock(blk *TortoiseBlock) uint32 {
+func (alg *tortoise) assignIdForBlock(blk *TortoiseBlock) uint32 {
 	//todo: should this section be protected by a mutex?
 	alg.allBlocks[blk.Id] = blk
 	if len(alg.idQueue) > 0 {
@@ -204,7 +204,7 @@ func (alg *Algorithm) assignIdForBlock(blk *TortoiseBlock) uint32 {
 
 }
 
-func (alg *Algorithm) HandleIncomingLayer(ll *mesh.Layer) {
+func (alg *tortoise) HandleIncomingLayer(ll *mesh.Layer) {
 	l := FromLayerToTortoiseLayer(ll)
 	alg.mu.Lock()
 	alg.layers[l.index] = l
@@ -231,6 +231,6 @@ func (alg *Algorithm) HandleIncomingLayer(ll *mesh.Layer) {
 	alg.mu.Unlock()
 }
 
-func (alg *Algorithm) HandleLateBlock(b *mesh.Block) {
+func (alg *tortoise) HandleLateBlock(b *mesh.Block) {
 	log.Info("received block with layer Id %v block id: %v ", b.Layer(), b.ID())
 }

consensus/tortoise_test.go

@@ -12,7 +12,7 @@ func TestAlgorithm_Sanity(t *testing.T) {
 	layerSize := 50
 	cachedLayers := 100
 
-	alg := NewAlgorithm(uint32(layerSize), uint32(cachedLayers))
+	alg := NewTortoise(uint32(layerSize), uint32(cachedLayers))
 	l := createGenesisLayer()
 	alg.HandleIncomingLayer(l)
 	alg.RegisterLayerCallback(func(id mesh.LayerID) {})

mesh/mesh.go

@@ -62,6 +62,7 @@ func NewMesh(layers, blocks, validity database.DB, mesh MeshValidator, state Sta
 		state:    state,
 		meshDB:   NewMeshDB(layers, blocks, validity),
 	}
+	mesh.RegisterLayerCallback(ll.LayerCompleteCallback)
 	return ll
 }