From 1c13cc1a00ec55cb89b7e61797a3794e0c9dac61 Mon Sep 17 00:00:00 2001 From: Anton Lerner Date: Mon, 15 Apr 2019 10:29:04 +0300 Subject: [PATCH] inverted control between nipst builder and atx builder (#806) * inverted control between nipst builder and atx builder * added tests to poet challange builder * fixed refactoring error * validate nipst * fix typo + refactor miss * add cleanup and post init to nipst test * nipst init post * test validation failures * update post * added atx validation * remove todo Verified with @barakshani * use EpochID where appropriate * added positive and negative tests to flow * after review and fmt * fixed a ut * go mod tidy --- activation/activation.go | 134 +++++++--- activation/activation_test.go | 87 ++++++- activation/activationdb_test.go | 91 +++++++ activation/atxdb.go | 73 +++++- go.mod | 2 +- go.sum | 4 +- nipst/nipst.go | 378 +++++++++++----------------- nipst/nipst_test.go | 226 ++++++++++++----- nipst/poet.go | 12 +- nipst/poet_test.go | 6 +- nipst/post.go | 43 +++- nipst/post_test.go | 1 + nipst/validator.go | 79 ++++++ oracle/blockeligibilityvalidator.go | 2 +- oracle/blockoracle.go | 14 +- oracle/blockoracle_test.go | 2 +- oracle/epochbeacon.go | 9 +- types/activation.go | 61 +++-- types/block.go | 4 +- 19 files changed, 841 insertions(+), 387 deletions(-) create mode 100644 nipst/validator.go diff --git a/activation/activation.go b/activation/activation.go index a1e6fb033b..04c23ff7e9 100644 --- a/activation/activation.go +++ b/activation/activation.go @@ -1,6 +1,8 @@ package activation import ( + "fmt" + "github.com/davecgh/go-xdr/xdr" "github.com/spacemeshos/go-spacemesh/database" "github.com/spacemeshos/go-spacemesh/log" "github.com/spacemeshos/go-spacemesh/mesh" @@ -30,13 +32,27 @@ type Broadcaster interface { Broadcast(channel string, data []byte) error } +type PoETNumberOfTickProvider struct { +} + +func (provider *PoETNumberOfTickProvider) NumOfTicks() uint64 { + return 0 +} + +type NipstBuilder interface { + BuildNIPST(challange []byte) (*nipst.NIPST, error) +} + type Builder struct { - nodeId types.NodeId - db *ActivationDb - net Broadcaster - activeSet ActiveSetProvider - mesh MeshProvider - epochProvider EpochProvider + nodeId types.NodeId + db *ActivationDb + net Broadcaster + activeSet ActiveSetProvider + mesh MeshProvider + epochProvider EpochProvider + layersPerEpoch uint64 + tickProvider PoETNumberOfTickProvider + nipstBuilder NipstBuilder } type Processor struct { @@ -44,41 +60,75 @@ type Processor struct { epochProvider EpochProvider } -func NewBuilder(nodeId types.NodeId, db database.DB, meshdb *mesh.MeshDB, net Broadcaster, activeSet ActiveSetProvider, view MeshProvider, epochDuration EpochProvider, layersPerEpoch uint64) *Builder { +func NewBuilder(nodeId types.NodeId, db database.DB, + meshdb *mesh.MeshDB, + net Broadcaster, + activeSet ActiveSetProvider, + view MeshProvider, + epochDuration EpochProvider, + layersPerEpoch uint64, + nipstBuilder NipstBuilder) *Builder { return &Builder{ - nodeId, NewActivationDb(db, meshdb, layersPerEpoch), net, activeSet, view, epochDuration, + nodeId, + NewActivationDb(db, meshdb, layersPerEpoch), + net, + activeSet, + view, + epochDuration, + layersPerEpoch, + PoETNumberOfTickProvider{}, + nipstBuilder, } } -func (b *Builder) PublishActivationTx(nipst *nipst.NIPST) error { - var seq uint64 +func (b *Builder) PublishActivationTx(epoch types.EpochId) error { prevAtx, err := b.GetPrevAtxId(b.nodeId) - if prevAtx == nil { - log.Info("previous ATX not found") - prevAtx = &types.EmptyAtx - seq = 0 - } else { - seq = b.GetLastSequence(b.nodeId) - if seq > 0 && prevAtx == nil { - log.Error("cannot find prev ATX for nodeid %v ", b.nodeId) + seq := uint64(0) + if err == nil { + atx, err := b.db.GetAtx(*prevAtx) + if err != nil { return err } - seq++ + seq = atx.Sequence + 1 + } else { + prevAtx = &types.EmptyAtx } - - l := b.mesh.LatestLayerId() - ech := b.epochProvider.Epoch(l) - var posAtx *types.AtxId = nil - if ech > 0 { - posAtx, err = b.GetPositioningAtxId(ech - 1) + posAtxId := &types.EmptyAtx + endTick := uint64(0) + LayerIdx := b.mesh.LatestLayerId() + if epoch > 0 { + //positioning atx is from the last epoch + posAtxId, err = b.GetPositioningAtxId(epoch - 1) if err != nil { return err } - } else { - posAtx = &types.EmptyAtx + posAtx, err := b.db.GetAtx(*posAtxId) + if err != nil { + return err + } + endTick = posAtx.EndTick + } + + challenge := types.PoETChallenge{ + NodeId: b.nodeId, + Sequence: seq, + PrevATXId: *prevAtx, + LayerIdx: types.LayerID(uint64(LayerIdx) + b.layersPerEpoch), + StartTick: endTick, + EndTick: b.tickProvider.NumOfTicks(), //todo: add provider when + PositioningAtx: *posAtxId, } - atx := types.NewActivationTx(b.nodeId, seq, *prevAtx, l, 0, *posAtx, b.activeSet.GetActiveSetSize(l-1), b.mesh.GetLatestView(), nipst) + bytes, err := xdr.Marshal(challenge) + if err != nil { + return err + } + npst, err := b.nipstBuilder.BuildNIPST(bytes) + if err != nil { + return err + } + //todo: check if view should be latest layer -1 + atx := types.NewActivationTxWithcChallenge(challenge, b.activeSet.GetActiveSetSize(b.mesh.LatestLayerId()-1), b.mesh.GetLatestView(), npst) buf, err := types.AtxAsBytes(atx) if err != nil { @@ -86,18 +136,31 @@ func (b *Builder) PublishActivationTx(nipst *nipst.NIPST) error { } //todo: should we do something about it? wait for something? return b.net.Broadcast(AtxProtocol, buf) + +} + +func (b *Builder) Persist(c *types.PoETChallenge) { + //todo: implement storing to persistent media +} + +func (b *Builder) Load() *types.PoETChallenge { + //todo: implement loading from persistent media + return nil } func (b *Builder) GetPrevAtxId(node types.NodeId) (*types.AtxId, error) { ids, err := b.db.GetNodeAtxIds(node) - - if err != nil || len(ids) == 0 { + if err != nil { return nil, err } + if len(ids) == 0 { + return nil, fmt.Errorf("no prev atxs for node %v", node.Key) + } return &ids[len(ids)-1], nil } func (b *Builder) GetPositioningAtxId(epochId types.EpochId) (*types.AtxId, error) { + //todo: make this on blocking until an atx is received atxs, err := b.db.GetEpochAtxIds(epochId) if err != nil { return nil, err @@ -119,14 +182,3 @@ func (b *Builder) GetLastSequence(node types.NodeId) uint64 { } return atx.Sequence } - -/*func (m *Mesh) UniqueAtxs(lyr *Layer) map[*ActivationTx]struct{} { - atxMap := make(map[*ActivationTx]struct{}) - for _, blk := range lyr.blocks { - for _, atx := range blk.ATXs { - atxMap[atx] = struct{}{} - } - - } - return atxMap -}*/ diff --git a/activation/activation_test.go b/activation/activation_test.go index 74ee28881c..c8544870fc 100644 --- a/activation/activation_test.go +++ b/activation/activation_test.go @@ -1,6 +1,8 @@ package activation import ( + "fmt" + "github.com/davecgh/go-xdr/xdr" "github.com/spacemeshos/go-spacemesh/common" "github.com/spacemeshos/go-spacemesh/database" "github.com/spacemeshos/go-spacemesh/log" @@ -42,13 +44,29 @@ func (n *NetMock) Broadcast(id string, d []byte) error { return nil } +type NipstBuilderMock struct { + Challenge []byte +} + +func (np *NipstBuilderMock) BuildNIPST(challange []byte) (*nipst.NIPST, error) { + np.Challenge = challange + return &nipst.NIPST{}, nil +} + +type NipstErrBuilderMock struct{} + +func (np *NipstErrBuilderMock) BuildNIPST(challange []byte) (*nipst.NIPST, error) { + return nil, fmt.Errorf("error") +} + func TestBuilder_BuildActivationTx(t *testing.T) { //todo: implement test id := types.NodeId{"aaaa", []byte("bbb")} net := &NetMock{} echp := &EchProvider{} layers := MeshProviderrMock{} - b := NewBuilder(id, database.NewMemDatabase(), mesh.NewMemMeshDB(log.NewDefault("")), net, ActiveSetProviderMock{}, layers, echp, 10) + layersPerEpcoh := types.LayerID(10) + b := NewBuilder(id, database.NewMemDatabase(), mesh.NewMemMeshDB(log.NewDefault("")), net, ActiveSetProviderMock{}, layers, echp, 10, &NipstBuilderMock{}) adb := b.db prevAtx := types.AtxId{Hash: common.HexToHash("0x111")} npst := nipst.NIPST{} @@ -61,9 +79,9 @@ func TestBuilder_BuildActivationTx(t *testing.T) { 5, []types.BlockID{1, 2, 3}, &npst) - adb.StoreAtx(echp.Epoch(atx.LayerIndex), atx) - act := types.NewActivationTx(b.nodeId, b.GetLastSequence(b.nodeId)+1, atx.Id(), layers.LatestLayerId(), 0, atx.Id(), b.activeSet.GetActiveSetSize(1), b.mesh.GetLatestView(), &npst) - err := b.PublishActivationTx(&nipst.NIPST{}) + adb.StoreAtx(echp.Epoch(atx.LayerIdx), atx) + act := types.NewActivationTx(b.nodeId, b.GetLastSequence(b.nodeId)+1, atx.Id(), layers.LatestLayerId()+layersPerEpcoh, 0, atx.Id(), b.activeSet.GetActiveSetSize(1), b.mesh.GetLatestView(), &npst) + err := b.PublishActivationTx(types.EpochId(layers.LatestLayerId() / layersPerEpcoh)) assert.NoError(t, err) bts, err := types.AtxAsBytes(act) assert.NoError(t, err) @@ -76,7 +94,64 @@ func TestBuilder_NoPrevATX(t *testing.T) { net := &NetMock{} echp := &EchProvider{} layers := MeshProviderrMock{} - b := NewBuilder(id, database.NewMemDatabase(), mesh.NewMemMeshDB(log.NewDefault("")), net, ActiveSetProviderMock{}, layers, echp, 10) - err := b.PublishActivationTx(&nipst.NIPST{}) + b := NewBuilder(id, database.NewMemDatabase(), mesh.NewMemMeshDB(log.NewDefault("")), net, ActiveSetProviderMock{}, layers, echp, 10, &NipstBuilderMock{}) + err := b.PublishActivationTx(1) assert.Error(t, err) } + +func TestBuilder_PublishActivationTx(t *testing.T) { + id := types.NodeId{"aaaa", []byte("bbb")} + net := &NetMock{} + echp := &EchProvider{} + layers := MeshProviderrMock{} + nipstBuilder := &NipstBuilderMock{} + layersPerEpcoh := uint64(10) + b := NewBuilder(id, database.NewMemDatabase(), mesh.NewMemMeshDB(log.NewDefault("")), net, ActiveSetProviderMock{}, layers, echp, layersPerEpcoh, nipstBuilder) + adb := b.db + prevAtx := types.AtxId{Hash: common.HexToHash("0x111")} + npst := nipst.NIPST{} + + atx := types.NewActivationTx(types.NodeId{"aaaa", []byte("bbb")}, + 1, + prevAtx, + 5, + 1, + prevAtx, + 5, + []types.BlockID{1, 2, 3}, + &npst) + + err := adb.StoreAtx(echp.Epoch(types.LayerID(uint64(atx.LayerIdx)/layersPerEpcoh)), atx) + assert.NoError(t, err) + + challenge := types.PoETChallenge{ + NodeId: b.nodeId, + Sequence: b.GetLastSequence(b.nodeId) + 1, + PrevATXId: atx.Id(), + LayerIdx: types.LayerID(uint64(layers.LatestLayerId()) + b.layersPerEpoch), + StartTick: atx.EndTick, + EndTick: b.tickProvider.NumOfTicks(), //todo: add provider when + PositioningAtx: atx.Id(), + } + + bytes, err := xdr.Marshal(challenge) + assert.NoError(t, err) + + act := types.NewActivationTx(b.nodeId, b.GetLastSequence(b.nodeId)+1, atx.Id(), layers.LatestLayerId()+10, 0, atx.Id(), b.activeSet.GetActiveSetSize(1), b.mesh.GetLatestView(), &npst) + err = b.PublishActivationTx(1) + assert.NoError(t, err) + bts, err := types.AtxAsBytes(act) + assert.NoError(t, err) + assert.Equal(t, bts, net.bt) + assert.Equal(t, bytes, nipstBuilder.Challenge) + + b.nipstBuilder = &NipstErrBuilderMock{} + err = b.PublishActivationTx(echp.Epoch(1)) + assert.Error(t, err) + assert.Equal(t, err.Error(), "error") + + bt := NewBuilder(id, database.NewMemDatabase(), mesh.NewMemMeshDB(log.NewDefault("")), net, ActiveSetProviderMock{}, layers, echp, layersPerEpcoh, &NipstBuilderMock{}) + err = bt.PublishActivationTx(1) + assert.Error(t, err) + +} diff --git a/activation/activationdb_test.go b/activation/activationdb_test.go index c792a6946a..a94d425ac6 100644 --- a/activation/activationdb_test.go +++ b/activation/activationdb_test.go @@ -165,3 +165,94 @@ func TestMesh_processBlockATXs(t *testing.T) { assert.Equal(t, 3, int(atxdb.ActiveIds(1))) assert.Equal(t, 3, int(atxdb.ActiveIds(2))) } + +func TestActivationDB_ValidateAtx(t *testing.T) { + atxdb, layers := getAtxDb("t8") + + idx1 := types.NodeId{Key: uuid.New().String()} + + id1 := types.NodeId{Key: uuid.New().String()} + id2 := types.NodeId{Key: uuid.New().String()} + id3 := types.NodeId{Key: uuid.New().String()} + atxs := []*types.ActivationTx{ + types.NewActivationTx(id1, 0, types.EmptyAtx, 1, 0, types.EmptyAtx, 3, []types.BlockID{}, &nipst.NIPST{}), + types.NewActivationTx(id2, 0, types.EmptyAtx, 1, 0, types.EmptyAtx, 3, []types.BlockID{}, &nipst.NIPST{}), + types.NewActivationTx(id3, 0, types.EmptyAtx, 1, 0, types.EmptyAtx, 3, []types.BlockID{}, &nipst.NIPST{}), + } + + blocks := createLayerWithAtx(layers, 1, 10, atxs, []types.BlockID{}, []types.BlockID{}) + blocks = createLayerWithAtx(layers, 10, 10, []*types.ActivationTx{}, blocks, blocks) + blocks = createLayerWithAtx(layers, 100, 10, []*types.ActivationTx{}, blocks, blocks) + + //atx := types.NewActivationTx(id1, 1, atxs[0].Id(), 1000, 0, atxs[0].Id(), 3, blocks, &nipst.NIPST{}) + prevAtx := types.NewActivationTx(idx1, 0, types.EmptyAtx, 100, 0, types.EmptyAtx, 3, blocks, &nipst.NIPST{}) + prevAtx.Valid = true + + atx := types.NewActivationTx(idx1, 1, prevAtx.Id(), 1012, 0, prevAtx.Id(), 3, []types.BlockID{}, &nipst.NIPST{}) + atx.VerifiedActiveSet = 3 + err := atxdb.StoreAtx(1, prevAtx) + assert.NoError(t, err) + + err = atxdb.ValidateAtx(atx) + assert.NoError(t, err) +} + +func TestActivationDB_ValidateAtxErrors(t *testing.T) { + atxdb, layers := getAtxDb("t8") + + idx1 := types.NodeId{Key: uuid.New().String()} + + id1 := types.NodeId{Key: uuid.New().String()} + id2 := types.NodeId{Key: uuid.New().String()} + id3 := types.NodeId{Key: uuid.New().String()} + atxs := []*types.ActivationTx{ + types.NewActivationTx(id1, 0, types.EmptyAtx, 1, 0, types.EmptyAtx, 3, []types.BlockID{}, &nipst.NIPST{}), + types.NewActivationTx(id2, 0, types.EmptyAtx, 1, 0, types.EmptyAtx, 3, []types.BlockID{}, &nipst.NIPST{}), + types.NewActivationTx(id3, 0, types.EmptyAtx, 1, 0, types.EmptyAtx, 3, []types.BlockID{}, &nipst.NIPST{}), + } + + blocks := createLayerWithAtx(layers, 1, 10, atxs, []types.BlockID{}, []types.BlockID{}) + blocks = createLayerWithAtx(layers, 10, 10, []*types.ActivationTx{}, blocks, blocks) + blocks = createLayerWithAtx(layers, 100, 10, []*types.ActivationTx{}, blocks, blocks) + + //atx := types.NewActivationTx(id1, 1, atxs[0].Id(), 1000, 0, atxs[0].Id(), 3, blocks, &nipst.NIPST{}) + prevAtx := types.NewActivationTx(idx1, 0, types.EmptyAtx, 100, 0, types.EmptyAtx, 3, blocks, &nipst.NIPST{}) + prevAtx.Valid = true + + err := atxdb.StoreAtx(1, prevAtx) + assert.NoError(t, err) + + //todo: can test against exact error strings + //wrong sequnce + atx := types.NewActivationTx(idx1, 0, prevAtx.Id(), 1012, 0, prevAtx.Id(), 3, []types.BlockID{}, &nipst.NIPST{}) + err = atxdb.ValidateAtx(atx) + assert.Error(t, err) + + //wrong active set + atx = types.NewActivationTx(idx1, 1, prevAtx.Id(), 1012, 0, prevAtx.Id(), 10, []types.BlockID{}, &nipst.NIPST{}) + err = atxdb.ValidateAtx(atx) + assert.Error(t, err) + + //wrong positioning atx + atx = types.NewActivationTx(idx1, 1, prevAtx.Id(), 1012, 0, atxs[0].Id(), 3, []types.BlockID{}, &nipst.NIPST{}) + err = atxdb.ValidateAtx(atx) + assert.Error(t, err) + + //wrong prevATx + atx = types.NewActivationTx(idx1, 1, atxs[0].Id(), 1012, 0, prevAtx.Id(), 3, []types.BlockID{}, &nipst.NIPST{}) + err = atxdb.ValidateAtx(atx) + assert.Error(t, err) + + //wrong layerId + atx = types.NewActivationTx(idx1, 1, prevAtx.Id(), 12, 0, prevAtx.Id(), 3, []types.BlockID{}, &nipst.NIPST{}) + err = atxdb.ValidateAtx(atx) + assert.Error(t, err) + + //atx already exists + err = atxdb.StoreAtx(1, atx) + assert.NoError(t, err) + atx = types.NewActivationTx(idx1, 1, prevAtx.Id(), 12, 0, prevAtx.Id(), 3, []types.BlockID{}, &nipst.NIPST{}) + err = atxdb.ValidateAtx(atx) + assert.Error(t, err) + //atx = types.NewActivationTx(idx1, 1, prevAtx.Id(), 1012, 0, prevAtx.Id(), 3, []types.BlockID{}, &nipst.NIPST{}) +} diff --git a/activation/atxdb.go b/activation/atxdb.go index 04c1aed1ae..de77f96422 100644 --- a/activation/atxdb.go +++ b/activation/atxdb.go @@ -3,6 +3,7 @@ package activation import ( "bytes" "errors" + "fmt" "github.com/davecgh/go-xdr/xdr2" "github.com/spacemeshos/go-spacemesh/common" "github.com/spacemeshos/go-spacemesh/database" @@ -25,21 +26,22 @@ func NewActivationDb(dbstore database.DB, meshDb *mesh.MeshDB, layersPerEpoch ui return &ActivationDb{atxs: dbstore, meshDb: meshDb, LayersPerEpoch: types.LayerID(layersPerEpoch)} } -func (m *ActivationDb) ProcessBlockATXs(blk *types.Block) { +func (db *ActivationDb) ProcessBlockATXs(blk *types.Block) { for _, atx := range blk.ATXs { - activeSet, err := m.CalcActiveSetFromView(atx) + activeSet, err := db.CalcActiveSetFromView(atx) if err != nil { log.Error("could not calculate active set for %v", atx.Id()) } + //todo: maybe there is a potential bug in this case if count for the view can change between calls to this function atx.VerifiedActiveSet = activeSet - err = m.StoreAtx(types.EpochId(atx.LayerIndex/m.LayersPerEpoch), atx) + err = db.StoreAtx(types.EpochId(atx.LayerIdx/db.LayersPerEpoch), atx) if err != nil { log.Error("cannot store atx: %v", atx) } } } -func (m *ActivationDb) CalcActiveSetFromView(a *types.ActivationTx) (uint32, error) { +func (db *ActivationDb) CalcActiveSetFromView(a *types.ActivationTx) (uint32, error) { bytes, err := types.ViewAsBytes(a.View) if err != nil { return 0, err @@ -52,13 +54,13 @@ func (m *ActivationDb) CalcActiveSetFromView(a *types.ActivationTx) (uint32, err var counter uint32 = 0 set := make(map[types.AtxId]struct{}) - firstLayerOfLastEpoch := a.LayerIndex - m.LayersPerEpoch - (a.LayerIndex % m.LayersPerEpoch) - lastLayerOfLastEpoch := firstLayerOfLastEpoch + m.LayersPerEpoch + firstLayerOfLastEpoch := a.LayerIdx - db.LayersPerEpoch - (a.LayerIdx % db.LayersPerEpoch) + lastLayerOfLastEpoch := firstLayerOfLastEpoch + db.LayersPerEpoch traversalFunc := func(blkh *types.BlockHeader) error { - blk, err := m.meshDb.GetBlock(blkh.Id) + blk, err := db.meshDb.GetBlock(blkh.Id) if err != nil { - log.Error("cannot validate atx, block %v not found", blk.Id) + log.Error("cannot validate atx, block %v not found", blkh.Id) return err } //skip blocks not from atx epoch @@ -87,13 +89,66 @@ func (m *ActivationDb) CalcActiveSetFromView(a *types.ActivationTx) (uint32, err mp[blk] = struct{}{} } - m.meshDb.ForBlockInView(mp, firstLayerOfLastEpoch, traversalFunc, errHandler) + db.meshDb.ForBlockInView(mp, firstLayerOfLastEpoch, traversalFunc, errHandler) activesetCache.Add(common.BytesToHash(bytes), counter) return counter, nil } +//todo: move to config +const NIPST_LAYERTIME = 1000 + +func (db *ActivationDb) ValidateAtx(atx *types.ActivationTx) error { + // validation rules: no other atx with same id and sequence number + // if s != 0 the prevAtx is valid and it's seq num is s -1 + // positioning atx is valid + // validate nipst duration? + // fields 1-7 of the atx are the challenge of the poet + // layer index i^ satisfies i -i^ < (layers_passed during nipst creation) ANTON: maybe should be ==? + // the atx view contains d active Ids + + eatx, _ := db.GetAtx(atx.Id()) + if eatx != nil { + return fmt.Errorf("found atx with same id") + } + prevAtxIds, err := db.GetNodeAtxIds(atx.NodeId) + if err == nil && len(prevAtxIds) > 0 { + prevAtx, err := db.GetAtx(prevAtxIds[len(prevAtxIds)-1]) + if err == nil { + if prevAtx.Sequence >= atx.Sequence { + return fmt.Errorf("found atx with same seq or greater") + } + } + } else { + if prevAtxIds == nil && atx.PrevATXId != types.EmptyAtx { + return fmt.Errorf("found atx with invalid prev atx %v", atx.PrevATXId) + } + } + if atx.PositioningAtx != types.EmptyAtx { + posAtx, err := db.GetAtx(atx.PositioningAtx) + if err != nil { + return fmt.Errorf("positioning atx not found") + } + if !posAtx.Valid { + return fmt.Errorf("positioning atx is not valid") + } + if atx.LayerIdx-posAtx.LayerIdx > NIPST_LAYERTIME { + return fmt.Errorf("distance between pos atx invalid %v ", atx.LayerIdx-posAtx.LayerIdx) + } + } else { + if atx.LayerIdx/db.LayersPerEpoch != 0 { + return fmt.Errorf("no positioning atx found") + } + } + + //todo: verify NIPST challange + if atx.VerifiedActiveSet <= uint32(db.ActiveIds(types.EpochId(uint64(atx.LayerIdx)/uint64(db.LayersPerEpoch)))) { + return fmt.Errorf("positioning atx conatins view with more active ids than seen %v %v", atx.VerifiedActiveSet, db.ActiveIds(types.EpochId(uint64(atx.LayerIdx)/uint64(db.LayersPerEpoch)))) + } + return nil +} + func (db *ActivationDb) StoreAtx(ech types.EpochId, atx *types.ActivationTx) error { log.Debug("storing atx %v, in epoch %v", atx, ech) diff --git a/go.mod b/go.mod index 2445541a0c..80e4b791fa 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/seehuhn/mt19937 v0.0.0-20180715112136-cc7708819361 github.com/spacemeshos/merkle-tree v0.0.0-20190327145446-3651a544f849 github.com/spacemeshos/poet-ref v0.0.0-20190404082615-4bb5581dbcba - github.com/spacemeshos/post v0.0.0-20190407095053-5f62914e669a + github.com/spacemeshos/post v0.0.0-20190414091440-9e41a560833b github.com/spacemeshos/sha256-simd v0.0.0-20190111104731-8575aafc88c9 github.com/spf13/cobra v0.0.3 github.com/spf13/pflag v1.0.3 diff --git a/go.sum b/go.sum index a073aa59c9..c9c23ceb45 100644 --- a/go.sum +++ b/go.sum @@ -132,8 +132,8 @@ github.com/spacemeshos/merkle-tree v0.0.0-20190327145446-3651a544f849 h1:RJw90V/ github.com/spacemeshos/merkle-tree v0.0.0-20190327145446-3651a544f849/go.mod h1:mPxjt4RONPxSUhxOq4bhSJyKVGQJ0VMSyRiE51dDLgE= github.com/spacemeshos/poet-ref v0.0.0-20190404082615-4bb5581dbcba h1:SD7ogYNWvANB0hZhDSsBeOpfOTujfJokCf5PcouJCHk= github.com/spacemeshos/poet-ref v0.0.0-20190404082615-4bb5581dbcba/go.mod h1:mQDOJ2bhXYw7a30aPgtzOJGj1tRG+dkmcnfG5RsMONw= -github.com/spacemeshos/post v0.0.0-20190407095053-5f62914e669a h1:qne3iaKPpPWDFqyWlqd5tDowQDm738RkU/Ew0tyE1Qo= -github.com/spacemeshos/post v0.0.0-20190407095053-5f62914e669a/go.mod h1:2FA26QXziszAuvENyHv6pjld6adNW24q8Ls96yZPX60= +github.com/spacemeshos/post v0.0.0-20190414091440-9e41a560833b h1:/R/ocumeVdnFDoLMjLiElMx6K+basPVWOENLvxf/nFc= +github.com/spacemeshos/post v0.0.0-20190414091440-9e41a560833b/go.mod h1:2FA26QXziszAuvENyHv6pjld6adNW24q8Ls96yZPX60= github.com/spacemeshos/sha256-simd v0.0.0-20190111104731-8575aafc88c9 h1:Cc+np6ORem5wrvO+YHQ1sdu71ItiQVRiYB4ugazpgxM= github.com/spacemeshos/sha256-simd v0.0.0-20190111104731-8575aafc88c9/go.mod h1:Pz5LyRghtiEuSixOVGO0da5AxKBZa6+Yq8ZJfjTAPI0= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= diff --git a/nipst/nipst.go b/nipst/nipst.go index 58cd84e356..bf1cf2b274 100644 --- a/nipst/nipst.go +++ b/nipst/nipst.go @@ -1,11 +1,16 @@ package nipst import ( + "encoding/hex" + "errors" "fmt" "github.com/spacemeshos/go-spacemesh/common" - "github.com/spacemeshos/go-spacemesh/crypto" + "github.com/spacemeshos/go-spacemesh/filesystem" "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/post/config" "github.com/spacemeshos/post/proving" + "os" + "path/filepath" "sync" "time" ) @@ -16,14 +21,14 @@ type PostProverClient interface { // to store some data, by having its storage being filled with // pseudo-random data with respect to a specific id. // This data is the result of a computationally-expensive operation. - initialize(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (commitment *postProof, err error) + initialize(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (commitment *PostProof, err error) // execute is the phase in which the prover received a challenge, // and proves that his data is still stored (or was recomputed). // This phase can be repeated arbitrarily many times without repeating initialization; // thus despite the initialization essentially serving as a proof-of-work, // the amortized computational complexity can be made arbitrarily small. - execute(id []byte, challenge common.Hash, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (proof *postProof, err error) + execute(id []byte, challenge common.Hash, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (proof *PostProof, err error) } // PoetProvingServiceClient provides a gateway to a trust-less public proving @@ -71,15 +76,6 @@ type NIPST struct { // a part of PoET. duration SeqWorkTicks - // prev is the previous NIPST, creating a NIPST chain - // in respect to a specific id. - // TODO(moshababo): check whether the previous NIPST is sufficient, or that the ATX is necessary - prev *NIPST - - // postCommitment is the result of the PoST - // initialization process. - postCommitment *postProof - // poetChallenge is the challenge for PoET which is // constructed from the PoST commitment for the initial NIPST (per id), // and from the previous NIPSTs chain for all the following. @@ -104,16 +100,15 @@ type NIPST struct { // postProof is the proof that the prover data // is still stored (or was recomputed). - postProof *postProof + postProof *PostProof } // initialNIPST returns an initial NIPST instance to be used in the NIPST construction. -func initialNIPST(id []byte, space uint64, duration SeqWorkTicks, prev *NIPST) *NIPST { +func initialNIPST(id []byte, space uint64, duration SeqWorkTicks) *NIPST { return &NIPST{ id: id, space: space, duration: duration, - prev: prev, } } @@ -135,19 +130,17 @@ type ActivationBuilder interface { } type NIPSTBuilder struct { - id []byte - space uint64 - difficulty proving.Difficulty - numberOfProvenLabels uint8 - duration SeqWorkTicks - postProver PostProverClient - poetProver PoetProvingServiceClient - verifyPost func(proof *postProof, leafCount uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty) (bool, error) - verifyMembership func(member *common.Hash, proof *membershipProof) (bool, error) - verifyPoet func(p *poetProof) (bool, error) - verifyPoetMembership func(*membershipProof, *poetProof) bool - - activationBuilder ActivationBuilder + id []byte + space uint64 + difficulty proving.Difficulty + numberOfProvenLabels uint8 + duration SeqWorkTicks + postProver PostProverClient + poetProver PoetProvingServiceClient + verifyPost verifyPostFunc + verifyPoetMembership verifyPoetMembershipFunc + verifyPoet verifyPoetFunc + verifyPoetMatchesMembership verifyPoetMatchesMembershipFunc stop bool stopM sync.Mutex @@ -164,241 +157,176 @@ func NewNIPSTBuilder( duration SeqWorkTicks, postProver PostProverClient, poetProver PoetProvingServiceClient, - verifyPost func(proof *postProof, leafCount uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty) (bool, error), - verifyMembership func(member *common.Hash, proof *membershipProof) (bool, error), - verifyPoet func(p *poetProof) (bool, error), - verifyPoetMembership func(*membershipProof, *poetProof) bool, - activationBuilder ActivationBuilder, + verifyPost verifyPostFunc, + verifyPoetMembership verifyPoetMembershipFunc, + verifyPoet verifyPoetFunc, + verifyPoetMatchesMembership verifyPoetMatchesMembershipFunc, ) *NIPSTBuilder { return &NIPSTBuilder{ - id: id, - space: space, - duration: duration, - difficulty: difficulty, - numberOfProvenLabels: numberOfProvenLabels, - postProver: postProver, - poetProver: poetProver, - verifyPost: verifyPost, - verifyMembership: verifyMembership, - verifyPoet: verifyPoet, - verifyPoetMembership: verifyPoetMembership, - activationBuilder: activationBuilder, - stop: false, - errChan: make(chan error), - nipst: initialNIPST(id, space, duration, nil), + id: id, + space: space, + duration: duration, + difficulty: difficulty, + numberOfProvenLabels: numberOfProvenLabels, + postProver: postProver, + poetProver: poetProver, + verifyPost: verifyPost, + verifyPoetMembership: verifyPoetMembership, + verifyPoet: verifyPoet, + verifyPoetMatchesMembership: verifyPoetMatchesMembership, + stop: false, + errChan: make(chan error), + nipst: initialNIPST(id, space, duration), } } -func (nb *NIPSTBuilder) stopped() bool { - nb.stopM.Lock() - defer nb.stopM.Unlock() - return nb.stop -} +func (nb *NIPSTBuilder) BuildNIPST(challenge common.Hash) (*NIPST, error) { + defTimeout := 5 * time.Second // TODO: replace temporary solution + nb.nipst.load() -func (nb *NIPSTBuilder) Stop() { - nb.stopM.Lock() - nb.stop = true - defer nb.stopM.Unlock() -} + if !nb.IsPostInitialized() { + return nil, errors.New("PoST not initialized") + } -func (nb *NIPSTBuilder) Start() { - nb.stopM.Lock() - nb.stop = false - go nb.loop() - defer nb.stopM.Unlock() -} + // Phase 0: Submit challenge to PoET service. + if nb.nipst.poetRound == nil { + poetChallenge := challenge -func (nb *NIPSTBuilder) loop() { - defTimeout := 5 * time.Second // temporary solution - nb.nipst.load() + log.Info("submitting challenge to PoET proving service "+ + "(service id: %v, challenge: %x)", + nb.poetProver.id(), poetChallenge) + + round, err := nb.poetProver.submit(poetChallenge, nb.duration) + if err != nil { + return nil, fmt.Errorf("failed to submit challenge to poet service: %v", err) + } - // Phase 0: PoST initialization. - if nb.nipst.postCommitment == nil { - log.Info("starting PoST initialization (id: %x, space: %v)", - nb.id, nb.space) + log.Info("challenge submitted to PoET proving service "+ + "(service id: %v, round id: %v)", + nb.poetProver.id(), round.id) - commitment, err := nb.postProver.initialize(nb.id, nb.space, nb.numberOfProvenLabels, nb.difficulty, defTimeout) + nb.nipst.poetChallenge = &poetChallenge + nb.nipst.poetRound = round + nb.nipst.persist() + } + + // Phase 1: Wait for PoET service round membership proof. + if nb.nipst.poetMembershipProof == nil { + log.Info("querying round membership proof from PoET proving service "+ + "(service id: %v, round id: %v, challenge: %x)", + nb.poetProver.id(), nb.nipst.poetRound.id, nb.nipst.poetChallenge) + + mproof, err := nb.poetProver.subscribeMembershipProof(nb.nipst.poetRound, *nb.nipst.poetChallenge, defTimeout) if err != nil { - nb.error("failed to initialize PoST: %v", err) - return + return nil, fmt.Errorf("failed to receive PoET round membership proof: %v", err) } - res, err := nb.verifyPost(commitment, nb.space, nb.numberOfProvenLabels, nb.difficulty) + res, err := nb.verifyPoetMembership(nb.nipst.poetChallenge, mproof) if err != nil { - nb.error("received an invalid PoST commitment: %v", err) - return + return nil, fmt.Errorf("received an invalid PoET round membership proof: %v", err) } if !res { - nb.error("received an invalid PoST commitment") - return + return nil, fmt.Errorf("received an invalid PoET round membership proof") } - log.Info("finished PoST initialization (commitment: %x)", commitment.serialize()) + log.Info("received a valid round membership proof from PoET proving service "+ + "(service id: %v, round id: %v, challenge: %x)", + nb.poetProver.id(), nb.nipst.poetRound.id, nb.nipst.poetChallenge) - nb.nipst.postCommitment = commitment + nb.nipst.poetMembershipProof = mproof nb.nipst.persist() } - for { - if nb.stopped() { - return - } + // Phase 2: Wait for PoET service proof. + if nb.nipst.poetProof == nil { + log.Info("waiting for PoET proof from PoET proving service "+ + "(service id: %v, round id: %v, round root commitment: %x)", + nb.poetProver.id(), nb.nipst.poetRound.id, nb.nipst.poetMembershipProof.root) - // Phase 1: Submit challenge to PoET service. - if nb.nipst.poetRound == nil { - var poetChallenge common.Hash - - // If it's the first NIPST in the chain, use the PoST commitment as - // the PoET challenge. Otherwise, use the previous NIPST/ATX hash. - if nb.nipst.prev == nil { - // TODO(moshababo): check what exactly need to be hashed. - poetChallenge = crypto.Keccak256Hash(nb.id, nb.nipst.postCommitment.serialize()) - } else { - // TODO(moshababo): check what exactly need to be hashed. - poetChallenge = crypto.Keccak256Hash(nb.nipst.prev.postProof.serialize()) - } - - log.Info("submitting challenge to PoET proving service "+ - "(service id: %v, challenge: %x)", - nb.poetProver.id(), poetChallenge) - - round, err := nb.poetProver.submit(poetChallenge, nb.duration) - if err != nil { - nb.error("failed to submit challenge to poet service: %v", err) - break - } - - log.Info("challenge submitted to PoET proving service "+ - "(service id: %v, round id: %v)", - nb.poetProver.id(), round.id) - - nb.nipst.poetChallenge = &poetChallenge - nb.nipst.poetRound = round - nb.nipst.persist() - } + proof, err := nb.poetProver.subscribeProof(nb.nipst.poetRound, defTimeout) + if err != nil { + return nil, fmt.Errorf("failed to receive PoET proof: %v", err) - if nb.stopped() { - return } - // Phase 2: Wait for PoET service round membership proof. - if nb.nipst.poetMembershipProof == nil { - log.Info("querying round membership proof from PoET proving service "+ - "(service id: %v, round id: %v, challenge: %x)", - nb.poetProver.id(), nb.nipst.poetRound.id, nb.nipst.poetChallenge) - - mproof, err := nb.poetProver.subscribeMembershipProof(nb.nipst.poetRound, *nb.nipst.poetChallenge, defTimeout) - if err != nil { - nb.error("failed to receive PoET round membership proof: %v", err) - break - } - - res, err := nb.verifyMembership(nb.nipst.poetChallenge, mproof) - if err != nil { - nb.error("received an invalid PoET round membership proof: %v", err) - break - } - if !res { - nb.error("received an invalid PoET round membership proof") - break - } - - log.Info("received a valid round membership proof from PoET proving service "+ - "(service id: %v, round id: %v, challenge: %x)", - nb.poetProver.id(), nb.nipst.poetRound.id, nb.nipst.poetChallenge) - - nb.nipst.poetMembershipProof = mproof - nb.nipst.persist() + if !nb.verifyPoetMatchesMembership(nb.nipst.poetMembershipProof, proof) { + return nil, fmt.Errorf("received an invalid PoET proof due to "+ + "commitment value (expected: %x, found: %x)", + nb.nipst.poetMembershipProof.root, proof.commitment) } - if nb.stopped() { - return + res, err := nb.verifyPoet(proof) + if err != nil { + return nil, fmt.Errorf("received an invalid PoET proof: %v", err) } - - // Phase 3: Wait for PoET service proof. - if nb.nipst.poetProof == nil { - log.Info("waiting for PoET proof from PoET proving service "+ - "(service id: %v, round id: %v, round root commitment: %x)", - nb.poetProver.id(), nb.nipst.poetRound.id, nb.nipst.poetMembershipProof.root) - - proof, err := nb.poetProver.subscribeProof(nb.nipst.poetRound, defTimeout) - if err != nil { - nb.error("failed to receive PoET proof: %v", err) - break - } - - if !nb.verifyPoetMembership(nb.nipst.poetMembershipProof, proof) { - nb.error("received an invalid PoET proof due to "+ - "commitment value (expected: %x, found: %x)", - nb.nipst.poetMembershipProof.root, proof.commitment) - } - - res, err := nb.verifyPoet(proof) - if err != nil { - nb.error("received an invalid PoET proof: %v", err) - break - } - if !res { - nb.error("received an invalid PoET proof") - break - } - - log.Info("received a valid PoET proof from PoET proving service "+ - "(service id: %v, round id: %v)", - nb.poetProver.id(), nb.nipst.poetRound.id) - - nb.nipst.poetProof = proof - nb.nipst.persist() + if !res { + return nil, fmt.Errorf("received an invalid PoET proof") } - if nb.stopped() { - return - } + log.Info("received a valid PoET proof from PoET proving service "+ + "(service id: %v, round id: %v)", + nb.poetProver.id(), nb.nipst.poetRound.id) - // Phase 4: PoST execution. - if nb.nipst.postProof == nil { - // TODO(moshababo): check what exactly need to be hashed. - postChallenge := crypto.Keccak256Hash(nb.nipst.poetProof.serialize()) - - log.Info("starting PoST execution (challenge: %x)", - postChallenge) - - proof, err := nb.postProver.execute(nb.id, postChallenge, nb.numberOfProvenLabels, nb.difficulty, defTimeout) - if err != nil { - nb.error("failed to execute PoST: %v", err) - break - } - - res, err := nb.verifyPost(proof, nb.space, nb.numberOfProvenLabels, nb.difficulty) - if err != nil { - nb.error("received an invalid PoST proof: %v", err) - break - } - if !res { - nb.error("received an invalid PoST proof") - break - } - - log.Info("finished PoST execution (proof: %x)", proof.serialize()) - - nb.nipst.postChallenge = &postChallenge - nb.nipst.postProof = proof - nb.nipst.persist() + nb.nipst.poetProof = proof + nb.nipst.persist() + } + + // Phase 3: PoST execution. + if nb.nipst.postProof == nil { + postChallenge := common.BytesToHash(nb.nipst.poetProof.commitment) + + log.Info("starting PoST execution (challenge: %x)", + postChallenge) + + proof, err := nb.postProver.execute(nb.id, postChallenge, nb.numberOfProvenLabels, nb.difficulty, defTimeout) + if err != nil { + return nil, fmt.Errorf("failed to execute PoST: %v", err) } - log.Info("finished NIPST construction") + res, err := nb.verifyPost(proof, nb.space, nb.numberOfProvenLabels, nb.difficulty) + if err != nil { + return nil, fmt.Errorf("received an invalid PoST proof: %v", err) + } + if !res { + return nil, fmt.Errorf("received an invalid PoST proof") + } - // build the ATX from the NIPST. - nb.activationBuilder.BuildActivationTx(nb.nipst) + log.Info("finished PoST execution (proof: %v)", proof) - // create and set a new NIPST instance for the next iteration. - nb.nipst = initialNIPST(nb.id, nb.space, nb.duration, nb.nipst) + nb.nipst.postChallenge = &postChallenge + nb.nipst.postProof = proof nb.nipst.persist() } + + log.Info("finished NIPST construction") + + return nb.nipst, nil +} + +func (nb *NIPSTBuilder) IsPostInitialized() bool { + postDataPath := filesystem.GetCanonicalPath(config.Post.DataFolder) + labelsPath := filepath.Join(postDataPath, hex.EncodeToString(nb.id)) + _, err := os.Stat(labelsPath) + if os.IsNotExist(err) { + log.Info("could not find labels path at %v", labelsPath) + return false + } + return true } -func (nb *NIPSTBuilder) error(format string, a ...interface{}) { - err := fmt.Errorf(format, a...) - log.Error(err.Error()) - nb.errChan <- err +func (nb *NIPSTBuilder) InitializePost() (*PostProof, error) { + defTimeout := 5 * time.Second // TODO: replace temporary solution + + if nb.IsPostInitialized() { + return nil, errors.New("PoST already initialized") + } + + commitment, err := nb.postProver.initialize(nb.id, nb.space, nb.numberOfProvenLabels, nb.difficulty, defTimeout) + if err != nil { + return nil, fmt.Errorf("failed to initialize PoST: %v", err) + } + + log.Info("finished PoST initialization (commitment: %v)", commitment) + + return commitment, nil } diff --git a/nipst/nipst_test.go b/nipst/nipst_test.go index 311fe38f96..e0396fba45 100644 --- a/nipst/nipst_test.go +++ b/nipst/nipst_test.go @@ -1,21 +1,33 @@ package nipst import ( + "encoding/hex" + "flag" "github.com/spacemeshos/go-spacemesh/common" + "github.com/spacemeshos/go-spacemesh/filesystem" + "github.com/spacemeshos/post/config" "github.com/spacemeshos/post/proving" "github.com/stretchr/testify/require" + "os" + "path/filepath" "testing" "time" ) +var minerID = []byte("id") +var idsToCleanup [][]byte +var spaceUnit = uint64(1024) +var difficulty = proving.Difficulty(5) +var numberOfProvenLabels = uint8(10) + type PostProverClientMock struct{} -func (p *PostProverClientMock) initialize(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (*postProof, error) { - return &postProof{}, nil +func (p *PostProverClientMock) initialize(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (*PostProof, error) { + return &PostProof{}, nil } -func (p *PostProverClientMock) execute(id []byte, challenge common.Hash, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (*postProof, error) { - return &postProof{}, nil +func (p *PostProverClientMock) execute(id []byte, challenge common.Hash, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (*PostProof, error) { + return &PostProof{}, nil } type PoetProvingServiceClientMock struct{} @@ -38,29 +50,18 @@ func (p *PoetProvingServiceClientMock) subscribeProof(r *poetRound, timeout time return &poetProof{}, nil } -type ActivationBuilderMock struct { - nipst chan *NIPST -} - -func (a *ActivationBuilderMock) BuildActivationTx(proof *NIPST) { - a.nipst <- proof -} - func TestNIPSTBuilderWithMocks(t *testing.T) { assert := require.New(t) postProverMock := &PostProverClientMock{} poetProverMock := &PoetProvingServiceClientMock{} - verifyPostMock := func(*postProof, uint64, uint8, proving.Difficulty) (bool, error) { return true, nil } + verifyPostMock := func(*PostProof, uint64, uint8, proving.Difficulty) (bool, error) { return true, nil } verifyMembershipMock := func(*common.Hash, *membershipProof) (bool, error) { return true, nil } verifyPoetMock := func(*poetProof) (bool, error) { return true, nil } verifyPoetMembershipMock := func(*membershipProof, *poetProof) bool { return true } - nipstChan := make(chan *NIPST) - activationBuilder := &ActivationBuilderMock{nipst: nipstChan} - nb := NewNIPSTBuilder( - []byte("id"), + minerID, 1024, 5, proving.NumberOfProvenLabels, @@ -71,19 +72,11 @@ func TestNIPSTBuilderWithMocks(t *testing.T) { verifyMembershipMock, verifyPoetMock, verifyPoetMembershipMock, - activationBuilder, ) - nb.Start() - - select { - case nipst := <-nipstChan: - assert.True(nipst.Valid()) - case <-time.After(5 * time.Second): - assert.Fail("nipst creation timeout") - return - } + npst, err := nb.BuildNIPST(common.BytesToHash([]byte("anton"))) + assert.NoError(err) - nb.Stop() + assert.True(npst.Valid()) } func TestNIPSTBuilderWithClients(t *testing.T) { @@ -91,54 +84,169 @@ func TestNIPSTBuilderWithClients(t *testing.T) { t.Skip() } - assert := require.New(t) + r := require.New(t) - postProver := newPostClient() + nipstChallenge := common.BytesToHash([]byte("anton")) + + npst := buildNIPST(r, spaceUnit, difficulty, numberOfProvenLabels, nipstChallenge) + + err := validateNIPST(npst, spaceUnit, difficulty, numberOfProvenLabels, nipstChallenge) + r.NoError(err) +} +func buildNIPST(r *require.Assertions, spaceUnit uint64, difficulty proving.Difficulty, numberOfProvenLabels uint8, nipstChallenge common.Hash) *NIPST { + postProver := newPostClient() poetProver, err := newRPCPoetHarnessClient() + r.NotNil(poetProver) defer func() { - err := poetProver.CleanUp() - assert.NoError(err) + err = poetProver.CleanUp() + r.NoError(err) }() - assert.NoError(err) - assert.NotNil(poetProver) + r.NoError(err) + nb := NewNIPSTBuilder( + minerID, + spaceUnit, + difficulty, + numberOfProvenLabels, + 600, + postProver, + poetProver, + verifyPost, + verifyPoetMembership, + verifyPoet, + verifyPoetMatchesMembership, + ) + npst, err := nb.BuildNIPST(nipstChallenge) + r.NoError(err) + return npst +} + +func TestNewNIPSTBuilderNotInitialized(t *testing.T) { + if testing.Short() { + t.Skip() + } + + r := require.New(t) - nipstChan := make(chan *NIPST) - activationBuilder := &ActivationBuilderMock{nipst: nipstChan} + minerIDNotInitialized := []byte("not initialized") + nipstChallenge := common.BytesToHash([]byte("anton")) + postProver := newPostClient() + poetProver, err := newRPCPoetHarnessClient() + r.NotNil(poetProver) + defer func() { + err = poetProver.CleanUp() + r.NoError(err) + }() + r.NoError(err) nb := NewNIPSTBuilder( - []byte("id"), - 1024, - 5, - proving.NumberOfProvenLabels, + minerIDNotInitialized, + spaceUnit, + difficulty, + numberOfProvenLabels, 600, postProver, poetProver, verifyPost, - verifyMembership, - verifyPoet, verifyPoetMembership, - activationBuilder, + verifyPoet, + verifyPoetMatchesMembership, ) - done := make(chan struct{}) - go func() { - select { - case err := <-nb.errChan: - assert.Fail(err.Error()) - case <-done: - } - }() + npst, err := nb.BuildNIPST(nipstChallenge) + r.EqualError(err, "PoST not initialized") + r.Nil(npst) + + idsToCleanup = append(idsToCleanup, minerIDNotInitialized) + initialProof, err := nb.InitializePost() + r.NoError(err) + r.NotNil(initialProof) - nb.Start() + npst, err = nb.BuildNIPST(nipstChallenge) + r.NoError(err) + r.NotNil(npst) + + err = validateNIPST(npst, spaceUnit, difficulty, numberOfProvenLabels, nipstChallenge) + r.NoError(err) +} - select { - case nipst := <-nipstChan: - assert.True(nipst.Valid()) - case <-time.After(5 * time.Second): - assert.Fail("timeout") +func TestValidator_Validate(t *testing.T) { + if testing.Short() { + t.Skip() } - nb.Stop() - close(done) + r := require.New(t) + + nipstChallenge := common.BytesToHash([]byte("anton")) + + npst := buildNIPST(r, spaceUnit, difficulty, numberOfProvenLabels, nipstChallenge) + + err := validateNIPST(npst, spaceUnit, difficulty, numberOfProvenLabels, nipstChallenge) + r.NoError(err) + + err = validateNIPST(npst, spaceUnit+1, difficulty, numberOfProvenLabels, nipstChallenge) + r.EqualError(err, "PoST space (1024) is less than a single space unit (1025)") + + err = validateNIPST(npst, spaceUnit, difficulty+1, numberOfProvenLabels, nipstChallenge) + r.EqualError(err, "PoST proof invalid: validation failed: number of derived leaf indices (9) doesn't match number of included proven leaves (8)") + + err = validateNIPST(npst, spaceUnit, difficulty, numberOfProvenLabels+5, nipstChallenge) + r.EqualError(err, "PoST proof invalid: validation failed: number of derived leaf indices (10) doesn't match number of included proven leaves (8)") + + err = validateNIPST(npst, spaceUnit, difficulty, numberOfProvenLabels, common.BytesToHash([]byte("lerner"))) + r.EqualError(err, "NIPST challenge is not equal to expected challenge") +} + +func validateNIPST(npst *NIPST, spaceUnit uint64, difficulty proving.Difficulty, numberOfProvenLabels uint8, + nipstChallenge common.Hash) error { + + v := &Validator{ + PostParams: PostParams{ + Difficulty: difficulty, + NumberOfProvenLabels: numberOfProvenLabels, + SpaceUnit: spaceUnit, + }, + verifyPost: verifyPost, + verifyPoetMembership: verifyPoetMembership, + verifyPoet: verifyPoet, + verifyPoetMatchesMembership: verifyPoetMatchesMembership, + } + return v.Validate(npst, nipstChallenge) +} + +func TestMain(m *testing.M) { + flag.Parse() + initPost(minerID, spaceUnit, 0, difficulty) + res := m.Run() + cleanup() + os.Exit(res) +} + +func initPost(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty) { + defTimeout := 5 * time.Second + idsToCleanup = append(idsToCleanup, id) + _, err := newPostClient().initialize(id, space, numberOfProvenLabels, difficulty, defTimeout) + logIfError(err) +} + +func cleanup() { + matches, err := filepath.Glob("*.bin") + logIfError(err) + for _, f := range matches { + err = os.Remove(f) + logIfError(err) + } + + postDataPath := filesystem.GetCanonicalPath(config.Post.DataFolder) + for _, id := range idsToCleanup { + labelsPath := filepath.Join(postDataPath, hex.EncodeToString(id)) + err = os.RemoveAll(labelsPath) + logIfError(err) + } +} + +func logIfError(err error) { + if err != nil { + _, _ = os.Stderr.WriteString(err.Error()) + } } diff --git a/nipst/poet.go b/nipst/poet.go index a9b29554d9..6c06ec8460 100644 --- a/nipst/poet.go +++ b/nipst/poet.go @@ -38,7 +38,9 @@ func (p *poetProof) serialize() []byte { return []byte("") } -func verifyMembership(member *common.Hash, proof *membershipProof) (bool, error) { +var _ verifyPoetMembershipFunc = verifyPoetMembership + +func verifyPoetMembership(member *common.Hash, proof *membershipProof) (bool, error) { valid, err := merkle.ValidatePartialTree( []uint64{uint64(proof.index)}, [][]byte{member[:]}, @@ -54,6 +56,8 @@ func verifyMembership(member *common.Hash, proof *membershipProof) (bool, error) return valid, nil } +var _ verifyPoetFunc = verifyPoet + func verifyPoet(p *poetProof) (bool, error) { v, err := verifier.New(p.commitment, p.n, shared.NewHashFunc(p.commitment)) if err != nil { @@ -68,9 +72,11 @@ func verifyPoet(p *poetProof) (bool, error) { return res, nil } -// verifyPoetMembership verifies that the poet proof commitment +var _ verifyPoetMatchesMembershipFunc = verifyPoetMatchesMembership + +// verifyPoetMatchesMembership verifies that the poet proof commitment // is the root in which the membership was proven to. -func verifyPoetMembership(m *membershipProof, p *poetProof) bool { +func verifyPoetMatchesMembership(m *membershipProof, p *poetProof) bool { return bytes.Equal(m.root[:], p.commitment) } diff --git a/nipst/poet_test.go b/nipst/poet_test.go index 8f13319068..89069e5944 100644 --- a/nipst/poet_test.go +++ b/nipst/poet_test.go @@ -38,12 +38,12 @@ func TestRPCPoet(t *testing.T) { assert := require.New(t) c, err := newRPCPoetHarnessClient() + assert.NotNil(c) defer func() { err := c.CleanUp() assert.NoError(err) }() assert.NoError(err) - assert.NotNil(c) for _, testCase := range rpcPoetTestCases { success := t.Run(testCase.name, func(t1 *testing.T) { @@ -68,7 +68,7 @@ func testRPCPoetClient(c *RPCPoetClient, assert *require.Assertions) { mProof, err := c.subscribeMembershipProof(poetRound, ch, 10*time.Second) assert.NoError(err) assert.NotNil(mProof) - res, err := verifyMembership(&ch, mProof) + res, err := verifyPoetMembership(&ch, mProof) assert.NoError(err) assert.True(res) @@ -79,7 +79,7 @@ func testRPCPoetClient(c *RPCPoetClient, assert *require.Assertions) { assert.NoError(err) assert.True(res) - assert.True(verifyPoetMembership(mProof, proof)) + assert.True(verifyPoetMatchesMembership(mProof, proof)) } func testRPCPoetClientTimeouts(c *RPCPoetClient, assert *require.Assertions) { diff --git a/nipst/post.go b/nipst/post.go index 5977e00fdc..31edabd534 100644 --- a/nipst/post.go +++ b/nipst/post.go @@ -1,7 +1,10 @@ package nipst import ( + "encoding/hex" + "fmt" "github.com/spacemeshos/go-spacemesh/common" + "github.com/spacemeshos/merkle-tree" "github.com/spacemeshos/post/initialization" "github.com/spacemeshos/post/proving" "github.com/spacemeshos/post/validation" @@ -9,14 +12,34 @@ import ( "time" ) -type postProof proving.Proof +type PostProof proving.Proof -func (p *postProof) serialize() []byte { +func (p *PostProof) String() string { + return fmt.Sprintf("id: %v, challenge: %v, root: %v", + bytesToShortString(p.Identity), bytesToShortString(p.Challenge), bytesToShortString(p.MerkleRoot)) +} + +func bytesToShortString(b []byte) string { + l := len(b) + if l == 0 { + return "empty" + } + if l > 5 { + l = 5 + } + return fmt.Sprintf("\"%s…\"", hex.EncodeToString(b)[:l]) +} + +func (p *PostProof) serialize() []byte { // TODO(moshababo): implement return []byte("") } -func verifyPost(proof *postProof, leafCount uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty) (bool, error) { +func verifyPost(proof *PostProof, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty) (bool, error) { + if space%merkle.NodeSize != 0 { + return false, fmt.Errorf("space (%d) is not a multiple of merkle.NodeSize (%d)", space, merkle.NodeSize) + } + leafCount := space / merkle.NodeSize err := validation.Validate(proving.Proof(*proof), leafCount, numberOfProvenLabels, difficulty) if err != nil { return false, err @@ -31,14 +54,18 @@ func newPostClient() *PostClient { return &PostClient{} } -func (c *PostClient) initialize(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (commitment *postProof, err error) { +func (c *PostClient) initialize(id []byte, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (commitment *PostProof, err error) { // TODO(moshababo): implement timeout - proof, err := initialization.Initialize(id, space, numberOfProvenLabels, difficulty) - return (*postProof)(&proof), err + if space%merkle.NodeSize != 0 { + return nil, fmt.Errorf("space (%d) is not a multiple of merkle.NodeSize (%d)", space, merkle.NodeSize) + } + leafCount := space / merkle.NodeSize + proof, err := initialization.Initialize(id, leafCount, numberOfProvenLabels, difficulty) + return (*PostProof)(&proof), err } -func (c *PostClient) execute(id []byte, challenge common.Hash, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (*postProof, error) { +func (c *PostClient) execute(id []byte, challenge common.Hash, numberOfProvenLabels uint8, difficulty proving.Difficulty, timeout time.Duration) (*PostProof, error) { // TODO(moshababo): implement timeout proof, err := proving.GenerateProof(id, challenge[:], numberOfProvenLabels, difficulty) - return (*postProof)(&proof), err + return (*PostProof)(&proof), err } diff --git a/nipst/post_test.go b/nipst/post_test.go index 6df5682c5b..e4aa706414 100644 --- a/nipst/post_test.go +++ b/nipst/post_test.go @@ -22,6 +22,7 @@ func TestPostClient(t *testing.T) { c := newPostClient() assert.NotNil(c) + idsToCleanup = append(idsToCleanup, id) commitment, err := c.initialize(id, space, numberOfProvenLabels, difficulty, 0) assert.NoError(err) assert.NotNil(commitment) diff --git a/nipst/validator.go b/nipst/validator.go new file mode 100644 index 0000000000..0b5051d2c1 --- /dev/null +++ b/nipst/validator.go @@ -0,0 +1,79 @@ +package nipst + +import ( + "bytes" + "errors" + "fmt" + "github.com/spacemeshos/go-spacemesh/common" + "github.com/spacemeshos/go-spacemesh/log" + "github.com/spacemeshos/post/proving" +) + +type verifyPostFunc func(proof *PostProof, space uint64, numberOfProvenLabels uint8, difficulty proving.Difficulty) (bool, error) +type verifyPoetMembershipFunc func(member *common.Hash, proof *membershipProof) (bool, error) +type verifyPoetFunc func(p *poetProof) (bool, error) +type verifyPoetMatchesMembershipFunc func(*membershipProof, *poetProof) bool + +type Validator struct { + PostParams + verifyPost verifyPostFunc + verifyPoetMembership verifyPoetMembershipFunc + verifyPoet verifyPoetFunc + verifyPoetMatchesMembership verifyPoetMatchesMembershipFunc +} + +type PostParams struct { + Difficulty proving.Difficulty + NumberOfProvenLabels uint8 + SpaceUnit uint64 +} + +func (v *Validator) Validate(nipst *NIPST, expectedChallenge common.Hash) error { + if !bytes.Equal(nipst.poetChallenge[:], expectedChallenge[:]) { + log.Warning("NIPST challenge is not equal to expected challenge") + return errors.New("NIPST challenge is not equal to expected challenge") + } + + if valid, err := v.verifyPoetMembership(nipst.poetChallenge, nipst.poetMembershipProof); err != nil || !valid { + log.Warning("PoET membership proof invalid: %v", err) + return fmt.Errorf("PoET membership proof invalid: %v", err) + } + + if valid := v.verifyPoetMatchesMembership(nipst.poetMembershipProof, nipst.poetProof); !valid { + log.Warning("PoET membership root does not match PoET proof") + return errors.New("PoET membership root does not match PoET proof") + } + + if valid, err := v.verifyPoet(nipst.poetProof); err != nil || !valid { + log.Warning("PoET proof invalid: %v", err) + return fmt.Errorf("PoET proof invalid: %v", err) + } + + if !bytes.Equal(nipst.postChallenge[:], nipst.postProof.Challenge) { + log.Warning("PoST challenge in NIPST does not match the one in the PoST proof") + return errors.New("PoST challenge in NIPST does not match the one in the PoST proof") + } + + if !bytes.Equal(nipst.postChallenge[:], nipst.poetProof.commitment) { + log.Warning("PoST challenge does not match PoET commitment") + return errors.New("PoST challenge does not match PoET commitment") + } + + if !bytes.Equal(nipst.postProof.Identity, nipst.id) { + log.Warning("PoST proof identity does not match NIPST identity") + return errors.New("PoST proof identity does not match NIPST identity") + } + + if nipst.space < v.SpaceUnit { + log.Warning("PoST space (%d) is less than a single space unit (%d)", nipst.space, v.SpaceUnit) + return fmt.Errorf("PoST space (%d) is less than a single space unit (%d)", nipst.space, v.SpaceUnit) + } + + if valid, err := v.verifyPost(nipst.postProof, nipst.space, v.NumberOfProvenLabels, v.Difficulty); err != nil || !valid { + + log.Warning("PoST proof invalid: %v", err) + return fmt.Errorf("PoST proof invalid: %v", err) + } + + return nil +} diff --git a/oracle/blockeligibilityvalidator.go b/oracle/blockeligibilityvalidator.go index 2c7940fda8..12278d6496 100644 --- a/oracle/blockeligibilityvalidator.go +++ b/oracle/blockeligibilityvalidator.go @@ -42,7 +42,7 @@ func (v BlockEligibilityValidator) BlockEligible(block *types.Block) (bool, erro log.Error("ATX is invalid: %v", err) return false, err } - if atxEpochNumber := atx.LayerIndex.GetEpoch(v.layersPerEpoch); epochNumber != atxEpochNumber { + if atxEpochNumber := atx.LayerIdx.GetEpoch(v.layersPerEpoch); epochNumber != atxEpochNumber { log.Error("ATX epoch (%d) doesn't match layer ID epoch (%d)", atxEpochNumber, epochNumber) return false, fmt.Errorf("activation epoch (%d) mismatch with layer epoch (%d)", atxEpochNumber, epochNumber) diff --git a/oracle/blockoracle.go b/oracle/blockoracle.go index 9ee1f7a611..c96178ced3 100644 --- a/oracle/blockoracle.go +++ b/oracle/blockoracle.go @@ -22,7 +22,7 @@ type MinerBlockOracle struct { vrfSigner *crypto.VRFSigner nodeID types.NodeId - proofsEpoch uint64 + proofsEpoch types.EpochId eligibilityProofs map[types.LayerID][]types.BlockEligibilityProof atxID types.AtxId } @@ -37,7 +37,7 @@ func NewMinerBlockOracle(committeeSize int32, layersPerEpoch uint16, activationD beaconProvider: beaconProvider, vrfSigner: vrfSigner, nodeID: nodeId, - proofsEpoch: ^uint64(0), + proofsEpoch: ^types.EpochId(0), } } @@ -55,7 +55,7 @@ func (bo *MinerBlockOracle) BlockEligible(layerID types.LayerID) (types.AtxId, [ return bo.atxID, bo.eligibilityProofs[layerID], nil } -func (bo *MinerBlockOracle) calcEligibilityProofs(epochNumber uint64) error { +func (bo *MinerBlockOracle) calcEligibilityProofs(epochNumber types.EpochId) error { epochBeacon := bo.beaconProvider.GetBeacon(epochNumber) latestATXID, err := getLatestATXID(bo.activationDb, bo.nodeID) @@ -92,8 +92,8 @@ func (bo *MinerBlockOracle) calcEligibilityProofs(epochNumber uint64) error { return nil } -func calcEligibleLayer(epochNumber uint64, layersPerEpoch uint16, vrfHash [32]byte) types.LayerID { - epochOffset := epochNumber * uint64(layersPerEpoch) +func calcEligibleLayer(epochNumber types.EpochId, layersPerEpoch uint16, vrfHash [32]byte) types.LayerID { + epochOffset := uint64(epochNumber) * uint64(layersPerEpoch) vrfInteger := binary.LittleEndian.Uint64(vrfHash[:8]) eligibleLayerRelativeToEpochStart := vrfInteger % uint64(layersPerEpoch) return types.LayerID(eligibleLayerRelativeToEpochStart + epochOffset) @@ -126,10 +126,10 @@ func getLatestATXID(activationDb ActivationDb, nodeID types.NodeId) (types.AtxId return latestATXID, err } -func serializeVRFMessage(epochBeacon []byte, epochNumber uint64, counter uint32) []byte { +func serializeVRFMessage(epochBeacon []byte, epochNumber types.EpochId, counter uint32) []byte { message := make([]byte, len(epochBeacon)+binary.Size(epochNumber)+binary.Size(counter)) copy(message, epochBeacon) - binary.LittleEndian.PutUint64(message[len(epochBeacon):], epochNumber) + binary.LittleEndian.PutUint64(message[len(epochBeacon):], uint64(epochNumber)) binary.LittleEndian.PutUint32(message[len(epochBeacon)+binary.Size(epochNumber):], counter) return message } diff --git a/oracle/blockoracle_test.go b/oracle/blockoracle_test.go index f7f88bc9ab..d5d46839aa 100644 --- a/oracle/blockoracle_test.go +++ b/oracle/blockoracle_test.go @@ -35,7 +35,7 @@ func (a mockActivationDB) GetNodeAtxIds(node types.NodeId) ([]types.AtxId, error func (a mockActivationDB) GetAtx(id types.AtxId) (*types.ActivationTx, error) { if id == atxID { return &types.ActivationTx{ActivationTxHeader: types.ActivationTxHeader{ActiveSetSize: a.activeSetSize, - LayerIndex: a.layerIndex}}, nil + PoETChallenge: types.PoETChallenge{LayerIdx: a.layerIndex}}}, nil } return nil, errors.New("wrong atx id") } diff --git a/oracle/epochbeacon.go b/oracle/epochbeacon.go index ee257b6450..4e88a0f43e 100644 --- a/oracle/epochbeacon.go +++ b/oracle/epochbeacon.go @@ -1,12 +1,15 @@ package oracle -import "encoding/binary" +import ( + "encoding/binary" + "github.com/spacemeshos/go-spacemesh/types" +) type EpochBeaconProvider struct { } -func (p *EpochBeaconProvider) GetBeacon(epochNumber uint64) []byte { +func (p *EpochBeaconProvider) GetBeacon(epochNumber types.EpochId) []byte { ret := make([]byte, 32) - binary.LittleEndian.PutUint64(ret, epochNumber) + binary.LittleEndian.PutUint64(ret, uint64(epochNumber)) return ret } diff --git a/types/activation.go b/types/activation.go index d2970541d5..0734567a23 100644 --- a/types/activation.go +++ b/types/activation.go @@ -17,16 +17,21 @@ type AtxId struct { var EmptyAtx = AtxId{common.Hash{0}} type ActivationTxHeader struct { - NodeId NodeId - Sequence uint64 - PrevATXId AtxId - LayerIndex LayerID - StartTick uint64 //implement later - EndTick uint64 //implement later - PositioningATX AtxId - ActiveSetSize uint32 + PoETChallenge VerifiedActiveSet uint32 + ActiveSetSize uint32 View []BlockID + Valid bool +} + +type PoETChallenge struct { + NodeId NodeId + Sequence uint64 + PrevATXId AtxId + LayerIdx LayerID + StartTick uint64 + EndTick uint64 + PositioningAtx AtxId } type ActivationTx struct { @@ -39,15 +44,31 @@ func NewActivationTx(NodeId NodeId, Sequence uint64, PrevATX AtxId, LayerIndex L StartTick uint64, PositioningATX AtxId, ActiveSetSize uint32, View []BlockID, nipst *nipst.NIPST) *ActivationTx { return &ActivationTx{ ActivationTxHeader: ActivationTxHeader{ - NodeId: NodeId, - Sequence: Sequence, - PrevATXId: PrevATX, - LayerIndex: LayerIndex, - StartTick: StartTick, - PositioningATX: PositioningATX, - ActiveSetSize: ActiveSetSize, - View: View, + PoETChallenge: PoETChallenge{ + NodeId: NodeId, + Sequence: Sequence, + PrevATXId: PrevATX, + LayerIdx: LayerIndex, + StartTick: StartTick, + PositioningAtx: PositioningATX, + }, + ActiveSetSize: ActiveSetSize, + View: View, + }, + + Nipst: nipst, + } + +} + +func NewActivationTxWithcChallenge(poetChallenge PoETChallenge, ActiveSetSize uint32, View []BlockID, nipst *nipst.NIPST) *ActivationTx { + return &ActivationTx{ + ActivationTxHeader: ActivationTxHeader{ + PoETChallenge: poetChallenge, + ActiveSetSize: ActiveSetSize, + View: View, }, + Nipst: nipst, } @@ -64,5 +85,13 @@ func (t ActivationTx) Id() AtxId { func (t ActivationTx) Validate() error { //todo: implement + // valid signature + // no other atx with same id and sequence number + // if s != 0 the prevAtx is valid and it's seq num is s -1 + // positioning atx is valid + // validate nipst duration? + // fields 1-7 of the atx are the challenge of the poet + // layer index i^ satisfies i -i^ < (layers_passed during nipst creation) ANTON: maybe should be ==? + // the atx view contains d active Ids return nil } diff --git a/types/block.go b/types/block.go index 98ae96a51f..fddeb2ec59 100644 --- a/types/block.go +++ b/types/block.go @@ -13,8 +13,8 @@ type BlockID uint64 type TransactionId []byte type LayerID uint64 -func (l LayerID) GetEpoch(layersPerEpoch uint16) uint64 { - return uint64(l) / uint64(layersPerEpoch) +func (l LayerID) GetEpoch(layersPerEpoch uint16) EpochId { + return EpochId(uint64(l) / uint64(layersPerEpoch)) } //todo: choose which type is VRF