Merge pull request #3288 from tendermint/release/v0.30.0

Release/v0.30.0

CHANGELOG.md

@@ -1,5 +1,52 @@
 # Changelog
 
+## v0.30.0
+
+*February 8th, 2019*
+
+This release fixes yet another issue with the proposer selection algorithm.
+We hope it's the last one, but we won't be surprised if it's not.
+We plan to one day expose the selection algorithm more directly to
+the application ([\#3285](https://github.com/tendermint/tendermint/issues/3285)), and even to support randomness ([\#763](https://github.com/tendermint/tendermint/issues/763)).
+For more, see issues marked
+[proposer-selection](https://github.com/tendermint/tendermint/labels/proposer-selection).
+
+This release also includes a fix to prevent Tendermint from including the same
+piece of evidence in more than one block. This issue was reported by @chengwenxi in our
+[bug bounty program](https://hackerone.com/tendermint).
+
+### BREAKING CHANGES:
+
+* Apps
+  - [state] [\#3222](https://github.com/tendermint/tendermint/issues/3222)
+    Duplicate updates for the same validator are forbidden. Apps must ensure
+    that a given `ResponseEndBlock.ValidatorUpdates` contains only one entry per pubkey.
+
+* Go API
+  - [types] [\#3222](https://github.com/tendermint/tendermint/issues/3222)
+    Remove `Add` and `Update` methods from `ValidatorSet` in favor of new
+    `UpdateWithChangeSet`. This allows updates to be applied as a set, instead of
+    one at a time.
+
+* Block Protocol
+  - [state] [\#3286](https://github.com/tendermint/tendermint/issues/3286) Blocks that include already committed evidence are invalid.
+
+* P2P Protocol
+  - [consensus] [\#3222](https://github.com/tendermint/tendermint/issues/3222)
+    Validator updates are applied as a set, instead of one at a time, thus
+    impacting the proposer priority calculation. This ensures that the proposer
+    selection algorithm does not depend on the order of updates in
+    `ResponseEndBlock.ValidatorUpdates`.
+
+### IMPROVEMENTS:
+- [crypto] [\#3279](https://github.com/tendermint/tendermint/issues/3279) Use `btcec.S256().N` directly instead of hard coding a copy.
+
+### BUG FIXES:
+- [state] [\#3222](https://github.com/tendermint/tendermint/issues/3222) Fix validator set updates so they are applied as a set, rather
+  than one at a time. This makes the proposer selection algorithm independent of
+  the order of updates in `ResponseEndBlock.ValidatorUpdates`.
+- [evidence] [\#3286](https://github.com/tendermint/tendermint/issues/3286) Don't add committed evidence to evidence pool.
+
 ## v0.29.2
 
 *February 7th, 2019*
@@ -11,7 +58,7 @@ Special thanks to external contributors on this release:
 `crypto` packages:
 - p2p:
   - Partial fix for MITM attacks on the p2p connection. MITM conditions may
-    still exist. See \#3010.
+    still exist. See [\#3010](https://github.com/tendermint/tendermint/issues/3010).
 - crypto:
   - Eliminate our fork of `btcd` and use the `btcd/btcec` library directly for
     native secp256k1 signing. Note we still modify the signature encoding to

CHANGELOG_PENDING.md

@@ -1,11 +1,21 @@
-## v0.30
+## v0.31.0
 
 **
 
 Special thanks to external contributors on this release:
 
 ### BREAKING CHANGES:
 
+* CLI/RPC/Config
+
+* Apps
+
+* Go API
+
+* Blockchain Protocol
+
+* P2P Protocol
+
 ### FEATURES:
 
 ### IMPROVEMENTS:

UPGRADING.md

@@ -3,6 +3,29 @@
 This guide provides steps to be followed when you upgrade your applications to
 a newer version of Tendermint Core.
 
+## v0.30.0
+
+This release contains a breaking change to both the block and p2p protocols,
+however it may be compatible with blockchains created with v0.29.0 depending on
+the chain history. If your blockchain has not included any pieces of evidence,
+or no piece of evidence has been included in more than one block,
+and if your application has never returned multiple updates
+for the same validator in a single block, then v0.30.0 will work fine with
+blockchains created with v0.29.0.
+
+The p2p protocol change is to fix the proposer selection algorithm again.
+Note that proposer selection is purely a p2p concern right
+now since the algorithm is only relevant during real time consensus.
+This change is thus compatible with v0.29.0, but
+all nodes must be upgraded to avoid disagreements on the proposer.
+
+### Applications
+
+Applications must ensure they do not return duplicates in
+`ResponseEndBlock.ValidatorUpdates`. A pubkey must only appear once per set of
+updates. Duplicates will cause irrecoverable failure. If you have a very good
+reason why we shouldn't do this, please open an issue.
+
 ## v0.29.0
 
 This release contains some breaking changes to the block and p2p protocols,

blockchain/reactor_test.go

@@ -95,13 +95,13 @@ func newBlockchainReactor(logger log.Logger, genDoc *types.GenesisDoc, privVals
 
 	// let's add some blocks in
 	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
-		lastCommit := &types.Commit{}
+		lastCommit := types.NewCommit(types.BlockID{}, nil)
 		if blockHeight > 1 {
 			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
 			lastBlock := blockStore.LoadBlock(blockHeight - 1)
 
 			vote := makeVote(&lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVals[0]).CommitSig()
-			lastCommit = &types.Commit{Precommits: []*types.CommitSig{vote}, BlockID: lastBlockMeta.BlockID}
+			lastCommit = types.NewCommit(lastBlockMeta.BlockID, []*types.CommitSig{vote})
 		}
 
 		thisBlock := makeBlock(blockHeight, state, lastCommit)

blockchain/store_test.go

@@ -23,11 +23,8 @@ import (
 
 // make a Commit with a single vote containing just the height and a timestamp
 func makeTestCommit(height int64, timestamp time.Time) *types.Commit {
-	return &types.Commit{
-		Precommits: []*types.CommitSig{
-			{Height: height, Timestamp: timestamp},
-		},
-	}
+	commitSigs := []*types.CommitSig{{Height: height, Timestamp: timestamp}}
+	return types.NewCommit(types.BlockID{}, commitSigs)
 }
 
 func makeStateAndBlockStore(logger log.Logger) (sm.State, *BlockStore) {

cmd/tendermint/commands/reset_priv_validator.go

@@ -61,7 +61,7 @@ func resetFilePV(privValKeyFile, privValStateFile string, logger log.Logger) {
 	} else {
 		pv := privval.GenFilePV(privValKeyFile, privValStateFile)
 		pv.Save()
-		logger.Info("Generated private validator file", "file", "keyFile", privValKeyFile,
+		logger.Info("Generated private validator file", "keyFile", privValKeyFile,
 			"stateFile", privValStateFile)
 	}
 }

consensus/reactor_test.go

@@ -211,6 +211,7 @@ func (m *mockEvidencePool) Update(block *types.Block, state sm.State) {
 	}
 	m.height++
 }
+func (m *mockEvidencePool) IsCommitted(types.Evidence) bool { return false }
 
 //------------------------------------
 

consensus/replay_test.go

@@ -537,10 +537,8 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 			}
 		case *types.Vote:
 			if p.Type == types.PrecommitType {
-				thisBlockCommit = &types.Commit{
-					BlockID:    p.BlockID,
-					Precommits: []*types.CommitSig{p.CommitSig()},
-				}
+				commitSigs := []*types.CommitSig{p.CommitSig()}
+				thisBlockCommit = types.NewCommit(p.BlockID, commitSigs)
 			}
 		}
 	}

consensus/state.go

@@ -954,7 +954,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 	if cs.Height == 1 {
 		// We're creating a proposal for the first block.
 		// The commit is empty, but not nil.
-		commit = &types.Commit{}
+		commit = types.NewCommit(types.BlockID{}, nil)
 	} else if cs.LastCommit.HasTwoThirdsMajority() {
 		// Make the commit from LastCommit
 		commit = cs.LastCommit.MakeCommit()

consensus/types/round_state_test.go

@@ -53,11 +53,8 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 		Data: types.Data{
 			Txs: txs,
 		},
-		Evidence: types.EvidenceData{},
-		LastCommit: &types.Commit{
-			BlockID:    blockID,
-			Precommits: precommits,
-		},
+		Evidence:   types.EvidenceData{},
+		LastCommit: types.NewCommit(blockID, precommits),
 	}
 	parts := block.MakePartSet(4096)
 	// Random Proposal

crypto/secp256k1/secp256k1_nocgo.go

@@ -14,8 +14,7 @@ import (
 // see:
 //  - https://github.com/ethereum/go-ethereum/blob/f9401ae011ddf7f8d2d95020b7446c17f8d98dc1/crypto/signature_nocgo.go#L90-L93
 //  - https://github.com/ethereum/go-ethereum/blob/f9401ae011ddf7f8d2d95020b7446c17f8d98dc1/crypto/crypto.go#L39
-var secp256k1N, _ = new(big.Int).SetString("fffffffffffffffffffffffffffffffebaaedce6af48a03bbfd25e8cd0364141", 16)
-var secp256k1halfN = new(big.Int).Div(secp256k1N, big.NewInt(2))
+var secp256k1halfN = new(big.Int).Rsh(secp256k1.S256().N, 1)
 
 // Sign creates an ECDSA signature on curve Secp256k1, using SHA256 on the msg.
 // The returned signature will be of the form R || S (in lower-S form).

evidence/pool.go

@@ -28,7 +28,8 @@ type EvidencePool struct {
 	state sm.State
 }
 
-func NewEvidencePool(stateDB dbm.DB, evidenceStore *EvidenceStore) *EvidencePool {
+func NewEvidencePool(stateDB, evidenceDB dbm.DB) *EvidencePool {
+	evidenceStore := NewEvidenceStore(evidenceDB)
 	evpool := &EvidencePool{
 		stateDB:       stateDB,
 		state:         sm.LoadState(stateDB),
@@ -132,6 +133,12 @@ func (evpool *EvidencePool) MarkEvidenceAsCommitted(height int64, evidence []typ
 
 }
 
+// IsCommitted returns true if we have already seen this exact evidence and it is already marked as committed.
+func (evpool *EvidencePool) IsCommitted(evidence types.Evidence) bool {
+	ei := evpool.evidenceStore.getEvidenceInfo(evidence)
+	return ei.Evidence != nil && ei.Committed
+}
+
 func (evpool *EvidencePool) removeEvidence(height, maxAge int64, blockEvidenceMap map[string]struct{}) {
 	for e := evpool.evidenceList.Front(); e != nil; e = e.Next() {
 		ev := e.Value.(types.Evidence)

evidence/pool_test.go

@@ -56,8 +56,8 @@ func TestEvidencePool(t *testing.T) {
 	valAddr := []byte("val1")
 	height := int64(5)
 	stateDB := initializeValidatorState(valAddr, height)
-	store := NewEvidenceStore(dbm.NewMemDB())
-	pool := NewEvidencePool(stateDB, store)
+	evidenceDB := dbm.NewMemDB()
+	pool := NewEvidencePool(stateDB, evidenceDB)
 
 	goodEvidence := types.NewMockGoodEvidence(height, 0, valAddr)
 	badEvidence := types.MockBadEvidence{goodEvidence}
@@ -84,3 +84,24 @@ func TestEvidencePool(t *testing.T) {
 	assert.Nil(t, err)
 	assert.Equal(t, 1, pool.evidenceList.Len())
 }
+
+func TestEvidencePoolIsCommitted(t *testing.T) {
+	// Initialization:
+	valAddr := []byte("validator_address")
+	height := int64(42)
+	stateDB := initializeValidatorState(valAddr, height)
+	evidenceDB := dbm.NewMemDB()
+	pool := NewEvidencePool(stateDB, evidenceDB)
+
+	// evidence not seen yet:
+	evidence := types.NewMockGoodEvidence(height, 0, valAddr)
+	assert.False(t, pool.IsCommitted(evidence))
+
+	// evidence seen but not yet committed:
+	assert.NoError(t, pool.AddEvidence(evidence))
+	assert.False(t, pool.IsCommitted(evidence))
+
+	// evidence seen and committed:
+	pool.MarkEvidenceAsCommitted(height, []types.Evidence{evidence})
+	assert.True(t, pool.IsCommitted(evidence))
+}

evidence/reactor_test.go

@@ -37,8 +37,8 @@ func makeAndConnectEvidenceReactors(config *cfg.Config, stateDBs []dbm.DB) []*Ev
 	logger := evidenceLogger()
 	for i := 0; i < N; i++ {
 
-		store := NewEvidenceStore(dbm.NewMemDB())
-		pool := NewEvidencePool(stateDBs[i], store)
+		evidenceDB := dbm.NewMemDB()
+		pool := NewEvidencePool(stateDBs[i], evidenceDB)
 		reactors[i] = NewEvidenceReactor(pool)
 		reactors[i].SetLogger(logger.With("validator", i))
 	}

evidence/store.go

@@ -117,32 +117,33 @@ func (store *EvidenceStore) listEvidence(prefixKey string, maxNum int64) (eviden
 	return evidence
 }
 
-// GetEvidence fetches the evidence with the given height and hash.
-func (store *EvidenceStore) GetEvidence(height int64, hash []byte) *EvidenceInfo {
+// GetEvidenceInfo fetches the EvidenceInfo with the given height and hash.
+// If not found, ei.Evidence is nil.
+func (store *EvidenceStore) GetEvidenceInfo(height int64, hash []byte) EvidenceInfo {
 	key := keyLookupFromHeightAndHash(height, hash)
 	val := store.db.Get(key)
 
 	if len(val) == 0 {
-		return nil
+		return EvidenceInfo{}
 	}
 	var ei EvidenceInfo
 	err := cdc.UnmarshalBinaryBare(val, &ei)
 	if err != nil {
 		panic(err)
 	}
-	return &ei
+	return ei
 }
 
 // AddNewEvidence adds the given evidence to the database.
 // It returns false if the evidence is already stored.
 func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int64) bool {
 	// check if we already have seen it
-	ei_ := store.GetEvidence(evidence.Height(), evidence.Hash())
-	if ei_ != nil && ei_.Evidence != nil {
+	ei := store.getEvidenceInfo(evidence)
+	if ei.Evidence != nil {
 		return false
 	}
 
-	ei := EvidenceInfo{
+	ei = EvidenceInfo{
 		Committed: false,
 		Priority:  priority,
 		Evidence:  evidence,
@@ -165,6 +166,11 @@ func (store *EvidenceStore) AddNewEvidence(evidence types.Evidence, priority int
 // MarkEvidenceAsBroadcasted removes evidence from Outqueue.
 func (store *EvidenceStore) MarkEvidenceAsBroadcasted(evidence types.Evidence) {
 	ei := store.getEvidenceInfo(evidence)
+	if ei.Evidence == nil {
+		// nothing to do; we did not store the evidence yet (AddNewEvidence):
+		return
+	}
+	// remove from the outqueue
 	key := keyOutqueue(evidence, ei.Priority)
 	store.db.Delete(key)
 }
@@ -177,8 +183,12 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) {
 	pendingKey := keyPending(evidence)
 	store.db.Delete(pendingKey)
 
-	ei := store.getEvidenceInfo(evidence)
-	ei.Committed = true
+	// committed EvidenceInfo doens't need priority
+	ei := EvidenceInfo{
+		Committed: true,
+		Evidence:  evidence,
+		Priority:  0,
+	}
 
 	lookupKey := keyLookup(evidence)
 	store.db.SetSync(lookupKey, cdc.MustMarshalBinaryBare(ei))
@@ -187,13 +197,7 @@ func (store *EvidenceStore) MarkEvidenceAsCommitted(evidence types.Evidence) {
 //---------------------------------------------------
 // utils
 
+// getEvidenceInfo is convenience for calling GetEvidenceInfo if we have the full evidence.
 func (store *EvidenceStore) getEvidenceInfo(evidence types.Evidence) EvidenceInfo {
-	key := keyLookup(evidence)
-	var ei EvidenceInfo
-	b := store.db.Get(key)
-	err := cdc.UnmarshalBinaryBare(b, &ei)
-	if err != nil {
-		panic(err)
-	}
-	return ei
+	return store.GetEvidenceInfo(evidence.Height(), evidence.Hash())
 }