types: change Commit to consist of just signatures (#4146)

* types: change `Commit` to consist of just signatures

These are final changes towards removing votes from commit and leaving
only signatures (see ADR-25)

Fixes #1648

* bring back TestCommitToVoteSetWithVotesForAnotherBlockOrNilBlock

+ add absent flag to Vote to indicate that it's for another block

* encode nil votes as CommitSig with BlockIDFlagAbsent

+ make Commit#Precommits array of non-pointers
because precommit will never be nil

* add NewCommitSigAbsent and Absent() funcs

* uncomment validation in CommitSig#ValidateBasic

* add comments to ValidatorSet funcs

* add a changelog entry

* break instead of continue

continue does not make sense in these cases

* types: rename Commit#Precommits to Signatures

* swagger: fix /commit response

* swagger: change block_id_flag type

* fix merge conflicts

CHANGELOG_PENDING.md

@@ -12,6 +12,11 @@ different modules:
 - Go API: the `Header` broke
 - P2P: since blocks go over the wire, technically the P2P protocol broke
 
+Also, blocks are significantly smaller 🔥 because we got rid of the redundant
+information in `Block#LastCommit`. `Commit` now mainly consists of a signature
+and a validator address plus a timestamp. Note we may remove the validator
+address & timestamp fields in the future (see ADR-25).
+
 Special thanks to external contributors on this release:
 @erikgrinaker, @PSalant726, @gchaincl, @gregzaitsev
 
@@ -66,6 +71,7 @@ program](https://hackerone.com/tendermint).
 
   - [abci] \#2521 Remove `TotalTxs` and `NumTxs` from `Header`
   - [types][\#4151](https://github.com/tendermint/tendermint/pull/4151) Enforce ordering of votes in DuplicateVoteEvidence to be lexicographically sorted on BlockID
+  - [types] \#1648 Change `Commit` to consist of just signatures
 
 - P2P Protocol
 

blockchain/v0/reactor_test.go

@@ -87,7 +87,7 @@ func newBlockchainReactor(
 
 	// let's add some blocks in
 	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
-		lastCommit := types.NewCommit(types.BlockID{}, nil)
+		lastCommit := types.NewCommit(blockHeight-1, 0, types.BlockID{}, nil)
 		if blockHeight > 1 {
 			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
 			lastBlock := blockStore.LoadBlock(blockHeight - 1)
@@ -101,8 +101,8 @@ func newBlockchainReactor(
 			if err != nil {
 				panic(err)
 			}
-			voteCommitSig := vote.CommitSig()
-			lastCommit = types.NewCommit(lastBlockMeta.BlockID, []*types.CommitSig{voteCommitSig})
+			lastCommit = types.NewCommit(vote.Height, vote.Round,
+				lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
 		}
 
 		thisBlock := makeBlock(blockHeight, state, lastCommit)

blockchain/v1/reactor_test.go

@@ -109,13 +109,13 @@ func newBlockchainReactor(
 
 	// let's add some blocks in
 	for blockHeight := int64(1); blockHeight <= maxBlockHeight; blockHeight++ {
-		lastCommit := types.NewCommit(types.BlockID{}, nil)
+		lastCommit := types.NewCommit(blockHeight-1, 1, types.BlockID{}, nil)
 		if blockHeight > 1 {
 			lastBlockMeta := blockStore.LoadBlockMeta(blockHeight - 1)
 			lastBlock := blockStore.LoadBlock(blockHeight - 1)
 
-			vote := makeVote(&lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVals[0]).CommitSig()
-			lastCommit = types.NewCommit(lastBlockMeta.BlockID, []*types.CommitSig{vote})
+			vote := makeVote(&lastBlock.Header, lastBlockMeta.BlockID, state.Validators, privVals[0])
+			lastCommit = types.NewCommit(vote.Height, vote.Round, lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
 		}
 
 		thisBlock := makeBlock(blockHeight, state, lastCommit)

consensus/mempool_test.go

@@ -111,21 +111,19 @@ func TestMempoolTxConcurrentWithCommit(t *testing.T) {
 	blockDB := dbm.NewMemDB()
 	cs := newConsensusStateWithConfigAndBlockStore(config, state, privVals[0], NewCounterApplication(), blockDB)
 	sm.SaveState(blockDB, state)
-	height, round := cs.Height, cs.Round
-	newBlockCh := subscribe(cs.eventBus, types.EventQueryNewBlock)
+	newBlockHeaderCh := subscribe(cs.eventBus, types.EventQueryNewBlockHeader)
 
-	NTxs := 3000
-	go deliverTxsRange(cs, 0, NTxs)
+	const numTxs int64 = 3000
+	go deliverTxsRange(cs, 0, int(numTxs))
 
-	startTestRound(cs, height, round)
-	for nTxs := 0; nTxs < NTxs; {
-		ticker := time.NewTicker(time.Second * 30)
+	startTestRound(cs, cs.Height, cs.Round)
+	for n := int64(0); n < numTxs; {
 		select {
-		case msg := <-newBlockCh:
-			blockEvent := msg.Data().(types.EventDataNewBlock)
-			nTxs += len(blockEvent.Block.Txs)
-		case <-ticker.C:
-			panic("Timed out waiting to commit blocks with transactions")
+		case msg := <-newBlockHeaderCh:
+			headerEvent := msg.Data().(types.EventDataNewBlockHeader)
+			n += headerEvent.NumTxs
+		case <-time.After(30 * time.Second):
+			t.Fatal("Timed out waiting 30s to commit blocks with transactions")
 		}
 	}
 }

consensus/reactor.go

@@ -452,7 +452,7 @@ func makeRoundStepMessage(rs *cstypes.RoundState) (nrsMsg *NewRoundStepMessage)
 		Round:                 rs.Round,
 		Step:                  rs.Step,
 		SecondsSinceStartTime: int(time.Since(rs.StartTime).Seconds()),
-		LastCommitRound:       rs.LastCommit.Round(),
+		LastCommitRound:       rs.LastCommit.GetRound(),
 	}
 	return
 }
@@ -805,7 +805,7 @@ OUTER_LOOP:
 				commit := conR.conS.LoadCommit(prs.Height)
 				peer.TrySend(StateChannel, cdc.MustMarshalBinaryBare(&VoteSetMaj23Message{
 					Height:  prs.Height,
-					Round:   commit.Round(),
+					Round:   commit.Round,
 					Type:    types.PrecommitType,
 					BlockID: commit.BlockID,
 				}))
@@ -1053,15 +1053,15 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote
 		return nil, false
 	}
 
-	height, round, type_, size := votes.Height(), votes.Round(), types.SignedMsgType(votes.Type()), votes.Size()
+	height, round, votesType, size := votes.GetHeight(), votes.GetRound(), types.SignedMsgType(votes.Type()), votes.Size()
 
 	// Lazily set data using 'votes'.
 	if votes.IsCommit() {
 		ps.ensureCatchupCommitRound(height, round, size)
 	}
 	ps.ensureVoteBitArrays(height, size)
 
-	psVotes := ps.getVoteBitArray(height, round, type_)
+	psVotes := ps.getVoteBitArray(height, round, votesType)
 	if psVotes == nil {
 		return nil, false // Not something worth sending
 	}
@@ -1071,30 +1071,30 @@ func (ps *PeerState) PickVoteToSend(votes types.VoteSetReader) (vote *types.Vote
 	return nil, false
 }
 
-func (ps *PeerState) getVoteBitArray(height int64, round int, type_ types.SignedMsgType) *cmn.BitArray {
-	if !types.IsVoteTypeValid(type_) {
+func (ps *PeerState) getVoteBitArray(height int64, round int, votesType types.SignedMsgType) *cmn.BitArray {
+	if !types.IsVoteTypeValid(votesType) {
 		return nil
 	}
 
 	if ps.PRS.Height == height {
 		if ps.PRS.Round == round {
-			switch type_ {
+			switch votesType {
 			case types.PrevoteType:
 				return ps.PRS.Prevotes
 			case types.PrecommitType:
 				return ps.PRS.Precommits
 			}
 		}
 		if ps.PRS.CatchupCommitRound == round {
-			switch type_ {
+			switch votesType {
 			case types.PrevoteType:
 				return nil
 			case types.PrecommitType:
 				return ps.PRS.CatchupCommit
 			}
 		}
 		if ps.PRS.ProposalPOLRound == round {
-			switch type_ {
+			switch votesType {
 			case types.PrevoteType:
 				return ps.PRS.ProposalPOL
 			case types.PrecommitType:
@@ -1105,7 +1105,7 @@ func (ps *PeerState) getVoteBitArray(height int64, round int, type_ types.Signed
 	}
 	if ps.PRS.Height == height+1 {
 		if ps.PRS.LastCommitRound == round {
-			switch type_ {
+			switch votesType {
 			case types.PrevoteType:
 				return nil
 			case types.PrecommitType:
@@ -1223,16 +1223,16 @@ func (ps *PeerState) SetHasVote(vote *types.Vote) {
 	ps.setHasVote(vote.Height, vote.Round, vote.Type, vote.ValidatorIndex)
 }
 
-func (ps *PeerState) setHasVote(height int64, round int, type_ types.SignedMsgType, index int) {
+func (ps *PeerState) setHasVote(height int64, round int, voteType types.SignedMsgType, index int) {
 	logger := ps.logger.With(
 		"peerH/R",
 		fmt.Sprintf("%d/%d", ps.PRS.Height, ps.PRS.Round),
 		"H/R",
 		fmt.Sprintf("%d/%d", height, round))
-	logger.Debug("setHasVote", "type", type_, "index", index)
+	logger.Debug("setHasVote", "type", voteType, "index", index)
 
 	// NOTE: some may be nil BitArrays -> no side effects.
-	psVotes := ps.getVoteBitArray(height, round, type_)
+	psVotes := ps.getVoteBitArray(height, round, voteType)
 	if psVotes != nil {
 		psVotes.SetIndex(index, true)
 	}

consensus/reactor_test.go

@@ -613,9 +613,9 @@ func validateBlock(block *types.Block, activeVals map[string]struct{}) error {
 			len(activeVals))
 	}
 
-	for _, vote := range block.LastCommit.Precommits {
-		if _, ok := activeVals[string(vote.ValidatorAddress)]; !ok {
-			return fmt.Errorf("found vote for unactive validator %X", vote.ValidatorAddress)
+	for _, commitSig := range block.LastCommit.Signatures {
+		if _, ok := activeVals[string(commitSig.ValidatorAddress)]; !ok {
+			return fmt.Errorf("found vote for inactive validator %X", commitSig.ValidatorAddress)
 		}
 	}
 	return nil

consensus/replay_test.go

@@ -883,16 +883,16 @@ func makeBlocks(n int, state *sm.State, privVal types.PrivValidator) []*types.Bl
 func makeBlock(state sm.State, lastBlock *types.Block, lastBlockMeta *types.BlockMeta,
 	privVal types.PrivValidator, height int64) (*types.Block, *types.PartSet) {
 
-	lastCommit := types.NewCommit(types.BlockID{}, nil)
+	lastCommit := types.NewCommit(height-1, 0, types.BlockID{}, nil)
 	if height > 1 {
 		vote, _ := types.MakeVote(
 			lastBlock.Header.Height,
 			lastBlockMeta.BlockID,
 			state.Validators,
 			privVal,
 			lastBlock.Header.ChainID)
-		voteCommitSig := vote.CommitSig()
-		lastCommit = types.NewCommit(lastBlockMeta.BlockID, []*types.CommitSig{voteCommitSig})
+		lastCommit = types.NewCommit(vote.Height, vote.Round,
+			lastBlockMeta.BlockID, []types.CommitSig{vote.CommitSig()})
 	}
 
 	return state.MakeBlock(height, []types.Tx{}, lastCommit, nil, state.Validators.GetProposer().Address)
@@ -971,7 +971,7 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 				if block.Height != height+1 {
 					panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1))
 				}
-				commitHeight := thisBlockCommit.Precommits[0].Height
+				commitHeight := thisBlockCommit.Height
 				if commitHeight != height+1 {
 					panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1))
 				}
@@ -988,8 +988,8 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 			}
 		case *types.Vote:
 			if p.Type == types.PrecommitType {
-				commitSigs := []*types.CommitSig{p.CommitSig()}
-				thisBlockCommit = types.NewCommit(p.BlockID, commitSigs)
+				thisBlockCommit = types.NewCommit(p.Height, p.Round,
+					p.BlockID, []types.CommitSig{p.CommitSig()})
 			}
 		}
 	}
@@ -1002,7 +1002,7 @@ func makeBlockchainFromWAL(wal WAL) ([]*types.Block, []*types.Commit, error) {
 	if block.Height != height+1 {
 		panic(fmt.Sprintf("read bad block from wal. got height %d, expected %d", block.Height, height+1))
 	}
-	commitHeight := thisBlockCommit.Precommits[0].Height
+	commitHeight := thisBlockCommit.Height
 	if commitHeight != height+1 {
 		panic(fmt.Sprintf("commit doesnt match. got height %d, expected %d", commitHeight, height+1))
 	}

consensus/state.go

@@ -1013,7 +1013,7 @@ func (cs *ConsensusState) createProposalBlock() (block *types.Block, blockParts
 	case cs.Height == 1:
 		// We're creating a proposal for the first block.
 		// The commit is empty, but not nil.
-		commit = types.NewCommit(types.BlockID{}, nil)
+		commit = types.NewCommit(0, 0, types.BlockID{}, nil)
 	case cs.LastCommit.HasTwoThirdsMajority():
 		// Make the commit from LastCommit
 		commit = cs.LastCommit.MakeCommit()
@@ -1464,11 +1464,11 @@ func (cs *ConsensusState) recordMetrics(height int64, block *types.Block) {
 	missingValidators := 0
 	missingValidatorsPower := int64(0)
 	for i, val := range cs.Validators.Validators {
-		var vote *types.CommitSig
-		if i < len(block.LastCommit.Precommits) {
-			vote = block.LastCommit.Precommits[i]
+		if i >= len(block.LastCommit.Signatures) {
+			break
 		}
-		if vote == nil {
+		commitSig := block.LastCommit.Signatures[i]
+		if commitSig.Absent() {
 			missingValidators++
 			missingValidatorsPower += val.VotingPower
 		}

consensus/types/round_state_test.go

@@ -16,7 +16,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 	// Random validators
 	nval, ntxs := 100, 100
 	vset, _ := types.RandValidatorSet(nval, 1)
-	precommits := make([]*types.CommitSig, nval)
+	commitSigs := make([]types.CommitSig, nval)
 	blockID := types.BlockID{
 		Hash: cmn.RandBytes(20),
 		PartsHeader: types.PartSetHeader{
@@ -25,7 +25,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 	}
 	sig := make([]byte, ed25519.SignatureSize)
 	for i := 0; i < nval; i++ {
-		precommits[i] = (&types.Vote{
+		commitSigs[i] = (&types.Vote{
 			ValidatorAddress: types.Address(cmn.RandBytes(20)),
 			Timestamp:        tmtime.Now(),
 			BlockID:          blockID,
@@ -54,7 +54,7 @@ func BenchmarkRoundStateDeepCopy(b *testing.B) {
 			Txs: txs,
 		},
 		Evidence:   types.EvidenceData{},
-		LastCommit: types.NewCommit(blockID, precommits),
+		LastCommit: types.NewCommit(1, 0, blockID, commitSigs),
 	}
 	parts := block.MakePartSet(4096)
 	// Random Proposal

lite/helpers.go

@@ -70,21 +70,29 @@ func (pkz privKeys) ToValidators(init, inc int64) *types.ValidatorSet {
 
 // signHeader properly signs the header with all keys from first to last exclusive.
 func (pkz privKeys) signHeader(header *types.Header, first, last int) *types.Commit {
-	commitSigs := make([]*types.CommitSig, len(pkz))
+	commitSigs := make([]types.CommitSig, len(pkz))
+	for i := 0; i < len(pkz); i++ {
+		commitSigs[i] = types.NewCommitSigAbsent()
+	}
 
 	// We need this list to keep the ordering.
 	vset := pkz.ToValidators(1, 0)
 
+	blockID := types.BlockID{
+		Hash:        header.Hash(),
+		PartsHeader: types.PartSetHeader{Total: 1, Hash: crypto.CRandBytes(32)},
+	}
+
 	// Fill in the votes we want.
 	for i := first; i < last && i < len(pkz); i++ {
-		vote := makeVote(header, vset, pkz[i])
+		vote := makeVote(header, vset, pkz[i], blockID)
 		commitSigs[vote.ValidatorIndex] = vote.CommitSig()
 	}
-	blockID := types.BlockID{Hash: header.Hash()}
-	return types.NewCommit(blockID, commitSigs)
+
+	return types.NewCommit(header.Height, 1, blockID, commitSigs)
 }
 
-func makeVote(header *types.Header, valset *types.ValidatorSet, key crypto.PrivKey) *types.Vote {
+func makeVote(header *types.Header, valset *types.ValidatorSet, key crypto.PrivKey, blockID types.BlockID) *types.Vote {
 	addr := key.PubKey().Address()
 	idx, _ := valset.GetByAddress(addr)
 	vote := &types.Vote{
@@ -94,7 +102,7 @@ func makeVote(header *types.Header, valset *types.ValidatorSet, key crypto.PrivK
 		Round:            1,
 		Timestamp:        tmtime.Now(),
 		Type:             types.PrecommitType,
-		BlockID:          types.BlockID{Hash: header.Hash()},
+		BlockID:          blockID,
 	}
 	// Sign it
 	signBytes := vote.SignBytes(header.ChainID)

lite/proxy/validate_test.go

@@ -70,7 +70,7 @@ func TestValidateBlock(t *testing.T) {
 			},
 			signedHeader: types.SignedHeader{
 				Header: &types.Header{Height: 11},
-				Commit: types.NewCommit(types.BlockID{Hash: []byte("0xDEADBEEF")}, nil),
+				Commit: types.NewCommit(11, 0, types.BlockID{Hash: []byte("0xDEADBEEF")}, nil),
 			},
 			wantErr: "data hash doesn't match header",
 		},
@@ -81,7 +81,7 @@ func TestValidateBlock(t *testing.T) {
 			},
 			signedHeader: types.SignedHeader{
 				Header: &types.Header{Height: 11},
-				Commit: types.NewCommit(types.BlockID{Hash: []byte("DEADBEEF")}, nil),
+				Commit: types.NewCommit(11, 0, types.BlockID{Hash: []byte("DEADBEEF")}, nil),
 			},
 		},
 		// End Header.Data hash mismatch test
@@ -169,7 +169,7 @@ func TestValidateBlockMeta(t *testing.T) {
 					ValidatorsHash: []byte("Tendermint"),
 					Time:           testTime2,
 				},
-				Commit: types.NewCommit(types.BlockID{Hash: []byte("DEADBEEF")}, nil),
+				Commit: types.NewCommit(11, 0, types.BlockID{Hash: []byte("DEADBEEF")}, nil),
 			},
 			wantErr: "headers don't match",
 		},
@@ -188,7 +188,7 @@ func TestValidateBlockMeta(t *testing.T) {
 					ValidatorsHash: []byte("Tendermint-x"),
 					Time:           testTime2,
 				},
-				Commit: types.NewCommit(types.BlockID{Hash: []byte("DEADBEEF")}, nil),
+				Commit: types.NewCommit(11, 0, types.BlockID{Hash: []byte("DEADBEEF")}, nil),
 			},
 			wantErr: "headers don't match",
 		},