BFT chain unit tests reconfigure (#4767)

* BFT chain unit tests: reconfigure - add new node Signed-off-by: May Rosenbaum <mayro1595@gmail.com> * BFT chain unit tests: reconfigure while node is down Signed-off-by: May Rosenbaum <mayro1595@gmail.com> * Fix race condition Signed-off-by: May Rosenbaum <mayro1595@gmail.com> * Post-review changes Signed-off-by: May Rosenbaum <mayro1595@gmail.com> --------- Signed-off-by: May Rosenbaum <mayro1595@gmail.com>
hyperledger · Apr 7, 2024 · 254bd6f · 254bd6f
1 parent 210f1ca
commit 254bd6f
Show file tree

Hide file tree

Showing 4 changed files with 397 additions and 44 deletions.
diff --git a/orderer/consensus/smartbft/chain.go b/orderer/consensus/smartbft/chain.go
@@ -73,7 +73,7 @@ type BFTChain struct {
 	WALDir             string
 	consensus          *smartbft.Consensus
 	support            consensus.ConsenterSupport
-	clusterService     *cluster.ClusterService
+	ClusterService     *cluster.ClusterService
 	verifier           *Verifier
 	assembler          *Assembler
 	Metrics            *Metrics
@@ -423,6 +423,9 @@ func (c *BFTChain) Deliver(proposal types.Proposal, signatures []types.Signature
 	}
 
 	reconfig := c.updateRuntimeConfig(block)
+	if reconfig.InLatestDecision {
+		c.Logger.Infof("Reconfiguration was done and the current nodes are: %v", reconfig.CurrentNodes)
+	}
 	return reconfig
 }
 
@@ -554,7 +557,7 @@ func (c *BFTChain) updateRuntimeConfig(block *cb.Block) types.Reconfig {
 	c.RuntimeConfig.Store(newRTC)
 	if protoutil.IsConfigBlock(block) {
 		c.Comm.Configure(c.Channel, newRTC.RemoteNodes)
-		c.clusterService.ConfigureNodeCerts(c.Channel, newRTC.consenters)
+		c.ClusterService.ConfigureNodeCerts(c.Channel, newRTC.consenters)
 		c.pruneBadRequests()
 	}
 

diff --git a/orderer/consensus/smartbft/chain_test.go b/orderer/consensus/smartbft/chain_test.go
@@ -8,9 +8,17 @@ import (
 	"bytes"
 	"encoding/binary"
 	"fmt"
+	"os"
 	"testing"
 	"time"
 
+	"github.com/golang/protobuf/proto"
+	"github.com/hyperledger/fabric/common/channelconfig"
+	"github.com/hyperledger/fabric/common/crypto/tlsgen"
+	"github.com/hyperledger/fabric/internal/configtxlator/update"
+	smartBFTMocks "github.com/hyperledger/fabric/orderer/consensus/smartbft/mocks"
+	"github.com/stretchr/testify/mock"
+
 	cb "github.com/hyperledger/fabric-protos-go/common"
 	"github.com/hyperledger/fabric-protos-go/msp"
 	"github.com/hyperledger/fabric/protoutil"
@@ -168,7 +176,7 @@ func TestSyncNode(t *testing.T) {
 	}
 
 	// restart the old leader
-	nodeMap[leaderId].Restart()
+	nodeMap[leaderId].Restart(networkSetupInfo.configInfo)
 
 	// make sure the old leader has synced with the nodes in the network
 	nodeMap[leaderId].State.WaitLedgerHeightToBe(7)
@@ -182,6 +190,129 @@ func TestSyncNode(t *testing.T) {
 	}
 }
 
+// Scenario:
+// 1. Start a network of 4 nodes
+// 2. Submit a TX and wait for the TX to be received by all nodes
+// 3. Add new node to the network
+// 4. Submit a TX and wait for the TX to be received by all nodes
+func TestAddNode(t *testing.T) {
+	dir := t.TempDir()
+	channelId := "testchannel"
+
+	// start a network
+	networkSetupInfo := NewNetworkSetupInfo(t, channelId, dir)
+	nodeMap := networkSetupInfo.CreateNodes(4)
+	networkSetupInfo.StartAllNodes()
+
+	// wait until all nodes have the genesis block in their ledger
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(1)
+	}
+
+	// send a tx to all nodes and wait the tx will be added to each ledger
+	env := createEndorserTxEnvelope("TEST_MESSAGE #1", channelId)
+	err := networkSetupInfo.SendTxToAllAvailableNodes(env)
+	require.NoError(t, err)
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(2)
+	}
+
+	// send a new config block to all nodes, to notice them about the new node
+	env = addNodeToConfig(t, networkSetupInfo.genesisBlock, 5, networkSetupInfo.tlsCA, networkSetupInfo.dir, networkSetupInfo.channelId)
+	err = networkSetupInfo.SendTxToAllAvailableNodes(env)
+	require.NoError(t, err)
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(3)
+	}
+
+	// add new node to the network
+	nodesMap, newNode := networkSetupInfo.AddNewNode()
+	newNode.Start()
+	require.Equal(t, len(nodesMap), 5)
+	require.Equal(t, len(networkSetupInfo.nodeIdToNode), 5)
+
+	// send a tx to all nodes again and wait the tx will be added to each ledger
+	env = createEndorserTxEnvelope("TEST_ADDITION_OF_NODE", channelId)
+	err = networkSetupInfo.SendTxToAllAvailableNodes(env)
+	require.NoError(t, err)
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(4)
+	}
+}
+
+// Scenario:
+// 1. Start a network of 4 nodes
+// 2. Submit a TX and wait for the TX to be received by all nodes
+// 3. Stop the leader and wait for a new leader to be elected
+// 4. Add new node to the network
+// 5. Start the old leader and make sure he has synced with other nodes about the config change
+// 4. Submit a TX and wait for the TX to be received by all nodes
+func TestReconfigurationWhileNodeIsDown(t *testing.T) {
+	dir := t.TempDir()
+	channelId := "testchannel"
+
+	// start a network
+	networkSetupInfo := NewNetworkSetupInfo(t, channelId, dir)
+	nodeMap := networkSetupInfo.CreateNodes(4)
+	networkSetupInfo.StartAllNodes()
+
+	// wait until all nodes have the genesis block in their ledger
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(1)
+	}
+
+	// send a tx to all nodes and wait the tx will be added to each ledger
+	env := createEndorserTxEnvelope("TEST_MESSAGE #1", channelId)
+	err := networkSetupInfo.SendTxToAllAvailableNodes(env)
+	require.NoError(t, err)
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(2)
+	}
+
+	// get leader id
+	leaderId := networkSetupInfo.GetAgreedLeader()
+
+	// stop the leader
+	nodeMap[leaderId].Stop()
+
+	// wait for a new leader to be elected
+	require.Eventually(t, func() bool {
+		newLeaderId := networkSetupInfo.GetAgreedLeader()
+		return leaderId != newLeaderId
+	}, 1*time.Minute, 100*time.Millisecond)
+
+	// send a new config block to all available nodes, to notice them about the new node
+	env = addNodeToConfig(t, networkSetupInfo.genesisBlock, 5, networkSetupInfo.tlsCA, networkSetupInfo.dir, networkSetupInfo.channelId)
+	err = networkSetupInfo.SendTxToAllAvailableNodes(env)
+	require.NoError(t, err)
+	for _, node := range nodeMap {
+		if node.NodeId == leaderId {
+			continue
+		}
+		node.State.WaitLedgerHeightToBe(3)
+	}
+
+	// add new node to the network
+	nodesMap, newNode := networkSetupInfo.AddNewNode()
+	newNode.Start()
+	require.Equal(t, len(nodesMap), 5)
+	require.Equal(t, len(networkSetupInfo.nodeIdToNode), 5)
+
+	// restart the old leader
+	nodeMap[leaderId].Restart(networkSetupInfo.configInfo)
+
+	// make sure the old leader has synced with the nodes in the network
+	nodeMap[leaderId].State.WaitLedgerHeightToBe(3)
+
+	// send a tx to all nodes again and wait the tx will be added to each ledger
+	env = createEndorserTxEnvelope("TEST_ADDITION_OF_NODE", channelId)
+	err = networkSetupInfo.SendTxToAllAvailableNodes(env)
+	require.NoError(t, err)
+	for _, node := range nodeMap {
+		node.State.WaitLedgerHeightToBe(4)
+	}
+}
+
 func createEndorserTxEnvelope(message string, channelId string) *cb.Envelope {
 	return &cb.Envelope{
 		Payload: protoutil.MarshalOrPanic(&cb.Payload{
@@ -213,3 +344,89 @@ func generateNonce() []byte {
 	nonce++
 	return nonceBuf.Bytes()
 }
+
+// addNodeToConfig creates a config block based on the last config block. It is useful for addition or removal of a node
+func addNodeToConfig(t *testing.T, lastConfigBlock *cb.Block, nodeId uint32, tlsCA tlsgen.CA, certDir string, channelId string) *cb.Envelope {
+	// copy the last config block
+	clonedLastConfigBlock := proto.Clone(lastConfigBlock).(*cb.Block)
+
+	// fetch the ConfigEnvelope from the block
+	env := protoutil.UnmarshalEnvelopeOrPanic(clonedLastConfigBlock.Data.Data[0])
+	payload := protoutil.UnmarshalPayloadOrPanic(env.Payload)
+	configEnv, err := protoutil.UnmarshalConfigEnvelope(payload.Data)
+	require.NoError(t, err)
+	originalConfigEnv := proto.Clone(configEnv).(*cb.ConfigEnvelope)
+
+	// create the crypto material for the new node
+	host := fmt.Sprintf("bft%d.example.com", nodeId-1)
+	srvP, clnP := generateSingleCertificateSmartBFT(t, tlsCA, certDir, int(nodeId), host)
+	clientCert, err := os.ReadFile(clnP)
+	require.NoError(t, err)
+	serverCert, err := os.ReadFile(srvP)
+	require.NoError(t, err)
+
+	// update the consenter mapping to include the new node
+	newOrderer := &cb.Consenter{
+		Id:            nodeId,
+		Host:          host,
+		Port:          7050,
+		MspId:         "SampleOrg",
+		Identity:      clientCert,
+		ClientTlsCert: clientCert,
+		ServerTlsCert: serverCert,
+	}
+
+	currentOrderers := &cb.Orderers{}
+	err = proto.Unmarshal(configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey].Value, currentOrderers)
+	require.NoError(t, err)
+	currentOrderers.ConsenterMapping = append(currentOrderers.ConsenterMapping, newOrderer)
+	configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey] = &cb.ConfigValue{
+		Version:   configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Values[channelconfig.OrderersKey].Version + 1,
+		Value:     protoutil.MarshalOrPanic(currentOrderers),
+		ModPolicy: channelconfig.AdminsPolicyKey,
+	}
+
+	// update organization endpoints
+	ordererEndpoints := configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value
+	ordererEndpointsVal := &cb.OrdererAddresses{}
+	proto.Unmarshal(ordererEndpoints, ordererEndpointsVal)
+	ordererAddresses := ordererEndpointsVal.Addresses
+	ordererAddresses = append(ordererAddresses, fmt.Sprintf("%s:%d", newOrderer.Host, newOrderer.Port))
+	configEnv.Config.ChannelGroup.Groups[channelconfig.OrdererGroupKey].Groups["SampleOrg"].Values["Endpoints"].Value = protoutil.MarshalOrPanic(&cb.OrdererAddresses{
+		Addresses: ordererAddresses,
+	})
+
+	// increase the sequence
+	configEnv.Config.Sequence = configEnv.Config.Sequence + 1
+
+	// calculate config update tx
+	configUpdate, err := update.Compute(originalConfigEnv.Config, configEnv.Config)
+	require.NoError(t, err)
+	signerSerializer := smartBFTMocks.NewSignerSerializer(t)
+	signerSerializer.EXPECT().Serialize().RunAndReturn(
+		func() ([]byte, error) {
+			return []byte{1, 2, 3}, nil
+		}).Maybe()
+	signerSerializer.EXPECT().Sign(mock.Anything).RunAndReturn(
+		func(message []byte) ([]byte, error) {
+			return message, nil
+		}).Maybe()
+	configUpdateTx, err := protoutil.CreateSignedEnvelope(cb.HeaderType_CONFIG_UPDATE, channelId, signerSerializer, configUpdate, 0, 0)
+	require.NoError(t, err)
+
+	// return the updated Envelope
+	return &cb.Envelope{
+		Payload: protoutil.MarshalOrPanic(&cb.Payload{
+			Data: protoutil.MarshalOrPanic(&cb.ConfigEnvelope{
+				Config:     configEnv.Config,
+				LastUpdate: configUpdateTx,
+			}),
+			Header: &cb.Header{
+				ChannelHeader: protoutil.MarshalOrPanic(&cb.ChannelHeader{
+					Type:      int32(cb.HeaderType_CONFIG),
+					ChannelId: channelId,
+				}),
+			},
+		}),
+	}
+}
diff --git a/orderer/consensus/smartbft/consenter.go b/orderer/consensus/smartbft/consenter.go
@@ -239,7 +239,7 @@ func (c *Consenter) HandleChain(support consensus.ConsenterSupport, metadata *cb
 
 	// refresh cluster service with updated consenters
 	c.ClusterService.ConfigureNodeCerts(chain.Channel, consenters)
-	chain.clusterService = c.ClusterService
+	chain.ClusterService = c.ClusterService
 
 	return chain, nil
 }