Peer CLI communicate with orderers with expired TLS certs (#1863)

Implement a TLS handshake timeshift for the "peer channel fetch"
and "peer channel update" comands to allow fetching config blocks
and updating the config for orderers with expired TLS certificates.

FAB-18205 #done

Signed-off-by: Will Lahti <wtlahti@us.ibm.com>

integration/configtx/configtx_test.go

@@ -11,9 +11,7 @@ import (
 	"crypto/x509"
 	"encoding/pem"
 	"io/ioutil"
-	"net"
 	"os"
-	"strconv"
 	"syscall"
 	"time"
 
@@ -155,7 +153,7 @@ var _ = Describe("ConfigTx", func() {
 		oConfig.BatchTimeout = 2 * time.Second
 		err = o.SetConfiguration(oConfig)
 		Expect(err).NotTo(HaveOccurred())
-		host, port := ordererHostPort(network, orderer)
+		host, port := OrdererHostPort(network, orderer)
 		err = o.Organization(orderer.Organization).SetEndpoint(configtx.Address{Host: host, Port: port + 1})
 		Expect(err).NotTo(HaveOccurred())
 
@@ -257,7 +255,7 @@ var _ = Describe("ConfigTx", func() {
 			peerOrg := c.Application().Organization(peer.Organization)
 
 			By("adding the anchor peer for " + peer.Organization)
-			host, port := peerHostPort(network, peer)
+			host, port := PeerHostPort(network, peer)
 			err = peerOrg.AddAnchorPeer(configtx.Address{Host: host, Port: port})
 			Expect(err).NotTo(HaveOccurred())
 
@@ -330,19 +328,3 @@ func parsePrivateKey(filename string) crypto.PrivateKey {
 	Expect(err).NotTo(HaveOccurred())
 	return privateKey
 }
-
-func peerHostPort(n *nwo.Network, p *nwo.Peer) (string, int) {
-	return splitHostPort(n.PeerAddress(p, nwo.ListenPort))
-}
-
-func ordererHostPort(n *nwo.Network, o *nwo.Orderer) (string, int) {
-	return splitHostPort(n.OrdererAddress(o, nwo.ListenPort))
-}
-
-func splitHostPort(address string) (string, int) {
-	host, port, err := net.SplitHostPort(address)
-	Expect(err).NotTo(HaveOccurred())
-	portInt, err := strconv.Atoi(port)
-	Expect(err).NotTo(HaveOccurred())
-	return host, portInt
-}

integration/configtx/host_port.go

@@ -0,0 +1,40 @@
+/*
+Copyright IBM Corp All Rights Reserved.
+
+SPDX-License-Identifier: Apache-2.0
+*/
+
+package configtx
+
+import (
+	"net"
+	"strconv"
+
+	"github.com/hyperledger/fabric/integration/nwo"
+	. "github.com/onsi/gomega"
+)
+
+// PeerHostPort returns the host name and port number for the specified peer.
+func PeerHostPort(n *nwo.Network, p *nwo.Peer) (string, int) {
+	return splitHostPort(n.PeerAddress(p, nwo.ListenPort))
+}
+
+// OrdererHostPort returns the host name and port number for the specified
+// orderer.
+func OrdererHostPort(n *nwo.Network, o *nwo.Orderer) (string, int) {
+	return splitHostPort(n.OrdererAddress(o, nwo.ListenPort))
+}
+
+// OrdererClusterHostPort returns the host name and cluster port number for the
+// specified orderer.
+func OrdererClusterHostPort(n *nwo.Network, o *nwo.Orderer) (string, int) {
+	return splitHostPort(n.OrdererAddress(o, nwo.ClusterPort))
+}
+
+func splitHostPort(address string) (string, int) {
+	host, port, err := net.SplitHostPort(address)
+	Expect(err).NotTo(HaveOccurred())
+	portInt, err := strconv.Atoi(port)
+	Expect(err).NotTo(HaveOccurred())
+	return host, portInt
+}

integration/nwo/commands/peer.go

@@ -8,6 +8,7 @@ package commands
 
 import (
 	"strconv"
+	"time"
 )
 
 type NodeStart struct {
@@ -135,11 +136,12 @@ func (c ChannelJoin) Args() []string {
 }
 
 type ChannelFetch struct {
-	ChannelID  string
-	Block      string
-	Orderer    string
-	OutputFile string
-	ClientAuth bool
+	ChannelID             string
+	Block                 string
+	Orderer               string
+	OutputFile            string
+	ClientAuth            bool
+	TLSHandshakeTimeShift time.Duration
 }
 
 func (c ChannelFetch) SessionName() string {
@@ -149,15 +151,10 @@ func (c ChannelFetch) SessionName() string {
 func (c ChannelFetch) Args() []string {
 	args := []string{
 		"channel", "fetch", c.Block,
-	}
-	if c.ChannelID != "" {
-		args = append(args, "--channelID", c.ChannelID)
-	}
-	if c.Orderer != "" {
-		args = append(args, "--orderer", c.Orderer)
-	}
-	if c.OutputFile != "" {
-		args = append(args, c.OutputFile)
+		"--channelID", c.ChannelID,
+		"--orderer", c.Orderer,
+		"--tlsHandshakeTimeShift", c.TLSHandshakeTimeShift.String(),
+		c.OutputFile,
 	}
 	if c.ClientAuth {
 		args = append(args, "--clientauth")
@@ -747,10 +744,11 @@ func (s SignConfigTx) Args() []string {
 }
 
 type ChannelUpdate struct {
-	ChannelID  string
-	Orderer    string
-	File       string
-	ClientAuth bool
+	ChannelID             string
+	Orderer               string
+	File                  string
+	ClientAuth            bool
+	TLSHandshakeTimeShift time.Duration
 }
 
 func (c ChannelUpdate) SessionName() string {
@@ -763,6 +761,7 @@ func (c ChannelUpdate) Args() []string {
 		"--channelID", c.ChannelID,
 		"--orderer", c.Orderer,
 		"--file", c.File,
+		"--tlsHandshakeTimeShift", c.TLSHandshakeTimeShift.String(),
 	}
 	if c.ClientAuth {
 		args = append(args, "--clientauth")

integration/raft/cft_test.go

@@ -17,17 +17,21 @@ import (
 	"path"
 	"path/filepath"
 	"strconv"
+	"strings"
 	"sync"
 	"syscall"
 	"time"
 
 	docker "github.com/fsouza/go-dockerclient"
 	"github.com/golang/protobuf/proto"
+	conftx "github.com/hyperledger/fabric-config/configtx"
+	"github.com/hyperledger/fabric-config/configtx/orderer"
 	"github.com/hyperledger/fabric-protos-go/common"
 	"github.com/hyperledger/fabric-protos-go/msp"
 	"github.com/hyperledger/fabric/cmd/common/signer"
 	"github.com/hyperledger/fabric/common/configtx"
 	"github.com/hyperledger/fabric/common/util"
+	intconftx "github.com/hyperledger/fabric/integration/configtx"
 	"github.com/hyperledger/fabric/integration/nwo"
 	"github.com/hyperledger/fabric/integration/nwo/commands"
 	"github.com/hyperledger/fabric/protoutil"
@@ -426,7 +430,8 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			o3Runner = network.OrdererRunner(o3)
 
 			By("Launching orderers with a clustered timeshift")
-			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+			orderers := []*nwo.Orderer{o1, o2, o3}
+			for _, orderer := range orderers {
 				ordererConfig := network.ReadOrdererConfig(orderer)
 				ordererConfig.General.Cluster.TLSHandshakeTimeShift = 5 * time.Minute
 				network.WriteOrdererConfig(orderer, ordererConfig)
@@ -456,7 +461,7 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			o3Runner = network.OrdererRunner(o3)
 
 			By("Launching orderers again without a general timeshift re-using the cluster port")
-			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+			for _, orderer := range orderers {
 				ordererConfig := network.ReadOrdererConfig(orderer)
 				ordererConfig.General.ListenPort = ordererConfig.General.Cluster.ListenPort
 				ordererConfig.General.TLS.Certificate = ordererConfig.General.Cluster.ServerCertificate
@@ -495,7 +500,7 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			o3Runner = network.OrdererRunner(o3)
 
 			By("Launching orderers again with a general timeshift re-using the cluster port")
-			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+			for _, orderer := range orderers {
 				ordererConfig := network.ReadOrdererConfig(orderer)
 				ordererConfig.General.TLS.TLSHandshakeTimeShift = 5 * time.Minute
 				network.WriteOrdererConfig(orderer, ordererConfig)
@@ -511,6 +516,54 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 
 			By("Waiting for a leader to be elected")
 			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
+
+			By("submitting config updates to orderers with expired TLS certs to replace the expired certs")
+			for _, o := range orderers {
+				timeShift := 5 * time.Minute
+				configBlock := fetchConfigBlock(network, peer, o, nwo.ClusterPort, network.SystemChannel.Name, timeShift)
+				channelConfig := configFromBlock(configBlock)
+				c := conftx.New(channelConfig)
+				err = c.Orderer().RemoveConsenter(consenterChannelConfig(network, o))
+				Expect(err).NotTo(HaveOccurred())
+
+				By("renewing the orderer TLS certificates for " + o.Name)
+				renewOrdererCertificates(network, o)
+				err = c.Orderer().AddConsenter(consenterChannelConfig(network, o))
+				Expect(err).NotTo(HaveOccurred())
+
+				By("updating the config for " + o.Name)
+				updateOrdererConfig(network, o, nwo.ClusterPort, network.SystemChannel.Name, timeShift, c.OriginalConfig(), c.UpdatedConfig(), peer)
+			}
+
+			By("Killing orderers")
+			o1Proc.Signal(syscall.SIGTERM)
+			o2Proc.Signal(syscall.SIGTERM)
+			o3Proc.Signal(syscall.SIGTERM)
+			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+
+			o1Runner = network.OrdererRunner(o1)
+			o2Runner = network.OrdererRunner(o2)
+			o3Runner = network.OrdererRunner(o3)
+
+			By("Launching orderers again without a general timeshift")
+			for _, o := range orderers {
+				ordererConfig := network.ReadOrdererConfig(o)
+				ordererConfig.General.TLS.TLSHandshakeTimeShift = 0
+				network.WriteOrdererConfig(o, ordererConfig)
+			}
+
+			o1Proc = ifrit.Invoke(o1Runner)
+			o2Proc = ifrit.Invoke(o2Runner)
+			o3Proc = ifrit.Invoke(o3Runner)
+
+			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+
+			By("Waiting for a leader to be elected")
+			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
 		})
 
 		It("disregards certificate renewal if only the validity period changed", func() {
@@ -765,8 +818,11 @@ func findLeader(ordererRunners []*ginkgomon.Runner) int {
 	return firstLeader
 }
 
-func renewOrdererCertificates(network *nwo.Network, o1, o2, o3 *nwo.Orderer) {
-	ordererDomain := network.Organization(o1.Organization).Domain
+func renewOrdererCertificates(network *nwo.Network, orderers ...*nwo.Orderer) {
+	if len(orderers) == 0 {
+		return
+	}
+	ordererDomain := network.Organization(orderers[0].Organization).Domain
 	ordererTLSCAKeyPath := filepath.Join(network.RootDir, "crypto", "ordererOrganizations",
 		ordererDomain, "tlsca", "priv_sk")
 
@@ -778,8 +834,8 @@ func renewOrdererCertificates(network *nwo.Network, o1, o2, o3 *nwo.Orderer) {
 	ordererTLSCACert, err := ioutil.ReadFile(ordererTLSCACertPath)
 	Expect(err).NotTo(HaveOccurred())
 
-	serverTLSCerts := make(map[string][]byte)
-	for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+	serverTLSCerts := map[string][]byte{}
+	for _, orderer := range orderers {
 		tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(orderer), "server.crt")
 		serverTLSCerts[tlsCertPath], err = ioutil.ReadFile(tlsCertPath)
 		Expect(err).NotTo(HaveOccurred())
@@ -896,7 +952,10 @@ func configFromBootstrapBlock(bootstrapBlock []byte) *common.Config {
 	block := &common.Block{}
 	err := proto.Unmarshal(bootstrapBlock, block)
 	Expect(err).NotTo(HaveOccurred())
+	return configFromBlock(block)
+}
 
+func configFromBlock(block *common.Block) *common.Config {
 	envelope, err := protoutil.GetEnvelopeFromBlock(block.Data.Data[0])
 	Expect(err).NotTo(HaveOccurred())
 
@@ -908,5 +967,88 @@ func configFromBootstrapBlock(bootstrapBlock []byte) *common.Config {
 	Expect(err).NotTo(HaveOccurred())
 
 	return configEnv.Config
+}
+
+func fetchConfigBlock(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration) *common.Block {
+	tempDir, err := ioutil.TempDir(n.RootDir, "fetchConfigBlock")
+	Expect(err).NotTo(HaveOccurred())
+	defer os.RemoveAll(tempDir)
+
+	output := filepath.Join(tempDir, "config_block.pb")
+	sess, err := n.OrdererAdminSession(orderer, peer, commands.ChannelFetch{
+		ChannelID:             channel,
+		Block:                 "config",
+		Orderer:               n.OrdererAddress(orderer, port),
+		OutputFile:            output,
+		ClientAuth:            n.ClientAuthRequired,
+		TLSHandshakeTimeShift: tlsHandshakeTimeShift,
+	})
+	Expect(err).NotTo(HaveOccurred())
+	Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
+	Expect(sess.Err).To(gbytes.Say("Received block: "))
+
+	configBlock := nwo.UnmarshalBlockFromFile(output)
+	return configBlock
+}
+
+func currentConfigBlockNumber(n *nwo.Network, peer *nwo.Peer, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration) uint64 {
+	configBlock := fetchConfigBlock(n, peer, orderer, port, channel, tlsHandshakeTimeShift)
+	return configBlock.Header.Number
+}
 
+func updateOrdererConfig(n *nwo.Network, orderer *nwo.Orderer, port nwo.PortName, channel string, tlsHandshakeTimeShift time.Duration, current, updated *common.Config, submitter *nwo.Peer, additionalSigners ...*nwo.Orderer) {
+	tempDir, err := ioutil.TempDir(n.RootDir, "updateConfig")
+	Expect(err).NotTo(HaveOccurred())
+	updateFile := filepath.Join(tempDir, "update.pb")
+	defer os.RemoveAll(tempDir)
+
+	currentBlockNumber := currentConfigBlockNumber(n, submitter, orderer, port, channel, tlsHandshakeTimeShift)
+	nwo.ComputeUpdateOrdererConfig(updateFile, n, channel, current, updated, submitter, additionalSigners...)
+
+	Eventually(func() bool {
+		sess, err := n.OrdererAdminSession(orderer, submitter, commands.ChannelUpdate{
+			ChannelID:             channel,
+			Orderer:               n.OrdererAddress(orderer, port),
+			File:                  updateFile,
+			ClientAuth:            n.ClientAuthRequired,
+			TLSHandshakeTimeShift: tlsHandshakeTimeShift,
+		})
+		Expect(err).NotTo(HaveOccurred())
+
+		sess.Wait(n.EventuallyTimeout)
+		if sess.ExitCode() != 0 {
+			return false
+		}
+
+		return strings.Contains(string(sess.Err.Contents()), "Successfully submitted channel update")
+	}, n.EventuallyTimeout).Should(BeTrue())
+
+	ccb := func() uint64 {
+		return currentConfigBlockNumber(n, submitter, orderer, port, channel, tlsHandshakeTimeShift)
+	}
+	Eventually(ccb, n.EventuallyTimeout).Should(BeNumerically(">", currentBlockNumber))
+}
+
+func consenterChannelConfig(n *nwo.Network, o *nwo.Orderer) orderer.Consenter {
+	host, port := intconftx.OrdererClusterHostPort(n, o)
+	tlsCert := parseCertificate(filepath.Join(n.OrdererLocalTLSDir(o), "server.crt"))
+	return orderer.Consenter{
+		Address: orderer.EtcdAddress{
+			Host: host,
+			Port: port,
+		},
+		ClientTLSCert: tlsCert,
+		ServerTLSCert: tlsCert,
+	}
+}
+
+// parseCertificate loads the PEM-encoded x509 certificate at the specified
+// path.
+func parseCertificate(path string) *x509.Certificate {
+	certBytes, err := ioutil.ReadFile(path)
+	Expect(err).NotTo(HaveOccurred())
+	pemBlock, _ := pem.Decode(certBytes)
+	cert, err := x509.ParseCertificate(pemBlock.Bytes)
+	Expect(err).NotTo(HaveOccurred())
+	return cert
 }

internal/peer/chaincode/common.go

@@ -500,7 +500,6 @@ func InitCmdFactory(cmdName string, isEndorserRequired, isOrdererRequired bool,
 		}
 
 		broadcastClient, err = common.GetBroadcastClientFnc()
-
 		if err != nil {
 			return nil, errors.WithMessage(err, "error getting broadcast client")
 		}