FAB-18163 TLS Timeshift w/o Separate Cluster Port (#1724)

The existing orderer code allows for a TLS timeshift to be specified when starting the orderer with a separate cluster port. This is useful for situations where the consenter TLS certificates have accidentally expired, preventing quorum from forming, and preventing channel config updates to replace these certificates. This change extends the concept to the general orderer port, which can be optionally re-used, rather than binding to a new port. If the cluster port is not re-used, then the TLS timeshift parameter specified in the general TLS section takes precdence over the parameter specified in the cluster section. Likewise, if there is a separate cluster port, the parameter in the general TLS section has no impact on the cluster listener. Signed-off-by: Jason Yellick <jyellick@us.ibm.com>
hyperledger · Aug 14, 2020 · 5d07186 · 5d07186
1 parent 2837fc5
commit 5d07186
Show file tree

Hide file tree

Showing 4 changed files with 116 additions and 23 deletions.
diff --git a/integration/nwo/fabricconfig/orderer.go b/integration/nwo/fabricconfig/orderer.go
@@ -20,7 +20,7 @@ type Orderer struct {
 
 type General struct {
 	ListenAddress   string                 `yaml:"ListenAddress,omitempty"`
-	ListenPort      int                    `yaml:"ListenPort,omitempty"`
+	ListenPort      uint16                 `yaml:"ListenPort,omitempty"`
 	TLS             *OrdererTLS            `yaml:"TLS,omitempty"`
 	Keepalive       *OrdererKeepalive      `yaml:"Keepalive,omitempty"`
 	BootstrapMethod string                 `yaml:"BootstrapMethod,omitempty"`
@@ -32,17 +32,39 @@ type General struct {
 	Profile         *OrdererProfile        `yaml:"Profile,omitempty"`
 	BCCSP           *BCCSP                 `yaml:"BCCSP,omitempty"`
 	Authentication  *OrdererAuthentication `yaml:"Authentication,omitempty"`
+	Cluster         *Cluster               `yaml:"Cluster,omitempty"`
 
 	ExtraProperties map[string]interface{} `yaml:",inline,omitempty"`
 }
 
+type Cluster struct {
+	ListenAddress                        string        `yaml:"ListenAddress,omitempty"`
+	ListenPort                           uint16        `yaml:"ListenPort,omitempty"`
+	ServerCertificate                    string        `yaml:"ServerCertificate,omitempty"`
+	ServerPrivateKey                     string        `yaml:"ServerPrivateKey,omitempty"`
+	ClientCertificate                    string        `yaml:"ClientCertificate,omitempty"`
+	ClientPrivateKey                     string        `yaml:"ClientPrivateKey,omitempty"`
+	RootCAs                              []string      `yaml:"RootCAs,omitempty"`
+	DialTimeout                          time.Duration `yaml:"DialTimeout,omitempty"`
+	RPCTimeout                           time.Duration `yaml:"RPCTimeout,omitempty"`
+	ReplicationBufferSize                int           `yaml:"ReplicationBufferSize,omitempty"`
+	ReplicationPullTimeout               time.Duration `yaml:"ReplicationPullTimeout,omitempty"`
+	ReplicationRetryTimeout              time.Duration `yaml:"ReplicationRetryTimeout,omitempty"`
+	ReplicationBackgroundRefreshInterval time.Duration `yaml:"ReplicationBackgroundRefreshInterval,omitempty"`
+	ReplicationMaxRetries                int           `yaml:"ReplicationMaxRetries,omitempty"`
+	SendBufferSize                       int           `yaml:"SendBufferSize,omitempty"`
+	CertExpirationWarningThreshold       time.Duration `yaml:"CertExpirationWarningThreshold,omitempty"`
+	TLSHandshakeTimeShift                time.Duration `yaml:"TLSHandshakeTimeShift,omitempty"`
+}
+
 type OrdererTLS struct {
-	Enabled            bool     `yaml:"Enabled"`
-	PrivateKey         string   `yaml:"PrivateKey,omitempty"`
-	Certificate        string   `yaml:"Certificate,omitempty"`
-	RootCAs            []string `yaml:"RootCAs,omitempty"`
-	ClientAuthRequired bool     `yaml:"ClientAuthRequired"`
-	ClientRootCAs      []string `yaml:"ClientRootCAs,omitempty"`
+	Enabled               bool          `yaml:"Enabled"`
+	PrivateKey            string        `yaml:"PrivateKey,omitempty"`
+	Certificate           string        `yaml:"Certificate,omitempty"`
+	RootCAs               []string      `yaml:"RootCAs,omitempty"`
+	ClientAuthRequired    bool          `yaml:"ClientAuthRequired"`
+	ClientRootCAs         []string      `yaml:"ClientRootCAs,omitempty"`
+	TLSHandshakeTimeShift time.Duration `yaml:"TLSHandshakeTimeShift,omitempty"`
 }
 
 type OrdererSASLPlain struct {

diff --git a/integration/raft/cft_test.go b/integration/raft/cft_test.go
@@ -422,19 +422,15 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
 			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
 
-			By("Launching orderers again")
 			o1Runner = network.OrdererRunner(o1)
 			o2Runner = network.OrdererRunner(o2)
 			o3Runner = network.OrdererRunner(o3)
 
-			for i, runner := range []*ginkgomon.Runner{o1Runner, o2Runner, o3Runner} {
-				// Switch between the general port and the cluster listener port
-				runner.Command.Env = append(runner.Command.Env, "ORDERER_GENERAL_CLUSTER_TLSHANDSHAKETIMESHIFT=90s")
-				tlsCertPath := filepath.Join(network.OrdererLocalTLSDir(network.Orderers[i]), "server.crt")
-				tlsKeyPath := filepath.Join(network.OrdererLocalTLSDir(network.Orderers[i]), "server.key")
-				runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_SERVERCERTIFICATE=%s", tlsCertPath))
-				runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_SERVERPRIVATEKEY=%s", tlsKeyPath))
-				runner.Command.Env = append(runner.Command.Env, fmt.Sprintf("ORDERER_GENERAL_CLUSTER_ROOTCAS=%s", ordererTLSCACertPath))
+			By("Launching orderers with a clustered timeshift")
+			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+				ordererConfig := network.ReadOrdererConfig(orderer)
+				ordererConfig.General.Cluster.TLSHandshakeTimeShift = 5 * time.Minute
+				network.WriteOrdererConfig(orderer, ordererConfig)
 			}
 
 			o1Proc = ifrit.Invoke(o1Runner)
@@ -448,6 +444,74 @@ var _ = Describe("EndToEnd Crash Fault Tolerance", func() {
 			By("Waiting for a leader to be elected")
 			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
 
+			By("Killing orderers")
+			o1Proc.Signal(syscall.SIGTERM)
+			o2Proc.Signal(syscall.SIGTERM)
+			o3Proc.Signal(syscall.SIGTERM)
+			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+
+			o1Runner = network.OrdererRunner(o1)
+			o2Runner = network.OrdererRunner(o2)
+			o3Runner = network.OrdererRunner(o3)
+
+			By("Launching orderers again without a general timeshift re-using the cluster port")
+			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+				ordererConfig := network.ReadOrdererConfig(orderer)
+				ordererConfig.General.ListenPort = ordererConfig.General.Cluster.ListenPort
+				ordererConfig.General.TLS.Certificate = ordererConfig.General.Cluster.ServerCertificate
+				ordererConfig.General.TLS.PrivateKey = ordererConfig.General.Cluster.ServerPrivateKey
+				ordererConfig.General.Cluster.TLSHandshakeTimeShift = 0
+				ordererConfig.General.Cluster.ListenPort = 0
+				ordererConfig.General.Cluster.ListenAddress = ""
+				ordererConfig.General.Cluster.ServerCertificate = ""
+				ordererConfig.General.Cluster.ServerPrivateKey = ""
+				network.WriteOrdererConfig(orderer, ordererConfig)
+			}
+
+			o1Proc = ifrit.Invoke(o1Runner)
+			o2Proc = ifrit.Invoke(o2Runner)
+			o3Proc = ifrit.Invoke(o3Runner)
+
+			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+
+			By("Waiting for TLS handshakes to fail")
+			Eventually(o1Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
+			Eventually(o2Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
+			Eventually(o3Runner.Err(), network.EventuallyTimeout).Should(gbytes.Say("tls: bad certificate"))
+
+			By("Killing orderers")
+			o1Proc.Signal(syscall.SIGTERM)
+			o2Proc.Signal(syscall.SIGTERM)
+			o3Proc.Signal(syscall.SIGTERM)
+			Eventually(o1Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+			Eventually(o2Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+			Eventually(o3Proc.Wait(), network.EventuallyTimeout).Should(Receive())
+
+			o1Runner = network.OrdererRunner(o1)
+			o2Runner = network.OrdererRunner(o2)
+			o3Runner = network.OrdererRunner(o3)
+
+			By("Launching orderers again with a general timeshift re-using the cluster port")
+			for _, orderer := range []*nwo.Orderer{o1, o2, o3} {
+				ordererConfig := network.ReadOrdererConfig(orderer)
+				ordererConfig.General.TLS.TLSHandshakeTimeShift = 5 * time.Minute
+				network.WriteOrdererConfig(orderer, ordererConfig)
+			}
+
+			o1Proc = ifrit.Invoke(o1Runner)
+			o2Proc = ifrit.Invoke(o2Runner)
+			o3Proc = ifrit.Invoke(o3Runner)
+
+			Eventually(o1Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+			Eventually(o2Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+			Eventually(o3Proc.Ready(), network.EventuallyTimeout).Should(BeClosed())
+
+			By("Waiting for a leader to be elected")
+			findLeader([]*ginkgomon.Runner{o1Runner, o2Runner, o3Runner})
 		})
 	})
 

diff --git a/orderer/common/localconfig/config.go b/orderer/common/localconfig/config.go
@@ -79,12 +79,13 @@ type Keepalive struct {
 
 // TLS contains configuration for TLS connections.
 type TLS struct {
-	Enabled            bool
-	PrivateKey         string
-	Certificate        string
-	RootCAs            []string
-	ClientAuthRequired bool
-	ClientRootCAs      []string
+	Enabled               bool
+	PrivateKey            string
+	Certificate           string
+	RootCAs               []string
+	ClientAuthRequired    bool
+	ClientRootCAs         []string
+	TLSHandshakeTimeShift time.Duration
 }
 
 // SASLPlain contains configuration for SASL/PLAIN authentication

diff --git a/orderer/common/server/main.go b/orderer/common/server/main.go
@@ -492,8 +492,13 @@ func initializeClusterClientConfig(conf *localconfig.TopLevel) comm.ClientConfig
 		serverRootCAs = append(serverRootCAs, rootCACert)
 	}
 
+	timeShift := conf.General.TLS.TLSHandshakeTimeShift
+	if reuseGrpcListener := reuseListener(conf); !reuseGrpcListener {
+		timeShift = conf.General.Cluster.TLSHandshakeTimeShift
+	}
+
 	cc.SecOpts = comm.SecureOptions{
-		TimeShift:         conf.General.Cluster.TLSHandshakeTimeShift,
+		TimeShift:         timeShift,
 		RequireClientCert: true,
 		CipherSuites:      comm.DefaultTLSCipherSuites,
 		ServerRootCAs:     serverRootCAs,
@@ -510,6 +515,7 @@ func initializeServerConfig(conf *localconfig.TopLevel, metricsProvider metrics.
 	secureOpts := comm.SecureOptions{
 		UseTLS:            conf.General.TLS.Enabled,
 		RequireClientCert: conf.General.TLS.ClientAuthRequired,
+		TimeShift:         conf.General.TLS.TLSHandshakeTimeShift,
 	}
 	// check to see if TLS is enabled
 	if secureOpts.UseTLS {