[FAB-11960] Introduce TLS to integration tests

This change set makes integration tests run with TLS.

Change-Id: I917f512b3765e52ec27d17830e93e091b360f322
Signed-off-by: yacovm <yacovm@il.ibm.com>

Author: yacovm

integration/discovery/discovery_test.go

@@ -355,7 +355,7 @@ func toDiscoveredPeer(n *nwo.Network, p *nwo.Peer, chaincodes ...string) Discove
 
 	return DiscoveredPeer{
 		MSPID:      n.Organization(p.Organization).MSPID,
-		Endpoint:   fmt.Sprintf("0.0.0.0:%d", n.PeerPort(p, nwo.ListenPort)),
+		Endpoint:   fmt.Sprintf("127.0.0.1:%d", n.PeerPort(p, nwo.ListenPort)),
 		Identity:   string(peerCert),
 		Chaincodes: chaincodes,
 	}

integration/nwo/command.go

@@ -24,6 +24,15 @@ type WorkingDirer interface {
 	WorkingDir() string
 }
 
+func ConnectsToOrderer(c Command) bool {
+	for _, arg := range c.Args() {
+		if arg == "--orderer" {
+			return true
+		}
+	}
+	return false
+}
+
 func NewCommand(path string, command Command) *exec.Cmd {
 	cmd := exec.Command(path, command.Args()...)
 	cmd.Env = os.Environ()

integration/nwo/core_template.go

@@ -22,9 +22,9 @@ logging:
 peer:
   id: {{ Peer.ID }}
   networkId: {{ .NetworkID }}
-  address: 0.0.0.0:{{ .PeerPort Peer "Listen" }}
-  addressAutoDetect: false
-  listenAddress: 0.0.0.0:{{ .PeerPort Peer "Listen" }}
+  address: 127.0.0.1:{{ .PeerPort Peer "Listen" }}
+  addressAutoDetect: true
+  listenAddress: 127.0.0.1:{{ .PeerPort Peer "Listen" }}
   chaincodeListenAddress: 0.0.0.0:{{ .PeerPort Peer "Chaincode" }}
   gomaxprocs: -1
   keepalive:
@@ -36,7 +36,7 @@ peer:
       interval: 60s
       timeout: 20s
   gossip:
-    bootstrap: 0.0.0.0:{{ .PeerPort Peer "Listen" }}
+    bootstrap: 127.0.0.1:{{ .PeerPort Peer "Listen" }}
     useLeaderElection: true
     orgLeader: false
     endpoint:
@@ -61,7 +61,7 @@ peer:
     aliveTimeInterval: 5s
     aliveExpirationTimeout: 25s
     reconnectInterval: 25s
-    externalEndpoint: 0.0.0.0:{{ .PeerPort Peer "Listen" }}
+    externalEndpoint: 127.0.0.1:{{ .PeerPort Peer "Listen" }}
     election:
       startupGracePeriod: 15s
       membershipSampleInterval: 1s
@@ -72,24 +72,24 @@ peer:
       transientstoreMaxBlockRetention: 1000
       pushAckTimeout: 3s
   events:
-    address: 0.0.0.0:{{ .PeerPort Peer "Events" }}
+    address: 127.0.0.1:{{ .PeerPort Peer "Events" }}
     buffersize: 100
     timeout: 10ms
     timewindow: 15m
     keepalive:
       minInterval: 60s
   tls:
-    enabled:  false
+    enabled:  true
     clientAuthRequired: false
     cert:
-      file: tls/server.crt
+      file: {{ .PeerLocalTLSDir Peer }}/server.crt
     key:
-      file: tls/server.key
+      file: {{ .PeerLocalTLSDir Peer }}/server.key
     rootcert:
-      file: tls/ca.crt
+      file: {{ .PeerLocalTLSDir Peer }}/ca.crt
     clientRootCAs:
       files:
-      - tls/ca.crt
+      - {{ .PeerLocalTLSDir Peer }}/ca.crt
   authentication:
     timewindow: 15m
   fileSystemPath: filesystem

integration/nwo/crypto_template.go

@@ -43,6 +43,10 @@ PeerOrgs:{{ range .PeerOrgs }}
     Count: {{ .Users }}
   Specs:{{ range $w.PeersInOrg .Name }}
   - Hostname: {{ .Name }}
+    SANS:
+    - localhost
+    - 127.0.0.1
+    - ::1
   {{- end }}
 {{- end }}
 {{- end }}

integration/nwo/network.go

@@ -7,6 +7,7 @@ SPDX-License-Identifier: Apache-2.0
 package nwo
 
 import (
+	"bytes"
 	"fmt"
 	"io"
 	"io/ioutil"
@@ -19,7 +20,7 @@ import (
 	"text/template"
 	"time"
 
-	docker "github.com/fsouza/go-dockerclient"
+	"github.com/fsouza/go-dockerclient"
 	"github.com/hyperledger/fabric/integration/helpers"
 	"github.com/hyperledger/fabric/integration/nwo/commands"
 	"github.com/hyperledger/fabric/integration/nwo/fabricconfig"
@@ -30,7 +31,7 @@ import (
 	"github.com/tedsuo/ifrit"
 	"github.com/tedsuo/ifrit/ginkgomon"
 	"github.com/tedsuo/ifrit/grouper"
-	yaml "gopkg.in/yaml.v2"
+	"gopkg.in/yaml.v2"
 )
 
 // Organization models information about an Organization. It includes
@@ -134,18 +135,19 @@ type Network struct {
 	NetworkID         string
 	EventuallyTimeout time.Duration
 
-	PortsByBrokerID  map[string]Ports
-	PortsByOrdererID map[string]Ports
-	PortsByPeerID    map[string]Ports
-	Organizations    []*Organization
-	SystemChannel    *SystemChannel
-	Channels         []*Channel
-	Consensus        *Consensus
-	Orderers         []*Orderer
-	Peers            []*Peer
-	Profiles         []*Profile
-	Consortiums      []*Consortium
-	Templates        *Templates
+	CACertificateBundlePath string
+	PortsByBrokerID         map[string]Ports
+	PortsByOrdererID        map[string]Ports
+	PortsByPeerID           map[string]Ports
+	Organizations           []*Organization
+	SystemChannel           *SystemChannel
+	Channels                []*Channel
+	Consensus               *Consensus
+	Orderers                []*Orderer
+	Peers                   []*Peer
+	Profiles                []*Profile
+	Consortiums             []*Consortium
+	Templates               *Templates
 
 	colorIndex uint
 }
@@ -307,9 +309,9 @@ func (n *Network) WritePeerConfig(p *Peer, config *fabricconfig.Core) {
 	Expect(err).NotTo(HaveOccurred())
 }
 
-// PeerUserMSPDir returns the path to the MSP directory containing the
+// peerUserCrpytoDir returns the path to the directory containing the
 // certificates and keys for the specified user of the peer.
-func (n *Network) PeerUserMSPDir(p *Peer, user string) string {
+func (n *Network) peerUserCrpytoDir(p *Peer, user, cryptoMaterialType string) string {
 	org := n.Organization(p.Organization)
 	Expect(org).NotTo(BeNil())
 
@@ -320,10 +322,22 @@ func (n *Network) PeerUserMSPDir(p *Peer, user string) string {
 		org.Domain,
 		"users",
 		fmt.Sprintf("%s@%s", user, org.Domain),
-		"msp",
+		cryptoMaterialType,
 	)
 }
 
+// PeerUserMSPDir returns the path to the MSP directory containing the
+// certificates and keys for the specified user of the peer.
+func (n *Network) PeerUserMSPDir(p *Peer, user string) string {
+	return n.peerUserCrpytoDir(p, user, "msp")
+}
+
+// PeerUserTLSDir returns the path to the TLS directory containing the
+// certificates and keys for the specified user of the peer.
+func (n *Network) PeerUserTLSDir(p *Peer, user string) string {
+	return n.peerUserCrpytoDir(p, user, "tls")
+}
+
 // PeerUserCert returns the path to the certificate for the specified user in
 // the peer organization.
 func (n *Network) PeerUserCert(p *Peer, user string) string {
@@ -356,8 +370,8 @@ func (n *Network) PeerUserKey(p *Peer, user string) string {
 	return filepath.Join(keystore, keys[0].Name())
 }
 
-// PeerLocalMSPDir returns the path to the local MSP directory for the peer.
-func (n *Network) PeerLocalMSPDir(p *Peer) string {
+// peerLocalCryptoDir returns the path to the local crypto directory for the peer.
+func (n *Network) peerLocalCryptoDir(p *Peer, cryptoType string) string {
 	org := n.Organization(p.Organization)
 	Expect(org).NotTo(BeNil())
 
@@ -368,10 +382,20 @@ func (n *Network) PeerLocalMSPDir(p *Peer) string {
 		org.Domain,
 		"peers",
 		fmt.Sprintf("%s.%s", p.Name, org.Domain),
-		"msp",
+		cryptoType,
 	)
 }
 
+// PeerLocalMSPDir returns the path to the local MSP directory for the peer.
+func (n *Network) PeerLocalMSPDir(p *Peer) string {
+	return n.peerLocalCryptoDir(p, "msp")
+}
+
+// PeerLocalTLSDir returns the path to the local TLS directory for the peer.
+func (n *Network) PeerLocalTLSDir(p *Peer) string {
+	return n.peerLocalCryptoDir(p, "tls")
+}
+
 // PeerCert returns the path to the peer's certificate.
 func (n *Network) PeerCert(p *Peer) string {
 	org := n.Organization(p.Organization)
@@ -407,9 +431,9 @@ func (n *Network) OrdererOrgMSPDir(o *Organization) string {
 	)
 }
 
-// OrdererLocalMSPDir returns the path to the local MSP directory for the
+// ordererLocalCryptoDir returns the path to the local crypto directory for the
 // Orderer.
-func (n *Network) OrdererLocalMSPDir(o *Orderer) string {
+func (n *Network) ordererLocalCryptoDir(o *Orderer, cryptoType string) string {
 	org := n.Organization(o.Organization)
 	Expect(org).NotTo(BeNil())
 
@@ -420,10 +444,22 @@ func (n *Network) OrdererLocalMSPDir(o *Orderer) string {
 		org.Domain,
 		"orderers",
 		fmt.Sprintf("%s.%s", o.Name, org.Domain),
-		"msp",
+		cryptoType,
 	)
 }
 
+// OrdererLocalMSPDir returns the path to the local MSP directory for the
+// Orderer.
+func (n *Network) OrdererLocalMSPDir(o *Orderer) string {
+	return n.ordererLocalCryptoDir(o, "msp")
+}
+
+// OrdererLocalTLSDir returns the path to the local TLS directory for the
+// Orderer.
+func (n *Network) OrdererLocalTLSDir(o *Orderer) string {
+	return n.ordererLocalCryptoDir(o, "tls")
+}
+
 // ProfileForChannel gets the configtxgen profile name associated with the
 // specified channel.
 func (n *Network) ProfileForChannel(channelName string) string {
@@ -512,6 +548,42 @@ func (n *Network) Bootstrap() {
 		Expect(err).NotTo(HaveOccurred())
 		Eventually(sess, n.EventuallyTimeout).Should(gexec.Exit(0))
 	}
+
+	n.concatenateTLSCACertificates()
+}
+
+// concatenateTLSCACertificates concatenates all TLS CA certificates
+// into a single file to be used by peer CLI
+func (n *Network) concatenateTLSCACertificates() {
+	tlsCertificatesFilePath := filepath.Join(n.RootDir, "crypto", "tlsCACerts.pem")
+	concatenatedTLSCABytes := bytes.Buffer{}
+	for _, tlsCertPath := range n.listTLSCACertificates() {
+		certBytes, err := ioutil.ReadFile(tlsCertPath)
+		Expect(err).NotTo(HaveOccurred())
+		concatenatedTLSCABytes.Write(certBytes)
+	}
+	err := ioutil.WriteFile(tlsCertificatesFilePath, concatenatedTLSCABytes.Bytes(), 0660)
+	Expect(err).NotTo(HaveOccurred())
+	n.CACertificateBundlePath = tlsCertificatesFilePath
+}
+
+// listTLSCACertificates returns the paths of all TLS CA certificates
+// in the network, across all organizations.
+func (n *Network) listTLSCACertificates() []string {
+	fileName2Path := make(map[string]string)
+	filepath.Walk(filepath.Join(n.RootDir, "crypto"), func(path string, info os.FileInfo, err error) error {
+		// File starts with "tlsca" and has "-cert.pem" in it
+		if strings.Index(info.Name(), "tlsca") == 0 && strings.Contains(info.Name(), "-cert.pem") {
+			fileName2Path[info.Name()] = path
+		}
+		return nil
+	})
+
+	var tlsCACertificates []string
+	for _, path := range fileName2Path {
+		tlsCACertificates = append(tlsCACertificates, path)
+	}
+	return tlsCACertificates
 }
 
 // Cleanup attempts to cleanup docker related artifacts that may
@@ -692,6 +764,7 @@ func (n *Network) ConfigTxGen(command Command) (*gexec.Session, error) {
 // Discover starts a gexec.Session for the provided discover command.
 func (n *Network) Discover(command Command) (*gexec.Session, error) {
 	cmd := NewCommand(n.Components.Discover(), command)
+	cmd.Args = append(cmd.Args, "--peerTLSCA", n.CACertificateBundlePath)
 	return n.StartSession(cmd, command.SessionName())
 }
 
@@ -848,10 +921,35 @@ func (n *Network) NetworkGroupRunner() ifrit.Runner {
 
 func (n *Network) peerCommand(command Command, env ...string) *exec.Cmd {
 	cmd := NewCommand(n.Components.Peer(), command)
-	cmd.Env = append(env, cmd.Env...)
+	cmd.Env = append(cmd.Env, env...)
+	if ConnectsToOrderer(command) {
+		cmd.Args = append(cmd.Args, "--tls")
+		cmd.Args = append(cmd.Args, "--cafile", n.CACertificateBundlePath)
+	}
+
+	// In case we have a peer invoke with multiple certificates,
+	// we need to mimic the correct peer CLI usage,
+	// so we count the number of --peerAddresses usages
+	// we have, and add the same (concatenated TLS CA certificates file)
+	// the same number of times to bypass the peer CLI sanity checks
+	requiredPeerAddresses := flagCount("--peerAddresses", cmd.Args)
+	for i := 0; i < requiredPeerAddresses; i++ {
+		cmd.Args = append(cmd.Args, "--tlsRootCertFiles")
+		cmd.Args = append(cmd.Args, n.CACertificateBundlePath)
+	}
 	return cmd
 }
 
+func flagCount(flag string, args []string) int {
+	var c int
+	for _, arg := range args {
+		if arg == flag {
+			c++
+		}
+	}
+	return c
+}
+
 // PeerAdminSession starts a gexec.Session as a peer admin for the provided
 // peer command. This is intended to be used by short running peer cli commands
 // that execute in the context of a peer configuration.

integration/nwo/orderer_template.go

@@ -13,11 +13,11 @@ General:
   ListenAddress: 127.0.0.1
   ListenPort: {{ .OrdererPort Orderer "Listen" }}
   TLS:
-    Enabled: false
-    PrivateKey: tls/server.key
-    Certificate: tls/server.crt
+    Enabled: true
+    PrivateKey: {{ $w.OrdererLocalTLSDir Orderer }}/server.key
+    Certificate: {{ $w.OrdererLocalTLSDir Orderer }}/server.crt
     RootCAs:
-    - tls/ca.crt
+    -  {{ $w.OrdererLocalTLSDir Orderer }}/ca.crt
     ClientAuthRequired: false
     ClientRootCAs:
   Keepalive: