[FAB-9321] Panic in caching connector

A panic 'send on closed channel' results when
DialContext is called after the CachingConnector
has been closed.

Change-Id: I27914ac99e0269817d6995fbcde1ca82fcf162a7
Signed-off-by: Bob Stasyszyn <Bob.Stasyszyn@securekey.com>

pkg/fab/comm/connector.go

@@ -169,6 +169,10 @@ func (cc *CachingConnector) createConn(ctx context.Context, target string, opts
 	cc.lock.Lock()
 	defer cc.lock.Unlock()
 
+	if cc.janitorDone == nil {
+		return nil, errors.New("caching connector is closed")
+	}
+
 	cconn, ok := cc.loadConn(target)
 	if ok {
 		return cconn, nil
@@ -225,6 +229,11 @@ func (cc *CachingConnector) shutdownConn(cconn *cachedConn) {
 	cc.lock.Lock()
 	defer cc.lock.Unlock()
 
+	if cc.janitorDone == nil {
+		logger.Debug("Connector already closed")
+		return
+	}
+
 	logger.Debugf("connection was shutdown [%s]", cconn.target)
 	cc.conns.Delete(cconn.target)
 	delete(cc.index, cconn.conn)

pkg/fab/comm/connector_test.go

@@ -77,6 +77,20 @@ func TestReleaseAfterClose(t *testing.T) {
 	assert.Equal(t, connectivity.Shutdown, conn1.GetState(), "connection should be shutdown")
 	connector.ReleaseConn(conn1)
 }
+
+func TestDialAfterClose(t *testing.T) {
+	connector := NewCachingConnector(normalSweepTime, normalIdleTime)
+
+	ctx, cancel := context.WithTimeout(context.Background(), normalTimeout)
+	conn1, err := connector.DialContext(ctx, endorserAddr[0], grpc.WithInsecure())
+	cancel()
+	assert.Nil(t, err, "DialContext should have succeeded")
+	connector.Close()
+	assert.Equal(t, connectivity.Shutdown, conn1.GetState(), "connection should be shutdown")
+	_, err = connector.DialContext(ctx, endorserAddr[0], grpc.WithInsecure())
+	assert.Error(t, err, "expecting error when dialing after connector is closed")
+}
+
 func TestConnectorHappyFlushNumber1(t *testing.T) {
 	connector := NewCachingConnector(normalSweepTime, normalIdleTime)
 	defer connector.Close()