Fix periodic checks for expiring certificates (#1466)

common/rpc/encryption/localStoreCertProvider.go

@@ -165,9 +165,9 @@ func (s *localStoreCertProvider) GetExpiringCerts(timeWindow time.Duration,
 		return nil, nil, err
 	}
 
-	checkError := checkTLSCertForExpiration(s.certs.serverCert, when, expiring, expired)
+	checkError := checkTLSCertForExpiration(certs.serverCert, when, expiring, expired)
 	err = appendError(err, checkError)
-	checkError = checkTLSCertForExpiration(s.certs.workerCert, when, expiring, expired)
+	checkError = checkTLSCertForExpiration(certs.workerCert, when, expiring, expired)
 	err = appendError(err, checkError)
 
 	checkCertsForExpiration(certs.clientCACerts, when, expiring, expired)
@@ -521,12 +521,15 @@ func (s *localStoreCertProvider) refreshCerts() {
 			continue
 		}
 
-		if s.certs.isEqual(newCerts) {
+		s.RLock()
+		currentCerts := s.certs
+		s.RUnlock()
+		if currentCerts.isEqual(newCerts) {
 			continue
 		}
 
-		s.Lock()
 		s.logger.Info("loaded new TLS certificates")
+		s.Lock()
 		s.certs = newCerts
 		s.Unlock()
 	}

common/rpc/encryption/localStoreTlsProvider.go

@@ -97,6 +97,7 @@ func NewLocalStoreTlsProvider(tlsConfig *config.RootTLS, scope tally.Scope, cert
 		RWMutex:  sync.RWMutex{},
 		settings: tlsConfig,
 		scope:    scope,
+		logger:   log.NewDefaultLogger(),
 	}
 	provider.initialize()
 	return provider, nil
@@ -350,7 +351,10 @@ func (s *localStoreTlsProvider) timerCallback() {
 		}
 		if window != 0 {
 			expiring, expired, err := s.GetExpiringCerts(window)
-			s.logger.Error(fmt.Sprintf("error while checking for certificate expiration: %v", err))
+			if err != nil {
+				s.logger.Error(fmt.Sprintf("error while checking for certificate expiration: %v", err))
+				continue
+			}
 			if s.scope != nil {
 				s.scope.Gauge(metricCertsExpired).Update(float64(len(expired)))
 				s.scope.Gauge(metricCertsExpiring).Update(float64(len(expiring)))

common/rpc/test/rpc_localstore_tls_test.go

@@ -296,6 +296,11 @@ func (s *localStoreRPCSuite) setupFrontend() {
 			Frontend:        s.frontendConfigMutualTLSRefresh,
 			Internode:       s.frontendConfigMutualTLSRefresh,
 			RefreshInterval: time.Second,
+			ExpirationChecks: config.CertExpirationValidation{
+				WarningWindow: time.Hour * 24 * 14,
+				ErrorWindow:   time.Hour * 24 * 7,
+				CheckInterval: time.Second,
+			},
 		},
 	}