Skip to content

Commit

Permalink
[FIXED] Reload TLS certificates on reconnect
Browse files Browse the repository at this point in the history 
Signed-off-by: Piotr Piotrowski <piotr@synadia.com>
  • Loading branch information
@piotrpio
piotrpio committed May 12, 2023
1 parent 2765665 commit 2782dde
Show file tree
Hide file tree
Showing 5 changed files with 235 additions and 23 deletions.
80 changes: 59 additions & 21 deletions nats.go
View file
Expand Up @@ -211,6 +211,13 @@ type ErrHandler func(*Conn, *Subscription, error)
// JWT for this user.
type UserJWTHandler func() (string, error)

// TLSCertHandler is used to fetch and return tls certificate.
type TLSCertHandler func() (tls.Certificate, error)

// RootCAsHandler is used to fetch and return a set of root certificate
// authorities that clients use when verifying server certificates.
type RootCAsHandler func() (*x509.CertPool, error)

// SignatureHandler is used to sign a nonce from the server while
// authenticating with nkeys. The user should sign the nonce and
// return the raw signature. The client will base64 encode this to
Expand Down Expand Up @@ -299,6 +306,13 @@ type Options struct {
// transports.
TLSConfig *tls.Config

// TLSCertCB is used to fetch and return custom tls certificate.
TLSCertCB TLSCertHandler

// RootCAsCB is used to fetch and return a set of root certificate
// authorities that clients use when verifying server certificates.
RootCAsCB RootCAsHandler

// AllowReconnect enables reconnection logic to be used when we
// encounter a disconnect from the current server.
AllowReconnect bool
Expand Down Expand Up @@ -834,21 +848,24 @@ func Secure(tls ...*tls.Config) Option {
// If Secure is not already set this will set it as well.
func RootCAs(file ...string) Option {
return func(o *Options) error {
pool := x509.NewCertPool()
for _, f := range file {
rootPEM, err := os.ReadFile(f)
if err != nil || rootPEM == nil {
return fmt.Errorf("nats: error loading or parsing rootCA file: %w", err)
}
ok := pool.AppendCertsFromPEM(rootPEM)
if !ok {
return fmt.Errorf("nats: failed to parse root certificate from %q", f)
rootCAsCB := func() (*x509.CertPool, error) {
pool := x509.NewCertPool()
for _, f := range file {
rootPEM, err := os.ReadFile(f)
if err != nil || rootPEM == nil {
return nil, fmt.Errorf("nats: error loading or parsing rootCA file: %w", err)
}
ok := pool.AppendCertsFromPEM(rootPEM)
if !ok {
return nil, fmt.Errorf("nats: failed to parse root certificate from %q", f)
}
}
return pool, nil
}
if o.TLSConfig == nil {
o.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12}
}
o.TLSConfig.RootCAs = pool
o.RootCAsCB = rootCAsCB
o.Secure = true
return nil
}
Expand All @@ -858,18 +875,21 @@ func RootCAs(file ...string) Option {
// If Secure is not already set this will set it as well.
func ClientCert(certFile, keyFile string) Option {
return func(o *Options) error {
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return fmt.Errorf("nats: error loading client certificate: %w", err)
}
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return fmt.Errorf("nats: error parsing client certificate: %w", err)
tlsCertCB := func() (tls.Certificate, error) {
cert, err := tls.LoadX509KeyPair(certFile, keyFile)
if err != nil {
return tls.Certificate{}, fmt.Errorf("nats: error loading client certificate: %w", err)
}
cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
if err != nil {
return tls.Certificate{}, fmt.Errorf("nats: error parsing client certificate: %w", err)
}
return cert, nil
}
if o.TLSConfig == nil {
o.TLSConfig = &tls.Config{MinVersion: tls.VersionTLS12}
}
o.TLSConfig.Certificates = []tls.Certificate{cert}
o.TLSCertCB = tlsCertCB
o.Secure = true
return nil
}
Expand Down Expand Up @@ -1969,11 +1989,23 @@ func (nc *Conn) makeTLSConn() error {
}
}
// Allow the user to configure their own tls.Config structure.
var tlsCopy *tls.Config
tlsCopy := &tls.Config{}
if nc.Opts.TLSConfig != nil {
tlsCopy = util.CloneTLSConfig(nc.Opts.TLSConfig)
} else {
tlsCopy = &tls.Config{}
}
if nc.Opts.TLSCertCB != nil {
cert, err := nc.Opts.TLSCertCB()
if err != nil {
return err
}
tlsCopy.Certificates = []tls.Certificate{cert}
}
if nc.Opts.RootCAsCB != nil {
rootCAs, err := nc.Opts.RootCAsCB()
if err != nil {
return err
}
tlsCopy.RootCAs = rootCAs
}
// If its blank we will override it with the current host
if tlsCopy.ServerName == _EMPTY_ {
Expand Down Expand Up @@ -2466,6 +2498,9 @@ func (nc *Conn) sendConnect() error {
// reading byte-by-byte here is ok.
proto, err := nc.readProto()
if err != nil {
if !nc.initc && nc.Opts.AsyncErrorCB != nil {
nc.ach.push(func() { nc.Opts.AsyncErrorCB(nc, nil, err) })
}
return err
}

Expand All @@ -2474,6 +2509,9 @@ func (nc *Conn) sendConnect() error {
// Read the rest now...
proto, err = nc.readProto()
if err != nil {
if !nc.initc && nc.Opts.AsyncErrorCB != nil {
nc.ach.push(func() { nc.Opts.AsyncErrorCB(nc, nil, err) })
}
return err
}
}
Expand Down
28 changes: 28 additions & 0 deletions test/configs/certs/client-cert-invalid.pem
View file
@@ -0,0 +1,28 @@
-----BEGIN CERTIFICATE-----
MIIExDCCAyygAwIBAgIQEdLeZgsrEsLe37gR/voylTANBgkqhkiG9w0BAQsFADCB
szEeMBwGA1UEChMVbWtjZXJ0IGRldmVsb3BtZW50IENBMUQwQgYDVQQLDDtwaW90
cnBpb3Ryb3dza2lAUGlvdHJzLU1hY0Jvb2stUHJvLmxvY2FsIChQaW90ciBQaW90
cm93c2tpKTFLMEkGA1UEAwxCbWtjZXJ0IHBpb3RycGlvdHJvd3NraUBQaW90cnMt
TWFjQm9vay1Qcm8ubG9jYWwgKFBpb3RyIFBpb3Ryb3dza2kpMB4XDTIzMDUxMjEw
MTYyOFoXDTI1MDgxMjEwMTYyOFowbzEnMCUGA1UEChMebWtjZXJ0IGRldmVsb3Bt
ZW50IGNlcnRpZmljYXRlMUQwQgYDVQQLDDtwaW90cnBpb3Ryb3dza2lAUGlvdHJz
LU1hY0Jvb2stUHJvLmxvY2FsIChQaW90ciBQaW90cm93c2tpKTCCASIwDQYJKoZI
hvcNAQEBBQADggEPADCCAQoCggEBAJqlzdmIcPNu8ad7TlrJA2SdAtaxYJJK8lRs
oAdq+PO7JeWE8NyEPSEFpXclWsEvG49gS6AueLPRuVT4WqbIDVqm5Rvcx/a1K39i
6Ik3qpLerGY8vPngIhXoU4CUjNrEyJy22bhCPidJtPRnfkVv/eI6LSA2jWsikGNH
Iqnbu6KtUIKbnLuuH0NR8ycYaqeiOdaCMV6STSXmM5S96qH7h7NexGC08b+aersw
CLelFR04J0RV/cax7U1pgsWKKv8icnjiB4tq5IbYuEZE0g/uJZ3BsB5DXKq+WjNI
uRDJJUWyzrWhBPNIW/1I2zesEXSKCvDcUVAuUceYxiEokR4+pv0CAwEAAaOBljCB
kzAOBgNVHQ8BAf8EBAMCBaAwJwYDVR0lBCAwHgYIKwYBBQUHAwIGCCsGAQUFBwMB
BggrBgEFBQcDBDAfBgNVHSMEGDAWgBQbD6YymnmaX19FroClM52B8doDIDA3BgNV
HREEMDAugglsb2NhbGhvc3SBD2VtYWlsQGxvY2FsaG9zdIcQAAAAAAAAAAAAAAAA
AAAAATANBgkqhkiG9w0BAQsFAAOCAYEAJuFrQ0KdmwEc7UyaoTygW59f1JSJGbZa
Ii5EuMtpSon5DX5NaI5aRE350UtimNrQIu8LAPx1UGwSRuPkzvuNAA/l0HAJrqh3
gEorH6fbsRkqkDUvmNiqTfs+So6R0s2+6yVG6t8+NT1OBH616eQ9efvthwRO0AAL
L8LGJJdYMveEJv+GB/+Zs75MQUxniJ+ip/YxF8bcaRjVS/tb3J52yZ1Eb2UU18kN
uAlFOxiKnwvb2csFcZ6zc4Fpm0LfCrpzPCwGF5y6bsjzpqVej87ea6roG9BJ7vbX
xjbwGfchJZmDsG/g9MeoQoIifYqupQmtaQtlKUUD5MRjDhpOVUEJ4tsXDoZEz9DB
kviE+VlIGU2QJ5l9KU2rIdxfh95rrIaqCt5xsT6wUjNtv0wAfbhMannUhjLv+h+G
tIbMIEo0GFA/uY1eXLO4PTgF+EojqFfpUUM17Z3kubsOSvepxkwyipA5eI2fkThu
Yu5Oyyq9X9Y3vnDMvHKJfkzA56Sp19Oy
-----END CERTIFICATE-----
28 changes: 28 additions & 0 deletions test/configs/certs/client-key-invalid.pem
View file
@@ -0,0 +1,28 @@
-----BEGIN PRIVATE KEY-----
MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQCapc3ZiHDzbvGn
e05ayQNknQLWsWCSSvJUbKAHavjzuyXlhPDchD0hBaV3JVrBLxuPYEugLniz0blU
+FqmyA1apuUb3Mf2tSt/YuiJN6qS3qxmPLz54CIV6FOAlIzaxMicttm4Qj4nSbT0
Z35Fb/3iOi0gNo1rIpBjRyKp27uirVCCm5y7rh9DUfMnGGqnojnWgjFekk0l5jOU
veqh+4ezXsRgtPG/mnq7MAi3pRUdOCdEVf3Gse1NaYLFiir/InJ44geLauSG2LhG
RNIP7iWdwbAeQ1yqvlozSLkQySVFss61oQTzSFv9SNs3rBF0igrw3FFQLlHHmMYh
KJEePqb9AgMBAAECggEAMypWT/mPfUsgksv+IZVOFRTJoqSvEdfQE1SZIbsnwOQT
Zru0QRFTdEB8/U2TmETwtmAixU16y+vAiLderr2ThYGgXbaPRjWsvYnI69VKDyuz
GGRSFc4tGNh0AB+l9p+SzB7HK+pmy/Lb9tzi7zBdbGLZGUZTRbX61Y3sjwxPKUPw
mrTindOvSH6FbVevAC6UCl92R9vk4ugS/oDZWKPntHeJX8NzXM6MfZ68+oPKeGXE
DTQ98nZDfZMRDvyyClaAVfstsPV1pxYNYIWaB1w0saL2NZz08zZeGnkqFt621Q4V
gsE9t9Gjg1o5paq0MEm5vBoJo7VyCoR7w/sfEzQaAQKBgQDAk/yOY2yxGcjZjE8q
ozXq/EtbC/ldKcgngm1KYtA7ZyzRt7gGuAuv6sbmri4wpHL1D2UeKS8QPsywkXvM
Oto3NdJraXbC26ObCP+njwWHWD1Zh3BD2O2mYOFUsTWzsxaZJbsy6Sh4cLg+0gVc
UYzqOnUY5EJh6hCnGSZnK4v4fQKBgQDNk/QbK4RrRH33qaAly/Ihw5ZI76s3hc4Y
RcsGi05iAV/jiE9HVF8vWytp1EdoLsO0BPrA9RPP8SYZdHCWh0KFYJtFzU+o8+1W
ThtCIPdmOmAtQnoj52TMmwc+x/WbNIBvBrKQHIbTX9JHUiGFM9NqSuVjNhVDOzvM
/o2D38swgQKBgBRzou68QF7OjjYMYJv2mVNLV/VjYCg0t7z6bQDpXZPxcSEUkcak
5RjZpiX5eY5Q6KR97g818HmZMcPOr4cQ+PvEC4S8vpATI1zjp8LzvXKSPHG1oIaU
EykIQOXtq/ZZnpzFFQxjFpkz311MkKUtQ/ncG3N5SlN7uCkG0r1CMqtBAoGBAL/z
myVXb9Bc5qW+a7t+/7oJDyVRK/Su6m39lQGqR2j5UZh5qVS38hycqx+ox3f+2lsX
ny9WZsZtq55u+8WBzFoPh0wY1X2zLXO9gHQxpe99KFp6TOODZropMw2q1aiy0A1b
GpW3HSj2urg/du8SIiCIiEEnuZjKER9qu6Zb6zSBAoGBALxqe9jb7WLArV/eMEtx
zg7V/FZfFyqEGEbLMM9njM6uiSq0u17H5bsvmgi+dAot16BbDPKWdOw01zQDhphe
GbchPMuNOPNyBm3MIJ5zXi4pQcc5W+Z5z54X9BCBJwIHEp+Tt9VJ6J9/RkSoTXp9
iq9elhb5bfMSA/KliX3cBTge
-----END PRIVATE KEY-----
120 changes: 119 additions & 1 deletion test/conn_test.go
View file
@@ -1,4 +1,4 @@
// Copyright 2012-2022 The NATS Authors
// Copyright 2012-2023 The NATS Authors
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
Expand All @@ -21,6 +21,7 @@ import (
"fmt"
"net"
"os"
"path/filepath"
"runtime"
"strconv"
"strings"
Expand Down Expand Up @@ -300,6 +301,123 @@ func TestClientCertificate(t *testing.T) {
}
}

func TestClientCertificateReloadOnServerRestart(t *testing.T) {
copyFiles := func(t *testing.T, cpFiles map[string]string) {
for from, to := range cpFiles {
content, err := os.ReadFile(from)
if err != nil {
t.Fatalf("Error reading file: %s", err)
}
if err := os.WriteFile(to, content, 0640); err != nil {
t.Fatalf("Error writing file: %s", err)
}
}
}

s, opts := RunServerWithConfig("./configs/tlsverify.conf")
defer s.Shutdown()

endpoint := fmt.Sprintf("%s:%d", opts.Host, opts.Port)
secureURL := fmt.Sprintf("nats://%s", endpoint)

tmpCertDir := t.TempDir()
certFile := filepath.Join(tmpCertDir, "client-cert.pem")
keyFile := filepath.Join(tmpCertDir, "client-key.pem")
caFile := filepath.Join(tmpCertDir, "ca.pem")

// copy valid cert files to tmp dir
filesToCopy := map[string]string{
"./configs/certs/client-cert.pem": certFile,
"./configs/certs/client-key.pem": keyFile,
"./configs/certs/ca.pem": caFile,
}
copyFiles(t, filesToCopy)

dcChan, rcChan, errChan := make(chan bool, 1), make(chan bool, 1), make(chan error, 1)
nc, err := nats.Connect(secureURL,
nats.RootCAs(caFile),
nats.ClientCert(certFile, keyFile),
nats.ReconnectWait(100*time.Millisecond),
nats.ErrorHandler(func(_ *nats.Conn, _ *nats.Subscription, err error) {
errChan <- err
}),
nats.DisconnectErrHandler(func(_ *nats.Conn, _ error) {
dcChan <- true
}),
nats.ReconnectHandler(func(_ *nats.Conn) {
rcChan <- true
}),
)
if err != nil {
t.Fatalf("Failed to create (TLS) connection: %v", err)
}
defer nc.Close()

// overwrite client certificate files with invalid ones, those
// should be loaded on server restart
filesToCopy = map[string]string{
"./configs/certs/client-cert-invalid.pem": certFile,
"./configs/certs/client-key-invalid.pem": keyFile,
}
copyFiles(t, filesToCopy)

// restart server
s.Shutdown()
s, _ = RunServerWithConfig("./configs/tlsverify.conf")
defer s.Shutdown()

// wait for disconnected signal
if err := Wait(dcChan); err != nil {
t.Fatal("Failed to receive disconnect signal")
}

// wait for reconnection error (bad certificate)
select {
case err := <-errChan:
if !strings.Contains(err.Error(), "bad certificate") {
t.Fatalf("Expected bad certificate error; got: %s", err)
}
case <-time.After(5 * time.Second):
t.Fatalf("Timeout waiting for reconnect error")
}

// overwrite cert files with valid ones again,
// so that subsequent reconnect attempt should succeed
// when cert files are reloaded
filesToCopy = map[string]string{
"./configs/certs/client-cert.pem": certFile,
"./configs/certs/client-key.pem": keyFile,
}
copyFiles(t, filesToCopy)

// wait for reconnect signal
if err := Wait(rcChan); err != nil {
t.Fatal("Failed to receive reconnect signal")
}

// pub-sub test message to make sure connection is OK
omsg := []byte("Hello!")
checkRecv := make(chan bool)

received := 0
nc.Subscribe("foo", func(m *nats.Msg) {
received++
if !bytes.Equal(m.Data, omsg) {
t.Fatal("Message received does not match")
}
checkRecv <- true
})
err = nc.Publish("foo", omsg)
if err != nil {
t.Fatalf("Failed to publish on secure (TLS) connection: %v", err)
}
nc.Flush()

if err := Wait(checkRecv); err != nil {
t.Fatal("Failed to receive message")
}
}

func TestServerTLSHintConnections(t *testing.T) {
s, opts := RunServerWithConfig("./configs/tls.conf")
defer s.Shutdown()
Expand Down
2 changes: 1 addition & 1 deletion ws.go
View file
Expand Up @@ -555,7 +555,7 @@ func wsFillFrameHeader(fh []byte, compressed bool, frameType wsOpCode, l int) (i

func (nc *Conn) wsInitHandshake(u *url.URL) error {
compress := nc.Opts.Compression
tlsRequired := u.Scheme == wsSchemeTLS || nc.Opts.Secure || nc.Opts.TLSConfig != nil
tlsRequired := u.Scheme == wsSchemeTLS || nc.Opts.Secure || nc.Opts.TLSConfig != nil || nc.Opts.TLSCertCB != nil || nc.Opts.RootCAsCB != nil
// Do TLS here as needed.
if tlsRequired {
if err := nc.makeTLSConn(); err != nil {
Expand Down

0 comments on commit 2782dde

Please sign in to comment.