crypto/tls: TLS handshake issue with Eclipse Paho MQTT client and RabbitMQ

[adeveloper87]

Hi All,

I'm trying to connect to my RabbitMQ broker using the Eclipse Paho MQTT client (go lang version).

I'm using go1.14.6 linux/arm.

My goal is to establish a secure connection with mutual authentication between my Go client and RabbitMQ broker.

I got the following TLS error from the Go client:

panic: Network Error : remote error: tls: handshake failure

I cannot see any relevant logs on my RabitMQ broker:

2020-07-17 16:43:42.936 [debug] <0.17255.19> Supervisor {<0.17255.19>,rabbit_mqtt_connection_sup} started rabbit_mqtt_connection_sup:start_keepalive_link() at pid <0.17256.19>
2020-07-17 16:43:42.936 [debug] <0.17255.19> Supervisor {<0.17255.19>,rabbit_mqtt_connection_sup} started rabbit_mqtt_reader:start_link(<0.17256.19>, {acceptor,{0,0,0,0,0,0,0,0},8883}) at pid <0.17257.19>

Please note that if i use openssl CLI it works fine with the same broker and certificates:

openssl s_client -connect :8883 -debug -CAfile /tmp/ca.crt -key /tmp/private-key.crt -cert /tmp/client-cert.crt

Could you help me to solve this issue? I can share privately rootCA + client cert + private key + server host.

Below the code that i'm using:
#############

GO CLIENT

#############

package main

import (
    MQTT "github.com/eclipse/paho.mqtt.golang"
    "fmt"
    "time"
    "io/ioutil"
    "crypto/tls"
    "crypto/x509"
)

var (
    brokerUrl = "ssl://<server-host>:8883"
)

func main() {
    opts := MQTT.NewClientOptions()
    opts.SetClientID("MY-CLIENT-ID")
    opts.AddBroker(brokerUrl)   
    opts.SetPingTimeout(1 * time.Second)
    opts.SetAutoReconnect(true)
    opts.SetCleanSession(true)
    opts.SetKeepAlive(10 * time.Second)
    opts.SetConnectTimeout(10 * time.Second)
    opts.SetTLSConfig(NewTLSConfig())

    client := MQTT.NewClient(opts)
    if token := client.Connect(); token.Wait() && token.Error() != nil {
        panic(token.Error())
    }

    fmt.Println("Client Connected")
}

func NewTLSConfig() *tls.Config {
    certpool, err := x509.SystemCertPool()
    if err != nil {
        return nil
    }

    pemCert, err := ioutil.ReadFile("ca.crt")
    if err != nil {
        return nil   
    }
   
    certpool.AppendCertsFromPEM(pemCert)
   
    // Import client certificate/key pair
    cert, err := tls.LoadX509KeyPair("client-cert.crt", "private-key.crt.key")
    if err != nil {
        return nil
    }

    // Just to print out the client certificate...
    cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
    if err != nil {
        return nil
    }
   
    // Create tls.Config with desired tls properties
    return &tls.Config{
       
        // RootCAs = certs used to verify server cert.
        RootCAs: certpool,

        // Certificates = list of certs client sends to server.
        Certificates: []tls.Certificate{cert},
       
        PreferServerCipherSuites: true,
       
        CipherSuites: []uint16{
            tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
            tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
            tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
            tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
            tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
        },
    }
}

######################
# RabbitMQ configuration #
######################

{versions, ['tlsv1.2']},
{ciphers,  [
           {ecdhe_ecdsa,aes_256_gcm,aead,sha384}, TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 ECDHE-RSA-AES256-GCM-SHA384
           {ecdhe_rsa,aes_256_gcm,aead,sha384}, TLS_RSA_WITH_AES_256_GCM_SHA384
           {ecdh_ecdsa,aes_256_gcm,aead,sha384},
           {ecdh_rsa,aes_256_gcm,aead,sha384},
           {dhe_rsa,aes_256_gcm,aead,sha384},
           {dhe_dss,aes_256_gcm,aead,sha384},
           {ecdhe_ecdsa,aes_128_gcm,aead,sha256}, TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256
           {ecdhe_rsa,aes_128_gcm,aead,sha256}, TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256
           {ecdh_ecdsa,aes_128_gcm,aead,sha256},
           {ecdh_rsa,aes_128_gcm,aead,sha256},
           {dhe_rsa,aes_128_gcm,aead,sha256},
           {dhe_dss,aes_128_gcm,aead,sha256}
         ]},
{honor_cipher_order,   true},
{honor_ecc_order,      true},
{client_renegotiation, false},
{secure_renegotiate,   true},
{verify,               verify_peer},
{fail_if_no_peer_cert, true}]

Thanks in advance,
Dario

[davecheney]

Have a look at your error handling, in several cases where err != nil, you are returning nil.

[adeveloper87]

Hi @davecheney,

thanks for your reply.

I think that my issue does not depends from a wrong error handling (no error raised on NewTLSConfig function).

Regards,
Dario

[davecheney]

Thank you for your reply. Please understand that triaging an issue without the context you have it’s important not to be distracted by unimportant details. If you could address the error handling in your example and confirm the problem still occurs that would be a great help.

[adeveloper87]

Hi @davecheney,

sure.
Below the main.go updated:

package main

import (
	MQTT "github.com/eclipse/paho.mqtt.golang"
	"fmt"
	"time"
	"io/ioutil"
	"crypto/tls"
	"crypto/x509"
)

var (
	brokerUrl = "ssl://<RABBITMQ-HOST>:8883"
)

func main() {
	opts := MQTT.NewClientOptions()
	opts.SetClientID("aa6d566f-46ea-4ada-b4ef-07b18766429b")
	opts.AddBroker(brokerUrl)	
	opts.SetPingTimeout(1 * time.Second)
	opts.SetAutoReconnect(true)
	opts.SetCleanSession(true)
	opts.SetKeepAlive(10 * time.Second)
	opts.SetConnectTimeout(10 * time.Second)
	
	tlsConfig := NewTLSConfig()
	if tlsConfig == nil {
		panic("tlsConfig is nil")
	}
	
	opts.SetTLSConfig(tlsConfig)

	client := MQTT.NewClient(opts)
	if token := client.Connect(); token.Wait() && token.Error() != nil {
		panic(token.Error())
	}

	fmt.Println("Client Connected")
}

func NewTLSConfig() *tls.Config {
	
	// Import trusted certificates from CAfile.pem.
	// Alternatively, manually add CA certificates to
	// default openssl CA bundle.
	certpool, err := x509.SystemCertPool()
	if err != nil {
		fmt.Errorf("x509.SystemCertPool failed")
		return nil
	}

	pemCert, err := ioutil.ReadFile("ca.crt")
	if err != nil {
		fmt.Errorf("ioutil.ReadFile failed")
		return nil	
	}
	
	certpool.AppendCertsFromPEM(pemCert)
	
	// Import client certificate/key pair
	cert, err := tls.LoadX509KeyPair("client-cert.crt", "private-key.crt.key")
	if err != nil {
		fmt.Errorf("tls.LoadX509KeyPair failed")
		return nil
	}

	// Just to print out the client certificate...
	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		fmt.Errorf("x509.ParseCertificate failed")
		return nil
	}
	
	// Create tls.Config with desired tls properties
	return &tls.Config{
		
		// RootCAs = certs used to verify server cert.
		RootCAs: certpool,

		// Certificates = list of certs client sends to server.
		Certificates: []tls.Certificate{cert},
		
		PreferServerCipherSuites: true,
		
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}
}

I can privately sent you all the input that i'm using (server host + root CA + client certificate + private key) to replicate the issue.

[davecheney]

I’m sorry to be pedantic but fmt.Errorf does not log an error, it returns an error value, which this code is discarding.

[adeveloper87]

I updated again the main.go file and i got the same error:

MQTT Connection failed
panic: Network Error : remote error: tls: handshake failure

Below the main.go file used:

package main

import (
	MQTT "github.com/eclipse/paho.mqtt.golang"
	"fmt"
	"time"
	"io/ioutil"
	"crypto/tls"
	"crypto/x509"
)

var (
	brokerUrl = "ssl://<RABBITMQ-HOST>:8883"
)

func main() {
	opts := MQTT.NewClientOptions()
	opts.SetClientID("aa6d566f-46ea-4ada-b4ef-07b18766429b")
	opts.AddBroker(brokerUrl)	
	opts.SetPingTimeout(1 * time.Second)
	opts.SetAutoReconnect(true)
	opts.SetCleanSession(true)
	opts.SetKeepAlive(10 * time.Second)
	opts.SetConnectTimeout(10 * time.Second)
	
	tlsConfig, err := NewTLSConfig()
	if err != nil {
		panic(err.Error())
	}
	
	opts.SetTLSConfig(tlsConfig)

	client := MQTT.NewClient(opts)
	if token := client.Connect(); token.Wait() && token.Error() != nil {
		fmt.Println("MQTT Connection failed")
		panic(token.Error())
	}

	fmt.Println("Client Connected")
}

func NewTLSConfig() (*tls.Config, error) {
	
	// Import trusted certificates from CAfile.pem.
	// Alternatively, manually add CA certificates to
	// default openssl CA bundle.
	certpool, err := x509.SystemCertPool()
	if err != nil {
		fmt.Println("x509.SystemCertPool failed")
		return nil, err
	}

	pemCert, err := ioutil.ReadFile("ca.crt")
	if err != nil {
		fmt.Println("ioutil.ReadFile failed")
		return nil, err
	}
	
	certpool.AppendCertsFromPEM(pemCert)
	
	// Import client certificate/key pair
	cert, err := tls.LoadX509KeyPair("client-cert.crt", "private-key.crt.key")
	if err != nil {
		fmt.Println("tls.LoadX509KeyPair failed")
		return nil, err
	}

	// Just to print out the client certificate...
	cert.Leaf, err = x509.ParseCertificate(cert.Certificate[0])
	if err != nil {
		fmt.Println("x509.ParseCertificate failed")
		return nil, err
	}
	
	// Create tls.Config with desired tls properties
	return &tls.Config{
		
		// RootCAs = certs used to verify server cert.
		RootCAs: certpool,

		// Certificates = list of certs client sends to server.
		Certificates: []tls.Certificate{cert},
		
		PreferServerCipherSuites: true,
		
		CipherSuites: []uint16{
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,
			tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,
			tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384,
		},
	}, nil
}