Add support for custom CAs for TLS validation in clients

internal/certs/pool.go

@@ -0,0 +1,47 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package certs
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"fmt"
+	"io/ioutil"
+)
+
+// SystemPoolWithCACertificate returns a copy of the system pool, including the CA certificate
+// in the given path.
+func SystemPoolWithCACertificate(path string) (*x509.CertPool, error) {
+	pool, err := x509.SystemCertPool()
+	if err != nil {
+		return nil, fmt.Errorf("initializing root certificate pool: %w", err)
+	}
+	err = addCACertificateToPool(pool, path)
+	if err != nil {
+		return nil, err
+	}
+	return pool, nil
+}
+
+func addCACertificateToPool(pool *x509.CertPool, path string) error {
+	d, err := ioutil.ReadFile(path)
+	if err != nil {
+		return fmt.Errorf("failed to read certificate in %q: %w", path, err)
+	}
+
+	cert, _ := pem.Decode(d)
+	if cert == nil || cert.Type != "CERTIFICATE" {
+		return fmt.Errorf("no certificate found in %q", path)
+	}
+
+	ca, err := x509.ParseCertificate(cert.Bytes)
+	if err != nil {
+		return fmt.Errorf("parsing certificate found in %q: %w", path, err)
+	}
+
+	pool.AddCert(ca)
+
+	return nil
+}

internal/certs/pool_test.go

@@ -0,0 +1,58 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package certs
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+const (
+	// caCertPath is the path to a self-signed CA certificate used to sign
+	// the server key and certificates also found here.
+	// They were created with the code in https://github.com/elastic/elastic-package/pull/789.
+	caCertPath     = "testdata/ca-cert.pem"
+	serverCertPath = "testdata/server-cert.pem"
+	serverKeyPath  = "testdata/server-key.pem"
+)
+
+func TestSystemPoolWithCACertificate(t *testing.T) {
+	pool, err := SystemPoolWithCACertificate(caCertPath)
+	require.NoError(t, err)
+
+	verifyTestCertWithPool(t, pool)
+}
+
+func verifyTestCertWithPool(t *testing.T, pool *x509.CertPool) {
+	t.Helper()
+
+	p, err := tls.LoadX509KeyPair(serverCertPath, serverKeyPath)
+	require.NoError(t, err)
+	require.NotEmpty(t, p.Certificate)
+
+	cert, err := x509.ParseCertificate(p.Certificate[0])
+	require.NoError(t, err)
+
+	opts := x509.VerifyOptions{
+		// Test certificates were valid at this time.
+		CurrentTime: time.Date(2022, 06, 10, 0, 0, 0, 0, time.UTC),
+	}
+
+	// Check that verification would fail with current system pool.
+	opts.Roots, err = x509.SystemCertPool()
+	require.NoError(t, err)
+	_, err = cert.Verify(opts)
+	require.Error(t, err, "this certificate is signed by custom authority, it should fail verification")
+
+	// Now do the actual check.
+	opts.Roots = pool
+	_, err = cert.Verify(opts)
+	assert.NoError(t, err)
+}

internal/certs/testdata/ca-cert.pem

@@ -0,0 +1,10 @@
+-----BEGIN CERTIFICATE-----
+MIIBezCCASKgAwIBAgIQcNOQbAM4rLZ3aK0ATKaU0TAKBggqhkjOPQQDAjAdMRsw
+GQYDVQQDExJlbGFzdGljLXBhY2thZ2UgQ0EwIBcNMjIwNjA5MTgxODI2WhgPMjEy
+MjA1MTYxODE4MjZaMB0xGzAZBgNVBAMTEmVsYXN0aWMtcGFja2FnZSBDQTBZMBMG
+ByqGSM49AgEGCCqGSM49AwEHA0IABGP/KTLDkBSM1W8KPSGSpr0wUqcpdDKXx4a0
+wiVqegVU6hyPYMsPR8CUXXf0oISsRR5Rq8YqkEGAcVzSMT1AnhOjQjBAMA4GA1Ud
+DwEB/wQEAwIBpjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBS9TgjrQX9kQ/X6
+/yYhrew8Rhm5LTAKBggqhkjOPQQDAgNHADBEAiA7lsXo6ZDX5HwAeAmqSv1CgwF7
+wUJY+EmXyaCD7U+2cgIgSiORQUxgPhBxWdwGIUISLDactU/tgx/X2YCyn6De9ak=
+-----END CERTIFICATE-----

internal/certs/testdata/ca-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIK0UCOjU3wREwxmEHXsrZ0AeXdUFNlfp2u9kpR8Aqk5joAoGCCqGSM49
+AwEHoUQDQgAEY/8pMsOQFIzVbwo9IZKmvTBSpyl0MpfHhrTCJWp6BVTqHI9gyw9H
+wJRdd/SghKxFHlGrxiqQQYBxXNIxPUCeEw==
+-----END EC PRIVATE KEY-----

internal/certs/testdata/server-cert.pem

@@ -0,0 +1,22 @@
+-----BEGIN CERTIFICATE-----
+MIIBszCCAVqgAwIBAgIRAKFZR9I9ZCao+pAdyewoifIwCgYIKoZIzj0EAwIwHTEb
+MBkGA1UEAxMSZWxhc3RpYy1wYWNrYWdlIENBMCAXDTIyMDYwOTE4MTgyNloYDzIx
+MjIwNTE2MTgxODI2WjAYMRYwFAYDVQQDEw1lbGFzdGljc2VhcmNoMFkwEwYHKoZI
+zj0CAQYIKoZIzj0DAQcDQgAECSF2higEF7YIMgR634bszFRW5Aepv/5sumgB14ob
+h64bVtet045vXghvDNvvv1DoKAasBrJqpaMFJ71EuM6oVaN+MHwwDgYDVR0PAQH/
+BAQDAgWgMAwGA1UdEwEB/wQCMAAwHwYDVR0jBBgwFoAUvU4I60F/ZEP1+v8mIa3s
+PEYZuS0wOwYDVR0RBDQwMoIJbG9jYWxob3N0gg1lbGFzdGljc2VhcmNohwR/AAAB
+hxAAAAAAAAAAAAAAAAAAAAABMAoGCCqGSM49BAMCA0cAMEQCIG4SkDgjeL0PvbjE
+CtmobY+yP6ad7UjhrOD0jGyqzAE/AiBf4d04C++P82oMLtuffhGIRubjlFHKr9Nj
+VGjP3C/KmQ==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIBezCCASKgAwIBAgIQcNOQbAM4rLZ3aK0ATKaU0TAKBggqhkjOPQQDAjAdMRsw
+GQYDVQQDExJlbGFzdGljLXBhY2thZ2UgQ0EwIBcNMjIwNjA5MTgxODI2WhgPMjEy
+MjA1MTYxODE4MjZaMB0xGzAZBgNVBAMTEmVsYXN0aWMtcGFja2FnZSBDQTBZMBMG
+ByqGSM49AgEGCCqGSM49AwEHA0IABGP/KTLDkBSM1W8KPSGSpr0wUqcpdDKXx4a0
+wiVqegVU6hyPYMsPR8CUXXf0oISsRR5Rq8YqkEGAcVzSMT1AnhOjQjBAMA4GA1Ud
+DwEB/wQEAwIBpjAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQWBBS9TgjrQX9kQ/X6
+/yYhrew8Rhm5LTAKBggqhkjOPQQDAgNHADBEAiA7lsXo6ZDX5HwAeAmqSv1CgwF7
+wUJY+EmXyaCD7U+2cgIgSiORQUxgPhBxWdwGIUISLDactU/tgx/X2YCyn6De9ak=
+-----END CERTIFICATE-----

internal/certs/testdata/server-key.pem

@@ -0,0 +1,5 @@
+-----BEGIN EC PRIVATE KEY-----
+MHcCAQEEIMzoSTnIerRjaUPumAtInV/p4LvGe9WOTiJmwtzBYKqdoAoGCCqGSM49
+AwEHoUQDQgAECSF2higEF7YIMgR634bszFRW5Aepv/5sumgB14obh64bVtet045v
+XghvDNvvv1DoKAasBrJqpaMFJ71EuM6oVQ==
+-----END EC PRIVATE KEY-----

internal/elasticsearch/client.go

@@ -6,6 +6,7 @@ package elasticsearch
 
 import (
 	"crypto/tls"
+	"fmt"
 	"net/http"
 	"os"
 
@@ -14,6 +15,7 @@ import (
 	"github.com/elastic/go-elasticsearch/v7"
 	"github.com/elastic/go-elasticsearch/v7/esapi"
 
+	"github.com/elastic/elastic-package/internal/certs"
 	"github.com/elastic/elastic-package/internal/stack"
 )
 
@@ -32,16 +34,20 @@ type clientOptions struct {
 	username string
 	password string
 
+	// certificateAuthority is the certificate to validate the server certificate.
+	certificateAuthority string
+
 	// skipTLSVerify disables TLS validation.
 	skipTLSVerify bool
 }
 
 // defaultOptionsFromEnv returns clientOptions initialized with values from environmet variables.
 func defaultOptionsFromEnv() clientOptions {
 	return clientOptions{
-		address:  os.Getenv(stack.ElasticsearchHostEnv),
-		username: os.Getenv(stack.ElasticsearchUsernameEnv),
-		password: os.Getenv(stack.ElasticsearchPasswordEnv),
+		address:              os.Getenv(stack.ElasticsearchHostEnv),
+		username:             os.Getenv(stack.ElasticsearchUsernameEnv),
+		password:             os.Getenv(stack.ElasticsearchPasswordEnv),
+		certificateAuthority: os.Getenv(stack.CACertificateEnv),
 	}
 }
 
@@ -54,6 +60,13 @@ func OptionWithAddress(address string) ClientOption {
 	}
 }
 
+// OptionWithCertificateAuthority sets the certificate authority to be used by the client.
+func OptionWithCertificateAuthority(certificateAuthority string) ClientOption {
+	return func(opts *clientOptions) {
+		opts.certificateAuthority = certificateAuthority
+	}
+}
+
 // OptionWithSkipTLSVerify disables TLS validation.
 func OptionWithSkipTLSVerify() ClientOption {
 	return func(opts *clientOptions) {
@@ -81,6 +94,14 @@ func Client(customOptions ...ClientOption) (*elasticsearch.Client, error) {
 		config.Transport = &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		}
+	} else if options.certificateAuthority != "" {
+		rootCAs, err := certs.SystemPoolWithCACertificate(options.certificateAuthority)
+		if err != nil {
+			return nil, fmt.Errorf("reading CA certificate: %w", err)
+		}
+		config.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{RootCAs: rootCAs},
+		}
 	}
 
 	client, err := elasticsearch.NewClient(config)

internal/elasticsearch/client_test.go

@@ -0,0 +1,66 @@
+// Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+// or more contributor license agreements. Licensed under the Elastic License;
+// you may not use this file except in compliance with the Elastic License.
+
+package elasticsearch
+
+import (
+	"bytes"
+	"crypto/x509"
+	"encoding/pem"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestClientWithTLS(t *testing.T) {
+	server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("X-elastic-product", "Elasticsearch")
+	}))
+
+	caCertFile := writeCACertFile(t, server.Certificate())
+
+	t.Run("no TLS config, should fail", func(t *testing.T) {
+		client, err := Client(OptionWithAddress(server.URL))
+		require.NoError(t, err)
+
+		_, err = client.Ping()
+		assert.Error(t, err)
+	})
+
+	t.Run("with CA", func(t *testing.T) {
+		client, err := Client(OptionWithAddress(server.URL), OptionWithCertificateAuthority(caCertFile))
+		require.NoError(t, err)
+
+		_, err = client.Ping()
+		assert.NoError(t, err)
+	})
+
+	t.Run("skip TLS verify", func(t *testing.T) {
+		client, err := Client(OptionWithAddress(server.URL), OptionWithSkipTLSVerify())
+		require.NoError(t, err)
+
+		_, err = client.Ping()
+		assert.NoError(t, err)
+	})
+}
+
+func writeCACertFile(t *testing.T, cert *x509.Certificate) string {
+	var d bytes.Buffer
+	err := pem.Encode(&d, &pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: cert.Raw,
+	})
+	require.NoError(t, err)
+
+	caCertFile := filepath.Join(t.TempDir(), "ca.pem")
+	err = ioutil.WriteFile(caCertFile, d.Bytes(), 0644)
+	require.NoError(t, err)
+
+	return caCertFile
+}

internal/kibana/client.go

@@ -7,13 +7,15 @@ package kibana
 import (
 	"bytes"
 	"crypto/tls"
+	"fmt"
 	"io"
 	"net/http"
 	"net/url"
 	"os"
 
 	"github.com/pkg/errors"
 
+	"github.com/elastic/elastic-package/internal/certs"
 	"github.com/elastic/elastic-package/internal/install"
 	"github.com/elastic/elastic-package/internal/logger"
 	"github.com/elastic/elastic-package/internal/stack"
@@ -25,7 +27,8 @@ type Client struct {
 	username string
 	password string
 
-	tlSkipVerify bool
+	certificateAuthority string
+	tlSkipVerify         bool
 }
 
 // ClientOption is functional option modifying Kibana client.
@@ -34,32 +37,49 @@ type ClientOption func(*Client)
 // NewClient creates a new instance of the client.
 func NewClient(opts ...ClientOption) (*Client, error) {
 	host := os.Getenv(stack.KibanaHostEnv)
-	if host == "" {
-		return nil, stack.UndefinedEnvError(stack.KibanaHostEnv)
-	}
-
 	username := os.Getenv(stack.ElasticsearchUsernameEnv)
 	password := os.Getenv(stack.ElasticsearchPasswordEnv)
+	certificateAuthority := os.Getenv(stack.CACertificateEnv)
 
 	c := &Client{
-		host:     host,
-		username: username,
-		password: password,
+		host:                 host,
+		username:             username,
+		password:             password,
+		certificateAuthority: certificateAuthority,
 	}
 
 	for _, opt := range opts {
 		opt(c)
 	}
+
+	if c.host == "" {
+		return nil, stack.UndefinedEnvError(stack.KibanaHostEnv)
+	}
+
 	return c, nil
 }
 
+// Address option sets the host to use to connect to Kibana.
+func Address(address string) ClientOption {
+	return func(c *Client) {
+		c.host = address
+	}
+}
+
 // TLSSkipVerify option disables TLS verification.
 func TLSSkipVerify() ClientOption {
 	return func(c *Client) {
 		c.tlSkipVerify = true
 	}
 }
 
+// CertificateAuthority sets the certificate authority to be used by the client.
+func CertificateAuthority(certificateAuthority string) ClientOption {
+	return func(c *Client) {
+		c.certificateAuthority = certificateAuthority
+	}
+}
+
 func (c *Client) get(resourcePath string) (int, []byte, error) {
 	return c.sendRequest(http.MethodGet, resourcePath, nil)
 }
@@ -106,6 +126,14 @@ func (c *Client) sendRequest(method, resourcePath string, body []byte) (int, []b
 		client.Transport = &http.Transport{
 			TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
 		}
+	} else if c.certificateAuthority != "" {
+		rootCAs, err := certs.SystemPoolWithCACertificate(c.certificateAuthority)
+		if err != nil {
+			return 0, nil, fmt.Errorf("reading CA certificate: %w", err)
+		}
+		client.Transport = &http.Transport{
+			TLSClientConfig: &tls.Config{RootCAs: rootCAs},
+		}
 	}
 
 	resp, err := client.Do(req)