Enhancements (#43)

* Enhancements

- Add mutual auth support for HTTPS between Prometheus and Exporter
- Add new metrics,
  - dup_res_ask
  - dup_res_respond_read
  - dup_res_respond_no_read
  - xdr_bin_cemeteries
  - conflict-resolve-writes
- PROD-1660 - Wait for network to be up, when starting under systemd
- Update dependencies and Go version

README.md

@@ -245,13 +245,33 @@ Packages will be generated under `./pkg/target/` directory.
 - To enable basic HTTP authentication and/or enable HTTPS between the Prometheus server and the exporter, use the below configurations keys,
     ```toml
     [Agent]
+    # Exporter HTTPS (TLS) configuration
+    # HTTPS between Prometheus and Exporter
 
-    # File paths should be double quoted.
-    # Certificate file for the metric servers for prometheus
+    # TLS certificates.
+    # Supports below formats,
+    # 1. Certificate file path                                      - "file:<file-path>"
+    # 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+    # 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+    # Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
+
+    # Server certificate
     cert_file = ""
-    # Key file for the metric servers for prometheus
+
+    # Private key associated with server certificate
     key_file = ""
 
+    # Root CA to validate client certificates (for mutual TLS)
+    root_ca = ""
+
+    # Passphrase for encrypted key_file. Supports below formats,
+    # 1. Passphrase directly                                                      - "<passphrase>"
+    # 2. Passphrase via file                                                      - "file:<file-that-contains-passphrase>"
+    # 3. Passphrase via environment variable                                      - "env:<environment-variable-that-holds-passphrase>"
+    # 4. Passphrase via environment variable containing base64 encoded passphrase - "env-b64:<environment-variable-that-contains-base64-encoded-passphrase>"
+    # 5. Passphrase in base64 encoded form                                        - "b64:<base64-encoded-passphrase>"
+    key_file_passphrase = ""
+
     # Basic HTTP authentication for '/metrics'.
     # Supports below formats,
     # 1. Credential directly                                                      - "<credential>"

ape.toml

@@ -1,10 +1,30 @@
 [Agent]
-# File paths should be double quoted.
-# Certificate file for the metric servers for prometheus
-#cert_file = "cert.pem"
+# Exporter HTTPS (TLS) configuration
+# HTTPS between Prometheus and Exporter
 
-# Key file for the metric servers for prometheus
-#key_file = "key.pem"
+# TLS certificates.
+# Supports below formats,
+# 1. Certificate file path                                      - "file:<file-path>"
+# 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+# 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+# Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
+
+# Server certificate
+cert_file = ""
+
+# Private key associated with server certificate
+key_file = ""
+
+# Root CA to validate client certificates (for mutual TLS)
+root_ca = ""
+
+# Passphrase for encrypted key_file. Supports below formats,
+# 1. Passphrase directly                                                      - "<passphrase>"
+# 2. Passphrase via file                                                      - "file:<file-that-contains-passphrase>"
+# 3. Passphrase via environment variable                                      - "env:<environment-variable-that-holds-passphrase>"
+# 4. Passphrase via environment variable containing base64 encoded passphrase - "env-b64:<environment-variable-that-contains-base64-encoded-passphrase>"
+# 5. Passphrase in base64 encoded form                                        - "b64:<base64-encoded-passphrase>"
+key_file_passphrase = ""
 
 # labels to add to the prometheus metrics for e.g. labels={zone="asia-south1-a", platform="google compute engine"}
 labels={}
@@ -36,9 +56,9 @@ db_port=3000
 
 # TLS certificates.
 # Supports below formats,
-# 2. Certificate file path                                      - "file:<file-path>"
-# 3. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
-# 4. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+# 1. Certificate file path                                      - "file:<file-path>"
+# 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+# 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
 # Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
 
 # root certificate file

ape.toml.template

@@ -1,11 +1,31 @@
 [Agent]
-# File paths should be double quoted.
-# Certificate file for the metric servers for prometheus
+# Exporter HTTPS (TLS) configuration
+# HTTPS between Prometheus and Exporter
+
+# TLS certificates.
+# Supports below formats,
+# 1. Certificate file path                                      - "file:<file-path>"
+# 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+# 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+# Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
+
+# Server certificate
 cert_file = "${AGENT_CERT_FILE}"
 
-# Key file for the metric servers for prometheus
+# Private key associated with server certificate
 key_file = "${AGENT_KEY_FILE}"
 
+# Root CA to validate client certificates (for mutual TLS)
+root_ca = "${AGENT_ROOT_CA}"
+
+# Passphrase for encrypted key_file. Supports below formats,
+# 1. Passphrase directly                                                      - "<passphrase>"
+# 2. Passphrase via file                                                      - "file:<file-that-contains-passphrase>"
+# 3. Passphrase via environment variable                                      - "env:<environment-variable-that-holds-passphrase>"
+# 4. Passphrase via environment variable containing base64 encoded passphrase - "env-b64:<environment-variable-that-contains-base64-encoded-passphrase>"
+# 5. Passphrase in base64 encoded form                                        - "b64:<base64-encoded-passphrase>"
+key_file_passphrase = "${AGENT_KEY_FILE_PASSPHRASE}"
+
 # labels to add to the prometheus metrics for e.g. labels={zone="asia-south1-a", platform="google compute engine"}
 labels = {${METRIC_LABELS}}
 
@@ -36,9 +56,9 @@ db_port=${AS_PORT}
 
 # TLS certificates.
 # Supports below formats,
-# 2. Certificate file path                                      - "file:<file-path>"
-# 3. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
-# 4. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
+# 1. Certificate file path                                      - "file:<file-path>"
+# 2. Environment variable containing base64 encoded certificate - "env-b64:<environment-variable-that-contains-base64-encoded-certificate>"
+# 3. Base64 encoded certificate                                 - "b64:<base64-encoded-certificate>"
 # Applicable to 'root_ca', 'cert_file' and 'key_file' configurations.
 
 # root certificate file

common.go

@@ -3,7 +3,10 @@ package main
 import (
 	"bytes"
 	"crypto/subtle"
+	"crypto/tls"
+	"crypto/x509"
 	"encoding/base64"
+	"encoding/pem"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -269,6 +272,86 @@ func getValueFromBase64(b64Value string) ([]byte, error) {
 	return value, nil
 }
 
+// loadCACert returns CA set of certificates (cert pool)
+// reads CA certificate based on the certConfig and adds it to the pool
+func loadCACert(certConfig string) (*x509.CertPool, error) {
+	certificates, err := x509.SystemCertPool()
+	if certificates == nil || err != nil {
+		certificates = x509.NewCertPool()
+	}
+
+	if len(certConfig) > 0 {
+		caCert, err := getCertificate(certConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		certificates.AppendCertsFromPEM(caCert)
+	}
+
+	return certificates, nil
+}
+
+// loadServerCertAndKey reads server certificate and associated key file based on certConfig and keyConfig
+// returns parsed server certificate
+// if the private key is encrypted, it will be decrypted using key file passphrase
+func loadServerCertAndKey(certConfig, keyConfig, keyPassConfig string) ([]tls.Certificate, error) {
+	var certificates []tls.Certificate
+
+	// Read cert file
+	certFileBytes, err := getCertificate(certConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read key file
+	keyFileBytes, err := getCertificate(keyConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	// Decode PEM data
+	keyBlock, _ := pem.Decode(keyFileBytes)
+	certBlock, _ := pem.Decode(certFileBytes)
+
+	if keyBlock == nil || certBlock == nil {
+		return nil, fmt.Errorf("Failed to decode PEM data for key or certificate")
+	}
+
+	// Check and Decrypt the the Key Block using passphrase
+	if x509.IsEncryptedPEMBlock(keyBlock) {
+		keyFilePassphraseBytes, err := getSecret(keyPassConfig)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to get key passphrase: `%s`", err)
+		}
+
+		decryptedDERBytes, err := x509.DecryptPEMBlock(keyBlock, keyFilePassphraseBytes)
+		if err != nil {
+			return nil, fmt.Errorf("Failed to decrypt PEM Block: `%s`", err)
+		}
+
+		keyBlock.Bytes = decryptedDERBytes
+		keyBlock.Headers = nil
+	}
+
+	// Encode PEM data
+	keyPEM := pem.EncodeToMemory(keyBlock)
+	certPEM := pem.EncodeToMemory(certBlock)
+
+	if keyPEM == nil || certPEM == nil {
+		return nil, fmt.Errorf("Failed to encode PEM data for key or certificate")
+	}
+
+	cert, err := tls.X509KeyPair(certPEM, keyPEM)
+	if err != nil {
+		return nil, fmt.Errorf("Failed to add certificate and key to the pool: `%s`", err)
+	}
+
+	certificates = append(certificates, cert)
+
+	return certificates, nil
+}
+
 func sanitizeUTF8(lv string) string {
 	if utf8.ValidString(lv) {
 		return lv

config.go

@@ -14,8 +14,10 @@ import (
 // Config represents the aerospike-prometheus-exporter configuration
 type Config struct {
 	AeroProm struct {
-		CertFile string `toml:"cert_file"`
-		KeyFile  string `toml:"key_file"`
+		CertFile          string `toml:"cert_file"`
+		KeyFile           string `toml:"key_file"`
+		RootCA            string `toml:"root_ca"`
+		KeyFilePassphrase string `toml:"key_file_passphrase"`
 
 		MetricLabels map[string]string `toml:"labels"`
 

go.mod

@@ -1,20 +1,20 @@
 module github.com/aerospike/aerospike-prometheus-exporter
 
-go 1.14
+go 1.16
 
 require (
 	github.com/BurntSushi/toml v0.3.1
-	github.com/aerospike/aerospike-client-go v4.1.0+incompatible
+	github.com/aerospike/aerospike-client-go v4.4.0+incompatible
 	github.com/gobwas/glob v0.2.3
-	github.com/hashicorp/go-version v1.2.1
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/hashicorp/go-version v1.3.0
 	github.com/jameskeane/bcrypt v0.0.0-20120420032655-c3cd44c1e20f
 	github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect
 	github.com/onsi/ginkgo v1.14.0 // indirect
-	github.com/prometheus/client_golang v1.9.0
-	github.com/prometheus/procfs v0.3.0 // indirect
-	github.com/sirupsen/logrus v1.7.0
+	github.com/prometheus/client_golang v1.10.0
+	github.com/prometheus/common v0.20.0 // indirect
+	github.com/sirupsen/logrus v1.8.1
 	github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect
-	golang.org/x/sync v0.0.0-20201207232520-09787c993a3a // indirect
-	golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c // indirect
-	google.golang.org/protobuf v1.25.0 // indirect
+	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
 )