Implement mTLS support in both the mysql backend and probes.

.github/workflows/main.yml

@@ -10,11 +10,10 @@ jobs:
     steps:
     - uses: actions/checkout@master
 
-    - name: Set up Go 1.12
-      uses: actions/setup-go@v1
+    - name: Set up Go
+      uses: actions/setup-go@v2
       with:
-        version: 1.12
-      id: go
+        go-version: 1.15
 
     - name: Build
       run: script/cibuild

doc/mysql-backend.md

@@ -31,6 +31,9 @@ Let's dissect the general section of the [sample config file](../resources/freno
   "BackendMySQLSchema": "freno_backend",
   "BackendMySQLUser": "freno_daemon",
   "BackendMySQLPassword": "123456",
+  "BackendMySQLTlsCaCertPath": "/usr/local/share/certs/ca.crt",
+  "BackendMySQLTlsClientCertPath": "/usr/local/share/certs/client.crt",
+  "BackendMySQLTlsClientKeyPath": "/usr/local/share/certs/client.key",
   "Domain": "us-east-1/production",
   "ShareDomain": "production",
 }
@@ -40,19 +43,24 @@ Let's dissect the general section of the [sample config file](../resources/freno
 - `BackendMySQLPort`: MySQL master port
 - `BackendMySQLSchema`: schema where `freno` will read/write state (see below)
 - `BackendMySQLUser`: user with read+write privileges on backend schema
+- `BackendMySQLTlsCaCertPath´: optional file system path for the PEM-encoded CA certificate to be used when connecting to mysql using TLS
+- `BackendMySQLTlsClientCertPath`: optional file system path for the PEM-encoded client certificate
+- `BackendMySQLTlsClientKeyPath`:  optional file system path for the PEM-encoded client key
 - `BackendMySQLPassword`: password
 - `Domain`: the same MySQL backend can serve multiple, unrelated `freno` clusters. Nodes within the same cluster should have the same `Domain` value and will compete for leadership.
 - `ShareDomain`: it is possible for clusters to collaborate. Clusters with same `ShareDomain` will consul with each other's metric health reports. A cluster may reject a `check` request if another cluster considers the `check` metrics unhealthy.
 
 You may exchange the above for environment variables:
-
 ```json
 {
   "BackendMySQLHost": "${MYSQL_BACKEND_HOST}",
   "BackendMySQLPort": 3306,
   "BackendMySQLSchema": "${MYSQL_BACKEND_SCHEMA}",
   "BackendMySQLUser": "${MYSQL_BACKEND_RW_USER}",
   "BackendMySQLPassword": "${MYSQL_BACKEND_RW_PASSWORD}",
+  "BackendMySQLTlsCaCertPath": "${MYSQL_BACKEND_CA_CERT}",
+  "BackendMySQLTlsClientCertPath": "${MYSQL_BACKEND_CLIENT_CERT}",
+  "BackendMySQLTlsClientKeyPath": "${MYSQL_BACKEND_CLIENT_KEY}",
   "Domain": "us-east-1/production",
   "ShareDomain": "production",
 }

doc/mysql.md

@@ -34,6 +34,9 @@ You will find the top-level configuration:
 "MySQL": {
   "User": "some_user",
   "Password": "${mysql_password_env_variable}",
+  "TlsCaCertPath": "/usr/local/share/certs/ca_freno.pem",
+  "TlsClientCertPath": "${client_cert_path_env_variable}",
+  "TlsClientKeyPath": "${client_key_path_env_variable}",
   "MetricQuery": "select unix_timestamp(now(6)) - unix_timestamp(ts) as lag_check from meta.heartbeat order by ts desc limit 1",
   "CacheMillis": 0,
   "ThrottleThreshold": 1.0,
@@ -53,6 +56,8 @@ You will find the top-level configuration:
 These params apply in general to all MySQL clusters, unless specified differently (overridden) on a per-cluster basis.
 
 - `User`, `Password`: these can be specified as plaintext, or in a `${some_env_variable}` format, in which case `freno` will look up its environment for specified variable. (e.g. to match the above config, a `shell` script invoking `freno` can `export mysql_password_env_variable=flyingcircus`)
+- `TlsCaCertPath`, `TlsClientCertPath`, `TlsClientKeyCertPath`. Are the file system paths of the PEM-encoded, TLS certificates needed to connect to MySQL using TLS. 
+   They can also be specified as text or in `${some_env_variable}` format.  
 - `MetricQuery`:
   - Note: returned value is expected to be `[0..)` (`0` or more), where lower values are "better" and higher values are "worse".
   - if not provided, `freno` will assume you're interested in replication lag, and will issue a `SHOW SLAVE STATUS` to extract `Seconds_behind_master`

pkg/config/config.go

@@ -92,24 +92,29 @@ func (config *Configuration) Reload() error {
 // Some of the settinges have reasonable default values, and some other
 // (like database credentials) are strictly expected from user.
 type ConfigurationSettings struct {
-	ListenPort           int
-	DataCenter           string
-	Environment          string
-	Domain               string
-	ShareDomain          string
-	RaftBind             string
-	RaftDataDir          string
-	DefaultRaftPort      int      // if a RaftNodes entry does not specify port, use this one
-	RaftNodes            []string // Raft nodes to make initial connection with
-	BackendMySQLHost     string
-	BackendMySQLPort     int
-	BackendMySQLSchema   string
-	BackendMySQLUser     string
-	BackendMySQLPassword string
-	MemcacheServers      []string // if given, freno will report to aggregated values to given memcache
-	MemcachePath         string   // use as prefix to metric path in memcache key, e.g. if `MemcachePath` is "myprefix" the key would be "myprefix/mysql/maincluster". Default: "freno"
-	EnableProfiling      bool     // enable pprof profiling http api
-	Stores               StoresSettings
+	ListenPort                    int
+	DataCenter                    string
+	Environment                   string
+	Domain                        string
+	ShareDomain                   string
+	RaftBind                      string
+	RaftDataDir                   string
+	DefaultRaftPort               int      // if a RaftNodes entry does not specify port, use this one
+	RaftNodes                     []string // Raft nodes to make initial connection with
+	BackendMySQLHost              string
+	BackendMySQLPort              int
+	BackendMySQLSchema            string
+	BackendMySQLUser              string
+	BackendMySQLPassword          string
+	BackendMySQLTlsCaCertPath     string // optional, if not provided it won't setup any TLS connection
+	BackendMySQLTlsClientCertPath string
+	BackendMySQLTlsClientKeyPath  string
+	BackendMySQLTlsSkipVerify     bool // optional, set it to true to skip certificate verification. Not recommended in production
+
+	MemcacheServers []string // if given, freno will report to aggregated values to given memcache
+	MemcachePath    string   // use as prefix to metric path in memcache key, e.g. if `MemcachePath` is "myprefix" the key would be "myprefix/mysql/maincluster". Default: "freno"
+	EnableProfiling bool     // enable pprof profiling http api
+	Stores          StoresSettings
 }
 
 func newConfigurationSettings() *ConfigurationSettings {
@@ -145,6 +150,15 @@ func (settings *ConfigurationSettings) postReadAdjustments() error {
 	if submatch := envVariableRegexp.FindStringSubmatch(settings.BackendMySQLPassword); len(submatch) > 1 {
 		settings.BackendMySQLPassword = os.Getenv(submatch[1])
 	}
+	if submatch := envVariableRegexp.FindStringSubmatch(settings.BackendMySQLTlsCaCertPath); len(submatch) > 1 {
+		settings.BackendMySQLTlsCaCertPath = os.Getenv(submatch[1])
+	}
+	if submatch := envVariableRegexp.FindStringSubmatch(settings.BackendMySQLTlsClientCertPath); len(submatch) > 1 {
+		settings.BackendMySQLTlsClientCertPath = os.Getenv(submatch[1])
+	}
+	if submatch := envVariableRegexp.FindStringSubmatch(settings.BackendMySQLTlsClientKeyPath); len(submatch) > 1 {
+		settings.BackendMySQLTlsClientKeyPath = os.Getenv(submatch[1])
+	}
 	if settings.RaftDataDir == "" && settings.BackendMySQLHost == "" {
 		return fmt.Errorf("Either RaftDataDir or BackendMySQLHost must be set")
 	}

pkg/config/mysql_config.go

@@ -11,8 +11,12 @@ import (
 const DefaultMySQLPort = 3306
 
 type MySQLClusterConfigurationSettings struct {
-	User                 string   // override MySQLConfigurationSettings's, or leave empty to inherit those settings
-	Password             string   // override MySQLConfigurationSettings's, or leave empty to inherit those settings
+	User                 string // override MySQLConfigurationSettings's, or leave empty to inherit those settings
+	Password             string // override MySQLConfigurationSettings's, or leave empty to inherit those settings
+	TlsCaCertPath        string // optional, if not provided it won't setup any TLS connection
+	TlsClientCertPath    string
+	TlsClientKeyPath     string
+	TlsSkipVerify        bool     // optional, set it to true to skip certificate verification. Not recommended in production.
 	MetricQuery          string   // override MySQLConfigurationSettings's, or leave empty to inherit those settings
 	CacheMillis          int      // override MySQLConfigurationSettings's, or leave empty to inherit those settings
 	ThrottleThreshold    float64  // override MySQLConfigurationSettings's, or leave empty to inherit those settings
@@ -49,6 +53,10 @@ func (settings *MySQLClusterConfigurationSettings) postReadAdjustments() error {
 type MySQLConfigurationSettings struct {
 	User                 string
 	Password             string
+	TlsCaCertPath        string // optional, if not provided it won't setup any TLS connection
+	TlsClientCertPath    string
+	TlsClientKeyPath     string
+	TlsSkipVerify        bool // optional, set it to true to skip certificate verification. Not recommended in production.
 	MetricQuery          string
 	CacheMillis          int // optional, if defined then probe result will be cached, and future probes may use cached value
 	ThrottleThreshold    float64
@@ -81,6 +89,15 @@ func (settings *MySQLConfigurationSettings) postReadAdjustments() error {
 	if submatch := envVariableRegexp.FindStringSubmatch(settings.Password); len(submatch) > 1 {
 		settings.Password = os.Getenv(submatch[1])
 	}
+	if submatch := envVariableRegexp.FindStringSubmatch(settings.TlsCaCertPath); len(submatch) > 1 {
+		settings.TlsCaCertPath = os.Getenv(submatch[1])
+	}
+	if submatch := envVariableRegexp.FindStringSubmatch(settings.TlsClientCertPath); len(submatch) > 1 {
+		settings.TlsClientCertPath = os.Getenv(submatch[1])
+	}
+	if submatch := envVariableRegexp.FindStringSubmatch(settings.TlsClientKeyPath); len(submatch) > 1 {
+		settings.TlsClientKeyPath = os.Getenv(submatch[1])
+	}
 
 	for _, clusterSettings := range settings.Clusters {
 		if err := clusterSettings.postReadAdjustments(); err != nil {
@@ -92,6 +109,15 @@ func (settings *MySQLConfigurationSettings) postReadAdjustments() error {
 		if clusterSettings.Password == "" {
 			clusterSettings.Password = settings.Password
 		}
+		if clusterSettings.TlsCaCertPath == "" {
+			clusterSettings.TlsCaCertPath = settings.TlsCaCertPath
+		}
+		if clusterSettings.TlsClientCertPath == "" {
+			clusterSettings.TlsClientCertPath = settings.TlsClientCertPath
+		}
+		if clusterSettings.TlsClientKeyPath == "" {
+			clusterSettings.TlsClientKeyPath = settings.TlsClientKeyPath
+		}
 		if clusterSettings.MetricQuery == "" {
 			clusterSettings.MetricQuery = settings.MetricQuery
 		}

pkg/group/mysql.go

@@ -24,7 +24,7 @@ service_id varchar(128) NOT NULL,
 CREATE TABLE throttled_apps (
   app_name varchar(128) NOT NULL,
 	throttled_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
-  expires_at TIMESTAMP NOT NULL,
+  	expires_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
 	ratio DOUBLE,
   PRIMARY KEY (app_name)
 );
@@ -40,6 +40,8 @@ import (
 	"sync/atomic"
 	"time"
 
+	"github.com/github/freno/pkg/mysql"
+
 	"github.com/github/freno/pkg/base"
 	"github.com/github/freno/pkg/config"
 	"github.com/github/freno/pkg/throttle"
@@ -59,27 +61,46 @@ type MySQLBackend struct {
 	throttler   *throttle.Throttler
 }
 
-const maxConnections = 3
-const electionExpireSeconds = 5
+const (
+	maxConnections        = 3
+	electionExpireSeconds = 5
+
+	electionInterval = time.Second
+	healthInterval   = 2 * electionInterval
+	stateInterval    = 10 * time.Second
 
-const electionInterval = time.Second
-const healthInterval = 2 * electionInterval
-const stateInterval = 10 * time.Second
+	connectionTimeout = 500 * time.Millisecond
+)
 
 func NewMySQLBackend(throttler *throttle.Throttler) (*MySQLBackend, error) {
-	if config.Settings().BackendMySQLHost == "" {
+	settings := config.Settings()
+	if settings.BackendMySQLHost == "" {
 		return nil, nil
 	}
-	dbUri := fmt.Sprintf("%s:%s@tcp(%s:%d)/%s?interpolateParams=true&charset=utf8mb4,utf8,latin1&timeout=500ms",
-		config.Settings().BackendMySQLUser, config.Settings().BackendMySQLPassword, config.Settings().BackendMySQLHost, config.Settings().BackendMySQLPort, config.Settings().BackendMySQLSchema,
-	)
-	db, _, err := sqlutils.GetDB(dbUri)
+	uri, err := mysql.MakeUri(
+		settings.BackendMySQLHost,
+		settings.BackendMySQLPort,
+		settings.BackendMySQLSchema,
+		settings.BackendMySQLUser,
+		settings.BackendMySQLPassword,
+		settings.BackendMySQLTlsCaCertPath,
+		settings.BackendMySQLTlsClientCertPath,
+		settings.BackendMySQLTlsClientKeyPath,
+		settings.BackendMySQLTlsSkipVerify,
+		connectionTimeout)
+
 	if err != nil {
 		return nil, err
 	}
+
+	db, _, err := sqlutils.GetDB(uri)
+	if err != nil {
+		return nil, err
+	}
+
 	db.SetMaxOpenConns(maxConnections)
 	db.SetMaxIdleConns(maxConnections)
-	log.Debugf("created db at: %s", dbUri)
+	log.Debugf("created db at: %s", uri)
 	hostname, err := os.Hostname()
 	if err != nil {
 		return nil, err
@@ -368,7 +389,7 @@ func (backend *MySQLBackend) ThrottleApp(appName string, ttlMinutes int64, expir
 			on duplicate key update
 				ratio=values(ratio)
 		`
-		args = sqlutils.Args(appName, throttle.DefaultThrottleTTLMinutes, ratio)
+		args = sqlutils.Args(appName, throttle.DefaultThrottleTTL.Minutes(), ratio)
 	}
 	_, err := sqlutils.ExecNoPrepare(backend.db, query, args...)
 	backend.throttler.ThrottleApp(appName, expireAt, ratio)

pkg/mysql/instance_key.go

@@ -15,7 +15,7 @@ const (
 	DefaultMySQLPort = 3306
 )
 
-// InstanceKey is an instance indicator, identifued by hostname and port
+// InstanceKey is an instance indicator, identified by hostname and port
 type InstanceKey struct {
 	Hostname string
 	Port     int
@@ -46,14 +46,6 @@ func ParseInstanceKey(hostPort string, defaultPort int) (*InstanceKey, error) {
 	return newRawInstanceKey(hostPort)
 }
 
-// Equals tests equality between this key and another key
-func (this *InstanceKey) Equals(other *InstanceKey) bool {
-	if other == nil {
-		return false
-	}
-	return this.Hostname == other.Hostname && this.Port == other.Port
-}
-
 // SmallerThan returns true if this key is dictionary-smaller than another.
 // This is used for consistent sorting/ordering; there's nothing magical about it.
 func (this *InstanceKey) SmallerThan(other *InstanceKey) bool {
@@ -79,11 +71,6 @@ func (this *InstanceKey) StringCode() string {
 	return fmt.Sprintf("%s:%d", this.Hostname, this.Port)
 }
 
-// DisplayString returns a user-friendly string representation of this key
-func (this *InstanceKey) DisplayString() string {
-	return this.StringCode()
-}
-
 // String returns a user-friendly string representation of this key
 func (this InstanceKey) String() string {
 	return this.StringCode()

pkg/mysql/instance_key_test.go

@@ -52,15 +52,6 @@ func TestParseInstanceKey(t *testing.T) {
 	}
 }
 
-func TestEquals(t *testing.T) {
-	{
-		expect := &InstanceKey{Hostname: "127.0.0.1", Port: 3306}
-		key, err := ParseInstanceKey("127.0.0.1", 3306)
-		test.S(t).ExpectNil(err)
-		test.S(t).ExpectTrue(key.Equals(expect))
-	}
-}
-
 func TestStringCode(t *testing.T) {
 	{
 		key := &InstanceKey{Hostname: "127.0.0.1", Port: 3306}

pkg/mysql/mysql_throttle_metric.go

@@ -88,9 +88,7 @@ func ReadThrottleMetric(probe *Probe, clusterName string) (mySQLThrottleMetric *
 		}()
 	}(mySQLThrottleMetric, started)
 
-	dbUri := probe.GetDBUri("information_schema")
-	db, fromCache, err := sqlutils.GetDB(dbUri)
-
+	db, fromCache, err := sqlutils.GetDB(probe.Uri)
 	if err != nil {
 		mySQLThrottleMetric.Err = err
 		return mySQLThrottleMetric