Start docs/

aws/aws.go

@@ -22,22 +22,31 @@ func NewConfigFactory() cfgFactory {
 }
 
 func (f cfgFactory) Make(ba blip.AWS) (aws.Config, error) {
-	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
-	defer cancel()
 	if ba.Region == "auto" {
-		ba.Region = Region(ctx)
-		blip.Debug("auto-detect region %s", ba.Region)
+		ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)
+		defer cancel()
+		var err error
+		ba.Region, err = Region(ctx)
+		if err != nil {
+			return aws.Config{}, err
+		}
 	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+	defer cancel()
 	return config.LoadDefaultConfig(ctx, config.WithRegion(ba.Region))
 }
 
 // Region auto-detects the region. Currently, the function relies on IMDS v2:
 // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html
 // If the region cannot be detect, it returns an empty string.
-func Region(ctx context.Context) string {
+func Region(ctx context.Context) (string, error) {
 	client := imds.New(imds.Options{})
-	ec2, _ := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
-	return ec2.Region
+	ec2, err := client.GetInstanceIdentityDocument(ctx, &imds.GetInstanceIdentityDocumentInput{})
+	if err != nil {
+		return "", err
+	}
+	return ec2.Region, nil
 }
 
 var once sync.Once

aws/rds.go

@@ -48,9 +48,10 @@ type RDSLoader struct {
 func (rl RDSLoader) Load(ctx context.Context, cfg blip.Config) ([]blip.ConfigMonitor, error) {
 	loaderCfg := cfg.MonitorLoader.AWS
 	if len(loaderCfg.Regions) == 0 {
-		if blip.Strict {
+		if loaderCfg.DisableAuto {
 			return nil, nil
 		}
+		blip.Debug("auto-detect AWS region")
 		loaderCfg.Regions = []string{"auto"}
 	}
 

aws/rds_test.go

@@ -20,18 +20,29 @@ func TestRDSClient(t *testing.T) {
 		Out: rds.DescribeDBInstancesOutput{
 			DBInstances: []types.DBInstance{
 				{
+					// Need ALL these fields else a debug statement will panic
 					DBInstanceIdentifier: aws.String("rds1"),
+					DBClusterIdentifier:  aws.String("rds-001"),
 					Endpoint: &types.Endpoint{
 						Address: aws.String("rds1"),
 						Port:    3306,
 					},
+
+					EngineVersion:    aws.String("v8.0.0"),
+					AvailabilityZone: aws.String("us-west-2a"),
+					DBInstanceStatus: aws.String("fantastic"),
 				},
 				{
 					DBInstanceIdentifier: aws.String("rds2"),
+					DBClusterIdentifier:  aws.String("rds-001"),
 					Endpoint: &types.Endpoint{
 						Address: aws.String("rds2"),
 						Port:    3307,
 					},
+
+					EngineVersion:    aws.String("v8.0.0"),
+					AvailabilityZone: aws.String("us-west-2a"),
+					DBInstanceStatus: aws.String("fantastic"),
 				},
 			},
 		},

config.go

@@ -211,7 +211,12 @@ type ConfigMonitorLoader struct {
 }
 
 type ConfigMonitorLoaderAWS struct {
-	Regions []string `yaml:"regions,omitempty"`
+	Regions     []string `yaml:"regions,omitempty"`
+	DisableAuto bool     `yaml:"disable-auto"`
+}
+
+func (c ConfigMonitorLoaderAWS) Automatic() bool {
+	return len(c.Regions) == 0 && c.DisableAuto == false
 }
 
 type ConfigMonitorLoaderLocal struct {
@@ -263,6 +268,8 @@ type ConfigMonitor struct {
 	Plans     ConfigPlans            `yaml:"plans,omitempty"`
 	Sinks     ConfigSinks            `yaml:"sinks,omitempty"`
 	TLS       ConfigTLS              `yaml:"tls,omitempty"`
+
+	Meta map[string]string `yaml:"meta,omitempty"`
 }
 
 func DefaultConfigMonitor() ConfigMonitor {
@@ -339,6 +346,9 @@ func (c *ConfigMonitor) InterpolateEnvVars() {
 	for k, v := range c.Tags {
 		c.Tags[k] = interpolateEnv(v)
 	}
+	for k, v := range c.Meta {
+		c.Meta[k] = interpolateEnv(v)
+	}
 	c.AWS.InterpolateEnvVars()
 	c.Exporter.InterpolateEnvVars()
 	c.HA.InterpolateEnvVars()
@@ -357,7 +367,12 @@ func (c *ConfigMonitor) InterpolateMonitor() {
 	c.Password = c.interpolateMon(c.Password)
 	c.PasswordFile = c.interpolateMon(c.PasswordFile)
 	c.TimeoutConnect = c.interpolateMon(c.TimeoutConnect)
-
+	for k, v := range c.Tags {
+		c.Tags[k] = c.interpolateMon(v)
+	}
+	for k, v := range c.Meta {
+		c.Meta[k] = c.interpolateMon(v)
+	}
 	c.AWS.InterpolateMonitor(c)
 	c.Exporter.InterpolateMonitor(c)
 	c.HA.InterpolateMonitor(c)
@@ -367,7 +382,7 @@ func (c *ConfigMonitor) InterpolateMonitor() {
 	c.TLS.InterpolateMonitor(c)
 }
 
-var monvar = regexp.MustCompile(`%{([\w_-]+)\.([\w_-]+)}`)
+var monvar = regexp.MustCompile(`%{([\w_-]+)\.([\w_.-]+)}`)
 
 func (c *ConfigMonitor) interpolateMon(v string) string {
 	if !strings.Contains(v, "%{monitor.") {
@@ -377,6 +392,20 @@ func (c *ConfigMonitor) interpolateMon(v string) string {
 	if len(m) != 3 {
 		// @todo error
 	}
+	if strings.HasPrefix(m[2], "tags.") {
+		if c.Tags == nil {
+			return ""
+		}
+		s := strings.SplitN(m[2], ".", 2)
+		return c.Tags[s[1]]
+	} else if strings.HasPrefix(m[2], "meta.") {
+		if c.Meta == nil {
+			return ""
+		}
+		s := strings.SplitN(m[2], ".", 2)
+		return c.Meta[s[1]]
+	}
+
 	return monvar.ReplaceAllString(v, c.fieldValue(m[2]))
 }
 
@@ -394,9 +423,9 @@ func (c *ConfigMonitor) fieldValue(f string) string {
 		return c.Username
 	case "password":
 		return c.Password
-	case "passwordfile", "password-file":
+	case "password-file":
 		return c.PasswordFile
-	case "timeoutconnect", "timeout-connect":
+	case "timeout-connect":
 		return c.TimeoutConnect
 	default:
 		return ""

dbconn/factory.go

@@ -38,10 +38,8 @@ func NewConnFactory(awsConfg blip.AWSConfigFactory, modifyDB func(*sql.DB, strin
 // copmlete: defaults, env var, and monitor var interpolations already applied,
 // which is done by the monitor.Loader in its private merge method.
 func (f factory) Make(cfg blip.ConfigMonitor) (*sql.DB, string, error) {
-	passwordFunc, err := f.Password(cfg)
-	if err != nil {
-		return nil, "", err
-	}
+	// ----------------------------------------------------------------------
+	// my.cnf
 
 	// Set values in cfg blip.ConfigMonitor from values in my.cnf. This does
 	// not overwrite any values in cfg already set. For exmaple, if username
@@ -59,7 +57,10 @@ func (f factory) Make(cfg blip.ConfigMonitor) (*sql.DB, string, error) {
 		}
 		cfg.ApplyDefaults(blip.Config{MySQL: def, TLS: tls})
 	}
-	blip.Debug("<< %+v", cfg)
+
+	// ----------------------------------------------------------------------
+	// TCP or Unix socket
+
 	net := ""
 	addr := ""
 	if cfg.Socket != "" {
@@ -70,69 +71,126 @@ func (f factory) Make(cfg blip.ConfigMonitor) (*sql.DB, string, error) {
 		addr = cfg.Hostname
 	}
 
+	// ----------------------------------------------------------------------
+	// Pasword reload func
+
+	// Blip presumes that passwords are rotated for security. So we create
+	// a callback that relaods the password based on its method: static, file,
+	// Amazon IAM auth token, etc. The special mysql-hotswap-dsn driver (below)
+	// calls this func when MySQL returns an authentication error.
+	passwordFunc, err := f.Password(cfg)
+	if err != nil {
+		return nil, "", err
+	}
+
+	// Test the password reload func, i.e. get the current password, which
+	// might just be a static password in the Blip config file or another file,
+	// but it could be something dynamic like an Amazon IAM auth token.
 	ctx, cancel := context.WithTimeout(context.Background(), 500*time.Millisecond)
 	defer cancel()
 	password, err := passwordFunc(ctx)
 	if err != nil {
 		return nil, "", err
 	}
 
+	// Credentials are username:password--part of the DSN created below
 	cred := cfg.Username
 	if password != "" {
 		cred += ":" + password
 	}
 
+	// ----------------------------------------------------------------------
+	// DSN params (including TLS)
+
 	params := []string{"parseTime=true"}
-	if (blip.True(cfg.AWS.AuthToken) || cfg.AWS.PasswordSecret != "") && !blip.True(cfg.AWS.DisableAutoTLS) {
+
+	// Load and register TLS, if any
+	tlsConfig, err := cfg.TLS.LoadTLS()
+	if tlsConfig != nil && err != nil {
+		mysql.RegisterTLSConfig(cfg.MonitorId, tlsConfig)
+		params = append(params, "tls="+cfg.MonitorId)
+		blip.Debug("TLS enabled for %s", cfg.MonitorId)
+	}
+
+	// Use built-in Amazon RDS CA if password is AWS IAM auth or Secrets Manager
+	// and auto-TLS is still enabled (default) and user didn't provide an explicit
+	// TLS config (above). This latter is really forward-looking: Amazon rotates
+	// its certs, so eventually the Blip built-in will be out of date. But user
+	// will never be blocked (waiting for a new Blip release) because they can
+	// override the built-in Amazon cert.
+	if (blip.True(cfg.AWS.AuthToken) || cfg.AWS.PasswordSecret != "") &&
+		!blip.True(cfg.AWS.DisableAutoTLS) &&
+		tlsConfig == nil {
+		aws.RegisterRDSCA() // safe to call multiple times
 		params = append(params, "tls=rds")
 	}
+
+	// IAM auto requires cleartext passwords (the auth token is already encrypted)
 	if blip.True(cfg.AWS.AuthToken) {
 		params = append(params, "allowCleartextPasswords=true")
 	}
 
-	if cfg.TLS.Cert != "" && cfg.TLS.Key != "" {
-		// @todo
-	}
+	// ----------------------------------------------------------------------
+	// Create DSN and *sql.DB
 
 	dsn := fmt.Sprintf("%s@%s(%s)/", cred, net, addr)
 	if len(params) > 0 {
 		dsn += "?" + strings.Join(params, "&")
 	}
 
+	// mysql-hotswap-dsn is a special driver; see reload_password.go.
+	// Remember: this does NOT connect to MySQL; it only creates a valid
+	// *sql.DB connection pool. Since the caller is Monitor.Run (indirectly
+	// via the blip.DbFactory it was given), actually connecting to MySQL
+	// happens (probably) by monitor/Engine.Prepare, or possibly by other
+	// components (plan loader, LPA, heartbeat, etc.)
 	db, err := sql.Open("mysql-hotswap-dsn", dsn)
 	if err != nil {
 		return nil, "", err
 	}
 
-	// Valid db/DSN, do not return error past here --------------------------
+	// ======================================================================
+	// Valid db/DSN, do not return error past here
+	// ======================================================================
 
+	// Now that we know the DSN/DB are valid, registry the password reload func.
+	// Don't do this earlier becuase there's no way to unregister it, which is
+	// probably a bug/leak if/when Blip allows dyanmically unloading monitors.
 	Repo.Add(addr, passwordFunc)
 
+	// Limit Blip to 3 MySQL conn by default: 1 or 2 for metrics, and 1 for
+	// LPA, heartbeat, etc. Since all metrics are supposed to collect in a
+	// matter of milliseconds, 3 should be more than enough.
 	db.SetMaxOpenConns(3)
 	db.SetMaxIdleConns(3)
 
+	// Let user-provided plugin set/change DB
 	if f.modifyDB != nil {
 		f.modifyDB(db, dsn)
 	}
 
-	dsncfg, err := mysql.ParseDSN(dsn)
+	// Parse DSN only so we can s/password/.../ to return a print-safe DSN string
+	// for info and debugging. This shouldn't erorr because we know from above
+	// that DSN is valid, which is why we can ignore the error.
+	redactedPassword, err := mysql.ParseDSN(dsn)
 	if err != nil { // ok to ignore
 		blip.Debug("mysql.ParseDSN error: %s", err)
 	}
-	if dsncfg.Passwd != "" {
-		dsncfg.Passwd = "..."
-	}
+	redactedPassword.Passwd = "..."
 
-	return db, dsncfg.FormatDSN(), nil
+	return db, redactedPassword.FormatDSN(), nil
 }
 
+// Password creates a password reload function (callback) based on the
+// configured password method. This function is used by the mysql-hotswap-dsn
+// driver (see reload_password.go). For a consistent abstraction, all
+// passwords are fetched via a reload func, even a static password specified
+// in the Blip config file.
 func (f factory) Password(cfg blip.ConfigMonitor) (PasswordFunc, error) {
+
+	// Amazon IAM auth token (valid 15 min)
 	if blip.True(cfg.AWS.AuthToken) {
-		// Password generated as IAM auth token (valid 15 min)
-		blip.Debug("password from AWS IAM auth token")
-		if !blip.True(cfg.AWS.DisableAutoTLS) {
-			aws.RegisterRDSCA()
-		}
+		blip.Debug("%s: AWS IAM auth token password", cfg.MonitorId)
 		awscfg, err := f.awsConfg.Make(blip.AWS{Region: cfg.AWS.Region})
 		if err != nil {
 			return nil, err
@@ -141,11 +199,9 @@ func (f factory) Password(cfg blip.ConfigMonitor) (PasswordFunc, error) {
 		return token.Password, nil
 	}
 
+	// Amazon Secrets Manager, could be rotated
 	if cfg.AWS.PasswordSecret != "" {
-		blip.Debug("password from AWS Secrets Manager")
-		if !blip.True(cfg.AWS.DisableAutoTLS) {
-			aws.RegisterRDSCA()
-		}
+		blip.Debug("%s: AWS Secrets Manager password", cfg.MonitorId)
 		awscfg, err := f.awsConfg.Make(blip.AWS{Region: cfg.AWS.Region})
 		if err != nil {
 			return nil, err
@@ -154,8 +210,9 @@ func (f factory) Password(cfg blip.ConfigMonitor) (PasswordFunc, error) {
 		return secret.Password, nil
 	}
 
+	// Password file, could be "rotated" (new password written to file)
 	if cfg.PasswordFile != "" {
-		blip.Debug("password from file %s", cfg.PasswordFile)
+		blip.Debug("%s: password file", cfg.MonitorId)
 		return func(context.Context) (string, error) {
 			bytes, err := ioutil.ReadFile(cfg.PasswordFile)
 			if err != nil {
@@ -165,13 +222,9 @@ func (f factory) Password(cfg blip.ConfigMonitor) (PasswordFunc, error) {
 		}, nil
 	}
 
-	if cfg.Password != "" {
-		blip.Debug("password from config")
-		return func(context.Context) (string, error) { return cfg.Password, nil }, nil
-	}
-
+	// Static password in my.cnf file, could be rotated (like password file)
 	if cfg.MyCnf != "" {
-		blip.Debug("password from my.cnf %s", cfg.MyCnf)
+		blip.Debug("%s my.cnf password", cfg.MonitorId)
 		return func(context.Context) (string, error) {
 			cfg, err := ParseMyCnf(cfg.MyCnf)
 			if err != nil {
@@ -181,12 +234,14 @@ func (f factory) Password(cfg blip.ConfigMonitor) (PasswordFunc, error) {
 		}, nil
 	}
 
-	if !blip.Strict {
-		blip.Debug("password blank")
-		return func(context.Context) (string, error) { return "", nil }, nil
+	// Static password in Blip config file, not rotated
+	if cfg.Password != "" {
+		blip.Debug("%s: static password", cfg.MonitorId)
+		return func(context.Context) (string, error) { return cfg.Password, nil }, nil
 	}
 
-	return nil, fmt.Errorf("no password")
+	blip.Debug("%s: no password", cfg.MonitorId)
+	return func(context.Context) (string, error) { return "", nil }, nil
 }
 
 // --------------------------------------------------------------------------