From 0cadce1e3ac6e223a439ab3d3032367f529bc91c Mon Sep 17 00:00:00 2001 From: Ian Oberst Date: Wed, 3 Jun 2026 15:17:12 -0700 Subject: [PATCH 1/3] Add AWS password secret parser plugin --- aws/secret.go | 40 ++++++--- blip.go | 53 +++++++++++ dbconn/factory.go | 84 +++++++++++------- dbconn/password_secret_test.go | 113 ++++++++++++++++++++++++ docs/content/cloud/aws.md | 15 +++- docs/content/config/config-file.md | 4 +- docs/content/develop/integration-api.md | 15 ++++ secret_test.go | 89 +++++++++++++++++++ server/server.go | 6 +- 9 files changed, 370 insertions(+), 49 deletions(-) create mode 100644 dbconn/password_secret_test.go create mode 100644 secret_test.go diff --git a/aws/secret.go b/aws/secret.go index b4927c72..c7b3eb78 100644 --- a/aws/secret.go +++ b/aws/secret.go @@ -26,6 +26,22 @@ func NewSecret(name string, cfg aws.Config) Secret { } func (s Secret) GetSecret(ctx context.Context) (map[string]interface{}, error) { + payload, err := s.GetSecretPayload(ctx) + if err != nil { + return nil, err + } + + var v map[string]interface{} + if err := json.Unmarshal(payload, &v); err != nil { + return nil, fmt.Errorf("cannot decode secret string as map[string]interface{}: %s", err) + } + if v == nil { + return nil, fmt.Errorf("secret value is 'null' literal") + } + return v, nil +} + +func (s Secret) GetSecretPayload(ctx context.Context) ([]byte, error) { input := &secretsmanager.GetSecretValueInput{ SecretId: aws.String(s.name), VersionStage: aws.String("AWSCURRENT"), @@ -35,18 +51,22 @@ func (s Secret) GetSecret(ctx context.Context) (map[string]interface{}, error) { if err != nil { return nil, fmt.Errorf("Secrets Manager API error: %s", err) } - blip.Debug("DEBUG: aws secret: %+v", *sv) - - if sv.SecretString == nil || *sv.SecretString == "" { - return nil, fmt.Errorf("secret string is nil or empty") + name := "" + if sv.Name != nil { + name = *sv.Name } + versionID := "" + if sv.VersionId != nil { + versionID = *sv.VersionId + } + blip.Debug("DEBUG: aws secret: name=%s version=%s", name, versionID) - var v map[string]interface{} - if err := json.Unmarshal([]byte(*sv.SecretString), &v); err != nil { - return nil, fmt.Errorf("cannot decode secret string as map[string]string: %s", err) + if sv.SecretString != nil && *sv.SecretString != "" { + return []byte(*sv.SecretString), nil } - if v == nil { - return nil, fmt.Errorf("secret value is 'null' literal") + if len(sv.SecretBinary) > 0 { + return append([]byte(nil), sv.SecretBinary...), nil } - return v, nil + + return nil, fmt.Errorf("secret string and secret binary are empty") } diff --git a/blip.go b/blip.go index 076128df..c135c321 100644 --- a/blip.go +++ b/blip.go @@ -6,6 +6,7 @@ package blip import ( "context" "database/sql" + "encoding/json" "fmt" "log" "math" @@ -98,6 +99,54 @@ type SinkFactoryArgs struct { Tags map[string]string // config.monitor.tags } +// Secret is a MySQL username/password parsed from a secret payload. +type Secret struct { + Username string + Password string +} + +// PasswordSecretParser maps an AWS Secrets Manager payload to a Secret. +// The Secret argument is pre-populated with config defaults and can be modified +// by the parser. The payload is the raw SecretString or SecretBinary value. +type PasswordSecretParser func(context.Context, ConfigMonitor, []byte, *Secret) error + +// DefaultPasswordSecretParser parses the default AWS RDS secret shape: +// "password" is required, and "username" is optional. +func DefaultPasswordSecretParser(_ context.Context, cfg ConfigMonitor, payload []byte, secret *Secret) error { + if secret == nil { + return fmt.Errorf("secret destination is nil") + } + secret.Username = cfg.Username + + var secretPayload map[string]interface{} + if err := json.Unmarshal(payload, &secretPayload); err != nil { + return fmt.Errorf("cannot decode secret as JSON object: %s", err) + } + if secretPayload == nil { + return fmt.Errorf("secret value is 'null' literal") + } + + username, ok := secretPayload["username"] + if ok { + usernameStr, ok := username.(string) + if ok { + secret.Username = usernameStr + } + } + + password, ok := secretPayload["password"] + if !ok { + return fmt.Errorf("error retrieving 'password' value of secret") + } + passwordStr, ok := password.(string) + if !ok { + return fmt.Errorf("invalid type for 'password' value of secret") + } + secret.Password = passwordStr + + return nil +} + // Plugins are function callbacks that override specific functionality of Blip. // Plugins are optional, but if specified it overrides the built-in functionality. type Plugins struct { @@ -123,6 +172,10 @@ type Plugins struct { // ModifyDB modifies the *sql.DB connection pool. Use with caution. ModifyDB func(*sql.DB, string) + // ParsePasswordSecret maps an AWS Secrets Manager payload to MySQL credentials. + // If nil, Blip uses DefaultPasswordSecretParser. + ParsePasswordSecret PasswordSecretParser + // StartMonitor allows a monitor to start by returning true. Else the monitor // is loaded but not started. This is used to load all monitors but start only // certain monitors. diff --git a/dbconn/factory.go b/dbconn/factory.go index b4f90aec..0169067f 100644 --- a/dbconn/factory.go +++ b/dbconn/factory.go @@ -32,17 +32,33 @@ var portSuffix = regexp.MustCompile(`:\d+$`) // factory is the internal implementation of blip.DbFactory. type factory struct { - awsConfig blip.AWSConfigFactory - modifyDB func(*sql.DB, string) + awsConfig blip.AWSConfigFactory + modifyDB func(*sql.DB, string) + passwordSecretParser blip.PasswordSecretParser +} + +// ConnFactoryOption configures the default MySQL connection factory. +type ConnFactoryOption func(*factory) + +// WithPasswordSecretParser sets the parser used for AWS Secrets Manager +// password-secret payloads. +func WithPasswordSecretParser(parser blip.PasswordSecretParser) ConnFactoryOption { + return func(f *factory) { + f.passwordSecretParser = parser + } } // NewConnFactory returns a blip.NewConnFactory that connects to MySQL. // This is the only blip.NewConnFactor. It is created in Server.Defaults. -func NewConnFactory(awsConfig blip.AWSConfigFactory, modifyDB func(*sql.DB, string)) factory { - return factory{ +func NewConnFactory(awsConfig blip.AWSConfigFactory, modifyDB func(*sql.DB, string), opts ...ConnFactoryOption) factory { + f := factory{ awsConfig: awsConfig, modifyDB: modifyDB, } + for _, opt := range opts { + opt(&f) + } + return f } // Make makes a *sql.DB for the given monitor config. On success, it also returns @@ -288,35 +304,7 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { return nil, err } secret := aws.NewSecret(cfg.AWS.PasswordSecret, awscfg) - return func(ctx context.Context) (Credentials, error) { - newSecret, err := secret.GetSecret(ctx) - if err != nil { - return Credentials{}, err - } - - username, ok := newSecret["username"] - if !ok { - // The username key is optional. Default to config - username = cfg.Username - } - usernameStr, ok := username.(string) - if !ok { - username = cfg.Username - } - password, ok := newSecret["password"] - if !ok { - return Credentials{}, fmt.Errorf("error retrieving 'password' value of secret") - } - passwordStr, ok := password.(string) - if !ok { - return Credentials{}, fmt.Errorf("invalid type for 'password' value of secret") - } - - return Credentials{ - Password: passwordStr, - Username: usernameStr, - }, nil - }, nil + return f.passwordSecretCredentialFunc(cfg, secret), nil } // Password file, could be "rotated" (new password written to file) @@ -364,6 +352,36 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { }, nil } +type passwordSecretGetter interface { + GetSecretPayload(context.Context) ([]byte, error) +} + +func (f factory) passwordSecretCredentialFunc(cfg blip.ConfigMonitor, secret passwordSecretGetter) CredentialFunc { + parser := f.passwordSecretParser + if parser == nil { + parser = blip.DefaultPasswordSecretParser + } + + return func(ctx context.Context) (Credentials, error) { + payload, err := secret.GetSecretPayload(ctx) + if err != nil { + return Credentials{}, err + } + + parsedSecret := blip.Secret{ + Username: cfg.Username, + } + if err := parser(ctx, cfg, payload, &parsedSecret); err != nil { + return Credentials{}, err + } + + return Credentials{ + Password: parsedSecret.Password, + Username: parsedSecret.Username, + }, nil + } +} + // -------------------------------------------------------------------------- const ( diff --git a/dbconn/password_secret_test.go b/dbconn/password_secret_test.go new file mode 100644 index 00000000..689b7c31 --- /dev/null +++ b/dbconn/password_secret_test.go @@ -0,0 +1,113 @@ +// Copyright 2024 Block, Inc. + +package dbconn + +import ( + "context" + "errors" + "testing" + + "github.com/cashapp/blip" +) + +type testPasswordSecret struct { + payload []byte + err error +} + +func (s testPasswordSecret) GetSecretPayload(context.Context) ([]byte, error) { + if s.err != nil { + return nil, s.err + } + return s.payload, nil +} + +func TestPasswordSecretCredentialFuncDefaultParser(t *testing.T) { + f := factory{} + credentialFunc := f.passwordSecretCredentialFunc( + blip.ConfigMonitor{Username: "config-user"}, + testPasswordSecret{payload: []byte(`{"password":"secret-pass"}`)}, + ) + + creds, err := credentialFunc(context.Background()) + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } + if creds.Username != "config-user" { + t.Errorf("Username=%q, expected config-user", creds.Username) + } + if creds.Password != "secret-pass" { + t.Errorf("Password=%q, expected secret-pass", creds.Password) + } +} + +func TestPasswordSecretCredentialFuncCustomParser(t *testing.T) { + called := false + f := factory{ + passwordSecretParser: func(_ context.Context, cfg blip.ConfigMonitor, payload []byte, secret *blip.Secret) error { + called = true + if cfg.Username != "config-user" { + t.Errorf("cfg.Username=%q, expected config-user", cfg.Username) + } + if secret.Username != "config-user" { + t.Errorf("pre-populated Username=%q, expected config-user", secret.Username) + } + if string(payload) != "secret-user:secret-pass" { + t.Errorf("payload=%q, expected secret-user:secret-pass", string(payload)) + } + secret.Username = "secret-user" + secret.Password = "secret-pass" + return nil + }, + } + credentialFunc := f.passwordSecretCredentialFunc( + blip.ConfigMonitor{Username: "config-user"}, + testPasswordSecret{payload: []byte("secret-user:secret-pass")}, + ) + + creds, err := credentialFunc(context.Background()) + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } + if !called { + t.Fatal("custom parser was not called") + } + if creds.Username != "secret-user" { + t.Errorf("Username=%q, expected secret-user", creds.Username) + } + if creds.Password != "secret-pass" { + t.Errorf("Password=%q, expected secret-pass", creds.Password) + } +} + +func TestPasswordSecretCredentialFuncParserError(t *testing.T) { + parseErr := errors.New("parse secret") + f := factory{ + passwordSecretParser: func(context.Context, blip.ConfigMonitor, []byte, *blip.Secret) error { + return parseErr + }, + } + credentialFunc := f.passwordSecretCredentialFunc( + blip.ConfigMonitor{Username: "config-user"}, + testPasswordSecret{payload: []byte("secret-pass")}, + ) + + _, err := credentialFunc(context.Background()) + if !errors.Is(err, parseErr) { + t.Fatalf("got error %v, expected %v", err, parseErr) + } +} + +func TestPasswordSecretCredentialFuncGetSecretError(t *testing.T) { + getErr := errors.New("get secret") + f := factory{} + credentialFunc := f.passwordSecretCredentialFunc( + blip.ConfigMonitor{Username: "config-user"}, + testPasswordSecret{err: getErr}, + ) + + _, err := credentialFunc(context.Background()) + if !errors.Is(err, getErr) { + t.Fatalf("got error %v, expected %v", err, getErr) + } +} diff --git a/docs/content/cloud/aws.md b/docs/content/cloud/aws.md index 071eb96a..e108ca5e 100644 --- a/docs/content/cloud/aws.md +++ b/docs/content/cloud/aws.md @@ -107,12 +107,12 @@ If everything is configured correctly in both Blip and AWS, Blip should work as Blip can fetch its MySQL password from a secret in [AWS Secrets Manager](https://aws.amazon.com/secrets-manager/). Set [`config.aws.password-secret`]({{< ref "/config/config-file#password-secret" >}}) to the ARN of the secret. -The secret value must be a key-value map with a `password` key, like: +By default, the secret value must be a JSON object with a string `password` key, like: ```json { "username": "blip", - "password": "...", // Blip uses only this value + "password": "...", "engine": "mysql", "host": "db.cluster.us-east-1.rds.amazonaws.com", "port": 3306, @@ -120,13 +120,20 @@ The secret value must be a key-value map with a `password` key, like: } ``` -Blip ignores other keys in the secret value; it only reads the `password` key-value. +By default, Blip ignores other keys in the secret value and reads only `username` and `password`. +The `username` key is optional; if it is not set, Blip uses the configured monitor username. {{< hint type=note >}} The example above is the default secret map that AWS creates for RDS. -It works with Blip, but Blip currently ignores the non-password fields. +It works with Blip, but Blip ignores the connection fields like `host` and `port`. {{< /hint >}} +If you embed Blip and use a different secret payload shape, set the +[`blip.Plugins.ParsePasswordSecret`](https://pkg.go.dev/github.com/cashapp/blip#Plugins) +callback before calling `Server.Boot`. +The callback receives one raw payload: `SecretString` bytes when present, otherwise `SecretBinary` bytes. +It should set the password and, if needed, username on the secret value passed to it. + The [AWS credentials](#credentials) that Blip uses must be allowed to read the secret with a policy privilege like: ```json diff --git a/docs/content/config/config-file.md b/docs/content/config/config-file.md index a32444c8..3848a63d 100644 --- a/docs/content/config/config-file.md +++ b/docs/content/config/config-file.md @@ -264,7 +264,9 @@ See [Cloud / AWS / IAM Authentication]({{< ref "/cloud/aws#iam-authentication" > |**Valid values**|AWS Secrets Manager ARN| |**Default value**|| -The `password-secret` variables sets the AWS Secrets Manager ARN that contains the MySQL user password. +The `password-secret` variable sets the AWS Secrets Manager ARN that contains the MySQL user password. +When using the default parser, the secret JSON must contain a string `password` field, and it can optionally contain a string `username` field. +Custom integrations can override this with [`Plugins.ParsePasswordSecret`]({{< ref "/develop/integration-api#aws-password-secrets" >}}). #### `region` diff --git a/docs/content/develop/integration-api.md b/docs/content/develop/integration-api.md index cf0d2771..05aac41d 100644 --- a/docs/content/develop/integration-api.md +++ b/docs/content/develop/integration-api.md @@ -16,6 +16,7 @@ How you integrate with Blip depends on what you're trying to customize: |Loading Blip config|Plugins| |Loading monitors|Plugins| |Loading plans|Plugins| +|Parsing AWS password secrets|Plugins| |AWS configs|Factories| |Database connections|Factories| |HTTP clients|Factories| @@ -58,6 +59,20 @@ Every factory is optional: if specified, it overrides the built-in factory. [Plugins](https://pkg.go.dev/github.com/cashapp/blip#Plugins) are function callbacks that let you override specific functionality of Blip. Every plugin is optional: if specified, it overrides the built-in functionality. +### AWS Password Secrets + +Set [`Plugins.ParsePasswordSecret`](https://pkg.go.dev/github.com/cashapp/blip#Plugins) to customize how Blip maps the raw AWS Secrets Manager payload from [`config.aws.password-secret`]({{< ref "/config/config-file#password-secret" >}}) to MySQL credentials. +If this callback is not set, Blip uses [`DefaultPasswordSecretParser`](https://pkg.go.dev/github.com/cashapp/blip#DefaultPasswordSecretParser): `password` is required, and `username` is optional. +Blip passes `SecretString` bytes when present; otherwise, it passes `SecretBinary` bytes. +The `secret` argument is initialized with the configured monitor username; custom parsers must set `secret.Password` and can override `secret.Username`. + +```go +plugins.ParsePasswordSecret = func(ctx context.Context, cfg blip.ConfigMonitor, payload []byte, secret *blip.Secret) error { + secret.Password = string(payload) + return nil +} +``` + ## Events Implement a [Receiver](https://pkg.go.dev/github.com/cashapp/blip/event#Receiver), then call [event.SetReceiver](https://pkg.go.dev/github.com/cashapp/blip/event#SetReceiver) to override the default. diff --git a/secret_test.go b/secret_test.go new file mode 100644 index 00000000..a00fc0e8 --- /dev/null +++ b/secret_test.go @@ -0,0 +1,89 @@ +// Copyright 2024 Block, Inc. + +package blip_test + +import ( + "context" + "testing" + + "github.com/cashapp/blip" +) + +func TestDefaultPasswordSecretParser(t *testing.T) { + tests := []struct { + name string + payload []byte + expectUsername string + expectPassword string + expectErr bool + }{ + { + name: "username and password", + payload: []byte(`{"username":"secret-user","password":"secret-pass"}`), + expectUsername: "secret-user", + expectPassword: "secret-pass", + }, + { + name: "password only", + payload: []byte(`{"password":"secret-pass"}`), + expectUsername: "config-user", + expectPassword: "secret-pass", + }, + { + name: "non-string username falls back to config", + payload: []byte(`{"username":123,"password":"secret-pass"}`), + expectUsername: "config-user", + expectPassword: "secret-pass", + }, + { + name: "missing password", + payload: []byte(`{"username":"secret-user"}`), + expectErr: true, + }, + { + name: "non-string password", + payload: []byte(`{"username":"secret-user","password":123}`), + expectErr: true, + }, + { + name: "malformed JSON", + payload: []byte(`{`), + expectErr: true, + }, + { + name: "null literal", + payload: []byte(`null`), + expectErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + cfg := blip.ConfigMonitor{Username: "config-user"} + secret := blip.Secret{} + err := blip.DefaultPasswordSecretParser(context.Background(), cfg, tt.payload, &secret) + if tt.expectErr { + if err == nil { + t.Fatal("got nil error, expected non-nil error") + } + return + } + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } + if secret.Username != tt.expectUsername { + t.Errorf("Username=%q, expected %q", secret.Username, tt.expectUsername) + } + if secret.Password != tt.expectPassword { + t.Errorf("Password=%q, expected %q", secret.Password, tt.expectPassword) + } + }) + } +} + +func TestDefaultPasswordSecretParserNilSecret(t *testing.T) { + err := blip.DefaultPasswordSecretParser(context.Background(), blip.ConfigMonitor{}, []byte(`{}`), nil) + if err == nil { + t.Fatal("got nil error, expected non-nil error") + } +} diff --git a/server/server.go b/server/server.go index 32625c67..498334f3 100644 --- a/server/server.go +++ b/server/server.go @@ -178,7 +178,11 @@ func (s *Server) Boot(env blip.Env, plugins blip.Plugins, factories blip.Factori } if factories.DbConn == nil { - factories.DbConn = dbconn.NewConnFactory(factories.AWSConfig, plugins.ModifyDB) + factories.DbConn = dbconn.NewConnFactory( + factories.AWSConfig, + plugins.ModifyDB, + dbconn.WithPasswordSecretParser(plugins.ParsePasswordSecret), + ) } sink.InitFactory(factories) From f20ff66a51b5a252336a995818cb9de30ee2ea5d Mon Sep 17 00:00:00 2001 From: Ian Oberst Date: Thu, 4 Jun 2026 09:09:05 -0700 Subject: [PATCH 2/3] Move password secret setup into helper --- dbconn/factory.go | 22 ++--- dbconn/password_secret_test.go | 168 +++++++++++++++++++++++++++------ 2 files changed, 148 insertions(+), 42 deletions(-) diff --git a/dbconn/factory.go b/dbconn/factory.go index 0169067f..59ea7a78 100644 --- a/dbconn/factory.go +++ b/dbconn/factory.go @@ -298,13 +298,7 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { // Amazon Secrets Manager, could be rotated if cfg.AWS.PasswordSecret != "" { - blip.Debug("%s: AWS Secrets Manager password", cfg.MonitorId) - awscfg, err := f.awsConfig.Make(blip.AWS{Region: cfg.AWS.Region}, cfg.Hostname) - if err != nil { - return nil, err - } - secret := aws.NewSecret(cfg.AWS.PasswordSecret, awscfg) - return f.passwordSecretCredentialFunc(cfg, secret), nil + return f.passwordSecretCredentialFunc(cfg) } // Password file, could be "rotated" (new password written to file) @@ -352,11 +346,13 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { }, nil } -type passwordSecretGetter interface { - GetSecretPayload(context.Context) ([]byte, error) -} - -func (f factory) passwordSecretCredentialFunc(cfg blip.ConfigMonitor, secret passwordSecretGetter) CredentialFunc { +func (f factory) passwordSecretCredentialFunc(cfg blip.ConfigMonitor) (CredentialFunc, error) { + blip.Debug("%s: AWS Secrets Manager password", cfg.MonitorId) + awscfg, err := f.awsConfig.Make(blip.AWS{Region: cfg.AWS.Region}, cfg.Hostname) + if err != nil { + return nil, err + } + secret := aws.NewSecret(cfg.AWS.PasswordSecret, awscfg) parser := f.passwordSecretParser if parser == nil { parser = blip.DefaultPasswordSecretParser @@ -379,7 +375,7 @@ func (f factory) passwordSecretCredentialFunc(cfg blip.ConfigMonitor, secret pas Password: parsedSecret.Password, Username: parsedSecret.Username, }, nil - } + }, nil } // -------------------------------------------------------------------------- diff --git a/dbconn/password_secret_test.go b/dbconn/password_secret_test.go index 689b7c31..ab2bddd9 100644 --- a/dbconn/password_secret_test.go +++ b/dbconn/password_secret_test.go @@ -4,30 +4,82 @@ package dbconn import ( "context" + "encoding/json" "errors" + "net/http" + "net/http/httptest" + "strings" "testing" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/cashapp/blip" ) -type testPasswordSecret struct { - payload []byte - err error +type testAWSConfigFactory struct { + cfg aws.Config + err error } -func (s testPasswordSecret) GetSecretPayload(context.Context) ([]byte, error) { - if s.err != nil { - return nil, s.err +func (f testAWSConfigFactory) Make(blip.AWS, string) (aws.Config, error) { + if f.err != nil { + return aws.Config{}, f.err } - return s.payload, nil + return f.cfg, nil +} + +type errHTTPClient struct { + err error +} + +func (c errHTTPClient) Do(*http.Request) (*http.Response, error) { + return nil, c.err +} + +func testPasswordSecretConfig(t *testing.T, secretString string) (aws.Config, func()) { + t.Helper() + + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/x-amz-json-1.1") + if err := json.NewEncoder(w).Encode(map[string]string{ + "Name": "test-secret", + "SecretString": secretString, + "VersionId": "test-version", + }); err != nil { + t.Errorf("cannot encode Secrets Manager response: %s", err) + } + })) + + return aws.Config{ + Credentials: credentials.NewStaticCredentialsProvider("access-key", "secret-key", ""), + EndpointResolver: aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{URL: server.URL, SigningRegion: "us-east-1"}, nil + }), + HTTPClient: server.Client(), + Region: "us-east-1", + Retryer: func() aws.Retryer { + return aws.NopRetryer{} + }, + }, server.Close } func TestPasswordSecretCredentialFuncDefaultParser(t *testing.T) { - f := factory{} - credentialFunc := f.passwordSecretCredentialFunc( - blip.ConfigMonitor{Username: "config-user"}, - testPasswordSecret{payload: []byte(`{"password":"secret-pass"}`)}, - ) + awscfg, cleanup := testPasswordSecretConfig(t, `{"password":"secret-pass"}`) + defer cleanup() + + f := factory{awsConfig: testAWSConfigFactory{cfg: awscfg}} + credentialFunc, err := f.passwordSecretCredentialFunc(blip.ConfigMonitor{ + Hostname: "db.example.com", + Username: "config-user", + AWS: blip.ConfigAWS{ + PasswordSecret: "test-secret", + Region: "us-east-1", + }, + }) + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } creds, err := credentialFunc(context.Background()) if err != nil { @@ -42,8 +94,12 @@ func TestPasswordSecretCredentialFuncDefaultParser(t *testing.T) { } func TestPasswordSecretCredentialFuncCustomParser(t *testing.T) { + awscfg, cleanup := testPasswordSecretConfig(t, "secret-user:secret-pass") + defer cleanup() + called := false f := factory{ + awsConfig: testAWSConfigFactory{cfg: awscfg}, passwordSecretParser: func(_ context.Context, cfg blip.ConfigMonitor, payload []byte, secret *blip.Secret) error { called = true if cfg.Username != "config-user" { @@ -60,10 +116,17 @@ func TestPasswordSecretCredentialFuncCustomParser(t *testing.T) { return nil }, } - credentialFunc := f.passwordSecretCredentialFunc( - blip.ConfigMonitor{Username: "config-user"}, - testPasswordSecret{payload: []byte("secret-user:secret-pass")}, - ) + credentialFunc, err := f.passwordSecretCredentialFunc(blip.ConfigMonitor{ + Hostname: "db.example.com", + Username: "config-user", + AWS: blip.ConfigAWS{ + PasswordSecret: "test-secret", + Region: "us-east-1", + }, + }) + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } creds, err := credentialFunc(context.Background()) if err != nil { @@ -81,18 +144,29 @@ func TestPasswordSecretCredentialFuncCustomParser(t *testing.T) { } func TestPasswordSecretCredentialFuncParserError(t *testing.T) { + awscfg, cleanup := testPasswordSecretConfig(t, "secret-pass") + defer cleanup() + parseErr := errors.New("parse secret") f := factory{ + awsConfig: testAWSConfigFactory{cfg: awscfg}, passwordSecretParser: func(context.Context, blip.ConfigMonitor, []byte, *blip.Secret) error { return parseErr }, } - credentialFunc := f.passwordSecretCredentialFunc( - blip.ConfigMonitor{Username: "config-user"}, - testPasswordSecret{payload: []byte("secret-pass")}, - ) + credentialFunc, err := f.passwordSecretCredentialFunc(blip.ConfigMonitor{ + Hostname: "db.example.com", + Username: "config-user", + AWS: blip.ConfigAWS{ + PasswordSecret: "test-secret", + Region: "us-east-1", + }, + }) + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } - _, err := credentialFunc(context.Background()) + _, err = credentialFunc(context.Background()) if !errors.Is(err, parseErr) { t.Fatalf("got error %v, expected %v", err, parseErr) } @@ -100,14 +174,50 @@ func TestPasswordSecretCredentialFuncParserError(t *testing.T) { func TestPasswordSecretCredentialFuncGetSecretError(t *testing.T) { getErr := errors.New("get secret") - f := factory{} - credentialFunc := f.passwordSecretCredentialFunc( - blip.ConfigMonitor{Username: "config-user"}, - testPasswordSecret{err: getErr}, - ) - - _, err := credentialFunc(context.Background()) - if !errors.Is(err, getErr) { + f := factory{ + awsConfig: testAWSConfigFactory{cfg: aws.Config{ + Credentials: credentials.NewStaticCredentialsProvider("access-key", "secret-key", ""), + EndpointResolver: aws.EndpointResolverFunc(func(service, region string) (aws.Endpoint, error) { + return aws.Endpoint{URL: "http://127.0.0.1", SigningRegion: "us-east-1"}, nil + }), + HTTPClient: errHTTPClient{err: getErr}, + Region: "us-east-1", + Retryer: func() aws.Retryer { + return aws.NopRetryer{} + }, + }}, + } + credentialFunc, err := f.passwordSecretCredentialFunc(blip.ConfigMonitor{ + Hostname: "db.example.com", + Username: "config-user", + AWS: blip.ConfigAWS{ + PasswordSecret: "test-secret", + Region: "us-east-1", + }, + }) + if err != nil { + t.Fatalf("got error %s, expected nil", err) + } + + _, err = credentialFunc(context.Background()) + if err == nil || !strings.Contains(err.Error(), getErr.Error()) { t.Fatalf("got error %v, expected %v", err, getErr) } } + +func TestPasswordSecretCredentialFuncAWSConfigError(t *testing.T) { + configErr := errors.New("aws config") + f := factory{awsConfig: testAWSConfigFactory{err: configErr}} + + _, err := f.passwordSecretCredentialFunc(blip.ConfigMonitor{ + Hostname: "db.example.com", + Username: "config-user", + AWS: blip.ConfigAWS{ + PasswordSecret: "test-secret", + Region: "us-east-1", + }, + }) + if !errors.Is(err, configErr) { + t.Fatalf("got error %v, expected %v", err, configErr) + } +} From 2756f7827bba481c627bac272b9dbfdc400b219d Mon Sep 17 00:00:00 2001 From: Ian Oberst Date: Thu, 4 Jun 2026 12:03:47 -0700 Subject: [PATCH 3/3] Expose password secret credentials type --- blip.go | 23 ++++++------- dbconn/factory.go | 43 ++++++++++++------------- dbconn/password_secret_test.go | 12 +++---- dbconn/reload_password.go | 8 +---- docs/content/cloud/aws.md | 2 +- docs/content/develop/integration-api.md | 6 ++-- secret_test.go | 14 ++++---- 7 files changed, 50 insertions(+), 58 deletions(-) diff --git a/blip.go b/blip.go index c135c321..689e9fba 100644 --- a/blip.go +++ b/blip.go @@ -99,24 +99,25 @@ type SinkFactoryArgs struct { Tags map[string]string // config.monitor.tags } -// Secret is a MySQL username/password parsed from a secret payload. -type Secret struct { +// DbCredentials are MySQL credentials parsed or loaded for a connection. +type DbCredentials struct { Username string Password string + TLS ConfigTLS } -// PasswordSecretParser maps an AWS Secrets Manager payload to a Secret. -// The Secret argument is pre-populated with config defaults and can be modified +// PasswordSecretParser maps an AWS Secrets Manager payload to database credentials. +// The DbCredentials argument is pre-populated with config defaults and can be modified // by the parser. The payload is the raw SecretString or SecretBinary value. -type PasswordSecretParser func(context.Context, ConfigMonitor, []byte, *Secret) error +type PasswordSecretParser func(context.Context, ConfigMonitor, []byte, *DbCredentials) error // DefaultPasswordSecretParser parses the default AWS RDS secret shape: // "password" is required, and "username" is optional. -func DefaultPasswordSecretParser(_ context.Context, cfg ConfigMonitor, payload []byte, secret *Secret) error { - if secret == nil { - return fmt.Errorf("secret destination is nil") +func DefaultPasswordSecretParser(_ context.Context, cfg ConfigMonitor, payload []byte, credentials *DbCredentials) error { + if credentials == nil { + return fmt.Errorf("credentials destination is nil") } - secret.Username = cfg.Username + credentials.Username = cfg.Username var secretPayload map[string]interface{} if err := json.Unmarshal(payload, &secretPayload); err != nil { @@ -130,7 +131,7 @@ func DefaultPasswordSecretParser(_ context.Context, cfg ConfigMonitor, payload [ if ok { usernameStr, ok := username.(string) if ok { - secret.Username = usernameStr + credentials.Username = usernameStr } } @@ -142,7 +143,7 @@ func DefaultPasswordSecretParser(_ context.Context, cfg ConfigMonitor, payload [ if !ok { return fmt.Errorf("invalid type for 'password' value of secret") } - secret.Password = passwordStr + credentials.Password = passwordStr return nil } diff --git a/dbconn/factory.go b/dbconn/factory.go index 59ea7a78..263b3678 100644 --- a/dbconn/factory.go +++ b/dbconn/factory.go @@ -157,7 +157,7 @@ func (f factory) Make(cfg blip.ConfigMonitor) (*sql.DB, string, error) { // TLS is configured, so make sure we reload it when the credentials are reloaded in case // it was changed origCredentialFunc := credentialFunc - credentialFunc = func(ctx context.Context) (Credentials, error) { + credentialFunc = func(ctx context.Context) (blip.DbCredentials, error) { creds, err := origCredentialFunc(ctx) if err != nil { return creds, err @@ -283,13 +283,13 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { return nil, err } token := aws.NewAuthToken(cfg.Username, cfg.Hostname, awscfg) - return func(ctx context.Context) (Credentials, error) { + return func(ctx context.Context) (blip.DbCredentials, error) { passwd, err := token.Password(ctx) if err != nil { - return Credentials{}, err + return blip.DbCredentials{}, err } - return Credentials{ + return blip.DbCredentials{ Password: passwd, Username: cfg.Username, }, nil @@ -304,12 +304,12 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { // Password file, could be "rotated" (new password written to file) if cfg.PasswordFile != "" { blip.Debug("%s: password file", cfg.MonitorId) - return func(context.Context) (Credentials, error) { + return func(context.Context) (blip.DbCredentials, error) { bytes, err := os.ReadFile(cfg.PasswordFile) if err != nil { - return Credentials{}, err + return blip.DbCredentials{}, err } - return Credentials{ + return blip.DbCredentials{ Password: string(bytes), Username: cfg.Username, }, err @@ -319,12 +319,12 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { // Credentials in my.cnf file, could be rotated (username and/or password, along with TLS config) if cfg.MyCnf != "" { blip.Debug("%s my.cnf credentials", cfg.MonitorId) - return func(context.Context) (Credentials, error) { + return func(context.Context) (blip.DbCredentials, error) { cfg, tlscfg, err := ParseMyCnf(cfg.MyCnf) if err != nil { - return Credentials{}, err + return blip.DbCredentials{}, err } - return Credentials{ + return blip.DbCredentials{ Password: cfg.Password, Username: cfg.Username, TLS: tlscfg, @@ -335,14 +335,14 @@ func (f factory) Credentials(cfg blip.ConfigMonitor) (CredentialFunc, error) { // Static password in Blip config file, not rotated if cfg.Password != "" { blip.Debug("%s: static password credentials", cfg.MonitorId) - return func(context.Context) (Credentials, error) { - return Credentials{Password: cfg.Password, Username: cfg.Username}, nil + return func(context.Context) (blip.DbCredentials, error) { + return blip.DbCredentials{Password: cfg.Password, Username: cfg.Username}, nil }, nil } blip.Debug("%s: no password", cfg.MonitorId) - return func(context.Context) (Credentials, error) { - return Credentials{Password: "", Username: cfg.Username}, nil + return func(context.Context) (blip.DbCredentials, error) { + return blip.DbCredentials{Password: "", Username: cfg.Username}, nil }, nil } @@ -358,23 +358,20 @@ func (f factory) passwordSecretCredentialFunc(cfg blip.ConfigMonitor) (Credentia parser = blip.DefaultPasswordSecretParser } - return func(ctx context.Context) (Credentials, error) { + return func(ctx context.Context) (blip.DbCredentials, error) { payload, err := secret.GetSecretPayload(ctx) if err != nil { - return Credentials{}, err + return blip.DbCredentials{}, err } - parsedSecret := blip.Secret{ + credentials := blip.DbCredentials{ Username: cfg.Username, } - if err := parser(ctx, cfg, payload, &parsedSecret); err != nil { - return Credentials{}, err + if err := parser(ctx, cfg, payload, &credentials); err != nil { + return blip.DbCredentials{}, err } - return Credentials{ - Password: parsedSecret.Password, - Username: parsedSecret.Username, - }, nil + return credentials, nil }, nil } diff --git a/dbconn/password_secret_test.go b/dbconn/password_secret_test.go index ab2bddd9..a6d78e05 100644 --- a/dbconn/password_secret_test.go +++ b/dbconn/password_secret_test.go @@ -100,19 +100,19 @@ func TestPasswordSecretCredentialFuncCustomParser(t *testing.T) { called := false f := factory{ awsConfig: testAWSConfigFactory{cfg: awscfg}, - passwordSecretParser: func(_ context.Context, cfg blip.ConfigMonitor, payload []byte, secret *blip.Secret) error { + passwordSecretParser: func(_ context.Context, cfg blip.ConfigMonitor, payload []byte, credentials *blip.DbCredentials) error { called = true if cfg.Username != "config-user" { t.Errorf("cfg.Username=%q, expected config-user", cfg.Username) } - if secret.Username != "config-user" { - t.Errorf("pre-populated Username=%q, expected config-user", secret.Username) + if credentials.Username != "config-user" { + t.Errorf("pre-populated Username=%q, expected config-user", credentials.Username) } if string(payload) != "secret-user:secret-pass" { t.Errorf("payload=%q, expected secret-user:secret-pass", string(payload)) } - secret.Username = "secret-user" - secret.Password = "secret-pass" + credentials.Username = "secret-user" + credentials.Password = "secret-pass" return nil }, } @@ -150,7 +150,7 @@ func TestPasswordSecretCredentialFuncParserError(t *testing.T) { parseErr := errors.New("parse secret") f := factory{ awsConfig: testAWSConfigFactory{cfg: awscfg}, - passwordSecretParser: func(context.Context, blip.ConfigMonitor, []byte, *blip.Secret) error { + passwordSecretParser: func(context.Context, blip.ConfigMonitor, []byte, *blip.DbCredentials) error { return parseErr }, } diff --git a/dbconn/reload_password.go b/dbconn/reload_password.go index 6af0555d..43f2df1f 100644 --- a/dbconn/reload_password.go +++ b/dbconn/reload_password.go @@ -17,13 +17,7 @@ func init() { dsndriver.SetHotswapFunc(Repo.ReloadDSN) } -type Credentials struct { - Username string - Password string - TLS blip.ConfigTLS -} - -type CredentialFunc func(context.Context) (Credentials, error) +type CredentialFunc func(context.Context) (blip.DbCredentials, error) type repo struct { m *sync.Map diff --git a/docs/content/cloud/aws.md b/docs/content/cloud/aws.md index e108ca5e..e2b36172 100644 --- a/docs/content/cloud/aws.md +++ b/docs/content/cloud/aws.md @@ -132,7 +132,7 @@ If you embed Blip and use a different secret payload shape, set the [`blip.Plugins.ParsePasswordSecret`](https://pkg.go.dev/github.com/cashapp/blip#Plugins) callback before calling `Server.Boot`. The callback receives one raw payload: `SecretString` bytes when present, otherwise `SecretBinary` bytes. -It should set the password and, if needed, username on the secret value passed to it. +It should set the password and, if needed, username on the credentials value passed to it. The [AWS credentials](#credentials) that Blip uses must be allowed to read the secret with a policy privilege like: diff --git a/docs/content/develop/integration-api.md b/docs/content/develop/integration-api.md index 05aac41d..64eb51fc 100644 --- a/docs/content/develop/integration-api.md +++ b/docs/content/develop/integration-api.md @@ -64,11 +64,11 @@ Every plugin is optional: if specified, it overrides the built-in functionality. Set [`Plugins.ParsePasswordSecret`](https://pkg.go.dev/github.com/cashapp/blip#Plugins) to customize how Blip maps the raw AWS Secrets Manager payload from [`config.aws.password-secret`]({{< ref "/config/config-file#password-secret" >}}) to MySQL credentials. If this callback is not set, Blip uses [`DefaultPasswordSecretParser`](https://pkg.go.dev/github.com/cashapp/blip#DefaultPasswordSecretParser): `password` is required, and `username` is optional. Blip passes `SecretString` bytes when present; otherwise, it passes `SecretBinary` bytes. -The `secret` argument is initialized with the configured monitor username; custom parsers must set `secret.Password` and can override `secret.Username`. +The `credentials` argument is initialized with the configured monitor username; custom parsers must set `credentials.Password` and can override `credentials.Username`. ```go -plugins.ParsePasswordSecret = func(ctx context.Context, cfg blip.ConfigMonitor, payload []byte, secret *blip.Secret) error { - secret.Password = string(payload) +plugins.ParsePasswordSecret = func(ctx context.Context, cfg blip.ConfigMonitor, payload []byte, credentials *blip.DbCredentials) error { + credentials.Password = string(payload) return nil } ``` diff --git a/secret_test.go b/secret_test.go index a00fc0e8..be9df081 100644 --- a/secret_test.go +++ b/secret_test.go @@ -60,8 +60,8 @@ func TestDefaultPasswordSecretParser(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { cfg := blip.ConfigMonitor{Username: "config-user"} - secret := blip.Secret{} - err := blip.DefaultPasswordSecretParser(context.Background(), cfg, tt.payload, &secret) + credentials := blip.DbCredentials{} + err := blip.DefaultPasswordSecretParser(context.Background(), cfg, tt.payload, &credentials) if tt.expectErr { if err == nil { t.Fatal("got nil error, expected non-nil error") @@ -71,17 +71,17 @@ func TestDefaultPasswordSecretParser(t *testing.T) { if err != nil { t.Fatalf("got error %s, expected nil", err) } - if secret.Username != tt.expectUsername { - t.Errorf("Username=%q, expected %q", secret.Username, tt.expectUsername) + if credentials.Username != tt.expectUsername { + t.Errorf("Username=%q, expected %q", credentials.Username, tt.expectUsername) } - if secret.Password != tt.expectPassword { - t.Errorf("Password=%q, expected %q", secret.Password, tt.expectPassword) + if credentials.Password != tt.expectPassword { + t.Errorf("Password=%q, expected %q", credentials.Password, tt.expectPassword) } }) } } -func TestDefaultPasswordSecretParserNilSecret(t *testing.T) { +func TestDefaultPasswordSecretParserNilCredentials(t *testing.T) { err := blip.DefaultPasswordSecretParser(context.Background(), blip.ConfigMonitor{}, []byte(`{}`), nil) if err == nil { t.Fatal("got nil error, expected non-nil error")