Vault arbitrary env passthrough (#52)

* resolve #46

Adds a new option: pass-env, which when true will pass the pico process environment to children. Defaults to false to promote separation of environments.

Adds support for passing prefixed variables from the global Pico. The prefix is GLOBAL_ and is not configurable because I felt the config flags are growing.

Adds some better unit tests for execution config and environment merging.

* correctly strip the prefix for global configuration values pulled from the secret store

Author: Southclaws

executor/cmd.go

@@ -12,13 +12,24 @@ var _ Executor = &CommandExecutor{}
 
 // CommandExecutor handles command invocation targets
 type CommandExecutor struct {
-	secrets secret.Store
+	secrets            secret.Store
+	passEnvironment    bool   // pass the Pico process environment to children
+	configSecretPath   string // path to global secrets to pass to children
+	configSecretPrefix string // only pass secrets with this prefix, usually GLOBAL_
 }
 
 // NewCommandExecutor creates a new CommandExecutor
-func NewCommandExecutor(secrets secret.Store) CommandExecutor {
+func NewCommandExecutor(
+	secrets secret.Store,
+	passEnvironment bool,
+	configSecretPath string,
+	configSecretPrefix string,
+) CommandExecutor {
 	return CommandExecutor{
-		secrets: secrets,
+		secrets:            secrets,
+		passEnvironment:    passEnvironment,
+		configSecretPath:   configSecretPath,
+		configSecretPrefix: configSecretPrefix,
 	}
 }
 
@@ -34,34 +45,66 @@ func (e *CommandExecutor) Subscribe(bus chan task.ExecutionTask) {
 	}
 }
 
-func (e *CommandExecutor) execute(
-	target task.Target,
+type exec struct {
+	path            string
+	env             map[string]string
+	shutdown        bool
+	passEnvironment bool
+}
+
+func (e *CommandExecutor) prepare(
+	name string,
 	path string,
 	shutdown bool,
 	execEnv map[string]string,
-) (err error) {
-	secrets, err := e.secrets.GetSecretsForTarget(target.Name)
+) (exec, error) {
+	// get global secrets from the Pico config path in the secret store.
+	// only secrets with the prefix are retrieved.
+	global, err := secret.GetPrefixedSecrets(e.secrets, e.configSecretPath, e.configSecretPrefix)
 	if err != nil {
-		return errors.Wrap(err, "failed to get secrets for target")
+		return exec{}, errors.Wrap(err, "failed to get global secrets for target")
+	}
+
+	secrets, err := e.secrets.GetSecretsForTarget(name)
+	if err != nil {
+		return exec{}, errors.Wrap(err, "failed to get secrets for target")
 	}
 
 	env := make(map[string]string)
 
-	// merge execution environment with secrets
+	// merge execution environment with secrets in the following order:
+	// globals first, then execution environment, then per-target secrets
+	for k, v := range global {
+		env[k] = v
+	}
 	for k, v := range execEnv {
 		env[k] = v
 	}
 	for k, v := range secrets {
 		env[k] = v
 	}
 
+	return exec{path, env, shutdown, e.passEnvironment}, nil
+}
+
+func (e *CommandExecutor) execute(
+	target task.Target,
+	path string,
+	shutdown bool,
+	execEnv map[string]string,
+) (err error) {
+	ex, err := e.prepare(target.Name, path, shutdown, execEnv)
+	if err != nil {
+		return err
+	}
+
 	zap.L().Debug("executing with secrets",
 		zap.String("target", target.Name),
 		zap.Strings("cmd", target.Up),
 		zap.String("url", target.RepoURL),
 		zap.String("dir", path),
-		zap.Int("env", len(env)),
-		zap.Int("secrets", len(secrets)))
+		zap.Int("env", len(ex.env)),
+		zap.Bool("passthrough", e.passEnvironment))
 
-	return target.Execute(path, env, shutdown)
+	return target.Execute(ex.path, ex.env, ex.shutdown, ex.passEnvironment)
 }

executor/cmd_test.go

@@ -1,4 +1,4 @@
-package executor_test
+package executor
 
 import (
 	"os"
@@ -7,9 +7,9 @@ import (
 
 	"golang.org/x/sync/errgroup"
 
-	"github.com/picostack/pico/executor"
 	"github.com/picostack/pico/secret/memory"
 	"github.com/picostack/pico/task"
+	"github.com/stretchr/testify/assert"
 
 	_ "github.com/picostack/pico/logger"
 )
@@ -20,11 +20,13 @@ func TestMain(m *testing.M) {
 }
 
 func TestCommandExecutor(t *testing.T) {
-	ce := executor.NewCommandExecutor(&memory.MemorySecrets{
-		Secrets: map[string]string{
-			"SOME_SECRET": "123",
+	ce := NewCommandExecutor(&memory.MemorySecrets{
+		Secrets: map[string]map[string]string{
+			"test": map[string]string{
+				"SOME_SECRET": "123",
+			},
 		},
-	})
+	}, false, "pico", "GLOBAL_")
 	bus := make(chan task.ExecutionTask)
 
 	g := errgroup.Group{}
@@ -55,3 +57,50 @@ func TestCommandExecutor(t *testing.T) {
 
 	os.RemoveAll(".test/.git")
 }
+
+func TestCommandPrepareWithoutPassthrough(t *testing.T) {
+	ce := NewCommandExecutor(&memory.MemorySecrets{
+		Secrets: map[string]map[string]string{
+			"test": map[string]string{
+				"SOME_SECRET": "123",
+			},
+		},
+	}, false, "pico", "GLOBAL_")
+
+	ex, err := ce.prepare("test", "./", false, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, exec{
+		path: "./",
+		env: map[string]string{
+			"SOME_SECRET": "123",
+		},
+		shutdown:        false,
+		passEnvironment: false,
+	}, ex)
+}
+
+func TestCommandPrepareWithGlobal(t *testing.T) {
+	ce := NewCommandExecutor(&memory.MemorySecrets{
+		Secrets: map[string]map[string]string{
+			"test": map[string]string{
+				"SOME_SECRET": "123",
+			},
+			"pico": map[string]string{
+				"GLOBAL_SECRET": "456",
+				"IGNORE":        "this",
+			},
+		},
+	}, false, "pico", "GLOBAL_")
+
+	ex, err := ce.prepare("test", "./", false, nil)
+	assert.NoError(t, err)
+	assert.Equal(t, exec{
+		path: "./",
+		env: map[string]string{
+			"SOME_SECRET": "123",
+			"SECRET":      "456",
+		},
+		shutdown:        false,
+		passEnvironment: false,
+	}, ex)
+}

main.go

@@ -43,6 +43,7 @@ this repository has new commits, Pico will automatically reconfigure.`,
 				cli.StringFlag{Name: "git-password", EnvVar: "GIT_PASSWORD"},
 				cli.StringFlag{Name: "hostname", EnvVar: "HOSTNAME"},
 				cli.StringFlag{Name: "directory", EnvVar: "DIRECTORY", Value: "./cache/"},
+				cli.DurationFlag{Name: "pass-env", EnvVar: "PASS_ENV"},
 				cli.BoolFlag{Name: "ssh", EnvVar: "SSH"},
 				cli.DurationFlag{Name: "check-interval", EnvVar: "CHECK_INTERVAL", Value: time.Second * 10},
 				cli.StringFlag{Name: "vault-addr", EnvVar: "VAULT_ADDR"},
@@ -77,15 +78,16 @@ this repository has new commits, Pico will automatically reconfigure.`,
 						User: c.String("git-username"),
 						Pass: c.String("git-password"),
 					},
-					Hostname:      hostname,
-					Directory:     c.String("directory"),
-					SSH:           c.Bool("ssh"),
-					CheckInterval: c.Duration("check-interval"),
-					VaultAddress:  c.String("vault-addr"),
-					VaultToken:    c.String("vault-token"),
-					VaultPath:     c.String("vault-path"),
-					VaultRenewal:  c.Duration("vault-renew-interval"),
-					VaultConfig:   c.String("vault-config-path"),
+					Hostname:        hostname,
+					Directory:       c.String("directory"),
+					PassEnvironment: c.Bool("pass-env"),
+					SSH:             c.Bool("ssh"),
+					CheckInterval:   c.Duration("check-interval"),
+					VaultAddress:    c.String("vault-addr"),
+					VaultToken:      c.String("vault-token"),
+					VaultPath:       c.String("vault-path"),
+					VaultRenewal:    c.Duration("vault-renew-interval"),
+					VaultConfig:     c.String("vault-config-path"),
 				})
 				if err != nil {
 					return errors.Wrap(err, "failed to initialise")

secret/memory/memory.go

@@ -6,12 +6,16 @@ import (
 
 // MemorySecrets implements a simple in-memory secret.Store for testing
 type MemorySecrets struct {
-	Secrets map[string]string
+	Secrets map[string]map[string]string
 }
 
 var _ secret.Store = &MemorySecrets{}
 
 // GetSecretsForTarget implements secret.Store
 func (v *MemorySecrets) GetSecretsForTarget(name string) (map[string]string, error) {
-	return v.Secrets, nil
+	table, ok := v.Secrets[name]
+	if !ok {
+		return nil, nil
+	}
+	return table, nil
 }

secret/secret.go

@@ -3,7 +3,24 @@
 // any secrets that match it.
 package secret
 
+import "strings"
+
 // Store describes a type that can securely obtain secrets for services.
 type Store interface {
 	GetSecretsForTarget(name string) (map[string]string, error)
 }
+
+// GetPrefixedSecrets uses a Store to get a set of secrets that use a prefix.
+func GetPrefixedSecrets(s Store, path, prefix string) (map[string]string, error) {
+	all, err := s.GetSecretsForTarget(path)
+	if err != nil {
+		return nil, err
+	}
+	pass := make(map[string]string)
+	for k, v := range all {
+		if strings.HasPrefix(k, prefix) {
+			pass[strings.TrimPrefix(k, prefix)] = v
+		}
+	}
+	return pass, nil
+}

service/service.go

@@ -26,16 +26,17 @@ import (
 
 // Config specifies static configuration parameters (from CLI or environment)
 type Config struct {
-	Target        task.Repo
-	Hostname      string
-	SSH           bool
-	Directory     string
-	CheckInterval time.Duration
-	VaultAddress  string
-	VaultToken    string
-	VaultPath     string
-	VaultRenewal  time.Duration
-	VaultConfig   string
+	Target          task.Repo
+	Hostname        string
+	SSH             bool
+	Directory       string
+	PassEnvironment bool
+	CheckInterval   time.Duration
+	VaultAddress    string
+	VaultToken      string
+	VaultPath       string
+	VaultRenewal    time.Duration
+	VaultConfig     string
 }
 
 // App stores application state
@@ -119,7 +120,7 @@ func (app *App) Start(ctx context.Context) error {
 	// states and potentially retry in some circumstances. Pico should be the
 	// kind of service that barely goes down, only when absolutely necessary.
 
-	ce := executor.NewCommandExecutor(app.secrets)
+	ce := executor.NewCommandExecutor(app.secrets, app.config.PassEnvironment, app.config.VaultConfig, "GLOBAL_")
 	g.Go(func() error {
 		ce.Subscribe(app.bus)
 		return nil

task/target.go

@@ -52,7 +52,7 @@ type Target struct {
 
 // Execute runs the target's command in the specified directory with the
 // specified environment variables
-func (t *Target) Execute(dir string, env map[string]string, shutdown bool) (err error) {
+func (t *Target) Execute(dir string, env map[string]string, shutdown bool, inheritEnv bool) (err error) {
 	if env == nil {
 		env = make(map[string]string)
 	}
@@ -67,10 +67,10 @@ func (t *Target) Execute(dir string, env map[string]string, shutdown bool) (err
 		command = t.Up
 	}
 
-	return execute(dir, env, command)
+	return execute(dir, env, command, inheritEnv)
 }
 
-func execute(dir string, env map[string]string, command []string) (err error) {
+func execute(dir string, env map[string]string, command []string, inheritEnv bool) (err error) {
 	if len(command) == 0 {
 		return errors.New("attempt to execute target with empty command")
 	}
@@ -83,10 +83,14 @@ func execute(dir string, env map[string]string, command []string) (err error) {
 	cmd.Stdout = os.Stdout
 	cmd.Stderr = os.Stdout
 
-	cmd.Env = os.Environ()
+	var cmdEnv []string
+	if inheritEnv {
+		cmdEnv = os.Environ()
+	}
 	for k, v := range env {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
+		cmdEnv = append(cmd.Env, fmt.Sprintf("%s=%s", k, v))
 	}
+	cmd.Env = cmdEnv
 
 	return cmd.Run()
 }