diff --git a/vault-manager/Dockerfile b/vault-manager/Dockerfile index 53a381e..835d7a1 100644 --- a/vault-manager/Dockerfile +++ b/vault-manager/Dockerfile @@ -10,11 +10,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o FROM alpine:latest -RUN apk add --no-cache ca-certificates \ - && addgroup -S app \ - && adduser -S -G app app +RUN apk add --no-cache ca-certificates COPY --from=build /out/vault-manager /usr/local/bin/vault-manager -USER app ENTRYPOINT ["vault-manager"] diff --git a/vault-manager/README.md b/vault-manager/README.md index afaaa4e..23b4685 100644 --- a/vault-manager/README.md +++ b/vault-manager/README.md @@ -47,7 +47,7 @@ vault-manager/ ```bash cd vault-manager -go run ./cmd/vault-manager --vault-addr=http://localhost:8200 +go run ./cmd/main.go --vault-addr=http://localhost:8200 ``` On first run against an uninitialized Vault, the unseal key and root token @@ -56,14 +56,14 @@ afterward. On subsequent runs against an already-initialized Vault, set `VAULT_ROOT_TOKEN` in the environment instead: ```bash -VAULT_ROOT_TOKEN=hvs.xxxxx go run ./cmd/vault-manager --vault-addr=http://localhost:8200 +VAULT_ROOT_TOKEN=hvs.xxxxx go run ./cmd/main.go --vault-addr=http://localhost:8200 ``` To provision once with root and then drop root privileges for the life of the process: ```bash -go run ./cmd/vault-manager --vault-addr=http://localhost:8200 --secure +go run ./cmd/main.go --vault-addr=http://localhost:8200 --secure ``` ## CLI flags @@ -78,6 +78,7 @@ go run ./cmd/vault-manager --vault-addr=http://localhost:8200 --secure | `--manager-role` | `vault-manager-bootstrap` | AppRole name used for the `--secure` bootstrap handoff | | `--manager-policy` | `vault-manager-bootstrap-policy` | ACL policy name for the bootstrap manager AppRole | | `--services` | `persys-gateway,persys-scheduler,persysctl,compute-agent,persys-forgery,persys-services,persys-automation,persys-intelligence,persys-sdk` | Comma-separated list of services to provision | +| `--bootstrap-file` | `/var/lib/persys/vault/bootstrap.json` | Persistent recovery credentials (unseal keys + auth) | | `--secure` | `false` | Provision a bootstrap AppRole and revoke the root token after setup | ## Environment variables @@ -125,6 +126,27 @@ logs (method, status code, duration, and any error) for each call. In docker compose, this is used by the `vault-manager` profile in `infra/docker/docker-compose.yml`. + +## Restart / recovery + +Unseal keys and auth credentials are written to `--bootstrap-file` +(default `/var/lib/persys/vault/bootstrap.json`) on first init. + +After `docker compose down` (without `-v`) and `up` again: + +1. Vault comes back **sealed**. +2. vault-manager loads the bootstrap file from the `vault_manager_data` volume. +3. It unseals Vault with the stored keys, then authenticates (manager AppRole or root token). + +**Requirements:** + +- Named volume mounted at `/var/lib/persys/vault/` (as in compose). +- Do **not** use `docker compose down -v` unless you intend to wipe recovery state. +- The image entrypoint chowns the volume so the non-root `app` user can write `bootstrap.json`. + +If the bootstrap file is missing on an already-initialized Vault, set +`VAULT_ROOT_TOKEN` once; after a successful run the file is rewritten. + ## Operational notes - Single key-share initialization (`secret_shares: 1`, `secret_threshold: diff --git a/vault-manager/cmd/main.go b/vault-manager/cmd/main.go index b1a38eb..e44eb64 100644 --- a/vault-manager/cmd/main.go +++ b/vault-manager/cmd/main.go @@ -2,9 +2,14 @@ // and unseals Vault if needed, sets up the PKI CA chain, provisions // per-service AppRoles and policies, then serves a gRPC API so other // services can fetch or rotate their credentials at runtime. +// +// On first run the unseal key(s) and auth credentials are written to +// --bootstrap-file so subsequent restarts (of vault-manager or of Vault +// itself) can recover without operator intervention. package main import ( + "errors" "fmt" "os" "os/signal" @@ -15,6 +20,7 @@ import ( "github.com/sirupsen/logrus" "github.com/persys-dev/persys-cloud/vault-manager/internal/approle" + "github.com/persys-dev/persys-cloud/vault-manager/internal/bootstrap" "github.com/persys-dev/persys-cloud/vault-manager/internal/config" "github.com/persys-dev/persys-cloud/vault-manager/internal/pki" "github.com/persys-dev/persys-cloud/vault-manager/internal/policy" @@ -28,34 +34,23 @@ func main() { config.Log.Fatal("no valid services found in --services") } - baseClient, err := vaultclient.New(cfg.VaultAddr, "") - if err != nil { - config.Log.Fatal(err) - } - vaultclient.WaitUntilReady(baseClient) + // Sealed Vault is fine — we unseal from bootstrap state next. + vaultclient.WaitUntilReady(cfg.VaultAddr) - rootToken, err := bootstrapOrUnseal(cfg) + workClient, state, err := recoverOrBootstrap(cfg) if err != nil { config.Log.Fatal(err) } - rootClient, err := vaultclient.New(cfg.VaultAddr, rootToken) - if err != nil { + if err := provision(workClient, cfg); err != nil { config.Log.Fatal(err) } - workClient := rootClient - if cfg.Secure { - config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token") - workClient, err = vaultclient.SwitchToSecure(rootClient, cfg) - if err != nil { - config.Log.Fatal(err) - } - } - - if err := provision(workClient, cfg); err != nil { - config.Log.Fatal(err) + // Persist latest state after successful provision. + if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil { + config.Log.Fatalf("save bootstrap state to %s: %v", cfg.BootstrapFile, err) } + config.Log.Printf("Bootstrap state saved to %s", cfg.BootstrapFile) secrets, err := approle.GatherSecrets(workClient, cfg) if err != nil { @@ -77,39 +72,141 @@ func main() { waitForShutdown() } -// bootstrapOrUnseal initializes and unseals Vault if it hasn't been set up -// yet, then returns the root token to use for provisioning: the freshly -// generated one, or VAULT_ROOT_TOKEN if Vault was already initialized. -func bootstrapOrUnseal(cfg *config.Config) (string, error) { +// recoverOrBootstrap is the restart-safe entry point. +// +// 1. Load bootstrap file if present. +// 2. Uninitialized Vault → init, unseal, persist keys + root token. +// 3. Initialized + sealed → unseal with stored keys. +// 4. Authenticate: manager AppRole (preferred) → stored root token → VAULT_ROOT_TOKEN. +// 5. --secure without manager creds → hand off, revoke root, persist manager creds. +func recoverOrBootstrap(cfg *config.Config) (*vault.Client, *bootstrap.State, error) { + state, err := bootstrap.Load(cfg.BootstrapFile) + if err != nil && !errors.Is(err, bootstrap.ErrNotFound) { + return nil, nil, fmt.Errorf("load bootstrap state from %s: %w", cfg.BootstrapFile, err) + } + if state == nil { + state = &bootstrap.State{} + config.Log.Printf("No bootstrap state at %s (first run or missing volume)", cfg.BootstrapFile) + } else { + config.Log.Printf("Loaded bootstrap state from %s (unseal_keys=%d manager=%v root=%v)", + cfg.BootstrapFile, len(state.UnsealKeys), state.HasManagerCreds(), state.RootToken != "") + } + initialized, err := vaultclient.IsInitialized(cfg.VaultAddr) if err != nil { - return "", err + return nil, nil, err } if !initialized { - config.Log.Println("Vault not initialized. Initializing...") - initResult, err := vaultclient.Initialize(cfg.VaultAddr) + return firstTimeInit(cfg, state) + } + + if err := vaultclient.EnsureUnsealed(cfg.VaultAddr, state.UnsealKeys); err != nil { + return nil, nil, err + } + + client, err := authenticate(cfg, state) + if err != nil { + return nil, nil, err + } + + if cfg.Secure && !state.HasManagerCreds() { + config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token") + handoff, err := vaultclient.SwitchToSecure(client, cfg) if err != nil { - return "", err + return nil, nil, err } - fmt.Println("Vault initialized credentials (store securely):") - fmt.Printf("unseal_key: %s\n", initResult.UnsealKey) - fmt.Printf("root_token: %s\n", initResult.RootToken) - if err := vaultclient.Unseal(cfg.VaultAddr, initResult.UnsealKey); err != nil { - return "", err + state.ManagerRoleID = handoff.RoleID + state.ManagerSecretID = handoff.SecretID + state.RootToken = "" + client = handoff.Client + } + + return client, state, nil +} + +func firstTimeInit(cfg *config.Config, state *bootstrap.State) (*vault.Client, *bootstrap.State, error) { + config.Log.Println("Vault not initialized. Initializing...") + initResult, err := vaultclient.Initialize(cfg.VaultAddr) + if err != nil { + return nil, nil, err + } + + fmt.Println("Vault initialized credentials (also saved to bootstrap file):") + fmt.Printf("unseal_keys: %v\n", initResult.UnsealKeys) + fmt.Printf("root_token: %s\n", initResult.RootToken) + + if err := vaultclient.UnsealAll(cfg.VaultAddr, initResult.UnsealKeys); err != nil { + return nil, nil, err + } + config.Log.Println("Vault initialized and unsealed.") + + state.UnsealKeys = initResult.UnsealKeys + state.RootToken = initResult.RootToken + + // Persist immediately so a crash between init and provision is recoverable. + if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil { + return nil, nil, fmt.Errorf("save bootstrap state after init to %s: %w", cfg.BootstrapFile, err) + } + config.Log.Printf("Bootstrap state saved to %s", cfg.BootstrapFile) + + client, err := vaultclient.New(cfg.VaultAddr, initResult.RootToken) + if err != nil { + return nil, nil, err + } + + if cfg.Secure { + config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token") + handoff, err := vaultclient.SwitchToSecure(client, cfg) + if err != nil { + return nil, nil, err + } + state.ManagerRoleID = handoff.RoleID + state.ManagerSecretID = handoff.SecretID + state.RootToken = "" + client = handoff.Client + + if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil { + return nil, nil, fmt.Errorf("save bootstrap state after secure handoff: %w", err) } - config.Log.Println("Vault initialized and unsealed.") - return initResult.RootToken, nil } - rootToken := strings.TrimSpace(os.Getenv("VAULT_ROOT_TOKEN")) + return client, state, nil +} + +// authenticate picks the best available credential source. +// +// Priority: +// 1. Manager AppRole from bootstrap file (restart after --secure) +// 2. Root token from bootstrap file +// 3. VAULT_ROOT_TOKEN environment variable +func authenticate(cfg *config.Config, state *bootstrap.State) (*vault.Client, error) { + if state.HasManagerCreds() { + config.Log.Println("Authenticating with stored manager AppRole credentials") + client, err := vaultclient.LoginAppRole(cfg.VaultAddr, state.ManagerRoleID, state.ManagerSecretID) + if err != nil { + return nil, fmt.Errorf("manager AppRole login: %w", err) + } + return client, nil + } + + rootToken := strings.TrimSpace(state.RootToken) if rootToken == "" { - return "", fmt.Errorf("VAULT_ROOT_TOKEN required when Vault is already initialized") + rootToken = strings.TrimSpace(os.Getenv("VAULT_ROOT_TOKEN")) } - return rootToken, nil + if rootToken == "" { + return nil, fmt.Errorf( + "vault is initialized but no credentials available: "+ + "ensure %s contains unseal_keys and root_token/manager creds, "+ + "or set VAULT_ROOT_TOKEN (file missing usually means the volume was not persisted)", + cfg.BootstrapFile, + ) + } + + config.Log.Println("Authenticating with root token") + return vaultclient.New(cfg.VaultAddr, rootToken) } -// provision ensures the PKI chain, service policies, and AppRoles all exist. func provision(client *vault.Client, cfg *config.Config) error { if err := pki.Ensure(client, cfg); err != nil { return err diff --git a/vault-manager/internal/bootstrap/store.go b/vault-manager/internal/bootstrap/store.go new file mode 100644 index 0000000..41eb842 --- /dev/null +++ b/vault-manager/internal/bootstrap/store.go @@ -0,0 +1,74 @@ +// Package bootstrap persists the credentials needed to recover after a +// vault-manager (or Vault itself) restart: unseal keys and either a root +// token or the bootstrap-manager AppRole credentials. +package bootstrap + +import ( + "encoding/json" + "errors" + "os" + "path/filepath" +) + +// State is the on-disk recovery record written after a successful bootstrap. +type State struct { + UnsealKeys []string `json:"unseal_keys"` + + // RootToken is kept when running without --secure. Cleared after a + // successful --secure handoff so the file never holds a live root token. + RootToken string `json:"root_token,omitempty"` + + // Manager AppRole credentials used when --secure is enabled (and on + // subsequent restarts of a previously secured deployment). + ManagerRoleID string `json:"manager_role_id,omitempty"` + ManagerSecretID string `json:"manager_secret_id,omitempty"` +} + +// ErrNotFound is returned by Load when the bootstrap file does not exist. +var ErrNotFound = errors.New("bootstrap state file not found") + +// Load reads and decodes the bootstrap state from path. +// Returns ErrNotFound if the file does not exist. +func Load(path string) (*State, error) { + b, err := os.ReadFile(path) + if err != nil { + if os.IsNotExist(err) { + return nil, ErrNotFound + } + return nil, err + } + + var s State + if err := json.Unmarshal(b, &s); err != nil { + return nil, err + } + return &s, nil +} + +// Save writes state to path atomically (tmp + rename) with restrictive perms. +func Save(path string, s *State) error { + dir := filepath.Dir(path) + if err := os.MkdirAll(dir, 0700); err != nil { + return err + } + + tmp := path + ".tmp" + b, err := json.MarshalIndent(s, "", " ") + if err != nil { + return err + } + if err := os.WriteFile(tmp, b, 0600); err != nil { + return err + } + return os.Rename(tmp, path) +} + +// HasManagerCreds reports whether the state holds usable manager AppRole credentials. +func (s *State) HasManagerCreds() bool { + return s != nil && s.ManagerRoleID != "" && s.ManagerSecretID != "" +} + +// HasUnsealKeys reports whether the state holds at least one unseal key. +func (s *State) HasUnsealKeys() bool { + return s != nil && len(s.UnsealKeys) > 0 +} diff --git a/vault-manager/internal/config/config.go b/vault-manager/internal/config/config.go index 2967d30..084f750 100644 --- a/vault-manager/internal/config/config.go +++ b/vault-manager/internal/config/config.go @@ -10,6 +10,7 @@ import ( ) const ( + DefaultBootstrapFile = "/opt/persys/vault/bootstrap.json" DefaultVaultAddr = "https://vault:8200" DefaultPKIRootMount = "pki" DefaultPKIIntermediateMount = "pki_int" @@ -17,7 +18,7 @@ const ( DefaultIntCommonName = "Persys Cloud Intermediate CA" DefaultManagerRoleName = "vault-manager-bootstrap" DefaultManagerPolicyName = "vault-manager-bootstrap-policy" - DefaultServicesCSV = "persys-gateway,persys-scheduler,persysctl,compute-agent,persys-forgery,persys-services,persys-automation,persys-intelligence,persys-sdk" + DefaultServicesCSV = "persys-gateway,persys-scheduler,persysctl,compute-agent,persys-forgery,persys-services,persys-automation,persys-intelligence,persys-dashboard,persys-sdk" GRPCListenAddr = ":50069" ) @@ -34,6 +35,7 @@ type Config struct { ManagerRoleName string ManagerPolicyName string ServiceNames []string + BootstrapFile string Secure bool } @@ -52,6 +54,7 @@ func ParseFlags() *Config { flag.StringVar(&cfg.ManagerRoleName, "manager-role", DefaultManagerRoleName, "Bootstrap manager AppRole name") flag.StringVar(&cfg.ManagerPolicyName, "manager-policy", DefaultManagerPolicyName, "Bootstrap manager policy name") flag.StringVar(&servicesCSV, "services", DefaultServicesCSV, "Comma-separated service names to provision") + flag.StringVar(&cfg.BootstrapFile, "bootstrap-file", DefaultBootstrapFile, "bootstrap credential store") flag.Parse() cfg.ServiceNames = ParseServiceNames(servicesCSV) diff --git a/vault-manager/internal/vaultclient/client.go b/vault-manager/internal/vaultclient/client.go index f393313..408cf82 100644 --- a/vault-manager/internal/vaultclient/client.go +++ b/vault-manager/internal/vaultclient/client.go @@ -19,8 +19,8 @@ import ( // InitResult holds the credentials returned by a fresh Vault initialization. type InitResult struct { - RootToken string - UnsealKey string + RootToken string + UnsealKeys []string } // New creates a Vault API client pointed at addr, optionally authenticated @@ -38,12 +38,18 @@ func New(addr, token string) (*vault.Client, error) { return client, nil } -// WaitUntilReady blocks until Vault responds to a health check. -func WaitUntilReady(client *vault.Client) { +// WaitUntilReady blocks until Vault answers the seal-status endpoint. +// Sealed Vault is considered ready — callers unseal explicitly afterward. +// Using /sys/health is wrong here: sealed nodes return 503 and the API +// client treats that as an error, which would hang forever after a Vault restart. +func WaitUntilReady(addr string) { for { - _, err := client.Sys().Health() + resp, err := http.Get(addr + "/v1/sys/seal-status") if err == nil { - return + resp.Body.Close() + if resp.StatusCode >= 200 && resp.StatusCode < 500 { + return + } } config.Log.Println("Waiting for Vault...") time.Sleep(2 * time.Second) @@ -66,7 +72,7 @@ func IsInitialized(addr string) (bool, error) { } // Initialize performs a single-shard Vault initialization and returns the -// root token and unseal key. +// root token and unseal key(s). func Initialize(addr string) (*InitResult, error) { body := map[string]interface{}{ "secret_shares": 1, @@ -112,8 +118,8 @@ func Initialize(addr string) (*InitResult, error) { } return &InitResult{ - RootToken: data.RootToken, - UnsealKey: unsealKeys[0], + RootToken: data.RootToken, + UnsealKeys: unsealKeys, }, nil } @@ -144,3 +150,86 @@ func Unseal(addr, unsealKey string) error { } return nil } + +// IsSealed reports whether the Vault at addr is currently sealed. +func IsSealed(addr string) (bool, error) { + resp, err := http.Get(addr + "/v1/sys/seal-status") + if err != nil { + return false, err + } + defer resp.Body.Close() + + var out struct { + Sealed bool `json:"sealed"` + } + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + return false, err + } + return out.Sealed, nil +} + +// UnsealAll submits every key in keys until Vault reports unsealed, or +// returns an error if it remains sealed after all keys. +func UnsealAll(addr string, keys []string) error { + if len(keys) == 0 { + return errors.New("no unseal keys provided") + } + for _, key := range keys { + if strings.TrimSpace(key) == "" { + continue + } + if err := Unseal(addr, key); err != nil { + return err + } + sealed, err := IsSealed(addr) + if err != nil { + return err + } + if !sealed { + return nil + } + } + return errors.New("vault still sealed after all keys") +} + +// EnsureUnsealed unseals Vault if it is currently sealed, using the given keys. +// It is a no-op when Vault is already unsealed. +func EnsureUnsealed(addr string, keys []string) error { + sealed, err := IsSealed(addr) + if err != nil { + return err + } + if !sealed { + config.Log.Println("Vault is already unsealed.") + return nil + } + if len(keys) == 0 { + return errors.New("vault is sealed but no unseal keys are available in bootstrap state") + } + config.Log.Println("Vault is sealed; unsealing with stored keys...") + if err := UnsealAll(addr, keys); err != nil { + return err + } + config.Log.Println("Vault unsealed.") + return nil +} + +// LoginAppRole authenticates with role_id/secret_id and returns a client +// holding the resulting token. +func LoginAppRole(addr, roleID, secretID string) (*vault.Client, error) { + base, err := New(addr, "") + if err != nil { + return nil, err + } + loginSecret, err := base.Logical().Write("auth/approle/login", map[string]interface{}{ + "role_id": roleID, + "secret_id": secretID, + }) + if err != nil { + return nil, fmt.Errorf("approle login failed: %w", err) + } + if loginSecret == nil || loginSecret.Auth == nil || loginSecret.Auth.ClientToken == "" { + return nil, errors.New("approle login returned empty token") + } + return New(addr, loginSecret.Auth.ClientToken) +} diff --git a/vault-manager/internal/vaultclient/secure.go b/vault-manager/internal/vaultclient/secure.go index 7097ca0..7253ea2 100644 --- a/vault-manager/internal/vaultclient/secure.go +++ b/vault-manager/internal/vaultclient/secure.go @@ -11,10 +11,18 @@ import ( "github.com/persys-dev/persys-cloud/vault-manager/internal/policy" ) +// SecureHandoff holds the result of a successful root → manager AppRole switch. +type SecureHandoff struct { + Client *vault.Client + RoleID string + SecretID string +} + // SwitchToSecure provisions a bootstrap AppRole, logs in with it, then // revokes the root token so all further provisioning runs without root -// privileges. -func SwitchToSecure(rootClient *vault.Client, cfg *config.Config) (*vault.Client, error) { +// privileges. The returned SecureHandoff includes the manager credentials +// so the caller can persist them for restart recovery. +func SwitchToSecure(rootClient *vault.Client, cfg *config.Config) (*SecureHandoff, error) { if err := approle.EnsureAuthMethod(rootClient); err != nil { return nil, err } @@ -23,9 +31,11 @@ func SwitchToSecure(rootClient *vault.Client, cfg *config.Config) (*vault.Client } _, err := rootClient.Logical().Write("auth/approle/role/"+cfg.ManagerRoleName, map[string]interface{}{ - "token_policies": []string{cfg.ManagerPolicyName}, - "token_ttl": "1h", - "token_max_ttl": "4h", + "token_policies": []string{cfg.ManagerPolicyName}, + "token_ttl": "1h", + "token_max_ttl": "4h", + "secret_id_ttl": "0", // unlimited — required for restart re-login + "secret_id_num_uses": 0, }) if err != nil { return nil, fmt.Errorf("ensure manager approle: %w", err) @@ -57,5 +67,9 @@ func SwitchToSecure(rootClient *vault.Client, cfg *config.Config) (*vault.Client } config.Log.Println("Root token revoked after secure AppRole handoff.") - return secureClient, nil + return &SecureHandoff{ + Client: secureClient, + RoleID: roleID, + SecretID: secretID, + }, nil }