Feat/vault manager crashproof

vault-manager/Dockerfile

@@ -10,11 +10,8 @@ RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -trimpath -ldflags='-s -w' -o
 
 FROM alpine:latest
 
-RUN apk add --no-cache ca-certificates \
-	&& addgroup -S app \
-	&& adduser -S -G app app
+RUN apk add --no-cache ca-certificates
 
 COPY --from=build /out/vault-manager /usr/local/bin/vault-manager
 
-USER app
 ENTRYPOINT ["vault-manager"]

vault-manager/README.md

@@ -47,7 +47,7 @@ vault-manager/
 
 ```bash
 cd vault-manager
-go run ./cmd/vault-manager --vault-addr=http://localhost:8200
+go run ./cmd/main.go --vault-addr=http://localhost:8200
 ```
 
 On first run against an uninitialized Vault, the unseal key and root token
@@ -56,14 +56,14 @@ afterward. On subsequent runs against an already-initialized Vault, set
 `VAULT_ROOT_TOKEN` in the environment instead:
 
 ```bash
-VAULT_ROOT_TOKEN=hvs.xxxxx go run ./cmd/vault-manager --vault-addr=http://localhost:8200
+VAULT_ROOT_TOKEN=hvs.xxxxx go run ./cmd/main.go --vault-addr=http://localhost:8200
 ```
 
 To provision once with root and then drop root privileges for the life of
 the process:
 
 ```bash
-go run ./cmd/vault-manager --vault-addr=http://localhost:8200 --secure
+go run ./cmd/main.go --vault-addr=http://localhost:8200 --secure
 ```
 
 ## CLI flags
@@ -78,6 +78,7 @@ go run ./cmd/vault-manager --vault-addr=http://localhost:8200 --secure
 | `--manager-role`     | `vault-manager-bootstrap`                                                                                                                        | AppRole name used for the `--secure` bootstrap handoff    |
 | `--manager-policy`   | `vault-manager-bootstrap-policy`                                                                                                                 | ACL policy name for the bootstrap manager AppRole         |
 | `--services`         | `persys-gateway,persys-scheduler,persysctl,compute-agent,persys-forgery,persys-services,persys-automation,persys-intelligence,persys-sdk`        | Comma-separated list of services to provision             |
+| `--bootstrap-file`   | `/var/lib/persys/vault/bootstrap.json`                                                                                                           | Persistent recovery credentials (unseal keys + auth)        |
 | `--secure`           | `false`                                                                                                                                          | Provision a bootstrap AppRole and revoke the root token after setup |
 
 ## Environment variables
@@ -125,6 +126,27 @@ logs (method, status code, duration, and any error) for each call.
 In docker compose, this is used by the `vault-manager` profile in
 `infra/docker/docker-compose.yml`.
 
+
+## Restart / recovery
+
+Unseal keys and auth credentials are written to `--bootstrap-file`
+(default `/var/lib/persys/vault/bootstrap.json`) on first init.
+
+After `docker compose down` (without `-v`) and `up` again:
+
+1. Vault comes back **sealed**.
+2. vault-manager loads the bootstrap file from the `vault_manager_data` volume.
+3. It unseals Vault with the stored keys, then authenticates (manager AppRole or root token).
+
+**Requirements:**
+
+- Named volume mounted at `/var/lib/persys/vault/` (as in compose).
+- Do **not** use `docker compose down -v` unless you intend to wipe recovery state.
+- The image entrypoint chowns the volume so the non-root `app` user can write `bootstrap.json`.
+
+If the bootstrap file is missing on an already-initialized Vault, set
+`VAULT_ROOT_TOKEN` once; after a successful run the file is rewritten.
+
 ## Operational notes
 
 - Single key-share initialization (`secret_shares: 1`, `secret_threshold:

vault-manager/cmd/main.go

@@ -2,9 +2,14 @@
 // and unseals Vault if needed, sets up the PKI CA chain, provisions
 // per-service AppRoles and policies, then serves a gRPC API so other
 // services can fetch or rotate their credentials at runtime.
+//
+// On first run the unseal key(s) and auth credentials are written to
+// --bootstrap-file so subsequent restarts (of vault-manager or of Vault
+// itself) can recover without operator intervention.
 package main
 
 import (
+	"errors"
 	"fmt"
 	"os"
 	"os/signal"
@@ -15,6 +20,7 @@ import (
 	"github.com/sirupsen/logrus"
 
 	"github.com/persys-dev/persys-cloud/vault-manager/internal/approle"
+	"github.com/persys-dev/persys-cloud/vault-manager/internal/bootstrap"
 	"github.com/persys-dev/persys-cloud/vault-manager/internal/config"
 	"github.com/persys-dev/persys-cloud/vault-manager/internal/pki"
 	"github.com/persys-dev/persys-cloud/vault-manager/internal/policy"
@@ -28,34 +34,23 @@ func main() {
 		config.Log.Fatal("no valid services found in --services")
 	}
 
-	baseClient, err := vaultclient.New(cfg.VaultAddr, "")
-	if err != nil {
-		config.Log.Fatal(err)
-	}
-	vaultclient.WaitUntilReady(baseClient)
+	// Sealed Vault is fine — we unseal from bootstrap state next.
+	vaultclient.WaitUntilReady(cfg.VaultAddr)
 
-	rootToken, err := bootstrapOrUnseal(cfg)
+	workClient, state, err := recoverOrBootstrap(cfg)
 	if err != nil {
 		config.Log.Fatal(err)
 	}
 
-	rootClient, err := vaultclient.New(cfg.VaultAddr, rootToken)
-	if err != nil {
+	if err := provision(workClient, cfg); err != nil {
 		config.Log.Fatal(err)
 	}
 
-	workClient := rootClient
-	if cfg.Secure {
-		config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token")
-		workClient, err = vaultclient.SwitchToSecure(rootClient, cfg)
-		if err != nil {
-			config.Log.Fatal(err)
-		}
-	}
-
-	if err := provision(workClient, cfg); err != nil {
-		config.Log.Fatal(err)
+	// Persist latest state after successful provision.
+	if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil {
+		config.Log.Fatalf("save bootstrap state to %s: %v", cfg.BootstrapFile, err)
 	}
+	config.Log.Printf("Bootstrap state saved to %s", cfg.BootstrapFile)
 
 	secrets, err := approle.GatherSecrets(workClient, cfg)
 	if err != nil {
@@ -77,39 +72,141 @@ func main() {
 	waitForShutdown()
 }
 
-// bootstrapOrUnseal initializes and unseals Vault if it hasn't been set up
-// yet, then returns the root token to use for provisioning: the freshly
-// generated one, or VAULT_ROOT_TOKEN if Vault was already initialized.
-func bootstrapOrUnseal(cfg *config.Config) (string, error) {
+// recoverOrBootstrap is the restart-safe entry point.
+//
+//  1. Load bootstrap file if present.
+//  2. Uninitialized Vault → init, unseal, persist keys + root token.
+//  3. Initialized + sealed → unseal with stored keys.
+//  4. Authenticate: manager AppRole (preferred) → stored root token → VAULT_ROOT_TOKEN.
+//  5. --secure without manager creds → hand off, revoke root, persist manager creds.
+func recoverOrBootstrap(cfg *config.Config) (*vault.Client, *bootstrap.State, error) {
+	state, err := bootstrap.Load(cfg.BootstrapFile)
+	if err != nil && !errors.Is(err, bootstrap.ErrNotFound) {
+		return nil, nil, fmt.Errorf("load bootstrap state from %s: %w", cfg.BootstrapFile, err)
+	}
+	if state == nil {
+		state = &bootstrap.State{}
+		config.Log.Printf("No bootstrap state at %s (first run or missing volume)", cfg.BootstrapFile)
+	} else {
+		config.Log.Printf("Loaded bootstrap state from %s (unseal_keys=%d manager=%v root=%v)",
+			cfg.BootstrapFile, len(state.UnsealKeys), state.HasManagerCreds(), state.RootToken != "")
+	}
+
 	initialized, err := vaultclient.IsInitialized(cfg.VaultAddr)
 	if err != nil {
-		return "", err
+		return nil, nil, err
 	}
 
 	if !initialized {
-		config.Log.Println("Vault not initialized. Initializing...")
-		initResult, err := vaultclient.Initialize(cfg.VaultAddr)
+		return firstTimeInit(cfg, state)
+	}
+
+	if err := vaultclient.EnsureUnsealed(cfg.VaultAddr, state.UnsealKeys); err != nil {
+		return nil, nil, err
+	}
+
+	client, err := authenticate(cfg, state)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if cfg.Secure && !state.HasManagerCreds() {
+		config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token")
+		handoff, err := vaultclient.SwitchToSecure(client, cfg)
 		if err != nil {
-			return "", err
+			return nil, nil, err
 		}
-		fmt.Println("Vault initialized credentials (store securely):")
-		fmt.Printf("unseal_key: %s\n", initResult.UnsealKey)
-		fmt.Printf("root_token: %s\n", initResult.RootToken)
-		if err := vaultclient.Unseal(cfg.VaultAddr, initResult.UnsealKey); err != nil {
-			return "", err
+		state.ManagerRoleID = handoff.RoleID
+		state.ManagerSecretID = handoff.SecretID
+		state.RootToken = ""
+		client = handoff.Client
+	}
+
+	return client, state, nil
+}
+
+func firstTimeInit(cfg *config.Config, state *bootstrap.State) (*vault.Client, *bootstrap.State, error) {
+	config.Log.Println("Vault not initialized. Initializing...")
+	initResult, err := vaultclient.Initialize(cfg.VaultAddr)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	fmt.Println("Vault initialized credentials (also saved to bootstrap file):")
+	fmt.Printf("unseal_keys: %v\n", initResult.UnsealKeys)
+	fmt.Printf("root_token: %s\n", initResult.RootToken)
+
+	if err := vaultclient.UnsealAll(cfg.VaultAddr, initResult.UnsealKeys); err != nil {
+		return nil, nil, err
+	}
+	config.Log.Println("Vault initialized and unsealed.")
+
+	state.UnsealKeys = initResult.UnsealKeys
+	state.RootToken = initResult.RootToken
+
+	// Persist immediately so a crash between init and provision is recoverable.
+	if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil {
+		return nil, nil, fmt.Errorf("save bootstrap state after init to %s: %w", cfg.BootstrapFile, err)
+	}
+	config.Log.Printf("Bootstrap state saved to %s", cfg.BootstrapFile)
+
+	client, err := vaultclient.New(cfg.VaultAddr, initResult.RootToken)
+	if err != nil {
+		return nil, nil, err
+	}
+
+	if cfg.Secure {
+		config.Log.Println("--secure enabled: creating bootstrap AppRole and switching off root token")
+		handoff, err := vaultclient.SwitchToSecure(client, cfg)
+		if err != nil {
+			return nil, nil, err
+		}
+		state.ManagerRoleID = handoff.RoleID
+		state.ManagerSecretID = handoff.SecretID
+		state.RootToken = ""
+		client = handoff.Client
+
+		if err := bootstrap.Save(cfg.BootstrapFile, state); err != nil {
+			return nil, nil, fmt.Errorf("save bootstrap state after secure handoff: %w", err)
 		}
-		config.Log.Println("Vault initialized and unsealed.")
-		return initResult.RootToken, nil
 	}
 
-	rootToken := strings.TrimSpace(os.Getenv("VAULT_ROOT_TOKEN"))
+	return client, state, nil
+}
+
+// authenticate picks the best available credential source.
+//
+// Priority:
+//  1. Manager AppRole from bootstrap file (restart after --secure)
+//  2. Root token from bootstrap file
+//  3. VAULT_ROOT_TOKEN environment variable
+func authenticate(cfg *config.Config, state *bootstrap.State) (*vault.Client, error) {
+	if state.HasManagerCreds() {
+		config.Log.Println("Authenticating with stored manager AppRole credentials")
+		client, err := vaultclient.LoginAppRole(cfg.VaultAddr, state.ManagerRoleID, state.ManagerSecretID)
+		if err != nil {
+			return nil, fmt.Errorf("manager AppRole login: %w", err)
+		}
+		return client, nil
+	}
+
+	rootToken := strings.TrimSpace(state.RootToken)
 	if rootToken == "" {
-		return "", fmt.Errorf("VAULT_ROOT_TOKEN required when Vault is already initialized")
+		rootToken = strings.TrimSpace(os.Getenv("VAULT_ROOT_TOKEN"))
 	}
-	return rootToken, nil
+	if rootToken == "" {
+		return nil, fmt.Errorf(
+			"vault is initialized but no credentials available: "+
+				"ensure %s contains unseal_keys and root_token/manager creds, "+
+				"or set VAULT_ROOT_TOKEN (file missing usually means the volume was not persisted)",
+			cfg.BootstrapFile,
+		)
+	}
+
+	config.Log.Println("Authenticating with root token")
+	return vaultclient.New(cfg.VaultAddr, rootToken)
 }
 
-// provision ensures the PKI chain, service policies, and AppRoles all exist.
 func provision(client *vault.Client, cfg *config.Config) error {
 	if err := pki.Ensure(client, cfg); err != nil {
 		return err

vault-manager/internal/bootstrap/store.go

@@ -0,0 +1,74 @@
+// Package bootstrap persists the credentials needed to recover after a
+// vault-manager (or Vault itself) restart: unseal keys and either a root
+// token or the bootstrap-manager AppRole credentials.
+package bootstrap
+
+import (
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+)
+
+// State is the on-disk recovery record written after a successful bootstrap.
+type State struct {
+	UnsealKeys []string `json:"unseal_keys"`
+
+	// RootToken is kept when running without --secure. Cleared after a
+	// successful --secure handoff so the file never holds a live root token.
+	RootToken string `json:"root_token,omitempty"`
+
+	// Manager AppRole credentials used when --secure is enabled (and on
+	// subsequent restarts of a previously secured deployment).
+	ManagerRoleID   string `json:"manager_role_id,omitempty"`
+	ManagerSecretID string `json:"manager_secret_id,omitempty"`
+}
+
+// ErrNotFound is returned by Load when the bootstrap file does not exist.
+var ErrNotFound = errors.New("bootstrap state file not found")
+
+// Load reads and decodes the bootstrap state from path.
+// Returns ErrNotFound if the file does not exist.
+func Load(path string) (*State, error) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return nil, ErrNotFound
+		}
+		return nil, err
+	}
+
+	var s State
+	if err := json.Unmarshal(b, &s); err != nil {
+		return nil, err
+	}
+	return &s, nil
+}
+
+// Save writes state to path atomically (tmp + rename) with restrictive perms.
+func Save(path string, s *State) error {
+	dir := filepath.Dir(path)
+	if err := os.MkdirAll(dir, 0700); err != nil {
+		return err
+	}
+
+	tmp := path + ".tmp"
+	b, err := json.MarshalIndent(s, "", "  ")
+	if err != nil {
+		return err
+	}
+	if err := os.WriteFile(tmp, b, 0600); err != nil {
+		return err
+	}
+	return os.Rename(tmp, path)
+}
+
+// HasManagerCreds reports whether the state holds usable manager AppRole credentials.
+func (s *State) HasManagerCreds() bool {
+	return s != nil && s.ManagerRoleID != "" && s.ManagerSecretID != ""
+}
+
+// HasUnsealKeys reports whether the state holds at least one unseal key.
+func (s *State) HasUnsealKeys() bool {
+	return s != nil && len(s.UnsealKeys) > 0
+}