backport of commit 2f43534

.changelog/23577.txt

@@ -0,0 +1,15 @@
+```release-note:improvement
+keyring: Added support for prepublishing keys
+```
+
+```release-note:bug
+keyring: Fixed a bug where periodic key rotation would not occur
+```
+
+```release-note:bug
+keyring: Fixed a bug where keys could be garbage collected before workload identities expire
+```
+
+```release-note:bug
+keyring: Fixed a bug where keys would never exit the "rekeying" state after a rotation with the `-full` flag
+```

api/keyring.go

@@ -34,16 +34,18 @@ type RootKeyMeta struct {
 	CreateIndex uint64
 	ModifyIndex uint64
 	State       RootKeyState
+	PublishTime int64
 }
 
 // RootKeyState enum describes the lifecycle of a root key.
 type RootKeyState string
 
 const (
-	RootKeyStateInactive   RootKeyState = "inactive"
-	RootKeyStateActive                  = "active"
-	RootKeyStateRekeying                = "rekeying"
-	RootKeyStateDeprecated              = "deprecated"
+	RootKeyStateInactive     RootKeyState = "inactive"
+	RootKeyStateActive                    = "active"
+	RootKeyStateRekeying                  = "rekeying"
+	RootKeyStateDeprecated                = "deprecated"
+	RootKeyStatePrepublished              = "prepublished"
 )
 
 // List lists all the keyring metadata
@@ -78,6 +80,9 @@ func (k *Keyring) Rotate(opts *KeyringRotateOptions, w *WriteOptions) (*RootKeyM
 		if opts.Full {
 			qp.Set("full", "true")
 		}
+		if opts.PublishTime > 0 {
+			qp.Set("publish_time", fmt.Sprintf("%d", opts.PublishTime))
+		}
 	}
 	resp := &struct{ Key *RootKeyMeta }{}
 	wm, err := k.client.put("/v1/operator/keyring/rotate?"+qp.Encode(), nil, resp, w)
@@ -86,6 +91,7 @@ func (k *Keyring) Rotate(opts *KeyringRotateOptions, w *WriteOptions) (*RootKeyM
 
 // KeyringRotateOptions are parameters for the Rotate API
 type KeyringRotateOptions struct {
-	Full      bool
-	Algorithm EncryptionAlgorithm
+	Full        bool
+	Algorithm   EncryptionAlgorithm
+	PublishTime int64
 }

command/agent/keyring_endpoint.go

@@ -6,6 +6,7 @@ package agent
 import (
 	"fmt"
 	"net/http"
+	"strconv"
 	"strings"
 	"time"
 
@@ -167,6 +168,15 @@ func (s *HTTPServer) keyringRotateRequest(resp http.ResponseWriter, req *http.Re
 		args.Full = true
 	}
 
+	ptRaw := query.Get("publish_time")
+	if ptRaw != "" {
+		publishTime, err := strconv.ParseInt(ptRaw, 10, 64)
+		if err != nil {
+			return nil, fmt.Errorf("invalid publish_time: %w", err)
+		}
+		args.PublishTime = publishTime
+	}
+
 	var out structs.KeyringRotateRootKeyResponse
 	if err := s.agent.RPC("Keyring.Rotate", &args, &out); err != nil {
 		return nil, err

command/agent/keyring_endpoint_test.go

@@ -4,6 +4,7 @@
 package agent
 
 import (
+	"fmt"
 	"net/http"
 	"net/http/httptest"
 	"strconv"
@@ -13,7 +14,6 @@ import (
 
 	"github.com/go-jose/go-jose/v3"
 	"github.com/shoenig/test/must"
-	"github.com/stretchr/testify/require"
 
 	"github.com/hashicorp/nomad/ci"
 	"github.com/hashicorp/nomad/nomad/structs"
@@ -29,57 +29,83 @@ func TestHTTP_Keyring_CRUD(t *testing.T) {
 		// List (get bootstrap key)
 
 		req, err := http.NewRequest(http.MethodGet, "/v1/operator/keyring/keys", nil)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		obj, err := s.Server.KeyringRequest(respW, req)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		listResp := obj.([]*structs.RootKeyMeta)
-		require.Len(t, listResp, 1)
-		oldKeyID := listResp[0].KeyID
+		must.Len(t, 1, listResp)
+		key0 := listResp[0].KeyID
 
 		// Rotate
 
 		req, err = http.NewRequest(http.MethodPut, "/v1/operator/keyring/rotate", nil)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		obj, err = s.Server.KeyringRequest(respW, req)
-		require.NoError(t, err)
-		require.NotZero(t, respW.HeaderMap.Get("X-Nomad-Index"))
+		must.NoError(t, err)
+		must.NotEq(t, "", respW.HeaderMap.Get("X-Nomad-Index"))
 		rotateResp := obj.(structs.KeyringRotateRootKeyResponse)
-		require.NotNil(t, rotateResp.Key)
-		require.True(t, rotateResp.Key.Active())
-		newID1 := rotateResp.Key.KeyID
+		must.NotNil(t, rotateResp.Key)
+		must.True(t, rotateResp.Key.IsActive())
+		key1 := rotateResp.Key.KeyID
+
+		// Rotate with prepublish
+
+		publishTime := time.Now().Add(24 * time.Hour).UnixNano()
+		req, err = http.NewRequest(http.MethodPut,
+			fmt.Sprintf("/v1/operator/keyring/rotate?publish_time=%d", publishTime), nil)
+		must.NoError(t, err)
+		obj, err = s.Server.KeyringRequest(respW, req)
+		must.NoError(t, err)
+		must.NotEq(t, "", respW.HeaderMap.Get("X-Nomad-Index"))
+		rotateResp = obj.(structs.KeyringRotateRootKeyResponse)
+		must.NotNil(t, rotateResp.Key)
+		must.True(t, rotateResp.Key.IsPrepublished())
+		key2 := rotateResp.Key.KeyID
 
 		// List
 
 		req, err = http.NewRequest(http.MethodGet, "/v1/operator/keyring/keys", nil)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		obj, err = s.Server.KeyringRequest(respW, req)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		listResp = obj.([]*structs.RootKeyMeta)
-		require.Len(t, listResp, 2)
+		must.Len(t, 3, listResp)
 		for _, key := range listResp {
-			if key.KeyID == newID1 {
-				require.True(t, key.Active(), "new key should be active")
-			} else {
-				require.False(t, key.Active(), "initial key should be inactive")
+			switch key.KeyID {
+			case key0:
+				must.True(t, key.IsInactive(), must.Sprint("initial key should be inactive"))
+			case key1:
+				must.True(t, key.IsActive(), must.Sprint("new key should be active"))
+			case key2:
+				must.True(t, key.IsPrepublished(),
+					must.Sprint("prepublished key should not be active"))
 			}
 		}
 
-		// Delete the old key and verify its gone
+		// Delete the original key and verify its gone
 
-		req, err = http.NewRequest(http.MethodDelete, "/v1/operator/keyring/key/"+oldKeyID, nil)
-		require.NoError(t, err)
+		req, err = http.NewRequest(http.MethodDelete, "/v1/operator/keyring/key/"+key0, nil)
+		must.NoError(t, err)
 		obj, err = s.Server.KeyringRequest(respW, req)
-		require.NoError(t, err)
+		must.NoError(t, err)
 
 		req, err = http.NewRequest(http.MethodGet, "/v1/operator/keyring/keys", nil)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		obj, err = s.Server.KeyringRequest(respW, req)
-		require.NoError(t, err)
+		must.NoError(t, err)
 		listResp = obj.([]*structs.RootKeyMeta)
-		require.Len(t, listResp, 1)
-		require.Equal(t, newID1, listResp[0].KeyID)
-		require.True(t, listResp[0].Active())
-		require.Len(t, listResp, 1)
+		must.Len(t, 2, listResp)
+		for _, key := range listResp {
+			switch key.KeyID {
+			case key0:
+				t.Fatalf("initial key should have been deleted")
+			case key1:
+				must.True(t, key.IsActive(), must.Sprint("new key should be active"))
+			case key2:
+				must.True(t, key.IsPrepublished(),
+					must.Sprint("prepublished key should not be active"))
+			}
+		}
 	})
 }
 

command/operator_root_keyring.go

@@ -75,11 +75,15 @@ func renderVariablesKeysResponse(keys []*api.RootKeyMeta, verbose bool) string {
 		length = 8
 	}
 	out := make([]string, len(keys)+1)
-	out[0] = "Key|State|Create Time"
+	out[0] = "Key|State|Create Time|Publish Time"
 	i := 1
 	for _, k := range keys {
-		out[i] = fmt.Sprintf("%s|%v|%s",
-			k.KeyID[:length], k.State, formatUnixNanoTime(k.CreateTime))
+		publishTime := ""
+		if k.PublishTime > 0 {
+			publishTime = formatUnixNanoTime(k.PublishTime)
+		}
+		out[i] = fmt.Sprintf("%s|%v|%s|%s",
+			k.KeyID[:length], k.State, formatUnixNanoTime(k.CreateTime), publishTime)
 		i = i + 1
 	}
 	return formatList(out)

command/operator_root_keyring_rotate.go

@@ -6,6 +6,7 @@ package command
 import (
 	"fmt"
 	"strings"
+	"time"
 
 	"github.com/hashicorp/nomad/api"
 	"github.com/posener/complete"
@@ -36,6 +37,17 @@ Keyring Options:
     will immediately return and the re-encryption process will run
     asynchronously on the leader.
 
+  -now
+    Publish the new key immediately without prepublishing. One of -now or
+    -prepublish must be set.
+
+  -prepublish
+    Set a duration for which to prepublish the new key (ex. "1h"). The currently
+    active key will be unchanged but the new public key will be available in the
+    JWKS endpoint. Multiple keys can be prepublished and they will be promoted to
+    active in order of publish time, at most once every root_key_gc_interval. One
+    of -now or -prepublish must be set.
+
   -verbose
     Show full information.
 `
@@ -50,8 +62,10 @@ func (c *OperatorRootKeyringRotateCommand) Synopsis() string {
 func (c *OperatorRootKeyringRotateCommand) AutocompleteFlags() complete.Flags {
 	return mergeAutocompleteFlags(c.Meta.AutocompleteFlags(FlagSetClient),
 		complete.Flags{
-			"-full":    complete.PredictNothing,
-			"-verbose": complete.PredictNothing,
+			"-full":       complete.PredictNothing,
+			"-now":        complete.PredictNothing,
+			"-prepublish": complete.PredictNothing,
+			"-verbose":    complete.PredictNothing,
 		})
 }
 
@@ -64,12 +78,15 @@ func (c *OperatorRootKeyringRotateCommand) Name() string {
 }
 
 func (c *OperatorRootKeyringRotateCommand) Run(args []string) int {
-	var rotateFull, verbose bool
+	var rotateFull, rotateNow, verbose bool
+	var prepublishDuration time.Duration
 
 	flags := c.Meta.FlagSet("root keyring rotate", FlagSetClient)
 	flags.Usage = func() { c.Ui.Output(c.Help()) }
 	flags.BoolVar(&rotateFull, "full", false, "full key rotation")
+	flags.BoolVar(&rotateNow, "now", false, "immediately rotate without prepublish")
 	flags.BoolVar(&verbose, "verbose", false, "")
+	flags.DurationVar(&prepublishDuration, "prepublish", 0, "prepublish key")
 
 	if err := flags.Parse(args); err != nil {
 		return 1
@@ -88,8 +105,28 @@ func (c *OperatorRootKeyringRotateCommand) Run(args []string) int {
 		return 1
 	}
 
+	if !rotateNow && prepublishDuration == 0 || rotateNow && prepublishDuration != 0 {
+		c.Ui.Error(`
+One of "-now" or "-prepublish" must be used.
+
+If a key has been leaked use "-now" to force immediate rotation.
+
+Otherwise please use "-prepublish <duration>" to ensure the new key is not used
+to sign workload identities before JWKS endpoints are updated.
+`)
+		return 1
+	}
+
+	publishTime := int64(0)
+	if prepublishDuration > 0 {
+		publishTime = time.Now().UnixNano() + prepublishDuration.Nanoseconds()
+	}
+
 	resp, _, err := client.Keyring().Rotate(
-		&api.KeyringRotateOptions{Full: rotateFull}, nil)
+		&api.KeyringRotateOptions{
+			Full:        rotateFull,
+			PublishTime: publishTime,
+		}, nil)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("error: %s", err))
 		return 1