Add kv custom key metadata (#12218)

* add custom-metdata flag to "kv metadata put" command

* add kv metadata put command test for custom-metadata flag

* add custom_metadata to kv-v2 api docs

* add custom_metadata to kv-v2 cli docs

* update go.mod

* Add custom metadata limits to docs

* add changelog entry

* update vault-plugin-secrets-kv to @master

changelog/12218.txt

@@ -0,0 +1,3 @@
+```release-note:feature
+secrets/kv: Add ability to specify version-agnostic custom key metadata
+```

command/kv_metadata_put.go

@@ -21,6 +21,7 @@ type KVMetadataPutCommand struct {
 	flagMaxVersions        int
 	flagCASRequired        bool
 	flagDeleteVersionAfter time.Duration
+	flagCustomMetadata     map[string]string
 	testStdin              io.Reader // for tests
 }
 
@@ -51,6 +52,10 @@ Usage: vault metadata kv put [options] KEY
 
       $ vault kv metadata put -cas-required secret/foo
 
+  Set custom metadata on the key:
+
+      $ vault kv metadata put -custom-metadata=foo=abc -custom-metadata=bar=123 secret/foo
+
   Additional flags and more advanced use cases are detailed below.
 
 ` + c.Flags().Help()
@@ -90,6 +95,14 @@ func (c *KVMetadataPutCommand) Flags() *FlagSets {
 		"3h25m19s".`,
 	})
 
+	f.StringMapVar(&StringMapVar{
+		Name: "custom-metadata",
+		Target: &c.flagCustomMetadata,
+		Default: map[string]string{},
+		Usage: "Specifies arbitrary version-agnostic key=value metadata meant to describe a secret." +
+			"This can be specified multiple times to add multiple pieces of metadata.",
+	})
+
 	return set
 }
 
@@ -139,8 +152,9 @@ func (c *KVMetadataPutCommand) Run(args []string) int {
 
 	path = addPrefixToVKVPath(path, mountPath, "metadata")
 	data := map[string]interface{}{
-		"max_versions": c.flagMaxVersions,
-		"cas_required": c.flagCASRequired,
+		"max_versions":  c.flagMaxVersions,
+		"cas_required":  c.flagCASRequired,
+		"custom_metadata": c.flagCustomMetadata,
 	}
 
 	if c.flagDeleteVersionAfter >= 0 {

command/kv_metadata_put_test.go

@@ -1,11 +1,11 @@
 package command
 
 import (
-	"strings"
-	"testing"
-
+	"github.com/go-test/deep"
 	"github.com/hashicorp/vault/api"
 	"github.com/mitchellh/cli"
+	"strings"
+	"testing"
 )
 
 func testKVMetadataPutCommand(tb testing.TB) (*cli.MockUi, *KVMetadataPutCommand) {
@@ -77,3 +77,80 @@ func TestKvMetadataPutCommandDeleteVersionAfter(t *testing.T) {
 		t.Fatalf("expected 0s but received %q", secret.Data["delete_version_after"])
 	}
 }
+
+func TestKvMetadataPutCommandCustomMetadata(t *testing.T) {
+	client, closer := testVaultServer(t)
+	defer closer()
+
+	basePath := t.Name() + "/"
+	secretPath := basePath + "secret/my-secret"
+
+	if err := client.Sys().Mount(basePath, &api.MountInput{
+		Type: "kv-v2",
+	}); err != nil {
+		t.Fatalf("kv-v2 mount error: %#v", err)
+	}
+
+	ui, cmd := testKVMetadataPutCommand(t)
+	cmd.client = client
+
+	exitStatus := cmd.Run([]string{"-custom-metadata=foo=abc", "-custom-metadata=bar=123", secretPath})
+
+	if exitStatus != 0 {
+		t.Fatalf("Expected 0 exit status but received %d", exitStatus)
+	}
+
+	metaFullPath := basePath + "metadata/secret/my-secret"
+	commandOutput := ui.OutputWriter.String() + ui.ErrorWriter.String()
+	expectedOutput := "Success! Data written to: " + metaFullPath
+
+	if !strings.Contains(commandOutput, expectedOutput) {
+		t.Fatalf("Expected command output %q but received %q", expectedOutput, commandOutput)
+	}
+
+	metadata, err := client.Logical().Read(metaFullPath)
+
+	if err != nil {
+		t.Fatalf("Metadata read error: %#v", err)
+	}
+
+	// JSON output from read decoded into map[string]interface{}
+	expectedCustomMetadata := map[string]interface{}{
+		"foo": "abc",
+		"bar": "123",
+	}
+
+	if diff := deep.Equal(metadata.Data["custom_metadata"], expectedCustomMetadata); len(diff) > 0 {
+		t.Fatal(diff)
+	}
+
+	ui, cmd = testKVMetadataPutCommand(t)
+	cmd.client = client
+
+	// Overwrite entire custom metadata with a single key
+	exitStatus = cmd.Run([]string{"-custom-metadata=baz=abc123", secretPath})
+
+	if exitStatus != 0 {
+		t.Fatalf("Expected 0 exit status but received %d", exitStatus)
+	}
+
+	commandOutput = ui.OutputWriter.String() + ui.ErrorWriter.String()
+
+	if !strings.Contains(commandOutput, expectedOutput) {
+		t.Fatalf("Expected command output %q but received %q", expectedOutput, commandOutput)
+	}
+
+	metadata, err = client.Logical().Read(metaFullPath)
+
+	if err != nil {
+		t.Fatalf("Metadata read error: %#v", err)
+	}
+
+	expectedCustomMetadata = map[string]interface{}{
+		"baz": "abc123",
+	}
+
+	if diff := deep.Equal(metadata.Data["custom_metadata"], expectedCustomMetadata); len(diff) > 0 {
+		t.Fatal(diff)
+	}
+}

go.mod

@@ -51,7 +51,7 @@ require (
 	github.com/go-sql-driver/mysql v1.5.0
 	github.com/go-test/deep v1.0.7
 	github.com/gocql/gocql v0.0.0-20210401103645-80ab1e13e309
-	github.com/golang/protobuf v1.4.2
+	github.com/golang/protobuf v1.5.2
 	github.com/google/go-cmp v0.5.5
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-metrics-stackdriver v0.2.0
@@ -109,7 +109,7 @@ require (
 	github.com/hashicorp/vault-plugin-secrets-azure v0.10.0
 	github.com/hashicorp/vault-plugin-secrets-gcp v0.10.2
 	github.com/hashicorp/vault-plugin-secrets-gcpkms v0.9.0
-	github.com/hashicorp/vault-plugin-secrets-kv v0.9.0
+	github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20210811133805-e060c2307b24
 	github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.4.0
 	github.com/hashicorp/vault-plugin-secrets-openldap v0.5.1
 	github.com/hashicorp/vault-plugin-secrets-terraform v0.1.1-0.20210715043003-e02ca8f6408e
@@ -186,7 +186,7 @@ require (
 	golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c
 	google.golang.org/api v0.29.0
 	google.golang.org/grpc v1.29.1
-	google.golang.org/protobuf v1.25.0
+	google.golang.org/protobuf v1.27.1
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce
 	gopkg.in/ory-am/dockertest.v3 v3.3.4
 	gopkg.in/square/go-jose.v2 v2.5.1

go.sum

@@ -491,6 +491,9 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
 github.com/golang/protobuf v1.4.2 h1:+Z5KGCizgyZCbGh1KZqA0fcLLkwbsjIzS4aV2v7wJX0=
 github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
+github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
 github.com/golang/snappy v0.0.0-20170215233205-553a64147049/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
 github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
@@ -678,6 +681,8 @@ github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09
 github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI=
 github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
+github.com/hashicorp/go-version v1.3.0 h1:McDWVJIU/y+u1BRV06dPaLfLCaT7fUTJLp5r04x7iNw=
+github.com/hashicorp/go-version v1.3.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA=
 github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
@@ -754,6 +759,8 @@ github.com/hashicorp/vault-plugin-secrets-gcp v0.10.2 h1:+DtlYJTsrFRInQpAo09KkYN
 github.com/hashicorp/vault-plugin-secrets-gcp v0.10.2/go.mod h1:psRQ/dm5XatoUKLDUeWrpP9icMJNtu/jmscUr37YGK4=
 github.com/hashicorp/vault-plugin-secrets-gcpkms v0.9.0 h1:7a0iWuFA/YNinQ1xXogyZHStolxMVtLV+sy1LpEHaZs=
 github.com/hashicorp/vault-plugin-secrets-gcpkms v0.9.0/go.mod h1:hhwps56f2ATeC4Smgghrc5JH9dXR31b4ehSf1HblP5Q=
+github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20210811133805-e060c2307b24 h1:uqPKQzkmO5vybOqk2aOdviXXi5088bcl2MrE0D1MhjM=
+github.com/hashicorp/vault-plugin-secrets-kv v0.5.7-0.20210811133805-e060c2307b24/go.mod h1:4j2pZrSynPuUAAYrZQVgSSHD0A9xj7GK9Ji1sWtnO4s=
 github.com/hashicorp/vault-plugin-secrets-kv v0.9.0 h1:nCw2IfWw2bWUGFZsNk8BvTEg9k7jDpRn48+VAqjdQ3s=
 github.com/hashicorp/vault-plugin-secrets-kv v0.9.0/go.mod h1:B/Cybh5aVF7LNAMHwVBxY8t7r2eL0C6HVGgTyP4nKK4=
 github.com/hashicorp/vault-plugin-secrets-mongodbatlas v0.4.0 h1:6ve+7hZmGn7OpML81iZUxYj2AaJptwys323S5XsvVas=
@@ -1676,6 +1683,11 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4=
 google.golang.org/protobuf v1.25.0 h1:Ejskq+SyPohKW+1uil0JJMtmHCgJPJ/qWTxr8qp+R4c=
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
+google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
+google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
+google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U=
 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
 gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw=

website/content/api-docs/secret/kv/kv-v2.mdx

@@ -399,6 +399,11 @@ $ curl \
     "max_versions": 0,
     "oldest_version": 0,
     "updated_time": "2018-03-22T02:36:43.986212308Z",
+    "custom_metadata": {
+      "foo": "abc",
+      "bar": "123",
+      "baz": "5c07d823-3810-48f6-a147-4c06b5219e84"
+    },
     "versions": {
       "1": {
         "created_time": "2018-03-22T02:24:06.945319214Z",
@@ -447,13 +452,21 @@ It does not create a new version.
   backend's `delete_version_after` will be used. Accepts [Go duration
   format string][duration-godoc].
 
+- `custom_metadata` `(map<string|string>: nil)` - A map of arbitrary string to string valued user-provided metadata meant
+  to describe the secret.
+
 ### Sample Payload
 
 ```json
 {
   "max_versions": 5,
   "cas_required": false,
-  "delete_version_after": "3h25m19s"
+  "delete_version_after": "3h25m19s",
+  "custom_metadata": {
+    "foo": "abc",
+    "bar": "123",
+    "baz": "5c07d823-3810-48f6-a147-4c06b5219e84"
+  }
 }
 ```
 

website/content/docs/internals/limits.mdx

@@ -194,6 +194,15 @@ key-value pairs are converted to JSON before storage.
 Version metadata consumes 21 bytes per version and must fit in a
 single storage entry, separate from the stored data.
 
+Each secret also has version-agnostic metadata. This data can contain a `custom_metadata` field of
+user-provided key-value pairs. Vault imposes the following custom metadata limits:
+
+|                                           | Limit     |
+| ----------------------------------------- | --------- |
+| Number of custom metadata key-value pairs | 64        |
+| Custom metadata key size                  | 128 bytes |
+| Custom metadata value size                | 512 bytes |
+
 ### Transit secret engine
 
 The maximum size of a Transit ciphertext or plaintext is limited by Vault's

website/content/docs/secrets/kv/kv-v2.mdx

@@ -318,6 +318,7 @@ See the commands below for more information:
    cas_required            false
    created_time            2019-06-19T17:20:22.985303Z
    current_version         2
+   custom_metadata         map[bar:123 foo:abc]
    delete_version_after    0s
    max_versions            0
    oldest_version          0
@@ -389,6 +390,15 @@ See the commands below for more information:
    destroyed        false
    ```
 
+  A secret's key metadata can contain custom metadata used to describe the secret. The
+  data will be stored as string-to-string key-value pairs. If the `-custom-metadata` flag
+  is set, the value of `custom_metadata` will be fully overwritten. The `-custom-metadata`
+  flag can be repeated to add multiple key-value pairs:
+
+  ```text
+  vault kv metadata put -custom-metadata=foo=abc -custom-metadata=bar=123 secret/my-secret
+  ```
+
 1. Permanently delete all metadata and versions for a key:
 
    ```text