Support custom Vault secret prefix (#6283)

* add settings api

* generated code'

* inject pathPrefix and add TODOs

* bump solo-kit, pass pathPrefix to vault instance + tests

* generate-code

* Add changelog

* Update projects/gloo/pkg/bootstrap/vault.go

Co-authored-by: Nathan Fudenberg <nathan.fudenberg@solo.io>

* Update projects/gloo/pkg/bootstrap/vault.go

Co-authored-by: Nathan Fudenberg <nathan.fudenberg@solo.io>

* generate-code

* Add custom pathPrefix to consul_vault tests

* Update Vault tests to enable a secrets path if testing custom prefixes + add create test

* update create test to use test prefix

* Update secrets creationg address + add PathPrefix to writeSettings

* remove secrets command from vaultInstance cmd/session, so the vault server can be killed on Clean()

* Add test pathPrefix to e2e_suite_test

* Undo e2e test updates with custom pathPrefix

* Doc updates

Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com>
Co-authored-by: Fabian Gonzalez <fabiangonz98@gmail.com>
Co-authored-by: Nathan Fudenberg <nathan.fudenberg@solo.io>

changelog/v1.12.0-beta4/support-custom-vault-secret-prefix.yaml

@@ -0,0 +1,6 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/gloo/issues/6184
+    resolvesIssue: true
+    description: |
+      Exposes pathPrefix for Vault's integrations, allowing users to set a custom secrets engine path to their client.

docs/content/installation/gateway/nomad/_index.md

@@ -37,7 +37,7 @@ Before proceeding to the installation, you will need to complete some prerequisi
 Installation on Nomad requires the following:
 
 - [Levant](https://github.com/jrasell/levant) installed on your local machine
-- [Docker](https://github.com/jrasell/levant), [Consul](https://www.consul.io), [Vault](https://www.vaultproject.io), and [Nomad](https://www.nomadproject.io/) installed on the target host machine (which can also be your local machine). A [Vagrantfile](https://github.com/solo-io/gloo/blob/master/install/nomad/Vagrantfile) is provided that includes everything needed to run Nomad.
+- [Docker](https://www.docker.com), [Consul](https://www.consul.io), [Vault](https://www.vaultproject.io), and [Nomad](https://www.nomadproject.io/) installed on the target host machine (which can also be your local machine). A [Vagrantfile](https://github.com/solo-io/gloo/blob/master/install/nomad/Vagrantfile) is provided that includes everything needed to run Nomad.
 
 If you want to run locally on macOS, you will also need to install [Weave Network](https://www.weave.works/docs/net/latest/install/installing-weave/).
 

docs/content/reference/cli/glooctl_create_secret.md

@@ -24,7 +24,8 @@ glooctl create secret [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

docs/content/reference/cli/glooctl_create_secret_apikey.md

@@ -45,7 +45,8 @@ glooctl create secret apikey [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

docs/content/reference/cli/glooctl_create_secret_aws.md

@@ -45,7 +45,8 @@ glooctl create secret aws [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

docs/content/reference/cli/glooctl_create_secret_azure.md

@@ -43,7 +43,8 @@ glooctl create secret azure [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

docs/content/reference/cli/glooctl_create_secret_header.md

@@ -43,7 +43,8 @@ glooctl create secret header [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

docs/content/reference/cli/glooctl_create_secret_oauth.md

@@ -43,7 +43,8 @@ glooctl create secret oauth [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

docs/content/reference/cli/glooctl_create_secret_tls.md

@@ -45,7 +45,8 @@ glooctl create secret tls [flags]
       --vault-ca-path string           CAPath is the path to a directory of PEM-encoded CA cert files to verify the Vault server SSL certificate.Use with --use-vault
       --vault-client-cert string       ClientCert is the path to the certificate for Vault communication.Use with --use-vault
       --vault-client-key string        ClientKey is the path to the private key for Vault communication.Use with --use-vault
-      --vault-root-key string          key prefix for for Vault key-value storage. (default "gloo")
+      --vault-path-prefix string       The Secrets Engine to which Vault should route traffic. (default "secret")
+      --vault-root-key string          key prefix for Vault key-value storage inside a storage engine. (default "gloo")
       --vault-tls-insecure             Insecure enables or disables SSL verification.Use with --use-vault
       --vault-tls-server-name string   TLSServerName, if set, is used to set the SNI host when connecting via TLS.Use with --use-vault
       --vault-token string             address of the Vault server. This should be a complete  URL such as "http://vault.example.com". Use with --use-vault

go.mod

@@ -71,7 +71,7 @@ require (
 	github.com/solo-io/skv2 v0.21.6
 	// Pinned to the `gloo-namespaced-statuses` tag of solo-apis
 	github.com/solo-io/solo-apis v0.0.0-20210922150112-505473b2e66c
-	github.com/solo-io/solo-kit v0.25.0
+	github.com/solo-io/solo-kit v0.26.0
 	github.com/spf13/afero v1.6.0
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/pflag v1.0.5

go.sum

@@ -1399,8 +1399,8 @@ github.com/solo-io/skv2 v0.21.6/go.mod h1:8jNcMWuAkBxdGhlRFMSgsK94q/jZGPEas8VHTr
 github.com/solo-io/solo-apis v0.0.0-20210922150112-505473b2e66c h1:4/yTroUmyUJonldE5EyC3AinNG4KVLgG0FVMrI2SQ04=
 github.com/solo-io/solo-apis v0.0.0-20210922150112-505473b2e66c/go.mod h1:4HQsQO4Cy/4V7ZZxWncvnMvIq7pYnb66jAT5hvDJBgQ=
 github.com/solo-io/solo-kit v0.23.0/go.mod h1:uCOi8RQ3MetHXsRFvVKPzafYySUvFuPxB+gvo7ScRR8=
-github.com/solo-io/solo-kit v0.25.0 h1:IcGHWSqId4W4GU7BF+Snn9ROZfi3Fg9khbI6VSvZo4Q=
-github.com/solo-io/solo-kit v0.25.0/go.mod h1:ZsqKgFsoZKz0MMWGD0J2K62u6qwBsk6X8P91fHTQUPg=
+github.com/solo-io/solo-kit v0.26.0 h1:MyWQrzmRPWgyZe44mCAx7Xn67gpqVrro+fGyeMlGgzQ=
+github.com/solo-io/solo-kit v0.26.0/go.mod h1:ZsqKgFsoZKz0MMWGD0J2K62u6qwBsk6X8P91fHTQUPg=
 github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
 github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
 github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ=

install/helm/gloo/crds/gloo.solo.io_v1_Settings.yaml

@@ -571,6 +571,8 @@ spec:
                   insecure:
                     nullable: true
                     type: boolean
+                  pathPrefix:
+                    type: string
                   rootKey:
                     type: string
                   tlsServerName:

projects/gloo/api/v1/settings.proto

@@ -113,6 +113,11 @@ message Settings {
         // this can be used to run multiple instances of Gloo against the same Consul cluster
         // defaults to `gloo`
         string root_key = 9;
+
+        // Optional: The name of a Vault Secrets Engine to which Vault should route traffic.
+        // For more info see https://learn.hashicorp.com/tutorials/vault/getting-started-secrets-engines.
+        // Defaults to 'secret'
+        string path_prefix = 10;
     }
 
     // Use [HashiCorp Consul Key-Value](https://www.consul.io/api/kv.html/) as storage for config data.

projects/gloo/cli/pkg/cmd/create/secret/create_test.go

@@ -1,6 +1,7 @@
 package secret_test
 
 import (
+	"fmt"
 	"log"
 	"os"
 
@@ -26,7 +27,7 @@ var _ = Describe("Create", func() {
 
 	BeforeSuite(func() {
 		var err error
-		vaultFactory, err = services.NewVaultFactory()
+		vaultFactory, err = services.NewVaultFactory(&services.VaultFactoryConfig{PathPrefix: services.TestPathPrefix})
 		Expect(err).NotTo(HaveOccurred())
 		client, err = api.NewClient(api.DefaultConfig())
 		Expect(err).NotTo(HaveOccurred())
@@ -65,5 +66,13 @@ var _ = Describe("Create", func() {
 			Expect(err).NotTo(HaveOccurred())
 			Expect(secret).NotTo(BeNil())
 		})
+
+		It("works with custom secrets engine path secrets", func() {
+			err := testutils.Glooctl(fmt.Sprintf("create secret aws --name test --access-key foo --secret-key bar --use-vault --vault-address=http://localhost:8200 --vault-token=root --vault-path-prefix=%s", services.TestPathPrefix))
+			Expect(err).NotTo(HaveOccurred())
+			secret, err := client.Logical().Read(fmt.Sprintf("%s/data/gloo/gloo.solo.io/v1/Secret/gloo-system/test", services.TestPathPrefix))
+			Expect(err).NotTo(HaveOccurred())
+			Expect(secret).NotTo(BeNil())
+		})
 	})
 })

projects/gloo/cli/pkg/cmd/options/options.go

@@ -123,9 +123,19 @@ type Consul struct {
 }
 
 type Vault struct {
-	UseVault bool // enable vault secret clients
-	RootKey  string
-	Client   func() (*vaultapi.Client, error)
+	// enable vault secret clients
+	UseVault bool
+
+	// https://learn.hashicorp.com/tutorials/vault/getting-started-secrets-engines
+	// PathPrefix tells Vault which secrets engine to which it should route traffic.
+	PathPrefix string
+
+	// Secrets are persisted using a resource client constructed in solo-kit
+	// https://github.com/solo-io/solo-kit/blob/1d799ae290c2f516f01fc4ad20272d7d2d5db1e7/pkg/api/v1/clients/vault/resource_client.go#L311
+	// The RootKey is used to configure the path for the particular Gloo installation
+	// This ensures that you can run multiple instances of Gloo against the same Consul cluster
+	RootKey string
+	Client  func() (*vaultapi.Client, error)
 }
 
 type Create struct {

projects/gloo/cli/pkg/flagutils/common.go

@@ -70,7 +70,8 @@ func AddVaultSecretFlags(set *pflag.FlagSet, vault *options.Vault) {
 
 	set.BoolVar(&vault.UseVault, "use-vault", false, "use Vault Key-Value storage as the "+
 		"backend for reading and writing secrets")
-	set.StringVar(&vault.RootKey, "vault-root-key", bootstrap.DefaultRootKey, "key prefix for for Vault key-value storage.")
+	set.StringVar(&vault.PathPrefix, "vault-path-prefix", bootstrap.DefaultPathPrefix, "The Secrets Engine to which Vault should route traffic.")
+	set.StringVar(&vault.RootKey, "vault-root-key", bootstrap.DefaultRootKey, "key prefix for Vault key-value storage inside a storage engine.")
 
 	set.StringVar(&config.Address, "vault-address", config.Address, "address of the Vault server. This should be a complete  URL such as \"http://vault.example.com\". "+
 		"Use with --use-vault")

projects/gloo/cli/pkg/helpers/clients.go

@@ -7,6 +7,8 @@ import (
 	"sync"
 	"time"
 
+	"github.com/solo-io/gloo/projects/gloo/pkg/bootstrap"
+
 	v1alpha1 "github.com/solo-io/gloo/projects/gloo/pkg/api/external/solo/ratelimit"
 
 	kubeconverters "github.com/solo-io/gloo/projects/gloo/pkg/api/converters/kube"
@@ -38,7 +40,7 @@ var (
 	fakeKubeClientset *fake.Clientset
 	memResourceClient *factory.MemoryResourceClientFactory
 	consulClient      *factory.ConsulResourceClientFactory
-	vaultClient       *factory.VaultSecretClientFactory
+	vaultClient       factory.ResourceClientFactory
 
 	lock sync.Mutex
 )
@@ -103,13 +105,10 @@ func UseConsulClients(client *api.Client, rootKey string) {
 }
 
 // only applies to secret clients
-func UseVaultClients(client *vaultapi.Client, rootKey string) {
+func UseVaultClients(client *vaultapi.Client, pathPrefix, rootKey string) {
 	lock.Lock()
 	defer lock.Unlock()
-	vaultClient = &factory.VaultSecretClientFactory{
-		Vault:   client,
-		RootKey: rootKey,
-	}
+	vaultClient = bootstrap.NewVaultSecretClientFactory(client, pathPrefix, rootKey)
 }
 
 func MustKubeClient() kubernetes.Interface {

projects/gloo/cli/pkg/helpers/clients_test.go

@@ -75,7 +75,7 @@ var _ = Describe("Clients", func() {
 	})
 	Describe("UseVaultClients", func() {
 		BeforeEach(func() {
-			UseVaultClients(&api2.Client{}, "")
+			UseVaultClients(&api2.Client{}, "", "")
 		})
 		AfterEach(func() {
 			UseDefaultClients()

projects/gloo/cli/pkg/prerun/prerun_consul.go

@@ -24,7 +24,7 @@ func EnableVaultClients(vault options.Vault) error {
 		if err != nil {
 			return eris.Wrapf(err, "creating Vault client")
 		}
-		helpers.UseVaultClients(client, vault.RootKey)
+		helpers.UseVaultClients(client, vault.PathPrefix, vault.RootKey)
 	}
 	return nil
 }