Merge pull request #497 from classmarkets/vault-0.10-tests

Make tests compatible with Vault 0.10

Makefile

@@ -21,8 +21,8 @@ GOVERSION = $(shell go version | awk '{print $$3;}')
 GORELEASER = $(shell which goreleaser)
 
 # pin versions for CI builds
-CI_CONSUL_VERSION=1.0.6
-CI_VAULT_VERSION=0.9.6
+CI_CONSUL_VERSION=1.1.0
+CI_VAULT_VERSION=0.10.1
 CI_GO_VERSION=1.10.3
 
 # all is the default target

cert/source_test.go

@@ -9,6 +9,7 @@ import (
 	"crypto/x509/pkix"
 	"encoding/pem"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"log"
 	"math/big"
@@ -250,8 +251,18 @@ func TestConsulSource(t *testing.T) {
 			resp, err := http.Get("http://127.0.0.1:8500/v1/status/leader")
 			// /v1/status/leader returns '\n""' while consul is in leader election mode
 			// and '"127.0.0.1:8300"' when not. So we punt by checking the
-			// Content-Length header instead of the actual body content :)
-			return err == nil && resp.StatusCode == 200 && resp.ContentLength > 10
+			// body length instead of the actual body content :)
+			if err != nil {
+				return false
+			}
+			defer resp.Body.Close()
+
+			if resp.StatusCode != 200 {
+				return false
+			}
+
+			n, err := io.Copy(ioutil.Discard, resp.Body)
+			return err == nil && n > 10
 		}
 
 		// We need give consul ~8-10 seconds to become ready until I've
@@ -339,6 +350,15 @@ func vaultServer(t *testing.T, addr, rootToken string) (*exec.Cmd, *vaultapi.Cli
 	  capabilities = ["read"]
 	}
 
+	# Vault >= 0.10. (KV Version 2)
+	path "secret/metadata/fabio/cert/" {
+	  capabilities = ["list"]
+	}
+
+	path "secret/data/fabio/cert/*" {
+	  capabilities = ["read"]
+	}
+
 	path "test-pki/issue/fabio" {
 	  capabilities = ["update"]
 	}
@@ -425,7 +445,26 @@ func TestVaultSource(t *testing.T) {
 	// create a cert and store it in vault
 	certPEM, keyPEM := makePEM("localhost", time.Minute)
 	data := map[string]interface{}{"cert": string(certPEM), "key": string(keyPEM)}
-	if _, err := client.Logical().Write(certPath+"/localhost", data); err != nil {
+
+	var nilSource *VaultSource // for calling helper methods
+
+	mountPath, v2, err := nilSource.isKVv2(certPath, client)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	p := certPath + "/localhost"
+	if v2 {
+		t.Log("Vault: KV backend: V2")
+		data = map[string]interface{}{
+			"data":    data,
+			"options": map[string]interface{}{},
+		}
+		p = nilSource.addPrefixToVKVPath(p, mountPath, "data")
+	} else {
+		t.Log("Vault: KV backend: V1")
+	}
+	if _, err := client.Logical().Write(p, data); err != nil {
 		t.Fatalf("logical.Write failed: %s", err)
 	}
 

cert/vault_source.go

@@ -5,6 +5,8 @@ import (
 	"crypto/x509"
 	"fmt"
 	"log"
+	"path"
+	"strings"
 	"time"
 
 	"github.com/hashicorp/vault/api"
@@ -47,8 +49,20 @@ func (s *VaultSource) load(path string) (pemBlocks map[string][]byte, err error)
 	// they are recognized by the post-processing function
 	// which assembles the certificates.
 	// The value can be stored either as string or []byte.
-	get := func(name, typ string, secret *api.Secret) {
-		v := secret.Data[typ]
+	get := func(name, typ string, secret *api.Secret, v2 bool) {
+		data := secret.Data
+		if v2 {
+			x, ok := secret.Data["data"]
+			if !ok {
+				return
+			}
+			data, ok = x.(map[string]interface{})
+			if !ok {
+				return
+			}
+		}
+
+		v := data[typ]
 		if v == nil {
 			return
 		}
@@ -72,27 +86,97 @@ func (s *VaultSource) load(path string) (pemBlocks map[string][]byte, err error)
 		return nil, fmt.Errorf("vault: client: %s", err)
 	}
 
+	mountPath, v2, err := s.isKVv2(path, c)
+	if err != nil {
+		return nil, fmt.Errorf("vault: query mount path: %s", err)
+	}
+
 	// get the subkeys under 'path'.
 	// Each subkey refers to a certificate.
-	certs, err := c.Logical().List(path)
+	p := path
+	if v2 {
+		p = s.addPrefixToVKVPath(p, mountPath, "metadata")
+	}
+
+	certs, err := c.Logical().List(p)
 	if err != nil {
 		return nil, fmt.Errorf("vault: list: %s", err)
 	}
 	if certs == nil || certs.Data["keys"] == nil {
 		return nil, nil
 	}
 
-	for _, s := range certs.Data["keys"].([]interface{}) {
-		name := s.(string)
+	for _, x := range certs.Data["keys"].([]interface{}) {
+		name := x.(string)
 		p := path + "/" + name
+		if v2 {
+			p = s.addPrefixToVKVPath(p, mountPath, "data")
+		}
 		secret, err := c.Logical().Read(p)
 		if err != nil {
 			log.Printf("[WARN] cert: Failed to read %s from Vault: %s", p, err)
 			continue
 		}
-		get(name, "cert", secret)
-		get(name, "key", secret)
+		get(name, "cert", secret, v2)
+		get(name, "key", secret, v2)
 	}
 
 	return pemBlocks, nil
 }
+
+func (s *VaultSource) addPrefixToVKVPath(p, mountPath, apiPrefix string) string {
+	p = strings.TrimPrefix(p, mountPath)
+	return path.Join(mountPath, apiPrefix, p)
+}
+
+func (s *VaultSource) isKVv2(path string, client *api.Client) (string, bool, error) {
+	mountPath, version, err := s.kvPreflightVersionRequest(client, path)
+	if err != nil {
+		return "", false, err
+	}
+
+	return mountPath, version == 2, nil
+}
+
+func (s *VaultSource) kvPreflightVersionRequest(client *api.Client, path string) (string, int, error) {
+	r := client.NewRequest("GET", "/v1/sys/internal/ui/mounts/"+path)
+	resp, err := client.RawRequest(r)
+	if resp != nil {
+		defer resp.Body.Close()
+	}
+	if err != nil {
+		// If we get a 404 we are using an older version of vault, default to
+		// version 1
+		if resp != nil && resp.StatusCode == 404 {
+			return "", 1, nil
+		}
+
+		return "", 0, err
+	}
+
+	secret, err := api.ParseSecret(resp.Body)
+	if err != nil {
+		return "", 0, err
+	}
+	var mountPath string
+	if mountPathRaw, ok := secret.Data["path"]; ok {
+		mountPath = mountPathRaw.(string)
+	}
+	options := secret.Data["options"]
+	if options == nil {
+		return mountPath, 1, nil
+	}
+	versionRaw := options.(map[string]interface{})["version"]
+	if versionRaw == nil {
+		return mountPath, 1, nil
+	}
+	version := versionRaw.(string)
+	switch version {
+	case "", "1":
+		return mountPath, 1, nil
+	case "2":
+		return mountPath, 2, nil
+	}
+
+	return mountPath, 1, nil
+}