Merge pull request #2363 from agentgonzo/vault-install-secrets-yaml

Vault install secrets yaml

pkg/auth/types.go

@@ -1,7 +1,7 @@
 package auth
 
 import (
-	"github.com/hashicorp/vault/api"
+	"github.com/jenkins-x/jx/pkg/vault"
 )
 
 const (
@@ -46,6 +46,6 @@ type FileAuthConfigSaver struct {
 
 // VaultAuthConfigSaver is a ConfigSaver that saves configs to Vault
 type VaultAuthConfigSaver struct {
-	vaultClient *api.Client
+	vaultClient vault.Client
 	secretName  string
 }

pkg/auth/vault_config_saver.go

@@ -1,20 +1,20 @@
 package auth
 
 import (
-	"github.com/hashicorp/vault/api"
 	"github.com/jenkins-x/jx/pkg/util"
+	"github.com/jenkins-x/jx/pkg/vault"
 )
 
 // LoadConfig loads the config from the vault
 func (v *VaultAuthConfigSaver) LoadConfig() (*AuthConfig, error) {
-	data, err := v.vaultClient.Logical().Read(secretPath(v.secretName))
+	data, err := v.vaultClient.Read(v.secretName)
 	if err != nil {
 		return nil, err
 	}
 	config := AuthConfig{}
 
 	if data != nil {
-		err = util.ToStructFromMapStringInterface(data.Data, &config)
+		err = util.ToStructFromMapStringInterface(data, &config)
 	}
 	return &config, err
 }
@@ -24,27 +24,21 @@ func (v *VaultAuthConfigSaver) SaveConfig(config *AuthConfig) error {
 	// Marshall the AuthConfig to a generic map to save in vault (as that's what vault takes)
 	m, err := util.ToMapStringInterfaceFromStruct(&config)
 	if err == nil {
-		v.vaultClient.Logical().Write(secretPath(v.secretName), m)
+		_, err = v.vaultClient.Write(v.secretName, m)
 	}
 	return err
 }
 
 // NewVaultAuthConfigService creates a new ConfigService that saves it config to a Vault
-func NewVaultAuthConfigService(secretName string, vaultClient *api.Client) ConfigService {
+func NewVaultAuthConfigService(secretName string, vaultClient vault.Client) ConfigService {
 	saver := newVaultAuthConfigSaver(secretName, vaultClient)
 	return NewAuthConfigService(&saver)
 }
 
 // newVaultAuthConfigSaver creates a ConfigSaver that saves the Configs under a specified secretname in a vault
-func newVaultAuthConfigSaver(secretName string, vaultClient *api.Client) VaultAuthConfigSaver {
+func newVaultAuthConfigSaver(secretName string, vaultClient vault.Client) VaultAuthConfigSaver {
 	return VaultAuthConfigSaver{
 		secretName:  secretName,
 		vaultClient: vaultClient,
 	}
 }
-
-// secretPath generates a secret path from the secret name for storing in vault
-// this just makes sure it gets stored under /secret
-func secretPath(secretName string) string {
-	return "secret/" + secretName
-}

pkg/buildnum/http_build_num_test.go

@@ -10,7 +10,7 @@ import (
 	"github.com/jenkins-x/jx/pkg/kube"
 	. "github.com/petergtz/pegomock"
 
-	"github.com/jenkins-x/jx/pkg/buildnum/mocks"
+	build_num_test "github.com/jenkins-x/jx/pkg/buildnum/mocks"
 
 	"github.com/stretchr/testify/assert"
 )

pkg/buildnum/interface.go

@@ -4,7 +4,7 @@ package buildnum
 import "github.com/jenkins-x/jx/pkg/kube"
 
 // BuildNumberIssuer generates build numbers for activities.
-//go:generate pegomock generate github.com/jenkins-x/jx/pkg/buildnum BuildNumberIssuer -o mocks/buildnum.go --generate-matchers
+//go:generate pegomock generate github.com/jenkins-x/jx/pkg/buildnum BuildNumberIssuer -o mocks/build_num.go --generate-matchers
 type BuildNumberIssuer interface {
 
 	//Generate the next build number for the supplied pipeline.

pkg/cloud/gke/vault/vault_backend.go

@@ -1,15 +1,16 @@
 package vault
 
 import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
 	"github.com/jenkins-x/jx/pkg/cloud/gke"
 	"github.com/jenkins-x/jx/pkg/kube/serviceaccount"
-	"github.com/jenkins-x/jx/pkg/vault"
 	"github.com/pkg/errors"
-	"io/ioutil"
 	"k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/client-go/kubernetes"
-	"os"
 )
 
 const (
@@ -23,6 +24,7 @@ var (
 	}
 )
 
+// KmsConfig keeps the configuration for Google KMS service
 type KmsConfig struct {
 	Keyring  string
 	Key      string
@@ -110,13 +112,18 @@ func CreateAuthServiceAccount(client kubernetes.Interface, vaultName, namespace,
 	return serviceAccountName, nil
 }
 
+// GcpServiceAccountSecretName builds the secret name where the GCP service account is stored
+func GcpServiceAccountSecretName(vaultName string, clusterName string) string {
+	return fmt.Sprintf("%s-%s-gcp-sa", clusterName, vaultName)
+}
+
 func storeGCPServiceAccountIntoSecret(client kubernetes.Interface, serviceAccountPath, vaultName, namespace, clusterName string) (string, error) {
 	serviceAccount, err := ioutil.ReadFile(serviceAccountPath)
 	if err != nil {
 		return "", errors.Wrapf(err, "reading the service account from file '%s'", serviceAccountPath)
 	}
 
-	secretName := vault.VaultGcpServiceAccountSecretName(vaultName, clusterName)
+	secretName := GcpServiceAccountSecretName(vaultName, clusterName)
 	secret := &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: secretName,

pkg/config/admin_secrets.go

@@ -121,14 +121,6 @@ func (s *AdminSecretsService) AddAdminSecretsValues(cmd *cobra.Command) {
 	cmd.Flags().StringVarP(&s.Flags.DefaultAdminPassword, "default-admin-password", "", "", "the default admin password to access Jenkins, Kubernetes Dashboard, Chartmuseum and Nexus")
 }
 
-func (c AdminSecretsConfig) String() (string, error) {
-	b, err := yaml.Marshal(c)
-	if err != nil {
-		return "", fmt.Errorf("failed to marshall helm values %v", err)
-	}
-	return string(b), nil
-}
-
 func (s *AdminSecretsService) NewAdminSecretsConfig() error {
 	s.Secrets = AdminSecretsConfig{
 		ChartMuseum:     &ChartMuseum{},

pkg/io/config_store.go

@@ -0,0 +1,59 @@
+package io
+
+import (
+	"io/ioutil"
+
+	"github.com/jenkins-x/jx/pkg/util"
+	"github.com/pkg/errors"
+	"gopkg.in/yaml.v2"
+)
+
+// ConfigStore provides an interface for storing configs
+type ConfigStore interface {
+	// Write saves some secret data to the store
+	Write(name string, bytes []byte) error
+
+	// WriteObject writes a named object to the store
+	WriteObject(name string, obj interface{}) error
+
+	// Read reads some secret data from the store
+	Read(name string) ([]byte, error)
+
+	// ReadObject reads an object from the store
+	ReadObject(name string, object interface{}) error
+}
+
+type fileStore struct {
+}
+
+// NewFileStore creates a ConfigStore that stores its data to the filesystem in YAML
+func NewFileStore() ConfigStore {
+	return &fileStore{}
+}
+
+// Write writes a secret to the filesystem in YAML format
+func (f *fileStore) Write(fileName string, bytes []byte) error {
+	return ioutil.WriteFile(fileName, bytes, util.DefaultWritePermissions)
+}
+
+// WriteObject writes a secret to the filesystem in YAML format
+func (f *fileStore) WriteObject(fileName string, obj interface{}) error {
+	y, err := yaml.Marshal(obj)
+	if err != nil {
+		return errors.Wrapf(err, "Unable to marshal object to yaml: %v", obj)
+	}
+	return f.Write(fileName, y)
+}
+
+func (f *fileStore) Read(fileName string) ([]byte, error) {
+	return ioutil.ReadFile(fileName)
+}
+
+// ReadObject reads an object from the filesystem as yaml
+func (f *fileStore) ReadObject(fileName string, object interface{}) error {
+	data, err := f.Read(fileName)
+	if err != nil {
+		return errors.Wrapf(err, "Unable to read %s", fileName)
+	}
+	return yaml.Unmarshal(data, object)
+}

pkg/jx/cmd/common_install.go

@@ -650,6 +650,42 @@ func (o *CommonOptions) installhyperv() error {
 	return nil
 }
 
+func (o *CommonOptions) installVaultCli() error {
+	binDir, err := util.JXBinLocation()
+	if err != nil {
+		return err
+	}
+	binary := "vault"
+	fileName, flag, err := shouldInstallBinary(binary)
+	if err != nil || !flag {
+		return err
+	}
+	latestVersion, err := util.GetLatestFullTagFromGithub("hashicorp", "vault")
+	if err != nil {
+		return err
+	}
+	// Strip the v off the beginning of the version number
+	latestVersion = strings.Replace(latestVersion, "v", "", 1)
+
+	clientURL := fmt.Sprintf("https://releases.hashicorp.com/vault/%s/vault_%s_%s_%s.zip", latestVersion, latestVersion, runtime.GOOS, runtime.GOARCH)
+	fullPath := filepath.Join(binDir, fileName)
+	tarFile := fullPath + ".zip"
+	err = binaries.DownloadFile(clientURL, tarFile)
+	if err != nil {
+		return err
+	}
+	err = util.UnzipSpecificFiles(tarFile, binDir, fileName)
+	if err != nil {
+		return err
+	}
+	err = os.Remove(tarFile)
+	if err != nil {
+		return err
+	}
+	err = os.Chmod(fullPath, 0755)
+	return err
+}
+
 func (o *CommonOptions) installHelm() error {
 	// TODO temporary hack while we are on the 2.10-rc version:
 	/*

pkg/jx/cmd/create_jenkins_token.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
-	"os"
 	"path/filepath"
 	"strings"
 	"time"
@@ -354,7 +353,7 @@ func (o *CreateJenkinsUserOptions) tryFindAPITokenFromBrowser(tokenUrl string, n
 	if err != nil {
 		return errors.Wrap(err, "creating the chrome user data dir")
 	}
-	defer os.RemoveAll(userDataDir)
+	defer util.DestroyFile(userDataDir)
 	netLogFile := filepath.Join(userDataDir, "net-logs.json")
 
 	c, err := o.createChromeClientWithNetLog(ctx, userDataDir, netLogFile)

pkg/jx/cmd/create_vault.go

@@ -15,6 +15,7 @@ import (
 	gkevault "github.com/jenkins-x/jx/pkg/cloud/gke/vault"
 	"github.com/jenkins-x/jx/pkg/jx/cmd/templates"
 	"github.com/jenkins-x/jx/pkg/kube"
+	kubevault "github.com/jenkins-x/jx/pkg/kube/vault"
 	"github.com/jenkins-x/jx/pkg/log"
 	"github.com/jenkins-x/jx/pkg/util"
 	"github.com/jenkins-x/jx/pkg/vault"
@@ -149,7 +150,7 @@ func (o *CreateVaultOptions) createVault(vaultOperatorClient versioned.Interface
 		return err
 	}
 	// Checks if the vault already exists
-	found := vault.FindVault(vaultOperatorClient, vaultName, o.Namespace)
+	found := kubevault.FindVault(vaultOperatorClient, vaultName, o.Namespace)
 	if found {
 		return fmt.Errorf("Vault with name '%s' already exists in namespace '%s'", vaultName, o.Namespace)
 	}
@@ -213,14 +214,14 @@ func (o *CreateVaultOptions) createVault(vaultOperatorClient versioned.Interface
 	log.Infof("Created service account %s for Vault authentication\n", util.ColorInfo(vaultAuthServiceAccount))
 
 	log.Infof("Creating Vault...\n")
-	gcpConfig := &vault.GCPConfig{
+	gcpConfig := &kubevault.GCPConfig{
 		ProjectId:   o.GKEProjectID,
 		KmsKeyring:  kmsConfig.Keyring,
 		KmsKey:      kmsConfig.Key,
 		KmsLocation: kmsConfig.Location,
 		GcsBucket:   vaultBucket,
 	}
-	err = vault.CreateVault(kubeClient, vaultOperatorClient, vaultName, o.Namespace, gcpServiceAccountSecretName,
+	err = kubevault.CreateVault(kubeClient, vaultOperatorClient, vaultName, o.Namespace, gcpServiceAccountSecretName,
 		gcpConfig, vaultAuthServiceAccount, o.Namespace, o.SecretsPathPrefix)
 	if err != nil {
 		return errors.Wrap(err, "creating vault")