Add basic auth and ssh support for private helm repo

charts/fleet-crd/templates/crds.yaml

@@ -1831,6 +1831,9 @@ spec:
               type: string
             forceSyncGeneration:
               type: integer
+            helmSecretName:
+              nullable: true
+              type: string
             insecureSkipTLSVerify:
               type: boolean
             paths:

modules/cli/apply/apply.go

@@ -37,6 +37,7 @@ type Options struct {
 	Paused          bool
 	Labels          map[string]string
 	SyncGeneration  int64
+	Auth            bundle.Auth
 }
 
 func globDirs(baseDir string) (result []string, err error) {
@@ -125,6 +126,7 @@ func readBundle(ctx context.Context, name, baseDir string, opts *Options) (*bund
 		TargetNamespace: opts.TargetNamespace,
 		Paused:          opts.Paused,
 		SyncGeneration:  opts.SyncGeneration,
+		Auth:            opts.Auth,
 	})
 }
 

modules/cli/cmds/apply.go

@@ -3,6 +3,7 @@ package cmds
 import (
 	"bytes"
 	"fmt"
+	"io/ioutil"
 	"os"
 	"os/exec"
 	"strings"
@@ -23,14 +24,18 @@ func NewApply() *cobra.Command {
 type Apply struct {
 	BundleInputArgs
 	OutputArgsNoDefault
-	Label           map[string]string `usage:"Labels to apply to created bundles" short:"l"`
-	TargetsFile     string            `usage:"Addition source of targets and restrictions to be append"`
-	Compress        bool              `usage:"Force all resources to be compress" short:"c"`
-	ServiceAccount  string            `usage:"Service account to assign to bundle created" short:"a"`
-	SyncGeneration  int               `usage:"Generation number used to force sync the deployment"`
-	TargetNamespace string            `usage:"Ensure this bundle goes to this target namespace"`
-	Paused          bool              `usage:"Create bundles in a paused state"`
-	Commit          string            `usage:"Commit to assign to the bundle" env:"COMMIT"`
+	Label             map[string]string `usage:"Labels to apply to created bundles" short:"l"`
+	TargetsFile       string            `usage:"Addition source of targets and restrictions to be append"`
+	Compress          bool              `usage:"Force all resources to be compress" short:"c"`
+	ServiceAccount    string            `usage:"Service account to assign to bundle created" short:"a"`
+	SyncGeneration    int               `usage:"Generation number used to force sync the deployment"`
+	TargetNamespace   string            `usage:"Ensure this bundle goes to this target namespace"`
+	Paused            bool              `usage:"Create bundles in a paused state"`
+	Commit            string            `usage:"Commit to assign to the bundle" env:"COMMIT"`
+	Username          string            `usage:"Basic auth username for helm repo" env:"HELM_USERNAME"`
+	PasswordFile      string            `usage:"Path of file containing basic auth password for helm repo"`
+	CACertsFile       string            `usage:"Path of custom cacerts for helm repo" name:"cacerts-file"`
+	SSHPrivateKeyFile string            `usage:"Path of ssh-private-key for helm repo" name:"ssh-privatekey-file"`
 }
 
 func (a *Apply) Run(cmd *cobra.Command, args []string) error {
@@ -58,6 +63,30 @@ func (a *Apply) Run(cmd *cobra.Command, args []string) error {
 		SyncGeneration:  int64(a.SyncGeneration),
 	}
 
+	if a.Username != "" && a.PasswordFile != "" {
+		password, err := ioutil.ReadFile(a.PasswordFile)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+
+		opts.Auth.Username = a.Username
+		opts.Auth.Password = string(password)
+	}
+	if a.CACertsFile != "" {
+		cabundle, err := ioutil.ReadFile(a.CACertsFile)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		opts.Auth.CABundle = cabundle
+	}
+	if a.SSHPrivateKeyFile != "" {
+		privateKey, err := ioutil.ReadFile(a.SSHPrivateKeyFile)
+		if err != nil && !os.IsNotExist(err) {
+			return err
+		}
+		opts.Auth.SSHPrivateKey = privateKey
+	}
+
 	if a.File == "-" {
 		opts.BundleReader = os.Stdin
 		if len(args) != 1 {

package/Dockerfile.agent

@@ -1,7 +1,7 @@
 FROM alpine:3.12.3
 ARG ARCH
 ENV ARCH=$ARCH
-RUN apk add -U --no-cache git bash
+RUN apk add -U --no-cache git bash openssh && adduser -u 1000 -D fleet-apply
 COPY bin/fleetagent-linux-$ARCH /usr/bin/fleetagent
 COPY bin/fleet-linux-$ARCH /usr/bin/fleet
 COPY package/log.sh /usr/bin/

pkg/apis/fleet.cattle.io/v1alpha1/git.go

@@ -40,6 +40,9 @@ type GitRepoSpec struct {
 	// It is expected the secret be of type "kubernetes.io/basic-auth" or "kubernetes.io/ssh-auth".
 	ClientSecretName string `json:"clientSecretName,omitempty"`
 
+	// HelmSecretName contains the auth secret for private helm repository
+	HelmSecretName string `json:"helmSecretName,omitempty"`
+
 	// CABundle is a PEM encoded CA bundle which will be used to validate the repo's certificate.
 	CABundle []byte `json:"caBundle,omitempty"`
 

pkg/bundle/read.go

@@ -23,6 +23,7 @@ type Options struct {
 	TargetNamespace string
 	Paused          bool
 	SyncGeneration  int64
+	Auth            Auth
 }
 
 func Open(ctx context.Context, name, baseDir, file string, opts *Options) (*Bundle, error) {
@@ -127,7 +128,7 @@ func read(ctx context.Context, name, baseDir string, bundleSpecReader io.Reader,
 	meta.Name = name
 	setTargetNames(&bundle.BundleSpec)
 
-	resources, err := readResources(ctx, &bundle.BundleSpec, opts.Compress, baseDir)
+	resources, err := readResources(ctx, &bundle.BundleSpec, opts.Compress, baseDir, opts.Auth)
 	if err != nil {
 		return nil, err
 	}

pkg/bundle/resources.go

@@ -3,6 +3,9 @@ package bundle
 import (
 	"context"
 	"crypto/sha256"
+	"crypto/tls"
+	"crypto/x509"
+	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -14,19 +17,18 @@ import (
 	"sync"
 	"unicode/utf8"
 
-	"helm.sh/helm/v3/pkg/repo"
-	"sigs.k8s.io/yaml"
-
 	"github.com/hashicorp/go-getter"
 	"github.com/pkg/errors"
 	"github.com/rancher/fleet/modules/cli/pkg/progress"
 	fleet "github.com/rancher/fleet/pkg/apis/fleet.cattle.io/v1alpha1"
 	"github.com/rancher/fleet/pkg/content"
 	"golang.org/x/sync/errgroup"
 	"golang.org/x/sync/semaphore"
+	"helm.sh/helm/v3/pkg/repo"
+	"sigs.k8s.io/yaml"
 )
 
-func readResources(ctx context.Context, spec *fleet.BundleSpec, compress bool, base string) ([]fleet.BundleResource, error) {
+func readResources(ctx context.Context, spec *fleet.BundleSpec, compress bool, base string, auth Auth) ([]fleet.BundleResource, error) {
 	var directories []directory
 
 	directories, err := addDirectory(directories, base, ".", ".")
@@ -55,7 +57,7 @@ func readResources(ctx context.Context, spec *fleet.BundleSpec, compress bool, b
 		}
 	}
 
-	directories, err = addCharts(directories, base, chartDirs)
+	directories, err = addCharts(directories, base, chartDirs, auth)
 	if err != nil {
 		return nil, err
 	}
@@ -84,7 +86,14 @@ func ChartPath(helm *fleet.HelmOptions) string {
 	return fmt.Sprintf(".chart/%x", sha256.Sum256([]byte(helm.Chart + ":" + helm.Repo + ":" + helm.Version)[:]))
 }
 
-func chartURL(location *fleet.HelmOptions) (string, error) {
+type Auth struct {
+	Username      string
+	Password      string
+	CABundle      []byte
+	SSHPrivateKey []byte
+}
+
+func chartURL(location *fleet.HelmOptions, auth Auth) (string, error) {
 	if location.Repo == "" {
 		return location.Chart, nil
 	}
@@ -93,7 +102,29 @@ func chartURL(location *fleet.HelmOptions) (string, error) {
 		location.Repo = location.Repo + "/"
 	}
 
-	resp, err := http.Get(location.Repo + "index.yaml")
+	request, err := http.NewRequest("GET", location.Repo+"index.yaml", nil)
+	if err != nil {
+		return "", err
+	}
+
+	if auth.Username != "" && auth.Password != "" {
+		request.SetBasicAuth(auth.Username, auth.Password)
+	}
+	client := &http.Client{}
+	if auth.CABundle != nil {
+		pool, err := x509.SystemCertPool()
+		if err != nil {
+			pool = x509.NewCertPool()
+		}
+		pool.AppendCertsFromPEM(auth.CABundle)
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{
+			RootCAs: pool,
+		}
+		client.Transport = transport
+	}
+
+	resp, err := client.Do(request)
 	if err != nil {
 		return "", err
 	}
@@ -137,10 +168,10 @@ func chartURL(location *fleet.HelmOptions) (string, error) {
 	return repoURL.ResolveReference(chartURL).String(), nil
 }
 
-func addCharts(directories []directory, base string, charts []*fleet.HelmOptions) ([]directory, error) {
+func addCharts(directories []directory, base string, charts []*fleet.HelmOptions, auth Auth) ([]directory, error) {
 	for _, chart := range charts {
 		if _, err := os.Stat(filepath.Join(base, chart.Chart)); os.IsNotExist(err) || chart.Repo != "" {
-			chartURL, err := chartURL(chart)
+			chartURL, err := chartURL(chart, auth)
 			if err != nil {
 				return nil, err
 			}
@@ -150,6 +181,7 @@ func addCharts(directories []directory, base string, charts []*fleet.HelmOptions
 				base:   base,
 				path:   chartURL,
 				key:    ChartPath(chart),
+				auth:   auth,
 			})
 		}
 	}
@@ -179,6 +211,7 @@ type directory struct {
 	base   string
 	path   string
 	key    string
+	auth   Auth
 }
 
 func readDirectories(ctx context.Context, compress bool, directories ...directory) (map[string][]fleet.BundleResource, error) {
@@ -199,7 +232,7 @@ func readDirectories(ctx context.Context, compress bool, directories ...director
 		dir := dir
 		eg.Go(func() error {
 			defer sem.Release(1)
-			resources, err := readDirectory(ctx, p, compress, dir.prefix, dir.base, dir.path)
+			resources, err := readDirectory(ctx, p, compress, dir.prefix, dir.base, dir.path, dir.auth)
 			if err != nil {
 				return err
 			}
@@ -219,10 +252,10 @@ func readDirectories(ctx context.Context, compress bool, directories ...director
 	return result, eg.Wait()
 }
 
-func readDirectory(ctx context.Context, progress *progress.Progress, compress bool, prefix, base, name string) ([]fleet.BundleResource, error) {
+func readDirectory(ctx context.Context, progress *progress.Progress, compress bool, prefix, base, name string, auth Auth) ([]fleet.BundleResource, error) {
 	var resources []fleet.BundleResource
 
-	files, err := readContent(ctx, progress, base, name)
+	files, err := readContent(ctx, progress, base, name, auth)
 	if err != nil {
 		return nil, err
 	}
@@ -253,7 +286,7 @@ func readDirectory(ctx context.Context, progress *progress.Progress, compress bo
 	return resources, nil
 }
 
-func readContent(ctx context.Context, progress *progress.Progress, base, name string) (map[string][]byte, error) {
+func readContent(ctx context.Context, progress *progress.Progress, base, name string, auth Auth) (map[string][]byte, error) {
 	temp, err := ioutil.TempDir("", "fleet")
 	if err != nil {
 		return nil, err
@@ -268,15 +301,48 @@ func readContent(ctx context.Context, progress *progress.Progress, base, name st
 	}
 
 	c := getter.Client{
-		Ctx:  ctx,
-		Src:  name,
-		Dst:  temp,
-		Pwd:  base,
-		Mode: getter.ClientModeDir,
+		Ctx:     ctx,
+		Src:     name,
+		Dst:     temp,
+		Pwd:     base,
+		Mode:    getter.ClientModeDir,
+		Getters: getter.Getters,
 		// TODO: why doesn't this work anymore
 		//ProgressListener: progress,
 	}
 
+	httpGetter := &getter.HttpGetter{
+		Client: &http.Client{},
+	}
+
+	if auth.Username != "" && auth.Password != "" {
+		header := http.Header{}
+		header.Add("Authorization", "Basic "+basicAuth(auth.Username, auth.Password))
+		httpGetter.Header = header
+	}
+	if auth.CABundle != nil {
+		pool, err := x509.SystemCertPool()
+		if err != nil {
+			pool = x509.NewCertPool()
+		}
+		pool.AppendCertsFromPEM(auth.CABundle)
+		transport := http.DefaultTransport.(*http.Transport).Clone()
+		transport.TLSClientConfig = &tls.Config{
+			RootCAs: pool,
+		}
+		httpGetter.Client.Transport = transport
+	}
+	if auth.SSHPrivateKey != nil {
+		if strings.IndexAny(c.Src, "&;") == -1 {
+			c.Src += "?"
+		} else {
+			c.Src += "&"
+		}
+		c.Src += fmt.Sprintf("sshkey=%s", base64.StdEncoding.EncodeToString(auth.SSHPrivateKey))
+	}
+	c.Getters["http"] = httpGetter
+	c.Getters["https"] = httpGetter
+
 	if err := c.Get(); err != nil {
 		return nil, err
 	}
@@ -366,3 +432,8 @@ func mergeGenericMap(first, second *fleet.GenericMap) *fleet.GenericMap {
 	}
 	return result
 }
+
+func basicAuth(username, password string) string {
+	auth := username + ":" + password
+	return base64.StdEncoding.EncodeToString([]byte(auth))
+}