Increase docker registry and git-server push fault-tolerance (#586)

* Stop adding zarf registry to local docker config
* allow dynamic port binding for git/registry
* fix updates to git-server test & optimize test sequence
* clean connect table and zarf connect msg output
* remove test time buffer now that ports are dynamic
* Cleanup repo/image pushing & add retry logic
* Only dump tunnel endpoint if auto-open is disabled

examples/Vagrantfile

@@ -18,9 +18,6 @@ Vagrant.configure("2") do |config|
   config.vm.network "forwarded_port", guest: 443, host: 8443
   config.vm.network "forwarded_port", guest: 9080, host: 9080
   config.vm.network "forwarded_port", guest: 9443, host: 9443
-  config.vm.network "forwarded_port", guest: 45001, host: 45001
-  config.vm.network "forwarded_port", guest: 45002, host: 45002
-  config.vm.network "forwarded_port", guest: 45003, host: 45003
 
   config.ssh.insert_key = false
   config.ssh.extra_args = [ "-t", "cd /examples; sudo su" ]

go.mod

@@ -17,7 +17,7 @@ require (
 	github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b
 	github.com/derailed/k9s v0.25.21
 	github.com/distribution/distribution/v3 v3.0.0-20220612151901-b5e2f3f33dbc
-	github.com/docker/cli v20.10.17+incompatible
+	github.com/docker/cli v20.10.16+incompatible // indirect
 	github.com/fatih/color v1.13.0
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/go-logr/logr v1.2.3

go.sum

@@ -692,14 +692,10 @@ github.com/denis-tingajkin/go-header v0.4.2/go.mod h1:eLRHAVXzE5atsKAnNRDB90WHCF
 github.com/denisenkom/go-mssqldb v0.9.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/denisenkom/go-mssqldb v0.11.0/go.mod h1:xbL0rPBG9cCiLr28tMa8zpbdarY27NDyej4t/EjAShU=
 github.com/denverdino/aliyungo v0.0.0-20190125010748-a747050bb1ba/go.mod h1:dV8lFg6daOBZbT6/BDGIz6Y3WFGn8juu6G+CQ6LHtl0=
-github.com/derailed/k9s v0.25.20 h1:rHXXD2qtOT+bsB82RbdwzB2CpHNBzdajKASCqPNlxKY=
-github.com/derailed/k9s v0.25.20/go.mod h1:Lyz70/1F/xB5VwhxyOQpEJnlAYO/vjM/nzPzxvkPpMg=
 github.com/derailed/k9s v0.25.21 h1:lcAi66Fr+TVRs/wZNl/S7M06WSHgseOUoWphce2pgww=
 github.com/derailed/k9s v0.25.21/go.mod h1:N4KW6B22evx7gm+wOCbDs+0u/ZnWO3xObEUYQ7F91t8=
 github.com/derailed/popeye v0.10.0 h1:qhqM/SMaJcTNxIGQ3oA8+PV7WfIQim7IF3vQTKbo6aM=
 github.com/derailed/popeye v0.10.0/go.mod h1:u9/7aMBKbxreSPp/9evMrbwJH8SgHIPrpfEJzkBuhr4=
-github.com/derailed/tview v0.6.6 h1:hNqBewhRTYRgfLp1p5KGw0DFdbGMS68iocBSmGGNg4s=
-github.com/derailed/tview v0.6.6/go.mod h1:A1LXWlbx/YDMXr3GVTy+IgclAkBssJpw/FiZ7aqUgzU=
 github.com/derailed/tview v0.7.1 h1:kV8nPWUpftbduH1hlPXqA/09iyyP9ZH6nSfZF9m+d8U=
 github.com/derailed/tview v0.7.1/go.mod h1:zH4i4fRLbUKOuqQwdcMefak2YyAjUj9bmKsiKTMf9gg=
 github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY=
@@ -715,8 +711,8 @@ github.com/dnaeon/go-vcr v1.0.1/go.mod h1:aBB1+wY4s93YsC3HHjMBMrwTj2R9FHDzUr9KyG
 github.com/docker/cli v20.10.10+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v20.10.11+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/cli v20.10.12+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
-github.com/docker/cli v20.10.17+incompatible h1:eO2KS7ZFeov5UJeaDmIs1NFEDRf32PaqRpvoEkKBy5M=
-github.com/docker/cli v20.10.17+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
+github.com/docker/cli v20.10.16+incompatible h1:aLQ8XowgKpR3/IysPj8qZQJBVQ+Qws61icFuZl6iKYs=
+github.com/docker/cli v20.10.16+incompatible/go.mod h1:JLrzqnKDaYBop7H2jaqPtU4hHvMKP+vjCwu2uszcLI8=
 github.com/docker/distribution v0.0.0-20190905152932-14b96e55d84c/go.mod h1:0+TTO4EOBfRPhZXAeF1Vu+W3hHZ8eLp8PgKVZlcvtFY=
 github.com/docker/distribution v2.7.1-0.20190205005809-0d3efadf0154+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
 github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w=
@@ -2518,8 +2514,6 @@ golang.org/x/crypto v0.0.0-20211202192323-5770296d904e/go.mod h1:IxCIyHEi3zRg3s0
 golang.org/x/crypto v0.0.0-20211209193657-4570a0811e8b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20211215153901-e495a2d5b3d3/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e h1:T8NU3HyQ8ClP4SEE+KbFlg6n0NhuTsN4MyznaarGsZM=
-golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d h1:sK3txAijHtOK88l68nt020reeT1ZdKLIYetKl95FzVY=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=

src/config/config.go

@@ -29,7 +29,6 @@ const (
 	ZarfGitReadUser         = "zarf-git-read-user"
 	ZarfRegistryPushUser    = "zarf-push"
 	ZarfRegistryPullUser    = "zarf-pull"
-	ZarfRegistry            = IPV4Localhost + ":45001"
 	ZarfImagePullSecretName = "private-registry"
 	ZarfGitServerSecretName = "private-git-server"
 

src/internal/git/push.go

@@ -17,14 +17,18 @@ import (
 const offlineRemoteName = "offline-downstream"
 const onlineRemoteRefPrefix = "refs/remotes/" + onlineRemoteName + "/"
 
-func PushAllDirectories(localPath string) {
+func PushAllDirectories(localPath string) error {
 	// Establish a git tunnel to send the repos
 	tunnel := k8s.NewZarfTunnel()
 	tunnel.Connect(k8s.ZarfGit, false)
+	defer tunnel.Close()
+
+	tunnelUrl := fmt.Sprintf("http://%s", tunnel.Endpoint())
 
 	paths, err := utils.ListDirectories(localPath)
 	if err != nil {
-		message.Fatalf(err, "unable to list the %s directory", localPath)
+		message.Warnf("Unable to list the %s directory", localPath)
+		return err
 	}
 
 	spinner := message.NewProgressSpinner("Processing %d git repos", len(paths))
@@ -33,26 +37,27 @@ func PushAllDirectories(localPath string) {
 	for _, path := range paths {
 		basename := filepath.Base(path)
 		spinner.Updatef("Pushing git repo %s", basename)
-		if err := push(path, spinner); err != nil {
-			spinner.Fatalf(err, "Unable to push the git repo %s", basename)
+		if err := push(path, tunnelUrl, spinner); err != nil {
+			spinner.Warnf("Unable to push the git repo %s", basename)
+			return err
 		}
 
 		// Add the read-only user to this repo
 		repoPathSplit := strings.Split(path, "/")
 		repoNameWithGitTag := repoPathSplit[len(repoPathSplit)-1]
 		repoName := strings.Split(repoNameWithGitTag, ".git")[0]
-		err = addReadOnlyUserToRepo(repoName)
+		err = addReadOnlyUserToRepo(tunnelUrl, repoName)
 		if err != nil {
-			message.Debug(err)
 			message.Warnf("Unable to add the read-only user to the repo: %v\n", repoName)
+			return err
 		}
 	}
 
 	spinner.Success()
-	tunnel.Close()
+	return nil
 }
 
-func push(localPath string, spinner *message.Spinner) error {
+func push(localPath, tunnelUrl string, spinner *message.Spinner) error {
 
 	// Open the given repo
 	repo, err := git.PlainOpen(localPath)
@@ -67,8 +72,7 @@ func push(localPath string, spinner *message.Spinner) error {
 
 	}
 	remoteUrl := remote.Config().URLs[0]
-	targetHost := fmt.Sprintf("http://%s:%d", config.IPV4Localhost, k8s.PortGit)
-	targetUrl := transformURL(targetHost, remoteUrl)
+	targetUrl := transformURL(tunnelUrl, remoteUrl)
 
 	_, err = repo.CreateRemote(&goConfig.RemoteConfig{
 		Name: offlineRemoteName,

src/internal/git/utils.go

@@ -248,6 +248,8 @@ func CreateReadOnlyUser() error {
 	tunnel.Connect(k8s.ZarfGit, false)
 	defer tunnel.Close()
 
+	tunnelUrl := tunnel.Endpoint()
+
 	// Create json representation of the create-user request body
 	createUserBody := map[string]interface{}{
 		"username":             config.ZarfGitReadUser,
@@ -261,7 +263,7 @@ func CreateReadOnlyUser() error {
 	}
 
 	// Send API request to create the user
-	createUserEndpoint := fmt.Sprintf("http://%s:%d/api/v1/admin/users", config.IPV4Localhost, k8s.PortGit)
+	createUserEndpoint := fmt.Sprintf("http://%s/api/v1/admin/users", tunnelUrl)
 	createUserRequest, _ := netHttp.NewRequest("POST", createUserEndpoint, bytes.NewBuffer(createUserData))
 	_, err = DoHttpThings(createUserRequest, config.ZarfGitPushUser, config.GetSecret(config.StateGitPush))
 	if err != nil {
@@ -275,13 +277,13 @@ func CreateReadOnlyUser() error {
 		"allow_create_organization": false,
 	}
 	updateUserData, _ := json.Marshal(updateUserBody)
-	updateUserEndpoint := fmt.Sprintf("http://%s:%d/api/v1/admin/users/%s", config.IPV4Localhost, k8s.PortGit, config.ZarfGitReadUser)
+	updateUserEndpoint := fmt.Sprintf("http://%s/api/v1/admin/users/%s", tunnelUrl, config.ZarfGitReadUser)
 	updateUserRequest, _ := netHttp.NewRequest("PATCH", updateUserEndpoint, bytes.NewBuffer(updateUserData))
 	_, err = DoHttpThings(updateUserRequest, config.ZarfGitPushUser, config.GetSecret(config.StateGitPush))
 	return err
 }
 
-func addReadOnlyUserToRepo(repo string) error {
+func addReadOnlyUserToRepo(tunnelUrl, repo string) error {
 	// Add the readonly user to the repo
 	addColabBody := map[string]string{
 		"permission": "read",
@@ -292,7 +294,7 @@ func addReadOnlyUserToRepo(repo string) error {
 	}
 
 	// Send API request to add a user as a read-only collaborator to a repo
-	addColabEndpoint := fmt.Sprintf("http://%s:%d/api/v1/repos/%s/%s/collaborators/%s", config.IPV4Localhost, k8s.PortGit, config.ZarfGitPushUser, repo, config.ZarfGitReadUser)
+	addColabEndpoint := fmt.Sprintf("%s/api/v1/repos/%s/%s/collaborators/%s", tunnelUrl, config.ZarfGitPushUser, repo, config.ZarfGitReadUser)
 	addColabRequest, _ := netHttp.NewRequest("PUT", addColabEndpoint, bytes.NewBuffer(addColabData))
 	_, err = DoHttpThings(addColabRequest, config.ZarfGitPushUser, config.GetSecret(config.StateGitPush))
 	return err

src/internal/helm/chart.go

@@ -16,11 +16,6 @@ import (
 	"helm.sh/helm/v3/pkg/storage/driver"
 )
 
-type ConnectString struct {
-	Description string
-	Url         string
-}
-type ConnectStrings map[string]ConnectString
 type ChartOptions struct {
 	BasePath          string
 	Chart             types.ZarfChart
@@ -32,7 +27,7 @@ type ChartOptions struct {
 }
 
 // InstallOrUpgradeChart performs a helm install of the given chart
-func InstallOrUpgradeChart(options ChartOptions) ConnectStrings {
+func InstallOrUpgradeChart(options ChartOptions) types.ConnectStrings {
 	fromMessage := options.Chart.Url
 	if fromMessage == "" {
 		fromMessage = "Zarf-generated helm chart"
@@ -117,7 +112,7 @@ func InstallOrUpgradeChart(options ChartOptions) ConnectStrings {
 
 // TemplateChart generates a helm template from a given chart
 func TemplateChart(options ChartOptions) (string, error) {
-	message.Debugf("helm.TemplateChart(%v)", options)
+	message.Debugf("helm.TemplateChart(%#v)", options)
 	spinner := message.NewProgressSpinner("Templating helm chart %s", options.Chart.Name)
 	defer spinner.Stop()
 
@@ -161,8 +156,8 @@ func TemplateChart(options ChartOptions) (string, error) {
 	return templatedChart.Manifest, nil
 }
 
-func GenerateChart(basePath string, manifest types.ZarfManifest, component types.ZarfComponent) ConnectStrings {
-	message.Debugf("helm.GenerateChart(%s, %v, %s)", basePath, manifest, component.Name)
+func GenerateChart(basePath string, manifest types.ZarfManifest, component types.ZarfComponent) types.ConnectStrings {
+	message.Debugf("helm.GenerateChart(%s, %#v, %s)", basePath, manifest, component.Name)
 	spinner := message.NewProgressSpinner("Starting helm chart generation %s", manifest.Name)
 	defer spinner.Stop()
 
@@ -214,7 +209,7 @@ func GenerateChart(basePath string, manifest types.ZarfManifest, component types
 }
 
 func installChart(actionConfig *action.Configuration, options ChartOptions, postRender *renderer) (*release.Release, error) {
-	message.Debugf("helm.installChart(%v, %v, %v)", actionConfig, options, postRender)
+	message.Debugf("helm.installChart(%#v, %#v, %#v)", actionConfig, options, postRender)
 	// Bind the helm action
 	client := action.NewInstall(actionConfig)
 
@@ -245,7 +240,7 @@ func installChart(actionConfig *action.Configuration, options ChartOptions, post
 }
 
 func upgradeChart(actionConfig *action.Configuration, options ChartOptions, postRender *renderer) (*release.Release, error) {
-	message.Debugf("helm.upgradeChart(%v, %v, %v)", actionConfig, options, postRender)
+	message.Debugf("helm.upgradeChart(%#v, %#v, %#v)", actionConfig, options, postRender)
 	client := action.NewUpgrade(actionConfig)
 
 	// Let each chart run for 5 minutes
@@ -271,7 +266,7 @@ func upgradeChart(actionConfig *action.Configuration, options ChartOptions, post
 }
 
 func rollbackChart(actionConfig *action.Configuration, name string) error {
-	message.Debugf("helm.rollbackChart(%v, %s)", actionConfig, name)
+	message.Debugf("helm.rollbackChart(%#v, %s)", actionConfig, name)
 	client := action.NewRollback(actionConfig)
 	client.CleanupOnFail = true
 	client.Force = true
@@ -281,7 +276,7 @@ func rollbackChart(actionConfig *action.Configuration, name string) error {
 }
 
 func uninstallChart(actionConfig *action.Configuration, name string) (*release.UninstallReleaseResponse, error) {
-	message.Debugf("helm.uninstallChart(%v, %s)", actionConfig, name)
+	message.Debugf("helm.uninstallChart(%#v, %s)", actionConfig, name)
 	client := action.NewUninstall(actionConfig)
 	client.KeepHistory = false
 	client.Timeout = 3 * time.Minute
@@ -290,7 +285,7 @@ func uninstallChart(actionConfig *action.Configuration, name string) (*release.U
 }
 
 func loadChartData(options ChartOptions) (*chart.Chart, map[string]any, error) {
-	message.Debugf("helm.loadChartData(%v)", options)
+	message.Debugf("helm.loadChartData(%#v)", options)
 	var (
 		loadedChart *chart.Chart
 		chartValues map[string]any

src/internal/helm/post-render.go

@@ -10,6 +10,7 @@ import (
 	"github.com/defenseunicorns/zarf/src/internal/k8s"
 	"github.com/defenseunicorns/zarf/src/internal/message"
 	"github.com/defenseunicorns/zarf/src/internal/utils"
+	"github.com/defenseunicorns/zarf/src/types"
 	"helm.sh/helm/v3/pkg/action"
 	"helm.sh/helm/v3/pkg/releaseutil"
 	corev1 "k8s.io/api/core/v1"
@@ -21,16 +22,16 @@ import (
 
 type renderer struct {
 	actionConfig   *action.Configuration
-	connectStrings ConnectStrings
+	connectStrings types.ConnectStrings
 	options        ChartOptions
 	namespaces     map[string]*corev1.Namespace
 }
 
 func NewRenderer(options ChartOptions, actionConfig *action.Configuration) *renderer {
-	message.Debugf("helm.NewRenderer(%v)", options)
+	message.Debugf("helm.NewRenderer(%#v)", options)
 	return &renderer{
 		actionConfig:   actionConfig,
-		connectStrings: make(ConnectStrings),
+		connectStrings: make(types.ConnectStrings),
 		options:        options,
 		namespaces: map[string]*corev1.Namespace{
 			// Add the passed-in namespace to the list
@@ -84,7 +85,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
 			// parse to unstructured to have access to more data than just the name
 			rawData := &unstructured.Unstructured{}
 			if err := yaml.Unmarshal([]byte(resource.Content), rawData); err != nil {
-				return nil, fmt.Errorf("failed to unmarshal manifest: %v", err)
+				return nil, fmt.Errorf("failed to unmarshal manifest: %#v", err)
 			}
 
 			switch rawData.GetKind() {
@@ -119,7 +120,7 @@ func (r *renderer) Run(renderedManifests *bytes.Buffer) (*bytes.Buffer, error) {
 					message.Debugf("Match helm service %s for zarf connection %s", rawData.GetName(), key)
 
 					// Add the connectstring for processing later in the deployment
-					r.connectStrings[key] = ConnectString{
+					r.connectStrings[key] = types.ConnectString{
 						Description: annotations[config.ZarfConnectAnnotationDescription],
 						Url:         annotations[config.ZarfConnectAnnotationUrl],
 					}

src/internal/images/push.go

@@ -8,14 +8,16 @@ import (
 	"github.com/google/go-containerregistry/pkg/crane"
 )
 
-func PushToZarfRegistry(imageTarballPath string, buildImageList []string) {
+func PushToZarfRegistry(imageTarballPath string, buildImageList []string) error {
 	message.Debugf("images.PushToZarfRegistry(%v, %v)", imageTarballPath, buildImageList)
 
 	// Establish a registry tunnel to send the images to the zarf registry
 	tunnel := k8s.NewZarfTunnel()
 	tunnel.Connect(k8s.ZarfRegistry, false)
 	defer tunnel.Close()
 
+	tunnelUrl := tunnel.Endpoint()
+
 	spinner := message.NewProgressSpinner("Storing images in the zarf registry")
 	defer spinner.Stop()
 
@@ -26,17 +28,15 @@ func PushToZarfRegistry(imageTarballPath string, buildImageList []string) {
 		spinner.Updatef("Updating image %s", src)
 		img, err := crane.LoadTag(imageTarballPath, src, config.GetCraneOptions()...)
 		if err != nil {
-			spinner.Errorf(err, "Unable to load the image from the update package")
-			return
+			return err
 		}
 
-		offlineName := utils.SwapHost(src, config.ZarfRegistry)
-		err = crane.Push(img, offlineName, pushOptions)
-
-		if err != nil {
-			spinner.Fatalf(err, "Unable to push the image to the registry")
+		offlineName := utils.SwapHost(src, tunnelUrl)
+		if err = crane.Push(img, offlineName, pushOptions); err != nil {
+			return err
 		}
 	}
 
 	spinner.Success()
+	return nil
 }