Process Helm Midstream Images and Secrets (#2244)

* Unblock one chart at a time for midstream

* add chartname back to UseHelmInstall map when done

pkg/pull/pull.go

@@ -9,6 +9,7 @@ import (
 	"net/url"
 	"os"
 	"path/filepath"
+	"strings"
 	"time"
 
 	"github.com/pkg/errors"
@@ -416,8 +417,11 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 		NoProxyEnvValue:    pullOptions.NoProxyEnvValue,
 		NewHelmCharts:      newHelmCharts,
 	}
-	// this map contains chart names and useHelmInstall flag
-	// presence of chartname and the flag determines if the pullsecrets will be generated within each chart or at the top level
+
+	// the UseHelmInstall map blocks visibility into charts and subcharts when searching for private images
+	// any chart name listed here will be skipped when writing midstream kustomization.yaml and pullsecret.yaml
+	// when using Helm Install, each chart gets it's own kustomization and pullsecret yaml and MUST be skipped when processing higher level directories!
+	// for writing Common Midstream, every chart and subchart is in this map as Helm Midstreams will be processed later in the code
 	commonWriteMidstreamOptions.UseHelmInstall = map[string]bool{}
 	for _, v := range newHelmCharts {
 		commonWriteMidstreamOptions.UseHelmInstall[v.Spec.Chart.Name] = v.Spec.UseHelmInstall
@@ -443,7 +447,14 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 
 	helmMidstreams := []midstream.Midstream{}
 	for _, helmBase := range helmBases {
-		writeMidstreamOptions := commonWriteMidstreamOptions
+		// we must look at the current chart for private images, but must ignore subcharts
+		// to do this, we remove only the current helmBase name from the UseHelmInstall map to unblock visibility into the chart directory
+		// this ensures only the current chart resources are added to kustomization.yaml and pullsecret.yaml
+		chartName := strings.Split(helmBase.Path, "/")[len(strings.Split(helmBase.Path, "/"))-1]
+		// copy the bool setting in the map to restore it after this process loop
+		useHelmSetting := writeMidstreamOptions.UseHelmInstall[chartName]
+		delete(writeMidstreamOptions.UseHelmInstall, chartName)
+
 		writeMidstreamOptions.MidstreamDir = filepath.Join(helmBase.GetOverlaysDir(writeBaseOptions), "midstream", helmBase.Path)
 		writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), helmBase.Path)
 
@@ -452,6 +463,9 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 			return "", errors.Wrapf(err, "failed to write helm midstream %s", helmBase.Path)
 		}
 
+		// add this chart back into UseHelmInstall to make sure it's not processed again
+		writeMidstreamOptions.UseHelmInstall[chartName] = useHelmSetting
+
 		helmMidstreams = append(helmMidstreams, *helmMidstream)
 	}
 

pkg/rewrite/rewrite.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"os"
 	"path/filepath"
+	"strings"
 
 	"github.com/pkg/errors"
 	kotsv1beta1 "github.com/replicatedhq/kots/kotskinds/apis/kots/v1beta1"
@@ -222,6 +223,10 @@ func Rewrite(rewriteOptions RewriteOptions) error {
 		NewHelmCharts:      newHelmCharts,
 	}
 
+	// the UseHelmInstall map blocks visibility into charts and subcharts when searching for private images
+	// any chart name listed here will be skipped when writing midstream kustomization.yaml and pullsecret.yaml
+	// when using Helm Install, each chart gets it's own kustomization and pullsecret yaml and MUST be skipped when processing higher level directories!
+	// for writing Common Midstream, every chart and subchart is in this map as Helm Midstreams will be processed later in the code
 	commonWriteMidstreamOptions.UseHelmInstall = map[string]bool{}
 	for _, v := range newHelmCharts {
 		commonWriteMidstreamOptions.UseHelmInstall[v.Spec.Chart.Name] = v.Spec.UseHelmInstall
@@ -245,10 +250,16 @@ func Rewrite(rewriteOptions RewriteOptions) error {
 		return errors.Wrap(err, "failed to write common midstream")
 	}
 
-	//commonWriteMidstreamOptions.UseHelmInstall = map[string]bool{}
 	helmMidstreams := []midstream.Midstream{}
 	for _, helmBase := range helmBases {
-		writeMidstreamOptions := commonWriteMidstreamOptions
+		// we must look at the current chart for private images, but must ignore subcharts
+		// to do this, we remove only the current helmBase name from the UseHelmInstall map to unblock visibility into the chart directory
+		// this ensures only the current chart resources are added to kustomization.yaml and pullsecret.yaml
+		chartName := strings.Split(helmBase.Path, "/")[len(strings.Split(helmBase.Path, "/"))-1]
+		// copy the bool setting in the map to restore it after this process loop
+		useHelmSetting := writeMidstreamOptions.UseHelmInstall[chartName]
+		delete(writeMidstreamOptions.UseHelmInstall, chartName)
+
 		writeMidstreamOptions.MidstreamDir = filepath.Join(helmBase.GetOverlaysDir(writeBaseOptions), "midstream", helmBase.Path)
 		writeMidstreamOptions.BaseDir = filepath.Join(u.GetBaseDir(writeUpstreamOptions), helmBase.Path)
 
@@ -259,6 +270,9 @@ func Rewrite(rewriteOptions RewriteOptions) error {
 			return errors.Wrapf(err, "failed to write helm midstream %s", helmBase.Path)
 		}
 
+		// add this chart back into UseHelmInstall to make sure it's not processed again
+		writeMidstreamOptions.UseHelmInstall[chartName] = useHelmSetting
+
 		helmMidstreams = append(helmMidstreams, *helmMidstream)
 	}