Fix kotskinds yamlErrors

pkg/base/base.go

@@ -124,20 +124,7 @@ func (f BaseFile) ShouldBeIncludedInBaseKustomization(excludeKotsKinds bool) (bo
 
 	// Backup is never deployed. kots.io/exclude and kots.io/when are used to enable snapshots
 	if excludeKotsKinds {
-		if o.APIVersion == "velero.io/v1" && o.Kind == "Backup" {
-			return false, nil
-		}
-
-		if o.APIVersion == "kots.io/v1beta1" {
-			return false, nil
-		}
-
-		if o.APIVersion == "troubleshoot.replicated.com/v1beta1" {
-			return false, nil
-		}
-
-		// In addition to kotskinds, we exclude the application crd for now
-		if o.APIVersion == "app.k8s.io/v1beta1" {
+		if iskotsAPIVersionKind(o) {
 			return false, nil
 		}
 	}
@@ -185,6 +172,46 @@ func (f BaseFile) ShouldBeIncludedInBaseKustomization(excludeKotsKinds bool) (bo
 	return true, nil
 }
 
+func (f BaseFile) IsKotsKind() (bool, error) {
+	o := OverlySimpleGVK{}
+
+	if err := yaml.Unmarshal(f.Content, &o); err != nil {
+		// check if this is a yaml file
+		if ext := filepath.Ext(f.Path); ext == ".yaml" || ext == ".yml" {
+			return false, ParseError{Err: err}
+		}
+		return false, nil
+	}
+
+	// check if this is a kubernetes document
+	if o.APIVersion == "" || o.Kind == "" {
+		// check if this is a yaml file
+		if ext := filepath.Ext(f.Path); ext == ".yaml" || ext == ".yml" {
+			return false, ParseError{Err: errors.New("not a kubernetes document")}
+		}
+		return false, nil
+	}
+
+	return iskotsAPIVersionKind(o), nil
+}
+
+func iskotsAPIVersionKind(o OverlySimpleGVK) bool {
+	if o.APIVersion == "velero.io/v1" && o.Kind == "Backup" {
+		return true
+	}
+	if o.APIVersion == "kots.io/v1beta1" {
+		return true
+	}
+	if o.APIVersion == "troubleshoot.replicated.com/v1beta1" {
+		return true
+	}
+	// In addition to kotskinds, we exclude the application crd for now
+	if o.APIVersion == "app.k8s.io/v1beta1" {
+		return true
+	}
+	return false
+}
+
 func (b Base) ListErrorFiles() []BaseFile {
 	files := append([]BaseFile{}, b.ErrorFiles...)
 	for _, b := range b.Bases {

pkg/base/replicated.go

@@ -83,12 +83,32 @@ func renderReplicated(u *upstreamtypes.Upstream, renderOptions *RenderOptions) (
 	}
 
 	for _, upstreamFile := range u.Files {
+		if renderOptions.ExcludeKotsKinds {
+			// kots kinds are not expected to be valid yaml after builder.RenderTemplate
+			// this will prevent errors later from ShouldBeIncludedInBaseKustomization
+			newContent := [][]byte{}
+			isKotsKind := false
+			for _, doc := range convertToSingleDocs(upstreamFile.Content) {
+				file := BaseFile{Path: upstreamFile.Path, Content: doc}
+				// ignore the error here, we will catch it later in ShouldBeIncludedInBaseKustomization
+				if ok, _ := file.IsKotsKind(); ok {
+					isKotsKind = true
+				} else {
+					newContent = append(newContent, doc)
+				}
+			}
+			if isKotsKind && len(newContent) == 0 {
+				continue
+			}
+			upstreamFile.Content = bytes.Join(newContent, []byte("\n---\n"))
+		}
+
 		baseFile, err := upstreamFileToBaseFile(upstreamFile, builder, renderOptions.Log)
 		if err != nil {
 			return nil, errors.Wrapf(err, "failed to convert upstream file %s to base", upstreamFile.Path)
 		}
 
-		baseFiles := convertToSingleDocs([]BaseFile{baseFile})
+		baseFiles := convertToSingleDocBaseFiles([]BaseFile{baseFile})
 		for _, f := range baseFiles {
 			include, err := f.ShouldBeIncludedInBaseKustomization(renderOptions.ExcludeKotsKinds)
 			if err != nil {

pkg/base/write.go

@@ -117,7 +117,7 @@ func deduplicateOnContent(files []BaseFile, excludeKotsKinds bool, baseNS string
 
 	foundGVKNamesMap := map[string]bool{}
 
-	singleDocs := convertToSingleDocs(files)
+	singleDocs := convertToSingleDocBaseFiles(files)
 
 	for _, file := range singleDocs {
 		writeToKustomization, err := file.ShouldBeIncludedInBaseKustomization(excludeKotsKinds)
@@ -149,20 +149,19 @@ func deduplicateOnContent(files []BaseFile, excludeKotsKinds bool, baseNS string
 	return resources, patches, nil
 }
 
-func convertToSingleDocs(files []BaseFile) []BaseFile {
+func convertToSingleDocBaseFiles(files []BaseFile) []BaseFile {
 	singleDocs := []BaseFile{}
 	for _, file := range files {
-		docs := bytes.Split(file.Content, []byte("\n---\n"))
-		if len(docs) == 1 {
-			singleDocs = append(singleDocs, file)
+		docs := convertToSingleDocs(file.Content)
+		// This is here so as not to change previous behavior
+		if len(docs) == 0 {
+			singleDocs = append(singleDocs, BaseFile{
+				Path:    file.Path,
+				Content: []byte(""),
+			})
 			continue
 		}
-
 		for idx, doc := range docs {
-			if len(bytes.TrimSpace(doc)) == 0 {
-				continue
-			}
-
 			filename := file.Path
 			if idx > 0 {
 				filename = strings.TrimSuffix(file.Path, filepath.Ext(file.Path))
@@ -180,6 +179,18 @@ func convertToSingleDocs(files []BaseFile) []BaseFile {
 	return singleDocs
 }
 
+func convertToSingleDocs(doc []byte) [][]byte {
+	singleDocs := [][]byte{}
+	docs := bytes.Split(doc, []byte("\n---\n"))
+	for _, doc := range docs {
+		if len(bytes.TrimSpace(doc)) == 0 {
+			continue
+		}
+		singleDocs = append(singleDocs, doc)
+	}
+	return singleDocs
+}
+
 func (b *Base) GetOverlaysDir(options WriteOptions) string {
 	renderDir := options.BaseDir
 

pkg/base/write_test.go

@@ -1,6 +1,8 @@
 package base
 
 import (
+	"encoding/json"
+	"reflect"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -254,3 +256,56 @@ func Test_DeduplicateOnContent(t *testing.T) {
 	}
 
 }
+
+func Test_convertToSingleDocBaseFiles(t *testing.T) {
+	tests := []struct {
+		name  string
+		files []BaseFile
+		want  []BaseFile
+	}{
+		{
+			name: "basic",
+			files: []BaseFile{
+				{
+					Path:    "/dir/multi.yaml",
+					Content: []byte("---\na: b\n---\nc: d"),
+				},
+				{
+					Path:    "/dir/single.yaml",
+					Content: []byte("e: f"),
+				},
+				{
+					Path:    "/dir/empty.yaml",
+					Content: []byte(""),
+				},
+			},
+			want: []BaseFile{
+				{
+					Path:    "/dir/multi.yaml",
+					Content: []byte("---\na: b"),
+				},
+				{
+					Path:    "/dir/multi-2.yaml",
+					Content: []byte("c: d"),
+				},
+				{
+					Path:    "/dir/single.yaml",
+					Content: []byte("e: f"),
+				},
+				{
+					Path:    "/dir/empty.yaml",
+					Content: []byte(""),
+				},
+			},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got := convertToSingleDocBaseFiles(tt.files); !reflect.DeepEqual(got, tt.want) {
+				gotB, _ := json.MarshalIndent(got, "", "  ")
+				wantB, _ := json.MarshalIndent(tt.want, "", "  ")
+				t.Errorf("convertToSingleDocBaseFiles() = %s, want %s", gotB, wantB)
+			}
+		})
+	}
+}