Add yaml errors to file tree (#843)

* Add yaml errors to file tree * skipped * revert * always create dir * fix test
replicatedhq · Jul 24, 2020 · 8c753b6 · 8c753b6
1 parent 93d9a9b
commit 8c753b6
Show file tree

Hide file tree

Showing 13 changed files with 97 additions and 0 deletions.
diff --git a/ffi/main.go b/ffi/main.go
@@ -98,6 +98,7 @@ func UpdateCheck(socket, fromArchivePath, namespace string) {
 			filepath.Join(tmpRoot, "upstream"),
 			filepath.Join(tmpRoot, "base"),
 			filepath.Join(tmpRoot, "overlays"),
+			filepath.Join(tmpRoot, "skippedFiles"),
 		}
 
 		err = os.Remove(fromArchivePath)

diff --git a/ffi/online.go b/ffi/online.go
@@ -161,6 +161,7 @@ func RewriteVersion(socket, fromArchivePath, outputFile, downstreamsStr, k8sName
 			filepath.Join(tmpRoot, "upstream"),
 			filepath.Join(tmpRoot, "base"),
 			filepath.Join(tmpRoot, "overlays"),
+			filepath.Join(tmpRoot, "skippedFiles"),
 		}
 
 		if err := tarGz.Archive(paths, outputFile); err != nil {

diff --git a/ffi/updatedownload.go b/ffi/updatedownload.go
@@ -119,6 +119,7 @@ func UpdateDownload(socket, fromArchivePath, namespace, registryJson, cursor str
 			filepath.Join(tmpRoot, "upstream"),
 			filepath.Join(tmpRoot, "base"),
 			filepath.Join(tmpRoot, "overlays"),
+			filepath.Join(tmpRoot, "skippedFiles"),
 		}
 
 		err = os.Remove(fromArchivePath)
@@ -293,6 +294,7 @@ func UpdateDownloadFromAirgap(socket, fromArchivePath, namespace, registryJson,
 			filepath.Join(tmpRoot, "upstream"),
 			filepath.Join(tmpRoot, "base"),
 			filepath.Join(tmpRoot, "overlays"),
+			filepath.Join(tmpRoot, "skippedFiles"),
 		}
 
 		err = os.Remove(fromArchivePath)

diff --git a/integration/replicated/generate.go b/integration/replicated/generate.go
@@ -144,6 +144,7 @@ func generateExpectedFilesystem(namespace, rawArchivePath string) ([]byte, error
 		path.Join(tmpRootDir, "upstream"),
 		path.Join(tmpRootDir, "base"),
 		path.Join(tmpRootDir, "overlays"),
+		path.Join(tmpRootDir, "skippedFiles"),
 	}
 	if err := tarGz.Archive(paths, archiveFile); err != nil {
 		return nil, errors.Wrap(err, "failed to create archive")

diff --git a/integration/upload/generate.go b/integration/upload/generate.go
@@ -37,6 +37,7 @@ func GenerateTest(name string, applicationPath string) error {
 		path.Join(applicationPath, "upstream"),
 		path.Join(applicationPath, "base"),
 		path.Join(applicationPath, "overlays"),
+		path.Join(applicationPath, "skippedFiles"),
 	}
 
 	if err := tarGz.Archive(paths, path.Join(testRoot, "expected-archive.tar.gz")); err != nil {

diff --git a/kotsadm/pkg/handlers/download.go b/kotsadm/pkg/handlers/download.go
@@ -86,6 +86,7 @@ func DownloadApp(w http.ResponseWriter, r *http.Request) {
 		filepath.Join(archivePath, "upstream"),
 		filepath.Join(archivePath, "base"),
 		filepath.Join(archivePath, "overlays"),
+		filepath.Join(archivePath, "skippedFiles"),
 	}
 	tmpDir, err := ioutil.TempDir("", "kotsadm")
 	if err != nil {

diff --git a/kotsadm/pkg/version/archive.go b/kotsadm/pkg/version/archive.go
@@ -32,6 +32,7 @@ func CreateAppVersionArchive(appID string, sequence int64, archivePath string) e
 		filepath.Join(archivePath, "upstream"),
 		filepath.Join(archivePath, "base"),
 		filepath.Join(archivePath, "overlays"),
+		filepath.Join(archivePath, "skippedFiles"),
 	}
 
 	tmpDir, err := ioutil.TempDir("", "kotsadm")

diff --git a/pkg/base/write.go b/pkg/base/write.go
@@ -11,11 +11,13 @@ import (
 
 	"github.com/pkg/errors"
 	"github.com/replicatedhq/kots/pkg/k8sutil"
+	"gopkg.in/yaml.v2"
 	kustomizetypes "sigs.k8s.io/kustomize/api/types"
 )
 
 type WriteOptions struct {
 	BaseDir          string
+	SkippedDir       string
 	Overwrite        bool
 	ExcludeKotsKinds bool
 }
@@ -108,6 +110,76 @@ func (b *Base) WriteBase(options WriteOptions) error {
 		return errors.Wrap(err, "failed to write kustomization to file")
 	}
 
+	if err := b.writeSkippedFiles(options); err != nil {
+		return errors.Wrap(err, "failed to write skipped files")
+	}
+
+	return nil
+}
+
+type SkippedFilesIndex struct {
+	SkippedFiles []SkippedFile `yaml:"skippedFiles"`
+}
+
+type SkippedFile struct {
+	Path   string `yaml:"path"`
+	Reason string `yaml:"reason"`
+}
+
+func (b *Base) writeSkippedFiles(options WriteOptions) error {
+	// if we dont render this dir we will get an error when we create the archive
+	renderDir := filepath.Join(options.SkippedDir, b.Path)
+
+	_, err := os.Stat(renderDir)
+	if err == nil {
+		if options.Overwrite {
+			if err := os.RemoveAll(renderDir); err != nil {
+				return errors.Wrap(err, "failed to remove previous content in skipped files")
+			}
+		} else {
+			return fmt.Errorf("directory %s already exists", renderDir)
+		}
+	}
+
+	if _, err := os.Stat(renderDir); os.IsNotExist(err) {
+		if err := os.MkdirAll(renderDir, 0744); err != nil {
+			return errors.Wrap(err, "failed to mkdir for skipped files root")
+		}
+	}
+
+	if len(b.ErrorFiles) == 0 {
+		return nil
+	}
+
+	index := SkippedFilesIndex{SkippedFiles: []SkippedFile{}}
+	for _, file := range b.ErrorFiles {
+		fileRenderPath := path.Join(renderDir, file.Path)
+		d := path.Dir(fileRenderPath)
+		if _, err := os.Stat(d); os.IsNotExist(err) {
+			if err := os.MkdirAll(d, 0744); err != nil {
+				return errors.Wrap(err, "failed to mkdir")
+			}
+		}
+
+		if err := ioutil.WriteFile(fileRenderPath, file.Content, 0644); err != nil {
+			return errors.Wrapf(err, "failed to write skipped file %s", fileRenderPath)
+		}
+
+		index.SkippedFiles = append(index.SkippedFiles, SkippedFile{
+			Path:   file.Path,
+			Reason: fmt.Sprintf("%v", file.Error),
+		})
+	}
+
+	indexOut, err := yaml.Marshal(index)
+	if err != nil {
+		return errors.Wrap(err, "failed to marshal skipped files index")
+	}
+	fileRenderPath := path.Join(renderDir, "_index.yaml")
+	if err := ioutil.WriteFile(fileRenderPath, indexOut, 0644); err != nil {
+		return errors.Wrap(err, "failed to write skipped files index")
+	}
+
 	return nil
 }
 

diff --git a/pkg/pull/archive.go b/pkg/pull/archive.go
@@ -32,6 +32,7 @@ func writeArchiveAsConfigMap(pullOptions PullOptions, u *upstreamtypes.Upstream,
 		path.Join(pullOptions.RootDir, u.Name, "upstream"),
 		path.Join(pullOptions.RootDir, u.Name, "base"),
 		path.Join(pullOptions.RootDir, u.Name, "overlays"),
+		path.Join(pullOptions.RootDir, u.Name, "skippedFiles"),
 	}
 
 	tempDir, err := ioutil.TempDir("", "kots")

diff --git a/pkg/pull/pull.go b/pkg/pull/pull.go
@@ -263,6 +263,7 @@ func Pull(upstreamURI string, pullOptions PullOptions) (string, error) {
 
 	writeBaseOptions := base.WriteOptions{
 		BaseDir:          u.GetBaseDir(writeUpstreamOptions),
+		SkippedDir:       u.GetSkippedDir(writeUpstreamOptions),
 		Overwrite:        true,
 		ExcludeKotsKinds: pullOptions.ExcludeKotsKinds,
 	}

diff --git a/pkg/rewrite/rewrite.go b/pkg/rewrite/rewrite.go
@@ -135,6 +135,7 @@ func Rewrite(rewriteOptions RewriteOptions) error {
 
 	writeBaseOptions := base.WriteOptions{
 		BaseDir:          u.GetBaseDir(writeUpstreamOptions),
+		SkippedDir:       u.GetSkippedDir(writeUpstreamOptions),
 		Overwrite:        true,
 		ExcludeKotsKinds: rewriteOptions.ExcludeKotsKinds,
 	}

diff --git a/pkg/upload/archive.go b/pkg/upload/archive.go
@@ -29,6 +29,11 @@ func createUploadableArchive(rootPath string) (string, error) {
 		path.Join(rootPath, "overlays"),
 	}
 
+	skippedFilesPath := path.Join(rootPath, "skippedFiles")
+	if _, err := os.Stat(skippedFilesPath); err == nil {
+		paths = append(paths, skippedFilesPath)
+	}
+
 	// the caller of this function is repsonsible for deleting this file
 	tempDir, err := ioutil.TempDir("", "kots")
 	if err != nil {

diff --git a/pkg/upstream/types/types.go b/pkg/upstream/types/types.go
@@ -56,3 +56,12 @@ func (u *Upstream) GetBaseDir(options WriteOptions) string {
 
 	return path.Join(renderDir, "base")
 }
+
+func (u *Upstream) GetSkippedDir(options WriteOptions) string {
+	renderDir := options.RootDir
+	if options.CreateAppDir {
+		renderDir = path.Join(renderDir, u.Name)
+	}
+
+	return path.Join(renderDir, "skippedFiles")
+}