Allow resources to be in subdirectories (#1700)

Co-authored-by: Ken Sipe <kensipe@gmail.com>
Signed-off-by: Andreas Neumann <aneumann@mesosphere.com>

pkg/kudoctl/cmd/generate/testdata/maintainer.golden

@@ -45,6 +45,7 @@ tasks:
   spec:
     resources:
     - statefulset.yaml
+    - subfolder/statefulset2.yaml
 - kind: Apply
   name: validation
   spec:

pkg/kudoctl/cmd/generate/testdata/plan.golden

@@ -52,6 +52,7 @@ tasks:
   spec:
     resources:
     - statefulset.yaml
+    - subfolder/statefulset2.yaml
 - kind: Apply
   name: validation
   spec:

pkg/kudoctl/cmd/generate/testdata/task.golden

@@ -43,6 +43,7 @@ tasks:
   spec:
     resources:
     - statefulset.yaml
+    - subfolder/statefulset2.yaml
 - kind: Apply
   name: validation
   spec:

pkg/kudoctl/cmd/testdata/index.yaml.golden

@@ -2,7 +2,7 @@ apiVersion: v1
 entries:
   zookeeper:
   - appVersion: 3.4.10
-    digest: 050c5e67279c41e40e054d0e93d1361187ba8c9a289b5a00eb1c68cef4f0b18e
+    digest: 3b25556108002d8e922bb381c936827d4e34586a57453c286b2d9ed39cb129aa
     maintainers:
     - email: avarkockova@mesosphere.com
       name: Alena Varkockova

pkg/kudoctl/cmd/testdata/plans-with-tasks-list.golden

@@ -6,7 +6,8 @@ plans
 │           │   ├── services.yaml
 │           │   └── pdb.yaml
 │           └── [apply]  app
-│               └── statefulset.yaml
+│               ├── statefulset.yaml
+│               └── subfolder/statefulset2.yaml
 └── validation (serial)
     └── [phase]  connection (parallel)
         └── [step]  connection

pkg/kudoctl/cmd/testdata/tasks-list.golden

@@ -1,4 +1,4 @@
-Name      	Kind 	Resources               
-infra     	Apply	[services.yaml pdb.yaml]
-app       	Apply	[statefulset.yaml]      
-validation	Apply	[validation.yaml]       
+Name      	Kind 	Resources                                     
+infra     	Apply	[services.yaml pdb.yaml]                      
+app       	Apply	[statefulset.yaml subfolder/statefulset2.yaml]
+validation	Apply	[validation.yaml]                             

pkg/kudoctl/files/file.go

@@ -17,9 +17,10 @@ import (
 // CopyOperatorToFs used with afero usually for tests to copy files into a filesystem.
 // copy from local file system into in mem
 func CopyOperatorToFs(fs afero.Fs, opath string, base string) {
-	dir := filepath.Clean(base)
+	baseDir := filepath.Clean(base)
+	root, _ := filepath.Split(opath)
 	failed := false
-	err := fs.MkdirAll(dir, 0755)
+	err := fs.MkdirAll(baseDir, 0755)
 	if err != nil {
 		fmt.Println("FAILED: ", err)
 		failed = true
@@ -28,16 +29,16 @@ func CopyOperatorToFs(fs afero.Fs, opath string, base string) {
 		if e != nil {
 			return e
 		}
-
+		// remove original path base
+		fileBase := filepath.Clean(strings.Replace(path, root, "", 1))
+		// add to new base dir
+		file := filepath.Join(baseDir, fileBase)
 		// directory copy
 		if info.IsDir() {
-			if dir != info.Name() {
-				dir = filepath.Join(dir, info.Name())
-				err := fs.MkdirAll(dir, 0755)
-				if err != nil {
-					fmt.Println("FAILED: ", err)
-					failed = true
-				}
+			err := fs.MkdirAll(file, 0755)
+			if err != nil {
+				fmt.Println("FAILED: ", err)
+				failed = true
 			}
 			return nil
 		}
@@ -46,18 +47,17 @@ func CopyOperatorToFs(fs afero.Fs, opath string, base string) {
 			return errors.New("unable to write file, as mkdir failed")
 		}
 
-		fn := filepath.Join(dir, info.Name())
-		fmt.Println(fn)
+		fmt.Println(file)
 
-		w, err := fs.Create(fn)
+		w, err := fs.Create(file)
 		if err != nil {
 			fmt.Println("FAILED: ", err)
-			return fmt.Errorf("unable to create file %s", fn)
+			return fmt.Errorf("unable to create file %s", file)
 		}
 		defer func() {
 			if ferr := w.Close(); ferr != nil {
 				fmt.Println("FAILED: ", err)
-				err = fmt.Errorf("unable to close file %s", fn)
+				err = fmt.Errorf("unable to close file %s", file)
 			}
 		}()
 
@@ -76,7 +76,7 @@ func CopyOperatorToFs(fs afero.Fs, opath string, base string) {
 		_, err = io.Copy(w, r)
 		if err != nil {
 			fmt.Println("FAILED: ", err)
-			return fmt.Errorf("unable to copy from %s into %s", fn, path)
+			return fmt.Errorf("unable to copy from %s into %s", file, path)
 		}
 
 		return nil

pkg/kudoctl/packages/reader/parser.go

@@ -38,16 +38,20 @@ func parsePackageFile(filePath string, fileBytes []byte, currentPackage *package
 	}
 
 	isTemplateFile := func(name string) bool {
+		name = filepath.Clean(name)
+
 		dir, file := filepath.Split(name)
-		base := filepath.Base(dir)
+		if !strings.HasPrefix(dir, templateBase) {
+			return false
+		}
 
 		match, err := regexp.MatchString(templateFileName, file)
 		if err != nil {
 			clog.Printf("Failed to parse template file %s, err: %v", name, err)
 			os.Exit(1)
 		}
 
-		return base == templateBase && match
+		return match
 	}
 
 	isParametersFile := func(name string) bool {

pkg/kudoctl/packages/reader/parser_test.go

@@ -2,6 +2,7 @@ package reader
 
 import (
 	"errors"
+	"path/filepath"
 	"strings"
 	"testing"
 
@@ -35,38 +36,48 @@ func TestParsePackageFile(t *testing.T) {
 	}{
 		{filePath: "operator.yaml", fileContent: validOperator, isOperator: true},
 		{filePath: "params.yaml", fileContent: validParams, isParam: true},
-		{filePath: "templates/pod-params.yaml", isTemplate: true},
-		{filePath: "templates/pod-operator.yaml", isTemplate: true},
-		{filePath: "templates/some-template.yaml", isTemplate: true},
+		{filePath: "templates/pod-params.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "templates/pod-operator.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "templates/some-template.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "templates/some/template.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "templates/some/nested/template2.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "./templates/some-template.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "./templates/with/subdirectory/some-template.yaml", isTemplate: true, fileContent: "not-empty"},
+		{filePath: "templates_without_path.yaml", expectedError: errors.New("unexpected file when reading package from filesystem: templates_without_path.yaml"), fileContent: "not-empty"},
+		{filePath: "invalid.yaml", expectedError: errors.New("unexpected file when reading package from filesystem: invalid.yaml")},
 		{filePath: "operator.yaml", isOperator: true, expectedError: errors.New("failed to parse yaml into valid operator operator.yaml")},
 	}
 
 	for _, tt := range tests {
 		tt := tt
 
-		pf := newPackageFiles()
+		t.Run(tt.filePath, func(t *testing.T) {
+			pf := newPackageFiles()
 
-		err := parsePackageFile(tt.filePath, []byte(tt.fileContent), &pf)
+			err := parsePackageFile(tt.filePath, []byte(tt.fileContent), &pf)
 
-		if tt.expectedError != nil {
-			assert.Equal(t, tt.expectedError.Error(), err.Error())
-			continue
-		} else {
+			if tt.expectedError != nil {
+				assert.NotNil(t, err, "Expected error but got none")
+				if err != nil {
+					assert.Equal(t, tt.expectedError.Error(), err.Error())
+				}
+				return
+			}
 			assert.Nil(t, err)
-		}
 
-		if tt.isOperator {
-			assert.NotNil(t, pf.Operator, "%v was not parsed as an operator file", tt.filePath)
-		}
-		if tt.isParam {
-			assert.NotNil(t, pf.Params, "%v was not parsed as a param file", tt.filePath)
-		}
-		if tt.isTemplate {
-			assert.Equal(t, 1, len(pf.Templates), "%v was not parsed as a template file", tt.filePath)
-
-			fileName := strings.TrimPrefix(tt.filePath, "templates/")
-			assert.NotNil(t, pf.Templates[fileName], "%v was not stored in template map", tt.filePath)
-		}
+			if tt.isOperator {
+				assert.NotNil(t, pf.Operator, "%v was not parsed as an operator file", tt.filePath)
+			}
+			if tt.isParam {
+				assert.NotNil(t, pf.Params, "%v was not parsed as a param file", tt.filePath)
+			}
+			if tt.isTemplate {
+				assert.Equal(t, 1, len(pf.Templates), "%v was not parsed as a template file", tt.filePath)
 
+				fileName := filepath.Clean(tt.filePath)
+				fileName = strings.TrimPrefix(fileName, "templates/")
+				assert.NotEmpty(t, pf.Templates[fileName], "%v was not stored in template map", tt.filePath)
+			}
+		})
 	}
 }

pkg/kudoctl/packages/testdata/zk-crd-golden1/operatorversion.golden

@@ -159,6 +159,94 @@ spec:
               resources:
                 requests:
                   storage: 2Gi
+    subfolder/statefulset2.yaml: |
+      apiVersion: apps/v1
+      kind: StatefulSet
+      metadata:
+        name: {{ .Name }}
+        namespace: {{ .Namespace }}
+      spec:
+        selector:
+          matchLabels:
+            app: zookeeper
+            zookeeper: {{ .Name }}
+        serviceName: {{ .Name }}-hs
+        replicas: 3
+        updateStrategy:
+          type: RollingUpdate
+        podManagementPolicy: Parallel
+        template:
+          metadata:
+            labels:
+              app: zookeeper
+              zookeeper: {{ .Name }}
+          spec:
+            containers:
+              - name: kubernetes-zookeeper
+                imagePullPolicy: Always
+                image: "k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10"
+                resources:
+                  requests:
+                    memory: "{{ .Params.memory }}"
+                    cpu: "{{ .Params.cpus }}"
+                ports:
+                  - containerPort: 2181
+                    name: client
+                  - containerPort: 2888
+                    name: server
+                  - containerPort: 3888
+                    name: leader-election
+                command:
+                  - sh
+                  - -c
+                  - "start-zookeeper \
+                    --servers=3 \
+                    --data_dir=/var/lib/zookeeper/data \
+                    --data_log_dir=/var/lib/zookeeper/data/log \
+                    --conf_dir=/opt/zookeeper/conf \
+                    --client_port=2181 \
+                    --election_port=3888 \
+                    --server_port=2888 \
+                    --tick_time=2000 \
+                    --init_limit=10 \
+                    --sync_limit=5 \
+                    --heap=512M \
+                    --max_client_cnxns=60 \
+                    --snap_retain_count=3 \
+                    --purge_interval=12 \
+                    --max_session_timeout=40000 \
+                    --min_session_timeout=4000 \
+                    --log_level=INFO"
+                readinessProbe:
+                  exec:
+                    command:
+                      - sh
+                      - -c
+                      - "zookeeper-ready 2181"
+                  initialDelaySeconds: 10
+                  timeoutSeconds: 5
+                livenessProbe:
+                  exec:
+                    command:
+                      - sh
+                      - -c
+                      - "zookeeper-ready 2181"
+                  initialDelaySeconds: 10
+                  timeoutSeconds: 5
+                volumeMounts:
+                  - name: datadir
+                    mountPath: /var/lib/zookeeper
+            securityContext:
+              runAsUser: 1000
+              fsGroup: 1000
+        volumeClaimTemplates:
+          - metadata:
+              name: datadir
+            spec:
+              accessModes: ["ReadWriteOnce"]
+              resources:
+                requests:
+                  storage: 2Gi
     validation.yaml: |
       apiVersion: batch/v1
       kind: Job
@@ -199,6 +287,7 @@ spec:
       spec:
         resources:
           - statefulset.yaml
+          - subfolder/statefulset2.yaml
     - name: validation
       kind: Apply
       spec: