Adding sparseCheckoutDir functionality to devfiles (redhat-developer#…

…3042)

* Initial commit

* added devfile

* added tests

* commented out tests

* change to validpath statement

* changed method of extracting zip

* changed pathsToUnzip to pathToUnzip

* Added error message when no files are unzipped

* cleaned up conditional prefix trim

* Changes from feedback

* feedback changes and unit tests for util func

* check for empty pathToUnzip

* changed error message format

* cleaned up path separator for windows

* fixed return pathToUnzip

* use hassuffix

* moved to function

* fromslash

* minor fixes

* lowercase fix for test

pkg/odo/cli/component/create.go

@@ -791,7 +791,8 @@ func (co *CreateOptions) downloadProject(projectPassed string) error {
 		return errors.Errorf("Project type not supported")
 	}
 
-	err = util.GetAndExtractZip(url, path)
+	err = checkoutProject(project, url, path)
+
 	if err != nil {
 		return err
 	}
@@ -903,6 +904,30 @@ func ensureAndLogProperResourceUsage(resource, resourceMin, resourceMax, resourc
 	}
 }
 
+func checkoutProject(project common.DevfileProject, zipURL, path string) error {
+	var sparseCheckoutDir string
+	if project.Git.SparseCheckoutDir != "" {
+		sparseCheckoutDir = project.Git.SparseCheckoutDir
+	} else if project.Github.SparseCheckoutDir != "" {
+		sparseCheckoutDir = project.Github.SparseCheckoutDir
+	} else if project.Zip.SparseCheckoutDir != "" {
+		sparseCheckoutDir = project.Zip.SparseCheckoutDir
+	}
+	if sparseCheckoutDir != "" {
+		err := util.GetAndExtractZip(zipURL, path, sparseCheckoutDir)
+		if err != nil {
+			return errors.Wrap(err, "failed to download and extract project zip folder")
+		}
+	} else {
+		// extract project to current working directory
+		err := util.GetAndExtractZip(zipURL, path, "/")
+		if err != nil {
+			return errors.Wrap(err, "failed to download and extract project zip folder")
+		}
+	}
+	return nil
+}
+
 // NewCmdCreate implements the create odo command
 func NewCmdCreate(name, fullName string) *cobra.Command {
 	co := NewCreateOptions()

pkg/util/util.go

@@ -812,8 +812,9 @@ func GetGitHubZipURL(repoURL string) (string, error) {
 }
 
 // GetAndExtractZip downloads a zip file from a URL with a http prefix or
-// takes an absolute path prefixed with file:// and extracts it to a destination
-func GetAndExtractZip(zipURL string, destination string) error {
+// takes an absolute path prefixed with file:// and extracts it to a destination.
+// pathToUnzip specifies the path within the zip folder to extract
+func GetAndExtractZip(zipURL string, destination string, pathToUnzip string) error {
 	if zipURL == "" {
 		return errors.Errorf("Empty zip url: %s", zipURL)
 	}
@@ -847,18 +848,23 @@ func GetAndExtractZip(zipURL string, destination string) error {
 		return errors.Errorf("Invalid Zip URL: %s . Should either be prefixed with file://, http:// or https://", zipURL)
 	}
 
-	_, err := Unzip(pathToZip, destination)
+	filenames, err := Unzip(pathToZip, destination, pathToUnzip)
 	if err != nil {
 		return err
 	}
 
+	if len(filenames) == 0 {
+		return errors.New("no files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path")
+	}
+
 	return nil
 }
 
-// Unzip will decompress a zip archive, moving all files and folders
-// within the zip file (parameter 1) to an output directory (parameter 2).
+// Unzip will decompress a zip archive, moving specified files and folders
+// within the zip file (parameter 1) to an output directory (parameter 2)
 // Source: https://golangcode.com/unzip-files-in-go/
-func Unzip(src, dest string) ([]string, error) {
+// pathToUnzip (parameter 3) is the path within the zip folder to extract
+func Unzip(src, dest, pathToUnzip string) ([]string, error) {
 	var filenames []string
 
 	r, err := zip.OpenReader(src)
@@ -867,16 +873,46 @@ func Unzip(src, dest string) ([]string, error) {
 	}
 	defer r.Close()
 
-	for _, f := range r.File {
+	// change path separator to correct character
+	pathToUnzip = filepath.FromSlash(pathToUnzip)
 
+	for _, f := range r.File {
 		// Store filename/path for returning and using later on
-		index := strings.Index(f.Name, "/")
+		index := strings.Index(f.Name, string(os.PathSeparator))
 		filename := f.Name[index+1:]
 		if filename == "" {
 			continue
 		}
+
+		// if sparseCheckoutDir has a pattern
+		match, err := filepath.Match(pathToUnzip, filename)
+		if err != nil {
+			return filenames, err
+		}
+
+		// removes first slash of pathToUnzip if present, adds trailing slash
+		pathToUnzip = strings.TrimPrefix(pathToUnzip, string(os.PathSeparator))
+		if pathToUnzip != "" && !strings.HasSuffix(pathToUnzip, string(os.PathSeparator)) {
+			pathToUnzip = pathToUnzip + string(os.PathSeparator)
+		}
+		// destination filepath before trim
 		fpath := filepath.Join(dest, filename)
 
+		// used for pattern matching
+		fpathDir := filepath.Dir(fpath)
+
+		// check for prefix or match
+		if strings.HasPrefix(filename, pathToUnzip) {
+			filename = strings.TrimPrefix(filename, pathToUnzip)
+		} else if !strings.HasPrefix(filename, pathToUnzip) && !match && !sliceContainsString(fpathDir, filenames) {
+			continue
+		}
+		// adds trailing slash to destination if needed as filepath.Join removes it
+		if (len(filename) == 1 && os.IsPathSeparator(filename[0])) || filename == "" {
+			fpath = dest + string(os.PathSeparator)
+		} else {
+			fpath = filepath.Join(dest, filename)
+		}
 		// Check for ZipSlip. More Info: http://bit.ly/2MsjAWE
 		if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) {
 			return filenames, fmt.Errorf("%s: illegal file path", fpath)
@@ -1009,3 +1045,13 @@ func ValidateURL(sourceURL string) error {
 
 	return nil
 }
+
+// sliceContainsString checks for existence of given string in given slice
+func sliceContainsString(str string, slice []string) bool {
+	for _, b := range slice {
+		if b == str {
+			return true
+		}
+	}
+	return false
+}

pkg/util/util_test.go

@@ -1652,3 +1652,41 @@ func TestValidateURL(t *testing.T) {
 		})
 	}
 }
+
+func TestSliceContainsString(t *testing.T) {
+	tests := []struct {
+		name      string
+		stringVal string
+		slice     []string
+		wantVal   bool
+	}{
+		{
+			name:      "Case 1: string in valid slice",
+			stringVal: "string",
+			slice:     []string{"string", "string2"},
+			wantVal:   true,
+		},
+		{
+			name:      "Case 2: string not in valid slice",
+			stringVal: "string3",
+			slice:     []string{"string", "string2"},
+			wantVal:   false,
+		},
+		{
+			name:      "Case 3: string not in empty slice",
+			stringVal: "string",
+			slice:     []string{},
+			wantVal:   false,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotVal := sliceContainsString(tt.stringVal, tt.slice)
+
+			if !reflect.DeepEqual(gotVal, tt.wantVal) {
+				t.Errorf("Got %v, want %v", gotVal, tt.wantVal)
+			}
+		})
+	}
+}

tests/examples/source/devfiles/nodejs/devfile-with-sparseCheckoutDir.yaml

@@ -0,0 +1,47 @@
+apiVersion: 1.0.0
+metadata:
+  name: test-devfile
+projects:
+  -
+    name: nodejs-web-app
+    source:
+      type: git
+      location: "https://github.com/che-samples/web-nodejs-sample.git"
+      sparseCheckoutDir: /app/
+components:
+  - type: dockerimage
+    image: quay.io/eclipse/che-nodejs10-ubi:nightly
+    endpoints:
+      - name: "3000/tcp"
+        port: 3000
+    alias: runtime
+    env:
+      - name: FOO
+        value: "bar"
+    memoryLimit: 1024Mi
+    mountSources: true
+commands:
+  - name: build
+    actions:
+      - type: exec
+        component: runtime
+        command: "npm install"
+        workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app
+  - name: devbuild
+    actions:
+      - type: exec
+        component: runtime
+        command: "npm install"
+        workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app
+  - name: run
+    actions:
+      - type: exec
+        component: runtime
+        command: "nodemon app.js"
+        workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app
+  - name: devrun
+    actions:
+      - type: exec
+        component: runtime
+        command: "nodemon app.js"
+        workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app

tests/integration/devfile/cmd_devfile_create_test.go

@@ -230,4 +230,36 @@ var _ = Describe("odo devfile create command tests", func() {
 	//		helper.DeleteDir(contextDevfile)
 	//	})
 	//})
+
+	// Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has a valid value", func() {
+	// 	It("should only extract the specified path in the sparseContextDir field", func() {
+	// 		helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true")
+	// 		contextDevfile := helper.CreateNewContext()
+	// 		helper.Chdir(contextDevfile)
+	// 		devfile := "devfile.yaml"
+	// 		helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), filepath.Join(contextDevfile, devfile))
+	// 		componentNamespace := helper.RandString(6)
+	// 		helper.CmdShouldPass("odo", "create", "--downloadSource", "--project", componentNamespace)
+	// 		expectedFiles := []string{"app.js", devfile}
+	// 		Expect(helper.VerifyFilesExist(contextDevfile, expectedFiles)).To(Equal(true))
+	// 		helper.DeleteDir(contextDevfile)
+	// 	})
+	// })
+
+	// Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has an invalid value", func() {
+	// 	It("should fail and alert the user that the specified path in sparseContextDir does not exist", func() {
+	// 		helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true")
+	// 		contextDevfile := helper.CreateNewContext()
+	// 		helper.Chdir(contextDevfile)
+	// 		devfile := "devfile.yaml"
+	// 		devfilePath := filepath.Join(contextDevfile, devfile)
+	// 		helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), devfilePath)
+	// 		helper.ReplaceDevfileField(devfilePath, "sparseCheckoutDir", "/invalid/")
+	// 		componentNamespace := helper.RandString(6)
+	// 		output := helper.CmdShouldFail("odo", "create", "--downloadSource", "--project", componentNamespace)
+	// 		expectedString := "no files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path"
+	// 		helper.MatchAllInOutput(output, []string{expectedString})
+	// 		helper.DeleteDir(contextDevfile)
+	// 	})
+	// })
 })