From 63bd13a95a96278fb7f096de6ff5d100b605a969 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Tue, 28 Apr 2020 10:05:17 +0100 Subject: [PATCH 01/20] Initial commit --- pkg/odo/cli/component/create.go | 35 +++++++++++++++++++++-- pkg/util/util.go | 49 +++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 3 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 4893fb3659f..a5ac9aa4195 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -759,9 +759,38 @@ func (co *CreateOptions) downloadProject() error { return errors.Errorf("Project type not supported") } - err = util.GetAndExtractZip(zipUrl, path) - if err != nil { - return err + if project.Source.SparseCheckoutDir != nil && *project.Source.SparseCheckoutDir != "" { + + // extract project to an extract folder + extractFolder := filepath.Join(path, "extract") + err = util.GetAndExtractZip(zipUrl, extractFolder) + if err != nil { + return err + } + + defer os.RemoveAll(extractFolder) + + sparseCheckoutDir := *project.Source.SparseCheckoutDir + fullSparseCheckoutDirPath := filepath.Join(extractFolder, sparseCheckoutDir) + if _, err := os.Stat(fullSparseCheckoutDirPath); os.IsNotExist(err) { + return errors.Errorf("Path specified in sparseCheckoutDir does not exist in project", err) + } + + info, err := os.Stat(path) + if err != nil { + return err + } + + err = util.CopyDir(fullSparseCheckoutDirPath, path, info) + if err != nil { + return errors.Errorf("Error copying sparseCheckoutDir to project path") + } + + } else { + err = util.GetAndExtractZip(zipUrl, path) + if err != nil { + return err + } } return nil diff --git a/pkg/util/util.go b/pkg/util/util.go index 5e03dedfe50..1db6c2d0bad 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -982,3 +982,52 @@ func CheckKubeConfigExist() bool { return false } + +// copyDir copy one directory to the other +// this function is called recursively info should start as os.Stat(src) +func CopyDir(src string, dst string, info os.FileInfo) error { + + if info.IsDir() { + files, err := ioutil.ReadDir(src) + if err != nil { + return err + } + + for _, file := range files { + dsrt := filepath.Join(src, file.Name()) + ddst := filepath.Join(dst, file.Name()) + if err := CopyDir(dsrt, ddst, file); err != nil { + return err + } + } + return nil + } + + if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { + return err + } + + return CopyFile(src, dst, info) +} + +// copyFile copy one file to another location +func CopyFile(src, dst string, info os.FileInfo) error { + dFile, err := os.Create(dst) + if err != nil { + return err + } + defer dFile.Close() // #nosec G307 + + sFile, err := os.Open(src) + if err != nil { + return err + } + defer sFile.Close() // #nosec G307 + + if err = os.Chmod(dFile.Name(), info.Mode()); err != nil { + return err + } + + _, err = io.Copy(dFile, sFile) + return err +} From 2462f56775ab0b71f849b1383a6331d8edae4f09 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Tue, 28 Apr 2020 15:14:53 +0100 Subject: [PATCH 02/20] added devfile --- pkg/odo/cli/component/create.go | 1 + .../devfile-with-sparseCheckoutDir.yaml | 47 +++++++++++++++++++ .../devfile/cmd_devfile_create_test.go | 14 ++++++ 3 files changed, 62 insertions(+) create mode 100644 tests/examples/source/devfiles/nodejs/devfile-with-sparseCheckoutDir.yaml diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 2efe89f55cd..1ab17ee7fca 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -772,6 +772,7 @@ func (co *CreateOptions) downloadProject() error { sparseCheckoutDir := *project.Source.SparseCheckoutDir fullSparseCheckoutDirPath := filepath.Join(extractFolder, sparseCheckoutDir) + invalidPath := util.CheckPathExists() if _, err := os.Stat(fullSparseCheckoutDirPath); os.IsNotExist(err) { return errors.Errorf("Path specified in sparseCheckoutDir does not exist in project", err) } diff --git a/tests/examples/source/devfiles/nodejs/devfile-with-sparseCheckoutDir.yaml b/tests/examples/source/devfiles/nodejs/devfile-with-sparseCheckoutDir.yaml new file mode 100644 index 00000000000..d7d3fc2f5f5 --- /dev/null +++ b/tests/examples/source/devfiles/nodejs/devfile-with-sparseCheckoutDir.yaml @@ -0,0 +1,47 @@ +apiVersion: 1.0.0 +metadata: + name: test-devfile +projects: + - + name: nodejs-web-app + source: + type: git + location: "https://github.com/che-samples/web-nodejs-sample.git" + sparseCheckoutDir: /app/ +components: + - type: dockerimage + image: quay.io/eclipse/che-nodejs10-ubi:nightly + endpoints: + - name: "3000/tcp" + port: 3000 + alias: runtime + env: + - name: FOO + value: "bar" + memoryLimit: 1024Mi + mountSources: true +commands: + - name: build + actions: + - type: exec + component: runtime + command: "npm install" + workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app + - name: devbuild + actions: + - type: exec + component: runtime + command: "npm install" + workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app + - name: run + actions: + - type: exec + component: runtime + command: "nodemon app.js" + workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app + - name: devrun + actions: + - type: exec + component: runtime + command: "nodemon app.js" + workdir: ${CHE_PROJECTS_ROOT}/nodejs-web-app/app diff --git a/tests/integration/devfile/cmd_devfile_create_test.go b/tests/integration/devfile/cmd_devfile_create_test.go index e9e58702a59..d38ea3fd5dd 100644 --- a/tests/integration/devfile/cmd_devfile_create_test.go +++ b/tests/integration/devfile/cmd_devfile_create_test.go @@ -184,4 +184,18 @@ var _ = Describe("odo devfile create command tests", func() { // helper.DeleteDir(contextDevfile) // }) //}) + + Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has a valid value", func() { + It("should only download the specified path in the sparseContextDir field", func() { + helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") + helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs"), currentWorkingDirectory) + helper.RenameFile(currentWorkingDirectory, "devfile.yaml") + devfile := "devfile.yaml" + helper.CmdShouldPass("odo", "create", "--downloadSource") + expectedFiles := []string{"package.json", "package-lock.json", "README.md", devfile} + Expect(helper.VerifyFilesExist(contextDevfile, expectedFiles)).To(Equal(true)) + helper.DeleteDir(contextDevfile) + helper.Chdir(context) + }) + }) }) From 4e39e3649d6518d334cc2a8f232901fcdec4f544 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 29 Apr 2020 08:49:13 +0100 Subject: [PATCH 03/20] added tests --- pkg/odo/cli/component/create.go | 8 +++-- .../devfile/cmd_devfile_create_test.go | 30 +++++++++++++++---- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 1ab17ee7fca..94444810b96 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -772,9 +772,9 @@ func (co *CreateOptions) downloadProject() error { sparseCheckoutDir := *project.Source.SparseCheckoutDir fullSparseCheckoutDirPath := filepath.Join(extractFolder, sparseCheckoutDir) - invalidPath := util.CheckPathExists() - if _, err := os.Stat(fullSparseCheckoutDirPath); os.IsNotExist(err) { - return errors.Errorf("Path specified in sparseCheckoutDir does not exist in project", err) + validPath := util.CheckPathExists(fullSparseCheckoutDirPath) + if !validPath { + return errors.Errorf("Path: %v specified in sparseCheckoutDir does not exist in project", sparseCheckoutDir) } info, err := os.Stat(path) @@ -788,6 +788,8 @@ func (co *CreateOptions) downloadProject() error { } } else { + + // extract project to current working directory err = util.GetAndExtractZip(zipUrl, path) if err != nil { return err diff --git a/tests/integration/devfile/cmd_devfile_create_test.go b/tests/integration/devfile/cmd_devfile_create_test.go index d38ea3fd5dd..474887179c5 100644 --- a/tests/integration/devfile/cmd_devfile_create_test.go +++ b/tests/integration/devfile/cmd_devfile_create_test.go @@ -186,16 +186,34 @@ var _ = Describe("odo devfile create command tests", func() { //}) Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has a valid value", func() { - It("should only download the specified path in the sparseContextDir field", func() { + It("should only extract the specified path in the sparseContextDir field", func() { helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") - helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs"), currentWorkingDirectory) - helper.RenameFile(currentWorkingDirectory, "devfile.yaml") + contextDevfile := helper.CreateNewContext() + helper.Chdir(contextDevfile) devfile := "devfile.yaml" - helper.CmdShouldPass("odo", "create", "--downloadSource") - expectedFiles := []string{"package.json", "package-lock.json", "README.md", devfile} + helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), filepath.Join(contextDevfile, devfile)) + componentNamespace := helper.RandString(6) + helper.CmdShouldPass("odo", "create", "--downloadSource", "--project", componentNamespace) + expectedFiles := []string{"app.js", devfile} Expect(helper.VerifyFilesExist(contextDevfile, expectedFiles)).To(Equal(true)) helper.DeleteDir(contextDevfile) - helper.Chdir(context) + }) + }) + + Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has an invalid value", func() { + It("should fail and alert the user that the specified path in sparseContextDir does not exist", func() { + helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") + contextDevfile := helper.CreateNewContext() + helper.Chdir(contextDevfile) + devfile := "devfile.yaml" + devfilePath := filepath.Join(contextDevfile, devfile) + helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), devfilePath) + helper.ReplaceDevfileField(devfilePath, "sparseCheckoutDir", "/invalid/") + componentNamespace := helper.RandString(6) + output := helper.CmdShouldFail("odo", "create", "--downloadSource", "--project", componentNamespace) + expectedString := "Path: /invalid/ specified in sparseCheckoutDir does not exist in project" + helper.MatchAllInOutput(output, []string{expectedString}) + helper.DeleteDir(contextDevfile) }) }) }) From 9a6de10be31e6555abbd6df11046c9113dc5505b Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 29 Apr 2020 10:05:50 +0100 Subject: [PATCH 04/20] commented out tests --- .../devfile/cmd_devfile_create_test.go | 62 +++++++++---------- 1 file changed, 31 insertions(+), 31 deletions(-) diff --git a/tests/integration/devfile/cmd_devfile_create_test.go b/tests/integration/devfile/cmd_devfile_create_test.go index 474887179c5..678532ac85c 100644 --- a/tests/integration/devfile/cmd_devfile_create_test.go +++ b/tests/integration/devfile/cmd_devfile_create_test.go @@ -185,35 +185,35 @@ var _ = Describe("odo devfile create command tests", func() { // }) //}) - Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has a valid value", func() { - It("should only extract the specified path in the sparseContextDir field", func() { - helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") - contextDevfile := helper.CreateNewContext() - helper.Chdir(contextDevfile) - devfile := "devfile.yaml" - helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), filepath.Join(contextDevfile, devfile)) - componentNamespace := helper.RandString(6) - helper.CmdShouldPass("odo", "create", "--downloadSource", "--project", componentNamespace) - expectedFiles := []string{"app.js", devfile} - Expect(helper.VerifyFilesExist(contextDevfile, expectedFiles)).To(Equal(true)) - helper.DeleteDir(contextDevfile) - }) - }) - - Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has an invalid value", func() { - It("should fail and alert the user that the specified path in sparseContextDir does not exist", func() { - helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") - contextDevfile := helper.CreateNewContext() - helper.Chdir(contextDevfile) - devfile := "devfile.yaml" - devfilePath := filepath.Join(contextDevfile, devfile) - helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), devfilePath) - helper.ReplaceDevfileField(devfilePath, "sparseCheckoutDir", "/invalid/") - componentNamespace := helper.RandString(6) - output := helper.CmdShouldFail("odo", "create", "--downloadSource", "--project", componentNamespace) - expectedString := "Path: /invalid/ specified in sparseCheckoutDir does not exist in project" - helper.MatchAllInOutput(output, []string{expectedString}) - helper.DeleteDir(contextDevfile) - }) - }) + // Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has a valid value", func() { + // It("should only extract the specified path in the sparseContextDir field", func() { + // helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") + // contextDevfile := helper.CreateNewContext() + // helper.Chdir(contextDevfile) + // devfile := "devfile.yaml" + // helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), filepath.Join(contextDevfile, devfile)) + // componentNamespace := helper.RandString(6) + // helper.CmdShouldPass("odo", "create", "--downloadSource", "--project", componentNamespace) + // expectedFiles := []string{"app.js", devfile} + // Expect(helper.VerifyFilesExist(contextDevfile, expectedFiles)).To(Equal(true)) + // helper.DeleteDir(contextDevfile) + // }) + // }) + + // Context("When executing odo create with devfile component, --downloadSource flag and sparseContextDir has an invalid value", func() { + // It("should fail and alert the user that the specified path in sparseContextDir does not exist", func() { + // helper.CmdShouldPass("odo", "preference", "set", "Experimental", "true") + // contextDevfile := helper.CreateNewContext() + // helper.Chdir(contextDevfile) + // devfile := "devfile.yaml" + // devfilePath := filepath.Join(contextDevfile, devfile) + // helper.CopyExampleDevFile(filepath.Join("source", "devfiles", "nodejs", "devfile-with-sparseCheckoutDir"), devfilePath) + // helper.ReplaceDevfileField(devfilePath, "sparseCheckoutDir", "/invalid/") + // componentNamespace := helper.RandString(6) + // output := helper.CmdShouldFail("odo", "create", "--downloadSource", "--project", componentNamespace) + // expectedString := "Path: /invalid/ specified in sparseCheckoutDir does not exist in project" + // helper.MatchAllInOutput(output, []string{expectedString}) + // helper.DeleteDir(contextDevfile) + // }) + // }) }) From 67681f540fb1fdf001156ec4671fa3b55d3ddd60 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 29 Apr 2020 10:06:57 +0100 Subject: [PATCH 05/20] change to validpath statement --- pkg/odo/cli/component/create.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 94444810b96..9ab91f2b3e4 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -772,8 +772,7 @@ func (co *CreateOptions) downloadProject() error { sparseCheckoutDir := *project.Source.SparseCheckoutDir fullSparseCheckoutDirPath := filepath.Join(extractFolder, sparseCheckoutDir) - validPath := util.CheckPathExists(fullSparseCheckoutDirPath) - if !validPath { + if !util.CheckPathExists(fullSparseCheckoutDirPath) { return errors.Errorf("Path: %v specified in sparseCheckoutDir does not exist in project", sparseCheckoutDir) } From cfbdd2a370e683c3f20de69d7c92f4f303daecb4 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Thu, 30 Apr 2020 11:29:58 +0100 Subject: [PATCH 06/20] changed method of extracting zip --- pkg/odo/cli/component/create.go | 23 +---------- pkg/util/util.go | 73 +++++++++------------------------ 2 files changed, 21 insertions(+), 75 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 9ab91f2b3e4..29491623907 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -761,35 +761,16 @@ func (co *CreateOptions) downloadProject() error { if project.Source.SparseCheckoutDir != nil && *project.Source.SparseCheckoutDir != "" { - // extract project to an extract folder - extractFolder := filepath.Join(path, "extract") - err = util.GetAndExtractZip(zipUrl, extractFolder) - if err != nil { - return err - } - - defer os.RemoveAll(extractFolder) - sparseCheckoutDir := *project.Source.SparseCheckoutDir - fullSparseCheckoutDirPath := filepath.Join(extractFolder, sparseCheckoutDir) - if !util.CheckPathExists(fullSparseCheckoutDirPath) { - return errors.Errorf("Path: %v specified in sparseCheckoutDir does not exist in project", sparseCheckoutDir) - } - - info, err := os.Stat(path) + err = util.GetAndExtractZip(zipUrl, path, sparseCheckoutDir) if err != nil { return err } - err = util.CopyDir(fullSparseCheckoutDirPath, path, info) - if err != nil { - return errors.Errorf("Error copying sparseCheckoutDir to project path") - } - } else { // extract project to current working directory - err = util.GetAndExtractZip(zipUrl, path) + err = util.GetAndExtractZip(zipUrl, path, "/") if err != nil { return err } diff --git a/pkg/util/util.go b/pkg/util/util.go index 1db6c2d0bad..2ecb974b82f 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -802,7 +802,7 @@ func GetGitHubZipURL(repoURL string) (string, error) { // GetAndExtractZip downloads a zip file from a URL with a http prefix or // takes an absolute path prefixed with file:// and extracts it to a destination -func GetAndExtractZip(zipURL string, destination string) error { +func GetAndExtractZip(zipURL string, destination string, pathsToUnzip string) error { if zipURL == "" { return errors.Errorf("Empty zip url: %s", zipURL) } @@ -836,7 +836,7 @@ func GetAndExtractZip(zipURL string, destination string) error { return errors.Errorf("Invalid Zip URL: %s . Should either be prefixed with file://, http:// or https://", zipURL) } - _, err := Unzip(pathToZip, destination) + _, err := Unzip(pathToZip, destination, pathsToUnzip) if err != nil { return err } @@ -847,7 +847,7 @@ func GetAndExtractZip(zipURL string, destination string) error { // Unzip will decompress a zip archive, moving all files and folders // within the zip file (parameter 1) to an output directory (parameter 2). // Source: https://golangcode.com/unzip-files-in-go/ -func Unzip(src, dest string) ([]string, error) { +func Unzip(src, dest, pathToUnzip string) ([]string, error) { var filenames []string r, err := zip.OpenReader(src) @@ -857,7 +857,6 @@ func Unzip(src, dest string) ([]string, error) { defer r.Close() for _, f := range r.File { - // Store filename/path for returning and using later on index := strings.Index(f.Name, "/") filename := f.Name[index+1:] @@ -871,8 +870,23 @@ func Unzip(src, dest string) ([]string, error) { return filenames, fmt.Errorf("%s: illegal file path", fpath) } - filenames = append(filenames, fpath) + // if sparseCheckoutDir has a pattern + match, err := filepath.Match(pathToUnzip, filename) + if err != nil { + return nil, err + } + // removes first slash of pathToUnzip + if strings.HasPrefix(pathToUnzip, "/") { + pathToUnzip = pathToUnzip[1:] + } + + // check for prefix or match + if strings.HasPrefix(filename, pathToUnzip) || match { + filenames = append(filenames, fpath) + } else { + continue + } if f.FileInfo().IsDir() { // Make Folder if err = os.MkdirAll(fpath, os.ModePerm); err != nil { @@ -982,52 +996,3 @@ func CheckKubeConfigExist() bool { return false } - -// copyDir copy one directory to the other -// this function is called recursively info should start as os.Stat(src) -func CopyDir(src string, dst string, info os.FileInfo) error { - - if info.IsDir() { - files, err := ioutil.ReadDir(src) - if err != nil { - return err - } - - for _, file := range files { - dsrt := filepath.Join(src, file.Name()) - ddst := filepath.Join(dst, file.Name()) - if err := CopyDir(dsrt, ddst, file); err != nil { - return err - } - } - return nil - } - - if err := os.MkdirAll(filepath.Dir(dst), os.ModePerm); err != nil { - return err - } - - return CopyFile(src, dst, info) -} - -// copyFile copy one file to another location -func CopyFile(src, dst string, info os.FileInfo) error { - dFile, err := os.Create(dst) - if err != nil { - return err - } - defer dFile.Close() // #nosec G307 - - sFile, err := os.Open(src) - if err != nil { - return err - } - defer sFile.Close() // #nosec G307 - - if err = os.Chmod(dFile.Name(), info.Mode()); err != nil { - return err - } - - _, err = io.Copy(dFile, sFile) - return err -} From b2c8fc8199e94187c87264a88ce6490a8519fb71 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Thu, 30 Apr 2020 12:54:43 +0100 Subject: [PATCH 07/20] changed pathsToUnzip to pathToUnzip --- pkg/util/util.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 2ecb974b82f..c2991fb307a 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -802,7 +802,7 @@ func GetGitHubZipURL(repoURL string) (string, error) { // GetAndExtractZip downloads a zip file from a URL with a http prefix or // takes an absolute path prefixed with file:// and extracts it to a destination -func GetAndExtractZip(zipURL string, destination string, pathsToUnzip string) error { +func GetAndExtractZip(zipURL string, destination string, pathToUnzip string) error { if zipURL == "" { return errors.Errorf("Empty zip url: %s", zipURL) } @@ -836,7 +836,7 @@ func GetAndExtractZip(zipURL string, destination string, pathsToUnzip string) er return errors.Errorf("Invalid Zip URL: %s . Should either be prefixed with file://, http:// or https://", zipURL) } - _, err := Unzip(pathToZip, destination, pathsToUnzip) + _, err := Unzip(pathToZip, destination, pathToUnzip) if err != nil { return err } From 26928c7c675b6352b99bf0facabb1613ed1a5f03 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Thu, 30 Apr 2020 13:39:14 +0100 Subject: [PATCH 08/20] Added error message when no files are unzipped --- pkg/util/util.go | 8 ++++++-- tests/integration/devfile/cmd_devfile_create_test.go | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index c2991fb307a..e2bb7b2962c 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -836,11 +836,15 @@ func GetAndExtractZip(zipURL string, destination string, pathToUnzip string) err return errors.Errorf("Invalid Zip URL: %s . Should either be prefixed with file://, http:// or https://", zipURL) } - _, err := Unzip(pathToZip, destination, pathToUnzip) + filenames, err := Unzip(pathToZip, destination, pathToUnzip) if err != nil { return err } + if len(filenames) == 0 { + return errors.Errorf("No files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path") + } + return nil } @@ -873,7 +877,7 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { // if sparseCheckoutDir has a pattern match, err := filepath.Match(pathToUnzip, filename) if err != nil { - return nil, err + return filenames, err } // removes first slash of pathToUnzip diff --git a/tests/integration/devfile/cmd_devfile_create_test.go b/tests/integration/devfile/cmd_devfile_create_test.go index 678532ac85c..6f816e19d52 100644 --- a/tests/integration/devfile/cmd_devfile_create_test.go +++ b/tests/integration/devfile/cmd_devfile_create_test.go @@ -211,7 +211,7 @@ var _ = Describe("odo devfile create command tests", func() { // helper.ReplaceDevfileField(devfilePath, "sparseCheckoutDir", "/invalid/") // componentNamespace := helper.RandString(6) // output := helper.CmdShouldFail("odo", "create", "--downloadSource", "--project", componentNamespace) - // expectedString := "Path: /invalid/ specified in sparseCheckoutDir does not exist in project" + // expectedString := "No files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path" // helper.MatchAllInOutput(output, []string{expectedString}) // helper.DeleteDir(contextDevfile) // }) From 4968786d8e7880507cfa4c172de373135aae21d2 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Thu, 30 Apr 2020 15:36:46 +0100 Subject: [PATCH 09/20] cleaned up conditional prefix trim --- pkg/util/util.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index e2bb7b2962c..2b32a4d7f9e 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -880,10 +880,8 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { return filenames, err } - // removes first slash of pathToUnzip - if strings.HasPrefix(pathToUnzip, "/") { - pathToUnzip = pathToUnzip[1:] - } + // removes first slash of pathToUnzip if present + pathToUnzip = strings.TrimPrefix(pathToUnzip, "/") // check for prefix or match if strings.HasPrefix(filename, pathToUnzip) || match { From dca7532fbf9bd2f2315a767643691e7826bb44dc Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Fri, 1 May 2020 15:57:52 +0100 Subject: [PATCH 10/20] Changes from feedback --- pkg/util/util.go | 51 ++++++++++++++++++++++++++++++++++++------------ 1 file changed, 38 insertions(+), 13 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 2b32a4d7f9e..35dfd0606b4 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -801,7 +801,8 @@ func GetGitHubZipURL(repoURL string) (string, error) { } // GetAndExtractZip downloads a zip file from a URL with a http prefix or -// takes an absolute path prefixed with file:// and extracts it to a destination +// takes an absolute path prefixed with file:// and extracts it to a destination. +// pathToUnzip specifies the path within the zip folder to extract func GetAndExtractZip(zipURL string, destination string, pathToUnzip string) error { if zipURL == "" { return errors.Errorf("Empty zip url: %s", zipURL) @@ -842,15 +843,16 @@ func GetAndExtractZip(zipURL string, destination string, pathToUnzip string) err } if len(filenames) == 0 { - return errors.Errorf("No files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path") + return errors.New("No files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path") } return nil } -// Unzip will decompress a zip archive, moving all files and folders -// within the zip file (parameter 1) to an output directory (parameter 2). +// Unzip will decompress a zip archive, moving specified files and folders +// within the zip file (parameter 1) to an output directory (parameter 2) // Source: https://golangcode.com/unzip-files-in-go/ +// pathToUnzip (parameter 3) is the path within the zip folder to extract func Unzip(src, dest, pathToUnzip string) ([]string, error) { var filenames []string @@ -867,12 +869,6 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { if filename == "" { continue } - fpath := filepath.Join(dest, filename) - - // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE - if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { - return filenames, fmt.Errorf("%s: illegal file path", fpath) - } // if sparseCheckoutDir has a pattern match, err := filepath.Match(pathToUnzip, filename) @@ -883,12 +879,31 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { // removes first slash of pathToUnzip if present pathToUnzip = strings.TrimPrefix(pathToUnzip, "/") + // destination filepath before trim + fpath := filepath.Join(dest, filename) + + // used for pattern matching + fpathDir := filepath.Dir(fpath) + // check for prefix or match - if strings.HasPrefix(filename, pathToUnzip) || match { - filenames = append(filenames, fpath) - } else { + if strings.HasPrefix(filename, pathToUnzip) { + filename = strings.TrimPrefix(filename, pathToUnzip) + } else if !strings.HasPrefix(filename, pathToUnzip) && !match && !sliceContainsString(fpathDir, filenames) { continue } + // adds trailing slash to destination if needed as filepath.Join removes it + if (len(filename) == 1 && os.IsPathSeparator(filename[0])) || len(filename) == 0 { + fpath = dest + string(os.PathSeparator) + } else { + fpath = filepath.Join(dest, filename) + } + // Check for ZipSlip. More Info: http://bit.ly/2MsjAWE + if !strings.HasPrefix(fpath, filepath.Clean(dest)+string(os.PathSeparator)) { + return filenames, fmt.Errorf("%s: illegal file path", fpath) + } + + filenames = append(filenames, fpath) + if f.FileInfo().IsDir() { // Make Folder if err = os.MkdirAll(fpath, os.ModePerm); err != nil { @@ -998,3 +1013,13 @@ func CheckKubeConfigExist() bool { return false } + +// sliceContainsString checks for existence of given string in given slice +func sliceContainsString(str string, slice []string) bool { + for _, b := range slice { + if b == str { + return true + } + } + return false +} From ef285fb2bdd73624a8deaf8706ce3f1b8aaaa6e4 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Mon, 11 May 2020 12:10:59 +0100 Subject: [PATCH 11/20] feedback changes and unit tests for util func --- pkg/odo/cli/component/create.go | 4 ++-- pkg/util/util.go | 6 ++++-- pkg/util/util_test.go | 38 +++++++++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 2d57d72ce2e..c46671bd359 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -771,7 +771,7 @@ func (co *CreateOptions) downloadProject() error { sparseCheckoutDir := *project.Source.SparseCheckoutDir err = util.GetAndExtractZip(zipUrl, path, sparseCheckoutDir) if err != nil { - return err + return errors.Wrap(err, "Error downloading and extracting project zip folder") } } else { @@ -779,7 +779,7 @@ func (co *CreateOptions) downloadProject() error { // extract project to current working directory err = util.GetAndExtractZip(zipUrl, path, "/") if err != nil { - return err + return errors.Wrap(err, "Error downloading and extracting project zip folder") } } diff --git a/pkg/util/util.go b/pkg/util/util.go index 0639669ca7b..571b3e4b82e 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -876,9 +876,11 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { return filenames, err } - // removes first slash of pathToUnzip if present + // removes first slash of pathToUnzip if present, adds trailing slash pathToUnzip = strings.TrimPrefix(pathToUnzip, "/") - + if pathToUnzip[len(pathToUnzip)-1:] != string(os.PathSeparator) { + pathToUnzip = pathToUnzip + string(os.PathSeparator) + } // destination filepath before trim fpath := filepath.Join(dest, filename) diff --git a/pkg/util/util_test.go b/pkg/util/util_test.go index 1dae55a141f..3da82199a2e 100644 --- a/pkg/util/util_test.go +++ b/pkg/util/util_test.go @@ -1652,3 +1652,41 @@ func TestValidateURL(t *testing.T) { }) } } + +func TestSliceContainsString(t *testing.T) { + tests := []struct { + name string + stringVal string + slice []string + wantVal bool + }{ + { + name: "Case 1: string in valid slice", + stringVal: "string", + slice: []string{"string", "string2"}, + wantVal: true, + }, + { + name: "Case 2: string not in valid slice", + stringVal: "string3", + slice: []string{"string", "string2"}, + wantVal: false, + }, + { + name: "Case 3: string not in empty slice", + stringVal: "string", + slice: []string{}, + wantVal: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotVal := sliceContainsString(tt.stringVal, tt.slice) + + if !reflect.DeepEqual(gotVal, tt.wantVal) { + t.Errorf("Got %v, want %v", gotVal, tt.wantVal) + } + }) + } +} From bf0f368c7c337b361bae15ebe21fb182223dc0fd Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Tue, 12 May 2020 10:21:03 +0100 Subject: [PATCH 12/20] check for empty pathToUnzip --- pkg/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 571b3e4b82e..1375dd0d52e 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -878,7 +878,7 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { // removes first slash of pathToUnzip if present, adds trailing slash pathToUnzip = strings.TrimPrefix(pathToUnzip, "/") - if pathToUnzip[len(pathToUnzip)-1:] != string(os.PathSeparator) { + if pathToUnzip != "" && pathToUnzip[len(pathToUnzip)-1:] != string(os.PathSeparator) { pathToUnzip = pathToUnzip + string(os.PathSeparator) } // destination filepath before trim From 4645969799052872e89a3e3ef1e21fb9e8fad3af Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Tue, 12 May 2020 12:38:34 +0100 Subject: [PATCH 13/20] changed error message format --- pkg/odo/cli/component/create.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index c46671bd359..4e147ebf095 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -771,7 +771,7 @@ func (co *CreateOptions) downloadProject() error { sparseCheckoutDir := *project.Source.SparseCheckoutDir err = util.GetAndExtractZip(zipUrl, path, sparseCheckoutDir) if err != nil { - return errors.Wrap(err, "Error downloading and extracting project zip folder") + return errors.Wrap(err, "failed to download and extract project zip folder") } } else { @@ -779,7 +779,7 @@ func (co *CreateOptions) downloadProject() error { // extract project to current working directory err = util.GetAndExtractZip(zipUrl, path, "/") if err != nil { - return errors.Wrap(err, "Error downloading and extracting project zip folder") + return errors.Wrap(err, "failed to download and extract project zip folder") } } From 8aa566b7730fd7f43e2879e7702a4f66591c72f3 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Tue, 12 May 2020 13:07:47 +0100 Subject: [PATCH 14/20] cleaned up path separator for windows --- pkg/util/util.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 1375dd0d52e..9c8efe36cb6 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -862,9 +862,14 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { } defer r.Close() + // change path separator to correct character for windows + if runtime.GOOS == "windows" { + strings.Replace(pathToUnzip, "/", string(os.PathSeparator), -1) + } + for _, f := range r.File { // Store filename/path for returning and using later on - index := strings.Index(f.Name, "/") + index := strings.Index(f.Name, string(os.PathSeparator)) filename := f.Name[index+1:] if filename == "" { continue @@ -877,7 +882,7 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { } // removes first slash of pathToUnzip if present, adds trailing slash - pathToUnzip = strings.TrimPrefix(pathToUnzip, "/") + pathToUnzip = strings.TrimPrefix(pathToUnzip, string(os.PathSeparator)) if pathToUnzip != "" && pathToUnzip[len(pathToUnzip)-1:] != string(os.PathSeparator) { pathToUnzip = pathToUnzip + string(os.PathSeparator) } @@ -894,7 +899,7 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { continue } // adds trailing slash to destination if needed as filepath.Join removes it - if (len(filename) == 1 && os.IsPathSeparator(filename[0])) || len(filename) == 0 { + if (len(filename) == 1 && os.IsPathSeparator(filename[0])) || filename == "" { fpath = dest + string(os.PathSeparator) } else { fpath = filepath.Join(dest, filename) From 2c416b8cbf79417b823a3dcd20772102d2aab970 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Tue, 12 May 2020 13:19:34 +0100 Subject: [PATCH 15/20] fixed return pathToUnzip --- pkg/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 9c8efe36cb6..a73fcb81699 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -864,7 +864,7 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { // change path separator to correct character for windows if runtime.GOOS == "windows" { - strings.Replace(pathToUnzip, "/", string(os.PathSeparator), -1) + pathToUnzip = strings.Replace(pathToUnzip, "/", string(os.PathSeparator), -1) } for _, f := range r.File { From 0824130fe9d078625f89637470e3da5f1d87602d Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 13 May 2020 14:12:09 +0100 Subject: [PATCH 16/20] use hassuffix --- pkg/util/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index a73fcb81699..8d04a21b59d 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -883,7 +883,7 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { // removes first slash of pathToUnzip if present, adds trailing slash pathToUnzip = strings.TrimPrefix(pathToUnzip, string(os.PathSeparator)) - if pathToUnzip != "" && pathToUnzip[len(pathToUnzip)-1:] != string(os.PathSeparator) { + if pathToUnzip != "" && !strings.HasSuffix(pathToUnzip, string(os.PathSeparator)) { pathToUnzip = pathToUnzip + string(os.PathSeparator) } // destination filepath before trim From e1092300e32039645d38fdbd1b5317edf9575236 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 13 May 2020 15:41:14 +0100 Subject: [PATCH 17/20] moved to function --- pkg/odo/cli/component/create.go | 36 +++++++++++++++++++-------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 4e147ebf095..3d3f37b6452 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -15,6 +15,7 @@ import ( "github.com/openshift/odo/pkg/component" "github.com/openshift/odo/pkg/config" "github.com/openshift/odo/pkg/devfile" + "github.com/openshift/odo/pkg/devfile/parser/data/common" "github.com/openshift/odo/pkg/envinfo" "github.com/openshift/odo/pkg/kclient" "github.com/openshift/odo/pkg/log" @@ -766,21 +767,9 @@ func (co *CreateOptions) downloadProject() error { return errors.Errorf("Project type not supported") } - if project.Source.SparseCheckoutDir != nil && *project.Source.SparseCheckoutDir != "" { - - sparseCheckoutDir := *project.Source.SparseCheckoutDir - err = util.GetAndExtractZip(zipUrl, path, sparseCheckoutDir) - if err != nil { - return errors.Wrap(err, "failed to download and extract project zip folder") - } - - } else { - - // extract project to current working directory - err = util.GetAndExtractZip(zipUrl, path, "/") - if err != nil { - return errors.Wrap(err, "failed to download and extract project zip folder") - } + err = sparseCheckoutDir(project, zipUrl, path) + if err != nil { + return err } return nil @@ -890,6 +879,23 @@ func ensureAndLogProperResourceUsage(resource, resourceMin, resourceMax, resourc } } +func sparseCheckoutDir(project common.DevfileProject, zipURL, path string) error { + if project.Source.SparseCheckoutDir != nil && *project.Source.SparseCheckoutDir != "" { + sparseCheckoutDir := *project.Source.SparseCheckoutDir + err := util.GetAndExtractZip(zipURL, path, sparseCheckoutDir) + if err != nil { + return errors.Wrap(err, "failed to download and extract project zip folder") + } + } else { + // extract project to current working directory + err := util.GetAndExtractZip(zipURL, path, "/") + if err != nil { + return errors.Wrap(err, "failed to download and extract project zip folder") + } + } + return nil +} + // NewCmdCreate implements the create odo command func NewCmdCreate(name, fullName string) *cobra.Command { co := NewCreateOptions() From f0b93b0a078f948ac954f1d58d173f6852ea1922 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 13 May 2020 15:59:38 +0100 Subject: [PATCH 18/20] fromslash --- pkg/util/util.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/pkg/util/util.go b/pkg/util/util.go index 8d04a21b59d..6c1eb8808cd 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -862,10 +862,8 @@ func Unzip(src, dest, pathToUnzip string) ([]string, error) { } defer r.Close() - // change path separator to correct character for windows - if runtime.GOOS == "windows" { - pathToUnzip = strings.Replace(pathToUnzip, "/", string(os.PathSeparator), -1) - } + // change path separator to correct character + pathToUnzip = filepath.FromSlash(pathToUnzip) for _, f := range r.File { // Store filename/path for returning and using later on From 14c6c44b29c14645db8b9b585bb9d589d9b526f9 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 20 May 2020 09:06:56 +0100 Subject: [PATCH 19/20] minor fixes --- pkg/odo/cli/component/create.go | 4 ++-- pkg/util/util.go | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/odo/cli/component/create.go b/pkg/odo/cli/component/create.go index 1ca9fd7629e..e04c288dff6 100644 --- a/pkg/odo/cli/component/create.go +++ b/pkg/odo/cli/component/create.go @@ -792,7 +792,7 @@ func (co *CreateOptions) downloadProject(projectPassed string) error { return errors.Errorf("Project type not supported") } - err = sparseCheckoutDir(project, zipUrl, path) + err = checkoutProject(project, zipUrl, path) if err != nil { return err } @@ -904,7 +904,7 @@ func ensureAndLogProperResourceUsage(resource, resourceMin, resourceMax, resourc } } -func sparseCheckoutDir(project common.DevfileProject, zipURL, path string) error { +func checkoutProject(project common.DevfileProject, zipURL, path string) error { if project.Source.SparseCheckoutDir != nil && *project.Source.SparseCheckoutDir != "" { sparseCheckoutDir := *project.Source.SparseCheckoutDir err := util.GetAndExtractZip(zipURL, path, sparseCheckoutDir) diff --git a/pkg/util/util.go b/pkg/util/util.go index f9f7869bc11..fa60fbb3928 100644 --- a/pkg/util/util.go +++ b/pkg/util/util.go @@ -854,7 +854,7 @@ func GetAndExtractZip(zipURL string, destination string, pathToUnzip string) err } if len(filenames) == 0 { - return errors.New("No files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path") + return errors.New("no files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path") } return nil From 24e341b71c106812ce5a20a69534ddfbbccab509 Mon Sep 17 00:00:00 2001 From: Cameron McWilliam Date: Wed, 20 May 2020 09:08:19 +0100 Subject: [PATCH 20/20] lowercase fix for test --- tests/integration/devfile/cmd_devfile_create_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/devfile/cmd_devfile_create_test.go b/tests/integration/devfile/cmd_devfile_create_test.go index 23b8cd8ee59..ff048219c57 100644 --- a/tests/integration/devfile/cmd_devfile_create_test.go +++ b/tests/integration/devfile/cmd_devfile_create_test.go @@ -249,7 +249,7 @@ var _ = Describe("odo devfile create command tests", func() { // helper.ReplaceDevfileField(devfilePath, "sparseCheckoutDir", "/invalid/") // componentNamespace := helper.RandString(6) // output := helper.CmdShouldFail("odo", "create", "--downloadSource", "--project", componentNamespace) - // expectedString := "No files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path" + // expectedString := "no files were unzipped, ensure that the project repo is not empty or that sparseCheckoutDir has a valid path" // helper.MatchAllInOutput(output, []string{expectedString}) // helper.DeleteDir(contextDevfile) // })