From 1fc844af30144acafb7654d70e2de3a1dabee3e0 Mon Sep 17 00:00:00 2001 From: Tuan Anh Tran Date: Fri, 19 Jun 2020 10:28:00 +0700 Subject: [PATCH] Add support for tgz/zip archives to the get command Introduces support for "get" command to use tgz and zip archives and adds helm3 option to "get" command. This commit squashes three commits from PR #130 and closes #130. Signed-off-by: Tuan Anh Tran Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- README.md | 1 + cmd/get.go | 165 +++++++++++++++++++++++++++++++++++++++++--- pkg/get/get.go | 31 +++++++++ pkg/get/get_test.go | 63 +++++++++++++++++ pkg/helm/untar.go | 75 ++++++++++++++++++++ 5 files changed, 327 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index c66db6157..0990a7904 100644 --- a/README.md +++ b/README.md @@ -59,6 +59,7 @@ arkade will determine the correct URL to download a CLI tool of your choice taki arkade get kubectl arkade get faas-cli arkade get kubectx +arkade get helm ``` > This is a time saver compared to searching for download pages every time you need a tool. diff --git a/cmd/get.go b/cmd/get.go index b1938e5ba..7cef4678e 100644 --- a/cmd/get.go +++ b/cmd/get.go @@ -4,18 +4,134 @@ package cmd import ( + "archive/tar" + "archive/zip" + "bytes" + "compress/gzip" "fmt" "io" + "io/ioutil" + "log" "net/http" "os" "path" + "path/filepath" "strings" + "time" "github.com/alexellis/arkade/pkg/env" "github.com/alexellis/arkade/pkg/get" + "github.com/alexellis/arkade/pkg/helm" "github.com/spf13/cobra" ) +// Untar reads the gzip-compressed tar file from r and writes it into dir. +// TODO: Untar logic is copied from helm.go. Need to refactor this later on. +func Untar(r io.Reader, dir string) error { + return untar(r, dir) +} + +func untar(r io.Reader, dir string) (err error) { + t0 := time.Now() + nFiles := 0 + madeDir := map[string]bool{} + defer func() { + td := time.Since(t0) + if err == nil { + log.Printf("extracted tarball into %s: %d files, %d dirs (%v)", dir, nFiles, len(madeDir), td) + } else { + log.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err) + } + }() + zr, err := gzip.NewReader(r) + if err != nil { + return fmt.Errorf("requires gzip-compressed body: %v", err) + } + tr := tar.NewReader(zr) + loggedChtimesError := false + for { + f, err := tr.Next() + if err == io.EOF { + break + } + if err != nil { + log.Printf("tar reading error: %v", err) + return fmt.Errorf("tar error: %v", err) + } + if !validRelPath(f.Name) { + return fmt.Errorf("tar contained invalid name error %q", f.Name) + } + baseFile := filepath.Base(f.Name) + abs := path.Join(dir, baseFile) + fmt.Println(abs, f.Name) + + fi := f.FileInfo() + mode := fi.Mode() + switch { + case mode.IsDir(): + + break + + case mode.IsRegular(): + + wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm()) + if err != nil { + return err + } + n, err := io.Copy(wf, tr) + if closeErr := wf.Close(); closeErr != nil && err == nil { + err = closeErr + } + if err != nil { + return fmt.Errorf("error writing to %s: %v", abs, err) + } + if n != f.Size { + return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size) + } + modTime := f.ModTime + if modTime.After(t0) { + // Clamp modtimes at system time. See + // golang.org/issue/19062 when clock on + // buildlet was behind the gitmirror server + // doing the git-archive. + modTime = t0 + } + if !modTime.IsZero() { + if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError { + // benign error. Gerrit doesn't even set the + // modtime in these, and we don't end up relying + // on it anywhere (the gomote push command relies + // on digests only), so this is a little pointless + // for now. + log.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err) + loggedChtimesError = true // once is enough + } + } + nFiles++ + default: + } + } + return nil +} + +func validRelativeDir(dir string) bool { + if strings.Contains(dir, `\`) || path.IsAbs(dir) { + return false + } + dir = path.Clean(dir) + if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." { + return false + } + return true +} + +func validRelPath(p string) bool { + if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") { + return false + } + return true +} + // MakeGet creates the Get command to download software func MakeGet() *cobra.Command { tools := get.MakeTools() @@ -27,8 +143,9 @@ func MakeGet() *cobra.Command { releases or downloads page. The tool is usually downloaded in binary format and provides a fast and easy alternative to a package manager.`, Example: ` arkade get kubectl + arkade get kubectx arkade get faas-cli - arkade get kubectx`, + arkade get helm`, SilenceUsage: true, } @@ -81,14 +198,45 @@ and provides a fast and easy alternative to a package manager.`, outFilePath := path.Join(tmp, fileName) - out, err := os.Create(outFilePath) - if err != nil { - return err - } - defer out.Close() + if tool.IsArchive() { + outFilePathDir := filepath.Dir(outFilePath) + outFilePath = path.Join(outFilePathDir, tool.Name) + if strings.Contains(strings.ToLower(operatingSystem), "mingw") && tool.NoExtension == false { + outFilePath += ".exe" + } + r := ioutil.NopCloser(res.Body) + if strings.HasSuffix(downloadURL, "tar.gz") { + untarErr := helm.Untar(r, outFilePathDir) + if untarErr != nil { + return untarErr + } + } else if strings.HasSuffix(downloadURL, "zip") { + buff := bytes.NewBuffer([]byte{}) + size, err := io.Copy(buff, res.Body) + if err != nil { + return err + } + reader := bytes.NewReader(buff.Bytes()) + zipReader, err := zip.NewReader(reader, size) + if err != nil { + return fmt.Errorf("error creating zip reader") + } + unzipErr := helm.Unzip(*zipReader, outFilePathDir) + if unzipErr != nil { + return unzipErr + } + } - if _, err = io.Copy(out, res.Body); err != nil { - return err + } else { + out, err := os.Create(outFilePath) + if err != nil { + return err + } + defer out.Close() + + if _, err = io.Copy(out, res.Body); err != nil { + return err + } } finalName := tool.Name @@ -115,4 +263,5 @@ const arkadeGet = `Use "arkade get TOOL" to download a tool or application: - kubectl - faas-cli - kubectx + - helm ` diff --git a/pkg/get/get.go b/pkg/get/get.go index c2fd70ed0..cbb5320a1 100644 --- a/pkg/get/get.go +++ b/pkg/get/get.go @@ -9,6 +9,8 @@ import ( "net/http" "strings" "time" + + "github.com/alexellis/arkade/pkg/env" ) type Tool struct { @@ -21,6 +23,14 @@ type Tool struct { NoExtension bool } +func (tool Tool) IsArchive() bool { + arch, operatingSystem := env.GetClientArch() + version := "" + + downloadURL, _ := GetDownloadURL(&tool, strings.ToLower(operatingSystem), strings.ToLower(arch), version) + return strings.HasSuffix(downloadURL, "tar.gz") || strings.HasSuffix(downloadURL, "zip") +} + var templateFuncs = map[string]interface{}{ "HasPrefix": func(s, prefix string) bool { return strings.HasPrefix(s, prefix) }, } @@ -197,6 +207,27 @@ https://storage.googleapis.com/kubernetes-release/release/{{.Version}}/bin/{{$os // Author recommends to keep using Bash version in this release https://github.com/ahmetb/kubectx/releases/tag/v0.9.0 NoExtension: true, }, + { + Owner: "helm", + Repo: "helm", + Name: "helm", + Version: "v3.2.4", + URLTemplate: `{{$arch := "arm"}} + +{{- if eq .Arch "x86_64" -}} +{{$arch = "amd64"}} +{{- end -}} + +{{$os := .OS}} +{{$ext := "tar.gz"}} + +{{ if HasPrefix .OS "ming" -}} +{{$os = "windows"}} +{{$ext = "zip"}} +{{- end -}} + +https://get.helm.sh/helm-{{.Version}}-{{$os}}-{{$arch}}.{{$ext}}`, + }, } return tools } diff --git a/pkg/get/get_test.go b/pkg/get/get_test.go index db36e62b7..ebdec6527 100644 --- a/pkg/get/get_test.go +++ b/pkg/get/get_test.go @@ -151,3 +151,66 @@ func Test_DownloadWindows(t *testing.T) { t.Fatalf("want: %s, got: %s", want, got) } } + +func Test_DownloadHelmDarwin(t *testing.T) { + tools := MakeTools() + name := "helm" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("darwin", arch64bit, tool.Version) + if err != nil { + t.Fatal(err) + } + want := "https://get.helm.sh/helm-v3.2.4-darwin-amd64.tar.gz" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadHelmLinux(t *testing.T) { + tools := MakeTools() + name := "helm" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("linux", arch64bit, tool.Version) + if err != nil { + t.Fatal(err) + } + want := "https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} + +func Test_DownloadHelmWindows(t *testing.T) { + tools := MakeTools() + name := "helm" + var tool *Tool + for _, target := range tools { + if name == target.Name { + tool = &target + break + } + } + + got, err := tool.GetURL("mingw64_nt-10.0-18362", arch64bit, tool.Version) + if err != nil { + t.Fatal(err) + } + want := "https://get.helm.sh/helm-v3.2.4-windows-amd64.zip" + if got != want { + t.Fatalf("want: %s, got: %s", want, got) + } +} diff --git a/pkg/helm/untar.go b/pkg/helm/untar.go index c28077291..f81490f35 100644 --- a/pkg/helm/untar.go +++ b/pkg/helm/untar.go @@ -9,6 +9,7 @@ package helm import ( "archive/tar" + "archive/zip" "compress/gzip" "fmt" "io" @@ -30,6 +31,80 @@ func Untar(r io.Reader, dir string) error { return untar(r, dir) } +// Untar reads the compressed zip file from r and writes it into dir. +func Unzip(r zip.Reader, dir string) error { + return unzip(r, dir) +} + +func unzip(r zip.Reader, dir string) (err error) { + if err != nil { + return err + } + t0 := time.Now() + nFiles := 0 + madeDir := map[string]bool{} + defer func() { + td := time.Since(t0) + if err == nil { + log.Printf("extracted zip into %s: %d files, %d dirs (%v)", dir, nFiles, len(madeDir), td) + } else { + log.Printf("error extracting zip into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err) + } + }() + + // Closure to address file descriptors issue with all the deferred .Close() methods + extractAndWriteFile := func(f *zip.File) error { + rc, err := f.Open() + if err != nil { + return err + } + defer func() { + if err := rc.Close(); err != nil { + panic(err) + } + }() + baseFile := filepath.Base(f.Name) + abs := path.Join(dir, baseFile) + + fmt.Println(abs) + + fi := f.FileInfo() + mode := fi.Mode() + + switch { + case mode.IsDir(): + break + case mode.IsRegular(): + f, err := os.OpenFile(abs, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, f.Mode()) + if err != nil { + return err + } + defer func() { + if err := f.Close(); err != nil { + panic(err) + } + }() + + _, err = io.Copy(f, rc) + if err != nil { + return err + } + default: + } + nFiles++ + return nil + } + + for _, f := range r.File { + err := extractAndWriteFile(f) + if err != nil { + return err + } + } + + return nil +} + func untar(r io.Reader, dir string) (err error) { t0 := time.Now() nFiles := 0