Add support for tgz/zip archives to the get command

Introduces support for "get" command to use tgz and zip archives
and adds helm3 option to "get" command.

This commit squashes three commits from PR #130 and closes #130.

Signed-off-by: Tuan Anh Tran <me@tuananh.org>
Signed-off-by: Alex Ellis (OpenFaaS Ltd) <alexellis2@gmail.com>

README.md

@@ -59,9 +59,12 @@ arkade will determine the correct URL to download a CLI tool of your choice taki
 arkade get kubectl
 arkade get faas-cli
 arkade get kubectx
+arkade get helm
 ```
 
-This is a time saver compared to searching for download pages every time you need a tool.
+> This is a time saver compared to searching for download pages every time you need a tool.
+
+Think of `arkade get TOOL` as a doing for CLIs, what `arkade install` does for helm.
 
 #### Reduce the repetition
 
@@ -208,11 +211,17 @@ When your sponsorship expires your app can be renewed at that time, or it will d
 
 Email [sales@openfaas.com](mailto:sales@openfaas.com) to find out more.
 
-### What about helm?
+### How does `arkade` compare to `helm`?
 
 In the same way that [brew](https://brew.sh) uses git and Makefiles to compile applications for your Mac, `arkade` uses upstream [helm](https://helm.sh) charts and `kubectl` to install applications to your Kubernetes cluster. arkade exposes strongly-typed flags for the various popular options for helm charts, and enables easier discovery through `arkade install --help` and `arkade install APP --help`.
 
-### Tools and cached versions of helm
+### What is in scope for `arkade get`?
+
+Generally speaking, tools that are used with the various arkade apps or with Kubernetes are in scope. If you want to propose a tool, raise a GitHub issue.
+
+What about package management? `arkade get` provides a faster alternative to package managers like `apt` and `brew`, you're free to use either or both at the same time.
+
+### Automatic download of tools
 
 When required, tools, CLIs, and the helm binaries are downloaded and extracted to `$HOME/.arkade`.
 

cmd/get.go

@@ -4,27 +4,148 @@
 package cmd
 
 import (
+	"archive/tar"
+	"archive/zip"
+	"bytes"
+	"compress/gzip"
 	"fmt"
 	"io"
+	"io/ioutil"
+	"log"
 	"net/http"
 	"os"
 	"path"
+	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/alexellis/arkade/pkg/env"
 	"github.com/alexellis/arkade/pkg/get"
+	"github.com/alexellis/arkade/pkg/helm"
 	"github.com/spf13/cobra"
 )
 
+// Untar reads the gzip-compressed tar file from r and writes it into dir.
+// TODO: Untar logic is copied from helm.go. Need to refactor this later on.
+func Untar(r io.Reader, dir string) error {
+	return untar(r, dir)
+}
+
+func untar(r io.Reader, dir string) (err error) {
+	t0 := time.Now()
+	nFiles := 0
+	madeDir := map[string]bool{}
+	defer func() {
+		td := time.Since(t0)
+		if err == nil {
+			log.Printf("extracted tarball into %s: %d files, %d dirs (%v)", dir, nFiles, len(madeDir), td)
+		} else {
+			log.Printf("error extracting tarball into %s after %d files, %d dirs, %v: %v", dir, nFiles, len(madeDir), td, err)
+		}
+	}()
+	zr, err := gzip.NewReader(r)
+	if err != nil {
+		return fmt.Errorf("requires gzip-compressed body: %v", err)
+	}
+	tr := tar.NewReader(zr)
+	loggedChtimesError := false
+	for {
+		f, err := tr.Next()
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			log.Printf("tar reading error: %v", err)
+			return fmt.Errorf("tar error: %v", err)
+		}
+		if !validRelPath(f.Name) {
+			return fmt.Errorf("tar contained invalid name error %q", f.Name)
+		}
+		baseFile := filepath.Base(f.Name)
+		abs := path.Join(dir, baseFile)
+		fmt.Println(abs, f.Name)
+
+		fi := f.FileInfo()
+		mode := fi.Mode()
+		switch {
+		case mode.IsDir():
+
+			break
+
+		case mode.IsRegular():
+
+			wf, err := os.OpenFile(abs, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode.Perm())
+			if err != nil {
+				return err
+			}
+			n, err := io.Copy(wf, tr)
+			if closeErr := wf.Close(); closeErr != nil && err == nil {
+				err = closeErr
+			}
+			if err != nil {
+				return fmt.Errorf("error writing to %s: %v", abs, err)
+			}
+			if n != f.Size {
+				return fmt.Errorf("only wrote %d bytes to %s; expected %d", n, abs, f.Size)
+			}
+			modTime := f.ModTime
+			if modTime.After(t0) {
+				// Clamp modtimes at system time. See
+				// golang.org/issue/19062 when clock on
+				// buildlet was behind the gitmirror server
+				// doing the git-archive.
+				modTime = t0
+			}
+			if !modTime.IsZero() {
+				if err := os.Chtimes(abs, modTime, modTime); err != nil && !loggedChtimesError {
+					// benign error. Gerrit doesn't even set the
+					// modtime in these, and we don't end up relying
+					// on it anywhere (the gomote push command relies
+					// on digests only), so this is a little pointless
+					// for now.
+					log.Printf("error changing modtime: %v (further Chtimes errors suppressed)", err)
+					loggedChtimesError = true // once is enough
+				}
+			}
+			nFiles++
+		default:
+		}
+	}
+	return nil
+}
+
+func validRelativeDir(dir string) bool {
+	if strings.Contains(dir, `\`) || path.IsAbs(dir) {
+		return false
+	}
+	dir = path.Clean(dir)
+	if strings.HasPrefix(dir, "../") || strings.HasSuffix(dir, "/..") || dir == ".." {
+		return false
+	}
+	return true
+}
+
+func validRelPath(p string) bool {
+	if p == "" || strings.Contains(p, `\`) || strings.HasPrefix(p, "/") || strings.Contains(p, "../") {
+		return false
+	}
+	return true
+}
+
+// MakeGet creates the Get command to download software
 func MakeGet() *cobra.Command {
 	tools := get.MakeTools()
 
 	var command = &cobra.Command{
 		Use:   "get",
-		Short: "Get a release of a tool or application and install it on your local computer.",
+		Short: `The get command downloads a tool`,
+		Long: `The get command downloads a CLI or application from the specific tool's 
+releases or downloads page. The tool is usually downloaded in binary format 
+and provides a fast and easy alternative to a package manager.`,
 		Example: `  arkade get kubectl
-  arkade get openfaas
-  arkade get kubectx`,
+  arkade get faas-cli
+  arkade get kubectx
+  arkade get helm`,
 		SilenceUsage: true,
 	}
 
@@ -77,14 +198,45 @@ func MakeGet() *cobra.Command {
 
 		outFilePath := path.Join(tmp, fileName)
 
-		out, err := os.Create(outFilePath)
-		if err != nil {
-			return err
-		}
-		defer out.Close()
+		if tool.IsArchive() {
+			outFilePathDir := filepath.Dir(outFilePath)
+			outFilePath = path.Join(outFilePathDir, tool.Name)
+			if strings.Contains(strings.ToLower(operatingSystem), "mingw") && tool.NoExtension == false {
+				outFilePath += ".exe"
+			}
+			r := ioutil.NopCloser(res.Body)
+			if strings.HasSuffix(downloadURL, "tar.gz") {
+				untarErr := helm.Untar(r, outFilePathDir)
+				if untarErr != nil {
+					return untarErr
+				}
+			} else if strings.HasSuffix(downloadURL, "zip") {
+				buff := bytes.NewBuffer([]byte{})
+				size, err := io.Copy(buff, res.Body)
+				if err != nil {
+					return err
+				}
+				reader := bytes.NewReader(buff.Bytes())
+				zipReader, err := zip.NewReader(reader, size)
+				if err != nil {
+					return fmt.Errorf("error creating zip reader")
+				}
+				unzipErr := helm.Unzip(*zipReader, outFilePathDir)
+				if unzipErr != nil {
+					return unzipErr
+				}
+			}
 
-		if _, err = io.Copy(out, res.Body); err != nil {
-			return err
+		} else {
+			out, err := os.Create(outFilePath)
+			if err != nil {
+				return err
+			}
+			defer out.Close()
+
+			if _, err = io.Copy(out, res.Body); err != nil {
+				return err
+			}
 		}
 
 		finalName := tool.Name
@@ -111,4 +263,5 @@ const arkadeGet = `Use "arkade get TOOL" to download a tool or application:
   - kubectl
   - faas-cli
   - kubectx
+  - helm
   `

pkg/get/get.go

@@ -9,6 +9,8 @@ import (
 	"net/http"
 	"strings"
 	"time"
+
+	"github.com/alexellis/arkade/pkg/env"
 )
 
 type Tool struct {
@@ -21,6 +23,14 @@ type Tool struct {
 	NoExtension    bool
 }
 
+func (tool Tool) IsArchive() bool {
+	arch, operatingSystem := env.GetClientArch()
+	version := ""
+
+	downloadURL, _ := GetDownloadURL(&tool, strings.ToLower(operatingSystem), strings.ToLower(arch), version)
+	return strings.HasSuffix(downloadURL, "tar.gz") || strings.HasSuffix(downloadURL, "zip")
+}
+
 var templateFuncs = map[string]interface{}{
 	"HasPrefix": func(s, prefix string) bool { return strings.HasPrefix(s, prefix) },
 }
@@ -197,6 +207,27 @@ https://storage.googleapis.com/kubernetes-release/release/{{.Version}}/bin/{{$os
 			// Author recommends to keep using Bash version in this release https://github.com/ahmetb/kubectx/releases/tag/v0.9.0
 			NoExtension: true,
 		},
+		{
+			Owner:   "helm",
+			Repo:    "helm",
+			Name:    "helm",
+			Version: "v3.2.4",
+			URLTemplate: `{{$arch := "arm"}}
+
+{{- if eq .Arch "x86_64" -}}
+{{$arch = "amd64"}}
+{{- end -}}
+
+{{$os := .OS}}
+{{$ext := "tar.gz"}}
+
+{{ if HasPrefix .OS "ming" -}}
+{{$os = "windows"}}
+{{$ext = "zip"}}
+{{- end -}}
+
+https://get.helm.sh/helm-{{.Version}}-{{$os}}-{{$arch}}.{{$ext}}`,
+		},
 	}
 	return tools
 }

pkg/get/get_test.go

@@ -151,3 +151,66 @@ func Test_DownloadWindows(t *testing.T) {
 		t.Fatalf("want: %s, got: %s", want, got)
 	}
 }
+
+func Test_DownloadHelmDarwin(t *testing.T) {
+	tools := MakeTools()
+	name := "helm"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("darwin", arch64bit, tool.Version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://get.helm.sh/helm-v3.2.4-darwin-amd64.tar.gz"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadHelmLinux(t *testing.T) {
+	tools := MakeTools()
+	name := "helm"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("linux", arch64bit, tool.Version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://get.helm.sh/helm-v3.2.4-linux-amd64.tar.gz"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}
+
+func Test_DownloadHelmWindows(t *testing.T) {
+	tools := MakeTools()
+	name := "helm"
+	var tool *Tool
+	for _, target := range tools {
+		if name == target.Name {
+			tool = &target
+			break
+		}
+	}
+
+	got, err := tool.GetURL("mingw64_nt-10.0-18362", arch64bit, tool.Version)
+	if err != nil {
+		t.Fatal(err)
+	}
+	want := "https://get.helm.sh/helm-v3.2.4-windows-amd64.zip"
+	if got != want {
+		t.Fatalf("want: %s, got: %s", want, got)
+	}
+}