From c228cd6cfbcb96b0cd9b80fef3dbfc6f294b7695 Mon Sep 17 00:00:00 2001 From: Marcos Lilljedahl Date: Sun, 5 Jul 2020 20:46:46 -0300 Subject: [PATCH] Move tar.gz parsing login to GH provider. This commit also adds support to install and update releases with slashes on the tag name. Signed-off-by: Marcos Lilljedahl --- cmd/install.go | 104 +--------------------- cmd/update.go | 73 ++++++++++----- pkg/providers/docker.go | 2 +- pkg/providers/github.go | 176 +++++++++++++++++++++++++++++-------- pkg/providers/providers.go | 8 +- 5 files changed, 197 insertions(+), 166 deletions(-) diff --git a/cmd/install.go b/cmd/install.go index 1c766b8..f4fb262 100644 --- a/cmd/install.go +++ b/cmd/install.go @@ -1,22 +1,13 @@ package cmd import ( - "archive/tar" - "bytes" - "compress/gzip" - "errors" "fmt" "io" - "io/ioutil" "os" "path/filepath" - "strings" "github.com/apex/log" - "github.com/h2non/filetype" - "github.com/h2non/filetype/matchers" "github.com/marcosnils/bin/pkg/config" - "github.com/marcosnils/bin/pkg/options" "github.com/marcosnils/bin/pkg/providers" "github.com/spf13/cobra" ) @@ -137,33 +128,6 @@ func getFinalPath(path, fileName string) (string, error) { //TODO check if other binary has the same hash and warn about it. //TODO if the file is zipped, tared, whatever then extract it func saveToDisk(f *providers.File, path string, overwrite bool) error { - defer f.Data.Close() - - var buf bytes.Buffer - tee := io.TeeReader(f.Data, &buf) - - t, err := filetype.MatchReader(tee) - if err != nil { - return err - } - - var outputFile = io.MultiReader(&buf, f.Data) - - // TODO: validating the type of the file will eventually be - // handled by each provider - // if t != matchers.TypeElf && t != matchers.TypeGz { - // return fmt.Errorf("File type [%v] not supported", t) - // } - - if t == matchers.TypeGz { - fileName, file, err := processTarGz(outputFile) - if err != nil { - return err - } - outputFile = file - path = strings.Replace(path, filepath.Base(path), fileName, -1) - - } var extraFlags int = os.O_EXCL @@ -181,76 +145,10 @@ func saveToDisk(f *providers.File, path string, overwrite bool) error { //TODO add a spinner here indicating that the binary is being downloaded log.Infof("Starting download for %s@%s into %s", f.Name, f.Version, path) - _, err = io.Copy(file, outputFile) + _, err = io.Copy(file, f.Data) if err != nil { return err } return nil } - -// processTar receives a tar.gz file and returns the -// correct file for bin to download -func processTarGz(r io.Reader) (string, io.Reader, error) { - // We're caching the whole file into memory so we can prompt - // the user which file they want to download - - b, err := ioutil.ReadAll(r) - if err != nil { - return "", nil, err - } - br := bytes.NewReader(b) - - gr, err := gzip.NewReader(br) - if err != nil { - return "", nil, err - } - - tr := tar.NewReader(gr) - tarFiles := []interface{}{} - for { - header, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return "", nil, err - } - - if header.Typeflag == tar.TypeReg { - tarFiles = append(tarFiles, header.Name) - } - } - if len(tarFiles) == 0 { - return "", nil, errors.New("No files found in tar archive") - } - - selectedFile := options.Select("Select file to download:", tarFiles).(string) - - // Reset readers so we can scan the tar file - // again to get the correct file reader - br.Seek(0, io.SeekStart) - gr, err = gzip.NewReader(br) - if err != nil { - return "", nil, err - } - tr = tar.NewReader(gr) - - var fr io.Reader - for { - header, err := tr.Next() - if err == io.EOF { - break - } else if err != nil { - return "", nil, err - } - - if header.Name == selectedFile { - fr = tr - break - } - } - // return base of selected file since tar - // files usually have folders inside - return filepath.Base(selectedFile), fr, nil - -} diff --git a/cmd/update.go b/cmd/update.go index bf0d24f..54b7fc5 100644 --- a/cmd/update.go +++ b/cmd/update.go @@ -18,19 +18,24 @@ type updateCmd struct { type updateOpts struct { } +type updateInfo struct{ version, url string } + func newUpdateCmd() *updateCmd { var root = &updateCmd{} // nolint: dupl var cmd = &cobra.Command{ - Use: "update", + Use: "update [binary_path]", Aliases: []string{"u"}, - Short: "Updates binaries managed by bin", + Short: "Updates one or multiple binaries managed by bin", SilenceUsage: true, + Args: cobra.MaximumNArgs(1), SilenceErrors: true, RunE: func(cmd *cobra.Command, args []string) error { - //TODO add support o update a single binary with - //`bin update ` + var bin string + if len(args) > 0 { + bin = args[0] + } //TODO update should check all binaries with a //certain configured parallelism (default 10, can be changed with -p) and report @@ -38,29 +43,29 @@ func newUpdateCmd() *updateCmd { //It's very likely that we have to extend the provider //interface to support this use-case - type updateInfo struct{ version, url string } - - toUpdate := map[updateInfo]*config.Binary{} + toUpdate := map[*updateInfo]*config.Binary{} cfg := config.Get() - for _, b := range cfg.Bins { - - p, err := providers.New(b.URL) - - if err != nil { - return err - } - log.Debugf("Checking updates for %s", b.Path) - v, u, err := p.GetLatestVersion(b.RemoteName) - - if err != nil { - return fmt.Errorf("Error checking updates for %s, %w", b.Path, err) + // Update single binary + if bin != "" { + if b, found := cfg.Bins[bin]; !found { + return fmt.Errorf("Binary path %s not found", bin) + + } else { + if ui, err := getLatestVersion(b); err != nil { + return err + } else if ui != nil { + toUpdate[ui] = b + } } - if b.Version != v { - log.Debugf("Found new version %s for %s", v, b.Path) - log.Infof("%s %s -> %s ", b.Path, color.YellowString(b.Version), color.GreenString(v)) - toUpdate[updateInfo{v, u}] = b + } else { + for _, b := range cfg.Bins { + if ui, err := getLatestVersion(b); err != nil { + return err + } else if ui != nil { + toUpdate[ui] = b + } } } @@ -122,3 +127,25 @@ func newUpdateCmd() *updateCmd { root.cmd = cmd return root } + +func getLatestVersion(b *config.Binary) (*updateInfo, error) { + p, err := providers.New(b.URL) + + if err != nil { + return nil, err + } + + log.Debugf("Checking updates for %s", b.Path) + v, u, err := p.GetLatestVersion() + + if err != nil { + return nil, fmt.Errorf("Error checking updates for %s, %w", b.Path, err) + } + + if b.Version == v { + return nil, nil + } + log.Debugf("Found new version %s for %s at %s", v, b.Path, u) + log.Infof("%s %s -> %s ", b.Path, color.YellowString(b.Version), color.GreenString(v)) + return &updateInfo{v, u}, nil +} diff --git a/pkg/providers/docker.go b/pkg/providers/docker.go index 184e7e8..ccef9ae 100644 --- a/pkg/providers/docker.go +++ b/pkg/providers/docker.go @@ -63,7 +63,7 @@ func getImageName(repo string) string { } // TODO: implement -func (d *docker) GetLatestVersion(name string) (string, string, error) { +func (d *docker) GetLatestVersion() (string, string, error) { return "", "", nil } diff --git a/pkg/providers/github.go b/pkg/providers/github.go index beaf407..1de2c70 100644 --- a/pkg/providers/github.go +++ b/pkg/providers/github.go @@ -1,16 +1,25 @@ package providers import ( + "archive/tar" + "bytes" + "compress/gzip" "context" "crypto/sha256" + "errors" "fmt" + "io" + "io/ioutil" "net/http" "net/url" "os" + "path/filepath" "strings" "github.com/apex/log" "github.com/google/go-github/v31/github" + "github.com/h2non/filetype" + "github.com/h2non/filetype/matchers" "github.com/marcosnils/bin/pkg/config" "github.com/marcosnils/bin/pkg/options" bstrings "github.com/marcosnils/bin/pkg/strings" @@ -29,6 +38,32 @@ type githubFileInfo struct{ url, name string } func (g *githubFileInfo) String() string { return g.name } +// filterAssets receives a slice of GH assets and tries to +// select the proper one and ask the user to manually select one +// in case it can't determine it +func filterAssets(as []*github.ReleaseAsset) (*githubFileInfo, error) { + matches := []interface{}{} + for _, a := range as { + lowerName := strings.ToLower(*a.Name) + if bstrings.ContainsAny(lowerName, config.GetOS()) && bstrings.ContainsAny(lowerName, config.GetArch()) { + matches = append(matches, &githubFileInfo{a.GetBrowserDownloadURL(), a.GetName()}) + } + } + + var gf *githubFileInfo + if len(matches) == 0 { + return nil, fmt.Errorf("Could not find any compatbile files") + } else if len(matches) > 1 { + gf = options.Select("Multiple matches found, please select one:", matches).(*githubFileInfo) + //TODO make user select the proper file + } else { + gf = matches[0].(*githubFileInfo) + } + + return gf, nil + +} + func (g *gitHub) Fetch() (*File, error) { var release *github.RepositoryRelease @@ -48,28 +83,15 @@ func (g *gitHub) Fetch() (*File, error) { return nil, err } - var f *File - matches := []interface{}{} - for _, a := range release.Assets { - lowerName := strings.ToLower(*a.Name) - if bstrings.ContainsAny(lowerName, config.GetOS()) && bstrings.ContainsAny(lowerName, config.GetArch()) { - matches = append(matches, &githubFileInfo{a.GetBrowserDownloadURL(), a.GetName()}) - } - } + gf, err := filterAssets(release.Assets) - var gf *githubFileInfo - if len(matches) == 0 { - return nil, fmt.Errorf("Could not find any compatbile files") - } else if len(matches) > 1 { - gf = options.Select("Multiple matches found, please select one:", matches).(*githubFileInfo) - //TODO make user select the proper file - } else { - gf = matches[0].(*githubFileInfo) + if err != nil { + return nil, err } // We're not closing the body here since the caller is in charge of that res, err := http.Get(gf.url) - log.Debugf("Checking binary form %s", gf.url) + log.Debugf("Checking binary from %s", gf.url) if err != nil { return nil, err } @@ -78,39 +100,118 @@ func (g *gitHub) Fetch() (*File, error) { return nil, fmt.Errorf("%d response when checking binary from %s", res.StatusCode, gf.url) } + var buf bytes.Buffer + tee := io.TeeReader(res.Body, &buf) + + t, err := filetype.MatchReader(tee) + if err != nil { + return nil, err + } + + var outputFile = io.MultiReader(&buf, res.Body) + + // TODO: validating the type of the file will eventually be + // handled by each provider since it's impossible to make it generic enough + // if t != matchers.TypeElf && t != matchers.TypeGz { + // return fmt.Errorf("File type [%v] not supported", t) + // } + + var name = gf.name + + if t == matchers.TypeGz { + fileName, file, err := processTarGz(outputFile) + if err != nil { + return nil, err + } + outputFile = file + name = fileName + + } + //TODO calculate file hash. Not sure if we can / should do it here //since we don't want to read the file unnecesarily. Additionally, sometimes //releases have .sha256 files, so it'd be nice to check for those also - f = &File{Data: res.Body, Name: gf.name, Hash: sha256.New(), Version: getVersion(gf.url)} + f := &File{Data: outputFile, Name: name, Hash: sha256.New(), Version: release.GetTagName()} + return f, nil } -//GetLatestVersion returns the version and the URL of the -//specified asset name -func (g *gitHub) GetLatestVersion(name string) (string, string, error) { - log.Debugf("Getting latest release for %s/%s", g.owner, g.repo) - release, _, err := g.client.Repositories.GetLatestRelease(context.TODO(), g.owner, g.repo) +// processTar receives a tar.gz file and returns the +// correct file for bin to download +func processTarGz(r io.Reader) (string, io.Reader, error) { + // We're caching the whole file into memory so we can prompt + // the user which file they want to download + + b, err := ioutil.ReadAll(r) if err != nil { - return "", "", err + return "", nil, err + } + br := bytes.NewReader(b) + + gr, err := gzip.NewReader(br) + if err != nil { + return "", nil, err } - var newDownloadUrl string - //TODO if asset can be found with the same name it had before, - //we should prompt the user if he wants to change the asset - for _, a := range release.Assets { - if a.GetName() == name { - newDownloadUrl = a.GetBrowserDownloadURL() + tr := tar.NewReader(gr) + tarFiles := []interface{}{} + for { + header, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return "", nil, err } + if header.Typeflag == tar.TypeReg { + tarFiles = append(tarFiles, header.Name) + } + } + if len(tarFiles) == 0 { + return "", nil, errors.New("No files found in tar archive") + } + + selectedFile := options.Select("Select file to download:", tarFiles).(string) + + // Reset readers so we can scan the tar file + // again to get the correct file reader + br.Seek(0, io.SeekStart) + gr, err = gzip.NewReader(br) + if err != nil { + return "", nil, err + } + tr = tar.NewReader(gr) + + var fr io.Reader + for { + header, err := tr.Next() + if err == io.EOF { + break + } else if err != nil { + return "", nil, err + } + + if header.Name == selectedFile { + fr = tr + break + } } - return release.GetTagName(), newDownloadUrl, nil + // return base of selected file since tar + // files usually have folders inside + return filepath.Base(selectedFile), fr, nil + } -// getVersion returns the asset version given the -// browser download URL -func getVersion(url string) string { - s := strings.Split(url, "/") - return s[len(s)-2] +//GetLatestVersion checks the latest repo release and +//returns the corresponding metadata +func (g *gitHub) GetLatestVersion() (string, string, error) { + log.Debugf("Getting latest release for %s/%s", g.owner, g.repo) + release, _, err := g.client.Repositories.GetLatestRelease(context.TODO(), g.owner, g.repo) + if err != nil { + return "", "", err + } + + return release.GetTagName(), release.GetHTMLURL(), nil } func newGitHub(u *url.URL) (Provider, error) { @@ -128,11 +229,12 @@ func newGitHub(u *url.URL) (Provider, error) { ps := strings.Split(u.Path, "/") for i, p := range ps { if p == "releases" { - tag = ps[i+2] + tag = strings.Join(ps[i+2:], "/") } } } + token := os.Getenv("GITHUB_AUTH_TOKEN") var tc *http.Client if token != "" { diff --git a/pkg/providers/providers.go b/pkg/providers/providers.go index dbca0b1..fbbd66c 100644 --- a/pkg/providers/providers.go +++ b/pkg/providers/providers.go @@ -13,15 +13,19 @@ import ( var ErrInvalidProvider = errors.New("invalid provider") type File struct { - Data io.ReadCloser + Data io.Reader Name string Hash hash.Hash Version string } type Provider interface { + // Fetch returns the file metadata to retrieve a specific binary given + // for a provider Fetch() (*File, error) - GetLatestVersion(string) (string, string, error) + // GetLatestVersion returns the version and the URL of the + // latest version for this binary + GetLatestVersion() (string, string, error) } var (