diff --git a/go.mod b/go.mod index a13221446..83f8f7d91 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,7 @@ require ( github.com/go-git/go-billy/v5 v5.0.0 github.com/go-git/go-git/v5 v5.1.0 github.com/golang/protobuf v1.4.2 // indirect - github.com/google/go-github/v30 v30.1.0 + github.com/google/go-github/v38 v38.1.0 github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729 @@ -46,7 +46,7 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.6.1 - github.com/xanzy/go-gitlab v0.50.1 + github.com/xanzy/go-gitlab v0.50.3 go.uber.org/zap v1.15.0 golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d diff --git a/go.sum b/go.sum index a81aec17c..f44f20010 100644 --- a/go.sum +++ b/go.sum @@ -10,8 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= -cuelang.org/go v0.4.0-rc.1 h1:X8fsqVhLCvXFhsWMGbI8rjTal45YOgt+ko+m7rOCySM= -cuelang.org/go v0.4.0-rc.1/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs= cuelang.org/go v0.4.0 h1:GLJblw6m2WGGCA3k1v6Wbk9gTOt2qto48ahO2MmSd6I= cuelang.org/go v0.4.0/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -138,10 +136,11 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo= -github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-github/v38 v38.1.0 h1:C6h1FkaITcBFK7gAmq4eFzt6gbhEhk7L5z6R3Uva+po= +github.com/google/go-github/v38 v38.1.0/go.mod h1:cStvrz/7nFr0FoENgG6GLbp53WaelXucT+BBz/3VKx4= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -340,8 +339,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= -github.com/xanzy/go-gitlab v0.50.1 h1:eH1G0/ZV1j81rhGrtbcePjbM5Ern7mPA4Xjt+yE+2PQ= -github.com/xanzy/go-gitlab v0.50.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= +github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY= +github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= diff --git a/lib/mod/cache/fetch.go b/lib/mod/cache/fetch.go index 5b4586d8e..3d7271b72 100644 --- a/lib/mod/cache/fetch.go +++ b/lib/mod/cache/fetch.go @@ -43,12 +43,12 @@ func fetch(lang, mod, ver string, private bool) error { switch remote { case "github.com": - if err := github.Fetch(FS, owner, repo, tag); err != nil { + if err := github.Fetch(FS, owner, repo, tag, private); err != nil { return fmt.Errorf("While fetching from github\n%w\n", err) } case "gitlab.com": - if err := gitlab.Fetch(FS, owner, repo, tag); err != nil { + if err := gitlab.Fetch(FS, owner, repo, tag, private); err != nil { return fmt.Errorf("While fetching from gitlab\n%w\n", err) } diff --git a/lib/mod/hack.go b/lib/mod/hack.go index c250acfc1..882362877 100644 --- a/lib/mod/hack.go +++ b/lib/mod/hack.go @@ -5,7 +5,7 @@ import ( "io/ioutil" "strings" - googithub "github.com/google/go-github/v30/github" + googithub "github.com/google/go-github/v38/github" "github.com/go-git/go-billy/v5/memfs" "github.com/hofstadter-io/hof/lib/mod/cache" @@ -43,7 +43,7 @@ func Hack(lang string, args []string) error { // Fetch and write to cache if tag found if T != nil { - zReader, err := github.FetchTagZip(client, T) + zReader, err := github.FetchTagZip(T) if err != nil { return fmt.Errorf("While fetching zipfile\n%w\n", err) } diff --git a/lib/yagu/repos/github/client.go b/lib/yagu/repos/github/client.go index fe083c325..7d9497e7c 100644 --- a/lib/yagu/repos/github/client.go +++ b/lib/yagu/repos/github/client.go @@ -6,7 +6,7 @@ import ( "golang.org/x/oauth2" - "github.com/google/go-github/v30/github" + "github.com/google/go-github/v38/github" ) const TokenEnv = "GITHUB_TOKEN" diff --git a/lib/yagu/repos/github/fetch.go b/lib/yagu/repos/github/fetch.go index 95e719a65..d79df411b 100644 --- a/lib/yagu/repos/github/fetch.go +++ b/lib/yagu/repos/github/fetch.go @@ -8,13 +8,18 @@ import ( "strings" "github.com/go-git/go-billy/v5" - "github.com/google/go-github/v30/github" + "github.com/google/go-github/v38/github" "github.com/parnurzeal/gorequest" "github.com/hofstadter-io/hof/lib/yagu" ) -func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) { +func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) { + // TODO, ensure private requests force auth + // by default, we look for a token + // how can we fall back to ssh if needed? + // note, GitHub has deprecated basic auth + client, err := NewClient() if err != nil { return err @@ -28,7 +33,7 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) { return err } - zReader, err = FetchBranchZip(client, owner, repo, *r.DefaultBranch) + zReader, err = FetchBranchZip(owner, repo, *r.DefaultBranch) if err != nil { return fmt.Errorf("While fetching branch zipfile for %s/%s@%s\n%w\n", owner, repo, *r.DefaultBranch, err) } @@ -52,7 +57,7 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) { return fmt.Errorf("Did not find tag %q for 'https://github.com/%s/%s' @%s", tag, owner, repo, tag) } - zReader, err = FetchTagZip(client, T) + zReader, err = FetchTagZip(T) if err != nil { fmt.Printf("Error: %v\n", err) return fmt.Errorf("While fetching tag zipfile\n%w\n", err) @@ -70,7 +75,7 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) { return nil } -func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader, error) { +func FetchTagZip(tag *github.RepositoryTag) (*zip.Reader, error) { url := *tag.ZipballURL @@ -78,7 +83,8 @@ func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader, req := gorequest.New().Get(url) - if token := os.Getenv("GITHUB_TOKEN"); token != "" { + // TODO, process auth logic here better, maybe find a way to DRY + if token := os.Getenv(TokenEnv); token != "" { req.SetBasicAuth("github-token", token) } @@ -106,13 +112,14 @@ func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader, return zfile, err } -func FetchBranchZip(client *github.Client, owner, repo, branch string) (*zip.Reader, error) { +func FetchBranchZip(owner, repo, branch string) (*zip.Reader, error) { url := fmt.Sprintf("https://github.com/%s/%s/archive/refs/heads/%s.zip", owner, repo, branch) req := gorequest.New().Get(url) - if token := os.Getenv("GITHUB_TOKEN"); token != "" { + // TODO, process auth logic here better, maybe find a way to DRY + if token := os.Getenv(TokenEnv); token != "" { req.SetBasicAuth("github-token", token) } diff --git a/lib/yagu/repos/github/info.go b/lib/yagu/repos/github/info.go index 8ba8fa163..2ab3f22c7 100644 --- a/lib/yagu/repos/github/info.go +++ b/lib/yagu/repos/github/info.go @@ -5,7 +5,7 @@ import ( "fmt" "strings" - "github.com/google/go-github/v30/github" + "github.com/google/go-github/v38/github" ) func GetTagsSplit(client *github.Client, module string) ([]*github.RepositoryTag, error) { @@ -31,7 +31,7 @@ func GetRepo(client *github.Client, owner, repo string) (*github.Repository, err } func GetBranch(client *github.Client, owner, repo, branch string) (*github.Branch, error) { - b, _, err := client.Repositories.GetBranch(context.Background(), owner, repo, branch) + b, _, err := client.Repositories.GetBranch(context.Background(), owner, repo, branch, true) return b, err } diff --git a/lib/yagu/repos/gitlab/client.go b/lib/yagu/repos/gitlab/client.go index 157cc6783..0df560503 100644 --- a/lib/yagu/repos/gitlab/client.go +++ b/lib/yagu/repos/gitlab/client.go @@ -1,6 +1,7 @@ package gitlab import ( + "fmt" "os" "github.com/xanzy/go-gitlab" @@ -8,6 +9,15 @@ import ( const TokenEnv = "GITLAB_TOKEN" -func NewClient() (client *gitlab.Client, err error) { - return gitlab.NewClient(os.Getenv(TokenEnv)) +func NewClient(private bool) (client *gitlab.Client, err error) { + // TODO, there are multiple NewClient<> methods + // how to determine which to use and the inputs to them + // https://pkg.go.dev/github.com/xanzy/go-gitlab#NewOAuthClient + // Noting also that we prefer auth over non-auth for API rate limits + + token := os.Getenv(TokenEnv) + if private && token == "" { + return nil, fmt.Errorf("Private module requested and no auth token available for %s", TokenEnv) + } + return gitlab.NewClient(token) } diff --git a/lib/yagu/repos/gitlab/fetch.go b/lib/yagu/repos/gitlab/fetch.go index 22e10f914..7a0da9bfb 100644 --- a/lib/yagu/repos/gitlab/fetch.go +++ b/lib/yagu/repos/gitlab/fetch.go @@ -12,23 +12,8 @@ import ( "github.com/hofstadter-io/hof/lib/yagu" ) -func fetchShaZip(client *gitlab.Client, pid interface{}, sha string) (*zip.Reader, error) { - format := "zip" - data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{ - Format: &format, - SHA: &sha, - }) - if err != nil { - return nil, err - } - - r := bytes.NewReader(data) - - return zip.NewReader(r, int64(len(data))) -} - -func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) { - client, err := NewClient() +func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) { + client, err := NewClient(private) if err != nil { return err } @@ -78,3 +63,19 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) { return nil } + +func fetchShaZip(client *gitlab.Client, pid, sha string) (*zip.Reader, error) { + format := "zip" + data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{ + Format: &format, + SHA: &sha, + }) + if err != nil { + return nil, err + } + + r := bytes.NewReader(data) + + return zip.NewReader(r, int64(len(data))) +} +