update some repo deps, cleanup a bit of logic, make notes about auth …

…impl

Signed-off-by: Tony Worm <tony@hofstadter.io>

go.mod

@@ -23,7 +23,7 @@ require (
 	github.com/go-git/go-billy/v5 v5.0.0
 	github.com/go-git/go-git/v5 v5.1.0
 	github.com/golang/protobuf v1.4.2 // indirect
-	github.com/google/go-github/v30 v30.1.0
+	github.com/google/go-github/v38 v38.1.0
 	github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed
 	github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d
 	github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729
@@ -46,7 +46,7 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.6.1
-	github.com/xanzy/go-gitlab v0.50.1
+	github.com/xanzy/go-gitlab v0.50.3
 	go.uber.org/zap v1.15.0
 	golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d

go.sum

@@ -10,8 +10,6 @@ cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7
 cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk=
 cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I=
 cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw=
-cuelang.org/go v0.4.0-rc.1 h1:X8fsqVhLCvXFhsWMGbI8rjTal45YOgt+ko+m7rOCySM=
-cuelang.org/go v0.4.0-rc.1/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs=
 cuelang.org/go v0.4.0 h1:GLJblw6m2WGGCA3k1v6Wbk9gTOt2qto48ahO2MmSd6I=
 cuelang.org/go v0.4.0/go.mod h1:tz/edkPi+T37AZcb5GlPY+WJkL6KiDlDVupKwL3vvjs=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
@@ -138,10 +136,11 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
-github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-github/v30 v30.1.0 h1:VLDx+UolQICEOKu2m4uAoMti1SxuEBAl7RSEG16L+Oo=
-github.com/google/go-github/v30 v30.1.0/go.mod h1:n8jBpHl45a/rlBUtRJMOG4GhNADUQFEufcolZ95JfU8=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-github/v38 v38.1.0 h1:C6h1FkaITcBFK7gAmq4eFzt6gbhEhk7L5z6R3Uva+po=
+github.com/google/go-github/v38 v38.1.0/go.mod h1:cStvrz/7nFr0FoENgG6GLbp53WaelXucT+BBz/3VKx4=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -340,8 +339,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U=
 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc=
-github.com/xanzy/go-gitlab v0.50.1 h1:eH1G0/ZV1j81rhGrtbcePjbM5Ern7mPA4Xjt+yE+2PQ=
-github.com/xanzy/go-gitlab v0.50.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
+github.com/xanzy/go-gitlab v0.50.3 h1:M7ncgNhCN4jaFNyXxarJhCLa9Qi6fdmCxFFhMTQPZiY=
+github.com/xanzy/go-gitlab v0.50.3/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=

lib/mod/cache/fetch.go

@@ -43,12 +43,12 @@ func fetch(lang, mod, ver string, private bool) error {
 
 	switch remote {
 	case "github.com":
-		if err := github.Fetch(FS, owner, repo, tag); err != nil {
+		if err := github.Fetch(FS, owner, repo, tag, private); err != nil {
 			return fmt.Errorf("While fetching from github\n%w\n", err)
 		}
 
 	case "gitlab.com":
-		if err := gitlab.Fetch(FS, owner, repo, tag); err != nil {
+		if err := gitlab.Fetch(FS, owner, repo, tag, private); err != nil {
 			return fmt.Errorf("While fetching from gitlab\n%w\n", err)
 		}
 

lib/mod/hack.go

@@ -5,7 +5,7 @@ import (
 	"io/ioutil"
 	"strings"
 
-	googithub "github.com/google/go-github/v30/github"
+	googithub "github.com/google/go-github/v38/github"
 	"github.com/go-git/go-billy/v5/memfs"
 
 	"github.com/hofstadter-io/hof/lib/mod/cache"
@@ -43,7 +43,7 @@ func Hack(lang string, args []string) error {
 
 	// Fetch and write to cache if tag found
 	if T != nil {
-		zReader, err := github.FetchTagZip(client, T)
+		zReader, err := github.FetchTagZip(T)
 		if err != nil {
 			return fmt.Errorf("While fetching zipfile\n%w\n", err)
 		}

lib/yagu/repos/github/client.go

@@ -6,7 +6,7 @@ import (
 
 	"golang.org/x/oauth2"
 
-	"github.com/google/go-github/v30/github"
+	"github.com/google/go-github/v38/github"
 )
 
 const TokenEnv = "GITHUB_TOKEN"

lib/yagu/repos/github/fetch.go

@@ -8,13 +8,18 @@ import (
 	"strings"
 
 	"github.com/go-git/go-billy/v5"
-	"github.com/google/go-github/v30/github"
+	"github.com/google/go-github/v38/github"
 	"github.com/parnurzeal/gorequest"
 
 	"github.com/hofstadter-io/hof/lib/yagu"
 )
 
-func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) {
+func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) {
+	// TODO, ensure private requests force auth
+	// by default, we look for a token
+	// how can we fall back to ssh if needed?
+	// note, GitHub has deprecated basic auth
+
 	client, err := NewClient()
 	if err != nil {
 		return err
@@ -28,7 +33,7 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) {
 			return err
 		}
 
-		zReader, err = FetchBranchZip(client, owner, repo, *r.DefaultBranch)
+		zReader, err = FetchBranchZip(owner, repo, *r.DefaultBranch)
 		if err != nil {
 			return fmt.Errorf("While fetching branch zipfile for %s/%s@%s\n%w\n", owner, repo, *r.DefaultBranch, err)
 		}
@@ -52,7 +57,7 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) {
 			return fmt.Errorf("Did not find tag %q for 'https://github.com/%s/%s' @%s", tag, owner, repo, tag)
 		}
 
-		zReader, err = FetchTagZip(client, T)
+		zReader, err = FetchTagZip(T)
 		if err != nil {
 			fmt.Printf("Error: %v\n", err)
 			return fmt.Errorf("While fetching tag zipfile\n%w\n", err)
@@ -70,15 +75,16 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) {
 	return nil
 }
 
-func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader, error) {
+func FetchTagZip(tag *github.RepositoryTag) (*zip.Reader, error) {
 
 	url := *tag.ZipballURL
 
 	fmt.Println("url:", url)
 
 	req := gorequest.New().Get(url)
 
-	if token := os.Getenv("GITHUB_TOKEN"); token != "" {
+	// TODO, process auth logic here better, maybe find a way to DRY
+	if token := os.Getenv(TokenEnv); token != "" {
 		req.SetBasicAuth("github-token", token)
 	}
 
@@ -106,13 +112,14 @@ func FetchTagZip(client *github.Client, tag *github.RepositoryTag) (*zip.Reader,
 	return zfile, err
 }
 
-func FetchBranchZip(client *github.Client, owner, repo, branch string) (*zip.Reader, error) {
+func FetchBranchZip(owner, repo, branch string) (*zip.Reader, error) {
 
 	url := fmt.Sprintf("https://github.com/%s/%s/archive/refs/heads/%s.zip", owner, repo, branch)
 
 	req := gorequest.New().Get(url)
 
-	if token := os.Getenv("GITHUB_TOKEN"); token != "" {
+	// TODO, process auth logic here better, maybe find a way to DRY
+	if token := os.Getenv(TokenEnv); token != "" {
 		req.SetBasicAuth("github-token", token)
 	}
 

lib/yagu/repos/github/info.go

@@ -5,7 +5,7 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/google/go-github/v30/github"
+	"github.com/google/go-github/v38/github"
 )
 
 func GetTagsSplit(client *github.Client, module string) ([]*github.RepositoryTag, error) {
@@ -31,7 +31,7 @@ func GetRepo(client *github.Client, owner, repo string) (*github.Repository, err
 }
 
 func GetBranch(client *github.Client, owner, repo, branch string) (*github.Branch, error) {
-	b, _, err := client.Repositories.GetBranch(context.Background(), owner, repo, branch)
+	b, _, err := client.Repositories.GetBranch(context.Background(), owner, repo, branch, true)
 	return b, err
 }
 

lib/yagu/repos/gitlab/client.go

@@ -1,13 +1,23 @@
 package gitlab
 
 import (
+	"fmt"
 	"os"
 
 	"github.com/xanzy/go-gitlab"
 )
 
 const TokenEnv = "GITLAB_TOKEN"
 
-func NewClient() (client *gitlab.Client, err error) {
-	return gitlab.NewClient(os.Getenv(TokenEnv))
+func NewClient(private bool) (client *gitlab.Client, err error) {
+	// TODO, there are multiple NewClient<> methods
+	// how to determine which to use and the inputs to them
+	// https://pkg.go.dev/github.com/xanzy/go-gitlab#NewOAuthClient
+	// Noting also that we prefer auth over non-auth for API rate limits
+
+	token := os.Getenv(TokenEnv)
+	if private && token == "" {
+		return nil, fmt.Errorf("Private module requested and no auth token available for %s", TokenEnv)
+	}
+	return gitlab.NewClient(token)
 }

lib/yagu/repos/gitlab/fetch.go

@@ -12,23 +12,8 @@ import (
 	"github.com/hofstadter-io/hof/lib/yagu"
 )
 
-func fetchShaZip(client *gitlab.Client, pid interface{}, sha string) (*zip.Reader, error) {
-	format := "zip"
-	data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{
-		Format: &format,
-		SHA:    &sha,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	r := bytes.NewReader(data)
-
-	return zip.NewReader(r, int64(len(data)))
-}
-
-func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) {
-	client, err := NewClient()
+func Fetch(FS billy.Filesystem, owner, repo, tag string, private bool) (error) {
+	client, err := NewClient(private)
 	if err != nil {
 		return err
 	}
@@ -78,3 +63,19 @@ func Fetch(FS billy.Filesystem, owner, repo, tag string) (error) {
 
 	return nil
 }
+
+func fetchShaZip(client *gitlab.Client, pid, sha string) (*zip.Reader, error) {
+	format := "zip"
+	data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{
+		Format: &format,
+		SHA:    &sha,
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	r := bytes.NewReader(data)
+
+	return zip.NewReader(r, int64(len(data)))
+}
+