diff --git a/go.mod b/go.mod index 8b0258251..a13221446 100644 --- a/go.mod +++ b/go.mod @@ -46,12 +46,10 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/spf13/viper v1.7.0 github.com/stretchr/testify v1.6.1 + github.com/xanzy/go-gitlab v0.50.1 go.uber.org/zap v1.15.0 - golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 // indirect golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449 - golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d - golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect golang.org/x/text v0.3.3 google.golang.org/appengine v1.6.6 // indirect google.golang.org/protobuf v1.24.0 // indirect diff --git a/go.sum b/go.sum index 1f697ad4f..a81aec17c 100644 --- a/go.sum +++ b/go.sum @@ -162,10 +162,15 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM= github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-hclog v0.9.2 h1:CG6TE5H9/JXsFWJCfoIVpKFIkFe6ysEuHirp4DxCsHI= +github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= +github.com/hashicorp/go-retryablehttp v0.6.8 h1:92lWxgpa+fF3FozM4B3UZtHZMJX8T5XT+TFdCxsPyWs= +github.com/hashicorp/go-retryablehttp v0.6.8/go.mod h1:vAew36LZh98gCBJNLH42IQ1ER/9wtLZZ8meHqQvEYWY= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= @@ -335,6 +340,8 @@ github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= +github.com/xanzy/go-gitlab v0.50.1 h1:eH1G0/ZV1j81rhGrtbcePjbM5Ern7mPA4Xjt+yE+2PQ= +github.com/xanzy/go-gitlab v0.50.1/go.mod h1:Q+hQhV508bDPoBijv7YjK/Lvlb4PhVhJdKqXVQrUoAE= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= @@ -362,8 +369,8 @@ golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200302210943-78000ba7a073/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 h1:vEg9joUBmeBcK9iSJftGNf3coIG4HqZElCPehJsfAYM= -golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -408,9 +415,10 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9 h1:pNX+40auqi2JqRfOP1akLGtYcn15TUbkhwuCO3foqqM= -golang.org/x/net v0.0.0-20200602114024-627f9648deb9/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20201021035429-f5854403a974 h1:IX6qOQeG5uLjB/hjjwjedwfjND0hgjPMMyO1RoIXQNI= +golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= @@ -421,6 +429,7 @@ golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -443,9 +452,8 @@ golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f h1:+Nyd8tzPX9R7BWHguqsrbFdRx3WQ/1ib8I44HXV5yTA= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -453,6 +461,8 @@ golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0 h1:/5xXl8Y5W96D+TtHSlonuFqGHIWVuyCkGJLwGh9JJFs= +golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -488,6 +498,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.13.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= diff --git a/lib/mod/cache/checksum.go b/lib/mod/cache/checksum.go index 0f0656031..cc91146a6 100644 --- a/lib/mod/cache/checksum.go +++ b/lib/mod/cache/checksum.go @@ -2,17 +2,12 @@ package cache import ( "os" - "strings" "golang.org/x/mod/sumdb/dirhash" ) func Checksum(lang, mod, ver string) (string, error) { - - flds := strings.SplitN(mod, "/", 3) - remote := flds[0] - owner := flds[1] - repo := flds[2] + remote, owner, repo := splitMod(mod) tag := ver dir := Outdir(lang, remote, owner, repo, tag) diff --git a/lib/mod/cache/fetch.go b/lib/mod/cache/fetch.go index 70677dd86..5ba2bfb48 100644 --- a/lib/mod/cache/fetch.go +++ b/lib/mod/cache/fetch.go @@ -17,13 +17,11 @@ import ( "github.com/hofstadter-io/hof/lib/yagu" "github.com/hofstadter-io/hof/lib/yagu/repos/github" + "github.com/hofstadter-io/hof/lib/yagu/repos/gitlab" ) func Fetch(lang, mod, ver string) (err error) { - flds := strings.SplitN(mod, "/", 3) - remote := flds[0] - owner := flds[1] - repo := flds[2] + remote, owner, repo := splitMod(mod) tag := ver dir := Outdir(lang, remote, owner, repo, tag) @@ -46,15 +44,14 @@ func Fetch(lang, mod, ver string) (err error) { } func fetch(lang, mod, ver string) error { - flds := strings.SplitN(mod, "/", 3) - remote := flds[0] - owner := flds[1] - repo := flds[2] + remote, owner, repo := splitMod(mod) tag := ver switch remote { case "github.com": return fetchGitHub(lang, owner, repo, tag) + case "gitlab.com": + return fetchGitLab(lang, owner, repo, tag) default: return fetchGit(lang, remote, owner, repo, tag) } @@ -120,6 +117,29 @@ func getSSHAuth(remote string) (string, *ssh.PublicKeys, error) { return fmt.Sprintf("%s@%s", usr, remote), pks, nil } +func fetchGitLab(lang, owner, repo, tag string) (err error) { + FS := memfs.New() + client, err := gitlab.NewClient() + if err != nil { + return err + } + + zReader, err := gitlab.FetchZip(client, owner, repo, tag) + if err != nil { + return fmt.Errorf("While fetching from GitLab\n%w\n", err) + } + + if err := yagu.BillyLoadFromZip(zReader, FS, true); err != nil { + return fmt.Errorf("While reading zipfile\n%w\n", err) + } + + if err := Write(lang, "gitlab.com", owner, repo, tag, FS); err != nil { + return fmt.Errorf("While writing to cache\n%w\n", err) + } + + return nil +} + func fetchGitHub(lang, owner, repo, tag string) (err error) { FS := memfs.New() diff --git a/lib/mod/cache/lookup.go b/lib/mod/cache/lookup.go index 950d65d42..91624c9b3 100644 --- a/lib/mod/cache/lookup.go +++ b/lib/mod/cache/lookup.go @@ -2,17 +2,13 @@ package cache import ( "os" - "strings" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" ) func Load(lang, mod, ver string) (FS billy.Filesystem, err error) { - flds := strings.SplitN(mod, "/", 3) - remote := flds[0] - owner := flds[1] - repo := flds[2] + remote, owner, repo := splitMod(mod) tag := ver dir := Outdir(lang, remote, owner, repo, tag) diff --git a/lib/mod/cache/mod.go b/lib/mod/cache/mod.go new file mode 100644 index 000000000..ea1babed1 --- /dev/null +++ b/lib/mod/cache/mod.go @@ -0,0 +1,14 @@ +package cache + +import "strings" + +func splitMod(mod string) (remote, owner, repo string) { + var flds []string + if i := strings.Index(mod, ".git"); i > -1 { + flds = strings.SplitN(mod[:i], "/", 3) + } else { + flds = strings.Split(mod, "/") + } + + return flds[0], flds[1], flds[2] +} diff --git a/lib/mod/cache/mod_test.go b/lib/mod/cache/mod_test.go new file mode 100644 index 000000000..f739d246c --- /dev/null +++ b/lib/mod/cache/mod_test.go @@ -0,0 +1,28 @@ +package cache + +import ( + "fmt" + "testing" +) + +func TestSplitMod(t *testing.T) { + tests := map[string]struct { + mod string + expected string + }{ + "simple": {mod: "github.com/owner/repo", expected: "github.com/owner/repo"}, + "submodule": {mod: "github.com/owner/repo/submodule", expected: "github.com/owner/repo"}, + "complex": {mod: "gitlab.com/owner/repo.git/submodule", expected: "gitlab.com/owner/repo"}, + "subgroup": {mod: "gitlab.com/owner/subgroup/repo.git", expected: "gitlab.com/owner/subgroup/repo"}, + "subgroup+submodule": {mod: "gitlab.com/owner/subgroup/repo.git/submodule", expected: "gitlab.com/owner/subgroup/repo"}, + } + + for name, tc := range tests { + t.Run(name, func(t *testing.T) { + rm, o, rp := splitMod(tc.mod) + if fmt.Sprintf("%s/%s/%s", rm, o, rp) != tc.expected { + t.Fatalf("expected: %v, got: %s/%s/%s", tc.expected, rm, o, rp) + } + }) + } +} diff --git a/lib/yagu/repos/gitlab/client.go b/lib/yagu/repos/gitlab/client.go new file mode 100644 index 000000000..6a95e711a --- /dev/null +++ b/lib/yagu/repos/gitlab/client.go @@ -0,0 +1,11 @@ +package gitlab + +import ( + "os" + + "github.com/xanzy/go-gitlab" +) + +func NewClient() (client *gitlab.Client, err error) { + return gitlab.NewClient(os.Getenv("GITLAB_TOKEN")) +} diff --git a/lib/yagu/repos/gitlab/fetch.go b/lib/yagu/repos/gitlab/fetch.go new file mode 100644 index 000000000..3c230f91e --- /dev/null +++ b/lib/yagu/repos/gitlab/fetch.go @@ -0,0 +1,62 @@ +package gitlab + +import ( + "archive/zip" + "bytes" + "fmt" + + "github.com/xanzy/go-gitlab" +) + +func fetchShaZip(client *gitlab.Client, pid interface{}, sha string) (*zip.Reader, error) { + format := "zip" + data, _, err := client.Repositories.Archive(pid, &gitlab.ArchiveOptions{ + Format: &format, + SHA: &sha, + }) + if err != nil { + return nil, err + } + + r := bytes.NewReader(data) + + return zip.NewReader(r, int64(len(data))) +} + +func FetchZip(client *gitlab.Client, owner, repo, tag string) (*zip.Reader, error) { + pid := fmt.Sprintf("%s/%s", owner, repo) + + var sha string + + if tag == "v0.0.0" { + bs, _, err := client.Branches.ListBranches(pid, nil) + if err != nil { + return nil, err + } + + var branch *gitlab.Branch + + for _, candidate := range bs { + if candidate.Default { + branch = candidate + + break + } + } + + if branch == nil { + return nil, fmt.Errorf("Could not find default branch for repository %s", pid) + } + + sha = branch.Commit.ID + } else { + t, _, err := client.Tags.GetTag(pid, tag) + if err != nil { + return nil, err + } + + sha = t.Commit.ID + } + + return fetchShaZip(client, pid, sha) +}