Merge pull request #69 from b4nst/feat/add-generic-git-package-support

feat: add generic git package manager

.gitignore

@@ -39,3 +39,26 @@
 /.sonar/
 /.scannerwork/
 /sonar-reports/
+
+### Vim ###
+# Swap
+[._]*.s[a-v][a-z]
+!*.svg  # comment out if you don't need vector files
+[._]*.sw[a-p]
+[._]s[a-rt-v][a-z]
+[._]ss[a-gi-z]
+[._]sw[a-p]
+
+# Session
+Session.vim
+Sessionx.vim
+
+# Temporary
+.netrwhist
+*~
+# Auto-generated tag files
+tags
+# Persistent undo
+[._]*.un~
+
+# End of https://www.toptal.com/developers/gitignore/api/vim

go.mod

@@ -27,6 +27,7 @@ require (
 	github.com/hofstadter-io/data-utils v0.0.0-20200128210141-0a3e569b27ed
 	github.com/hofstadter-io/dotpath v0.0.0-20191027071558-52e2819b7d2d
 	github.com/hofstadter-io/hof-lang v0.0.0-20180925221847-4d1c962b0729
+	github.com/kevinburke/ssh_config v1.1.0
 	github.com/kirsle/configdir v0.0.0-20170128060238-e45d2f54772f
 	github.com/kr/pretty v0.2.0
 	github.com/mattn/go-colorable v0.1.6 // indirect
@@ -45,12 +46,10 @@ require (
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/spf13/viper v1.7.0
 	github.com/stretchr/testify v1.6.1
+	github.com/xanzy/go-gitlab v0.50.1
 	go.uber.org/zap v1.15.0
-	golang.org/x/crypto v0.0.0-20200604202706-70a84ac30bf9 // indirect
 	golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449
-	golang.org/x/net v0.0.0-20200602114024-627f9648deb9 // indirect
 	golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
-	golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
 	golang.org/x/text v0.3.3
 	google.golang.org/appengine v1.6.6 // indirect
 	google.golang.org/protobuf v1.24.0 // indirect

lib/mod/cache/checksum.go

@@ -2,17 +2,12 @@ package cache
 
 import (
 	"os"
-	"strings"
 
 	"golang.org/x/mod/sumdb/dirhash"
 )
 
 func Checksum(lang, mod, ver string) (string, error) {
-
-	flds := strings.Split(mod, "/")
-	remote := flds[0]
-	owner := flds[1]
-	repo := flds[2]
+	remote, owner, repo := parseModURL(mod)
 	tag := ver
 
 	dir := Outdir(lang, remote, owner, repo, tag)
@@ -27,4 +22,3 @@ func Checksum(lang, mod, ver string) (string, error) {
 
 	return h, err
 }
-

lib/mod/cache/fetch.go

@@ -3,21 +3,19 @@ package cache
 import (
 	"fmt"
 	"os"
-	"strings"
 
-	googithub "github.com/google/go-github/v30/github"
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/memfs"
+	googithub "github.com/google/go-github/v30/github"
 
 	"github.com/hofstadter-io/hof/lib/yagu"
+	"github.com/hofstadter-io/hof/lib/yagu/repos/git"
 	"github.com/hofstadter-io/hof/lib/yagu/repos/github"
+	"github.com/hofstadter-io/hof/lib/yagu/repos/gitlab"
 )
 
-func Fetch(lang, mod, ver string) (err error) {
-	flds := strings.Split(mod, "/")
-	remote := flds[0]
-	owner := flds[1]
-	repo := flds[2]
+func Fetch(lang, mod, ver, pev string) (err error) {
+	remote, owner, repo := parseModURL(mod)
 	tag := ver
 
 	dir := Outdir(lang, remote, owner, repo, tag)
@@ -27,8 +25,11 @@ func Fetch(lang, mod, ver string) (err error) {
 		if _, ok := err.(*os.PathError); !ok && err.Error() != "file does not exist" && err.Error() != "no such file or directory" {
 			return err
 		}
-		// not found
-		fetch(lang, mod, ver)
+		// not found, try fetching deps
+		private := MatchPrefixPatterns(os.Getenv(pev), mod)
+		if err := fetch(lang, mod, ver, private); err != nil {
+			return err
+		}
 	}
 
 	// else we have it already
@@ -37,20 +38,55 @@ func Fetch(lang, mod, ver string) (err error) {
 	return nil
 }
 
-func fetch(lang, mod, ver string) error {
-	flds := strings.Split(mod, "/")
-	remote := flds[0]
-	owner := flds[1]
-	repo := flds[2]
+func fetch(lang, mod, ver string, private bool) error {
+	remote, owner, repo := parseModURL(mod)
 	tag := ver
 
-	switch remote {
-	case "github.com":
+	if private {
+		return fetchGit(lang, remote, owner, repo, tag, true)
+	} else if remote == "github.com" {
 		return fetchGitHub(lang, owner, repo, tag)
+	} else if remote == "gitlab.com" {
+		return fetchGitLab(lang, owner, repo, tag)
+	}
+	return fetchGit(lang, remote, owner, repo, tag, private)
+}
+
+func fetchGit(lang, remote, owner, repo, tag string, private bool) error {
+	FS := memfs.New()
+
+	if err := git.FetchGit(FS, remote, owner, repo, tag, private); err != nil {
+		return fmt.Errorf("While fetching from git\n%w\n", err)
+	}
+
+	if err := Write(lang, remote, owner, repo, tag, FS); err != nil {
+		return fmt.Errorf("While writing to cache\n%w\n", err)
+	}
+
+	return nil
+}
+
+func fetchGitLab(lang, owner, repo, tag string) (err error) {
+	FS := memfs.New()
+	client, err := gitlab.NewClient()
+	if err != nil {
+		return err
+	}
 
-	default:
-		return fmt.Errorf("Unknown remote: %q in %s", remote, mod)
+	zReader, err := gitlab.FetchZip(client, owner, repo, tag)
+	if err != nil {
+		return fmt.Errorf("While fetching from GitLab\n%w\n", err)
 	}
+
+	if err := yagu.BillyLoadFromZip(zReader, FS, true); err != nil {
+		return fmt.Errorf("While reading zipfile\n%w\n", err)
+	}
+
+	if err := Write(lang, "gitlab.com", owner, repo, tag, FS); err != nil {
+		return fmt.Errorf("While writing to cache\n%w\n", err)
+	}
+
+	return nil
 }
 
 func fetchGitHub(lang, owner, repo, tag string) (err error) {
@@ -65,19 +101,6 @@ func fetchGitHub(lang, owner, repo, tag string) (err error) {
 		return fmt.Errorf("While fetching from github\n%w\n", err)
 	}
 
-	/*
-	fmt.Println("filelist:")
-	files, err := yagu.BillyGetFilelist(FS)
-	if err != nil {
-		return fmt.Errorf("While getting filelist\n%w\n", err)
-	}
-
-	for _, f := range files {
-		fmt.Println(" -", f.Name())
-	}
-
-	fmt.Println("Writing...", )
-	*/
 	err = Write(lang, "github.com", owner, repo, tag, FS)
 	if err != nil {
 		return fmt.Errorf("While writing to cache\n%w\n", err)

lib/mod/cache/lookup.go

@@ -2,17 +2,13 @@ package cache
 
 import (
 	"os"
-	"strings"
 
 	"github.com/go-git/go-billy/v5"
 	"github.com/go-git/go-billy/v5/osfs"
 )
 
 func Load(lang, mod, ver string) (FS billy.Filesystem, err error) {
-	flds := strings.Split(mod, "/")
-	remote := flds[0]
-	owner := flds[1]
-	repo := flds[2]
+	remote, owner, repo := parseModURL(mod)
 	tag := ver
 
 	dir := Outdir(lang, remote, owner, repo, tag)

lib/mod/cache/util.go

@@ -0,0 +1,67 @@
+package cache
+
+import (
+	"path"
+	"strings"
+)
+
+func parseModURL(mod string) (remote, owner, repo string) {
+	var flds []string
+	if i := strings.Index(mod, ".git"); i > -1 {
+		flds = strings.SplitN(mod[:i], "/", 3)
+	} else {
+		flds = strings.Split(mod, "/")
+	}
+
+	if len(flds) < 3 {
+		return flds[0], "", flds[1]
+	}
+
+	return flds[0], flds[1], flds[2]
+}
+
+// MatchPrefixPatterns reports whether any path prefix of target matches one of
+// the glob patterns (as defined by path.Match) in the comma-separated globs
+// list. This implements the algorithm used when matching a module path to the
+// GOPRIVATE environment variable, as described by 'go help module-private'.
+//
+// It ignores any empty or malformed patterns in the list.
+func MatchPrefixPatterns(globs, target string) bool {
+	for globs != "" {
+		// Extract next non-empty glob in comma-separated list.
+		var glob string
+		if i := strings.Index(globs, ","); i >= 0 {
+			glob, globs = globs[:i], globs[i+1:]
+		} else {
+			glob, globs = globs, ""
+		}
+		if glob == "" {
+			continue
+		}
+
+		// A glob with N+1 path elements (N slashes) needs to be matched
+		// against the first N+1 path elements of target,
+		// which end just before the N+1'th slash.
+		n := strings.Count(glob, "/")
+		prefix := target
+		// Walk target, counting slashes, truncating at the N+1'th slash.
+		for i := 0; i < len(target); i++ {
+			if target[i] == '/' {
+				if n == 0 {
+					prefix = target[:i]
+					break
+				}
+				n--
+			}
+		}
+		if n > 0 {
+			// Not enough prefix elements.
+			continue
+		}
+		matched, _ := path.Match(glob, prefix)
+		if matched {
+			return true
+		}
+	}
+	return false
+}

lib/mod/cache/util_test.go

@@ -0,0 +1,30 @@
+package cache
+
+import (
+	"path"
+	"testing"
+)
+
+func TestSplitMod(t *testing.T) {
+	tests := map[string]struct {
+		mod      string
+		expected string
+	}{
+		"simple":             {mod: "github.com/owner/repo", expected: "github.com/owner/repo"},
+		"submodule":          {mod: "github.com/owner/repo/submodule", expected: "github.com/owner/repo"},
+		"complex":            {mod: "gitlab.com/owner/repo.git/submodule", expected: "gitlab.com/owner/repo"},
+		"subgroup":           {mod: "gitlab.com/owner/subgroup/repo.git", expected: "gitlab.com/owner/subgroup/repo"},
+		"subgroup+submodule": {mod: "gitlab.com/owner/subgroup/repo.git/submodule", expected: "gitlab.com/owner/subgroup/repo"},
+		"small":              {mod: "cuelang.org/go", expected: "cuelang.org/go"},
+	}
+
+	for name, tc := range tests {
+		t.Run(name, func(t *testing.T) {
+			rm, o, rp := parseModURL(tc.mod)
+			got := path.Join(rm, o, rp)
+			if got != tc.expected {
+				t.Fatalf("expected: %v, got: %s", tc.expected, got)
+			}
+		})
+	}
+}

lib/mod/langs/cue.go

@@ -2,12 +2,13 @@ package langs
 
 const CuelangModder = `
 langs: cue: {
-	Name:        "cue"
-	Version:     "v0.4.0"
-	ModFile:     "cue.mods"
-	SumFile:     "cue.sums"
-	ModsDir:     "cue.mod/pkg"
-	MappingFile: "cue.mod/modules.txt"
+	Name:          "cue"
+	Version:       "v0.4.0"
+	ModFile:       "cue.mods"
+	SumFile:       "cue.sums"
+	ModsDir:       "cue.mod/pkg"
+	MappingFile:   "cue.mod/modules.txt"
+	PrivateEnvVar: "CUEPRIVATE"
 	InitTemplates: {
 		"cue.mod/module.cue": """
 		module: "{{ .Module }}"

lib/mod/langs/go.go

@@ -8,6 +8,7 @@ langs: go: {
 	SumFile:       "go.sum",
 	ModsDir:       "vendor",
 	MappingFile:   "vendor/modules.txt",
+	PrivateEnvVar: "GOPRIVATE"
 	CommandInit:   [["go", "mod", "init"]],
 	CommandGraph:  [["go", "mod", "graph"]],
 	CommandTidy:   [["go", "mod", "tidy"]],

lib/mod/langs/spec.go

@@ -10,6 +10,7 @@ langs: #Spec
 	SumFile:  string,
 	ModsDir:  string,
 	MappingFile: string,
+	PrivateEnvVar?: string,
 
 	NoLoad?: bool,
 	CommandInit?: [...[...string]],

lib/mod/modder/modder.go

@@ -20,10 +20,11 @@ type Modder struct {
 	Version string `yaml:"Version",omitempty`
 
 	// Module information
-	ModFile     string `yaml:"ModFile",omitempty`
-	SumFile     string `yaml:"SumFile",omitempty`
-	ModsDir     string `yaml:"ModsDir",omitempty`
-	MappingFile string `yaml:"MappingFile",omitempty`
+	ModFile       string `yaml:"ModFile",omitempty`
+	SumFile       string `yaml:"SumFile",omitempty`
+	ModsDir       string `yaml:"ModsDir",omitempty`
+	MappingFile   string `yaml:"MappingFile",omitempty`
+	PrivateEnvVar string `yaml:"PrivateEnvVar",omitempty`
 
 	// Commands override default, configuragble processing
 	// for things like golang

lib/mod/modder/modder_vendor.go

@@ -123,15 +123,6 @@ func (mdr *Modder) LoadRemoteModule(R Replace) error {
 	// If sumfile, check integrity and possibly shortcut
 	if mdr.module.SumFile != nil {
 		mdr.CompareSumEntryToVendor(R)
-		/*
-		err := mdr.CompareSumEntryToVendor(R)
-		if err == nil {
-			// return nil
-			// TODO, add dependencies to processing
-		} else {
-			fmt.Println("Error:", err)
-		}
-		*/
 	}
 
 	// TODO, check if valid and just add m.deps to processing
@@ -149,7 +140,7 @@ func (mdr *Modder) LoadRemoteModule(R Replace) error {
 		m.Version = R.NewVersion
 	}
 
-	err := cache.Fetch(mdr.Name, R.NewPath, R.NewVersion)
+	err := cache.Fetch(mdr.Name, R.NewPath, R.NewVersion, mdr.PrivateEnvVar)
 	if err != nil {
 		return err
 	}