Skip to content

Commit

Permalink
refactor: move fetch git to yagu
Browse files Browse the repository at this point in the history
  • Loading branch information
b4nst committed Aug 2, 2021
1 parent 8851e71 commit c7d1d15
Show file tree
Hide file tree
Showing 5 changed files with 255 additions and 55 deletions.
59 changes: 4 additions & 55 deletions lib/mod/cache/fetch.go
Expand Up @@ -3,19 +3,13 @@ package cache
import (
"fmt"
"os"
"strings"

"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/transport"
"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/go-git/go-git/v5/storage/memory"
googithub "github.com/google/go-github/v30/github"
"github.com/kevinburke/ssh_config"

"github.com/hofstadter-io/hof/lib/yagu"
"github.com/hofstadter-io/hof/lib/yagu/repos/git"
"github.com/hofstadter-io/hof/lib/yagu/repos/github"
"github.com/hofstadter-io/hof/lib/yagu/repos/gitlab"
)
Expand Down Expand Up @@ -60,31 +54,9 @@ func fetch(lang, mod, ver string) error {
func fetchGit(lang, remote, owner, repo, tag string) error {
FS := memfs.New()

gco := &git.CloneOptions{
URL: fmt.Sprintf("https://%s/%s/%s", remote, owner, repo),
Depth: 1,
}
if tag != "v0.0.0" {
gco.ReferenceName = plumbing.NewTagReferenceName(tag)
gco.SingleBranch = true
}

if _, err := git.Clone(memory.NewStorage(), FS, gco); err != nil {
if err != transport.ErrAuthenticationRequired {
return err
}

// Needs auth
newRemote, auth, err := getSSHAuth(remote)
if err != nil {
return err
}
gco.URL = fmt.Sprintf("%s:%s/%s", newRemote, owner, repo)
gco.Auth = auth

if _, err := git.Clone(memory.NewStorage(), FS, gco); err != nil {
return err
}
// TODO retreive private config
if err := git.FetchGit(FS, remote, owner, repo, tag, false); err != nil {
return fmt.Errorf("While fetching from git\n%w\n", err)
}

if err := Write(lang, remote, owner, repo, tag, FS); err != nil {
Expand All @@ -94,29 +66,6 @@ func fetchGit(lang, remote, owner, repo, tag string) error {
return nil
}

func getSSHAuth(remote string) (string, *ssh.PublicKeys, error) {
pk, err := ssh_config.GetStrict(remote, "IdentityFile")
if err != nil {
return "", nil, err
}
if strings.HasPrefix(pk, "~") {
if hdir, err := os.UserHomeDir(); err == nil {
pk = strings.Replace(pk, "~", hdir, 1)
}
}
usr := ssh_config.Get(remote, "User")
if usr == "" {
usr = "git"
}

pks, err := ssh.NewPublicKeysFromFile(usr, pk, "")
if err != nil {
return "", nil, err
}

return fmt.Sprintf("%s@%s", usr, remote), pks, nil
}

func fetchGitLab(lang, owner, repo, tag string) (err error) {
FS := memfs.New()
client, err := gitlab.NewClient()
Expand Down
33 changes: 33 additions & 0 deletions lib/yagu/repos/git/fetch.go
Expand Up @@ -5,6 +5,7 @@ import (
"os"
"strings"

"github.com/go-git/go-billy/v5"
"github.com/go-git/go-billy/v5/memfs"
"github.com/go-git/go-billy/v5/osfs"
gogit "github.com/go-git/go-git/v5"
Expand Down Expand Up @@ -88,3 +89,35 @@ func CloneRepoRef(srcUrl string, ref *plumbing.Reference) (*GitRepo, error) {
Repo: r,
}, nil
}

// FetchGit clone the repository inside FS.
// If private flag is set, it will look for netrc credentials, fallbacking to SSH
func FetchGit(FS billy.Filesystem, remote, owner, repo, tag string, private bool) error {
gco := &gogit.CloneOptions{
URL: fmt.Sprintf("https://%s/%s/%s", remote, owner, repo),
Depth: 1,
}

if tag != "v0.0.0" {
gco.ReferenceName = plumbing.NewTagReferenceName(tag)
gco.SingleBranch = true
}

if private {
if netrc, err := NetrcCredentials(remote); err != nil {
gco.Auth = &http.BasicAuth{
Username: netrc.Login,
Password: netrc.Password,
}
} else if ssh, err := SSHCredentials(remote); err != nil {
gco.Auth = ssh.Keys
gco.URL = fmt.Sprintf("%s@%s:%s/%s", ssh.User, remote, owner, repo)
} else {
return err
}
}

_, err := gogit.Clone(memory.NewStorage(), FS, gco)

return err
}
127 changes: 127 additions & 0 deletions lib/yagu/repos/git/netrc.go
@@ -0,0 +1,127 @@
package git

import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"sync"
)

type netrcLine struct {
machine string
login string
password string
}

type NetrcMachine struct {
Login string
Password string
}

var (
netrcOnce sync.Once
netrc map[string]NetrcMachine
netrcErr error
)

func NetrcMachines() map[string]NetrcMachine {
netrcOnce.Do(readNetrc)

return netrc
}

func NetrcCredentials(machine string) (NetrcMachine, error) {
if cred, ok := netrc[machine]; ok {
return cred, nil
}

return NetrcMachine{}, fmt.Errorf("No credentials found for %s", machine)
}

func parseNetrc(data string) map[string]NetrcMachine {
// See https://www.gnu.org/software/inetutils/manual/html_node/The-_002enetrc-file.html
// for documentation on the .netrc format.
nrc := make(map[string]NetrcMachine)
var l netrcLine
inMacro := false
for _, line := range strings.Split(data, "\n") {
if inMacro {
if line == "" {
inMacro = false
}
continue
}

f := strings.Fields(line)
i := 0
for ; i < len(f)-1; i += 2 {
// Reset at each "machine" token.
// “The auto-login process searches the .netrc file for a machine token
// that matches […]. Once a match is made, the subsequent .netrc tokens
// are processed, stopping when the end of file is reached or another
// machine or a default token is encountered.”
switch f[i] {
case "machine":
l = netrcLine{machine: f[i+1]}
case "default":
break
case "login":
l.login = f[i+1]
case "password":
l.password = f[i+1]
case "macdef":
// “A macro is defined with the specified name; its contents begin with
// the next .netrc line and continue until a null line (consecutive
// new-line characters) is encountered.”
inMacro = true
}

if l.machine != "" && l.login != "" && l.password != "" {
nrc[l.machine] = NetrcMachine{l.login, l.password}
l = netrcLine{}
}
}

if i < len(f) && f[i] == "default" {
// “There can be only one default token, and it must be after all machine tokens.”
break
}
}

return nrc
}

func netrcPath() (string, error) {
if env := os.Getenv("NETRC"); env != "" {
return env, nil
}
dir, err := os.UserHomeDir()
if err != nil {
return "", err
}
base := ".netrc"
if runtime.GOOS == "windows" {
base = "_netrc"
}
return filepath.Join(dir, base), nil
}

func readNetrc() {
path, err := netrcPath()
if err != nil {
netrcErr = err
return
}

data, err := os.ReadFile(path)
if err != nil {
if !os.IsNotExist(err) {
netrcErr = err
}
return
}

netrc = parseNetrc(string(data))
}
54 changes: 54 additions & 0 deletions lib/yagu/repos/git/netrc_test.go
@@ -0,0 +1,54 @@
package git

import (
"reflect"
"testing"
)

var testNetrc = `
machine incomplete
password none
machine api.github.com
login user
password pwd
machine incomlete.host
login justlogin
machine test.host
login user2
password pwd2
machine oneline login user3 password pwd3
machine ignore.host macdef ignore
login nobody
password nothing
machine hasmacro.too macdef ignore-next-lines login user4 password pwd4
login nobody
password nothing
default
login anonymous
password gopher@golang.org
machine after.default
login oops
password too-late-in-file
`

func TestParseNetrc(t *testing.T) {
lines := parseNetrc(testNetrc)
want := map[string]NetrcMachine{
"api.github.com": {"user", "pwd"},
"test.host": {"user2", "pwd2"},
"oneline": {"user3", "pwd3"},
"hasmacro.too": {"user4", "pwd4"},
}

if !reflect.DeepEqual(lines, want) {
t.Errorf("parseNetrc:\nhave %q\nwant %q", lines, want)
}
}
37 changes: 37 additions & 0 deletions lib/yagu/repos/git/ssh.go
@@ -0,0 +1,37 @@
package git

import (
"os"
"strings"

"github.com/go-git/go-git/v5/plumbing/transport/ssh"
"github.com/kevinburke/ssh_config"
)

type SSHMachine struct {
User string
Keys *ssh.PublicKeys
}

func SSHCredentials(machine string) (SSHMachine, error) {
pk, err := ssh_config.GetStrict(machine, "IdentityFile")
if err != nil {
return SSHMachine{}, err
}
if strings.HasPrefix(pk, "~") {
if hdir, err := os.UserHomeDir(); err == nil {
pk = strings.Replace(pk, "~", hdir, 1)
}
}
usr := ssh_config.Get(machine, "User")
if usr == "" {
usr = "git"
}

pks, err := ssh.NewPublicKeysFromFile(usr, pk, "")
if err != nil {
return SSHMachine{}, err
}

return SSHMachine{usr, pks}, nil
}

0 comments on commit c7d1d15

Please sign in to comment.