x-motemen · Songmu · Apr 26, 2019 · Apr 22, 2019 · Apr 22, 2019 · Apr 22, 2019
diff --git a/.travis.yml b/.travis.yml
@@ -1,8 +1,7 @@
 language: go
 
 go:
-  - 1.8
-  - 1.9
+  - 1.x
   - tip
 
 script: make test VERBOSE=1
diff --git a/commands.go b/commands.go
@@ -11,8 +11,8 @@ import (
 	"strings"
 	"syscall"
 
-	"github.com/urfave/cli"
 	"github.com/motemen/ghq/utils"
+	"github.com/urfave/cli"
 )
 
 var Commands = []cli.Command{
@@ -202,13 +202,13 @@ func getRemoteRepository(remote RemoteRepository, doUpdate bool, isShallow bool)
 	if newPath {
 		utils.Log("clone", fmt.Sprintf("%s -> %s", remoteURL, path))
 
-		vcs := remote.VCS()
+		vcs, repoURL := remote.VCS()
 		if vcs == nil {
 			utils.Log("error", fmt.Sprintf("Could not find version control system: %s", remoteURL))
 			os.Exit(1)
 		}
 
-		err := vcs.Clone(remoteURL, path, isShallow)
+		err := vcs.Clone(repoURL, path, isShallow)
 		if err != nil {
 			utils.Log("error", err.Error())
 			os.Exit(1)

diff --git a/go_import.go b/go_import.go
@@ -0,0 +1,85 @@
+package main
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+func detectGoImport(u *url.URL) (string, *url.URL, error) {
+	goGetU, _ := url.Parse(u.String()) // clone
+	q := goGetU.Query()
+	q.Add("go-get", "1")
+	goGetU.RawQuery = q.Encode()
+
+	cli := &http.Client{
+		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+			// never follow redirection
+			return http.ErrUseLastResponse
+		},
+	}
+	req, _ := http.NewRequest(http.MethodGet, goGetU.String(), nil)
+	req.Header.Add("User-Agent", fmt.Sprintf("ghq/%s (+https://github.com/motemen/ghq)", Version))
+	resp, err := cli.Do(req)
+	if err != nil {
+		return "", nil, err
+	}
+	defer resp.Body.Close()
+
+	return detectVCSAndRepoURL(resp.Body)
+}
+
+// find meta tag like following from thml
+// <meta name="go-import" content="gopkg.in/yaml.v2 git https://gopkg.in/yaml.v2">
+// ref. https://golang.org/cmd/go/#hdr-Remote_import_paths
+func detectVCSAndRepoURL(r io.Reader) (string, *url.URL, error) {
+	doc, err := html.Parse(r)
+	if err != nil {
+		return "", nil, err
+	}
+
+	var goImportContent string
+
+	var f func(*html.Node)
+	f = func(n *html.Node) {
+		if goImportContent != "" {
+			return
+		}
+		if n.Type == html.ElementNode && n.Data == "meta" {
+			var (
+				goImportMeta = false
+				content      = ""
+			)
+			for _, a := range n.Attr {
+				if a.Key == "name" && a.Val == "go-import" {
+					goImportMeta = true
+					continue
+				}
+				if a.Key == "content" {
+					content = a.Val
+				}
+			}
+			if goImportMeta && content != "" {
+				goImportContent = content
+			}
+		}
+		for c := n.FirstChild; c != nil; c = c.NextSibling {
+			f(c)
+		}
+	}
+	f(doc)
+
+	stuffs := strings.Fields(goImportContent)
+	if len(stuffs) < 3 {
+		return "", nil, fmt.Errorf("no go-import meta tags detected")
+	}
+	u, err := url.Parse(stuffs[2])
+	if err != nil {
+		return "", nil, err
+	}
+	return stuffs[1], u, nil
+}
diff --git a/go_import_test.go b/go_import_test.go
@@ -0,0 +1,30 @@
+package main
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestDetectVCSAndRepoURL(t *testing.T) {
+	input := `<html>
+<head>
+<meta name="go-import" content="gopkg.in/yaml.v2 git https://gopkg.in/yaml.v2">
+<meta name="go-source" content="gopkg.in/yaml.v2 _ https://github.com/go-yaml/yaml/tree/v2.2.2{/dir} https://github.com/go-yaml/yaml/blob/v2.2.2{/dir}/{file}#L{line}">
+</head>
+<body>
+go get gopkg.in/yaml.v2
+</body>
+</html>`
+
+	vcs, u, err := detectVCSAndRepoURL(strings.NewReader(input))
+	if vcs != "git" {
+		t.Errorf("want: %q, got: %q", "git", vcs)
+	}
+	expectedURL := "https://gopkg.in/yaml.v2"
+	if u.String() != expectedURL {
+		t.Errorf("want: %q, got: %q", expectedURL, u.String())
+	}
+	if err != nil {
+		t.Errorf("something went wrong: %s", err)
+	}
+}
diff --git a/remote_repository.go b/remote_repository.go
@@ -16,7 +16,7 @@ type RemoteRepository interface {
 	// Checks if the URL is valid.
 	IsValid() bool
 	// The VCS backend that hosts the repository.
-	VCS() *VCSBackend
+	VCS() (*VCSBackend, *url.URL)
 }
 
 // A GitHubRepository represents a GitHub repository. Impliments RemoteRepository.
@@ -42,8 +42,8 @@ func (repo *GitHubRepository) IsValid() bool {
 	return true
 }
 
-func (repo *GitHubRepository) VCS() *VCSBackend {
-	return GitBackend
+func (repo *GitHubRepository) VCS() (*VCSBackend, *url.URL) {
+	return GitBackend, repo.URL()
 }
 
 // A GitHubGistRepository represents a GitHub Gist repository.
@@ -59,8 +59,8 @@ func (repo *GitHubGistRepository) IsValid() bool {
 	return true
 }
 
-func (repo *GitHubGistRepository) VCS() *VCSBackend {
-	return GitBackend
+func (repo *GitHubGistRepository) VCS() (*VCSBackend, *url.URL) {
+	return GitBackend, repo.URL()
 }
 
 type GoogleCodeRepository struct {
@@ -77,13 +77,13 @@ func (repo *GoogleCodeRepository) IsValid() bool {
 	return validGoogleCodePathPattern.MatchString(repo.url.Path)
 }
 
-func (repo *GoogleCodeRepository) VCS() *VCSBackend {
+func (repo *GoogleCodeRepository) VCS() (*VCSBackend, *url.URL) {
 	if utils.RunSilently("hg", "identify", repo.url.String()) == nil {
-		return MercurialBackend
+		return MercurialBackend, repo.URL()
 	} else if utils.RunSilently("git", "ls-remote", repo.url.String()) == nil {
-		return GitBackend
+		return GitBackend, repo.URL()
 	} else {
-		return nil
+		return nil, nil
 	}
 }
 
@@ -99,8 +99,8 @@ func (repo *DarksHubRepository) IsValid() bool {
 	return strings.Count(repo.url.Path, "/") == 2
 }
 
-func (repo *DarksHubRepository) VCS() *VCSBackend {
-	return DarcsBackend
+func (repo *DarksHubRepository) VCS() (*VCSBackend, *url.URL) {
+	return DarcsBackend, repo.URL()
 }
 
 type BluemixRepository struct {
@@ -117,8 +117,8 @@ func (repo *BluemixRepository) IsValid() bool {
 	return validBluemixPathPattern.MatchString(repo.url.Path)
 }
 
-func (repo *BluemixRepository) VCS() *VCSBackend {
-	return GitBackend
+func (repo *BluemixRepository) VCS() (*VCSBackend, *url.URL) {
+	return GitBackend, repo.URL()
 }
 
 type OtherRepository struct {
@@ -133,7 +133,7 @@ func (repo *OtherRepository) IsValid() bool {
 	return true
 }
 
-func (repo *OtherRepository) VCS() *VCSBackend {
+func (repo *OtherRepository) VCS() (*VCSBackend, *url.URL) {
 	if GitHasFeatureConfigURLMatch() {
 		// Respect 'ghq.url.https://ghe.example.com/.vcs' config variable
 		// (in gitconfig:)
@@ -145,38 +145,48 @@ func (repo *OtherRepository) VCS() *VCSBackend {
 		}
 
 		if vcs == "git" || vcs == "github" {
-			return GitBackend
+			return GitBackend, repo.URL()
 		}
 
 		if vcs == "svn" || vcs == "subversion" {
-			return SubversionBackend
+			return SubversionBackend, repo.URL()
 		}
 
 		if vcs == "git-svn" {
-			return GitsvnBackend
+			return GitsvnBackend, repo.URL()
 		}
 
 		if vcs == "hg" || vcs == "mercurial" {
-			return MercurialBackend
+			return MercurialBackend, repo.URL()
 		}
 
 		if vcs == "darcs" {
-			return DarcsBackend
+			return DarcsBackend, repo.URL()
 		}
 	} else {
 		utils.Log("warning", "This version of Git does not support `config --get-urlmatch`; per-URL settings are not available")
 	}
 
 	// Detect VCS backend automatically
 	if utils.RunSilently("git", "ls-remote", repo.url.String()) == nil {
-		return GitBackend
-	} else if utils.RunSilently("hg", "identify", repo.url.String()) == nil {
-		return MercurialBackend
-	} else if utils.RunSilently("svn", "info", repo.url.String()) == nil {
-		return SubversionBackend
-	} else {
-		return nil
+		return GitBackend, repo.URL()
+	}
+
+	vcs, repoURL, err := detectGoImport(repo.url)
+	if err == nil {
+		// vcs == "mod" (modproxy) not supported yet
+		return vcsBackendMap[vcs], repoURL
 	}
+
+	if utils.RunSilently("hg", "identify", repo.url.String()) == nil {
+		return MercurialBackend, repo.URL()
+	}
+
+	if utils.RunSilently("svn", "info", repo.url.String()) == nil {
+		return SubversionBackend, repo.URL()
+	}
+
+	return nil, nil
 }
 
 func NewRemoteRepository(url *url.URL) (RemoteRepository, error) {

diff --git a/remote_repository_test.go b/remote_repository_test.go
@@ -28,12 +28,14 @@ func TestNewRemoteRepositoryGitHub(t *testing.T) {
 	repo, err = NewRemoteRepository(parseURL("https://github.com/motemen/pusheen-explorer"))
 	Expect(err).To(BeNil())
 	Expect(repo.IsValid()).To(Equal(true))
-	Expect(repo.VCS()).To(Equal(GitBackend))
+	vcs, _ := repo.VCS()
+	Expect(vcs).To(Equal(GitBackend))
 
 	repo, err = NewRemoteRepository(parseURL("https://github.com/motemen/pusheen-explorer/"))
 	Expect(err).To(BeNil())
 	Expect(repo.IsValid()).To(Equal(true))
-	Expect(repo.VCS()).To(Equal(GitBackend))
+	vcs, _ = repo.VCS()
+	Expect(vcs).To(Equal(GitBackend))
 
 	repo, err = NewRemoteRepository(parseURL("https://github.com/motemen/pusheen-explorer/blob/master/README.md"))
 	Expect(err).To(BeNil())
@@ -55,7 +57,8 @@ func TestNewRemoteRepositoryGitHubGist(t *testing.T) {
 	repo, err = NewRemoteRepository(parseURL("https://gist.github.com/motemen/9733745"))
 	Expect(err).To(BeNil())
 	Expect(repo.IsValid()).To(Equal(true))
-	Expect(repo.VCS()).To(Equal(GitBackend))
+	vcs, _ := repo.VCS()
+	Expect(vcs).To(Equal(GitBackend))
 }
 
 func TestNewRemoteRepositoryGoogleCode(t *testing.T) {
@@ -73,7 +76,8 @@ func TestNewRemoteRepositoryGoogleCode(t *testing.T) {
 		"hg identify":   nil,
 		"git ls-remote": errors.New(""),
 	})
-	Expect(repo.VCS()).To(Equal(MercurialBackend))
+	vcs, _ := repo.VCS()
+	Expect(vcs).To(Equal(MercurialBackend))
 
 	repo, err = NewRemoteRepository(parseURL("https://code.google.com/p/git-core"))
 	Expect(err).To(BeNil())
@@ -82,7 +86,8 @@ func TestNewRemoteRepositoryGoogleCode(t *testing.T) {
 		"hg identify":   errors.New(""),
 		"git ls-remote": nil,
 	})
-	Expect(repo.VCS()).To(Equal(GitBackend))
+	vcs, _ = repo.VCS()
+	Expect(vcs).To(Equal(GitBackend))
 }
 
 func TestNewRemoteRepositoryDarcsHub(t *testing.T) {
@@ -91,5 +96,6 @@ func TestNewRemoteRepositoryDarcsHub(t *testing.T) {
 	repo, err := NewRemoteRepository(parseURL("http://hub.darcs.net/foo/bar"))
 	Expect(err).To(BeNil())
 	Expect(repo.IsValid()).To(Equal(true))
-	Expect(repo.VCS()).To(Equal(DarcsBackend))
+	vcs, _ := repo.VCS()
+	Expect(vcs).To(Equal(DarcsBackend))
 }
diff --git a/url.go b/url.go
@@ -12,8 +12,11 @@ import (
 // Convert SCP-like URL to SSH URL(e.g. [user@]host.xz:path/to/repo.git/)
 // ref. http://git-scm.com/docs/git-fetch#_git_urls
 // (golang hasn't supported Perl-like negative look-behind match)
-var hasSchemePattern = regexp.MustCompile("^[^:]+://")
-var scpLikeUrlPattern = regexp.MustCompile("^([^@]+@)?([^:]+):/?(.+)$")
+var (
+	hasSchemePattern          = regexp.MustCompile("^[^:]+://")
+	scpLikeUrlPattern         = regexp.MustCompile("^([^@]+@)?([^:]+):/?(.+)$")
+	looksLikeAuthorityPattern = regexp.MustCompile(`[A-Za-z0-9]\.[A-Za-z]+(?::\d{1,5})?`)
+)
 
 func NewURL(ref string) (*url.URL, error) {
 	if !hasSchemePattern.MatchString(ref) && scpLikeUrlPattern.MatchString(ref) {
@@ -36,6 +39,12 @@ func NewURL(ref string) (*url.URL, error) {
 			if err != nil {
 				return url, err
 			}
+		} else if url.Host == "" {
+			// If ref is like "github.com/motemen/ghq" consider it as "https://github.com/motemen/ghq"
+			paths := strings.Split(ref, "/")
+			if looksLikeAuthorityPattern.MatchString(paths[0]) {
+				return url.Parse("https://" + ref)
+			}
 		}
 		url.Scheme = "https"
 		url.Host = "github.com"