vcs.go

// Copyright 2018 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package codehost

import (
	"encoding/xml"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"sync"
	"time"

	"cmd/go/internal/lockedfile"
	"cmd/go/internal/par"
	"cmd/go/internal/str"
)

// A VCSError indicates an error using a version control system.
// The implication of a VCSError is that we know definitively where
// to get the code, but we can't access it due to the error.
// The caller should report this error instead of continuing to probe
// other possible module paths.
//
// TODO(bcmills): See if we can invert this. (Return a distinguished error for
// “repo not found” and treat everything else as terminal.)
type VCSError struct {
	Err error
}

func (e *VCSError) Error() string { return e.Err.Error() }

func vcsErrorf(format string, a ...interface{}) error {
	return &VCSError{Err: fmt.Errorf(format, a...)}
}

func NewRepo(vcs, remote string) (Repo, error) {
	type key struct {
		vcs    string
		remote string
	}
	type cached struct {
		repo Repo
		err  error
	}
	c := vcsRepoCache.Do(key{vcs, remote}, func() interface{} {
		repo, err := newVCSRepo(vcs, remote)
		if err != nil {
			err = &VCSError{err}
		}
		return cached{repo, err}
	}).(cached)

	return c.repo, c.err
}

var vcsRepoCache par.Cache

type vcsRepo struct {
	mu lockedfile.Mutex // protects all commands, so we don't have to decide which are safe on a per-VCS basis

	remote string
	cmd    *vcsCmd
	dir    string

	tagsOnce sync.Once
	tags     map[string]bool

	branchesOnce sync.Once
	branches     map[string]bool

	fetchOnce sync.Once
	fetchErr  error
}

func newVCSRepo(vcs, remote string) (Repo, error) {
	if vcs == "git" {
		return newGitRepo(remote, false)
	}
	cmd := vcsCmds[vcs]
	if cmd == nil {
		return nil, fmt.Errorf("unknown vcs: %s %s", vcs, remote)
	}
	if !strings.Contains(remote, "://") {
		return nil, fmt.Errorf("invalid vcs remote: %s %s", vcs, remote)
	}

	r := &vcsRepo{remote: remote, cmd: cmd}
	var err error
	r.dir, r.mu.Path, err = WorkDir(vcsWorkDirType+vcs, r.remote)
	if err != nil {
		return nil, err
	}

	if cmd.init == nil {
		return r, nil
	}

	unlock, err := r.mu.Lock()
	if err != nil {
		return nil, err
	}
	defer unlock()

	if _, err := os.Stat(filepath.Join(r.dir, "."+vcs)); err != nil {
		if _, err := Run(r.dir, cmd.init(r.remote)); err != nil {
			os.RemoveAll(r.dir)
			return nil, err
		}
	}
	return r, nil
}

const vcsWorkDirType = "vcs1."

type vcsCmd struct {
	vcs           string                                            // vcs name "hg"
	init          func(remote string) []string                      // cmd to init repo to track remote
	tags          func(remote string) []string                      // cmd to list local tags
	tagRE         *regexp.Regexp                                    // regexp to extract tag names from output of tags cmd
	branches      func(remote string) []string                      // cmd to list local branches
	branchRE      *regexp.Regexp                                    // regexp to extract branch names from output of tags cmd
	badLocalRevRE *regexp.Regexp                                    // regexp of names that must not be served out of local cache without doing fetch first
	statLocal     func(rev, remote string) []string                 // cmd to stat local rev
	parseStat     func(rev, out string) (*RevInfo, error)           // cmd to parse output of statLocal
	fetch         []string                                          // cmd to fetch everything from remote
	latest        string                                            // name of latest commit on remote (tip, HEAD, etc)
	readFile      func(rev, file, remote string) []string           // cmd to read rev's file
	readZip       func(rev, subdir, remote, target string) []string // cmd to read rev's subdir as zip file
}

var re = regexp.MustCompile

var vcsCmds = map[string]*vcsCmd{
	"hg": {
		vcs: "hg",
		init: func(remote string) []string {
			return []string{"hg", "clone", "-U", remote, "."}
		},
		tags: func(remote string) []string {
			return []string{"hg", "tags", "-q"}
		},
		tagRE: re(`(?m)^[^\n]+$`),
		branches: func(remote string) []string {
			return []string{"hg", "branches", "-c", "-q"}
		},
		branchRE:      re(`(?m)^[^\n]+$`),
		badLocalRevRE: re(`(?m)^(tip)$`),
		statLocal: func(rev, remote string) []string {
			return []string{"hg", "log", "-l1", "-r", rev, "--template", "{node} {date|hgdate} {tags}"}
		},
		parseStat: hgParseStat,
		fetch:     []string{"hg", "pull", "-f"},
		latest:    "tip",
		readFile: func(rev, file, remote string) []string {
			return []string{"hg", "cat", "-r", rev, file}
		},
		readZip: func(rev, subdir, remote, target string) []string {
			pattern := []string{}
			if subdir != "" {
				pattern = []string{"-I", subdir + "/**"}
			}
			return str.StringList("hg", "archive", "-t", "zip", "--no-decode", "-r", rev, "--prefix=prefix/", pattern, target)
		},
	},

	"svn": {
		vcs:  "svn",
		init: nil, // no local checkout
		tags: func(remote string) []string {
			return []string{"svn", "list", strings.TrimSuffix(remote, "/trunk") + "/tags"}
		},
		tagRE: re(`(?m)^(.*?)/?$`),
		statLocal: func(rev, remote string) []string {
			suffix := "@" + rev
			if rev == "latest" {
				suffix = ""
			}
			return []string{"svn", "log", "-l1", "--xml", remote + suffix}
		},
		parseStat: svnParseStat,
		latest:    "latest",
		readFile: func(rev, file, remote string) []string {
			return []string{"svn", "cat", remote + "/" + file + "@" + rev}
		},
		// TODO: zip
	},

	"bzr": {
		vcs: "bzr",
		init: func(remote string) []string {
			return []string{"bzr", "branch", "--use-existing-dir", remote, "."}
		},
		fetch: []string{
			"bzr", "pull", "--overwrite-tags",
		},
		tags: func(remote string) []string {
			return []string{"bzr", "tags"}
		},
		tagRE:         re(`(?m)^\S+`),
		badLocalRevRE: re(`^revno:-`),
		statLocal: func(rev, remote string) []string {
			return []string{"bzr", "log", "-l1", "--long", "--show-ids", "-r", rev}
		},
		parseStat: bzrParseStat,
		latest:    "revno:-1",
		readFile: func(rev, file, remote string) []string {
			return []string{"bzr", "cat", "-r", rev, file}
		},
		readZip: func(rev, subdir, remote, target string) []string {
			extra := []string{}
			if subdir != "" {
				extra = []string{"./" + subdir}
			}
			return str.StringList("bzr", "export", "--format=zip", "-r", rev, "--root=prefix/", target, extra)
		},
	},

	"fossil": {
		vcs: "fossil",
		init: func(remote string) []string {
			return []string{"fossil", "clone", remote, ".fossil"}
		},
		fetch: []string{"fossil", "pull", "-R", ".fossil"},
		tags: func(remote string) []string {
			return []string{"fossil", "tag", "-R", ".fossil", "list"}
		},
		tagRE: re(`XXXTODO`),
		statLocal: func(rev, remote string) []string {
			return []string{"fossil", "info", "-R", ".fossil", rev}
		},
		parseStat: fossilParseStat,
		latest:    "trunk",
		readFile: func(rev, file, remote string) []string {
			return []string{"fossil", "cat", "-R", ".fossil", "-r", rev, file}
		},
		readZip: func(rev, subdir, remote, target string) []string {
			extra := []string{}
			if subdir != "" && !strings.ContainsAny(subdir, "*?[],") {
				extra = []string{"--include", subdir}
			}
			// Note that vcsRepo.ReadZip below rewrites this command
			// to run in a different directory, to work around a fossil bug.
			return str.StringList("fossil", "zip", "-R", ".fossil", "--name", "prefix", extra, rev, target)
		},
	},
}

func (r *vcsRepo) loadTags() {
	out, err := Run(r.dir, r.cmd.tags(r.remote))
	if err != nil {
		return
	}

	// Run tag-listing command and extract tags.
	r.tags = make(map[string]bool)
	for _, tag := range r.cmd.tagRE.FindAllString(string(out), -1) {
		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(tag) {
			continue
		}
		r.tags[tag] = true
	}
}

func (r *vcsRepo) loadBranches() {
	if r.cmd.branches == nil {
		return
	}

	out, err := Run(r.dir, r.cmd.branches(r.remote))
	if err != nil {
		return
	}

	r.branches = make(map[string]bool)
	for _, branch := range r.cmd.branchRE.FindAllString(string(out), -1) {
		if r.cmd.badLocalRevRE != nil && r.cmd.badLocalRevRE.MatchString(branch) {
			continue
		}
		r.branches[branch] = true
	}
}

func (r *vcsRepo) Tags(prefix string) ([]string, error) {
	unlock, err := r.mu.Lock()
	if err != nil {
		return nil, err
	}
	defer unlock()

	r.tagsOnce.Do(r.loadTags)

	tags := []string{}
	for tag := range r.tags {
		if strings.HasPrefix(tag, prefix) {
			tags = append(tags, tag)
		}
	}
	sort.Strings(tags)
	return tags, nil
}

func (r *vcsRepo) Stat(rev string) (*RevInfo, error) {
	unlock, err := r.mu.Lock()
	if err != nil {
		return nil, err
	}
	defer unlock()

	if rev == "latest" {
		rev = r.cmd.latest
	}
	r.branchesOnce.Do(r.loadBranches)
	revOK := (r.cmd.badLocalRevRE == nil || !r.cmd.badLocalRevRE.MatchString(rev)) && !r.branches[rev]
	if revOK {
		if info, err := r.statLocal(rev); err == nil {
			return info, nil
		}
	}

	r.fetchOnce.Do(r.fetch)
	if r.fetchErr != nil {
		return nil, r.fetchErr
	}
	info, err := r.statLocal(rev)
	if err != nil {
		return nil, err
	}
	if !revOK {
		info.Version = info.Name
	}
	return info, nil
}

func (r *vcsRepo) fetch() {
	_, r.fetchErr = Run(r.dir, r.cmd.fetch)
}

func (r *vcsRepo) statLocal(rev string) (*RevInfo, error) {
	out, err := Run(r.dir, r.cmd.statLocal(rev, r.remote))
	if err != nil {
		return nil, vcsErrorf("unknown revision %s", rev)
	}
	return r.cmd.parseStat(rev, string(out))
}

func (r *vcsRepo) Latest() (*RevInfo, error) {
	return r.Stat("latest")
}

func (r *vcsRepo) ReadFile(rev, file string, maxSize int64) ([]byte, error) {
	if rev == "latest" {
		rev = r.cmd.latest
	}
	_, err := r.Stat(rev) // download rev into local repo
	if err != nil {
		return nil, err
	}

	// r.Stat acquires r.mu, so lock after that.
	unlock, err := r.mu.Lock()
	if err != nil {
		return nil, err
	}
	defer unlock()

	out, err := Run(r.dir, r.cmd.readFile(rev, file, r.remote))
	if err != nil {
		return nil, os.ErrNotExist
	}
	return out, nil
}

func (r *vcsRepo) ReadFileRevs(revs []string, file string, maxSize int64) (map[string]*FileRev, error) {
	// We don't technically need to lock here since we're returning an error
	// uncondititonally, but doing so anyway will help to avoid baking in
	// lock-inversion bugs.
	unlock, err := r.mu.Lock()
	if err != nil {
		return nil, err
	}
	defer unlock()

	return nil, vcsErrorf("ReadFileRevs not implemented")
}

func (r *vcsRepo) RecentTag(rev, prefix string) (tag string, err error) {
	// We don't technically need to lock here since we're returning an error
	// uncondititonally, but doing so anyway will help to avoid baking in
	// lock-inversion bugs.
	unlock, err := r.mu.Lock()
	if err != nil {
		return "", err
	}
	defer unlock()

	return "", vcsErrorf("RecentTag not implemented")
}

func (r *vcsRepo) ReadZip(rev, subdir string, maxSize int64) (zip io.ReadCloser, actualSubdir string, err error) {
	if r.cmd.readZip == nil {
		return nil, "", vcsErrorf("ReadZip not implemented for %s", r.cmd.vcs)
	}

	unlock, err := r.mu.Lock()
	if err != nil {
		return nil, "", err
	}
	defer unlock()

	if rev == "latest" {
		rev = r.cmd.latest
	}
	f, err := ioutil.TempFile("", "go-readzip-*.zip")
	if err != nil {
		return nil, "", err
	}
	if r.cmd.vcs == "fossil" {
		// If you run
		//	fossil zip -R .fossil --name prefix trunk /tmp/x.zip
		// fossil fails with "unable to create directory /tmp" [sic].
		// Change the command to run in /tmp instead,
		// replacing the -R argument with an absolute path.
		args := r.cmd.readZip(rev, subdir, r.remote, filepath.Base(f.Name()))
		for i := range args {
			if args[i] == ".fossil" {
				args[i] = filepath.Join(r.dir, ".fossil")
			}
		}
		_, err = Run(filepath.Dir(f.Name()), args)
	} else {
		_, err = Run(r.dir, r.cmd.readZip(rev, subdir, r.remote, f.Name()))
	}
	if err != nil {
		f.Close()
		os.Remove(f.Name())
		return nil, "", err
	}
	return &deleteCloser{f}, "", nil
}

// deleteCloser is a file that gets deleted on Close.
type deleteCloser struct {
	*os.File
}

func (d *deleteCloser) Close() error {
	defer os.Remove(d.File.Name())
	return d.File.Close()
}

func hgParseStat(rev, out string) (*RevInfo, error) {
	f := strings.Fields(string(out))
	if len(f) < 3 {
		return nil, vcsErrorf("unexpected response from hg log: %q", out)
	}
	hash := f[0]
	version := rev
	if strings.HasPrefix(hash, version) {
		version = hash // extend to full hash
	}
	t, err := strconv.ParseInt(f[1], 10, 64)
	if err != nil {
		return nil, vcsErrorf("invalid time from hg log: %q", out)
	}

	var tags []string
	for _, tag := range f[3:] {
		if tag != "tip" {
			tags = append(tags, tag)
		}
	}
	sort.Strings(tags)

	info := &RevInfo{
		Name:    hash,
		Short:   ShortenSHA1(hash),
		Time:    time.Unix(t, 0).UTC(),
		Version: version,
		Tags:    tags,
	}
	return info, nil
}

func svnParseStat(rev, out string) (*RevInfo, error) {
	var log struct {
		Logentry struct {
			Revision int64  `xml:"revision,attr"`
			Date     string `xml:"date"`
		} `xml:"logentry"`
	}
	if err := xml.Unmarshal([]byte(out), &log); err != nil {
		return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
	}

	t, err := time.Parse(time.RFC3339, log.Logentry.Date)
	if err != nil {
		return nil, vcsErrorf("unexpected response from svn log --xml: %v\n%s", err, out)
	}

	info := &RevInfo{
		Name:    fmt.Sprintf("%d", log.Logentry.Revision),
		Short:   fmt.Sprintf("%012d", log.Logentry.Revision),
		Time:    t.UTC(),
		Version: rev,
	}
	return info, nil
}

func bzrParseStat(rev, out string) (*RevInfo, error) {
	var revno int64
	var tm time.Time
	for _, line := range strings.Split(out, "\n") {
		if line == "" || line[0] == ' ' || line[0] == '\t' {
			// End of header, start of commit message.
			break
		}
		if line[0] == '-' {
			continue
		}
		i := strings.Index(line, ":")
		if i < 0 {
			// End of header, start of commit message.
			break
		}
		key, val := line[:i], strings.TrimSpace(line[i+1:])
		switch key {
		case "revno":
			if j := strings.Index(val, " "); j >= 0 {
				val = val[:j]
			}
			i, err := strconv.ParseInt(val, 10, 64)
			if err != nil {
				return nil, vcsErrorf("unexpected revno from bzr log: %q", line)
			}
			revno = i
		case "timestamp":
			j := strings.Index(val, " ")
			if j < 0 {
				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
			}
			t, err := time.Parse("2006-01-02 15:04:05 -0700", val[j+1:])
			if err != nil {
				return nil, vcsErrorf("unexpected timestamp from bzr log: %q", line)
			}
			tm = t.UTC()
		}
	}
	if revno == 0 || tm.IsZero() {
		return nil, vcsErrorf("unexpected response from bzr log: %q", out)
	}

	info := &RevInfo{
		Name:    fmt.Sprintf("%d", revno),
		Short:   fmt.Sprintf("%012d", revno),
		Time:    tm,
		Version: rev,
	}
	return info, nil
}

func fossilParseStat(rev, out string) (*RevInfo, error) {
	for _, line := range strings.Split(out, "\n") {
		if strings.HasPrefix(line, "uuid:") {
			f := strings.Fields(line)
			if len(f) != 5 || len(f[1]) != 40 || f[4] != "UTC" {
				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
			}
			t, err := time.Parse("2006-01-02 15:04:05", f[2]+" "+f[3])
			if err != nil {
				return nil, vcsErrorf("unexpected response from fossil info: %q", line)
			}
			hash := f[1]
			version := rev
			if strings.HasPrefix(hash, version) {
				version = hash // extend to full hash
			}
			info := &RevInfo{
				Name:    hash,
				Short:   ShortenSHA1(hash),
				Time:    t,
				Version: version,
			}
			return info, nil
		}
	}
	return nil, vcsErrorf("unexpected response from fossil info: %q", out)
}