gopath_scanner.go

// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"

	"github.com/golang/dep"
	"github.com/golang/dep/gps"
	"github.com/golang/dep/gps/paths"
	"github.com/golang/dep/gps/pkgtree"
	fb "github.com/golang/dep/internal/feedback"
	"github.com/golang/dep/internal/fs"
	"github.com/pkg/errors"
)

// gopathScanner supplies manifest/lock data by scanning the contents of GOPATH
// It uses its results to fill-in any missing details left by the rootAnalyzer.
type gopathScanner struct {
	ctx        *dep.Ctx
	directDeps map[gps.ProjectRoot]bool
	sm         gps.SourceManager

	pd    projectData
	origM *dep.Manifest
	origL *dep.Lock
}

func newGopathScanner(ctx *dep.Ctx, directDeps map[gps.ProjectRoot]bool, sm gps.SourceManager) *gopathScanner {
	return &gopathScanner{
		ctx:        ctx,
		directDeps: directDeps,
		sm:         sm,
	}
}

// InitializeRootManifestAndLock performs analysis of the filesystem tree rooted
// at path, with the root import path importRoot, to determine the project's
// constraints. Respect any initial constraints defined in the root manifest and
// lock.
func (g *gopathScanner) InitializeRootManifestAndLock(rootM *dep.Manifest, rootL *dep.Lock) error {
	var err error

	g.ctx.Err.Println("Searching GOPATH for projects...")
	g.pd, err = g.scanGopathForDependencies()
	if err != nil {
		return err
	}

	g.origM = dep.NewManifest()
	g.origM.Constraints = g.pd.constraints

	g.origL = &dep.Lock{
		P: make([]gps.LockedProject, 0, len(g.pd.ondisk)),
	}

	for pr, v := range g.pd.ondisk {
		// That we have to chop off these path prefixes is a symptom of
		// a problem in gps itself
		pkgs := make([]string, 0, len(g.pd.dependencies[pr]))
		prslash := string(pr) + "/"
		for _, pkg := range g.pd.dependencies[pr] {
			if pkg == string(pr) {
				pkgs = append(pkgs, ".")
			} else {
				pkgs = append(pkgs, trimPathPrefix(pkg, prslash))
			}
		}

		g.origL.P = append(g.origL.P, gps.NewLockedProject(
			gps.ProjectIdentifier{ProjectRoot: pr}, v, pkgs),
		)
	}

	g.overlay(rootM, rootL)

	return nil
}

// Fill in gaps in the root manifest/lock with data found from the GOPATH.
func (g *gopathScanner) overlay(rootM *dep.Manifest, rootL *dep.Lock) {
	for pkg, prj := range g.origM.Constraints {
		if _, has := rootM.Constraints[pkg]; has {
			continue
		}
		rootM.Constraints[pkg] = prj
		v := g.pd.ondisk[pkg]

		pi := gps.ProjectIdentifier{ProjectRoot: pkg, Source: prj.Source}
		f := fb.NewConstraintFeedback(gps.ProjectConstraint{Ident: pi, Constraint: v}, fb.DepTypeDirect)
		f.LogFeedback(g.ctx.Err)
		f = fb.NewLockedProjectFeedback(gps.NewLockedProject(pi, v, nil), fb.DepTypeDirect)
		f.LogFeedback(g.ctx.Err)
	}

	// Keep track of which projects have been locked
	lockedProjects := map[gps.ProjectRoot]bool{}
	for _, lp := range rootL.P {
		lockedProjects[lp.Ident().ProjectRoot] = true
	}

	for _, lp := range g.origL.P {
		pkg := lp.Ident().ProjectRoot
		if _, isLocked := lockedProjects[pkg]; isLocked {
			continue
		}
		rootL.P = append(rootL.P, lp)
		lockedProjects[pkg] = true

		if _, isDirect := g.directDeps[pkg]; !isDirect {
			f := fb.NewLockedProjectFeedback(lp, fb.DepTypeTransitive)
			f.LogFeedback(g.ctx.Err)
		}
	}

	// Identify projects whose version is unknown and will have to be solved for
	var missing []string    // all project roots missing from GOPATH
	var missingVCS []string // all project roots missing VCS information
	for pr := range g.pd.notondisk {
		if _, isLocked := lockedProjects[pr]; isLocked {
			continue
		}
		if g.pd.invalidSVC[pr] {
			missingVCS = append(missingVCS, string(pr))
		} else {
			missing = append(missing, string(pr))
		}
	}

	missingStr := ""
	missingVCSStr := ""
	if len(missing) > 0 {
		missingStr = fmt.Sprintf("The following dependencies were not found in GOPATH:\n  %s\n\n",
			strings.Join(missing, "\n  "))
	}
	if len(missingVCS) > 0 {
		missingVCSStr = fmt.Sprintf("The following dependencies found in GOPATH were missing VCS information (a remote source is required):\n  %s\n\n",
			strings.Join(missingVCS, "\n  "))
	}
	if len(missingVCS)+len(missing) > 0 {
		g.ctx.Err.Printf("\n%s%sThe most recent version of these projects will be used.\n\n", missingStr, missingVCSStr)
	}
}

func trimPathPrefix(p1, p2 string) string {
	if isPrefix, _ := fs.HasFilepathPrefix(p1, p2); isPrefix {
		return p1[len(p2):]
	}
	return p1
}

// contains checks if a array of strings contains a value
func contains(a []string, b string) bool {
	for _, v := range a {
		if b == v {
			return true
		}
	}
	return false
}

// getProjectPropertiesFromVersion takes a Version and returns a proper
// ProjectProperties with Constraint value based on the provided version.
func getProjectPropertiesFromVersion(v gps.Version) gps.ProjectProperties {
	pp := gps.ProjectProperties{}

	// extract version and ignore if it's revision only
	switch tv := v.(type) {
	case gps.PairedVersion:
		v = tv.Unpair()
	case gps.Revision:
		return pp
	}

	switch v.Type() {
	case gps.IsBranch, gps.IsVersion:
		pp.Constraint = v
	case gps.IsSemver:
		c, err := gps.NewSemverConstraintIC(v.String())
		if err != nil {
			panic(err)
		}
		pp.Constraint = c
	}

	return pp
}

type projectData struct {
	constraints  gps.ProjectConstraints          // constraints that could be found
	dependencies map[gps.ProjectRoot][]string    // all dependencies (imports) found by project root
	notondisk    map[gps.ProjectRoot]bool        // projects that were not found on disk
	invalidSVC   map[gps.ProjectRoot]bool        // projects that were found on disk but SVC data could not be read
	ondisk       map[gps.ProjectRoot]gps.Version // projects that were found on disk
}

func (g *gopathScanner) scanGopathForDependencies() (projectData, error) {
	constraints := make(gps.ProjectConstraints)
	dependencies := make(map[gps.ProjectRoot][]string)
	packages := make(map[string]bool)
	notondisk := make(map[gps.ProjectRoot]bool)
	invalidSVC := make(map[gps.ProjectRoot]bool)
	ondisk := make(map[gps.ProjectRoot]gps.Version)

	var syncDepGroup sync.WaitGroup
	syncDep := func(pr gps.ProjectRoot, sm gps.SourceManager) {
		if err := sm.SyncSourceFor(gps.ProjectIdentifier{ProjectRoot: pr}); err != nil {
			g.ctx.Err.Printf("%+v", errors.Wrapf(err, "Unable to cache %s", pr))
		}
		syncDepGroup.Done()
	}

	if len(g.directDeps) == 0 {
		return projectData{}, nil
	}

	for ippr := range g.directDeps {
		// TODO(sdboyer) these are not import paths by this point, they've
		// already been worked down to project roots.
		ip := string(ippr)
		pr, err := g.sm.DeduceProjectRoot(ip)
		if err != nil {
			return projectData{}, errors.Wrap(err, "sm.DeduceProjectRoot")
		}

		packages[ip] = true
		if _, has := dependencies[pr]; has {
			dependencies[pr] = append(dependencies[pr], ip)
			continue
		}
		syncDepGroup.Add(1)
		go syncDep(pr, g.sm)

		dependencies[pr] = []string{ip}
		abs, err := g.ctx.AbsForImport(string(pr))
		if err != nil {
			notondisk[pr] = true
			continue
		}
		v, err := gps.VCSVersion(abs)
		if err != nil {
			invalidSVC[pr] = true
			notondisk[pr] = true
			continue
		}

		ondisk[pr] = v
		pp := getProjectPropertiesFromVersion(v)
		if pp.Constraint != nil || pp.Source != "" {
			constraints[pr] = pp
		}
	}

	// Explore the packages we've found for transitive deps, either
	// completing the lock or identifying (more) missing projects that we'll
	// need to ask gps to solve for us.
	colors := make(map[string]uint8)
	const (
		white uint8 = iota
		grey
		black
	)

	// cache of PackageTrees, so we don't parse projects more than once
	ptrees := make(map[gps.ProjectRoot]pkgtree.PackageTree)

	// depth-first traverser
	var dft func(string) error
	dft = func(pkg string) error {
		switch colors[pkg] {
		case white:
			colors[pkg] = grey

			pr, err := g.sm.DeduceProjectRoot(pkg)
			if err != nil {
				return errors.Wrap(err, "could not deduce project root for "+pkg)
			}

			// We already visited this project root earlier via some other
			// pkg within it, and made the decision that it's not on disk.
			// Respect that decision, and pop the stack.
			if notondisk[pr] {
				colors[pkg] = black
				return nil
			}

			ptree, has := ptrees[pr]
			if !has {
				// It's fine if the root does not exist - it indicates that this
				// project is not present in the workspace, and so we need to
				// solve to deal with this dep.
				r := filepath.Join(g.ctx.GOPATH, "src", string(pr))
				fi, err := os.Stat(r)
				if os.IsNotExist(err) || !fi.IsDir() {
					colors[pkg] = black
					notondisk[pr] = true
					return nil
				}

				// We know the project is on disk; the question is whether we're
				// first seeing it here, in the transitive exploration, or if it
				// was found in the initial pass on direct imports. We know it's
				// the former if there's no entry for it in the ondisk map.
				if _, in := ondisk[pr]; !in {
					abs, err := g.ctx.AbsForImport(string(pr))
					if err != nil {
						colors[pkg] = black
						notondisk[pr] = true
						return nil
					}
					v, err := gps.VCSVersion(abs)
					if err != nil {
						// Even if we know it's on disk, errors are still
						// possible when trying to deduce version. If we
						// encounter such an error, just treat the project as
						// not being on disk; the solver will work it out.
						colors[pkg] = black
						notondisk[pr] = true
						return nil
					}
					ondisk[pr] = v
				}

				ptree, err = pkgtree.ListPackages(r, string(pr))
				if err != nil {
					// Any error here other than an a nonexistent dir (which
					// can't happen because we covered that case above) is
					// probably critical, so bail out.
					return errors.Wrap(err, "gps.ListPackages")
				}
				ptrees[pr] = ptree
			}

			// Get a reachmap that includes main pkgs (even though importing
			// them is an error, what we're checking right now is simply whether
			// there's a package with go code present on disk), and does not
			// backpropagate errors (again, because our only concern right now
			// is package existence).
			rm, errmap := ptree.ToReachMap(true, false, false, nil)
			reached, ok := rm[pkg]
			if !ok {
				colors[pkg] = black
				// not on disk...
				notondisk[pr] = true
				return nil
			}
			if _, ok := errmap[pkg]; ok {
				// The package is on disk, but contains some errors.
				colors[pkg] = black
				return nil
			}

			if deps, has := dependencies[pr]; has {
				if !contains(deps, pkg) {
					dependencies[pr] = append(deps, pkg)
				}
			} else {
				dependencies[pr] = []string{pkg}
				syncDepGroup.Add(1)
				go syncDep(pr, g.sm)
			}

			// recurse
			for _, rpkg := range reached.External {
				if paths.IsStandardImportPath(rpkg) {
					continue
				}

				err := dft(rpkg)
				if err != nil {
					// Bubble up any errors we encounter
					return err
				}
			}

			colors[pkg] = black
		case grey:
			return errors.Errorf("Import cycle detected on %s", pkg)
		}
		return nil
	}

	// run the depth-first traversal from the set of immediate external
	// package imports we found in the current project
	for pkg := range packages {
		err := dft(pkg)
		if err != nil {
			return projectData{}, err // already errors.Wrap()'d internally
		}
	}

	syncDepGroup.Wait()

	pd := projectData{
		constraints:  constraints,
		dependencies: dependencies,
		invalidSVC:   invalidSVC,
		notondisk:    notondisk,
		ondisk:       ondisk,
	}
	return pd, nil
}