go/packages: add name= query

Add an implementation of name= for go list. It will be used to
implement goimports and godoc-like lookups by package name.

Imported a copy of the semver package from the stdlib to do version
comparison, and tweaked the gopathwalk API to include a hint about what
kind of source directory is being traversed.

Note that the tests, despite my best efforts, are not hermetic: go list
insists on doing version lookups in situations where it seems to me like
it shouldn't need to.

I think this implementation is ready for serious use. The one thing I'm
nervous about is that it currently does a substring match when looking
for a package name, so if you look up a package named "a" you will get
a huge number of results. This matches goimports' behavior but I don't
know if it's suitable for general use.

Change-Id: I2b7f823b74571fe30d3bd9c7dfafb4e6a40df5d3
Reviewed-on: https://go-review.googlesource.com/c/138878
Run-TryBot: Heschi Kreinick <heschi@google.com>
Reviewed-by: Michael Matloob <matloob@golang.org>

go/packages/packages_test.go

@@ -1107,6 +1107,106 @@ func TestContains_FallbackSticks(t *testing.T) {
 	}
 }
 
+func TestName(t *testing.T) {
+	tmp, cleanup := makeTree(t, map[string]string{
+		"src/a/needle/needle.go":       `package needle; import "c"`,
+		"src/b/needle/needle.go":       `package needle;`,
+		"src/c/c.go":                   `package c;`,
+		"src/irrelevant/irrelevant.go": `package irrelevant;`,
+	})
+	defer cleanup()
+
+	cfg := &packages.Config{
+		Mode: packages.LoadImports,
+		Dir:  tmp,
+		Env:  append(os.Environ(), "GOPATH="+tmp, "GO111MODULE=off"),
+	}
+	initial, err := packages.Load(cfg, "name=needle")
+	if err != nil {
+		t.Fatal(err)
+	}
+	graph, _ := importGraph(initial)
+	wantGraph := `
+* a/needle
+* b/needle
+  c
+  a/needle -> c
+`[1:]
+	if graph != wantGraph {
+		t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
+	}
+}
+
+func TestName_Modules(t *testing.T) {
+	tmp, cleanup := makeTree(t, map[string]string{
+		"src/localmod/go.mod":     `module test`,
+		"src/localmod/pkg/pkg.go": `package pkg;`,
+	})
+	defer cleanup()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// testdata/TestNamed_Modules contains:
+	// - pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg
+	// - pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.0/pkg
+	// - src/b/pkg
+	cfg := &packages.Config{
+		Mode: packages.LoadImports,
+		Dir:  filepath.Join(tmp, "src/localmod"),
+		Env:  append(os.Environ(), "GOPATH="+wd+"/testdata/TestName_Modules", "GO111MODULE=on"),
+	}
+
+	initial, err := packages.Load(cfg, "name=pkg")
+	if err != nil {
+		t.Fatal(err)
+	}
+	graph, _ := importGraph(initial)
+	wantGraph := `
+* github.com/heschik/tools-testrepo/pkg
+* github.com/heschik/tools-testrepo/v2/pkg
+* test/pkg
+`[1:]
+	if graph != wantGraph {
+		t.Errorf("wrong import graph: got <<%s>>, want <<%s>>", graph, wantGraph)
+	}
+}
+
+func TestName_ModulesDedup(t *testing.T) {
+	tmp, cleanup := makeTree(t, map[string]string{
+		"src/localmod/go.mod": `module test`,
+	})
+	defer cleanup()
+
+	wd, err := os.Getwd()
+	if err != nil {
+		t.Fatal(err)
+	}
+	// testdata/TestNamed_ModulesDedup contains:
+	// - pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.2/pkg/pkg.go
+	// - pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.1/pkg/pkg.go
+	// - pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg/pkg.go
+	// but, inexplicably, not v2.0.0. Nobody knows why.
+	cfg := &packages.Config{
+		Mode: packages.LoadImports,
+		Dir:  filepath.Join(tmp, "src/localmod"),
+		Env:  append(os.Environ(), "GOPATH="+wd+"/testdata/TestName_ModulesDedup", "GO111MODULE=on"),
+	}
+	initial, err := packages.Load(cfg, "name=pkg")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for _, pkg := range initial {
+		if strings.Contains(pkg.PkgPath, "v2") {
+			if strings.Contains(pkg.GoFiles[0], "v2.0.2") {
+				return
+			}
+		}
+	}
+	t.Errorf("didn't find v2.0.2 of pkg in Load results: %v", initial)
+}
+
 func TestJSON(t *testing.T) {
 	//TODO: add in some errors
 	tmp, cleanup := makeTree(t, map[string]string{

go/packages/testdata/README

@@ -0,0 +1,3 @@
+Test data directories here were created by running go commands with GOPATH set as such:
+GOPATH=......./testdata/TestNamed_ModulesDedup go get github.com/heschik/tools-testrepo/v2@v2.0.1
+and then removing the vcs cache directories, which appear to be unnecessary.

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/list

@@ -0,0 +1 @@
+v1.0.0

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.info

@@ -0,0 +1 @@
+{"Version":"v1.0.0","Time":"2018-09-28T22:09:08Z"}

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.ziphash

@@ -0,0 +1 @@
+h1:D2qc+R2eCTCyoT8WAYoExXhPBThJWmlYSfB4coWbfBE=

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/list

@@ -0,0 +1 @@
+v2.0.0

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.info

@@ -0,0 +1 @@
+{"Version":"v2.0.0","Time":"2018-09-28T22:12:08Z"}

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo/v2

go/packages/testdata/TestName_Modules/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.0.ziphash

@@ -0,0 +1 @@
+h1:Ll4Bx8ZD8zg8lD4idX7CAhx/jh16o9dWC2m9SnT1qu0=

go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.0/go.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo/v2

go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.0/pkg/pkg.go

@@ -0,0 +1 @@
+package pkg

go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/go.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo

go/packages/testdata/TestName_Modules/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg/pkg.go

@@ -0,0 +1 @@
+package pkg

go/packages/testdata/TestName_Modules/src/b/pkg/pkg.go

@@ -0,0 +1 @@
+package pkg

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/list

@@ -0,0 +1 @@
+v1.0.0

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.info

@@ -0,0 +1 @@
+{"Version":"v1.0.0","Time":"2018-09-28T22:09:08Z"}

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/@v/v1.0.0.ziphash

@@ -0,0 +1 @@
+h1:D2qc+R2eCTCyoT8WAYoExXhPBThJWmlYSfB4coWbfBE=

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/list

@@ -0,0 +1,2 @@
+v2.0.1
+v2.0.2

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.info

@@ -0,0 +1 @@
+{"Version":"v2.0.1","Time":"2018-09-28T22:12:08Z"}

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo/v2

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.1.ziphash

@@ -0,0 +1 @@
+h1:efPBVdJ45IMcA/KXBOWyOZLo1TETKCXvzrZgfY+gqZk=

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.info

@@ -0,0 +1 @@
+{"Version":"v2.0.2","Time":"2018-09-28T22:12:08Z"}

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo/v2

go/packages/testdata/TestName_ModulesDedup/pkg/mod/cache/download/github.com/heschik/tools-testrepo/v2/@v/v2.0.2.ziphash

@@ -0,0 +1 @@
+h1:vUnR/JOkfEQt/wvMqbT9G2gODHVgVD1saTJ8x2ngAck=

go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.1/go.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo/v2

go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.1/pkg/pkg.go

@@ -0,0 +1 @@
+package pkg

go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.2/go.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo/v2

go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo/v2@v2.0.2/pkg/pkg.go

@@ -0,0 +1 @@
+package pkg

go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/go.mod

@@ -0,0 +1 @@
+module github.com/heschik/tools-testrepo

go/packages/testdata/TestName_ModulesDedup/pkg/mod/github.com/heschik/tools-testrepo@v1.0.0/pkg/pkg.go

@@ -0,0 +1 @@
+package pkg

imports/fix.go

@@ -526,21 +526,21 @@ func scanGoDirs() map[string]*pkg {
 	result := make(map[string]*pkg)
 	var mu sync.Mutex
 
-	add := func(srcDir, dir string) {
+	add := func(root gopathwalk.Root, dir string) {
 		mu.Lock()
 		defer mu.Unlock()
 
 		if _, dup := result[dir]; dup {
 			return
 		}
-		importpath := filepath.ToSlash(dir[len(srcDir)+len("/"):])
+		importpath := filepath.ToSlash(dir[len(root.Path)+len("/"):])
 		result[dir] = &pkg{
 			importPath:      importpath,
 			importPathShort: VendorlessPath(importpath),
 			dir:             dir,
 		}
 	}
-	gopathwalk.Walk(add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false})
+	gopathwalk.Walk(gopathwalk.SrcDirsRoots(), add, gopathwalk.Options{Debug: Debug, ModulesEnabled: false})
 	return result
 }
 

internal/gopathwalk/walk.go

@@ -25,57 +25,99 @@ type Options struct {
 	ModulesEnabled bool // Search module caches. Also disables legacy goimports ignore rules.
 }
 
+// RootType indicates the type of a Root.
+type RootType int
+
+const (
+	RootUnknown RootType = iota
+	RootGOROOT
+	RootGOPATH
+	RootCurrentModule
+	RootModuleCache
+)
+
+// A Root is a starting point for a Walk.
+type Root struct {
+	Path string
+	Type RootType
+}
+
+// SrcDirsRoots returns the roots from build.Default.SrcDirs(). Not modules-compatible.
+func SrcDirsRoots() []Root {
+	var roots []Root
+	roots = append(roots, Root{filepath.Join(build.Default.GOROOT, "src"), RootGOROOT})
+	for _, p := range filepath.SplitList(build.Default.GOPATH) {
+		roots = append(roots, Root{filepath.Join(p, "src"), RootGOPATH})
+	}
+	return roots
+}
+
 // Walk walks Go source directories ($GOROOT, $GOPATH, etc) to find packages.
 // For each package found, add will be called (concurrently) with the absolute
 // paths of the containing source directory and the package directory.
-func Walk(add func(srcDir string, dir string), opts Options) {
-	for _, srcDir := range build.Default.SrcDirs() {
-		walkDir(srcDir, add, opts)
+// add will be called concurrently.
+func Walk(roots []Root, add func(root Root, dir string), opts Options) {
+	for _, root := range roots {
+		walkDir(root, add, opts)
 	}
 }
 
-func walkDir(srcDir string, add func(string, string), opts Options) {
+func walkDir(root Root, add func(Root, string), opts Options) {
 	if opts.Debug {
-		log.Printf("scanning %s", srcDir)
+		log.Printf("scanning %s", root.Path)
 	}
 	w := &walker{
-		srcDir: srcDir,
-		srcV:   filepath.Join(srcDir, "v"),
-		srcMod: filepath.Join(srcDir, "mod"),
-		add:    add,
-		opts:   opts,
+		root: root,
+		add:  add,
+		opts: opts,
 	}
 	w.init()
-	if err := fastwalk.Walk(srcDir, w.walk); err != nil {
-		log.Printf("goimports: scanning directory %v: %v", srcDir, err)
+	if err := fastwalk.Walk(root.Path, w.walk); err != nil {
+		log.Printf("goimports: scanning directory %v: %v", root.Path, err)
 	}
 
 	if opts.Debug {
-		defer log.Printf("scanned %s", srcDir)
+		defer log.Printf("scanned %s", root.Path)
 	}
 }
 
 // walker is the callback for fastwalk.Walk.
 type walker struct {
-	srcDir       string               // The source directory to scan.
-	srcV, srcMod string               // vgo-style module cache dirs. Optional.
-	add          func(string, string) // The callback that will be invoked for every possible Go package dir.
-	opts         Options              // Options passed to Walk by the user.
+	root Root               // The source directory to scan.
+	add  func(Root, string) // The callback that will be invoked for every possible Go package dir.
+	opts Options            // Options passed to Walk by the user.
 
 	ignoredDirs []os.FileInfo // The ignored directories, loaded from .goimportsignore files.
 }
 
 // init initializes the walker based on its Options.
 func (w *walker) init() {
-	if !w.opts.ModulesEnabled {
-		w.ignoredDirs = w.getIgnoredDirs(w.srcDir)
+	var ignoredPaths []string
+	if w.root.Type == RootModuleCache {
+		ignoredPaths = []string{"cache"}
+	}
+	if !w.opts.ModulesEnabled && w.root.Type == RootGOPATH {
+		ignoredPaths = w.getIgnoredDirs(w.root.Path)
+		ignoredPaths = append(ignoredPaths, "v", "mod")
+	}
+
+	for _, p := range ignoredPaths {
+		full := filepath.Join(w.root.Path, p)
+		if fi, err := os.Stat(full); err == nil {
+			w.ignoredDirs = append(w.ignoredDirs, fi)
+			if w.opts.Debug {
+				log.Printf("Directory added to ignore list: %s", full)
+			}
+		} else if w.opts.Debug {
+			log.Printf("Error statting ignored directory: %v", err)
+		}
 	}
 }
 
 // getIgnoredDirs reads an optional config file at <path>/.goimportsignore
 // of relative directories to ignore when scanning for go files.
 // The provided path is one of the $GOPATH entries with "src" appended.
-func (w *walker) getIgnoredDirs(path string) []os.FileInfo {
+func (w *walker) getIgnoredDirs(path string) []string {
 	file := filepath.Join(path, ".goimportsignore")
 	slurp, err := ioutil.ReadFile(file)
 	if w.opts.Debug {
@@ -89,22 +131,14 @@ func (w *walker) getIgnoredDirs(path string) []os.FileInfo {
 		return nil
 	}
 
-	var ignoredDirs []os.FileInfo
+	var ignoredDirs []string
 	bs := bufio.NewScanner(bytes.NewReader(slurp))
 	for bs.Scan() {
 		line := strings.TrimSpace(bs.Text())
 		if line == "" || strings.HasPrefix(line, "#") {
 			continue
 		}
-		full := filepath.Join(path, line)
-		if fi, err := os.Stat(full); err == nil {
-			ignoredDirs = append(ignoredDirs, fi)
-			if w.opts.Debug {
-				log.Printf("Directory added to ignore list: %s", full)
-			}
-		} else if w.opts.Debug {
-			log.Printf("Error statting entry in .goimportsignore: %v", err)
-		}
+		ignoredDirs = append(ignoredDirs, line)
 	}
 	return ignoredDirs
 }
@@ -119,12 +153,9 @@ func (w *walker) shouldSkipDir(fi os.FileInfo) bool {
 }
 
 func (w *walker) walk(path string, typ os.FileMode) error {
-	if !w.opts.ModulesEnabled && (path == w.srcV || path == w.srcMod) {
-		return filepath.SkipDir
-	}
 	dir := filepath.Dir(path)
 	if typ.IsRegular() {
-		if dir == w.srcDir {
+		if dir == w.root.Path {
 			// Doesn't make sense to have regular files
 			// directly in your $GOPATH/src or $GOROOT/src.
 			return fastwalk.SkipFiles
@@ -133,7 +164,7 @@ func (w *walker) walk(path string, typ os.FileMode) error {
 			return nil
 		}
 
-		w.add(w.srcDir, dir)
+		w.add(w.root, dir)
 		return fastwalk.SkipFiles
 	}
 	if typ == os.ModeDir {

internal/gopathwalk/walk_test.go

@@ -107,9 +107,9 @@ func TestSkip(t *testing.T) {
 	}
 
 	var found []string
-	walkDir(filepath.Join(dir, "src"), func(srcDir string, dir string) {
-		found = append(found, dir[len(srcDir)+1:])
-	}, Options{ModulesEnabled: false})
+	walkDir(Root{filepath.Join(dir, "src"), RootGOPATH}, func(root Root, dir string) {
+		found = append(found, dir[len(root.Path)+1:])
+	}, Options{ModulesEnabled: false, Debug: true})
 	if want := []string{"shouldfind"}; !reflect.DeepEqual(found, want) {
 		t.Errorf("expected to find only %v, got %v", want, found)
 	}