cmd/doc: understand vendor directories in module mode

This change employs the same strategy as in CL 203017
to detect when vendoring is in use, and if so treats
the vendor directory as a (non-module, prefixless) root.

The integration test also verifies that the 'std' and 'cmd'
modules are included and their vendored dependencies are
visible (as they are with 'go list') even when outside of
those modules.

Fixes #35224

Change-Id: I18cd01218e9eb97c1fc6e2401c1907536b0b95f7
Reviewed-on: https://go-review.googlesource.com/c/go/+/205577
Run-TryBot: Bryan C. Mills <bcmills@google.com>
Reviewed-by: Jay Conrod <jayconrod@google.com>

src/cmd/doc/dirs.go

@@ -6,19 +6,24 @@ package main
 
 import (
 	"bytes"
+	"fmt"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
+	"regexp"
 	"strings"
 	"sync"
+
+	"golang.org/x/mod/semver"
 )
 
 // A Dir describes a directory holding code by specifying
 // the expected import path and the file system directory.
 type Dir struct {
 	importPath string // import path for that dir
 	dir        string // file system directory
+	inModule   bool
 }
 
 // Dirs is a structure for scanning the directory tree.
@@ -113,9 +118,14 @@ func (d *Dirs) bfsWalkRoot(root Dir) {
 				if name[0] == '.' || name[0] == '_' || name == "testdata" {
 					continue
 				}
-				// Ignore vendor when using modules.
-				if usingModules && name == "vendor" {
-					continue
+				// When in a module, ignore vendor directories and stop at module boundaries.
+				if root.inModule {
+					if name == "vendor" {
+						continue
+					}
+					if fi, err := os.Stat(filepath.Join(dir, name, "go.mod")); err == nil && !fi.IsDir() {
+						continue
+					}
 				}
 				// Remember this (fully qualified) directory for the next pass.
 				next = append(next, filepath.Join(dir, name))
@@ -129,7 +139,7 @@ func (d *Dirs) bfsWalkRoot(root Dir) {
 					}
 					importPath += filepath.ToSlash(dir[len(root.dir)+1:])
 				}
-				d.scan <- Dir{importPath, dir}
+				d.scan <- Dir{importPath, dir, root.inModule}
 			}
 		}
 
@@ -156,14 +166,20 @@ var codeRootsCache struct {
 var usingModules bool
 
 func findCodeRoots() []Dir {
-	list := []Dir{{"", filepath.Join(buildCtx.GOROOT, "src")}}
-
+	var list []Dir
 	if !testGOPATH {
 		// Check for use of modules by 'go env GOMOD',
 		// which reports a go.mod file path if modules are enabled.
 		stdout, _ := exec.Command("go", "env", "GOMOD").Output()
 		gomod := string(bytes.TrimSpace(stdout))
+
 		usingModules = len(gomod) > 0
+		if usingModules {
+			list = append(list,
+				Dir{dir: filepath.Join(buildCtx.GOROOT, "src"), inModule: true},
+				Dir{importPath: "cmd", dir: filepath.Join(buildCtx.GOROOT, "src", "cmd"), inModule: true})
+		}
+
 		if gomod == os.DevNull {
 			// Modules are enabled, but the working directory is outside any module.
 			// We can still access std, cmd, and packages specified as source files
@@ -174,8 +190,9 @@ func findCodeRoots() []Dir {
 	}
 
 	if !usingModules {
+		list = append(list, Dir{dir: filepath.Join(buildCtx.GOROOT, "src")})
 		for _, root := range splitGopath() {
-			list = append(list, Dir{"", filepath.Join(root, "src")})
+			list = append(list, Dir{dir: filepath.Join(root, "src")})
 		}
 		return list
 	}
@@ -185,6 +202,21 @@ func findCodeRoots() []Dir {
 	// to handle the entire file system search and become go/packages,
 	// but for now enumerating the module roots lets us fit modules
 	// into the current code with as few changes as possible.
+	mainMod, vendorEnabled, err := vendorEnabled()
+	if err != nil {
+		return list
+	}
+	if vendorEnabled {
+		// Add the vendor directory to the search path ahead of "std".
+		// That way, if the main module *is* "std", we will identify the path
+		// without the "vendor/" prefix before the one with that prefix.
+		list = append([]Dir{{dir: filepath.Join(mainMod.Dir, "vendor"), inModule: false}}, list...)
+		if mainMod.Path != "std" {
+			list = append(list, Dir{importPath: mainMod.Path, dir: mainMod.Dir, inModule: true})
+		}
+		return list
+	}
+
 	cmd := exec.Command("go", "list", "-m", "-f={{.Path}}\t{{.Dir}}", "all")
 	cmd.Stderr = os.Stderr
 	out, _ := cmd.Output()
@@ -195,9 +227,77 @@ func findCodeRoots() []Dir {
 		}
 		path, dir := line[:i], line[i+1:]
 		if dir != "" {
-			list = append(list, Dir{path, dir})
+			list = append(list, Dir{importPath: path, dir: dir, inModule: true})
 		}
 	}
 
 	return list
 }
+
+// The functions below are derived from x/tools/internal/imports at CL 203017.
+
+type moduleJSON struct {
+	Path, Dir, GoVersion string
+}
+
+var modFlagRegexp = regexp.MustCompile(`-mod[ =](\w+)`)
+
+// vendorEnabled indicates if vendoring is enabled.
+// Inspired by setDefaultBuildMod in modload/init.go
+func vendorEnabled() (*moduleJSON, bool, error) {
+	mainMod, go114, err := getMainModuleAnd114()
+	if err != nil {
+		return nil, false, err
+	}
+
+	stdout, _ := exec.Command("go", "env", "GOFLAGS").Output()
+	goflags := string(bytes.TrimSpace(stdout))
+	matches := modFlagRegexp.FindStringSubmatch(goflags)
+	var modFlag string
+	if len(matches) != 0 {
+		modFlag = matches[1]
+	}
+	if modFlag != "" {
+		// Don't override an explicit '-mod=' argument.
+		return mainMod, modFlag == "vendor", nil
+	}
+	if mainMod == nil || !go114 {
+		return mainMod, false, nil
+	}
+	// Check 1.14's automatic vendor mode.
+	if fi, err := os.Stat(filepath.Join(mainMod.Dir, "vendor")); err == nil && fi.IsDir() {
+		if mainMod.GoVersion != "" && semver.Compare("v"+mainMod.GoVersion, "v1.14") >= 0 {
+			// The Go version is at least 1.14, and a vendor directory exists.
+			// Set -mod=vendor by default.
+			return mainMod, true, nil
+		}
+	}
+	return mainMod, false, nil
+}
+
+// getMainModuleAnd114 gets the main module's information and whether the
+// go command in use is 1.14+. This is the information needed to figure out
+// if vendoring should be enabled.
+func getMainModuleAnd114() (*moduleJSON, bool, error) {
+	const format = `{{.Path}}
+{{.Dir}}
+{{.GoVersion}}
+{{range context.ReleaseTags}}{{if eq . "go1.14"}}{{.}}{{end}}{{end}}
+`
+	cmd := exec.Command("go", "list", "-m", "-f", format)
+	cmd.Stderr = os.Stderr
+	stdout, err := cmd.Output()
+	if err != nil {
+		return nil, false, nil
+	}
+	lines := strings.Split(string(stdout), "\n")
+	if len(lines) < 5 {
+		return nil, false, fmt.Errorf("unexpected stdout: %q", stdout)
+	}
+	mod := &moduleJSON{
+		Path:      lines[0],
+		Dir:       lines[1],
+		GoVersion: lines[2],
+	}
+	return mod, lines[3] == "go1.14", nil
+}

src/cmd/doc/doc_test.go

@@ -27,7 +27,10 @@ func TestMain(m *testing.M) {
 	if err != nil {
 		panic(err)
 	}
-	dirsInit(Dir{"testdata", testdataDir}, Dir{"testdata/nested", filepath.Join(testdataDir, "nested")}, Dir{"testdata/nested/nested", filepath.Join(testdataDir, "nested", "nested")})
+	dirsInit(
+		Dir{importPath: "testdata", dir: testdataDir},
+		Dir{importPath: "testdata/nested", dir: filepath.Join(testdataDir, "nested")},
+		Dir{importPath: "testdata/nested/nested", dir: filepath.Join(testdataDir, "nested", "nested")})
 
 	os.Exit(m.Run())
 }

src/cmd/go/testdata/script/mod_doc.txt

@@ -41,6 +41,30 @@ env GOPROXY=off
 ! go doc example.com/hello
 stderr '^doc: cannot find module providing package example.com/hello: module lookup disabled by GOPROXY=off$'
 
+# When in a module with a vendor directory, doc should use the vendored copies
+# of the packages. 'std' and 'cmd' are convenient examples of such modules.
+#
+# When in those modules, the "// import" comment should refer to the same import
+# path used in source code, not to the absolute path relative to GOROOT.
+
+cd $GOROOT/src
+go doc cryptobyte
+stdout '// import "golang.org/x/crypto/cryptobyte"'
+
+cd $GOROOT/src/cmd/go
+go doc modfile
+stdout '// import "golang.org/x/mod/modfile"'
+
+# When outside of the 'std' module, its vendored packages
+# remain accessible using the 'vendor/' prefix, but report
+# the correct "// import" comment as used within std.
+cd $GOPATH
+go doc vendor/golang.org/x/crypto/cryptobyte
+stdout '// import "vendor/golang.org/x/crypto/cryptobyte"'
+
+go doc cmd/vendor/golang.org/x/mod/modfile
+stdout '// import "cmd/vendor/golang.org/x/mod/modfile"'
+
 -- go.mod --
 module x
 require rsc.io/quote v1.5.2