cmd/go: automatically check and use vendored packages

This implements the proposal described in
https://golang.org/issue/33848#issuecomment-537222782.

Fixes golang#33848

Change-Id: Ia34d6500ca396b6aa644b920233716c6b83ef729
Reviewed-on: https://go-review.googlesource.com/c/go/+/198319
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Gobot Gobot <gobot@golang.org>
Reviewed-by: Jay Conrod <jayconrod@google.com>

doc/go1.14.html

@@ -51,20 +51,38 @@ <h2 id="tools">Tools</h2>
 
 <h3 id="go-command">Go command</h3>
 
-<p><!-- golang.org/issue/30748 -->
-  The <code>go</code> command now includes snippets of plain-text error messages
-  from module proxies and other HTTP servers.
-  An error message will only be shown if it is valid UTF-8 and consists of only
-  graphic characters and spaces.
+<!-- golang.org/issue/33848 -->
+<p>
+  When the main module contains a top-level <code>vendor</code> directory and
+  its <code>go.mod<code> file specifies <code>go</code> <code>1.14</code> or
+  higher, the <code>go</code> command now defaults to <code>-mod=vendor</code>
+  for operations that accept that flag. A new value for that flag,
+  <code>-mod=mod</code>, causes the <code>go</code> command to instead load
+  modules from the module cache (as when no <code>vendor<code> directory is
+  present).
+</p>
+
+<p>
+  When <code>-mod=vendor</code> is set (explicitly or by default), the
+  <code>go</code> command now verifies that the main module's
+  <code>vendor/modules.txt</code> file is consistent with its
+  <code>go.mod</code> file.
 </p>
 
 <p><!-- golang.org/issue/32502, golang.org/issue/30345 -->
-  The <code>go</code> <code>get</code> subcommand no longer accepts
+  The <code>go</code> <code>get</code> command no longer accepts
   the <code>-mod</code> flag. Previously, the flag's setting either
   <a href="https://golang.org/issue/30345">was ignored</a> or
   <a href="https://golang.org/issue/32502">caused the build to fail</a>.
 </p>
 
+<p><!-- golang.org/issue/30748 -->
+  The <code>go</code> command now includes snippets of plain-text error messages
+  from module proxies and other HTTP servers.
+  An error message will only be shown if it is valid UTF-8 and consists of only
+  graphic characters and spaces.
+</p>
+
 <h2 id="runtime">Runtime</h2>
 
 <p>

src/cmd/go.mod

@@ -1,6 +1,6 @@
 module cmd
 
-go 1.12
+go 1.14
 
 require (
 	github.com/google/pprof v0.0.0-20190515194954-54271f7e092f

src/cmd/go/internal/modcmd/vendor.go

@@ -59,19 +59,24 @@ func runVendor(cmd *base.Command, args []string) {
 		modpkgs[m] = append(modpkgs[m], pkg)
 	}
 
+	isExplicit := map[module.Version]bool{}
+	for _, r := range modload.ModFile().Require {
+		isExplicit[r.Mod] = true
+	}
+
 	var buf bytes.Buffer
 	for _, m := range modload.BuildList()[1:] {
-		if pkgs := modpkgs[m]; len(pkgs) > 0 {
-			repl := ""
-			if r := modload.Replacement(m); r.Path != "" {
-				repl = " => " + r.Path
-				if r.Version != "" {
-					repl += " " + r.Version
-				}
-			}
-			fmt.Fprintf(&buf, "# %s %s%s\n", m.Path, m.Version, repl)
+		if pkgs := modpkgs[m]; len(pkgs) > 0 || isExplicit[m] {
+			line := moduleLine(m, modload.Replacement(m))
+			buf.WriteString(line)
 			if cfg.BuildV {
-				fmt.Fprintf(os.Stderr, "# %s %s%s\n", m.Path, m.Version, repl)
+				os.Stderr.WriteString(line)
+			}
+			if isExplicit[m] {
+				buf.WriteString("## explicit\n")
+				if cfg.BuildV {
+					os.Stderr.WriteString("## explicit\n")
+				}
 			}
 			sort.Strings(pkgs)
 			for _, pkg := range pkgs {
@@ -83,6 +88,24 @@ func runVendor(cmd *base.Command, args []string) {
 			}
 		}
 	}
+
+	// Record unused and wildcard replacements at the end of the modules.txt file:
+	// without access to the complete build list, the consumer of the vendor
+	// directory can't otherwise determine that those replacements had no effect.
+	for _, r := range modload.ModFile().Replace {
+		if len(modpkgs[r.Old]) > 0 {
+			// We we already recorded this replacement in the entry for the replaced
+			// module with the packages it provides.
+			continue
+		}
+
+		line := moduleLine(r.Old, r.New)
+		buf.WriteString(line)
+		if cfg.BuildV {
+			os.Stderr.WriteString(line)
+		}
+	}
+
 	if buf.Len() == 0 {
 		fmt.Fprintf(os.Stderr, "go: no dependencies to vendor\n")
 		return
@@ -92,6 +115,26 @@ func runVendor(cmd *base.Command, args []string) {
 	}
 }
 
+func moduleLine(m, r module.Version) string {
+	b := new(strings.Builder)
+	b.WriteString("# ")
+	b.WriteString(m.Path)
+	if m.Version != "" {
+		b.WriteString(" ")
+		b.WriteString(m.Version)
+	}
+	if r.Path != "" {
+		b.WriteString(" => ")
+		b.WriteString(r.Path)
+		if r.Version != "" {
+			b.WriteString(" ")
+			b.WriteString(r.Version)
+		}
+	}
+	b.WriteString("\n")
+	return b.String()
+}
+
 func vendorPkg(vdir, pkg string) {
 	realPath := modload.ImportMap(pkg)
 	if realPath != pkg && modload.ImportMap(realPath) != "" {

src/cmd/go/internal/modload/import.go

@@ -139,7 +139,7 @@ func Import(path string) (m module.Version, dir string, err error) {
 			return Target, mainDir, nil
 		}
 		readVendorList()
-		return vendorMap[path], vendorDir, nil
+		return vendorPkgModule[path], vendorDir, nil
 	}
 
 	// Check each module on the build list.

src/cmd/go/internal/modload/init.go

@@ -30,6 +30,7 @@ import (
 	"cmd/go/internal/mvs"
 	"cmd/go/internal/renameio"
 	"cmd/go/internal/search"
+	"cmd/go/internal/semver"
 )
 
 var (
@@ -349,8 +350,14 @@ func InitMod() {
 		excluded[x.Mod] = true
 	}
 	modFileToBuildList()
-	stdVendorMode()
-	WriteGoMod()
+	setDefaultBuildMod()
+	if cfg.BuildMod == "vendor" {
+		readVendorList()
+		checkVendorConsistency()
+	} else {
+		// TODO(golang.org/issue/33326): if cfg.BuildMod != "readonly"?
+		WriteGoMod()
+	}
 }
 
 // modFileToBuildList initializes buildList from the modFile.
@@ -371,40 +378,133 @@ func modFileToBuildList() {
 	buildList = list
 }
 
-// stdVendorMode applies inside $GOROOT/src.
-// It checks that the go.mod matches vendor/modules.txt
-// and then sets -mod=vendor unless this is a command
-// that has to do explicitly with modules.
-func stdVendorMode() {
-	if !targetInGorootSrc {
+// setDefaultBuildMod sets a default value for cfg.BuildMod
+// if it is currently empty.
+func setDefaultBuildMod() {
+	if cfg.BuildMod != "" {
+		// Don't override an explicit '-mod=' argument.
 		return
 	}
+	cfg.BuildMod = "mod"
 	if cfg.CmdName == "get" || strings.HasPrefix(cfg.CmdName, "mod ") {
+		// Don't set -mod implicitly for commands whose purpose is to
+		// manipulate the build list.
 		return
 	}
+	if modRoot != "" {
+		if fi, err := os.Stat(filepath.Join(modRoot, "vendor")); err == nil && fi.IsDir() {
+			modGo := "unspecified"
+			if modFile.Go != nil {
+				if semver.Compare("v"+modFile.Go.Version, "v1.14") >= 0 {
+					// The Go version is at least 1.14, and a vendor directory exists.
+					// Set -mod=vendor by default.
+					cfg.BuildMod = "vendor"
+					return
+				} else {
+					modGo = modFile.Go.Version
+				}
+			}
+			fmt.Fprintf(os.Stderr, "go: not defaulting to -mod=vendor because go.mod 'go' version is %s\n", modGo)
+		}
+	}
 
+	// TODO(golang.org/issue/33326): set -mod=readonly implicitly if the go.mod
+	// file is itself read-only?
+}
+
+// checkVendorConsistency verifies that the vendor/modules.txt file matches (if
+// go 1.14) or at least does not contradict (go 1.13 or earlier) the
+// requirements and replacements listed in the main module's go.mod file.
+func checkVendorConsistency() {
 	readVendorList()
-BuildList:
-	for _, m := range buildList {
-		if m.Path == "cmd" || m.Path == "std" {
-			continue
+
+	pre114 := false
+	if modFile.Go == nil || semver.Compare("v"+modFile.Go.Version, "v1.14") < 0 {
+		// Go versions before 1.14 did not include enough information in
+		// vendor/modules.txt to check for consistency.
+		// If we know that we're on an earlier version, relax the consistency check.
+		pre114 = true
+	}
+
+	vendErrors := new(strings.Builder)
+	vendErrorf := func(mod module.Version, format string, args ...interface{}) {
+		detail := fmt.Sprintf(format, args...)
+		if mod.Version == "" {
+			fmt.Fprintf(vendErrors, "\n\t%s: %s", mod.Path, detail)
+		} else {
+			fmt.Fprintf(vendErrors, "\n\t%s@%s: %s", mod.Path, mod.Version, detail)
 		}
-		for _, v := range vendorList {
-			if m.Path == v.Path {
-				if m.Version != v.Version {
-					base.Fatalf("go: inconsistent vendoring in %s:\n"+
-						"\tgo.mod requires %s %s but vendor/modules.txt has %s.\n"+
-						"\trun 'go mod tidy; go mod vendor' to sync",
-						modRoot, m.Path, m.Version, v.Version)
+	}
+
+	explicitInGoMod := make(map[module.Version]bool, len(modFile.Require))
+	for _, r := range modFile.Require {
+		explicitInGoMod[r.Mod] = true
+		if !vendorMeta[r.Mod].Explicit {
+			if pre114 {
+				// Before 1.14, modules.txt did not indicate whether modules were listed
+				// explicitly in the main module's go.mod file.
+				// However, we can at least detect a version mismatch if packages were
+				// vendored from a non-matching version.
+				if vv, ok := vendorVersion[r.Mod.Path]; ok && vv != r.Mod.Version {
+					vendErrorf(r.Mod, fmt.Sprintf("is explicitly required in go.mod, but vendor/modules.txt indicates %s@%s", r.Mod.Path, vv))
 				}
-				continue BuildList
+			} else {
+				vendErrorf(r.Mod, "is explicitly required in go.mod, but not marked as explicit in vendor/modules.txt")
+			}
+		}
+	}
+
+	describe := func(m module.Version) string {
+		if m.Version == "" {
+			return m.Path
+		}
+		return m.Path + "@" + m.Version
+	}
+
+	// We need to verify *all* replacements that occur in modfile: even if they
+	// don't directly apply to any module in the vendor list, the replacement
+	// go.mod file can affect the selected versions of other (transitive)
+	// dependencies
+	goModReplacement := make(map[module.Version]module.Version, len(modFile.Replace))
+	for _, r := range modFile.Replace {
+		goModReplacement[r.Old] = r.New
+		vr := vendorMeta[r.Old].Replacement
+		if vr == (module.Version{}) {
+			if pre114 && (r.Old.Version == "" || vendorVersion[r.Old.Path] != r.Old.Version) {
+				// Before 1.14, modules.txt omitted wildcard replacements and
+				// replacements for modules that did not have any packages to vendor.
+			} else {
+				vendErrorf(r.Old, "is replaced in go.mod, but not marked as replaced in vendor/modules.txt")
 			}
+		} else if vr != r.New {
+			vendErrorf(r.Old, "is replaced by %s in go.mod, but marked as replaced by %s in vendor/modules.txt", describe(r.New), describe(vr))
 		}
-		base.Fatalf("go: inconsistent vendoring in %s:\n"+
-			"\tgo.mod requires %s %s but vendor/modules.txt does not include it.\n"+
-			"\trun 'go mod tidy; go mod vendor' to sync", modRoot, m.Path, m.Version)
 	}
-	cfg.BuildMod = "vendor"
+
+	for _, mod := range vendorList {
+		meta := vendorMeta[mod]
+		if meta.Explicit && !explicitInGoMod[mod] {
+			vendErrorf(mod, "is marked as explicit in vendor/modules.txt, but not explicitly required in go.mod")
+		}
+	}
+
+	for _, mod := range vendorReplaced {
+		r, ok := goModReplacement[mod]
+		if !ok {
+			r, ok = goModReplacement[module.Version{Path: mod.Path}]
+		}
+		if !ok {
+			vendErrorf(mod, "is marked as replaced in vendor/modules.txt, but not replaced in go.mod")
+			continue
+		}
+		if meta := vendorMeta[mod]; r != meta.Replacement {
+			vendErrorf(mod, "is marked as replaced by %s in vendor/modules.txt, but replaced by %s in go.mod", describe(meta.Replacement), describe(r))
+		}
+	}
+
+	if vendErrors.Len() > 0 {
+		base.Fatalf("go: inconsistent vendoring in %s:%s\n\nrun 'go mod vendor' to sync, or use -mod=mod or -mod=readonly to ignore the vendor directory", modRoot, vendErrors)
+	}
 }
 
 // Allowed reports whether module m is allowed (not excluded) by the main module's go.mod.