cmd/go: enable lazy loading

This change activates the dormant “lazy loading” codepaths added in CL
265777 and its predecessors. Dependencies of modules that declare 'go
1.17' or higher are loaded lazily, and the dependencies in the go.mod
file maintain additional invariants to support more efficient lazy
loading for downstream dependent modules.

See https://golang.org/design/36460-lazy-module-loading for the
detailed design.

For #36460

Change-Id: Ic12ee7842aef9580357fcf8909d87654fcb2ad12
Reviewed-on: https://go-review.googlesource.com/c/go/+/314634
Trust: Bryan C. Mills <bcmills@google.com>
Run-TryBot: Bryan C. Mills <bcmills@google.com>
TryBot-Result: Go Bot <gobot@golang.org>
Reviewed-by: Michael Matloob <matloob@golang.org>

doc/go1.17.html

@@ -43,7 +43,22 @@ <h2 id="tools">Tools</h2>
 
 <h3 id="go-command">Go command</h3>
 
-<h4 id="modules">Modules</h4>
+<h4 id="lazy-loading">Lazy module loading</h4>
+
+<p><!-- golang.org/issue/36460 -->
+  If a module specifies <code>go</code> <code>1.17</code> or higher in its
+  <code>go.mod</code> file, its transitive requirements are now loaded lazily,
+  avoding the need to download or read <code>go.mod</code> files for
+  otherwise-irrelevant dependencies. To support lazy loading, in Go 1.17 modules
+  the <code>go</code> command maintains <em>explicit</em> requirements in
+  the <code>go.mod</code> file for every dependency that provides any package
+  transitively imported by any package or test within the module.
+  See <a href="https://golang.org/design/36460-lazy-module-loading">the design
+  document</a> for more detail.
+  <!-- TODO(bcmills): replace the design-doc link with proper documentation. -->
+</p>
+
+<h4 id="module-deprecation-comments">Module deprecation comments</h4>
 
 <p><!-- golang.org/issue/40357 -->
   Module authors may deprecate a module by adding a

src/cmd/go.mod

@@ -4,9 +4,12 @@ go 1.17
 
 require (
 	github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5
+	github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639 // indirect
 	golang.org/x/arch v0.0.0-20210308155006-05f8f0431f72
 	golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 // indirect
 	golang.org/x/mod v0.4.3-0.20210409134425-858fdbee9c24
+	golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57 // indirect
 	golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d
 	golang.org/x/tools v0.1.1-0.20210422170518-f946a157eefe
+	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 )

src/cmd/go/internal/modload/modfile.go

@@ -41,7 +41,7 @@ const (
 	// go117EnableLazyLoading toggles whether lazy-loading code paths should be
 	// active. It will be removed once the lazy loading implementation is stable
 	// and well-tested.
-	go117EnableLazyLoading = false
+	go117EnableLazyLoading = true
 
 	// go1117LazyTODO is a constant that exists only until lazy loading is
 	// implemented. Its use indicates a condition that will need to change if the

src/cmd/go/testdata/script/mod_all.txt

@@ -189,19 +189,22 @@ stdout '^example.com/main/testonly_test \[example.com/main/testonly.test\]$'
 
 rm vendor
 
-# Convert all modules to go 1.16 to enable lazy loading.
-go mod edit -go=1.16 a/go.mod
-go mod edit -go=1.16 b/go.mod
-go mod edit -go=1.16 c/go.mod
-go mod edit -go=1.16 d/go.mod
-go mod edit -go=1.16 q/go.mod
-go mod edit -go=1.16 r/go.mod
-go mod edit -go=1.16 s/go.mod
-go mod edit -go=1.16 t/go.mod
-go mod edit -go=1.16 u/go.mod
-go mod edit -go=1.16 w/go.mod
-go mod edit -go=1.16 x/go.mod
-go mod edit -go=1.16
+# Convert all modules to go 1.17 to enable lazy loading.
+go mod edit -go=1.17 a/go.mod
+go mod edit -go=1.17 b/go.mod
+go mod edit -go=1.17 c/go.mod
+go mod edit -go=1.17 d/go.mod
+go mod edit -go=1.17 q/go.mod
+go mod edit -go=1.17 r/go.mod
+go mod edit -go=1.17 s/go.mod
+go mod edit -go=1.17 t/go.mod
+go mod edit -go=1.17 u/go.mod
+go mod edit -go=1.17 w/go.mod
+go mod edit -go=1.17 x/go.mod
+go mod edit -go=1.17
+cp go.mod go.mod.orig
+go mod tidy
+cmp go.mod go.mod.orig
 
 # With lazy loading, 'go list all' with neither -mod=vendor nor -test should
 # match -mod=vendor without -test in 1.15.
@@ -282,20 +285,41 @@ stdout '^example.com/t_test \[example.com/t.test\]$'
 stdout '^example.com/u.test$'
 stdout '^example.com/u_test \[example.com/u.test\]$'
 
+# 'go list -m all' should cover all of the modules providing packages in
+# 'go list -test -deps all', but should exclude modules d and x,
+# which are not relevant to the main module and are outside of the
+# lazy-loading horizon.
 
-# TODO(#36460):
-# 'go list -m all' should exactly cover the packages in 'go list -test all'.
+go list -m -f $MODFMT all
+stdout -count=10 '^.'
+stdout '^example.com/a$'
+stdout '^example.com/b$'
+stdout '^example.com/c$'
+! stdout '^example.com/d$'
+stdout '^example.com/main$'
+stdout '^example.com/q$'
+stdout '^example.com/r$'
+stdout '^example.com/s$'
+stdout '^example.com/t$'
+stdout '^example.com/u$'
+stdout '^example.com/w$'
+! stdout '^example.com/x$'
 
 -- go.mod --
 module example.com/main
 
+// Note: this go.mod file initially specifies go 1.15,
+// but includes some redundant roots so that it
+// also already obeys the 1.17 lazy loading invariants.
 go 1.15
 
 require (
 	example.com/a v0.1.0
 	example.com/b v0.1.0
 	example.com/q v0.1.0
+	example.com/r v0.1.0 // indirect
 	example.com/t v0.1.0
+	example.com/u v0.1.0 // indirect
 )
 
 replace (

src/cmd/go/testdata/script/mod_get_missing_ziphash.txt

@@ -29,7 +29,7 @@ go build -n use
 -- go.mod --
 module use
 
-go 1.17
+go 1.16
 
 require rsc.io/quote v1.5.2
 -- go.sum.tidy --

src/cmd/go/testdata/script/mod_install_pkg_version.txt

@@ -67,9 +67,9 @@ cd tmp
 go mod init tmp
 go mod edit -require=rsc.io/fortune@v1.0.0
 ! go install -mod=readonly $GOPATH/pkg/mod/rsc.io/fortune@v1.0.0
-stderr '^go: rsc.io/fortune@v1.0.0: missing go.sum entry; to add it:\n\tgo mod download rsc.io/fortune$'
+stderr '^rsc.io/fortune@v1.0.0: missing go.sum entry; to add it:\n\tgo mod download rsc.io/fortune$'
 ! go install -mod=readonly ../../pkg/mod/rsc.io/fortune@v1.0.0
-stderr '^go: rsc.io/fortune@v1.0.0: missing go.sum entry; to add it:\n\tgo mod download rsc.io/fortune$'
+stderr '^rsc.io/fortune@v1.0.0: missing go.sum entry; to add it:\n\tgo mod download rsc.io/fortune$'
 go get -d rsc.io/fortune@v1.0.0
 go install -mod=readonly $GOPATH/pkg/mod/rsc.io/fortune@v1.0.0
 exists $GOPATH/bin/fortune$GOEXE

src/cmd/go/testdata/script/mod_lazy_downgrade.txt

@@ -1,5 +1,5 @@
 # This test illustrates the interaction between lazy loading and downgrading in
-# 'go get.
+# 'go get'.
 
 # The package import graph used in this test looks like:
 #
@@ -46,7 +46,7 @@ go list -m all
 # outside of the deepening scan should not affect the downgrade.
 
 cp go.mod.orig go.mod
-go mod edit -go=1.16
+go mod edit -go=1.17
 
 go list -m all
 stdout '^example.com/a v0.1.0 '
@@ -59,12 +59,50 @@ stdout '^example.com/a v0.1.0 '
 stdout '^example.com/b v0.2.0 '
 stdout '^example.com/c v0.1.0 '
 
+# At this point, b.2 is still an explicit root, so its dependency on c
+# is still tracked, and it will still be downgraded away if we remove c.
+# ('go get' never makes a root into a non-root. Only 'go mod tidy' does that.)
+
 go get -d example.com/c@none
 go list -m all
-! stdout '^example.com/a '  # TODO(#36460): example.com/a v0.1.0
-! stdout '^example.com/b '  # TODO(#36460): example.com/b v0.1.0
+! stdout '^example.com/a '
+! stdout '^example.com/b '
 ! stdout '^example.com/c '
 
+
+# This time, we drop the explicit 'b' root by downgrading it to v0.1.0
+# (the version required by a.1) and running 'go mod tidy'.
+# It is still selected at v0.1.0 (as a dependency of a),
+# but its dependency on c is now pruned from the module graph, so it doesn't
+# result in any downgrades to b or a if we run 'go get c@none'.
+
+cp go.mod.orig go.mod
+go mod edit -go=1.17
+
+go list -m all
+stdout '^example.com/a v0.1.0 '
+stdout '^example.com/b v0.3.0 '
+stdout '^example.com/c v0.2.0 '
+
+go get -d example.com/c@v0.1.0 example.com/b@v0.1.0
+go list -m all
+stdout '^example.com/a v0.1.0 '
+stdout '^example.com/b v0.1.0 '
+stdout '^example.com/c v0.1.0 '
+
+go mod tidy
+go list -m all
+stdout '^example.com/a v0.1.0 '
+stdout '^example.com/b v0.1.0 '
+! stdout '^example.com/c '
+
+go get -d example.com/c@none
+go list -m all
+stdout '^example.com/a v0.1.0'
+stdout '^example.com/b v0.1.0'
+! stdout '^example.com/c '
+
+
 -- go.mod --
 module example.com/lazy
 
@@ -91,7 +129,7 @@ import _ "example.com/a"
 -- a/go.mod --
 module example.com/a
 
-go 1.15
+go 1.17
 
 require example.com/b v0.1.0
 -- a/a.go --
@@ -104,7 +142,7 @@ import _ "example.com/b"
 -- b1/go.mod --
 module example.com/b
 
-go 1.15
+go 1.17
 
 require example.com/c v0.1.0
 -- b1/b.go --
@@ -116,7 +154,7 @@ import _ "example.com/c"
 -- b2/go.mod --
 module example.com/b
 
-go 1.15
+go 1.17
 
 require example.com/c v0.1.0
 -- b2/b.go --
@@ -128,7 +166,7 @@ import _ "example.com/c"
 -- b3/go.mod --
 module example.com/b
 
-go 1.15
+go 1.17
 
 require example.com/c v0.2.0
 -- b3/b.go --
@@ -140,6 +178,6 @@ import _ "example.com/c"
 -- c/go.mod --
 module example.com/c
 
-go 1.15
+go 1.17
 -- c/c.go --
 package c

src/cmd/go/testdata/script/mod_lazy_import_allmod.txt

@@ -53,8 +53,8 @@ stdout '^c v0.1.0 '
 
 cp m.go.orig m.go
 cp go.mod.orig go.mod
-go mod edit -go=1.16
-go mod edit -go=1.16 go.mod.new
+go mod edit -go=1.17
+go mod edit -go=1.17 go.mod.new
 
 cp go.mod go.mod.orig
 go mod tidy
@@ -63,14 +63,15 @@ cmp go.mod.orig go.mod
 go list -m all
 stdout '^a v0.1.0 '
 stdout '^b v0.1.0 '
-stdout '^c v0.1.0 '  # TODO(#36460): This should be pruned out.
+! stdout '^c '
 
-# After adding a new import of b/y,
-# the import of c from b/y should again resolve to the version required by b.
+# After adding a new direct import of b/y,
+# the existing verison of b should be promoted to a root,
+# bringing the version of c required by b into the build list.
 
 cp m.go.new m.go
 go mod tidy
-cmp go.mod.new go.mod
+cmp go.mod.lazy go.mod
 
 go list -m all
 stdout '^a v0.1.0 '
@@ -124,6 +125,23 @@ require (
 	b v0.1.0
 )
 
+replace (
+	a v0.1.0 => ./a1
+	b v0.1.0 => ./b1
+	c v0.1.0 => ./c1
+	c v0.2.0 => ./c2
+)
+-- go.mod.lazy --
+module m
+
+go 1.17
+
+require (
+	a v0.1.0
+	b v0.1.0
+	c v0.1.0 // indirect
+)
+
 replace (
 	a v0.1.0 => ./a1
 	b v0.1.0 => ./b1
@@ -133,7 +151,7 @@ replace (
 -- a1/go.mod --
 module a
 
-go 1.16
+go 1.17
 
 require b v0.1.0
 -- a1/a.go --
@@ -145,7 +163,7 @@ import _ "b/x"
 -- b1/go.mod --
 module b
 
-go 1.16
+go 1.17
 
 require c v0.1.0
 -- b1/x/x.go --
@@ -161,7 +179,7 @@ func CVersion() string {
 -- c1/go.mod --
 module c
 
-go 1.16
+go 1.17
 -- c1/c.go --
 package c