cmd/pkgsite: more changes to use local stdlib

This cl makes more changes to enable using the local stdlib. It always
uses the local stdlib from cmd/pkgsite. It also changes code that
processes and shows versions to expect and pass through v0.0.0, the
fake version we use when we show local packages.

For #57742

Change-Id: Ic407e58e9658b0703dbc6df47f40bbe1f1bd2b2a
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/517915
kokoro-CI: kokoro <noreply+kokoro@google.com>
Run-TryBot: Michael Matloob <matloob@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>

cmd/pkgsite/main.go

@@ -110,6 +110,7 @@ func main() {
 	flag.BoolVar(&serverCfg.useCache, "cache", false, "fetch from the module cache")
 	flag.StringVar(&serverCfg.cacheDir, "cachedir", "", "module cache directory (defaults to `go env GOMODCACHE`)")
 	flag.BoolVar(&serverCfg.useListedMods, "list", true, "for each path, serve all modules in build list")
+	serverCfg.useLocalStdlib = true
 
 	flag.Usage = func() {
 		out := flag.CommandLine.Output()
@@ -380,11 +381,6 @@ func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter,
 		getters = append(getters, g)
 	}
 
-	// Add a proxy
-	if cfg.proxy != nil {
-		getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(time.Second)))
-	}
-
 	if cfg.useLocalStdlib {
 		goRepo := *goRepoPath
 		if goRepo == "" {
@@ -398,6 +394,11 @@ func buildGetters(ctx context.Context, cfg getterConfig) ([]fetch.ModuleGetter,
 		}
 	}
 
+	// Add a proxy
+	if cfg.proxy != nil {
+		getters = append(getters, fetch.NewProxyModuleGetter(cfg.proxy, source.NewClient(time.Second)))
+	}
+
 	getters = append(getters, fetch.NewStdlibZipModuleGetter())
 
 	return getters, nil

cmd/pkgsite/main_test.go

@@ -56,13 +56,12 @@ package a
 
 	cfg := func(modifyDefault func(*serverConfig)) serverConfig {
 		c := serverConfig{
-			paths:          []string{localModule},
-			gopathMode:     false,
-			useListedMods:  true,
-			useLocalStdlib: true,
-			useCache:       true,
-			cacheDir:       cacheDir,
-			proxy:          prox,
+			paths:         []string{localModule},
+			gopathMode:    false,
+			useListedMods: true,
+			useCache:      true,
+			cacheDir:      cacheDir,
+			proxy:         prox,
 		}
 		if modifyDefault != nil {
 			modifyDefault(&c)

internal/fetch/fetch.go

@@ -187,8 +187,10 @@ func processModuleContents(ctx context.Context, modulePath, resolvedVersion, req
 	defer span.End()
 
 	v := resolvedVersion
-	if modulePath == stdlib.ModulePath && stdlib.SupportedBranches[requestedVersion] {
-		v = requestedVersion
+	if _, ok := mg.(*stdlibZipModuleGetter); ok {
+		if modulePath == stdlib.ModulePath && stdlib.SupportedBranches[requestedVersion] {
+			v = requestedVersion
+		}
 	}
 	sourceInfo, err := mg.SourceInfo(ctx, modulePath, v)
 	if err != nil {

internal/fetchdatasource/fetchdatasource.go

@@ -23,7 +23,6 @@ import (
 	"golang.org/x/pkgsite/internal/fetch"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/proxy"
-	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
 )
 
@@ -166,15 +165,6 @@ func (ds *FetchDataSource) fetch(ctx context.Context, modulePath, version string
 			} else {
 				m.RemoveNonRedistributableData()
 			}
-			// There is special handling in FetchModule for the standard library,
-			// that bypasses the getter g. Don't record g as having fetch std.
-			//
-			// TODO(rfindley): it would be cleaner if the standard library could be
-			// its own module getter. This could also allow the go/packages getter to
-			// serve existing on-disk content for std. See also golang/go#58923.
-			if modulePath == stdlib.ModulePath {
-				g = nil
-			}
 			return m, g, nil
 		}
 		if !errors.Is(fr.Error, derrors.NotFound) {
@@ -187,7 +177,7 @@ func (ds *FetchDataSource) fetch(ctx context.Context, modulePath, version string
 func (ds *FetchDataSource) populateUnitSubdirectories(u *internal.Unit, m *internal.Module) {
 	p := u.Path + "/"
 	for _, u2 := range m.Units {
-		if strings.HasPrefix(u2.Path, p) {
+		if strings.HasPrefix(u2.Path, p) || u.Path == "std" {
 			var syn string
 			if len(u2.Documentation) > 0 {
 				syn = u2.Documentation[0].Synopsis

internal/frontend/urlinfo.go

@@ -15,6 +15,7 @@ import (
 	"golang.org/x/pkgsite/internal"
 	"golang.org/x/pkgsite/internal/derrors"
 	"golang.org/x/pkgsite/internal/experiment"
+	"golang.org/x/pkgsite/internal/fetch"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
@@ -155,6 +156,11 @@ func parseStdLibURLPath(urlPath string) (_ *urlPathInfo, err error) {
 	tag = strings.TrimSuffix(tag, "/")
 	info.requestedVersion = stdlib.VersionForTag(tag)
 	if info.requestedVersion == "" {
+		if tag == fetch.LocalVersion {
+			// Special case: 0.0.0 is the version for a local stdlib
+			info.requestedVersion = fetch.LocalVersion
+			return info, nil
+		}
 		return nil, &userError{
 			err:         fmt.Errorf("invalid Go tag for url: %q", urlPath),
 			userMessage: fmt.Sprintf("%q is not a valid tag for the standard library", tag),

internal/frontend/versions.go

@@ -14,6 +14,7 @@ import (
 
 	"golang.org/x/mod/semver"
 	"golang.org/x/pkgsite/internal"
+	"golang.org/x/pkgsite/internal/fetch"
 	"golang.org/x/pkgsite/internal/log"
 	"golang.org/x/pkgsite/internal/stdlib"
 	"golang.org/x/pkgsite/internal/version"
@@ -370,7 +371,7 @@ func pseudoVersionRev(v string) string {
 
 // displayVersion returns the version string, formatted for display.
 func displayVersion(modulePath, requestedVersion, resolvedVersion string) string {
-	if modulePath == stdlib.ModulePath {
+	if modulePath == stdlib.ModulePath && resolvedVersion != fetch.LocalVersion {
 		if stdlib.SupportedBranches[requestedVersion] ||
 			(strings.HasPrefix(resolvedVersion, "v0.0.0") && resolvedVersion != "v0.0.0") { // Plain v0.0.0 is from the go packages module getter
 			commit := strings.Split(resolvedVersion, "-")[2]
@@ -394,7 +395,7 @@ func displayVersion(modulePath, requestedVersion, resolvedVersion string) string
 // a link to this site.
 // See TestLinkVersion for examples.
 func linkVersion(modulePath, requestedVersion, resolvedVersion string) string {
-	if modulePath == stdlib.ModulePath {
+	if modulePath == stdlib.ModulePath && resolvedVersion != fetch.LocalVersion {
 		if strings.HasPrefix(resolvedVersion, "go") {
 			return resolvedVersion // already a go version
 		}