gopls/internal/lsp/cache: add dynamic build tag support

This CL leverages the new zero-config implementation to add support for
dynamic build tags (golang/go#29202). While this CL is large, the gist
of the change is relatively simple:
 - in bestView, check that the view GOOS/GOARCH actually matches the
   file
 - in defineView, loop through supported ports to find one that matches
   the file, and apply the necessary GOOS= GOARCH= env overlay
 - detect that views must be re-selected whenever a build constraint
   changes

Everything else in the CL is supporting / refactoring around this minor
adjustment to view selection. Notably, the logic to check whether a file
matches a port (using go/build) required some care, because the go/build
API is cumbersome and not particularly efficient. We therefore check
ports as little as possible, and trim the file content that is passed
into build.Context.MatchFile. Earlier attempts at this change were
simpler, because they simply matched all available ports all the time,
but this had significant cost (around a millisecond overhead added to
every operation, including change processing).

However, the good news is that with the logic as it is, I believe it is
safe to support all available ports, because we only loop through this
list when checking views, an infrequent operation.

For golang/go#57979
For golang/go#29202

Change-Id: Ib654e18038dda74164b57d51b2d5274f91a1306d
Reviewed-on: https://go-review.googlesource.com/c/tools/+/551897
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Alan Donovan <adonovan@google.com>
LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
Run-TryBot: Robert Findley <rfindley@google.com>

gopls/doc/commands.md

@@ -593,6 +593,7 @@ Result:
 	"Type": string,
 	"Root": string,
 	"Folder": string,
+	"EnvOverlay": []string,
 }
 ```
 

gopls/internal/lsp/cache/port.go

@@ -0,0 +1,204 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"bytes"
+	"go/build"
+	"go/parser"
+	"go/token"
+	"io"
+	"path/filepath"
+	"strings"
+
+	"golang.org/x/tools/gopls/internal/util/bug"
+)
+
+type port struct{ GOOS, GOARCH string }
+
+var (
+	// preferredPorts holds GOOS/GOARCH combinations for which we dynamically
+	// create new Views, by setting GOOS=... and GOARCH=... on top of
+	// user-provided configuration when we detect that the default build
+	// configuration does not match an open file. Ports are matched in the order
+	// defined below, so that when multiple ports match a file we use the port
+	// occurring at a lower index in the slice. For that reason, we sort first
+	// class ports ahead of secondary ports, and (among first class ports) 64-bit
+	// ports ahead of the less common 32-bit ports.
+	preferredPorts = []port{
+		// First class ports, from https://go.dev/wiki/PortingPolicy.
+		{"darwin", "amd64"},
+		{"darwin", "arm64"},
+		{"linux", "amd64"},
+		{"linux", "arm64"},
+		{"windows", "amd64"},
+		{"linux", "arm"},
+		{"linux", "386"},
+		{"windows", "386"},
+
+		// Secondary ports, from GOROOT/src/internal/platform/zosarch.go.
+		// (First class ports are commented out.)
+		{"aix", "ppc64"},
+		{"dragonfly", "amd64"},
+		{"freebsd", "386"},
+		{"freebsd", "amd64"},
+		{"freebsd", "arm"},
+		{"freebsd", "arm64"},
+		{"illumos", "amd64"},
+		{"linux", "ppc64"},
+		{"linux", "ppc64le"},
+		{"linux", "mips"},
+		{"linux", "mipsle"},
+		{"linux", "mips64"},
+		{"linux", "mips64le"},
+		{"linux", "riscv64"},
+		{"linux", "s390x"},
+		{"android", "386"},
+		{"android", "amd64"},
+		{"android", "arm"},
+		{"android", "arm64"},
+		{"ios", "arm64"},
+		{"ios", "amd64"},
+		{"js", "wasm"},
+		{"netbsd", "386"},
+		{"netbsd", "amd64"},
+		{"netbsd", "arm"},
+		{"netbsd", "arm64"},
+		{"openbsd", "386"},
+		{"openbsd", "amd64"},
+		{"openbsd", "arm"},
+		{"openbsd", "arm64"},
+		{"openbsd", "mips64"},
+		{"plan9", "386"},
+		{"plan9", "amd64"},
+		{"plan9", "arm"},
+		{"solaris", "amd64"},
+		{"windows", "arm"},
+		{"windows", "arm64"},
+
+		{"aix", "ppc64"},
+		{"android", "386"},
+		{"android", "amd64"},
+		{"android", "arm"},
+		{"android", "arm64"},
+		// {"darwin", "amd64"},
+		// {"darwin", "arm64"},
+		{"dragonfly", "amd64"},
+		{"freebsd", "386"},
+		{"freebsd", "amd64"},
+		{"freebsd", "arm"},
+		{"freebsd", "arm64"},
+		{"freebsd", "riscv64"},
+		{"illumos", "amd64"},
+		{"ios", "amd64"},
+		{"ios", "arm64"},
+		{"js", "wasm"},
+		// {"linux", "386"},
+		// {"linux", "amd64"},
+		// {"linux", "arm"},
+		// {"linux", "arm64"},
+		{"linux", "loong64"},
+		{"linux", "mips"},
+		{"linux", "mips64"},
+		{"linux", "mips64le"},
+		{"linux", "mipsle"},
+		{"linux", "ppc64"},
+		{"linux", "ppc64le"},
+		{"linux", "riscv64"},
+		{"linux", "s390x"},
+		{"linux", "sparc64"},
+		{"netbsd", "386"},
+		{"netbsd", "amd64"},
+		{"netbsd", "arm"},
+		{"netbsd", "arm64"},
+		{"openbsd", "386"},
+		{"openbsd", "amd64"},
+		{"openbsd", "arm"},
+		{"openbsd", "arm64"},
+		{"openbsd", "mips64"},
+		{"openbsd", "ppc64"},
+		{"openbsd", "riscv64"},
+		{"plan9", "386"},
+		{"plan9", "amd64"},
+		{"plan9", "arm"},
+		{"solaris", "amd64"},
+		{"wasip1", "wasm"},
+		// {"windows", "386"},
+		// {"windows", "amd64"},
+		{"windows", "arm"},
+		{"windows", "arm64"},
+	}
+)
+
+// matches reports whether the port matches a file with the given absolute path
+// and content.
+//
+// Note that this function accepts content rather than e.g. a file.Handle,
+// because we trim content before matching for performance reasons, and
+// therefore need to do this outside of matches when considering multiple ports.
+func (p port) matches(path string, content []byte) bool {
+	ctxt := build.Default // make a copy
+	ctxt.UseAllFiles = false
+	dir, name := filepath.Split(path)
+
+	// The only virtualized operation called by MatchFile is OpenFile.
+	ctxt.OpenFile = func(p string) (io.ReadCloser, error) {
+		if p != path {
+			return nil, bug.Errorf("unexpected file %q", p)
+		}
+		return io.NopCloser(bytes.NewReader(content)), nil
+	}
+
+	ctxt.GOOS = p.GOOS
+	ctxt.GOARCH = p.GOARCH
+	ok, err := ctxt.MatchFile(dir, name)
+	return err == nil && ok
+}
+
+// trimContentForPortMatch trims the given Go file content to a minimal file
+// containing the same build constraints, if any.
+//
+// This is an unfortunate but necessary optimization, as matching build
+// constraints using go/build has significant overhead, and involves parsing
+// more than just the build constraint.
+//
+// TestMatchingPortsConsistency enforces consistency by comparing results
+// without trimming content.
+func trimContentForPortMatch(content []byte) []byte {
+	buildComment := buildComment(content)
+	return []byte(buildComment + "\npackage p") // package name does not matter
+}
+
+// buildComment returns the first matching //go:build comment in the given
+// content, or "" if none exists.
+func buildComment(content []byte) string {
+	f, err := parser.ParseFile(token.NewFileSet(), "", content, parser.PackageClauseOnly|parser.ParseComments)
+	if err != nil {
+		return ""
+	}
+
+	for _, cg := range f.Comments {
+		for _, c := range cg.List {
+			if isGoBuildComment(c.Text) {
+				return c.Text
+			}
+		}
+	}
+	return ""
+}
+
+// Adapted from go/build/build.go.
+//
+// TODO(rfindley): use constraint.IsGoBuild once we are on 1.19+.
+func isGoBuildComment(line string) bool {
+	const goBuildComment = "//go:build"
+	if !strings.HasPrefix(line, goBuildComment) {
+		return false
+	}
+	// Report whether //go:build is followed by a word boundary.
+	line = strings.TrimSpace(line)
+	rest := line[len(goBuildComment):]
+	return len(rest) == 0 || len(strings.TrimSpace(rest)) < len(rest)
+}

gopls/internal/lsp/cache/port_test.go

@@ -0,0 +1,126 @@
+// Copyright 2023 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+	"os"
+	"testing"
+
+	"github.com/google/go-cmp/cmp"
+	"golang.org/x/sync/errgroup"
+	"golang.org/x/tools/go/packages"
+	"golang.org/x/tools/gopls/internal/file"
+	"golang.org/x/tools/gopls/internal/lsp/protocol"
+	"golang.org/x/tools/gopls/internal/util/bug"
+	"golang.org/x/tools/internal/testenv"
+)
+
+func TestMain(m *testing.M) {
+	bug.PanicOnBugs = true
+	os.Exit(m.Run())
+}
+
+func TestMatchingPortsStdlib(t *testing.T) {
+	// This test checks that we don't encounter a bug when matching ports, and
+	// sanity checks that the optimization to use trimmed/fake file content
+	// before delegating to go/build.Context.MatchFile does not affect
+	// correctness.
+	if testing.Short() {
+		t.Skip("skipping in short mode: takes to long on slow file systems")
+	}
+
+	testenv.NeedsTool(t, "go")
+
+	// Load, parse and type-check the program.
+	cfg := &packages.Config{
+		Mode:  packages.LoadFiles,
+		Tests: true,
+	}
+	pkgs, err := packages.Load(cfg, "std", "cmd")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	var g errgroup.Group
+	packages.Visit(pkgs, nil, func(pkg *packages.Package) {
+		for _, f := range pkg.CompiledGoFiles {
+			f := f
+			g.Go(func() error {
+				content, err := os.ReadFile(f)
+				// We report errors via t.Error, not by returning,
+				// so that a single test can report multiple test failures.
+				if err != nil {
+					t.Errorf("failed to read %s: %v", f, err)
+					return nil
+				}
+				fh := makeFakeFileHandle(protocol.URIFromPath(f), content)
+				fastPorts := matchingPreferredPorts(t, fh, true)
+				slowPorts := matchingPreferredPorts(t, fh, false)
+				if diff := cmp.Diff(fastPorts, slowPorts); diff != "" {
+					t.Errorf("%s: ports do not match (-trimmed +untrimmed):\n%s", f, diff)
+					return nil
+				}
+				return nil
+			})
+		}
+	})
+	g.Wait()
+}
+
+func matchingPreferredPorts(tb testing.TB, fh file.Handle, trimContent bool) map[port]unit {
+	content, err := fh.Content()
+	if err != nil {
+		tb.Fatal(err)
+	}
+	if trimContent {
+		content = trimContentForPortMatch(content)
+	}
+	path := fh.URI().Path()
+	matching := make(map[port]unit)
+	for _, port := range preferredPorts {
+		if port.matches(path, content) {
+			matching[port] = unit{}
+		}
+	}
+	return matching
+}
+
+func BenchmarkMatchingPreferredPorts(b *testing.B) {
+	// Copy of robustio_posix.go
+	const src = `
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build !windows && !plan9
+// +build !windows,!plan9
+
+// TODO(adonovan): use 'unix' tag when go1.19 can be assumed.
+
+package robustio
+
+import (
+	"os"
+	"syscall"
+	"time"
+)
+
+func getFileID(filename string) (FileID, time.Time, error) {
+	fi, err := os.Stat(filename)
+	if err != nil {
+		return FileID{}, time.Time{}, err
+	}
+	stat := fi.Sys().(*syscall.Stat_t)
+	return FileID{
+		device: uint64(stat.Dev), // (int32 on darwin, uint64 on linux)
+		inode:  stat.Ino,
+	}, fi.ModTime(), nil
+}
+`
+	fh := makeFakeFileHandle("file:///path/to/test/file.go", []byte(src))
+	for i := 0; i < b.N; i++ {
+		_ = matchingPreferredPorts(b, fh, true)
+	}
+}