From d463eb0e41917354246b72b3a0b0ec3a0a595a5e Mon Sep 17 00:00:00 2001 From: Rob Findley Date: Sat, 10 Oct 2020 09:04:36 -0400 Subject: [PATCH] internal/lsp/cache: introduce a workspace abstraction When incorporating the gopls.mod file, the invalidation logic for workspace module information becomes quite complicated. For example: + if a modfile changes we should only invalidate if it is in the set of active modules + the set of active modules can be provided by either the filesystem or gopls.mod + if gopls.mod changes, we may gain or lose active modules in the workspace + if gopls.mod is *deleted*, we may need to walk the filesystem to actually find all active modules Doing this with only concrete changes to the snapshot proved prohibitively complex (at least for me), so a new workspace type is introduced to manage the module state. This new abstraction is responsible for tracking the set of active modules, the workspace modfile, and the set of workspace directories. Its invalidation logic is factored out of snapshot.clone, so that it can be tested and to alleviate some of the growing complexity of snapshot.clone. The workspace type is idempotent, allowing it to be shared across snapshots without needing to use the cache. There is little benefit to the cache in this case, since workspace module computation should be infrequent, and the type itself consumes little memory. This is made possible because the workspace type itself depends only on file state, and therefore may be invalidated independently of the snapshot. The new source.FileState interface is used in testing, and so that the workspace module may be computed based on both the session file state as well as the snapshot file state. As a result of this change, the following improvements in functionality are possible: + in the presence of a gopls.mod file, we avoid walking the filesystem to detect modules. This could be helpful for working in large monorepos or in GOPATH, when discovering all modules would be expensive. + The set of active modules should always be consistent with the gopls.mod file, likely fixing some bugs (for example, computing diagnostics for modfiles that shouldn't be loaded) For golang/go#41837 Change-Id: I2da888c097748b659ee892ca2d6b3fbe29c1942e Reviewed-on: https://go-review.googlesource.com/c/tools/+/261237 Run-TryBot: Robert Findley gopls-CI: kokoro TryBot-Result: Go Bot Reviewed-by: Heschi Kreinick Trust: Robert Findley --- gopls/internal/regtest/workspace_test.go | 64 +++- gopls/internal/regtest/wrappers.go | 7 + internal/lsp/cache/cache.go | 23 +- internal/lsp/cache/imports.go | 31 +- internal/lsp/cache/load.go | 7 +- internal/lsp/cache/mod.go | 14 +- internal/lsp/cache/session.go | 95 ++---- internal/lsp/cache/snapshot.go | 402 ++++++---------------- internal/lsp/cache/view.go | 64 ++-- internal/lsp/cache/workspace.go | 409 +++++++++++++++++++++++ internal/lsp/cache/workspace_test.go | 257 ++++++++++++++ internal/lsp/command.go | 3 +- internal/lsp/fake/editor.go | 27 +- internal/lsp/source/view.go | 4 - 14 files changed, 936 insertions(+), 471 deletions(-) create mode 100644 internal/lsp/cache/workspace.go create mode 100644 internal/lsp/cache/workspace_test.go diff --git a/gopls/internal/regtest/workspace_test.go b/gopls/internal/regtest/workspace_test.go index 4e9559dba62..9eaff5596e3 100644 --- a/gopls/internal/regtest/workspace_test.go +++ b/gopls/internal/regtest/workspace_test.go @@ -158,6 +158,16 @@ replace random.org => %s } const workspaceModuleProxy = ` +-- example.com@v1.2.3/go.mod -- +module example.com + +go 1.12 +-- example.com@v1.2.3/blah/blah.go -- +package blah + +func SaySomething() { + fmt.Println("something") +} -- b.com@v1.2.3/go.mod -- module b.com @@ -364,6 +374,9 @@ func Hello() int { } func TestUseGoplsMod(t *testing.T) { + // This test validates certain functionality related to using a gopls.mod + // file to specify workspace modules. + testenv.NeedsGo1Point(t, 14) const multiModule = ` -- moda/a/go.mod -- module a.com @@ -384,6 +397,7 @@ func main() { -- modb/go.mod -- module b.com +require example.com v1.2.3 -- modb/b/b.go -- package b @@ -404,12 +418,18 @@ replace a.com => $SANDBOX_WORKDIR/moda/a WithProxyFiles(workspaceModuleProxy), WithModes(Experimental), ).run(t, multiModule, func(t *testing.T, env *Env) { + // Initially, the gopls.mod should cause only the a.com module to be + // loaded. Validate this by jumping to a definition in b.com and ensuring + // that we go to the module cache. env.OpenFile("moda/a/a.go") - original, _ := env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) - if want := "b.com@v1.2.3/b/b.go"; !strings.HasSuffix(original, want) { - t.Errorf("expected %s, got %v", want, original) + location, _ := env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) + if want := "b.com@v1.2.3/b/b.go"; !strings.HasSuffix(location, want) { + t.Errorf("expected %s, got %v", want, location) } workdir := env.Sandbox.Workdir.RootURI().SpanURI().Filename() + + // Now, modify the gopls.mod file on disk to activate the b.com module in + // the workspace. env.WriteWorkspaceFile("gopls.mod", fmt.Sprintf(`module gopls-workspace require ( @@ -426,9 +446,41 @@ replace b.com => %s/modb env.DiagnosticAtRegexp("modb/b/b.go", "x"), ), ) - newLocation, _ := env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) - if want := "modb/b/b.go"; !strings.HasSuffix(newLocation, want) { - t.Errorf("expected %s, got %v", want, newLocation) + env.OpenFile("modb/go.mod") + // Check that go.mod diagnostics picked up the newly active mod file. + env.Await(env.DiagnosticAtRegexp("modb/go.mod", `require example.com v1.2.3`)) + // ...and that jumping to definition now goes to b.com in the workspace. + location, _ = env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) + if want := "modb/b/b.go"; !strings.HasSuffix(location, want) { + t.Errorf("expected %s, got %v", want, location) + } + + // Now, let's modify the gopls.mod *overlay* (not on disk), and verify that + // this change is also picked up. + env.OpenFile("gopls.mod") + env.SetBufferContent("gopls.mod", fmt.Sprintf(`module gopls-workspace + +require ( + a.com v0.0.0-goplsworkspace +) + +replace a.com => %s/moda/a +`, workdir)) + env.Await(CompletedWork(lsp.DiagnosticWorkTitle(lsp.FromDidChange), 1)) + // TODO: diagnostics are not being cleared from the old go.mod location, + // because it's not treated as a 'deleted' file. Uncomment this after + // fixing. + /* + env.Await(OnceMet( + CompletedWork(lsp.DiagnosticWorkTitle(lsp.FromDidChange), 1), + EmptyDiagnostics("modb/go.mod"), + )) + */ + + // Just as before, check that we now jump to the module cache. + location, _ = env.GoToDefinition("moda/a/a.go", env.RegexpSearch("moda/a/a.go", "Hello")) + if want := "b.com@v1.2.3/b/b.go"; !strings.HasSuffix(location, want) { + t.Errorf("expected %s, got %v", want, location) } }) } diff --git a/gopls/internal/regtest/wrappers.go b/gopls/internal/regtest/wrappers.go index 53f79527d87..d7ca87fbe87 100644 --- a/gopls/internal/regtest/wrappers.go +++ b/gopls/internal/regtest/wrappers.go @@ -92,6 +92,13 @@ func (e *Env) EditBuffer(name string, edits ...fake.Edit) { } } +func (e *Env) SetBufferContent(name string, content string) { + e.T.Helper() + if err := e.Editor.SetBufferContent(e.Ctx, name, content); err != nil { + e.T.Fatal(err) + } +} + // RegexpRange returns the range of the first match for re in the buffer // specified by name, calling t.Fatal on any error. It first searches for the // position in open buffers, then in workspace files. diff --git a/internal/lsp/cache/cache.go b/internal/lsp/cache/cache.go index c2b4fe52f21..5e6180d73f8 100644 --- a/internal/lsp/cache/cache.go +++ b/internal/lsp/cache/cache.go @@ -79,14 +79,10 @@ func (c *Cache) getFile(ctx context.Context, uri span.URI) (*fileHandle, error) return fh, nil } - select { - case ioLimit <- struct{}{}: - case <-ctx.Done(): - return nil, ctx.Err() + fh, err := readFile(ctx, uri, fi.ModTime()) + if err != nil { + return nil, err } - defer func() { <-ioLimit }() - - fh = readFile(ctx, uri, fi.ModTime()) c.fileMu.Lock() c.fileContent[uri] = fh c.fileMu.Unlock() @@ -96,7 +92,14 @@ func (c *Cache) getFile(ctx context.Context, uri span.URI) (*fileHandle, error) // ioLimit limits the number of parallel file reads per process. var ioLimit = make(chan struct{}, 128) -func readFile(ctx context.Context, uri span.URI, modTime time.Time) *fileHandle { +func readFile(ctx context.Context, uri span.URI, modTime time.Time) (*fileHandle, error) { + select { + case ioLimit <- struct{}{}: + case <-ctx.Done(): + return nil, ctx.Err() + } + defer func() { <-ioLimit }() + ctx, done := event.Start(ctx, "cache.readFile", tag.File.Of(uri.Filename())) _ = ctx defer done() @@ -106,14 +109,14 @@ func readFile(ctx context.Context, uri span.URI, modTime time.Time) *fileHandle return &fileHandle{ modTime: modTime, err: err, - } + }, nil } return &fileHandle{ modTime: modTime, uri: uri, bytes: data, hash: hashContents(data), - } + }, nil } func (c *Cache) NewSession(ctx context.Context) *Session { diff --git a/internal/lsp/cache/imports.go b/internal/lsp/cache/imports.go index 5039eac520a..48216ab1d3c 100644 --- a/internal/lsp/cache/imports.go +++ b/internal/lsp/cache/imports.go @@ -33,34 +33,31 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot // Use temporary go.mod files, but always go to disk for the contents. // Rebuilding the cache is expensive, and we don't want to do it for // transient changes. - var modFH, sumFH source.FileHandle + var modFH source.FileHandle + var gosum []byte var modFileIdentifier string var err error - // TODO(heschik): Change the goimports logic to use a persistent workspace + // TODO(rfindley): Change the goimports logic to use a persistent workspace // module for workspace module mode. // // Get the go.mod file that corresponds to this view's root URI. This is // broken because it assumes that the view's root is a module, but this is // not more broken than the previous state--it is a temporary hack that // should be removed ASAP. - var match *moduleRoot - for _, m := range snapshot.modules { - if m.rootURI == snapshot.view.rootURI { - match = m + var matchURI span.URI + for modURI := range snapshot.workspace.activeModFiles() { + if dirURI(modURI) == snapshot.view.rootURI { + matchURI = modURI } } - if match != nil { - modFH, err = snapshot.GetFile(ctx, match.modURI) + // TODO(rFindley): should it be an error if matchURI is empty? + if matchURI != "" { + modFH, err = snapshot.GetFile(ctx, matchURI) if err != nil { return err } modFileIdentifier = modFH.FileIdentity().Hash - if match.sumURI != "" { - sumFH, err = snapshot.GetFile(ctx, match.sumURI) - if err != nil { - return err - } - } + gosum = snapshot.goSum(ctx, matchURI) } // v.goEnv is immutable -- changes make a new view. Options can change. // We can't compare build flags directly because we may add -modfile. @@ -87,7 +84,7 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot } s.cachedModFileIdentifier = modFileIdentifier s.cachedBuildFlags = currentBuildFlags - s.cleanupProcessEnv, err = s.populateProcessEnv(ctx, snapshot, modFH, sumFH) + s.cleanupProcessEnv, err = s.populateProcessEnv(ctx, snapshot, modFH, gosum) if err != nil { return err } @@ -125,7 +122,7 @@ func (s *importsState) runProcessEnvFunc(ctx context.Context, snapshot *snapshot // populateProcessEnv sets the dynamically configurable fields for the view's // process environment. Assumes that the caller is holding the s.view.importsMu. -func (s *importsState) populateProcessEnv(ctx context.Context, snapshot *snapshot, modFH, sumFH source.FileHandle) (cleanup func(), err error) { +func (s *importsState) populateProcessEnv(ctx context.Context, snapshot *snapshot, modFH source.FileHandle, gosum []byte) (cleanup func(), err error) { cleanup = func() {} pe := s.processEnv @@ -166,7 +163,7 @@ func (s *importsState) populateProcessEnv(ctx context.Context, snapshot *snapsho // Add -modfile to the build flags, if we are using it. if snapshot.workspaceMode()&tempModfile != 0 && modFH != nil { var tmpURI span.URI - tmpURI, cleanup, err = tempModFile(modFH, sumFH) + tmpURI, cleanup, err = tempModFile(modFH, gosum) if err != nil { return nil, err } diff --git a/internal/lsp/cache/load.go b/internal/lsp/cache/load.go index 0d07070df0f..f5bb720197d 100644 --- a/internal/lsp/cache/load.go +++ b/internal/lsp/cache/load.go @@ -191,14 +191,11 @@ func (s *snapshot) tempWorkspaceModule(ctx context.Context) (_ span.URI, cleanup if s.workspaceMode()&usesWorkspaceModule == 0 { return "", cleanup, nil } - wsModuleHandle, err := s.getWorkspaceModuleHandle(ctx) - if err != nil { - return "", nil, err - } - file, err := wsModuleHandle.build(ctx, s) + file, err := s.workspace.modFile(ctx, s) if err != nil { return "", nil, err } + content, err := file.Format() if err != nil { return "", cleanup, err diff --git a/internal/lsp/cache/mod.go b/internal/lsp/cache/mod.go index 6d84c501c23..0e4ed4e3dd7 100644 --- a/internal/lsp/cache/mod.go +++ b/internal/lsp/cache/mod.go @@ -98,24 +98,26 @@ func (s *snapshot) ParseMod(ctx context.Context, modFH source.FileHandle) (*sour return pmh.parse(ctx, s) } -func (s *snapshot) sumFH(ctx context.Context, modFH source.FileHandle) (source.FileHandle, error) { +// goSum reads the go.sum file for the go.mod file at modURI, if it exists. If +// it doesn't exist, it returns nil. +func (s *snapshot) goSum(ctx context.Context, modURI span.URI) []byte { // Get the go.sum file, either from the snapshot or directly from the // cache. Avoid (*snapshot).GetFile here, as we don't want to add // nonexistent file handles to the snapshot if the file does not exist. - sumURI := span.URIFromPath(sumFilename(modFH.URI())) + sumURI := span.URIFromPath(sumFilename(modURI)) var sumFH source.FileHandle = s.FindFile(sumURI) if sumFH == nil { var err error sumFH, err = s.view.session.cache.getFile(ctx, sumURI) if err != nil { - return nil, err + return nil } } - _, err := sumFH.Read() + content, err := sumFH.Read() if err != nil { - return nil, err + return nil } - return sumFH, nil + return content } func sumFilename(modURI span.URI) string { diff --git a/internal/lsp/cache/session.go b/internal/lsp/cache/session.go index 733c937215c..678e9c42de9 100644 --- a/internal/lsp/cache/session.go +++ b/internal/lsp/cache/session.go @@ -7,8 +7,6 @@ package cache import ( "context" "fmt" - "os" - "path/filepath" "strconv" "strings" "sync" @@ -171,10 +169,8 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, return nil, nil, func() {}, err } - // If workspace module mode is enabled, find all of the modules in the - // workspace. By default, we just find the root module. - var modules map[span.URI]*moduleRoot - modules, err = findWorkspaceModules(ctx, ws.rootURI, options) + // Build the gopls workspace, collecting active modules in the view. + workspace, err := newWorkspace(ctx, ws.rootURI, s, options.ExperimentalWorkspaceModule) if err != nil { return nil, nil, func() {}, err } @@ -225,11 +221,9 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, modTidyHandles: make(map[span.URI]*modTidyHandle), modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), modWhyHandles: make(map[span.URI]*modWhyHandle), - modules: modules, + workspace: workspace, } - v.snapshot.workspaceDirectories = v.snapshot.findWorkspaceDirectories(ctx) - // Initialize the view without blocking. initCtx, initCancel := context.WithCancel(xcontext.Detach(ctx)) v.initCancelFirstAttempt = initCancel @@ -242,52 +236,6 @@ func (s *Session) createView(ctx context.Context, name string, folder span.URI, return v, snapshot, snapshot.generation.Acquire(ctx), nil } -// findWorkspaceModules walks the view's root folder, looking for go.mod files. -// Any that are found are added to the view's set of modules, which are then -// used to construct the workspace module. -// -// It assumes that the caller has not yet created the view, and therefore does -// not lock any of the internal data structures before accessing them. -// -// TODO(rstambler): Check overlays for go.mod files. -func findWorkspaceModules(ctx context.Context, root span.URI, options *source.Options) (map[span.URI]*moduleRoot, error) { - // Walk the view's folder to find all modules in the view. - modules := make(map[span.URI]*moduleRoot) - if !options.ExperimentalWorkspaceModule { - path := filepath.Join(root.Filename(), "go.mod") - if info, _ := os.Stat(path); info != nil { - if m := getViewModule(ctx, root, span.URIFromPath(path), options); m != nil { - modules[m.rootURI] = m - } - } - return modules, nil - } - return modules, filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { - if err != nil { - // Probably a permission error. Keep looking. - return filepath.SkipDir - } - // For any path that is not the workspace folder, check if the path - // would be ignored by the go command. Vendor directories also do not - // contain workspace modules. - if info.IsDir() && path != root.Filename() { - suffix := strings.TrimPrefix(path, root.Filename()) - switch { - case checkIgnored(suffix), - strings.Contains(filepath.ToSlash(suffix), "/vendor/"): - return filepath.SkipDir - } - } - // We're only interested in go.mod files. - if filepath.Base(path) == "go.mod" { - if m := getViewModule(ctx, root, span.URIFromPath(path), options); m != nil { - modules[m.rootURI] = m - } - } - return nil - }) -} - // View returns the view by name. func (s *Session) View(name string) source.View { s.viewMu.Lock() @@ -439,8 +387,14 @@ func (s *Session) ModifyFiles(ctx context.Context, changes []source.FileModifica return err } +type fileChange struct { + content []byte + exists bool + fileHandle source.VersionedFileHandle +} + func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModification) (map[span.URI]source.View, map[source.View]source.Snapshot, []func(), []span.URI, error) { - views := make(map[*View]map[span.URI]source.VersionedFileHandle) + views := make(map[*View]map[span.URI]*fileChange) bestViews := map[span.URI]source.View{} // Keep track of deleted files so that we can clear their diagnostics. // A file might be re-created after deletion, so only mark files that @@ -485,23 +439,28 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif return nil, nil, nil, nil, err } if _, ok := views[view]; !ok { - views[view] = make(map[span.URI]source.VersionedFileHandle) + views[view] = make(map[span.URI]*fileChange) } - var ( - fh source.VersionedFileHandle - ok bool - ) - if fh, ok = overlays[c.URI]; ok { - views[view][c.URI] = fh + if fh, ok := overlays[c.URI]; ok { + views[view][c.URI] = &fileChange{ + content: fh.text, + exists: true, + fileHandle: fh, + } delete(deletions, c.URI) } else { fsFile, err := s.cache.getFile(ctx, c.URI) if err != nil { return nil, nil, nil, nil, err } - fh = &closedFile{fsFile} - views[view][c.URI] = fh - if _, err := fh.Read(); err != nil { + content, err := fsFile.Read() + fh := &closedFile{fsFile} + views[view][c.URI] = &fileChange{ + content: content, + exists: err == nil, + fileHandle: fh, + } + if err != nil { deletions[c.URI] = struct{}{} } } @@ -510,8 +469,8 @@ func (s *Session) DidModifyFiles(ctx context.Context, changes []source.FileModif snapshots := map[source.View]source.Snapshot{} var releases []func() - for view, uris := range views { - snapshot, release := view.invalidateContent(ctx, uris, forceReloadMetadata) + for view, changed := range views { + snapshot, release := view.invalidateContent(ctx, changed, forceReloadMetadata) snapshots[view] = snapshot releases = append(releases, release) } diff --git a/internal/lsp/cache/snapshot.go b/internal/lsp/cache/snapshot.go index f26772ec2c8..ff06817e215 100644 --- a/internal/lsp/cache/snapshot.go +++ b/internal/lsp/cache/snapshot.go @@ -78,10 +78,6 @@ type snapshot struct { // when the view is created. workspacePackages map[packageID]packagePath - // workspaceDirectories are the directories containing workspace packages. - // They are the view's root, as well as any replace targets. - workspaceDirectories map[span.URI]struct{} - // unloadableFiles keeps track of files that we've failed to load. unloadableFiles map[span.URI]struct{} @@ -96,12 +92,7 @@ type snapshot struct { modUpgradeHandles map[span.URI]*modUpgradeHandle modWhyHandles map[span.URI]*modWhyHandle - // modules is the set of modules currently in this workspace. - modules map[span.URI]*moduleRoot - - // workspaceModuleHandle keeps track of the in-memory representation of the - // go.mod file for the workspace module. - workspaceModuleHandle *workspaceModuleHandle + workspace *workspace } type packageKey struct { @@ -128,14 +119,14 @@ func (s *snapshot) FileSet() *token.FileSet { func (s *snapshot) ModFiles() []span.URI { var uris []span.URI - for _, m := range s.modules { - uris = append(uris, m.modURI) + for modURI := range s.workspace.activeModFiles() { + uris = append(uris, modURI) } return uris } func (s *snapshot) ValidBuildConfiguration() bool { - return validBuildConfiguration(s.view.rootURI, &s.view.workspaceInformation, s.modules) + return validBuildConfiguration(s.view.rootURI, &s.view.workspaceInformation, s.workspace.activeModFiles()) } // workspaceMode describes the way in which the snapshot's workspace should @@ -152,7 +143,7 @@ func (s *snapshot) workspaceMode() workspaceMode { // If the view is not in a module and contains no modules, but still has a // valid workspace configuration, do not create the workspace module. // It could be using GOPATH or a different build system entirely. - if len(s.modules) == 0 && validBuildConfiguration { + if len(s.workspace.activeModFiles()) == 0 && validBuildConfiguration { return mode } mode |= moduleMode @@ -255,11 +246,9 @@ func (s *snapshot) goCommandInvocation(ctx context.Context, mode source.Invocati // the passed-in working dir. if mode == source.ForTypeChecking { if s.workspaceMode()&usesWorkspaceModule == 0 { - var mod *moduleRoot - for _, m := range s.modules { // range to access the only element - mod = m + for m := range s.workspace.activeModFiles() { // range to access the only element + modURI = m } - modURI = mod.modURI } else { var tmpDir span.URI var err error @@ -291,9 +280,8 @@ func (s *snapshot) goCommandInvocation(ctx context.Context, mode source.Invocati return "", nil, cleanup, err } // Use the go.sum if it happens to be available. - sumFH, _ := s.sumFH(ctx, modFH) - - tmpURI, cleanup, err = tempModFile(modFH, sumFH) + gosum := s.goSum(ctx, modURI) + tmpURI, cleanup, err = tempModFile(modFH, gosum) if err != nil { return "", nil, cleanup, err } @@ -601,14 +589,7 @@ func (s *snapshot) workspacePackageIDs() (ids []packageID) { } func (s *snapshot) WorkspaceDirectories(ctx context.Context) []span.URI { - s.mu.Lock() - defer s.mu.Unlock() - - var dirs []span.URI - for d := range s.workspaceDirectories { - dirs = append(dirs, d) - } - return dirs + return s.workspace.dirs(ctx, s) } func (s *snapshot) WorkspacePackages(ctx context.Context) ([]source.Package, error) { @@ -684,12 +665,12 @@ func (s *snapshot) CachedImportPaths(ctx context.Context) (map[string]source.Pac func (s *snapshot) GoModForFile(ctx context.Context, uri span.URI) span.URI { var match span.URI - for _, m := range s.modules { - if !isSubdirectory(m.rootURI.Filename(), uri.Filename()) { + for modURI := range s.workspace.activeModFiles() { + if !isSubdirectory(dirURI(modURI).Filename(), uri.Filename()) { continue } - if len(m.modURI) > len(match) { - match = m.modURI + if len(modURI) > len(match) { + match = modURI } } return match @@ -810,7 +791,7 @@ func (s *snapshot) FindFile(uri span.URI) source.VersionedFileHandle { // GetVersionedFile returns a File for the given URI. If the file is unknown it // is added to the managed set. // -// GetFile succeeds even if the file does not exist. A non-nil error return +// GetVersionedFile succeeds even if the file does not exist. A non-nil error return // indicates some type of internal error, for example if ctx is cancelled. func (s *snapshot) GetVersionedFile(ctx context.Context, uri span.URI) (source.VersionedFileHandle, error) { f, err := s.view.getFile(uri) @@ -992,32 +973,32 @@ func generationName(v *View, snapshotID uint64) string { return fmt.Sprintf("v%v/%v", v.id, snapshotID) } -func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.VersionedFileHandle, forceReloadMetadata bool) (*snapshot, reinitializeView) { +func (s *snapshot) clone(ctx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (*snapshot, reinitializeView) { + newWorkspace, workspaceChanged := s.workspace.invalidate(ctx, changes) + s.mu.Lock() defer s.mu.Unlock() newGen := s.view.session.cache.store.Generation(generationName(s.view, s.id+1)) result := &snapshot{ - id: s.id + 1, - generation: newGen, - view: s.view, - builtin: s.builtin, - ids: make(map[span.URI][]packageID), - importedBy: make(map[packageID][]packageID), - metadata: make(map[packageID]*metadata), - packages: make(map[packageKey]*packageHandle), - actions: make(map[actionKey]*actionHandle), - files: make(map[span.URI]source.VersionedFileHandle), - goFiles: make(map[parseKey]*parseGoHandle), - workspaceDirectories: make(map[span.URI]struct{}), - workspacePackages: make(map[packageID]packagePath), - unloadableFiles: make(map[span.URI]struct{}), - parseModHandles: make(map[span.URI]*parseModHandle), - modTidyHandles: make(map[span.URI]*modTidyHandle), - modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), - modWhyHandles: make(map[span.URI]*modWhyHandle), - modules: make(map[span.URI]*moduleRoot), - workspaceModuleHandle: s.workspaceModuleHandle, + id: s.id + 1, + generation: newGen, + view: s.view, + builtin: s.builtin, + ids: make(map[span.URI][]packageID), + importedBy: make(map[packageID][]packageID), + metadata: make(map[packageID]*metadata), + packages: make(map[packageKey]*packageHandle), + actions: make(map[actionKey]*actionHandle), + files: make(map[span.URI]source.VersionedFileHandle), + goFiles: make(map[parseKey]*parseGoHandle), + workspacePackages: make(map[packageID]packagePath), + unloadableFiles: make(map[span.URI]struct{}), + parseModHandles: make(map[span.URI]*parseModHandle), + modTidyHandles: make(map[span.URI]*modTidyHandle), + modUpgradeHandles: make(map[span.URI]*modUpgradeHandle), + modWhyHandles: make(map[span.URI]*modWhyHandle), + workspace: newWorkspace, } if s.builtin != nil { @@ -1028,6 +1009,7 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve for k, v := range s.files { result.files[k] = v } + // Copy the set of unloadable files. for k, v := range s.unloadableFiles { result.unloadableFiles[k] = v @@ -1036,13 +1018,9 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve for k, v := range s.parseModHandles { result.parseModHandles[k] = v } - // Copy all of the workspace directories. They may be reset later. - for k, v := range s.workspaceDirectories { - result.workspaceDirectories[k] = v - } for k, v := range s.goFiles { - if _, ok := withoutURIs[k.file.URI]; ok { + if _, ok := changes[k.file.URI]; ok { continue } newGen.Inherit(v.handle) @@ -1053,53 +1031,55 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve // Copy all of the go.mod-related handles. They may be invalidated later, // so we inherit them at the end of the function. for k, v := range s.modTidyHandles { - if _, ok := withoutURIs[k]; ok { + if _, ok := changes[k]; ok { continue } result.modTidyHandles[k] = v } for k, v := range s.modUpgradeHandles { - if _, ok := withoutURIs[k]; ok { + if _, ok := changes[k]; ok { continue } result.modUpgradeHandles[k] = v } for k, v := range s.modWhyHandles { - if _, ok := withoutURIs[k]; ok { + if _, ok := changes[k]; ok { continue } result.modWhyHandles[k] = v } - // Add all of the modules now. They may be deleted or added to later. - for k, v := range s.modules { - result.modules[k] = v - } - - var modulesChanged, shouldReinitializeView bool + var reinitialize reinitializeView // directIDs keeps track of package IDs that have directly changed. // It maps id->invalidateMetadata. directIDs := map[packageID]bool{} - for withoutURI, currentFH := range withoutURIs { + // Invalidate all package metadata if the workspace module has changed. + if workspaceChanged { + reinitialize = definitelyReinit + for k := range s.metadata { + directIDs[k] = true + } + } + for uri, change := range changes { // The original FileHandle for this URI is cached on the snapshot. - originalFH := s.files[withoutURI] + originalFH := s.files[uri] // Check if the file's package name or imports have changed, // and if so, invalidate this file's packages' metadata. - invalidateMetadata := forceReloadMetadata || s.shouldInvalidateMetadata(ctx, result, originalFH, currentFH) + invalidateMetadata := forceReloadMetadata || s.shouldInvalidateMetadata(ctx, result, originalFH, change.fileHandle) // Mark all of the package IDs containing the given file. // TODO: if the file has moved into a new package, we should invalidate that too. - filePackages := guessPackagesForURI(withoutURI, s.ids) + filePackages := guessPackagesForURI(uri, s.ids) for _, id := range filePackages { directIDs[id] = directIDs[id] || invalidateMetadata } // Invalidate the previous modTidyHandle if any of the files have been // saved or if any of the metadata has been invalidated. - if invalidateMetadata || fileWasSaved(originalFH, currentFH) { + if invalidateMetadata || fileWasSaved(originalFH, change.fileHandle) { // TODO(rstambler): Only delete mod handles for which the // withoutURI is relevant. for k := range s.modTidyHandles { @@ -1112,75 +1092,22 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve delete(result.modWhyHandles, k) } } - currentExists := true - if _, err := currentFH.Read(); os.IsNotExist(err) { - currentExists = false - } - // If the file invalidation is for a go.mod. originalFH is nil if the - // file is newly created. - currentMod := currentExists && currentFH.Kind() == source.Mod - originalMod := originalFH != nil && originalFH.Kind() == source.Mod - if currentMod || originalMod { - modulesChanged = true - + if isGoMod(uri) { // If the view's go.mod file's contents have changed, invalidate // the metadata for every known package in the snapshot. - if invalidateMetadata { - for k := range s.metadata { - directIDs[k] = true - } - // If a go.mod file in the workspace has changed, we need to - // rebuild the workspace module. - result.workspaceModuleHandle = nil - } - delete(result.parseModHandles, withoutURI) - - // Check if this is a newly created go.mod file. When a new module - // is created, we have to retry the initial workspace load. - rootURI := span.URIFromPath(filepath.Dir(withoutURI.Filename())) - if currentMod { - if _, ok := result.modules[rootURI]; !ok { - if m := getViewModule(ctx, s.view.rootURI, currentFH.URI(), s.view.Options()); m != nil { - result.modules[m.rootURI] = m - shouldReinitializeView = true - } - - } - } else if originalMod { - // Similarly, we need to retry the IWL if a go.mod in the workspace - // was deleted. - if _, ok := result.modules[rootURI]; ok { - delete(result.modules, rootURI) - shouldReinitializeView = true - } - } - } - // Keep track of the creations and deletions of go.sum files. - // Creating a go.sum without an associated go.mod has no effect on the - // set of modules. - currentSum := currentExists && currentFH.Kind() == source.Sum - originalSum := originalFH != nil && originalFH.Kind() == source.Sum - if currentSum || originalSum { - rootURI := span.URIFromPath(filepath.Dir(withoutURI.Filename())) - if currentSum { - if mod, ok := result.modules[rootURI]; ok { - mod.sumURI = currentFH.URI() - } - } else if originalSum { - if mod, ok := result.modules[rootURI]; ok { - mod.sumURI = "" - } + delete(result.parseModHandles, uri) + if _, ok := result.workspace.activeModFiles()[uri]; ok && reinitialize < maybeReinit { + reinitialize = maybeReinit } } - // Handle the invalidated file; it may have new contents or not exist. - if !currentExists { - delete(result.files, withoutURI) + if !change.exists { + delete(result.files, uri) } else { - result.files[withoutURI] = currentFH + result.files[uri] = change.fileHandle } // Make sure to remove the changed file from the unloadable set. - delete(result.unloadableFiles, withoutURI) + delete(result.unloadableFiles, uri) } // Invalidate reverse dependencies too. @@ -1208,12 +1135,6 @@ func (s *snapshot) clone(ctx context.Context, withoutURIs map[span.URI]source.Ve addRevDeps(id, invalidateMetadata) } - // When modules change, we need to recompute their workspace directories, - // as replace directives may have changed. - if modulesChanged { - result.workspaceDirectories = result.findWorkspaceDirectories(ctx) - } - // Copy the package type information. for k, v := range s.packages { if _, ok := transitiveIDs[k.id]; ok { @@ -1292,20 +1213,9 @@ copyIDs: for _, v := range result.parseModHandles { newGen.Inherit(v.handle) } - if result.workspaceModuleHandle != nil { - newGen.Inherit(result.workspaceModuleHandle.handle) - } // Don't bother copying the importedBy graph, // as it changes each time we update metadata. - var reinitialize reinitializeView - if modulesChanged { - reinitialize = maybeReinit - } - if shouldReinitializeView { - reinitialize = definitelyReinit - } - // If the snapshot's workspace mode has changed, the packages loaded using // the previous mode are no longer relevant, so clear them out. if s.workspaceMode() != result.workspaceMode() { @@ -1443,45 +1353,6 @@ func (s *snapshot) shouldInvalidateMetadata(ctx context.Context, newSnapshot *sn return false } -// findWorkspaceDirectoriesLocked returns all of the directories that are -// considered to be part of the view's workspace. For GOPATH workspaces, this -// is just the view's root. For modules-based workspaces, this is the module -// root and any replace targets. It also returns the parseModHandle for the -// view's go.mod file if it has one. -// -// It assumes that the file handle is the view's go.mod file, if it has one. -// The caller need not be holding the snapshot's mutex, but it might be. -func (s *snapshot) findWorkspaceDirectories(ctx context.Context) map[span.URI]struct{} { - // If the view does not have a go.mod file, only the root directory - // is known. In GOPATH mode, we should really watch the entire GOPATH, - // but that's too expensive. - dirs := map[span.URI]struct{}{ - s.view.rootURI: {}, - } - for _, m := range s.modules { - fh, err := s.GetFile(ctx, m.modURI) - if err != nil { - continue - } - // Ignore parse errors. An invalid go.mod is not fatal. - // TODO(rstambler): Try to preserve existing watched directories as - // much as possible, otherwise we will thrash when a go.mod is edited. - mod, err := s.ParseMod(ctx, fh) - if err != nil { - continue - } - for _, r := range mod.File.Replace { - // We may be replacing a module with a different version, not a path - // on disk. - if r.New.Version != "" { - continue - } - dirs[span.URIFromPath(r.New.Path)] = struct{}{} - } - } - return dirs -} - func (s *snapshot) BuiltinPackage(ctx context.Context) (*source.BuiltinPackage, error) { s.AwaitInitialized(ctx) @@ -1533,111 +1404,40 @@ func (s *snapshot) buildBuiltinPackage(ctx context.Context, goFiles []string) er return nil } -type workspaceModuleHandle struct { - handle *memoize.Handle -} - -type workspaceModuleData struct { - file *modfile.File - err error -} - -type workspaceModuleKey string - -func (wmh *workspaceModuleHandle) build(ctx context.Context, snapshot *snapshot) (*modfile.File, error) { - v, err := wmh.handle.Get(ctx, snapshot.generation, snapshot) +// BuildGoplsMod generates a go.mod file for all modules in the workspace. It +// bypasses any existing gopls.mod. +func BuildGoplsMod(ctx context.Context, root span.URI, fs source.FileSource) (*modfile.File, error) { + allModules, err := findAllModules(ctx, root) if err != nil { return nil, err } - data := v.(*workspaceModuleData) - return data.file, data.err + return buildWorkspaceModFile(ctx, allModules, fs) } -func (s *snapshot) getWorkspaceModuleHandle(ctx context.Context) (*workspaceModuleHandle, error) { - s.mu.Lock() - wsModule := s.workspaceModuleHandle - s.mu.Unlock() - if wsModule != nil { - return wsModule, nil - } - var fhs []source.FileHandle - for _, mod := range s.modules { - fh, err := s.GetFile(ctx, mod.modURI) - if err != nil { - return nil, err - } - fhs = append(fhs, fh) - } - goplsModURI := span.URIFromPath(filepath.Join(s.view.Folder().Filename(), "gopls.mod")) - goplsModFH, err := s.GetFile(ctx, goplsModURI) - if err != nil { - return nil, err - } - _, err = goplsModFH.Read() - switch { - case err == nil: - // We have a gopls.mod. Our handle only depends on it. - fhs = []source.FileHandle{goplsModFH} - case os.IsNotExist(err): - // No gopls.mod, so we must build the workspace mod file automatically. - // Defensively ensure that the goplsModFH is nil as this controls automatic - // building of the workspace mod file. - goplsModFH = nil - default: - return nil, errors.Errorf("error getting gopls.mod: %w", err) - } - - sort.Slice(fhs, func(i, j int) bool { - return fhs[i].URI() < fhs[j].URI() - }) - var k string - for _, fh := range fhs { - k += fh.FileIdentity().String() - } - key := workspaceModuleKey(hashContents([]byte(k))) - h := s.generation.Bind(key, func(ctx context.Context, arg memoize.Arg) interface{} { - if goplsModFH != nil { - parsed, err := s.ParseMod(ctx, goplsModFH) - if err != nil { - return &workspaceModuleData{err: err} - } - return &workspaceModuleData{file: parsed.File} - } - s := arg.(*snapshot) - data := &workspaceModuleData{} - data.file, data.err = s.BuildWorkspaceModFile(ctx) - return data - }) - wsModule = &workspaceModuleHandle{ - handle: h, - } - s.mu.Lock() - defer s.mu.Unlock() - s.workspaceModuleHandle = wsModule - return s.workspaceModuleHandle, nil -} - -// BuildWorkspaceModFile generates a workspace module given the modules in the -// the workspace. It does not read gopls.mod. -func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, error) { +// TODO(rfindley): move this to workspacemodule.go +func buildWorkspaceModFile(ctx context.Context, modFiles map[span.URI]struct{}, fs source.FileSource) (*modfile.File, error) { file := &modfile.File{} file.AddModuleStmt("gopls-workspace") - paths := make(map[string]*moduleRoot) - for _, mod := range s.modules { - fh, err := s.GetFile(ctx, mod.modURI) + paths := make(map[string]span.URI) + for modURI := range modFiles { + fh, err := fs.GetFile(ctx, modURI) + if err != nil { + return nil, err + } + content, err := fh.Read() if err != nil { return nil, err } - parsed, err := s.ParseMod(ctx, fh) + parsed, err := modfile.Parse(fh.URI().Filename(), content, nil) if err != nil { return nil, err } - if parsed.File == nil || parsed.File.Module == nil { - return nil, fmt.Errorf("no module declaration for %s", mod.modURI) + if file == nil || parsed.Module == nil { + return nil, fmt.Errorf("no module declaration for %s", modURI) } - path := parsed.File.Module.Mod.Path - paths[path] = mod + path := parsed.Module.Mod.Path + paths[path] = modURI // If the module's path includes a major version, we expect it to have // a matching major version. _, majorVersion, _ := module.SplitPathVersion(path) @@ -1646,24 +1446,28 @@ func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, er } majorVersion = strings.TrimLeft(majorVersion, "/.") // handle gopkg.in versions file.AddNewRequire(path, source.WorkspaceModuleVersion(majorVersion), false) - if err := file.AddReplace(path, "", mod.rootURI.Filename(), ""); err != nil { + if err := file.AddReplace(path, "", dirURI(modURI).Filename(), ""); err != nil { return nil, err } } // Go back through all of the modules to handle any of their replace // statements. - for _, module := range s.modules { - fh, err := s.GetFile(ctx, module.modURI) + for modURI := range modFiles { + fh, err := fs.GetFile(ctx, modURI) if err != nil { return nil, err } - pmf, err := s.ParseMod(ctx, fh) + content, err := fh.Read() + if err != nil { + return nil, err + } + parsed, err := modfile.Parse(fh.URI().Filename(), content, nil) if err != nil { return nil, err } // If any of the workspace modules have replace directives, they need // to be reflected in the workspace module. - for _, rep := range pmf.File.Replace { + for _, rep := range parsed.Replace { // Don't replace any modules that are in our workspace--we should // always use the version in the workspace. if _, ok := paths[rep.Old.Path]; ok { @@ -1673,12 +1477,12 @@ func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, er newVersion := rep.New.Version // If a replace points to a module in the workspace, make sure we // direct it to version of the module in the workspace. - if mod, ok := paths[rep.New.Path]; ok { - newPath = mod.rootURI.Filename() + if m, ok := paths[rep.New.Path]; ok { + newPath = dirURI(m).Filename() newVersion = "" } else if rep.New.Version == "" && !filepath.IsAbs(rep.New.Path) { // Make any relative paths absolute. - newPath = filepath.Join(module.rootURI.Filename(), rep.New.Path) + newPath = filepath.Join(dirURI(modURI).Filename(), rep.New.Path) } if err := file.AddReplace(rep.Old.Path, rep.Old.Version, newPath, newVersion); err != nil { return nil, err @@ -1687,23 +1491,3 @@ func (s *snapshot) BuildWorkspaceModFile(ctx context.Context) (*modfile.File, er } return file, nil } - -func getViewModule(ctx context.Context, viewRootURI, modURI span.URI, options *source.Options) *moduleRoot { - rootURI := span.URIFromPath(filepath.Dir(modURI.Filename())) - // If we are not in multi-module mode, check that the affected module is - // in the workspace root. - if !options.ExperimentalWorkspaceModule { - if span.CompareURI(rootURI, viewRootURI) != 0 { - return nil - } - } - sumURI := span.URIFromPath(sumFilename(modURI)) - if info, _ := os.Stat(sumURI.Filename()); info == nil { - sumURI = "" - } - return &moduleRoot{ - rootURI: rootURI, - modURI: modURI, - sumURI: sumURI, - } -} diff --git a/internal/lsp/cache/view.go b/internal/lsp/cache/view.go index 42017a853f9..448273ab0c7 100644 --- a/internal/lsp/cache/view.go +++ b/internal/lsp/cache/view.go @@ -151,11 +151,6 @@ type builtinPackageData struct { err error } -type moduleRoot struct { - rootURI span.URI - modURI, sumURI span.URI -} - // fileBase holds the common functionality for all files. // It is intended to be embedded in the file implementations type fileBase struct { @@ -183,7 +178,7 @@ func (v *View) ID() string { return v.id } // tempModFile creates a temporary go.mod file based on the contents of the // given go.mod file. It is the caller's responsibility to clean up the files // when they are done using them. -func tempModFile(modFh, sumFH source.FileHandle) (tmpURI span.URI, cleanup func(), err error) { +func tempModFile(modFh source.FileHandle, gosum []byte) (tmpURI span.URI, cleanup func(), err error) { filenameHash := hashContents([]byte(modFh.URI().Filename())) tmpMod, err := ioutil.TempFile("", fmt.Sprintf("go.%s.*.mod", filenameHash)) if err != nil { @@ -217,12 +212,8 @@ func tempModFile(modFh, sumFH source.FileHandle) (tmpURI span.URI, cleanup func( }() // Create an analogous go.sum, if one exists. - if sumFH != nil { - sumContents, err := sumFH.Read() - if err != nil { - return "", cleanup, err - } - if err := ioutil.WriteFile(tmpSumName, sumContents, 0655); err != nil { + if gosum != nil { + if err := ioutil.WriteFile(tmpSumName, gosum, 0655); err != nil { return "", cleanup, err } } @@ -452,14 +443,14 @@ func (v *View) BackgroundContext() context.Context { func (s *snapshot) IgnoredFile(uri span.URI) bool { filename := uri.Filename() var prefixes []string - if len(s.modules) == 0 { + if len(s.workspace.activeModFiles()) == 0 { for _, entry := range filepath.SplitList(s.view.gopath) { prefixes = append(prefixes, filepath.Join(entry, "src")) } } else { prefixes = append(prefixes, s.view.gomodcache) - for _, m := range s.modules { - prefixes = append(prefixes, m.rootURI.Filename()) + for m := range s.workspace.activeModFiles() { + prefixes = append(prefixes, dirURI(m).Filename()) } } for _, prefix := range prefixes { @@ -527,25 +518,23 @@ func (s *snapshot) initialize(ctx context.Context, firstAttempt bool) { Message: err.Error(), }) } - if len(s.modules) > 0 { - for _, mod := range s.modules { - fh, err := s.GetFile(ctx, mod.modURI) - if err != nil { - addError(mod.modURI, err) - continue - } - parsed, err := s.ParseMod(ctx, fh) - if err != nil { - addError(mod.modURI, err) - continue - } - if parsed.File == nil || parsed.File.Module == nil { - addError(mod.modURI, fmt.Errorf("no module path for %s", mod.modURI)) - continue - } - path := parsed.File.Module.Mod.Path - scopes = append(scopes, moduleLoadScope(path)) + for modURI := range s.workspace.activeModFiles() { + fh, err := s.GetFile(ctx, modURI) + if err != nil { + addError(modURI, err) + continue + } + parsed, err := s.ParseMod(ctx, fh) + if err != nil { + addError(modURI, err) + continue } + if parsed.File == nil || parsed.File.Module == nil { + addError(modURI, fmt.Errorf("no module path for %s", modURI)) + continue + } + path := parsed.File.Module.Mod.Path + scopes = append(scopes, moduleLoadScope(path)) } if len(scopes) == 0 { scopes = append(scopes, viewLoadScope("LOAD_VIEW")) @@ -568,7 +557,7 @@ func (s *snapshot) initialize(ctx context.Context, firstAttempt bool) { // invalidateContent invalidates the content of a Go file, // including any position and type information that depends on it. // It returns true if we were already tracking the given file, false otherwise. -func (v *View) invalidateContent(ctx context.Context, uris map[span.URI]source.VersionedFileHandle, forceReloadMetadata bool) (source.Snapshot, func()) { +func (v *View) invalidateContent(ctx context.Context, changes map[span.URI]*fileChange, forceReloadMetadata bool) (source.Snapshot, func()) { // Detach the context so that content invalidation cannot be canceled. ctx = xcontext.Detach(ctx) @@ -584,8 +573,9 @@ func (v *View) invalidateContent(ctx context.Context, uris map[span.URI]source.V defer v.snapshotMu.Unlock() oldSnapshot := v.snapshot + var reinitialize reinitializeView - v.snapshot, reinitialize = oldSnapshot.clone(ctx, uris, forceReloadMetadata) + v.snapshot, reinitialize = oldSnapshot.clone(ctx, changes, forceReloadMetadata) go oldSnapshot.generation.Destroy() if reinitialize == maybeReinit || reinitialize == definitelyReinit { @@ -686,7 +676,7 @@ func defaultCheckPathCase(path string) error { return nil } -func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modules map[span.URI]*moduleRoot) bool { +func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modFiles map[span.URI]struct{}) bool { // Since we only really understand the `go` command, if the user has a // different GOPACKAGESDRIVER, assume that their configuration is valid. if ws.hasGopackagesDriver { @@ -694,7 +684,7 @@ func validBuildConfiguration(folder span.URI, ws *workspaceInformation, modules } // Check if the user is working within a module or if we have found // multiple modules in the workspace. - if len(modules) > 0 { + if len(modFiles) > 0 { return true } // The user may have a multiple directories in their GOPATH. diff --git a/internal/lsp/cache/workspace.go b/internal/lsp/cache/workspace.go new file mode 100644 index 00000000000..71d38423290 --- /dev/null +++ b/internal/lsp/cache/workspace.go @@ -0,0 +1,409 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "context" + "os" + "path/filepath" + "sort" + "strings" + "sync" + + "golang.org/x/mod/modfile" + "golang.org/x/tools/internal/event" + "golang.org/x/tools/internal/lsp/source" + "golang.org/x/tools/internal/span" + "golang.org/x/tools/internal/xcontext" + errors "golang.org/x/xerrors" +) + +type workspaceSource int + +const ( + legacyWorkspace = iota + goplsModWorkspace + fileSystemWorkspace +) + +func (s workspaceSource) String() string { + switch s { + case legacyWorkspace: + return "legacy" + case goplsModWorkspace: + return "gopls.mod" + case fileSystemWorkspace: + return "file system" + default: + return "!(unknown module source)" + } +} + +// workspace tracks go.mod files in the workspace, along with the +// gopls.mod file, to provide support for multi-module workspaces. +// +// Specifically, it provides: +// - the set of modules contained within in the workspace root considered to +// be 'active' +// - the workspace modfile, to be used for the go command `-modfile` flag +// - the set of workspace directories +// +// This type is immutable (or rather, idempotent), so that it may be shared +// across multiple snapshots. +type workspace struct { + root span.URI + moduleSource workspaceSource + + // modFiles holds the active go.mod files. + modFiles map[span.URI]struct{} + + // The workspace module is lazily re-built once after being invalidated. + // buildMu+built guards this reconstruction. + // + // file and wsDirs may be non-nil even if built == false, if they were copied + // from the previous workspace module version. In this case, they will be + // preserved if building fails. + buildMu sync.Mutex + built bool + buildErr error + file *modfile.File + wsDirs map[span.URI]struct{} +} + +func newWorkspace(ctx context.Context, root span.URI, fs source.FileSource, experimental bool) (*workspace, error) { + if !experimental { + modFiles, err := getLegacyModules(ctx, root, fs) + if err != nil { + return nil, err + } + return &workspace{ + root: root, + modFiles: modFiles, + moduleSource: legacyWorkspace, + }, nil + } + goplsModFH, err := fs.GetFile(ctx, goplsModURI(root)) + if err != nil { + return nil, err + } + contents, err := goplsModFH.Read() + if err == nil { + file, modFiles, err := parseGoplsMod(root, goplsModFH.URI(), contents) + if err != nil { + return nil, err + } + return &workspace{ + root: root, + modFiles: modFiles, + file: file, + moduleSource: goplsModWorkspace, + }, nil + } + modFiles, err := findAllModules(ctx, root) + if err != nil { + return nil, err + } + return &workspace{ + root: root, + modFiles: modFiles, + moduleSource: fileSystemWorkspace, + }, nil +} + +func (wm *workspace) activeModFiles() map[span.URI]struct{} { + return wm.modFiles +} + +// modFile gets the workspace modfile associated with this workspace, +// computing it if it doesn't exist. +// +// A fileSource must be passed in to solve a chicken-egg problem: it is not +// correct to pass in the snapshot file source to newWorkspace when +// invalidating, because at the time these are called the snapshot is locked. +// So we must pass it in later on when actually using the modFile. +func (wm *workspace) modFile(ctx context.Context, fs source.FileSource) (*modfile.File, error) { + wm.build(ctx, fs) + return wm.file, wm.buildErr +} + +func (wm *workspace) build(ctx context.Context, fs source.FileSource) { + wm.buildMu.Lock() + defer wm.buildMu.Unlock() + + if wm.built { + return + } + // Building should never be cancelled. Since the workspace module is shared + // across multiple snapshots, doing so would put us in a bad state, and it + // would not be obvious to the user how to recover. + ctx = xcontext.Detach(ctx) + + // If our module source is not gopls.mod, try to build the workspace module + // from modules. Fall back on the pre-existing mod file if parsing fails. + if wm.moduleSource != goplsModWorkspace { + file, err := buildWorkspaceModFile(ctx, wm.modFiles, fs) + switch { + case err == nil: + wm.file = file + case wm.file != nil: + // Parsing failed, but we have a previous file version. + event.Error(ctx, "building workspace mod file", err) + default: + // No file to fall back on. + wm.buildErr = err + } + } + if wm.file != nil { + wm.wsDirs = map[span.URI]struct{}{ + wm.root: {}, + } + for _, r := range wm.file.Replace { + // We may be replacing a module with a different version, not a path + // on disk. + if r.New.Version != "" { + continue + } + wm.wsDirs[span.URIFromPath(r.New.Path)] = struct{}{} + } + } + // Ensure that there is always at least the root dir. + if len(wm.wsDirs) == 0 { + wm.wsDirs = map[span.URI]struct{}{ + wm.root: {}, + } + } + wm.built = true +} + +// dirs returns the workspace directories for the loaded modules. +func (wm *workspace) dirs(ctx context.Context, fs source.FileSource) []span.URI { + wm.build(ctx, fs) + var dirs []span.URI + for d := range wm.wsDirs { + dirs = append(dirs, d) + } + sort.Slice(dirs, func(i, j int) bool { + return span.CompareURI(dirs[i], dirs[j]) < 0 + }) + return dirs +} + +// invalidate returns a (possibly) new workspaceModule after invalidating +// changedURIs. If wm is still valid in the presence of changedURIs, it returns +// itself unmodified. +func (wm *workspace) invalidate(ctx context.Context, changes map[span.URI]*fileChange) (*workspace, bool) { + // Prevent races to wm.modFile or wm.wsDirs below, if wm has not yet been + // built. + wm.buildMu.Lock() + defer wm.buildMu.Unlock() + // Any gopls.mod change is processed first, followed by go.mod changes, as + // changes to gopls.mod may affect the set of active go.mod files. + var ( + // New values. We return a new workspace module if and only if modFiles is + // non-nil. + modFiles map[span.URI]struct{} + moduleSource = wm.moduleSource + modFile = wm.file + err error + ) + if wm.moduleSource == goplsModWorkspace { + // If we are currently reading the modfile from gopls.mod, we default to + // preserving it even if module metadata changes (which may be the case if + // a go.sum file changes). + modFile = wm.file + } + // First handle changes to the gopls.mod file. + if wm.moduleSource != legacyWorkspace { + // If gopls.mod has changed we need to either re-read it if it exists or + // walk the filesystem if it doesn't exist. + gmURI := goplsModURI(wm.root) + if change, ok := changes[gmURI]; ok { + if change.exists { + // Only invalidate if the gopls.mod actually parses. Otherwise, stick with the current gopls.mod + parsedFile, parsedModules, err := parseGoplsMod(wm.root, gmURI, change.content) + if err == nil { + modFile = parsedFile + moduleSource = goplsModWorkspace + modFiles = parsedModules + } else { + // Note that modFile is not invalidated here. + event.Error(ctx, "parsing gopls.mod", err) + } + } else { + // gopls.mod is deleted. search for modules again. + moduleSource = fileSystemWorkspace + modFiles, err = findAllModules(ctx, wm.root) + // the modFile is no longer valid. + if err != nil { + event.Error(ctx, "finding file system modules", err) + } + modFile = nil + } + } + } + + // Next, handle go.mod changes that could affect our set of tracked modules. + // If we're reading our tracked modules from the gopls.mod, there's nothing + // to do here. + if wm.moduleSource != goplsModWorkspace { + for uri, change := range changes { + // If a go.mod file has changed, we may need to update the set of active + // modules. + if !isGoMod(uri) { + continue + } + if wm.moduleSource == legacyWorkspace && !equalURI(modURI(wm.root), uri) { + // Legacy mode only considers a module a workspace root. + continue + } + if !isSubdirectory(wm.root.Filename(), uri.Filename()) { + // Otherwise, the module must be contained within the workspace root. + continue + } + if modFiles == nil { + modFiles = make(map[span.URI]struct{}) + for k := range wm.modFiles { + modFiles[k] = struct{}{} + } + } + if change.exists { + modFiles[uri] = struct{}{} + } else { + delete(modFiles, uri) + } + } + } + if modFiles != nil { + // Any change to modules triggers a new version. + return &workspace{ + root: wm.root, + moduleSource: moduleSource, + modFiles: modFiles, + file: modFile, + wsDirs: wm.wsDirs, + }, true + } + // No change. Just return wm, since it is immutable. + return wm, false +} + +func equalURI(left, right span.URI) bool { + return span.CompareURI(left, right) == 0 +} + +// goplsModURI returns the URI for the gopls.mod file contained in root. +func goplsModURI(root span.URI) span.URI { + return span.URIFromPath(filepath.Join(root.Filename(), "gopls.mod")) +} + +// modURI returns the URI for the go.mod file contained in root. +func modURI(root span.URI) span.URI { + return span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) +} + +// isGoMod reports if uri is a go.mod file. +func isGoMod(uri span.URI) bool { + return filepath.Base(uri.Filename()) == "go.mod" +} + +// isGoMod reports if uri is a go.sum file. +func isGoSum(uri span.URI) bool { + return filepath.Base(uri.Filename()) == "go.sum" +} + +// fileExists reports if the file uri exists within source. +func fileExists(ctx context.Context, uri span.URI, source source.FileSource) (bool, error) { + fh, err := source.GetFile(ctx, uri) + if err != nil { + return false, err + } + return fileHandleExists(fh) +} + +// fileHandleExists reports if the file underlying fh actually exits. +func fileHandleExists(fh source.FileHandle) (bool, error) { + _, err := fh.Read() + if err == nil { + return true, nil + } + if os.IsNotExist(err) { + return false, nil + } + return false, err +} + +// TODO(rFindley): replace this (and similar) with a uripath package analogous +// to filepath. +func dirURI(uri span.URI) span.URI { + return span.URIFromPath(filepath.Dir(uri.Filename())) +} + +// getLegacyModules returns a module set containing at most the root module. +func getLegacyModules(ctx context.Context, root span.URI, fs source.FileSource) (map[span.URI]struct{}, error) { + uri := span.URIFromPath(filepath.Join(root.Filename(), "go.mod")) + modules := make(map[span.URI]struct{}) + exists, err := fileExists(ctx, uri, fs) + if err != nil { + return nil, err + } + if exists { + modules[uri] = struct{}{} + } + return modules, nil +} + +func parseGoplsMod(root, uri span.URI, contents []byte) (*modfile.File, map[span.URI]struct{}, error) { + modFile, err := modfile.Parse(uri.Filename(), contents, nil) + if err != nil { + return nil, nil, errors.Errorf("parsing gopls.mod: %w", err) + } + modFiles := make(map[span.URI]struct{}) + for _, replace := range modFile.Replace { + if replace.New.Version != "" { + return nil, nil, errors.Errorf("gopls.mod: replaced module %q@%q must not have version", replace.New.Path, replace.New.Version) + } + dirFP := filepath.FromSlash(replace.New.Path) + if !filepath.IsAbs(dirFP) { + dirFP = filepath.Join(root.Filename(), dirFP) + // The resulting modfile must use absolute paths, so that it can be + // written to a temp directory. + replace.New.Path = dirFP + } + modURI := span.URIFromPath(filepath.Join(dirFP, "go.mod")) + modFiles[modURI] = struct{}{} + } + return modFile, modFiles, nil +} + +// findAllModules recursively walks the root directory looking for go.mod +// files, returning the set of modules it discovers. +// TODO(rfindley): consider overlays. +func findAllModules(ctx context.Context, root span.URI) (map[span.URI]struct{}, error) { + // Walk the view's folder to find all modules in the view. + modFiles := make(map[span.URI]struct{}) + return modFiles, filepath.Walk(root.Filename(), func(path string, info os.FileInfo, err error) error { + if err != nil { + // Probably a permission error. Keep looking. + return filepath.SkipDir + } + // For any path that is not the workspace folder, check if the path + // would be ignored by the go command. Vendor directories also do not + // contain workspace modules. + if info.IsDir() && path != root.Filename() { + suffix := strings.TrimPrefix(path, root.Filename()) + switch { + case checkIgnored(suffix), + strings.Contains(filepath.ToSlash(suffix), "/vendor/"): + return filepath.SkipDir + } + } + // We're only interested in go.mod files. + uri := span.URIFromPath(path) + if isGoMod(uri) { + modFiles[uri] = struct{}{} + } + return nil + }) +} diff --git a/internal/lsp/cache/workspace_test.go b/internal/lsp/cache/workspace_test.go new file mode 100644 index 00000000000..7cd0ecee563 --- /dev/null +++ b/internal/lsp/cache/workspace_test.go @@ -0,0 +1,257 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package cache + +import ( + "context" + "os" + "testing" + + "golang.org/x/tools/internal/lsp/fake" + "golang.org/x/tools/internal/lsp/source" + "golang.org/x/tools/internal/span" +) + +// osFileSource is a fileSource that just reads from the operating system. +type osFileSource struct{} + +func (s osFileSource) GetFile(ctx context.Context, uri span.URI) (source.FileHandle, error) { + fi, statErr := os.Stat(uri.Filename()) + if statErr != nil { + return &fileHandle{ + err: statErr, + uri: uri, + }, nil + } + fh, err := readFile(ctx, uri, fi.ModTime()) + if err != nil { + return nil, err + } + return fh, nil +} + +func TestWorkspaceModule(t *testing.T) { + tests := []struct { + desc string + initial string // txtar-encoded + legacyMode bool + initialSource workspaceSource + initialModules []string + initialDirs []string + updates map[string]string + finalSource workspaceSource + finalModules []string + finalDirs []string + }{ + { + desc: "legacy mode", + initial: ` +-- go.mod -- +module mod.com +-- a/go.mod -- +module moda.com`, + legacyMode: true, + initialModules: []string{"./go.mod"}, + initialSource: legacyWorkspace, + initialDirs: []string{"."}, + }, + { + desc: "nested module", + initial: ` +-- go.mod -- +module mod.com +-- a/go.mod -- +module moda.com`, + initialModules: []string{"./go.mod", "a/go.mod"}, + initialSource: fileSystemWorkspace, + initialDirs: []string{".", "a"}, + }, + { + desc: "removing module", + initial: ` +-- a/go.mod -- +module moda.com +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod", "b/go.mod"}, + initialSource: fileSystemWorkspace, + initialDirs: []string{".", "a", "b"}, + updates: map[string]string{ + "gopls.mod": `module gopls-workspace + +require moda.com v0.0.0-goplsworkspace +replace moda.com => $SANDBOX_WORKDIR/a`, + }, + finalModules: []string{"a/go.mod"}, + finalSource: goplsModWorkspace, + finalDirs: []string{".", "a"}, + }, + { + desc: "adding module", + initial: ` +-- gopls.mod -- +require moda.com v0.0.0-goplsworkspace +replace moda.com => $SANDBOX_WORKDIR/a +-- a/go.mod -- +module moda.com +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod"}, + initialSource: goplsModWorkspace, + initialDirs: []string{".", "a"}, + updates: map[string]string{ + "gopls.mod": `module gopls-workspace + +require moda.com v0.0.0-goplsworkspace +require modb.com v0.0.0-goplsworkspace + +replace moda.com => $SANDBOX_WORKDIR/a +replace modb.com => $SANDBOX_WORKDIR/b`, + }, + finalModules: []string{"a/go.mod", "b/go.mod"}, + finalSource: goplsModWorkspace, + finalDirs: []string{".", "a", "b"}, + }, + { + desc: "deleting gopls.mod", + initial: ` +-- gopls.mod -- +module gopls-workspace + +require moda.com v0.0.0-goplsworkspace +replace moda.com => $SANDBOX_WORKDIR/a +-- a/go.mod -- +module moda.com +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod"}, + initialSource: goplsModWorkspace, + initialDirs: []string{".", "a"}, + updates: map[string]string{ + "gopls.mod": "", + }, + finalModules: []string{"a/go.mod", "b/go.mod"}, + finalSource: fileSystemWorkspace, + finalDirs: []string{".", "a", "b"}, + }, + { + desc: "broken module parsing", + initial: ` +-- a/go.mod -- +module moda.com + +require gopls.test v0.0.0-goplsworkspace +replace gopls.test => ../../gopls.test // (this path shouldn't matter) +-- b/go.mod -- +module modb.com`, + initialModules: []string{"a/go.mod", "b/go.mod"}, + initialSource: fileSystemWorkspace, + initialDirs: []string{".", "a", "b", "../gopls.test"}, + updates: map[string]string{ + "a/go.mod": `modul moda.com + +require gopls.test v0.0.0-goplsworkspace +replace gopls.test => ../../gopls.test2`, + }, + finalModules: []string{"a/go.mod", "b/go.mod"}, + finalSource: fileSystemWorkspace, + // finalDirs should be unchanged: we should preserve dirs in the presence + // of a broken modfile. + finalDirs: []string{".", "a", "b", "../gopls.test"}, + }, + } + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + ctx := context.Background() + dir, err := fake.Tempdir(test.initial) + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + root := span.URIFromPath(dir) + + fs := osFileSource{} + wm, err := newWorkspace(ctx, root, fs, !test.legacyMode) + if err != nil { + t.Fatal(err) + } + rel := fake.RelativeTo(dir) + checkWorkspaceModule(t, rel, wm, test.initialSource, test.initialModules) + gotDirs := wm.dirs(ctx, fs) + checkWorkspaceDirs(t, rel, gotDirs, test.initialDirs) + if test.updates != nil { + changes := make(map[span.URI]*fileChange) + for k, v := range test.updates { + if v == "" { + // for convenience, use this to signal a deletion. TODO: more doc + err := os.Remove(rel.AbsPath(k)) + if err != nil { + t.Fatal(err) + } + } else { + fake.WriteFileData(k, []byte(v), rel) + } + uri := span.URIFromPath(rel.AbsPath(k)) + fh, err := fs.GetFile(ctx, uri) + if err != nil { + t.Fatal(err) + } + content, err := fh.Read() + changes[uri] = &fileChange{ + content: content, + exists: err == nil, + fileHandle: &closedFile{fh}, + } + } + wm, _ := wm.invalidate(ctx, changes) + checkWorkspaceModule(t, rel, wm, test.finalSource, test.finalModules) + gotDirs := wm.dirs(ctx, fs) + checkWorkspaceDirs(t, rel, gotDirs, test.finalDirs) + } + }) + } +} + +func checkWorkspaceModule(t *testing.T, rel fake.RelativeTo, got *workspace, wantSource workspaceSource, want []string) { + t.Helper() + if got.moduleSource != wantSource { + t.Errorf("module source = %v, want %v", got.moduleSource, wantSource) + } + modules := make(map[span.URI]struct{}) + for k := range got.activeModFiles() { + modules[k] = struct{}{} + } + for _, modPath := range want { + path := rel.AbsPath(modPath) + uri := span.URIFromPath(path) + if _, ok := modules[uri]; !ok { + t.Errorf("missing module %q", uri) + } + delete(modules, uri) + } + for remaining := range modules { + t.Errorf("unexpected module %q", remaining) + } +} + +func checkWorkspaceDirs(t *testing.T, rel fake.RelativeTo, got []span.URI, want []string) { + t.Helper() + gotM := make(map[span.URI]bool) + for _, dir := range got { + gotM[dir] = true + } + for _, dir := range want { + path := rel.AbsPath(dir) + uri := span.URIFromPath(path) + if !gotM[uri] { + t.Errorf("missing dir %q", uri) + } + delete(gotM, uri) + } + for remaining := range gotM { + t.Errorf("unexpected dir %q", remaining) + } +} diff --git a/internal/lsp/command.go b/internal/lsp/command.go index 6dd5045bbac..d51a98b4c92 100644 --- a/internal/lsp/command.go +++ b/internal/lsp/command.go @@ -15,6 +15,7 @@ import ( "golang.org/x/tools/internal/event" "golang.org/x/tools/internal/gocommand" + "golang.org/x/tools/internal/lsp/cache" "golang.org/x/tools/internal/lsp/protocol" "golang.org/x/tools/internal/lsp/source" "golang.org/x/tools/internal/span" @@ -250,7 +251,7 @@ func (s *Server) runCommand(ctx context.Context, work *workDone, command *source } snapshot, release := v.Snapshot(ctx) defer release() - modFile, err := snapshot.BuildWorkspaceModFile(ctx) + modFile, err := cache.BuildGoplsMod(ctx, v.Folder(), snapshot) if err != nil { return errors.Errorf("getting workspace mod file: %w", err) } diff --git a/internal/lsp/fake/editor.go b/internal/lsp/fake/editor.go index dba89976962..b367293f16b 100644 --- a/internal/lsp/fake/editor.go +++ b/internal/lsp/fake/editor.go @@ -522,6 +522,13 @@ func (e *Editor) EditBuffer(ctx context.Context, path string, edits []Edit) erro return e.editBufferLocked(ctx, path, edits) } +func (e *Editor) SetBufferContent(ctx context.Context, path, content string) error { + e.mu.Lock() + defer e.mu.Unlock() + lines := strings.Split(content, "\n") + return e.setBufferContentLocked(ctx, path, lines, nil) +} + // BufferText returns the content of the buffer with the given name. func (e *Editor) BufferText(name string) string { e.mu.Lock() @@ -542,24 +549,28 @@ func (e *Editor) editBufferLocked(ctx context.Context, path string, edits []Edit if !ok { return fmt.Errorf("unknown buffer %q", path) } - var ( - content = make([]string, len(buf.content)) - err error - evts []protocol.TextDocumentContentChangeEvent - ) + content := make([]string, len(buf.content)) copy(content, buf.content) - content, err = editContent(content, edits) + content, err := editContent(content, edits) if err != nil { return err } + return e.setBufferContentLocked(ctx, path, content, edits) +} +func (e *Editor) setBufferContentLocked(ctx context.Context, path string, content []string, fromEdits []Edit) error { + buf, ok := e.buffers[path] + if !ok { + return fmt.Errorf("unknown buffer %q", path) + } buf.content = content buf.version++ e.buffers[path] = buf // A simple heuristic: if there is only one edit, send it incrementally. // Otherwise, send the entire content. - if len(edits) == 1 { - evts = append(evts, edits[0].toProtocolChangeEvent()) + var evts []protocol.TextDocumentContentChangeEvent + if len(fromEdits) == 1 { + evts = append(evts, fromEdits[0].toProtocolChangeEvent()) } else { evts = append(evts, protocol.TextDocumentContentChangeEvent{ Text: buf.text(), diff --git a/internal/lsp/source/view.go b/internal/lsp/source/view.go index bdb388bfae0..ccac54db9d8 100644 --- a/internal/lsp/source/view.go +++ b/internal/lsp/source/view.go @@ -116,10 +116,6 @@ type Snapshot interface { // GoModForFile returns the URI of the go.mod file for the given URI. GoModForFile(ctx context.Context, uri span.URI) span.URI - // BuildWorkspaceModFile builds the contents of mod file to be used for - // multi-module workspace. - BuildWorkspaceModFile(ctx context.Context) (*modfile.File, error) - // BuiltinPackage returns information about the special builtin package. BuiltinPackage(ctx context.Context) (*BuiltinPackage, error)