gopls: use go generate consistently

This change establishes that "go generate ./..." from within the gopls
directory will update all generated code in the module to its
canonical version. (Beware that this command doesn't work in the
parent directory due to golang/go#58723.)

Details:

- The main 'generate' command now includes the canonical git ref
  name of the LSP protocol used by gopls, and by default it
  clones the repo at this version instead of using the version in
  pjw's home directory. ;-)
  It should deliver identical results for all users without setup.
  The "generated" comment includes an accurate URL.
  The timestamp is removed since it is nondeterministic.
- In addition to the commit hash, the output now reports the git
  ref (branch/tag) name, which is not the same as the
  metadata.version (LSP protocol version) string.
- The go:generate comments are now more prominent in their
  respective files. We hide several that appear in testdata from
  the go command by splitting string literals.
- Improved command documentation for generators.
- Use a slice not map for genTypes so that protocol generation
  is deterministic. (Previously the "created for" comments
  would alternate between TextDocumentFilter_Item{0,1}.)

The generated output has changed trivially ("//x" -> "// x")
because the most recent committed generated files was stale
wrt recent generator changes. Also, line number comments now
start at 1, not zero (eliminating a TODO).

Change-Id: I40e9454d0042507bd64e0d8fab3ce6b1fbe87ddc
Reviewed-on: https://go-review.googlesource.com/c/tools/+/471115
Run-TryBot: Alan Donovan <adonovan@google.com>
gopls-CI: kokoro <noreply+kokoro@google.com>
TryBot-Result: Gopher Robot <gobot@golang.org>
Reviewed-by: Robert Findley <rfindley@google.com>
Reviewed-by: Peter Weinberger <pjw@google.com>

gopls/internal/lsp/command/generate.go

@@ -8,8 +8,7 @@
 package main
 
 import (
-	"fmt"
-	"io/ioutil"
+	"log"
 	"os"
 
 	"golang.org/x/tools/gopls/internal/lsp/command/gen"
@@ -18,8 +17,9 @@ import (
 func main() {
 	content, err := gen.Generate()
 	if err != nil {
-		fmt.Fprintf(os.Stderr, "%v\n", err)
-		os.Exit(1)
+		log.Fatal(err)
+	}
+	if err := os.WriteFile("command_gen.go", content, 0644); err != nil {
+		log.Fatal(err)
 	}
-	ioutil.WriteFile("command_gen.go", content, 0644)
 }

gopls/internal/lsp/command/interface.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:generate go run -tags=generate generate.go
+
 // Package command defines the interface provided by gopls for the
 // workspace/executeCommand LSP request.
 //
@@ -12,8 +14,6 @@
 // also provided by this package, via code generation.
 package command
 
-//go:generate go run -tags=generate generate.go
-
 import (
 	"context"
 

gopls/internal/lsp/helper/README.md

@@ -1,12 +1,14 @@
 # Generate server_gen.go
 
-`helper` generates boilerplate code for server.go by processing the
-generated code in `protocol/tsserver.go`.
+`helper` generates the file `../server_gen.go` (in package
+`internal/lsp`) which contains stub declarations of server methods.
 
-First, build `helper` in this directory (`go build .`).
+To invoke it, run `go generate` in the `gopls/internal/lsp` directory.
 
-In directory `lsp`, executing `go generate server.go` generates the stylized file
-`server_gen.go` that contains stubs for type `Server`.
+It is derived from `gopls/internal/lsp/protocol/tsserver.go`, which
+itself is generated from the protocol downloaded from VSCode, so be
+sure to run `go generate` in the protocol first. Or run `go generate
+./...` twice in the gopls directory.
 
 It decides what stubs are needed and their signatures
 by looking at the `Server` interface (`-t` flag). These all look somewhat like

gopls/internal/lsp/helper/helper.go

@@ -2,8 +2,14 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
-// Invoke with //go:generate helper/helper -t Server -d protocol/tsserver.go -u lsp -o server_gen.go
-// invoke in internal/lsp
+// The helper command generates the declaration of the concrete
+// 'server' type that implements the abstract Server interface defined
+// in protocol/tsserver.go (which is itself generated from the LSP
+// protocol).
+//
+// To run, invoke "go generate" in the parent (lsp) directory.
+//
+// TODO(adonovan): merge this into the main LSP generator.
 package main
 
 import (
@@ -33,7 +39,7 @@ func main() {
 	flag.Parse()
 	if *typ == "" || *def == "" || *use == "" || *out == "" {
 		flag.PrintDefaults()
-		return
+		os.Exit(1)
 	}
 	// read the type definition and see what methods we're looking for
 	doTypes()

gopls/internal/lsp/protocol/doc.go

@@ -2,6 +2,8 @@
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
+//go:generate go run ./generate
+
 // Package protocol contains the structs that map directly to the
 // request and response messages of the Language Server Protocol.
 //

gopls/internal/lsp/protocol/generate/README.md

@@ -10,9 +10,7 @@ The protocol is described in a
 [web page](https://microsoft.github.io/language-server-protocol/specifications/lsp/3.18/specification/),
 in words, and in a json file (metaModel.json) available either linked towards the bottom of the
 web page, or in the vscode-languageserver-node repository. This code uses the latter so the
-exact version can be tied to a githash. Download the repository with
-
-`git clone https://github.com/microsoft/vscode-languageserver-node.git`
+exact version can be tied to a githash. By default, the command will download the `github.com/microsoft/vscode-languageserver-node` repository to a temporary directory.
 
 The specification has five sections
 1. Requests, which describe the Request and Response types for request methods (e.g., *textDocument/didChange*),

gopls/internal/lsp/protocol/generate/main.go

@@ -5,6 +5,10 @@
 //go:build go1.19
 // +build go1.19
 
+// The generate command generates Go declarations from VSCode's
+// description of the Language Server Protocol.
+//
+// To run it, type 'go generate' in the parent (protocol) directory.
 package main
 
 import (
@@ -15,15 +19,23 @@ import (
 	"go/format"
 	"log"
 	"os"
+	"os/exec"
 	"path/filepath"
 	"strings"
-	"time"
 )
 
+const vscodeRepo = "https://github.com/microsoft/vscode-languageserver-node"
+
+// lspGitRef names a branch or tag in vscodeRepo.
+// It implicitly determines the protocol version of the LSP used by gopls.
+// For example, tag release/protocol/3.17.3 of the repo defines protocol version 3.17.0.
+// (Point releases are reflected in the git tag version even when they are cosmetic
+// and don't change the protocol.)
+var lspGitRef = "release/protocol/3.17.3-next.6"
+
 var (
-	// git clone https://github.com/microsoft/vscode-languageserver-node.git
-	repodir   = flag.String("d", "", "directory of vscode-languageserver-node")
-	outputdir = flag.String("o", "gen", "output directory")
+	repodir   = flag.String("d", "", "directory containing clone of "+vscodeRepo)
+	outputdir = flag.String("o", ".", "output directory")
 	// PJW: not for real code
 	cmpdir = flag.String("c", "", "directory of earlier code")
 	doboth = flag.String("b", "", "generate and compare")
@@ -37,8 +49,26 @@ func main() {
 }
 
 func processinline() {
+	// A local repository may be specified during debugging.
+	// The default behavior is to download the canonical version.
 	if *repodir == "" {
-		*repodir = filepath.Join(os.Getenv("HOME"), "vscode-languageserver-node")
+		tmpdir, err := os.MkdirTemp("", "")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(tmpdir) // ignore error
+
+		// Clone the repository.
+		cmd := exec.Command("git", "clone", "--quiet", "--depth=1", "-c", "advice.detachedHead=false", vscodeRepo, "--branch="+lspGitRef, "--single-branch", tmpdir)
+		cmd.Stdout = os.Stderr
+		cmd.Stderr = os.Stderr
+		if err := cmd.Run(); err != nil {
+			log.Fatal(err)
+		}
+
+		*repodir = tmpdir
+	} else {
+		lspGitRef = fmt.Sprintf("(not git, local dir %s)", *repodir)
 	}
 
 	model := parse(filepath.Join(*repodir, "protocol/metaModel.json"))
@@ -240,21 +270,25 @@ func fileHeader(model Model) string {
 		log.Fatalf("githash cannot be recovered from %s", fname)
 	}
 
-	format := `// Copyright 2022 The Go Authors. All rights reserved.
+	format := `// Copyright 2023 The Go Authors. All rights reserved.
 // Use of this source code is governed by a BSD-style
 // license that can be found in the LICENSE file.
 
 // Code generated for LSP. DO NOT EDIT.
 
 package protocol
 
-// Code generated from version %s of protocol/metaModel.json.
-// git hash %s (as of %s)
-
+// Code generated from %[1]s at ref %[2]s (hash %[3]s).
+// %[4]s/blob/%[2]s/%[1]s
+// LSP metaData.version = %[5]s.
 
 `
-	now := time.Now().Format("2006-01-02")
-	return fmt.Sprintf(format, model.Version.Version, githash, now)
+	return fmt.Sprintf(format,
+		"protocol/metaModel.json", // 1
+		lspGitRef,                 // 2
+		githash,                   // 3
+		vscodeRepo,                // 4
+		model.Version.Version)     // 5
 }
 
 func parse(fname string) Model {

gopls/internal/lsp/protocol/generate/typenames.go

@@ -14,7 +14,7 @@ import (
 )
 
 var typeNames = make(map[*Type]string)
-var genTypes = make(map[*Type]*newType)
+var genTypes []*newType
 
 func findTypeNames(model Model) {
 	for _, s := range model.Structures {
@@ -75,41 +75,41 @@ func nameType(t *Type, path []string) string {
 		for _, it := range t.Items {
 			nameType(it, append(path, "Item"))
 		}
-		genTypes[t] = &newType{
+		genTypes = append(genTypes, &newType{
 			name:  nm,
 			typ:   t,
 			kind:  "and",
 			items: t.Items,
 			line:  t.Line,
-		}
+		})
 		return nm
 	case "literal":
 		nm := nameFromPath("Lit", path)
 		typeNames[t] = nm
 		for _, p := range t.Value.(ParseLiteral).Properties {
 			nameType(p.Type, append(path, p.Name))
 		}
-		genTypes[t] = &newType{
+		genTypes = append(genTypes, &newType{
 			name:       nm,
 			typ:        t,
 			kind:       "literal",
 			properties: t.Value.(ParseLiteral).Properties,
 			line:       t.Line,
-		}
+		})
 		return nm
 	case "tuple":
 		nm := nameFromPath("Tuple", path)
 		typeNames[t] = nm
 		for _, it := range t.Items {
 			nameType(it, append(path, "Item"))
 		}
-		genTypes[t] = &newType{
+		genTypes = append(genTypes, &newType{
 			name:  nm,
 			typ:   t,
 			kind:  "tuple",
 			items: t.Items,
 			line:  t.Line,
-		}
+		})
 		return nm
 	case "or":
 		nm := nameFromPath("Or", path)
@@ -159,13 +159,13 @@ func nameType(t *Type, path []string) string {
 				return newNm
 			}
 		}
-		genTypes[t] = &newType{
+		genTypes = append(genTypes, &newType{
 			name:  nm,
 			typ:   t,
 			kind:  "or",
 			items: t.Items,
 			line:  t.Line,
-		}
+		})
 		return nm
 	case "stringLiteral": // a single type, like 'kind' or 'rename'
 		typeNames[t] = "string"

gopls/internal/lsp/protocol/generate/types.go

@@ -142,8 +142,7 @@ func addLineNumbers(buf []byte) []byte {
 	// always followed by a newline. There are other {s embedded in strings.
 	// json.Token does not return \n, or :, or , so using it would
 	// require parsing the json to reconstruct the missing information.
-	// TODO(pjw): should linecnt start at 1 (editor) or 0 (compatibility)?
-	for linecnt, i := 0, 0; i < len(buf); i++ {
+	for linecnt, i := 1, 0; i < len(buf); i++ {
 		ans = append(ans, buf[i])
 		switch buf[i] {
 		case '{':