tests: fix diagnostic assertions in our tests

This uber PR contains a number of changes to our testscript tests:

* Drop all errlogmatch's looking for assertions where we really should
  not need them. Historically we needed these because the threading
  model of gopls was a bit broken. i.e. despite us notifying of a change
  to a file, unless we waited for diagnostics to be available then we
  couldn't guarantee that, for example, completions would be correct
* Fix up all go.mod files to include go directives so that we don't fall
  foul of golang/go#36144
* Where we do require errlogmatch's, make sure we follow the existing
  semantics per golang/go#36243 and
  golang/go#36340
* Fix up some scripts so that we don't have initial errors in the file
  and hence don't need to soak up initial diagnostics
* Use sleep $DEFAULT_ERRLOGMATCH_WAIT where we expect there to be no
  diagnostic errors
* Only use errlogmatch on diagnostic publications where we need to
  know that a file has been opened, e.g. file watching, or where we
  need to soak up initial diagnostic notifications (see
  golang/go#36243)
* Move all commented-out errlogmatch commands matching on the number of
  errors to the end of each script. We still can't enable these

_scripts/dockerRun.sh

@@ -45,10 +45,10 @@ export GOVIM_RUN_INSTALL_TESTSCRIPTS=true
 go generate $(go list ./... | grep -v 'govim/internal/golang_org_x_tools')
 go test $(go list ./... | grep -v 'govim/internal/golang_org_x_tools')
 
-if [ "${CI:-}" == "true" ] && [ "${TRAVIS_BRANCH:-}_${TRAVIS_PULL_REQUEST_BRANCH:-}" == "master_" ]
-then
+# if [ "${CI:-}" == "true" ] && [ "${TRAVIS_BRANCH:-}_${TRAVIS_PULL_REQUEST_BRANCH:-}" == "master_" ]
+# then
 	go test -race $(go list ./... | grep -v 'govim/internal/golang_org_x_tools')
-fi
+# fi
 
 go vet $(go list ./... | grep -v 'govim/internal/golang_org_x_tools')
 go run honnef.co/go/tools/cmd/staticcheck $(go list ./... | grep -v 'govim/internal/golang_org_x_tools')

cmd/govim/main_test.go

@@ -47,6 +47,8 @@ func TestMain(m *testing.M) {
 }
 
 func TestScripts(t *testing.T) {
+	t.Parallel()
+
 	var waitLock sync.Mutex
 	var waitList []func() error
 
@@ -99,7 +101,6 @@ func TestScripts(t *testing.T) {
 						"GONOSUMDB=*",
 						"HOME="+home,
 						"PLUGIN_PATH="+govimPath,
-						"CURRENT_GOPATH="+strings.TrimSpace(runCmd(t, "go", "env", "GOPATH")),
 					)
 					testPluginPath := filepath.Join(home, ".vim", "pack", "plugins", "start", "govim")
 
@@ -210,6 +211,7 @@ func TestScripts(t *testing.T) {
 }
 
 func TestInstallScripts(t *testing.T) {
+	t.Parallel()
 	if os.Getenv(EnvInstallScripts) != "true" {
 		t.Skipf("Skipping install scripts; %v != true", EnvInstallScripts)
 	}
@@ -226,12 +228,12 @@ func TestInstallScripts(t *testing.T) {
 
 	t.Run("scripts", func(t *testing.T) {
 		testscript.Run(t, testscript.Params{
-			TestWork: os.Getenv("GOTMPDIR") != "",
-			Dir:      filepath.Join("testdata", "install"),
+			Dir: filepath.Join("testdata", "install"),
 			Setup: func(e *testscript.Env) error {
 				e.Vars = append(e.Vars,
 					"PLUGIN_PATH="+govimPath,
-					"CURRENT_GOPATH="+strings.TrimSpace(runCmd(t, "go", "env", "GOPATH")),
+					"GOPATH="+strings.TrimSpace(runCmd(t, "go", "env", "GOPATH")),
+					"GOCACHE="+strings.TrimSpace(runCmd(t, "go", "env", "GOCACHE")),
 					testsetup.EnvLoadTestAPI+"=true",
 				)
 				return nil
@@ -241,8 +243,7 @@ func TestInstallScripts(t *testing.T) {
 
 	t.Run("scripts-with-gopls-from-path", func(t *testing.T) {
 		testscript.Run(t, testscript.Params{
-			TestWork: os.Getenv("GOTMPDIR") != "",
-			Dir:      filepath.Join("testdata", "install"),
+			Dir: filepath.Join("testdata", "install"),
 			Setup: func(e *testscript.Env) error {
 				var path string
 				for i := len(e.Vars) - 1; i >= 0; i-- {
@@ -260,7 +261,8 @@ func TestInstallScripts(t *testing.T) {
 				e.Vars = append(e.Vars,
 					"PATH="+path,
 					"PLUGIN_PATH="+govimPath,
-					"CURRENT_GOPATH="+strings.TrimSpace(runCmd(t, "go", "env", "GOPATH")),
+					"GOPATH="+strings.TrimSpace(runCmd(t, "go", "env", "GOPATH")),
+					"GOCACHE="+strings.TrimSpace(runCmd(t, "go", "env", "GOCACHE")),
 					string(config.EnvVarUseGoplsFromPath)+"=true",
 					testsetup.EnvLoadTestAPI+"=true",
 				)

cmd/govim/testdata/install/install_pathogen.txt

@@ -3,7 +3,6 @@
 [!net] skip
 
 env HOME=$WORK/home
-env GOPATH=$CURRENT_GOPATH
 
 exec git clone -q $PLUGIN_PATH $HOME/.vim/bundle/govim
 

cmd/govim/testdata/install/install_vim-plug.txt

@@ -3,7 +3,6 @@
 [!net] skip
 
 env HOME=$WORK/home
-env GOPATH=$CURRENT_GOPATH
 
 exec git clone -q $PLUGIN_PATH $WORK/plugin
 

cmd/govim/testdata/install/install_vim8_packages.txt

@@ -3,7 +3,6 @@
 [!net] skip
 
 env HOME=$WORK/home
-env GOPATH=$CURRENT_GOPATH
 
 exec git clone -q $PLUGIN_PATH $HOME/.vim/pack/plugins/start/govim
 

cmd/govim/testdata/install/install_vundle.txt

@@ -3,7 +3,6 @@
 [!net] skip
 
 env HOME=$WORK/home
-env GOPATH=$CURRENT_GOPATH
 
 exec git clone -q $PLUGIN_PATH $WORK/plugin
 

cmd/govim/testdata/scenario_caseinsensitivecompletion/complete.txt

@@ -2,71 +2,66 @@
 # already has the relevant import required for the completion.
 
 vim ex 'e main.go'
-errlogmatch 'PublishDiagnostics callback: &protocol.PublishDiagnosticsParams{\n\S+:\s+URI:\s+"file://'$WORK/main.go
 
 # lower completed from lo
 vim ex 'call cursor(7,1)'
 vim normal Slo
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.lower
-# Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 # UPPER completed from UP
 vim ex 'call cursor(7,1)'
 vim normal SUP
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.upper
-# Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 # lower completed from LO
 vim ex 'call cursor(7,1)'
 vim normal SLO
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.lower
-# Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 # UPPER completed from up
 vim ex 'call cursor(7,1)'
 vim normal Sup
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.upper
+
 # Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
+# errlogmatch -start -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 -- go.mod --
 module mod.com
 
+go 1.12
 -- main.go --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	//
 }
 -- main.go.lower --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	lower
 }
 -- main.go.upper --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	UPPER
 }

cmd/govim/testdata/scenario_caseinsensitivecompletion/no_fuzzy_no_deep.txt

@@ -1,19 +1,20 @@
 # Test that deep fuzzy complete does not work.
 
-cp main.go.orig main.go
+cp main.go main.go.orig
 vim ex 'e main.go'
-errlogmatch 'PublishDiagnostics callback: &protocol.PublishDiagnosticsParams{\n\S+:\s+URI:\s+"file://'$WORK/main.go
 vim ex 'call cursor(11,1)'
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.orig
+
 # Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
+# errlogmatch -start -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 -- go.mod --
 module mod.com
 
--- main.go.orig --
+go 1.12
+-- main.go --
 package main
 
 func main() {

cmd/govim/testdata/scenario_casesensitivecompletion/complete.txt

@@ -2,89 +2,84 @@
 # already has the relevant import required for the completion.
 
 vim ex 'e main.go'
-errlogmatch 'PublishDiagnostics callback: &protocol.PublishDiagnosticsParams{\n\S+:\s+URI:\s+"file://'$WORK/main.go
 
 # lower from lo
 vim ex 'call cursor(7,1)'
 vim normal Slo
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.lowercomplete
-# Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 # UPPER from UP
 vim ex 'call cursor(7,1)'
 vim normal SUP
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.uppercomplete
-# Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 # lower from LO
 vim ex 'call cursor(7,1)'
 vim normal SLO
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.lowermiss
-# Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 # UPPER from up
 vim ex 'call cursor(7,1)'
 vim normal Sup
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\"'
 vim ex 'w'
 cmp main.go main.go.uppermiss
+
 # Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
+# errlogmatch -start -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 -- go.mod --
 module mod.com
 
+go 1.12
 -- main.go --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	//
 }
 -- main.go.lowercomplete --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	lower
 }
 -- main.go.uppercomplete --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	UPPER
 }
 -- main.go.lowermiss --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	LO
 }
 -- main.go.uppermiss --
 package main
 
-func main() {
-	var lower int
-	var UPPER int
+var lower int
+var UPPER int
 
+func main() {
 	up
 }

cmd/govim/testdata/scenario_completeunimported/complete.txt

@@ -1,20 +1,21 @@
 # Test that completing of unimported std library packages works.
 
 vim ex 'e main.go'
-errlogmatch 'PublishDiagnostics callback: &protocol.PublishDiagnosticsParams{\n\S+:\s+URI:\s+"file://'$WORK/main.go
 
 # lower from lo
 vim ex 'call cursor(4,1)'
 vim normal Sfmt.Pr
 vim ex 'execute \"normal A\\<C-X>\\<C-O>\\<C-N>\\<C-N>\"'
 vim ex 'noau w'
 cmp main.go main.go.golden
+
 # Disabled pending resolution to https://github.com/golang/go/issues/34103
-# errlogmatch -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
+# errlogmatch -start -count=0 'LogMessage callback: &protocol\.LogMessageParams\{Type:(1|2), Message:".*'
 
 -- go.mod --
 module mod.com
 
+go 1.12
 -- main.go --
 package main
 

cmd/govim/testdata/scenario_default/bug_gopls_36144.txt

@@ -0,0 +1,26 @@
+# A test that causes a failure similar to the scenario described by:
+# https://github.com/golang/go/issues/36144. This is marked as "skip" because
+# otherwise we would continually break the govim build. The issue has been
+# reported, its implications and severity are known, so there is no value in
+# having a red CI build.
+#
+# What's critical here is that the go.mod file is incomplete; it is missing
+# the go directive. Hence on the initial load that file gets changed, which
+# triggers a file watcher event in govim, which triggers a recalculation of
+# diagnostics when the go.mod file is changed, which sometimes causes a
+# cancellation of the initial diagnostics for the package, or the diagnostics
+# that are sent when a file is opened.
+
+skip 'Known issue reported as https://github.com/golang/go/issues/36144'
+
+vim ex 'e main.go'
+errlogmatch 'PublishDiagnostics callback: &protocol.PublishDiagnosticsParams{\n\S+:\s+URI:\s+"file://'$WORK/main.go
+
+-- go.mod --
+module mod.com
+
+-- main.go --
+package main
+
+func main() {
+}