cmd/cue: embed packages required for get go checking

Currently, 'cue get go' loads interface types from
cmd/cue/cmd/interfaces during initialisation. Go types that satisfy
these interface types are then used as special cases for the target CUE
definition types - see the complete logic in (*extractor).altType.

Given the nature of 'cue get go' (i.e. that we are looking to import CUE
definitions from Go code), it is assumed that the command will be run in
the context of a main Go module. That is to say, 'go env GOMOD' will be
non-empty. The package arguments to 'cue get go' (and their
dependencies) will be resolved via the main Go module.

Currently, 'cue get go' assumes that the package
cuelang.org/go/cmd/cue/cmd/interfaces will be available. That is to say,
either cuelang.org/go is the main module or, more likely, that
cuelang.org/go is a module dependency of the main module. However, this
is an unnecessary restriction and one that is potentially incorrect.
After all, the user is using a specific version of cmd/cue - loading a
different version of cuelang.org/go seems likely to lead to version-skew
related problems.

This change switches cmd/cue to instead load
cuelang.org/go/cmd/cue/cmd/interfaces via a simple embedding of the
GoFiles in that package.

As part of the same change we surface all loading and type checking
errors for either the embedded cuelang.org/go/cmd/cue/cmd/interfaces or
the arguments to 'cue get go'.

We also migrate the TestGetGo to be a testscript test:

    cmd/cue/cmd/testdata/script/get_go_types.txt

This test encompasses all the checks for the various rules/edge cases of
type mappins in 'cue get go', e.g. the mapping of certain Go types to _
or string.

We also introduce more basic tests that verify the behaviour of 'cue get
go' when given a main module package, as well as a non-main module
package (a dependency that should already be resolvable via the main Go
module).

Note: it doesn't seem worth adding/testing GOPATH support given that modules
will be GA in Go 1.16. Indeed we have not had explicit GOPATH tests for
some time/ever. This change also signifies an end to our support for
GOPATH - if it works it will be by accident.

In a later change we will enforce the entirely reasonable requirement
that the main Go module (which we are using to resolve the arguments to
'cue get go') and the main CUE module into which we are writing the
result of 'cue get go' (either within the main module or the cue.mod/pkg
directory) are aligned in terms of their module paths. This will allow
us to drop the --local flag as part of our renaming 'cue get go' to 'cue
import go'.

We leave the enforcing of setting GOFLAGS=-mod=readonly whilst loading
the arguments to 'cue get go' to a later change. This is preparation
towards 'cue import go' but will require a bit more work.

With thanks to @verdverm and @NicolasRouquette for the analysis and
suggestions that went into this fix.

Fixes #621

Change-Id: I630e2cb167c05d63186aa356eaf70a58749b61c5
Reviewed-on: https://cue-review.googlesource.com/c/cue/+/8021
Reviewed-by: CUE cueckoo <cueckoo@gmail.com>
Reviewed-by: Paul Jolly <paul@myitcv.org.uk>

cmd/cue/cmd/get_go.go

@@ -225,8 +225,6 @@ const (
 	flagLocal   flagName = "local"
 )
 
-var cueTestRoot string // the CUE module root for test purposes.
-
 func (e *extractor) initExclusions(str string) {
 	e.exclude = str
 	for _, re := range strings.Split(str, ",") {
@@ -281,13 +279,63 @@ func (e *extractor) usedPkg(pkg string) {
 	e.usedPkgs[pkg] = true
 }
 
-func initInterfaces() error {
+const cueGoMod = `
+module cuelang.org/go
+
+go 1.14
+`
+
+//go:generate go run cuelang.org/go/internal/cmd/embedpkg cuelang.org/go/cmd/cue/cmd/interfaces
+
+func initInterfaces() (err error) {
+	// tempdir needed for overlay
+	tmpDir, err := ioutil.TempDir("", "cuelang")
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		rerr := os.RemoveAll(tmpDir)
+		if err == nil {
+			err = rerr
+		}
+	}()
+
+	// write the cuelang go.mod
+	err = ioutil.WriteFile(filepath.Join(tmpDir, "go.mod"), []byte(cueGoMod), 0666)
+	if err != nil {
+		return err
+	}
+
+	for fn, contents := range interfacesFiles {
+		fn = filepath.Join(tmpDir, filepath.FromSlash(fn))
+		dir := filepath.Dir(fn)
+		if err := os.MkdirAll(dir, 0777); err != nil {
+			return err
+		}
+
+		if err = ioutil.WriteFile(fn, contents, 0666); err != nil {
+			return err
+		}
+	}
+
 	cfg := &packages.Config{
-		Mode: packages.LoadAllSyntax,
+		Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles |
+			packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes |
+			packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedDeps,
+		Dir: filepath.Join(tmpDir),
 	}
+
 	p, err := packages.Load(cfg, "cuelang.org/go/cmd/cue/cmd/interfaces")
 	if err != nil {
-		return err
+		return fmt.Errorf("error loading embedded cuelang.org/go/cmd/cue/cmd/interfaces package: %w", err)
+	}
+	if len(p[0].Errors) > 0 {
+		var buf bytes.Buffer
+		for _, e := range p[0].Errors {
+			fmt.Fprintf(&buf, "\t%v\n", e)
+		}
+		return fmt.Errorf("error loading embedded cuelang.org/go/cmd/cue/cmd/interfaces package:\n%s", buf.String())
 	}
 
 	for e, tt := range p[0].TypesInfo.Types {
@@ -321,6 +369,16 @@ var (
 // - consider not including types with any dropped fields.
 
 func extract(cmd *Command, args []string) error {
+	// TODO the CUE load using "." (below) assumes that a CUE module and a Go
+	// module will exist within the same directory (more precisely a Go module
+	// could be nested within a CUE module), such that the module path in any
+	// subdirectory below the current directory will be the same.  This seems an
+	// entirely reasonable restriction, but also one that we should enforce.
+	//
+	// Enforcing this restriction also makes --local entirely redundant.
+
+	// command specifies a Go package(s) that belong to the main module
+	// and where for some reason the
 	// determine module root:
 	binst := loadFromArgs(cmd, []string{"."}, nil)[0]
 
@@ -331,18 +389,25 @@ func extract(cmd *Command, args []string) error {
 	// TODO: require explicitly set root.
 	root := binst.Root
 
-	// Override root in testing mode.
-	if cueTestRoot != "" {
-		root = cueTestRoot
-	}
-
 	cfg := &packages.Config{
-		Mode: packages.LoadAllSyntax | packages.NeedModule,
+		Mode: packages.NeedName | packages.NeedFiles | packages.NeedCompiledGoFiles |
+			packages.NeedImports | packages.NeedTypes | packages.NeedTypesSizes |
+			packages.NeedSyntax | packages.NeedTypesInfo | packages.NeedDeps |
+			packages.NeedModule,
 	}
 	pkgs, err := packages.Load(cfg, args...)
 	if err != nil {
 		return err
 	}
+	var errs []string
+	for _, P := range pkgs {
+		if len(P.Errors) > 0 {
+			errs = append(errs, fmt.Sprintf("\t%s: %v", P.PkgPath, P.Errors))
+		}
+	}
+	if len(errs) > 0 {
+		return fmt.Errorf("could not load Go packages:\n%s", strings.Join(errs, "\n"))
+	}
 
 	e := extractor{
 		cmd:    cmd,
@@ -408,7 +473,7 @@ func (e *extractor) extractPkg(root string, p *packages.Package) error {
 		}
 	}
 
-	if err := os.MkdirAll(dir, 0755); err != nil {
+	if err := os.MkdirAll(dir, 0777); err != nil {
 		return err
 	}
 
@@ -481,7 +546,7 @@ func (e *extractor) extractPkg(root string, p *packages.Package) error {
 		if err != nil {
 			return err
 		}
-		err = ioutil.WriteFile(filepath.Join(dir, file), b, 0644)
+		err = ioutil.WriteFile(filepath.Join(dir, file), b, 0666)
 		if err != nil {
 			return err
 		}
@@ -539,7 +604,7 @@ func (e *extractor) importCUEFiles(p *packages.Package, dir, args string) error
 				w.Write(b)
 
 				dst := filepath.Join(dir, file)
-				if err := ioutil.WriteFile(dst, w.Bytes(), 0644); err != nil {
+				if err := ioutil.WriteFile(dst, w.Bytes(), 0666); err != nil {
 					return err
 				}
 			}