feat: precompile imports and fix gnodev test

cmd/gnodev/main.go

@@ -35,7 +35,7 @@ var mainApps AppList = []AppItem{
 		App:      precompileApp,
 		Name:     "precompile",
 		Desc:     "precompile .gno to .go",
-		Defaults: defaultPrecompileOptions,
+		Defaults: defaultPrecompileFlags,
 	},
 	{
 		App:      testApp,

cmd/gnodev/main_test.go

@@ -35,7 +35,7 @@ func TestMain(t *testing.T) {
 		// --help
 		{args: []string{"build", "--help"}, stdoutShouldContain: "# buildOptions options\n-"},
 		{args: []string{"test", "--help"}, stdoutShouldContain: "# testOptions options\n-"},
-		{args: []string{"precompile", "--help"}, stdoutShouldContain: "# precompileOptions options\n-"},
+		{args: []string{"precompile", "--help"}, stdoutShouldContain: "# precompileFlags options\n-"},
 		{args: []string{"repl", "--help"}, stdoutShouldContain: "# replOptions options\n-"},
 
 		// custom

cmd/gnodev/precompile.go

@@ -12,24 +12,52 @@ import (
 	gno "github.com/gnolang/gno/pkgs/gnolang"
 )
 
-type precompileOptions struct {
+type importPath string
+
+type precompileFlags struct {
 	Verbose     bool   `flag:"verbose" help:"verbose"`
 	SkipFmt     bool   `flag:"skip-fmt" help:"do not check syntax of generated .go files"`
+	SkipImports bool   `flag:"skip-imports" help:"do not precompile imports recursively"`
 	GoBinary    string `flag:"go-binary" help:"go binary to use for building"`
 	GofmtBinary string `flag:"go-binary" help:"gofmt binary to use for syntax checking"`
 	Output      string `flag:"output" help:"output directory"`
 }
 
-var defaultPrecompileOptions = precompileOptions{
+var defaultPrecompileFlags = precompileFlags{
 	Verbose:     false,
 	SkipFmt:     false,
+	SkipImports: false,
 	GoBinary:    "go",
 	GofmtBinary: "gofmt",
 	Output:      ".",
 }
 
-func precompileApp(cmd *command.Command, args []string, iopts interface{}) error {
-	opts := iopts.(precompileOptions)
+type precompileOptions struct {
+	flags precompileFlags
+	// precompiled is the set of packages already
+	// precompiled from .gno to .go.
+	precompiled map[importPath]struct{}
+}
+
+func newPrecompileOptions(flags precompileFlags) *precompileOptions {
+	return &precompileOptions{flags, map[importPath]struct{}{}}
+}
+
+func (p *precompileOptions) getFlags() precompileFlags {
+	return p.flags
+}
+
+func (p *precompileOptions) isPrecompiled(pkg importPath) bool {
+	_, precompiled := p.precompiled[pkg]
+	return precompiled
+}
+
+func (p *precompileOptions) markAsPrecompiled(pkg importPath) {
+	p.precompiled[pkg] = struct{}{}
+}
+
+func precompileApp(cmd *command.Command, args []string, f interface{}) error {
+	flags := f.(precompileFlags)
 	if len(args) < 1 {
 		cmd.ErrPrintfln("Usage: precompile [precompile flags] [packages]")
 		return errors.New("invalid args")
@@ -41,6 +69,7 @@ func precompileApp(cmd *command.Command, args []string, iopts interface{}) error
 		return fmt.Errorf("list paths: %w", err)
 	}
 
+	opts := newPrecompileOptions(flags)
 	errCount := 0
 	for _, filepath := range paths {
 		err = precompileFile(filepath, opts)
@@ -58,8 +87,13 @@ func precompileApp(cmd *command.Command, args []string, iopts interface{}) error
 	return nil
 }
 
-func precompilePkg(pkgPath string, opts precompileOptions) error {
-	files, err := filepath.Glob(filepath.Join(pkgPath, "*.gno"))
+func precompilePkg(pkgPath importPath, opts *precompileOptions) error {
+	if opts.isPrecompiled(pkgPath) {
+		return nil
+	}
+	opts.markAsPrecompiled(pkgPath)
+
+	files, err := filepath.Glob(filepath.Join(string(pkgPath), "*.gno"))
 	if err != nil {
 		log.Fatal(err)
 	}
@@ -73,15 +107,14 @@ func precompilePkg(pkgPath string, opts precompileOptions) error {
 	return nil
 }
 
-func precompileFile(srcPath string, opts precompileOptions) error {
-	shouldCheckFmt := !opts.SkipFmt
-	verbose := opts.Verbose
-	gofmt := opts.GofmtBinary
+func precompileFile(srcPath string, opts *precompileOptions) error {
+	flags := opts.getFlags()
+	gofmt := flags.GofmtBinary
 	if gofmt == "" {
-		gofmt = defaultPrecompileOptions.GofmtBinary
+		gofmt = defaultPrecompileFlags.GofmtBinary
 	}
 
-	if verbose {
+	if flags.Verbose {
 		fmt.Fprintf(os.Stderr, "%s\n", srcPath)
 	}
 
@@ -108,31 +141,44 @@ func precompileFile(srcPath string, opts precompileOptions) error {
 	}
 
 	// preprocess.
-	transformed, err := gno.Precompile(string(source), tags, srcPath)
+	precompileRes, err := gno.Precompile(string(source), tags, srcPath)
 	if err != nil {
 		return fmt.Errorf("%w", err)
 	}
 
-	// write .go file.
+	// resolve target path
 	var targetPath string
-	if opts.Output != defaultPrecompileOptions.Output {
-		targetPath = filepath.Join(opts.Output, targetFilename)
+	if flags.Output != defaultPrecompileFlags.Output {
+		path, err := ResolvePath(flags.Output, importPath(filepath.Dir(srcPath)))
+		if err != nil {
+			return fmt.Errorf("resolve output path: %w", err)
+		}
+		targetPath = filepath.Join(path, targetFilename)
 	} else {
-		dir := filepath.Dir(srcPath)
-		targetPath = filepath.Join(dir, targetFilename)
+		targetPath = filepath.Join(filepath.Dir(srcPath), targetFilename)
 	}
-	err = os.WriteFile(targetPath, []byte(transformed), 0o644)
+
+	// write .go file.
+	err = WriteDirFile(targetPath, []byte(precompileRes.Translated))
 	if err != nil {
 		return fmt.Errorf("write .go file: %w", err)
 	}
 
-	// check .go fmt.
-	if shouldCheckFmt {
+	// check .go fmt, if `SkipFmt` sets to false.
+	if !flags.SkipFmt {
 		err = gno.PrecompileVerifyFile(targetPath, gofmt)
 		if err != nil {
 			return fmt.Errorf("check .go file: %w", err)
 		}
 	}
 
+	// precompile imported packages, if `SkipImports` sets to false
+	if !flags.SkipImports {
+		importPaths := getPathsFromImportSpec(precompileRes.Imports)
+		for _, path := range importPaths {
+			precompilePkg(path, opts)
+		}
+	}
+
 	return nil
 }

cmd/gnodev/test.go

@@ -56,6 +56,13 @@ func testApp(cmd *command.Command, args []string, iopts interface{}) error {
 	}
 	defer os.RemoveAll(tempdirRoot)
 
+	// go.mod
+	modPath := filepath.Join(tempdirRoot, "go.mod")
+	err = makeTestGoMod(modPath, gno.ImportPrefix, "1.18")
+	if err != nil {
+		return fmt.Errorf("write .mod file: %w", err)
+	}
+
 	// guess opts.RootDir
 	if opts.RootDir == "" {
 		opts.RootDir = guessRootDir()
@@ -80,15 +87,10 @@ func testApp(cmd *command.Command, args []string, iopts interface{}) error {
 			if verbose {
 				cmd.ErrPrintfln("=== PREC  %s", pkgPath)
 			}
-			pkgPathSafe := strings.ReplaceAll(pkgPath, "/", "~")
-			tempdir := filepath.Join(tempdirRoot, pkgPathSafe)
-			if err = os.MkdirAll(tempdir, 0o755); err != nil {
-				log.Fatal(err)
-			}
-			precompileOpts := precompileOptions{
-				Output: tempdir,
-			}
-			err := precompilePkg(pkgPath, precompileOpts)
+			precompileOpts := newPrecompileOptions(precompileFlags{
+				Output: tempdirRoot,
+			})
+			err := precompilePkg(importPath(pkgPath), precompileOpts)
 			if err != nil {
 				cmd.ErrPrintln(err)
 				cmd.ErrPrintln("FAIL")
@@ -101,7 +103,11 @@ func testApp(cmd *command.Command, args []string, iopts interface{}) error {
 			if verbose {
 				cmd.ErrPrintfln("=== BUILD %s", pkgPath)
 			}
-			err = goBuildFileOrPkg(tempdir, defaultBuildOptions)
+			tempDir, err := ResolvePath(tempdirRoot, importPath(pkgPath))
+			if err != nil {
+				errors.New("cannot resolve build dir")
+			}
+			err = goBuildFileOrPkg(tempDir, defaultBuildOptions)
 			if err != nil {
 				cmd.ErrPrintln(err)
 				cmd.ErrPrintln("FAIL")

cmd/gnodev/util.go

@@ -2,13 +2,16 @@ package main
 
 import (
 	"fmt"
+	"go/ast"
 	"io/fs"
 	"log"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strings"
 	"time"
+
+	gno "github.com/gnolang/gno/pkgs/gnolang"
 )
 
 func isGnoFile(f fs.DirEntry) bool {
@@ -104,3 +107,55 @@ func guessRootDir() string {
 	rootDir := strings.TrimSpace(string(out))
 	return rootDir
 }
+
+// makeTestGoMod creates the temporary go.mod for test
+func makeTestGoMod(path string, packageName string, goversion string) error {
+	content := fmt.Sprintf("module %s\n\ngo %s\n", packageName, goversion)
+	return os.WriteFile(path, []byte(content), 0o644)
+}
+
+// getPathsFromImportSpec derive and returns ImportPaths
+// without ImportPrefix from *ast.ImportSpec
+func getPathsFromImportSpec(importSpec []*ast.ImportSpec) (importPaths []importPath) {
+	for _, i := range importSpec {
+		path := i.Path.Value[1 : len(i.Path.Value)-1] // trim leading and trailing `"`
+		if strings.HasPrefix(path, gno.ImportPrefix) {
+			res := strings.TrimPrefix(path, gno.ImportPrefix)
+			importPaths = append(importPaths, importPath("."+res))
+		}
+	}
+	return
+}
+
+// ResolvePath joins the output dir with relative pkg path
+// e.g
+// Output Dir: Temp/gno-precompile
+// Pkg Path: ../example/gno.land/p/pkg
+// Returns -> Temp/gno-precompile/example/gno.land/p/pkg
+func ResolvePath(output string, path importPath) (string, error) {
+	absOutput, err := filepath.Abs(output)
+	if err != nil {
+		return "", err
+	}
+	absPkgPath, err := filepath.Abs(string(path))
+	if err != nil {
+		return "", err
+	}
+	pkgPath := strings.TrimPrefix(absPkgPath, guessRootDir())
+
+	return filepath.Join(absOutput, pkgPath), nil
+}
+
+// WriteDirFile write file to the path and also create
+// directory if needed. with:
+// Dir perm -> 0755; File perm -> 0o644
+func WriteDirFile(pathWithName string, data []byte) error {
+	path := filepath.Dir(pathWithName)
+
+	// Create Dir if not exists
+	if _, err := os.Stat(path); os.IsNotExist(err) {
+		os.MkdirAll(path, 0o755)
+	}
+
+	return os.WriteFile(pathWithName, []byte(data), 0o644)
+}

examples/gno.land/r/demo/x/upgrade/upgrade-a/integration_test.gno

@@ -1,4 +1,4 @@
-package upgrade
+package upgradea
 
 import (
 	v1 "gno.land/r/demo/x/upgrade/upgrade-a/v1"

pkgs/gnolang/precompile.go

@@ -64,9 +64,32 @@ var importPrefixWhitelist = []string{
 	"github.com/gnolang/gno/_test",
 }
 
+const ImportPrefix = "github.com/gnolang/gno"
+
+type precompileResult struct {
+	Imports    []*ast.ImportSpec
+	Translated string
+}
+
 // TODO: func PrecompileFile: supports caching.
 // TODO: func PrecompilePkg: supports directories.
 
+func guessRootDir(fileOrPkg string, goBinary string) (string, error) {
+	abs, err := filepath.Abs(fileOrPkg)
+	if err != nil {
+		return "", err
+	}
+	args := []string{"list", "-m", "-mod=mod", "-f", "{{.Dir}}", ImportPrefix}
+	cmd := exec.Command(goBinary, args...)
+	cmd.Dir = abs
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return "", fmt.Errorf("can't guess --root-dir")
+	}
+	rootDir := strings.TrimSpace(string(out))
+	return rootDir, nil
+}
+
 func PrecompileAndCheckMempkg(mempkg *std.MemPackage) error {
 	gofmt := "gofmt"
 
@@ -81,13 +104,13 @@ func PrecompileAndCheckMempkg(mempkg *std.MemPackage) error {
 		if !strings.HasSuffix(mfile.Name, ".gno") {
 			continue // skip spurious file.
 		}
-		translated, err := Precompile(string(mfile.Body), "gno,tmp", mfile.Name)
+		res, err := Precompile(string(mfile.Body), "gno,tmp", mfile.Name)
 		if err != nil {
 			errs = multierr.Append(errs, err)
 			continue
 		}
 		tmpFile := filepath.Join(tmpDir, mfile.Name)
-		err = os.WriteFile(tmpFile, []byte(translated), 0o644)
+		err = os.WriteFile(tmpFile, []byte(res.Translated), 0o644)
 		if err != nil {
 			errs = multierr.Append(errs, err)
 			continue
@@ -105,29 +128,34 @@ func PrecompileAndCheckMempkg(mempkg *std.MemPackage) error {
 	return nil
 }
 
-func Precompile(source string, tags string, filename string) (string, error) {
+func Precompile(source string, tags string, filename string) (*precompileResult, error) {
 	var out bytes.Buffer
 
 	fset := token.NewFileSet()
 	f, err := parser.ParseFile(fset, "tmp.gno", source, parser.ParseComments)
 	if err != nil {
-		return "", fmt.Errorf("parse: %w", err)
+		return nil, fmt.Errorf("parse: %w", err)
 	}
 
 	isTestFile := strings.HasSuffix(filename, "_test.gno") || strings.HasSuffix(filename, "_filetest.gno")
 	shouldCheckWhitelist := !isTestFile
 
 	transformed, err := precompileAST(fset, f, shouldCheckWhitelist)
 	if err != nil {
-		return "", fmt.Errorf("%w", err)
+		return nil, fmt.Errorf("%w", err)
 	}
 
 	_, err = out.WriteString("// Code generated by github.com/gnolang/gno. DO NOT EDIT.\n\n//go:build " + tags + "\n// +build " + tags + "\n\n")
 	if err != nil {
-		return "", fmt.Errorf("write to buffer: %w", err)
+		return nil, fmt.Errorf("write to buffer: %w", err)
 	}
 	err = format.Node(&out, fset, transformed)
-	return out.String(), nil
+
+	res := &precompileResult{
+		Imports:    f.Imports,
+		Translated: out.String(),
+	}
+	return res, nil
 }
 
 // PrecompileVerifyFile tries to run `go fmt` against a precompiled .go file.
@@ -190,6 +218,10 @@ func PrecompileBuildPackage(fileOrPkg string, goBinary string) error {
 	sort.Strings(files)
 	args := append([]string{"build", "-v", "-tags=gno"}, files...)
 	cmd := exec.Command(goBinary, args...)
+	rootDir, err := guessRootDir(fileOrPkg, goBinary)
+	if err == nil {
+		cmd.Dir = rootDir
+	}
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, string(out))