cmd/gg: initial commit of -infiles: flag prefix support

cmd/gg/README.md

@@ -96,8 +96,8 @@ The following is a rough list of TODOs for gg:
 * consider supporting concurrent execution of go generate directives
 * define semantics for when generated files are removed by a generator
 * add full tests for cgo
+* document how to use -outdir: and -infiles: prefixed-flags
 
-exit status 1
 
 ```
 <!-- END -->

cmd/gg/dep.go

@@ -126,3 +126,34 @@ func parseOutDirs(dir string, args []string) ([]string, error) {
 	sort.Strings(dirs)
 	return dirs, nil
 }
+
+func parseInFilePatts(dir string, args []string) ([]string, error) {
+	inFilePatts := make(map[string]bool)
+	for i := 0; i < len(args); i++ {
+		v := args[i]
+		if v == "--" {
+			break
+		}
+		if !strings.HasPrefix(v, "-"+gogenerate.FlagInFilesPrefix) {
+			continue
+		}
+		v = strings.TrimPrefix(v, "-"+gogenerate.FlagInFilesPrefix)
+		j := strings.Index(v, "=")
+		var p string
+		if j == -1 {
+			if i+1 == len(args) || args[i+1] == "--" {
+				return nil, fmt.Errorf("invalid in files pattern flag amongst: %v", args)
+			}
+			p = args[i+1]
+		} else {
+			p = v[j+1:]
+		}
+		inFilePatts[p] = true
+	}
+	var patts []string
+	for p := range inFilePatts {
+		patts = append(patts, p)
+	}
+	sort.Strings(patts)
+	return patts, nil
+}

cmd/gg/directive.go

@@ -3,16 +3,17 @@ package main
 import "fmt"
 
 type directive struct {
-	pkgName string
-	file    string
-	line    int
-	args    []string
-	gen     generator
-	outDirs []string
+	pkgName     string
+	file        string
+	line        int
+	args        []string
+	gen         generator
+	outDirs     []string
+	inFilePatts []string
 }
 
 func (d directive) String() string {
-	return fmt.Sprintf("{pkgName: %v, pos: %v:%v, args: %v, gen: [%v], outDirs: %v}", d.pkgName, d.file, d.line, d.args, d.gen, d.outDirs)
+	return fmt.Sprintf("{pkgName: %v, pos: %v:%v, args: %v, gen: [%v], outDirs: %v, inFilePatts: %v}", d.pkgName, d.file, d.line, d.args, d.gen, d.outDirs, d.inFilePatts)
 }
 
 func (d directive) HashString() string {

cmd/gg/help.go

@@ -94,5 +94,6 @@ The following is a rough list of TODOs for gg:
 * consider supporting concurrent execution of go generate directives
 * define semantics for when generated files are removed by a generator
 * add full tests for cgo
+* document how to use -outdir: and -infiles: prefixed-flags
 
 `[1:]

cmd/gg/main.go

@@ -1358,7 +1358,12 @@ func (g *gg) refreshDirectiveDeps(p *pkg, misses missingDeps) {
 			if err != nil {
 				return fmt.Errorf("failed to parse out dirs in %v:%v: %v", filepath.Join(p.Dir, file), line, err)
 			}
+			inFilePatts, err := parseInFilePatts(p.Dir, dirArgs)
+			if err != nil {
+				return fmt.Errorf("failed to parse in files in %v:%v: %v", filepath.Join(p.Dir, file), line, err)
+			}
 			dir.outDirs = outDirs
+			dir.inFilePatts = inFilePatts
 			for i, v := range outDirs {
 				if v == p.Dir {
 					outDirs = append(outDirs[:i], outDirs[i+1:]...)
@@ -1415,7 +1420,10 @@ func (g *gg) undo(d dep) {
 }
 
 func (g *gg) hashFile(hw io.Writer, dir, file string) {
-	fp := filepath.Join(dir, file)
+	fp := file
+	if !filepath.IsAbs(file) {
+		fp = filepath.Join(dir, file)
+	}
 	fmt.Fprintf(hw, "file: %v\n", fp)
 	f, err := os.Open(fp)
 	if err != nil {
@@ -1454,6 +1462,7 @@ func (g *gg) hashOutDirs(outDirs []string, p *pkg, outHash io.Writer, dirNames m
 			p.SysoFiles,
 			p.SwigFiles,
 			p.SwigCXXFiles,
+			g.inFiles(p),
 		)
 		for _, f := range inputFiles {
 			seen[f] = true
@@ -1514,6 +1523,72 @@ func (g *gg) hashOutDirs(outDirs []string, p *pkg, outHash io.Writer, dirNames m
 	return fileHash
 }
 
+func (g *gg) inFiles(p *pkg) []string {
+	// verify that the inFilePatts for the directives are contained within directories
+	// in the package's dependency graph. This also populates *pkg.inFiles
+	var patts []string
+	for _, d := range p.dirs {
+		patts = append(patts, d.inFilePatts...)
+	}
+	if len(patts) == 0 {
+		return nil
+	}
+	inFiles := make(map[string]bool)
+	work := []dep{p}
+	seen := make(map[dep]bool)
+	pkgDirs := map[string]bool{
+		".": true,
+	}
+	for len(work) > 0 {
+		w := work[0]
+		work = work[1:]
+		if seen[w] {
+			continue
+		}
+		seen[w] = true
+		switch w := w.(type) {
+		case *pkg:
+			pkgDirs[w.Dir] = true
+		case *gobinModDep:
+			pkgDirs[w.pkg.Dir] = true
+		}
+		for d := range w.Deps().deps {
+			work = append(work, d)
+		}
+	}
+
+Patts:
+	for _, patt := range patts {
+		absPatt := patt
+		if !filepath.IsAbs(patt) {
+			absPatt = filepath.Join(p.Dir, patt)
+		}
+		ms, err := filepath.Glob(absPatt)
+		if err != nil {
+			g.fatalf("failed to glob %v: %v", absPatt, err)
+		}
+		for _, m := range ms {
+			dir := filepath.Dir(m)
+			if _, ok := pkgDirs[dir]; ok {
+				inFiles[m] = true
+				continue Patts
+			}
+			var dirs []string
+			for d := range pkgDirs {
+				dirs = append(dirs, d)
+			}
+			sort.Strings(dirs)
+			g.fatalf("inFile pattern %v (%v) does not resolve to a directory within the dependency graph for %v: %v", patt, absPatt, p.ImportPath, dirs)
+		}
+	}
+	var res []string
+	for f := range inFiles {
+		res = append(res, f)
+	}
+	sort.Strings(res)
+	return res
+}
+
 func (g *gg) shouldBuildFile(path string) bool {
 	mf := imports.MatchFile(path, g.tagsMap)
 	if !mf {

cmd/gg/testdata/infiles.txt

@@ -0,0 +1,127 @@
+# Test for special -infiles: flag
+
+# first run
+gg -p 1 -trace ./...
+cmpenv stderr trace1
+cmp p1/gen_input_txt_p2.go p1/input.txt
+cmp p1/gen_another_txt_p2.go p2/another.txt
+
+cp p1/input.txt.2 p1/input.txt
+
+# second run
+gg -p 1 -trace ./...
+cmpenv stderr trace2
+cmp p1/gen_input_txt_p2.go p1/input.txt.2
+cmp p1/gen_another_txt_p2.go p2/another.txt
+
+cp p2/another.txt.2 p2/another.txt
+
+# third run
+gg -p 1 -trace ./...
+cmpenv stderr trace3
+cmp p1/gen_input_txt_p2.go p1/input.txt.2
+cmp p1/gen_another_txt_p2.go p2/another.txt.2
+
+-- go.mod --
+module mod.com
+
+-- p1/p1.go --
+package p1
+
+//go:generate gobin -m -run mod.com/p2 -infiles:test input.txt
+//go:generate gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+
+-- p1/input.txt --
+package p1
+
+// this is input.txt
+
+-- p1/input.txt.2 --
+package p1
+
+// this is input.txt.2
+
+-- p2/p2.go --
+package main
+
+import (
+	"flag"
+	"io"
+	"os"
+	"strings"
+	"path/filepath"
+)
+
+var (
+	fTest = flag.String("infiles:test", "", "our test file")
+)
+
+func main() {
+	flag.Parse()
+
+	infile, err := os.Open(*fTest)
+	if err != nil {
+		panic(err)
+	}
+	outfile, err := os.Create("gen_" + strings.Replace(filepath.Base(*fTest), ".", "_", -1) + "_p2.go")
+	if err != nil {
+		panic(err)
+	}
+	if _, err := io.Copy(outfile, infile); err != nil {
+		panic(err)
+	}
+}
+
+-- p2/another.txt --
+package p1
+
+// another.txt in p2
+-- p2/another.txt.2 --
+package p1
+
+// another.txt.2 in p2
+-- trace1 --
+go list -deps -test -json ./...
+hash {Pkg: mod.com/p2 [G]}
+hash gobinModDep gobinModDep: mod.com/p2 (mod.com/p2)
+generate {Pkg: mod.com/p1 [G]}
+run generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+run generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+generate {Pkg: mod.com/p1 [G]}
+run generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+run generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+hash {Pkg: mod.com/p1 [G]}
+-- trace2 --
+go list -deps -test -json ./...
+hash {Pkg: mod.com/p2 [G]}
+hash gobinModDep gobinModDep: mod.com/p2 (mod.com/p2)
+generate {Pkg: mod.com/p1 [G]}
+run generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+run generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+generate {Pkg: mod.com/p1 [G]}
+run generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+run generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+hash {Pkg: mod.com/p1 [G]}
+-- trace3 --
+go list -deps -test -json ./...
+hash {Pkg: mod.com/p2 [G]}
+hash gobinModDep gobinModDep: mod.com/p2 (mod.com/p2)
+generate {Pkg: mod.com/p1 [G]}
+run generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+run generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+generate {Pkg: mod.com/p1 [G]}
+run generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test input.txt
+run generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+ran generator: gobin -m -run mod.com/p2 -infiles:test ../p2/another.txt
+hash {Pkg: mod.com/p1 [G]}

gogenerate/gogenerate.go

@@ -54,9 +54,14 @@ const (
 	// to control logging verbosity.
 	FlagLog = "gglog"
 
-	// FlagOutDirPrefix is the prefix used for flags generated by OutPkgFlag.
+	// FlagOutDirPrefix is the prefix used for flags that indicate an output
+	// directory for a generator
 	FlagOutDirPrefix = "outdir:"
 
+	// FlagInFilesPrefix is the prefix used for flags that indicate a glob
+	// pattern of input files for a generator
+	FlagInFilesPrefix = "infiles:"
+
 	// FlagLicenseFile is the name of the common flag shared between go generate generators
 	// to provide a license header file.
 	FlagLicenseFile = "licenseFile"
@@ -99,18 +104,23 @@ func AnyFileIsGenerated(path string) (string, string, bool) {
 
 	fn = strings.TrimPrefix(fn, genFilePrefix)
 
-	if strings.HasSuffix(fn, "_test") {
-		fn = strings.TrimSuffix(fn, "_test")
+	// remove any test, GOOS and GOARCH suffixes
+	l := strings.Split(fn, "_")
+	if n := len(l); n > 0 && l[n-1] == "test" {
+		l = l[:n-1]
+	}
+	n := len(l)
+	if n >= 2 && imports.KnownOS[l[n-2]] && imports.KnownArch[l[n-1]] {
+		l = l[:n-2]
+	} else if n >= 1 && (imports.KnownOS[l[n-1]] || imports.KnownArch[l[n-1]]) {
+		l = l[:n-1]
 	}
 
-	// deals with the edge case gen_.go or gen__test.go
-	if fn == "" {
+	if len(l) == 0 {
 		return "", ext, false
 	}
 
-	parts := strings.Split(fn, sep)
-
-	return parts[len(parts)-1], ext, true
+	return l[len(l)-1], ext, true
 }
 
 // AnyFileGeneratedBy returns true if the base name of the supplied path is a

gogenerate/gogenerate_test.go

@@ -134,6 +134,18 @@ func TestFileGeneratedBy(t *testing.T) {
 		{"gen_bananaGen", true, "example.com/bananaGen", true},
 		{"gen_bananaGen_test", true, "bananaGen", true},
 		{"gen_bananaGen_test", true, "exmaple.com/bananaGen", true},
+		{"gen_bananaGen_linux", true, "bananaGen", true},
+		{"gen_bananaGen_linux", true, "example.com/bananaGen", true},
+		{"gen_bananaGen_linux_test", true, "bananaGen", true},
+		{"gen_bananaGen_linux_test", true, "exmaple.com/bananaGen", true},
+		{"gen_bananaGen_amd64", true, "bananaGen", true},
+		{"gen_bananaGen_amd64", true, "example.com/bananaGen", true},
+		{"gen_bananaGen_amd64_test", true, "bananaGen", true},
+		{"gen_bananaGen_amd64_test", true, "exmaple.com/bananaGen", true},
+		{"gen_bananaGen_linux_amd64", true, "bananaGen", true},
+		{"gen_bananaGen_linux_amd64", true, "example.com/bananaGen", true},
+		{"gen_bananaGen_linux_amd64_test", true, "bananaGen", true},
+		{"gen_bananaGen_linux_amd64_test", true, "exmaple.com/bananaGen", true},
 		{"gen_a_bananaGen", true, "bananaGen", true},
 		{"gen_a_bananaGen", true, "example.com/bananaGen", true},
 		{"gen_a_bananaGen_test", true, "bananaGen", true},