Improve file support and misc changes (#35)

* Checkpoint.

* Checkpoint.

* Checkpoint.

* Tests work again.

* Fix TestGoYAML

* Make YAMLStringList and YAMLStringListFromNodes methods on Controller.

* Update snapshots.

_testdata/compile/go.mod

@@ -2,18 +2,18 @@ module q
 
 go 1.20
 
-require github.com/bobg/fab v0.29.0
+require github.com/bobg/fab v0.48.4
 
 require (
 	github.com/benbjohnson/clock v1.3.0
 	github.com/bobg/errors v0.10.0
-	github.com/bobg/go-generics/v2 v2.1.1
+	github.com/bobg/go-generics/v2 v2.1.2
 	github.com/bobg/tsdecls v0.1.0
+	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
+	github.com/davecgh/go-spew v1.1.1
 	github.com/gibson042/canonicaljson-go v1.0.3
-	github.com/mattn/go-shellwords v1.0.12
 	github.com/mattn/go-sqlite3 v1.14.15
 	github.com/otiai10/copy v1.7.0
-	github.com/pressly/goose/v3 v3.6.1
 	golang.org/x/mod v0.6.0
 	golang.org/x/tools v0.2.0
 	gopkg.in/yaml.v3 v3.0.1
@@ -23,8 +23,8 @@ require (
 	github.com/bobg/go-generics v1.5.0 // indirect
 	github.com/fatih/camelcase v1.0.0 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb // indirect
-	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
 	golang.org/x/sys v0.1.0 // indirect
 )
 

_testdata/compile/go.sum

@@ -4,20 +4,20 @@ github.com/bobg/errors v0.10.0 h1:zlGq7hLqgaJILpwDmCDTnPvlvKI8M9Rh3uTRCuaMbiU=
 github.com/bobg/errors v0.10.0/go.mod h1:lJenauJJF2tAdzEmND/wGVfA9kCChcj2p4KO/bNCz24=
 github.com/bobg/go-generics v1.5.0 h1:BiVzUWzJKn6wMC3eAfwy3PctCQhwRUFaTncl0R57s74=
 github.com/bobg/go-generics v1.5.0/go.mod h1:B7O5x+EeOyI02YDBuDi+Wv6Z3itw+BoRr318oQ2mbNY=
-github.com/bobg/go-generics/v2 v2.1.1 h1:4rN9upY6Xm4TASSMeH+NzUghgO4h/SbNrQphIjRd/R0=
-github.com/bobg/go-generics/v2 v2.1.1/go.mod h1:iPMSRVFlzkJSYOCXQ0n92RA3Vxw0RBv2E8j9ZODXgHk=
+github.com/bobg/go-generics/v2 v2.1.2 h1:Mx9vgC7p5vJ0dFLUGaq6TtHisbK1d16vFEwzMCUY8jI=
+github.com/bobg/go-generics/v2 v2.1.2/go.mod h1:iPMSRVFlzkJSYOCXQ0n92RA3Vxw0RBv2E8j9ZODXgHk=
 github.com/bobg/tsdecls v0.1.0 h1:zlYDnZZSLjVCIZMZu78PfZQ63VWZ9I6wKNXJjlv1XV4=
 github.com/bobg/tsdecls v0.1.0/go.mod h1:C+D/p1UVhHCqFNMPffaNr+xGi9zhYqvF/u7qETNqLbs=
+github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=
+github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0=
+github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/fatih/camelcase v1.0.0 h1:hxNvNX/xYBp0ovncs8WyWZrOrpBNub/JfaMvbURyft8=
 github.com/fatih/camelcase v1.0.0/go.mod h1:yN2Sb0lFhZJUdVvtELVWefmrXpuZESvPmqwoZc+/fpc=
 github.com/gibson042/canonicaljson-go v1.0.3 h1:EAyF8L74AWabkyUmrvEFHEt/AGFQeD6RfwbAuf0j1bI=
 github.com/gibson042/canonicaljson-go v1.0.3/go.mod h1:DsLpJTThXyGNO+KZlI85C1/KDcImpP67k/RKVjcaEqo=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=
-github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY=
-github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk=
-github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y=
 github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI=
 github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/otiai10/copy v1.7.0 h1:hVoPiN+t+7d2nzzwMiDHPSOogsWAStewq3TwU05+clE=
@@ -29,30 +29,21 @@ github.com/otiai10/mint v1.3.3 h1:7JgpsBaN0uMkyju4tbYHu0mnM55hNKVYLsXmwr15NQI=
 github.com/otiai10/mint v1.3.3/go.mod h1:/yxELlJQ0ufhjUwhshSj+wFjZ78CnZ48/1wtmBH1OTc=
 github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
-github.com/pressly/goose/v3 v3.6.1 h1:DB7/eKhn98vWOz90OSXqMf4OwuKCdQ6GbvxhtjO4Uak=
-github.com/pressly/goose/v3 v3.6.1/go.mod h1:fpaav/TpxygOn1+OAdzwswN2NbvadBOktQpiDOxewvY=
-github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk=
+github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
+github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
+github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
+github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb h1:PaBZQdo+iSDyHT053FjUCgZQ/9uqVwPOcl7KSWhKn6w=
 golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc=
 golang.org/x/mod v0.6.0 h1:b9gGHsz9/HhJ3HF5DHQytPpuwocVTChQJK3AvoLRD5I=
 golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI=
-golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde h1:ejfdSekXMDxDLbRrJMwUk6KnSLZ2McaUCVcIKM+N6jc=
-golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.1.0 h1:kunALQeHf1/185U1i0GOB/fy1IPRDDpuoOOqRReG57U=
 golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/tools v0.2.0 h1:G6AHpWxTMGY1KyEYoAQ5WTtIekUUvDNjan3ugu60JvE=
 golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-lukechampine.com/uint128 v1.1.1 h1:pnxCASz787iMf+02ssImqk6OLt+Z5QHMoZyUXR4z6JU=
-modernc.org/cc/v3 v3.36.0 h1:0kmRkTmqNidmu3c7BNDSdVHCxXCkWLmWmCIVX4LUboo=
-modernc.org/ccgo/v3 v3.16.6 h1:3l18poV+iUemQ98O3X5OMr97LOqlzis+ytivU4NqGhA=
-modernc.org/libc v1.16.7 h1:qzQtHhsZNpVPpeCu+aMIQldXeV1P0vRhSqCL0nOIJOA=
-modernc.org/mathutil v1.4.1 h1:ij3fYGe8zBF4Vu+g0oT7mB06r8sqGWKuJu1yXeR4by8=
-modernc.org/memory v1.1.1 h1:bDOL0DIDLQv7bWhP3gMvIrnoFw+Eo6F7a2QK9HPDiFU=
-modernc.org/opt v0.1.1 h1:/0RX92k9vwVeDXj+Xn23DKp2VJubL7k8qNffND6qn3A=
-modernc.org/sqlite v1.17.3 h1:iE+coC5g17LtByDYDWKpR6m2Z9022YrSh3bumwOnIrI=
-modernc.org/strutil v1.1.1 h1:xv+J1BXY3Opl2ALrBwyfEikFAj8pmqcpnfmuwUwcozs=
-modernc.org/token v1.0.0 h1:a0jaWiNMDhDUtqOj09wvjWWAqd3q7WpBulmL9H2egsk=

_testdata/glob/fab.yaml

@@ -0,0 +1,6 @@
+Doxating: !Files
+  In: !Glob
+    - x*
+    - y*
+  Target: !Command
+    Shell: echo Hello

_testdata/glob/x1

@@ -0,0 +1 @@
+x1

_testdata/glob/x2

@@ -0,0 +1 @@
+x2

_testdata/glob/y1

@@ -0,0 +1 @@
+y1

_testdata/glob/y2

@@ -0,0 +1 @@
+y2

_testdata/glob/z1

@@ -0,0 +1 @@
+z1

_testdata/glob/z2

@@ -0,0 +1 @@
+z2

argtarg.go

@@ -60,7 +60,7 @@ func argTargetDecoder(con *Controller, node *yaml.Node, dir string) (Target, err
 		return nil, errors.Wrap(err, "YAML error in target child of AllTarget node")
 	}
 
-	args, err := YAMLStringListFromNodes(node.Content[1:])
+	args, err := con.YAMLStringListFromNodes(node.Content[1:], dir)
 	if err != nil {
 		return nil, errors.Wrap(err, "YAML error in ArgTarget args")
 	}

command.go

@@ -380,17 +380,17 @@ func commandDecoder(con *Controller, node *yaml.Node, dir string) (Target, error
 		return nil, errors.Wrap(err, "YAML error decoding Command")
 	}
 
-	args, err := YAMLStringList(&c.Args)
+	args, err := con.YAMLStringList(&c.Args, dir)
 	if err != nil {
 		return nil, errors.Wrap(err, "YAML error decoding Command.Args")
 	}
-	env, err := YAMLStringList(&c.Env)
+	env, err := con.YAMLStringList(&c.Env, dir)
 	if err != nil {
 		return nil, errors.Wrap(err, "YAML error decoding Command.Env")
 	}
 
 	if c.Cmd == "" {
-		strs, err := YAMLStringList(&c.Shell)
+		strs, err := con.YAMLStringList(&c.Shell, dir)
 
 		var e BadYAMLNodeKindError
 		switch {

files.go

@@ -26,19 +26,29 @@ var filesRegistry = newRegistry[*files]()
 // whose Run method should produce or update the expected output files.
 //
 // When the Files target runs,
-// a hash is computed from the nested subtarget
-// and all the input and output files.
-// If none of those has changed since the last time the output files were built,
-// then the output files are up to date and running of this Files target can be skipped.
+// it does the following:
+//
+//   - It checks to see whether any of its input files
+//     are listed as output files in other Files targets.
+//     Other targets found in this way are run first,
+//     as prerequisites.
+//   - It then computes a hash from the nested subtarget
+//     and all the input and output files.
+//     If this hash is found in the “hash database”
+//     (obtained with [GetHashDB]),
+//     that means none of the files has changed
+//     since the last time the output files were built,
+//     so running of the subtarget can be skipped.
+//   - Otherwise the subtarget is run.
+//     The hash is then recomputed
+//     and added to the hash database,
+//     telling the next run of this target
+//     that this collection of input and output files
+//     can be considered up-to-date.
 //
 // The nested subtarget must be of a type that can be JSON-marshaled.
 // Notably this excludes [F].
 //
-// When a Files target runs,
-// it checks to see whether any of its input files
-// are listed as output files in other Files targets.
-// Other targets found in this way are [Run] first, as prerequisites.
-//
 // The list of input files should mention every file where a change should cause a rebuild.
 // Ideally this includes any files required by the nested subtarget
 // plus any transitive dependencies.
@@ -49,7 +59,13 @@ var filesRegistry = newRegistry[*files]()
 // causes the output files to be added to the "autoclean registry."
 // A [Clean] target may then choose to remove the files listed in that registry
 // (instead of, or in addition to, any explicitly listed files)
-// by setting its Autoclean field to true.
+// by setting _its_ Autoclean field to true.
+//
+// The list of input and output files may include directories too.
+// These are walked recursively for computing the hash described above.
+// Be careful when using directories in the output-file list
+// together with the Autoclean feature:
+// the entire directory tree will be deleted.
 //
 // When [GetDryRun] is true,
 // checking and updating of the hash DB is skipped.
@@ -58,8 +74,9 @@ var filesRegistry = newRegistry[*files]()
 // which introduces a mapping whose fields are:
 //
 //   - Target: the nested subtarget, or target name
-//   - In: the list of input files, interpreted with [YAMLStringList]
-//   - Out: the list of output files, interpreted with [YAMLStringList]
+//   - In: the list of input files, interpreted with [YAMLFilesList]
+//   - Out: the list of output files, interpreted with [YAMLFilesList]
+//   - Autoclean: a boolean
 //
 // Example:
 //
@@ -181,7 +198,7 @@ func (ft *files) runPrereqs(ctx context.Context, con *Controller) error {
 	var prereqs []Target
 
 	for _, in := range ft.In {
-		if target, ok := filesRegistry.lookup(in); ok {
+		if target := findInFilesRegistry(in); target != nil {
 			prereqs = append(prereqs, target)
 		}
 	}
@@ -192,6 +209,21 @@ func (ft *files) runPrereqs(ctx context.Context, con *Controller) error {
 	return con.Run(ctx, prereqs...)
 }
 
+func findInFilesRegistry(name string) Target {
+	for {
+		if target, ok := filesRegistry.lookup(name); ok {
+			return target
+		}
+
+		dir := filepath.Dir(name)
+		switch dir {
+		case "", ".", "/", name:
+			return nil
+		}
+		name = dir
+	}
+}
+
 type FilesOpt func(*files)
 
 // Autoclean is an option for passing to [Files].
@@ -322,6 +354,43 @@ func filesDecoder(con *Controller, node *yaml.Node, dir string) (Target, error)
 	return Files(target, in, out, Autoclean(yfiles.Autoclean)), nil
 }
 
+func globDecoder(con *Controller, node *yaml.Node, dir string) ([]string, error) {
+	if node.Kind != yaml.SequenceNode {
+		return nil, BadYAMLNodeKindError{Got: node.Kind, Want: yaml.SequenceNode}
+	}
+
+	patterns, err := con.YAMLStringListFromNodes(node.Content, dir)
+	if err != nil {
+		return nil, errors.Wrap(err, "in children of Glob node")
+	}
+
+	jdir := con.JoinPath(dir)
+
+	var result []string
+	for _, pattern := range patterns {
+		// Each pattern has to be joined to dir
+		// in order to evaluate the glob in the right place.
+		// But then dir is removed from the resulting matches
+		// (via filepath.Rel).
+
+		j := con.JoinPath(dir, pattern)
+		matches, err := filepath.Glob(j)
+		if err != nil {
+			return nil, errors.Wrap(err, "in Glob pattern")
+		}
+		matches, err = slices.Mapx(matches, func(_ int, m string) (string, error) {
+			return filepath.Rel(jdir, m)
+		})
+		if err != nil {
+			return nil, errors.Wrap(err, "making matches relative to their directory")
+		}
+		result = append(result, matches...)
+	}
+
+	return result, nil
+}
+
 func init() {
 	RegisterYAMLTarget("Files", filesDecoder)
+	RegisterYAMLStringList("Glob", globDecoder)
 }

files_test.go

@@ -2,12 +2,14 @@ package fab
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"reflect"
 	"testing"
 
 	"github.com/bobg/go-generics/v2/set"
+	"github.com/davecgh/go-spew/spew"
 )
 
 func TestFileChaining(t *testing.T) {
@@ -171,3 +173,60 @@ func TestFileHashes(t *testing.T) {
 		t.Errorf("got %v, want %v", got, want)
 	}
 }
+
+func TestFilesRegistry(t *testing.T) {
+	targ := &files{}
+	filesRegistry.add("TestFilesRegistry/a/b/c.d", targ)
+	filesRegistry.add("TestFilesRegistry/a/e", targ)
+
+	cases := []struct {
+		probe string
+		want  bool
+	}{{
+		probe: "TestFilesRegistry/a/b/c.d",
+		want:  true,
+	}, {
+		probe: "TestFilesRegistry/a/b/x",
+		want:  false,
+	}, {
+		probe: "TestFilesRegistry/a/e/x",
+		want:  true,
+	}}
+
+	for i, tc := range cases {
+		t.Run(fmt.Sprintf("case_%02d", i+1), func(t *testing.T) {
+			got := findInFilesRegistry(tc.probe)
+			if got != nil && !tc.want {
+				t.Errorf("got a hit but didn't want one")
+			} else if got == nil && tc.want {
+				t.Errorf("got no hit but wanted one")
+			}
+		})
+	}
+}
+
+func TestGlob(t *testing.T) {
+	con := NewController("_testdata/glob")
+	if err := con.ReadYAMLFile(""); err != nil {
+		t.Fatal(err)
+	}
+
+	got, _ := con.RegistryTarget("Doxating")
+	want := Files(
+		&Command{
+			Shell: "echo Hello",
+			Dir:   "_testdata/glob",
+		},
+		[]string{
+			"_testdata/glob/x1",
+			"_testdata/glob/x2",
+			"_testdata/glob/y1",
+			"_testdata/glob/y2",
+		},
+		nil,
+	)
+
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("got:\n%s\nwant:\n%s", spew.Sdump(got), spew.Sdump(want))
+	}
+}

go.mod

@@ -5,7 +5,7 @@ go 1.20
 require (
 	github.com/benbjohnson/clock v1.3.0
 	github.com/bobg/errors v0.10.0
-	github.com/bobg/go-generics/v2 v2.1.1
+	github.com/bobg/go-generics/v2 v2.1.2
 	github.com/bobg/tsdecls v0.1.0
 	github.com/bradleyjkemp/cupaloy/v2 v2.8.0
 	github.com/davecgh/go-spew v1.1.1

go.sum

@@ -4,8 +4,8 @@ github.com/bobg/errors v0.10.0 h1:zlGq7hLqgaJILpwDmCDTnPvlvKI8M9Rh3uTRCuaMbiU=
 github.com/bobg/errors v0.10.0/go.mod h1:lJenauJJF2tAdzEmND/wGVfA9kCChcj2p4KO/bNCz24=
 github.com/bobg/go-generics v1.5.0 h1:BiVzUWzJKn6wMC3eAfwy3PctCQhwRUFaTncl0R57s74=
 github.com/bobg/go-generics v1.5.0/go.mod h1:B7O5x+EeOyI02YDBuDi+Wv6Z3itw+BoRr318oQ2mbNY=
-github.com/bobg/go-generics/v2 v2.1.1 h1:4rN9upY6Xm4TASSMeH+NzUghgO4h/SbNrQphIjRd/R0=
-github.com/bobg/go-generics/v2 v2.1.1/go.mod h1:iPMSRVFlzkJSYOCXQ0n92RA3Vxw0RBv2E8j9ZODXgHk=
+github.com/bobg/go-generics/v2 v2.1.2 h1:Mx9vgC7p5vJ0dFLUGaq6TtHisbK1d16vFEwzMCUY8jI=
+github.com/bobg/go-generics/v2 v2.1.2/go.mod h1:iPMSRVFlzkJSYOCXQ0n92RA3Vxw0RBv2E8j9ZODXgHk=
 github.com/bobg/tsdecls v0.1.0 h1:zlYDnZZSLjVCIZMZu78PfZQ63VWZ9I6wKNXJjlv1XV4=
 github.com/bobg/tsdecls v0.1.0/go.mod h1:C+D/p1UVhHCqFNMPffaNr+xGi9zhYqvF/u7qETNqLbs=
 github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M=

golang/_testdata/go.yaml

@@ -6,7 +6,7 @@ Foo: !go.Binary
 
 Bar: !Files
   In: !go.Deps
-    Dir: .
+    Dir: ..
   Out: [bar]
   Target: !Command
     Shell: echo bar