diff --git a/README.md b/README.md index dcbd9dec..1a0560af 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,13 @@ ### `myitcv.io/...` mono-repo + + + diff --git a/_scripts/run_tests.sh b/_scripts/run_tests.sh index 65af2876..7e4e5d37 100755 --- a/_scripts/run_tests.sh +++ b/_scripts/run_tests.sh @@ -10,8 +10,11 @@ if [ "${CI:-}" == "true" ] then go get -u golang.org/x/vgo pushd $(go list -f "{{.Dir}}" golang.org/x/vgo) > /dev/null - git checkout -qf $VGO_COMMIT + + # git checkout -qf $VGO_COMMIT + git fetch -q https://go.googlesource.com/vgo refs/changes/55/105855/3 && git checkout -qf FETCH_HEAD go install + popd > /dev/null # so we can access Github without hitting rate limits @@ -63,17 +66,33 @@ do then ./_scripts/run_tests.sh else + if [ -f ./_scripts/pre_run_tests.sh ] + then + ./_scripts/pre_run_tests.sh + fi + $go generate ./... $go test ./... - # we can remove this once we resolve https://github.com/golang/go/issues/24661 - $go install ./... + if [ -f ./_scripts/post_run_tests.sh ] + then + ./_scripts/post_run_tests.sh + fi fi popd > /dev/null echo "----" echo "" done +# we use regular go to list here because of https://github.com/golang/go/issues/24749; +# this is also the reason why we need to change to the directory to do the vgo install +for i in $(go list -f "{{if eq .Name \"main\"}}{{.Dir}}{{end}}" ./...) +do + pushd $i > /dev/null + $go install + popd > /dev/null +done + echo Checking markdown files are current # by this point we will have mdreplace installed. Hence check that # committed .md files are "fresh" diff --git a/cmd/mdreplace/README.md b/cmd/mdreplace/README.md index 7b646449..8bb99be6 100644 --- a/cmd/mdreplace/README.md +++ b/cmd/mdreplace/README.md @@ -55,7 +55,7 @@ _To see this in action, look at the [source of the ### Usage ``` -{{.}} +{{. -}} ``` --> ### Usage @@ -69,12 +69,13 @@ Usage: When called with no file arguments, mdreplace works with stdin Flags: + -debug + whether to print debug information of not -strip whether to strip special comments from the file -w whether to write back to input files (cannot be used when reading from stdin) - ``` diff --git a/cmd/mdreplace/mdreplace.go b/cmd/mdreplace/mdreplace.go index ca3cad61..38db5027 100644 --- a/cmd/mdreplace/mdreplace.go +++ b/cmd/mdreplace/mdreplace.go @@ -50,6 +50,7 @@ import ( var ( fWrite = flag.Bool("w", false, "whether to write back to input files (cannot be used when reading from stdin)") fStrip = flag.Bool("strip", false, "whether to strip special comments from the file") + fDebug = flag.Bool("debug", false, "whether to print debug information of not") ) //go:generate pkgconcat -out gen_cliflag.go myitcv.io/_tmpls/cliflag @@ -150,7 +151,7 @@ func infof(format string, args ...interface{}) { } func debugf(format string, args ...interface{}) { - if debug { + if debug || *fDebug { fmt.Fprintf(os.Stderr, format, args...) } } diff --git a/vgo/go.mod b/vgo/go.mod new file mode 100644 index 00000000..f0040fee --- /dev/null +++ b/vgo/go.mod @@ -0,0 +1,3 @@ +module "myitcv.io/vgo" + +require "golang.org/x/net" v0.0.0-20180406214816-61147c48b25b diff --git a/vgo/loader.go b/vgo/loader.go new file mode 100644 index 00000000..30dc9514 --- /dev/null +++ b/vgo/loader.go @@ -0,0 +1,155 @@ +// package vgo provides some utility types, functions etc to support vgo +package vgo // import "myitcv.io/vgo" + +import ( + "bytes" + "encoding/json" + "fmt" + "go/build" + "go/importer" + "go/types" + "io" + "os" + "os/exec" + "strings" + "sync" +) + +// Loader supports loading of vgo-build cached packages. NewLoader returns a +// correctly initialised *Loader. A Loader must not be copied once created. +type Loader struct { + mu sync.Mutex + + dir string + compiler string + resCache map[string]map[string]*types.Package + importers map[string]types.ImporterFrom + test bool +} + +func NewLoader(dir string) *Loader { + res := &Loader{ + dir: dir, + compiler: "gc", + resCache: make(map[string]map[string]*types.Package), + importers: make(map[string]types.ImporterFrom), + } + + return res +} + +func NewTestLoader(dir string) *Loader { + res := NewLoader(dir) + res.test = true + return res +} + +var _ types.ImporterFrom = new(Loader) + +func (l *Loader) Import(path string) (*types.Package, error) { + return nil, fmt.Errorf("did not expect this method to be used; we implement types.ImporterFrom") +} + +func (l *Loader) ImportFrom(path, dir string, mode types.ImportMode) (*types.Package, error) { + if mode != 0 { + panic(fmt.Errorf("unknown types.ImportMode %v", mode)) + } + + l.mu.Lock() + defer l.mu.Unlock() + + // TODO optimise mutex usage later... keep it simple for now + dirCache, ok := l.resCache[dir] + if ok { + if p, ok := dirCache[path]; ok { + return p, nil + } + } else { + // ensures dirCache is now set + dirCache = make(map[string]*types.Package) + l.resCache[dir] = dirCache + } + + // res cache miss + imp, ok := l.importers[dir] + if !ok { + // we need to load the results for this dir and build an importer + + // resolve the package found in dir + bpkg, err := build.ImportDir(dir, 0) + if err != nil { + return nil, fmt.Errorf("unable to resolve %v to a package: %v", dir, err) + } + + // now run vgo depbuildlist with the import path + args := []string{"vgo", "deplist", "-build"} + + if l.test { + args = append(args, "-test") + } + + args = append(args, bpkg.ImportPath) + + cmd := exec.Command(args[0], args[1:]...) + cmd.Dir = l.dir + + out, err := cmd.CombinedOutput() + if err != nil { + return nil, fmt.Errorf("unable to run %v: %v [%q]", strings.Join(cmd.Args, " "), err, string(out)) + } + + // parse the JSON + + lookup := make(map[string]string) + + dec := json.NewDecoder(bytes.NewBuffer(out)) + + for { + var d struct { + ImportPath string + PackageFile string + } + + if err := dec.Decode(&d); err != nil { + if err == io.EOF { + break + } + + return nil, fmt.Errorf("failed to parse vgo output: %v\noutput was:\n%v", err, string(out)) + } + + lookup[d.ImportPath] = d.PackageFile + } + + i := importer.For(l.compiler, func(path string) (io.ReadCloser, error) { + file, ok := lookup[path] + if !ok { + return nil, fmt.Errorf("failed to resolve import path %q", path) + } + + f, err := os.Open(file) + if err != nil { + return nil, fmt.Errorf("failed to open file %v: %v", file, err) + } + + return f, nil + }) + + from, ok := i.(types.ImporterFrom) + if !ok { + return nil, fmt.Errorf("failed to get an importer that implements go/types.ImporterFrom; got %T", i) + } + + imp = from + l.importers[dir] = imp + } + + p, err := imp.ImportFrom(path, dir, mode) + if err != nil { + return nil, fmt.Errorf("failed to import: %v", err) + } + + dirCache[path] = p + + return p, nil +} diff --git a/vgo/loader_test.go b/vgo/loader_test.go new file mode 100644 index 00000000..a17e39cf --- /dev/null +++ b/vgo/loader_test.go @@ -0,0 +1,49 @@ +package vgo_test + +import ( + "os" + "testing" + + // import a non-standard library package for its side effects. + // vgo will then detect this + _ "golang.org/x/net/html" + "myitcv.io/vgo" +) + +func TestLoader(t *testing.T) { + // given the side-effect import above, we can now create a Loader + // to load "golang.org/x/net/html" in the context of the current + // directory + + l := vgo.NewTestLoader(".") + + cwd, err := os.Getwd() + if err != nil { + t.Fatalf("failed to get cwd: %v", cwd) + } + + cases := []string{ + "golang.org/x/net/html", + + // this is a dependency of x/net/html; hence an indirect + // test dependency of vgo_test + "golang.org/x/net/html/atom", + } + + for _, c := range cases { + t.Run(c, func(t *testing.T) { + p, err := l.ImportFrom(c, cwd, 0) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + if p == nil { + t.Fatal("expected response; got nil") + } + + if v := p.Path(); v != c { + t.Fatalf("expected ImportPath %q; got %q", c, v) + } + }) + } +}