Fix a bunch of GO15VENDOREXPERIMENT issues

- Find package directories better. Previously we used build.FindOnly
  which didn't work the way I expected it to (any dir would work w/o
  error).
- Set the VendorExperiment bool based on go version as 1.6 defaults to
  on.
- A bunch of extra debugging for use while sanity checking myself.
- vendor flag for test structs.
- Some tests for vendor/ stuff:
    - Basic Test
    - Transitive
    - Transitive, across GOPATHs + collapse vendor/ directories.

Should Fix #358

Changelog.md

@@ -1,3 +1,16 @@
+# v42 2015/12/22
+
+* Fix a bunch of GO15VENDOREXPERIMENT issues
+    * Find package directories better. Previously we used build.FindOnly which didn't work the way I expected it to (any dir would work w/o error).
+    * Set the VendorExperiment bool based on go version as 1.6 defaults to on.
+    * A bunch of extra debugging for use while sanity checking myself.
+    * vendor flag for test structs.
+    * Some tests for vendor/ stuff:
+        * Basic Test
+        * Transitive
+        * Transitive, across GOPATHs + collapse vendor/ directories.
+* Should Fix #358
+
 # v41 2015/12/17
 
 * Don't rewrite packages outside of the project. This would happen if you specified

errors.go

@@ -8,3 +8,11 @@ var (
 	errorCopyingSourceCode   = errors.New("error copying source code")
 	errorNoPackagesUpdatable = errors.New("no packages can be updated")
 )
+
+type errPackageNotFound struct {
+	path string
+}
+
+func (e errPackageNotFound) Error() string {
+	return "Package (" + e.path + ") not found"
+}

godepfile.go

@@ -54,6 +54,8 @@ func loadDefaultGodepsFile() (Godeps, error) {
 
 // pkgs is the list of packages to read dependencies for
 func (g *Godeps) fill(pkgs []*Package, destImportPath string) error {
+	debugln("fill", destImportPath)
+	ppln(pkgs)
 	var err1 error
 	var path, testImports []string
 	for _, p := range pkgs {
@@ -87,10 +89,12 @@ func (g *Godeps) fill(pkgs []*Package, destImportPath string) error {
 		path = append(path, p.ImportPath)
 		path = append(path, p.Deps...)
 	}
+	debugln("path", path)
 	for i, p := range path {
 		path[i] = unqualify(p)
 	}
 	path = uniq(path)
+	debugln("uniq, unqualify'd path", path)
 	ps, err = LoadPackages(path...)
 	if err != nil {
 		return err

list.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"errors"
 	"fmt"
 	"go/build"
 	"go/parser"
@@ -104,27 +105,11 @@ func fullPackageInDir(dir string) (*build.Package, error) {
 
 // listPackage specified by path
 func listPackage(path string) (*Package, error) {
-	var dir string
+	debugln("listPackage", path)
 	var lp *build.Package
-	var err error
-	if build.IsLocalImport(path) {
-		dir = path
-		if !filepath.IsAbs(dir) {
-			if abs, err := filepath.Abs(dir); err == nil {
-				// interpret relative to current directory
-				dir = abs
-			}
-		}
-	} else {
-		dir, err = os.Getwd()
-		if err != nil {
-			return nil, err
-		}
-		lp, err = build.Import(path, dir, build.FindOnly)
-		if err != nil {
-			return nil, err
-		}
-		dir = lp.Dir
+	dir, err := findDirForPath(path, nil)
+	if err != nil {
+		return nil, err
 	}
 	lp, err = fullPackageInDir(dir)
 	p := &Package{
@@ -152,32 +137,11 @@ func listPackage(path string) (*Package, error) {
 		ip, i := ds.Next()
 
 		debugf("Processing import %s for %s\n", i, ip.Dir)
-		// We need to check to see if the import exists in vendor/ folders up the hierachy of the importing package,
-		var dp *build.Package
-		if VendorExperiment && !ip.Goroot {
-			for base := ip.Dir; base != ip.Root; base = filepath.Dir(base) {
-				vdir := filepath.Join(base, "vendor", i)
-				debugln("checking for vendor dir:", vdir)
-				dp, err = build.ImportDir(vdir, build.FindOnly)
-				if err != nil {
-					if os.IsNotExist(err) {
-						continue
-					}
-					debugln(err.Error())
-					ppln(err)
-					panic("Unknown error attempt to find vendor/")
-				}
-				goto Found
-			}
-		}
-		// Wasn't found above, so resolve it using the build.Context
-		dp, err = build.Import(i, ip.Dir, build.FindOnly)
+		pdir, err := findDirForPath(i, ip)
 		if err != nil {
-			ppln(err)
-			return nil, errorMissingDep{i: i, dir: ip.Dir}
+			return nil, err
 		}
-	Found:
-		dp, err = fullPackageInDir(dp.Dir)
+		dp, err := fullPackageInDir(pdir)
 		if err != nil { // This really should happen in this context though
 			ppln(err)
 			return nil, errorMissingDep{i: i, dir: ip.Dir}
@@ -199,18 +163,75 @@ func listPackage(path string) (*Package, error) {
 	}
 	p.Imports = uniq(p.Imports)
 	p.Deps = uniq(p.Deps)
-	debugln("Looking For Package:", path, "in", dir)
+	debugln("Done Looking For Package:", path, "in", dir)
 	ppln(p)
 	return p, nil
 }
 
-// fillPackage full of info. Assumes a build.Package discovered in build.FindOnly mode
+// finds the directory for the given import path in the context of the provided build.Package (if provided)
+func findDirForPath(path string, ip *build.Package) (string, error) {
+	debugln("findDirForPath", path, ip)
+	var search []string
+
+	if build.IsLocalImport(path) {
+		dir := path
+		if !filepath.IsAbs(dir) {
+			if abs, err := filepath.Abs(dir); err == nil {
+				// interpret relative to current directory
+				dir = abs
+			}
+		}
+		return dir, nil
+	}
+
+	// We need to check to see if the import exists in vendor/ folders up the hierachy of the importing package
+	if VendorExperiment && ip != nil {
+		debugln("resolving vendor posibilities:", ip.Dir, ip.Root)
+		for base := ip.Dir; base != ip.Root; base = filepath.Dir(base) {
+			s := filepath.Join(base, "vendor", path)
+			debugln("Adding search dir:", s)
+			search = append(search, s)
+		}
+	}
+
+	for _, base := range build.Default.SrcDirs() {
+		search = append(search, filepath.Join(base, path))
+	}
+
+	for _, dir := range search {
+		debugln("searching", dir)
+		fi, err := os.Stat(dir)
+		if err == nil && fi.IsDir() {
+			return dir, nil
+		}
+	}
+
+	return "", errPackageNotFound{path}
+}
+
+// fillPackage full of info. Assumes p.Dir is set at a minimum
 func fillPackage(p *build.Package) error {
 
 	if p.Goroot {
 		return nil
 	}
 
+	if p.SrcRoot == "" {
+		for _, base := range build.Default.SrcDirs() {
+			if strings.HasPrefix(p.Dir, base) {
+				p.SrcRoot = base
+			}
+		}
+	}
+
+	if p.SrcRoot == "" {
+		return errors.New("Unable to find SrcRoot for package " + p.ImportPath)
+	}
+
+	if p.Root == "" {
+		p.Root = filepath.Dir(p.SrcRoot)
+	}
+
 	var buildMatch = "+build "
 	var buildFieldSplit = func(r rune) bool {
 		return unicode.IsSpace(r) || r == ','

main.go

@@ -12,9 +12,12 @@ import (
 )
 
 var (
-	cpuprofile string
-	verbose    bool // Verbose flag for commands that support it
-	debug      bool // Debug flag for commands that support it
+	cpuprofile       string
+	verbose          bool // Verbose flag for commands that support it
+	debug            bool // Debug flag for commands that support it
+	majorGoVersion   string
+	VendorExperiment bool
+	sep              string
 )
 
 // Command is an implementation of a godep command
@@ -77,13 +80,33 @@ func main() {
 		return
 	}
 
+	var err error
+	majorGoVersion, err = goVersion()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	// VendorExperiment is the Go 1.5 vendor directory experiment flag, see
+	// https://github.com/golang/go/commit/183cc0cd41f06f83cb7a2490a499e3f9101befff
+	VendorExperiment = (majorGoVersion == "go1.5" && os.Getenv("GO15VENDOREXPERIMENT") == "1") ||
+		(majorGoVersion == "go1.6" && os.Getenv("GO15VENDOREXPERIMENT") != "0")
+
+	// sep is the signature set of path elements that
+	// precede the original path of an imported package.
+	sep = defaultSep(VendorExperiment)
+
 	for _, cmd := range commands {
 		if cmd.Name == args[0] {
 			cmd.Flag.BoolVar(&verbose, "v", false, "enable verbose output")
 			cmd.Flag.BoolVar(&debug, "d", false, "enable debug output")
 			cmd.Flag.StringVar(&cpuprofile, "cpuprofile", "", "Write cpu profile to this file")
 			cmd.Flag.Usage = func() { cmd.UsageExit() }
 			cmd.Flag.Parse(args[1:])
+
+			debugln("majorGoVersion", majorGoVersion)
+			debugln("VendorExperiment", VendorExperiment)
+			debugln("sep", sep)
+
 			if cpuprofile != "" {
 				f, err := os.Create(cpuprofile)
 				if err != nil {

pkg.go

@@ -38,6 +38,7 @@ type Package struct {
 // Files with a build tag of `ignore` are skipped. Files with other build tags
 // are however processed.
 func LoadPackages(names ...string) (a []*Package, err error) {
+	debugln("LoadPackages", names)
 	if len(names) == 0 {
 		return nil, nil
 	}

rewrite.go

@@ -21,6 +21,7 @@ import (
 // according to the rules for func qualify.
 func rewrite(pkgs []*Package, qual string, paths []string) error {
 	for _, path := range pkgFiles(pkgs) {
+		debugln("rewrite", path)
 		err := rewriteTree(path, qual, paths)
 		if err != nil {
 			return err
@@ -51,17 +52,14 @@ func rewriteTree(path, qual string, paths []string) error {
 			continue
 		}
 		s := w.Stat()
-		switch s.IsDir() {
-		case true:
-			if s.Name() == "testdata" {
-				w.SkipDir()
-			}
-		case false:
-			if strings.HasSuffix(w.Path(), ".go") {
-				err := rewriteGoFile(w.Path(), qual, paths)
-				if err != nil {
-					return err
-				}
+		if s.IsDir() && s.Name() == "testdata" {
+			w.SkipDir()
+			continue
+		}
+		if strings.HasSuffix(w.Path(), ".go") {
+			err := rewriteGoFile(w.Path(), qual, paths)
+			if err != nil {
+				return err
 			}
 		}
 	}
@@ -71,6 +69,7 @@ func rewriteTree(path, qual string, paths []string) error {
 // rewriteGoFile rewrites import statments in the named file
 // according to the rules for func qualify.
 func rewriteGoFile(name, qual string, paths []string) error {
+	debugln("rewriteGoFile", name, ",", qual, ",", paths)
 	printerConfig := &printer.Config{Mode: printer.TabIndent | printer.UseSpaces, Tabwidth: 8}
 	fset := token.NewFileSet()
 	f, err := parser.ParseFile(fset, name, nil, parser.ParseComments)
@@ -118,14 +117,6 @@ func rewriteGoFile(name, qual string, paths []string) error {
 	return os.Rename(tpath, name)
 }
 
-// VendorExperiment is the Go 1.5 vendor directory experiment flag, see
-// https://github.com/golang/go/commit/183cc0cd41f06f83cb7a2490a499e3f9101befff
-var VendorExperiment = os.Getenv("GO15VENDOREXPERIMENT") == "1"
-
-// sep is the signature set of path elements that
-// precede the original path of an imported package.
-var sep = defaultSep(VendorExperiment)
-
 func defaultSep(experiment bool) string {
 	if experiment {
 		return "/vendor/"

rewrite_test.go

@@ -7,6 +7,7 @@ import (
 )
 
 func TestUnqualify(t *testing.T) {
+	setGlobals(false)
 	var cases = []struct {
 		path string
 		want string
@@ -243,7 +244,7 @@ func TestRewrite(t *testing.T) {
 	const gopath = "godeptest"
 	defer os.RemoveAll(gopath)
 	for pos, test := range cases {
-		clearPkgCache()
+		setGlobals(false)
 		err := os.RemoveAll(gopath)
 		if err != nil {
 			t.Fatal(err)

save.go

@@ -142,15 +142,19 @@ func save(pkgs []string) error {
 	if err != nil {
 		return err
 	}
-	debugln("LoadPackages", pkgs)
+	debugln("Project Packages", pkgs)
 	ppln(a)
 
 	projA := projectPackages(dp.Dir, a)
+	debugln("Filtered projectPackages")
+	ppln(projA)
+
 	err = gnew.fill(a, dp.ImportPath)
 	if err != nil {
 		return err
 	}
 	debugln("New Godeps Filled")
+	ppln(gnew)
 
 	if gnew.Deps == nil {
 		gnew.Deps = make([]Dependency, 0) // produce json [], not null
@@ -160,6 +164,7 @@ func save(pkgs []string) error {
 	if err != nil {
 		return err
 	}
+
 	if gold.isOldFile {
 		// If we are migrating from an old format file,
 		// we require that the listed version of every
@@ -186,7 +191,11 @@ func save(pkgs []string) error {
 	//   godep go list ./...
 	srcdir := filepath.FromSlash(strings.Trim(sep, "/"))
 	rem := subDeps(gold.Deps, gnew.Deps)
+	debugln("Deps to remove")
+	ppln(rem)
 	add := subDeps(gnew.Deps, gold.Deps)
+	debugln("Deps to add")
+	ppln(add)
 	err = removeSrc(srcdir, rem)
 	if err != nil {
 		return err
@@ -205,6 +214,8 @@ func save(pkgs []string) error {
 			rewritePaths = append(rewritePaths, dep.ImportPath)
 		}
 	}
+	debugln("rewritePaths")
+	ppln(rewritePaths)
 	return rewrite(projA, dp.ImportPath, rewritePaths)
 }