[WIP] minimal module awareness

gps/pkgtree/gomod.go

@@ -0,0 +1,73 @@
+// Copyright 2016 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package pkgtree
+
+import (
+	"bytes"
+	"io/ioutil"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// goModPath was taken, nearly, verbatim from: go1.10.3/src/cmd/go/internal/load/pkg.go
+
+var (
+	modulePrefix       = []byte("\nmodule ")
+	goModPathCache     = make(map[string]string)
+	goModPathCacheLock sync.RWMutex
+)
+
+// goModPath returns the module path in the go.mod in dir, if any.
+func goModPath(dir string) (path string) {
+	goModPathCacheLock.RLock()
+	path, ok := goModPathCache[dir]
+	goModPathCacheLock.RUnlock()
+	if ok {
+		return path
+	}
+	defer func() {
+		goModPathCacheLock.Lock()
+		goModPathCache[dir] = path
+		goModPathCacheLock.Unlock()
+	}()
+
+	data, err := ioutil.ReadFile(filepath.Join(dir, "go.mod"))
+	if err != nil {
+		return ""
+	}
+	var i int
+	if bytes.HasPrefix(data, modulePrefix[1:]) {
+		i = 0
+	} else {
+		i = bytes.Index(data, modulePrefix)
+		if i < 0 {
+			return ""
+		}
+		i++
+	}
+	line := data[i:]
+
+	// Cut line at \n, drop trailing \r if present.
+	if j := bytes.IndexByte(line, '\n'); j >= 0 {
+		line = line[:j]
+	}
+	if line[len(line)-1] == '\r' {
+		line = line[:len(line)-1]
+	}
+	line = line[len("module "):]
+
+	// If quoted, unquote.
+	path = strings.TrimSpace(string(line))
+	if path != "" && path[0] == '"' {
+		s, err := strconv.Unquote(path)
+		if err != nil {
+			return ""
+		}
+		path = s
+	}
+	return path
+}

gps/pkgtree/pkgtree.go

@@ -25,6 +25,8 @@ import (
 type Package struct {
 	Name        string   // Package name, as declared in the package statement
 	ImportPath  string   // Full import path, including the prefix provided to ListPackages()
+	RelModPath  string   // Location of go.mod file for package, relative to project root
+	Module      string   // The name (import path) of the go module
 	CommentPath string   // Import path given in the comment on the package statement
 	Imports     []string // Imports from all go and cgo files
 	TestImports []string // Imports from all go test files (in go/build parlance: both TestImports and XTestImports)
@@ -126,10 +128,38 @@ func ListPackages(fileRoot, importRoot string) (PackageTree, error) {
 			f.Close()
 		}
 
+		ip := filepath.Join(importRoot, strings.TrimPrefix(wp, fileRoot))
+		var modpath, module string
+
+		// walk back up the filesystem to the import root, inclusive, to check
+		// to see if a go.mod file requires an adjustment to the import path.
+		for i := len(wp); i >= len(fileRoot); i-- {
+			if i < len(wp) && wp[i] != filepath.Separator {
+				continue
+			}
+
+			var gomod string
+			if gomod = goModPath(wp[:i]); gomod == "" {
+				continue
+			}
+
+			// there was a go.mod file in this directory which states that the
+			// module, from this point in the file tree, should be imported as gomod
+			ip = filepath.Join(gomod, wp[i:])
+			modpath = strings.TrimPrefix(wp[:i], fileRoot)
+			module = gomod
+
+			if modpath == "" {
+				modpath = "."
+			}
+
+			break
+		}
+
 		// Compute the import path. Run the result through ToSlash(), so that
 		// windows file paths are normalized to slashes, as is expected of
 		// import paths.
-		ip := filepath.ToSlash(filepath.Join(importRoot, strings.TrimPrefix(wp, fileRoot)))
+		ip = filepath.ToSlash(ip)
 
 		// Find all the imports, across all os/arch combos
 		p := &build.Package{
@@ -154,6 +184,8 @@ func ListPackages(fileRoot, importRoot string) (PackageTree, error) {
 
 		pkg := Package{
 			ImportPath:  ip,
+			RelModPath:  modpath,
+			Module:      module,
 			CommentPath: p.ImportComment,
 			Name:        p.Name,
 			Imports:     p.Imports,

gps/pkgtree/pkgtree_test.go

@@ -2247,6 +2247,8 @@ func TestCanaryPackageTreeCopy(t *testing.T) {
 	packageFields := []string{
 		"Name",
 		"ImportPath",
+		"RelModPath",
+		"Module",
 		"CommentPath",
 		"Imports",
 		"TestImports",

gps/source.go

@@ -9,6 +9,9 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
+	"path/filepath"
+	"strings"
 	"sync"
 
 	"github.com/golang/dep/gps/pkgtree"
@@ -322,11 +325,66 @@ func (sg *sourceGateway) existsUpstream(ctx context.Context) error {
 	return err
 }
 
-func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to string) error {
+func moveModules(to string, projectRoot ProjectRoot, pkgs pkgtree.PackageTree) error {
+	basedir := strings.TrimSuffix(to, string(projectRoot))
+	modules := map[string]struct{}{}
+	for _, pkg := range pkgs.Packages {
+		if pkg.P.RelModPath == "" {
+			continue
+		}
+
+		// the filesystem path containing go.mod
+		modpath := filepath.FromSlash(filepath.Join(string(projectRoot), pkg.P.RelModPath))
+
+		// the filesystem path of the module (before renaming)
+		godir := filepath.FromSlash(filepath.Join(basedir, modpath))
+
+		// where the module will have to be moved
+		moddir := filepath.FromSlash(filepath.Join(basedir, pkg.P.Module))
+
+		// ensure we have to move the directory
+		if godir == moddir {
+			continue
+		}
+
+		// only do this once per modpath
+		if _, ok := modules[modpath]; ok {
+			continue
+		}
+
+		modules[modpath] = struct{}{}
+
+		tmppath := filepath.Join(filepath.Dir(godir), "_"+filepath.Base(godir))
+
+		// move the dir to a temp dir
+		if err := os.Rename(godir, tmppath); err != nil {
+			return errors.Wrap(err, "error moving module to tempdir")
+		}
+
+		// create the dir containing the module
+		if err := os.MkdirAll(filepath.Dir(moddir), 0755); err != nil {
+			return errors.Wrap(err, "error making module dir")
+		}
+
+		// move the module into the proper place
+		if err := os.Rename(tmppath, moddir); err != nil {
+			return errors.Wrap(err, "error moving module from tempdir")
+		}
+	}
+
+	return nil
+}
+
+func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to string, projectRoot ProjectRoot) error {
+	pkgs, err := sg.listPackages(ctx, projectRoot, v)
+	if err != nil {
+		return err
+	}
+
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
 
-	err := sg.require(ctx, sourceExistsLocally)
+	err = sg.require(ctx, sourceExistsLocally)
 	if err != nil {
 		return err
 	}
@@ -337,7 +395,11 @@ func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to stri
 	}
 
 	err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
-		return sg.src.exportRevisionTo(ctx, r, to)
+		if eerr := sg.src.exportRevisionTo(ctx, r, to); eerr != nil {
+			return eerr
+		}
+
+		return moveModules(to, projectRoot, pkgs)
 	})
 
 	// It's possible (in git) that we may have tried this against a version that
@@ -349,7 +411,11 @@ func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to stri
 	if err != nil && sg.srcState&sourceHasLatestLocally == 0 {
 		if err = sg.require(ctx, sourceHasLatestLocally); err == nil {
 			err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
-				return sg.src.exportRevisionTo(ctx, r, to)
+				if eerr := sg.src.exportRevisionTo(ctx, r, to); eerr != nil {
+					return eerr
+				}
+
+				return moveModules(to, projectRoot, pkgs)
 			})
 		}
 	}
@@ -358,10 +424,15 @@ func (sg *sourceGateway) exportVersionTo(ctx context.Context, v Version, to stri
 }
 
 func (sg *sourceGateway) exportPrunedVersionTo(ctx context.Context, lp LockedProject, prune PruneOptions, to string) error {
+	pkgs, err := sg.listPackages(ctx, lp.Ident().ProjectRoot, lp.Version())
+	if err != nil {
+		return err
+	}
+
 	sg.mu.Lock()
 	defer sg.mu.Unlock()
 
-	err := sg.require(ctx, sourceExistsLocally)
+	err = sg.require(ctx, sourceExistsLocally)
 	if err != nil {
 		return err
 	}
@@ -378,7 +449,11 @@ func (sg *sourceGateway) exportPrunedVersionTo(ctx context.Context, lp LockedPro
 	}
 
 	if err = sg.suprvsr.do(ctx, sg.src.upstreamURL(), ctExportTree, func(ctx context.Context) error {
-		return sg.src.exportRevisionTo(ctx, r, to)
+		if eerr := sg.src.exportRevisionTo(ctx, r, to); eerr != nil {
+			return eerr
+		}
+
+		return moveModules(to, lp.Ident().ProjectRoot, pkgs)
 	}); err != nil {
 		return err
 	}

gps/source_cache_bolt_encode.go

@@ -18,6 +18,8 @@ import (
 )
 
 var (
+	cacheKeyModule       = []byte("d")
+	cacheKeyModPath      = []byte("a")
 	cacheKeyComment      = []byte("c")
 	cacheKeyConstraint   = cacheKeyComment
 	cacheKeyError        = []byte("e")
@@ -372,6 +374,18 @@ func cachePutPackageOrErr(b *bolt.Bucket, poe pkgtree.PackageOrErr) error {
 		err := b.Put(cacheKeyError, []byte(poe.Err.Error()))
 		return errors.Wrapf(err, "failed to put error: %v", poe.Err)
 	}
+	if len(poe.P.RelModPath) > 0 {
+		err := b.Put(cacheKeyModPath, []byte(poe.P.RelModPath))
+		if err != nil {
+			return errors.Wrapf(err, "failed to put package: %v", poe.P)
+		}
+	}
+	if len(poe.P.Module) > 0 {
+		err := b.Put(cacheKeyModule, []byte(poe.P.Module))
+		if err != nil {
+			return errors.Wrapf(err, "failed to put package: %v", poe.P)
+		}
+	}
 	if len(poe.P.CommentPath) > 0 {
 		err := b.Put(cacheKeyComment, []byte(poe.P.CommentPath))
 		if err != nil {
@@ -427,6 +441,8 @@ func cacheGetPackageOrErr(b *bolt.Bucket) (pkgtree.PackageOrErr, error) {
 	}
 
 	var p pkgtree.Package
+	p.Module = string(b.Get(cacheKeyModule))
+	p.RelModPath = string(b.Get(cacheKeyModPath))
 	p.CommentPath = string(b.Get(cacheKeyComment))
 	if ip := b.Bucket(cacheKeyImport); ip != nil {
 		err := ip.ForEach(func(_, v []byte) error {

gps/source_manager.go

@@ -556,7 +556,7 @@ func (sm *SourceMgr) ExportProject(ctx context.Context, id ProjectIdentifier, v
 		return err
 	}
 
-	return srcg.exportVersionTo(ctx, v, to)
+	return srcg.exportVersionTo(ctx, v, to, id.ProjectRoot)
 }
 
 // ExportPrunedProject writes out a tree of the provided LockedProject, applying

gps/source_test.go

@@ -151,7 +151,7 @@ func testSourceGateway(t *testing.T) {
 				t.Fatalf("wanted nonexistent err when passing bad version, got: %s", err)
 			}
 
-			err = sg.exportVersionTo(ctx, badver, cachedir)
+			err = sg.exportVersionTo(ctx, badver, cachedir, ProjectRoot("github.com/sdboyer/deptest"))
 			if err == nil {
 				t.Fatal("wanted err on nonexistent version")
 			} else if err.Error() != wanterr.Error() {