WIP: Go dep analysis

.fossa.yml

@@ -9,5 +9,5 @@ cli:
 analyze:
   modules:
   - name: fossa
-    path: cmd/fossa
+    path: ./cmd/fossa
     type: go

Gopkg.toml

@@ -1,3 +1,11 @@
+ignored = [
+  "github.com/fossas/fossa-cli/fixtures/*"
+]
+
+[prune]
+  go-tests = true
+  unused-packages = true
+
 [[constraint]]
   name = "github.com/BurntSushi/toml"
   version = "0.3.0"
@@ -45,7 +53,3 @@
 [[constraint]]
   name = "gopkg.in/yaml.v2"
   version = "2.2.1"
-
-[prune]
-  go-tests = true
-  unused-packages = true

analyzers/golang/golang.go

@@ -15,11 +15,13 @@ package golang
 
 import (
 	"os"
+	"strings"
 
 	"github.com/mitchellh/mapstructure"
 	"github.com/pkg/errors"
 
 	"github.com/fossas/fossa-cli/buildtools/gocmd"
+	"github.com/fossas/fossa-cli/errutil"
 	"github.com/fossas/fossa-cli/exec"
 	"github.com/fossas/fossa-cli/log"
 	"github.com/fossas/fossa-cli/module"
@@ -102,6 +104,7 @@ func (a *Analyzer) Build(m module.Module) error {
 
 // IsBuilt runs `go list $PKG` and checks for errors.
 func (a *Analyzer) IsBuilt(m module.Module) (bool, error) {
+	log.Logger.Debugf("%#v", m)
 	pkg, err := a.Go.ListOne(m.BuildTarget)
 	if err != nil {
 		return false, err
@@ -112,6 +115,7 @@ func (a *Analyzer) IsBuilt(m module.Module) (bool, error) {
 // Analyze builds a dependency graph using go list and then looks up revisions
 // using tool-specific lockfiles.
 func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
+	log.Logger.Debugf("%#v", m)
 	// Get Go project.
 	project, err := a.GetProject(m.BuildTarget)
 	if err != nil {
@@ -149,21 +153,118 @@ func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
 		}
 	}
 
-	_ = lockfile
+	log.Logger.Debugf("Lockfile: %#v", lockfile)
 
-	// // Use `go list` to get imports and deps of module.
-	// gopkg, err := a.Go.ListOne(m.BuildTarget)
-	// if err != nil {
-	// 	return module.Module{}, err
-	// }
-	// var imports []pkg.ID
-	// for _, i := range gopkg.Imports {
-	// 	imports = append(imports, pkg.ID{})
-	// }
+	// Use `go list` to get imports and deps of module.
+	main, err := a.Go.ListOne(m.BuildTarget)
+	if err != nil {
+		return m, err
+	}
+	log.Logger.Debugf("Go main package: %#v", main)
+	deps, err := a.Go.List(main.Deps)
+	if err != nil {
+		return m, err
+	}
+
+	// Construct map of unvendored import path to package.
+	gopkgs := append(deps, main)
+	gopkgMap := make(map[string]gocmd.Package)
+	for _, p := range gopkgs {
+		gopkgMap[Unvendor(p.ImportPath)] = p
+	}
+	// cgo imports don't have revisions.
+	gopkgMap["C"] = gocmd.Package{
+		Name:     "C",
+		IsStdLib: true,
+	}
+
+	// Construct dependency graph.
+	pkgs := make(map[pkg.ID]pkg.Package)
+	_ = pkgs
+	for _, gopkg := range gopkgs {
+		log.Logger.Debugf("Getting revision for: %#v", gopkg)
+
+		// Get revision.
+		revision, err := GetRevision(project, lockfile, gopkg)
+		if err != nil {
+			return m, err
+		}
+		id := pkg.ID{
+			Type:     pkg.Go,
+			Name:     Unvendor(gopkg.ImportPath),
+			Revision: revision,
+			Location: "", // TODO: fill this field with something useful?
+		}
+
+		// Resolve imports.
+		var imports []pkg.Import
+		for _, i := range gopkg.Imports {
+			name := Unvendor(i)
+			revision, err := GetRevision(project, lockfile, gopkgMap[name])
+			if err != nil {
+				return m, err
+			}
+			imports = append(imports, pkg.Import{
+				Target: name,
+				Resolved: pkg.ID{
+					Type:     pkg.Go,
+					Name:     name,
+					Revision: revision,
+					Location: "",
+				},
+			})
+		}
 
-	// // Use `go list` to get imports of deps of module.
+		pkgs[id] = pkg.Package{
+			ID:      id,
+			Imports: imports,
+		}
+	}
 
-	// // If import tracing fails, warn and upload the lockfile.
+	// Construct imports list.
+	var imports []pkg.ID
+	for _, i := range main.Imports {
+		name := Unvendor(i)
+		revision, err := GetRevision(project, lockfile, gopkgMap[name])
+		if err != nil {
+			return m, err
+		}
 
+		imports = append(imports, pkg.ID{
+			Type:     pkg.Go,
+			Name:     name,
+			Revision: revision,
+			Location: "",
+		})
+	}
+
+	m.Deps = pkgs
+	m.Imports = imports
 	return m, nil
 }
+
+// GetRevision resolves a revision, returning errutil.ErrNoRevisionForPackage
+// when no revision is found unless a revision is not required.
+//
+// Packages require a resolved revision unless:
+//
+//   1. The package is part of the standard library.
+//   2. The package is internal.
+//   3. The package is within the project.
+//
+func GetRevision(project Project, resolver Resolver, gopkg gocmd.Package) (string, error) {
+	log.Logger.Debugf("GetRevision: %#v", gopkg)
+	name := Unvendor(gopkg.ImportPath)
+	revision, err := resolver.Resolve(name)
+	if err == errutil.ErrNoRevisionForPackage {
+		log.Logger.Debugf("Could not find revision for package %#v", name)
+		if gopkg.IsStdLib || gopkg.IsInternal || strings.HasPrefix(gopkg.Dir, project.Dir) {
+			log.Logger.Debugf("Skipping package: %#v", gopkg)
+			return "", nil
+		}
+	}
+	if err != nil {
+		return "", err
+	}
+	return revision, nil
+}

analyzers/golang/manifest.go

@@ -70,7 +70,7 @@ func (a *Analyzer) ResolveManifest(p module.Module) (module.Module, error) {
 			}
 			imports = append(imports, pkg.Import{
 				Target: "",
-				Resolved: &pkg.ID{
+				Resolved: pkg.ID{
 					Type:     pkg.Go,
 					Name:     imported,
 					Revision: importedRevision.Rev,

analyzers/golang/project.go

@@ -9,7 +9,7 @@ import (
 type Project struct {
 	Tool     string // Name of the dependency management tool used by the project, if any.
 	Manifest string // Absolute path to the tool's manifest file for this project, if any.
-	Internal string // Absolute path of the first-party code folder, if any.
+	Dir      string // Absolute path of the first-party code folder, if any.
 }
 
 // GetProject gets the project containing any Go package.
@@ -58,7 +58,7 @@ func (a *Analyzer) GetProject(pkg string) (Project, error) {
 	// Find the nearest lockfile.
 	var toolName string
 	manifestDir, err := files.WalkUp(dir, func(d string) (bool, error) {
-		either := &resultExists{}
+		either := &eitherStr{}
 		either.Find("godep", d, "Godeps", "Godeps.json")
 		either.Find("govendor", d, "vendor", "vendor.json")
 		either.Find("dep", d, "Gopkg.toml")
@@ -68,23 +68,23 @@ func (a *Analyzer) GetProject(pkg string) (Project, error) {
 		if either.err != nil {
 			return false, either.err
 		}
-		if either.which != "" {
-			toolName = either.which
+		if either.result != "" {
+			toolName = either.result
 		}
-		return either.which != "", nil
+		return either.result != "", nil
 	})
 
 	// Find the nearest VCS repository.
 	repoRoot, err := files.WalkUp(dir, func(d string) (bool, error) {
-		either := &resultExists{}
+		either := &eitherStr{}
 		either.FindFolder("git", d, ".git")
 		either.FindFolder("svn", d, ".svn")
 		either.FindFolder("hg", d, ".hg")
 		either.FindFolder("bzr", d, ".bzr")
 		if either.err != nil {
 			return false, either.err
 		}
-		return either.which != "", nil
+		return either.result != "", nil
 	})
 	if err != nil {
 		return Project{}, err
@@ -93,17 +93,17 @@ func (a *Analyzer) GetProject(pkg string) (Project, error) {
 	return Project{
 		Tool:     toolName,
 		Manifest: manifestDir,
-		Internal: repoRoot,
+		Dir:      repoRoot,
 	}, nil
 }
 
 // This is a monomorphic Either monad. I miss Haskell.
-type resultExists struct {
-	which string
-	err   error
+type eitherStr struct {
+	result string
+	err    error
 }
 
-func (r *resultExists) Bind(which string, find func(pathElems ...string) (bool, error), pathElems ...string) {
+func (r *eitherStr) Bind(tool string, find func(pathElems ...string) (bool, error), pathElems ...string) {
 	if r.err != nil {
 		return
 	}
@@ -113,14 +113,14 @@ func (r *resultExists) Bind(which string, find func(pathElems ...string) (bool,
 		r.err = err
 	}
 	if ok {
-		r.which = which
+		r.result = tool
 	}
 }
 
-func (r *resultExists) Find(which string, pathElems ...string) {
-	r.Bind(which, files.Exists, pathElems...)
+func (r *eitherStr) Find(tool string, pathElems ...string) {
+	r.Bind(tool, files.Exists, pathElems...)
 }
 
-func (r *resultExists) FindFolder(which string, pathElems ...string) {
-	r.Bind(which, files.ExistsFolder, pathElems...)
+func (r *eitherStr) FindFolder(tool string, pathElems ...string) {
+	r.Bind(tool, files.ExistsFolder, pathElems...)
 }

analyzers/golang/resolve.go

@@ -3,7 +3,7 @@ package golang
 import (
 	"errors"
 
-	"github.com/fossas/fossa-cli/pkg"
+	"github.com/fossas/fossa-cli/buildtools/dep"
 )
 
 var (
@@ -13,13 +13,13 @@ var (
 // A Resolver provides a single method for resolving the revision of a Go
 // package.
 type Resolver interface {
-	Resolve(importpath string) pkg.ID
+	Resolve(importpath string) (string, error)
 }
 
 func NewResolver(resolver, dir string) (Resolver, error) {
 	switch resolver {
 	case "dep":
-		return nil, errors.New("not yet implemented")
+		return dep.New(dir)
 	case "gdm":
 		return nil, errors.New("not yet implemented")
 	case "glide":

analyzers/golang/util.go

@@ -1,5 +1,9 @@
 package golang
 
+import (
+	"strings"
+)
+
 // Dir returns the absolute path to a Go package.
 func (a *Analyzer) Dir(importpath string) (string, error) {
 	pkg, err := a.Go.ListOne(importpath)
@@ -8,3 +12,10 @@ func (a *Analyzer) Dir(importpath string) (string, error) {
 	}
 	return pkg.Dir, nil
 }
+
+// Unvendor takes a vendorized import path and strips all vendor folder
+// prefixes.
+func Unvendor(importpath string) string {
+	sections := strings.Split(importpath, "/vendor/")
+	return sections[len(sections)-1]
+}

analyzers/golang/util_test.go

@@ -0,0 +1,13 @@
+package golang_test
+
+import (
+	"testing"
+
+	"github.com/fossas/fossa-cli/analyzers/golang"
+)
+
+func TestUnvendorPathThatIsNotVendored(t *testing.T) {
+	if "foobar" != golang.Unvendor("foobar") {
+		t.Fail()
+	}
+}