refactor(go): Support Godep, add integration tests

.circleci/config.yml

@@ -22,28 +22,34 @@ jobs:
           command: |
             # Invalidate pkg cache for FOSSA CLI binary
             rm -rf /home/fossa/go/pkg/linux_amd64/github.com/fossas/fossa-cli
-            dep ensure
             make
       - save_cache:
           key: v1-gopkg-cache-{{ checksum "Gopkg.lock" }}
           paths:
             - "/home/fossa/go/pkg"
             - "/home/fossa/go/src/github.com/fossas/fossa-cli/vendor"
       - run:
-          name: Run integration test
+          name: Run unit tests
           command: |
             # Load shell helpers (e.g. sdkman)
             source /home/fossa/.bashrc
             # Run tests
-            go test ./builders
+            go test ./...
+      - run:
+          name: Run integration tests
+          command: |
+            # Load shell helpers (e.g. sdkman)
+            source /home/fossa/.bashrc
+            # Run tests
+            ./test.sh
       - run:
           name: Run FOSSA build
           command: |
-            fossa --debug 2>$ARTIFACTS/fossa-build-stderr
+            fossa --debug 2> tee $ARTIFACTS/fossa-build-stderr
       - run:
           name: Run FOSSA license check
           command: |
-            fossa test --debug 2>$ARTIFACTS/fossa-test-stderr
+            fossa test --debug 2> tee $ARTIFACTS/fossa-test-stderr
       - store_test_results:
           path: /tmp/test-results
       - store_artifacts:

Dockerfile

@@ -49,7 +49,7 @@ RUN wget https://dl.google.com/android/repository/sdk-tools-linux-3859397.zip -O
 ENV PATH=$PATH:/opt/android-sdk/tools/bin ANDROID_HOME=/opt/android-sdk
 
 # Install Go compiler
-RUN wget https://dl.google.com/go/go1.9.4.linux-amd64.tar.gz -O /tmp/go.tar.gz && \
+RUN wget https://dl.google.com/go/go1.10.3.linux-amd64.tar.gz -O /tmp/go.tar.gz && \
     sudo tar -xf /tmp/go.tar.gz -C /usr/local 
 ENV GOPATH=/home/fossa/go PATH=$PATH:/usr/local/go/bin:/home/fossa/go/bin
 

analyzers/golang/golang.go

@@ -81,12 +81,13 @@ func (a *Analyzer) Discover(dir string) ([]module.Module, error) {
 	}
 
 	var projects []module.Module
-	for _, p := range found {
+	for _, f := range found {
+		log.Logger.Debugf("Found Go module: %#v", f)
 		projects = append(projects, module.Module{
-			Name:         p.Name,
+			Name:         Unvendor(f.ImportPath),
 			Type:         pkg.Go,
-			IsExecutable: p.Name == "main",
-			BuildTarget:  p.ImportPath,
+			IsExecutable: f.Name == "main",
+			BuildTarget:  f.ImportPath,
 		})
 	}
 	return projects, nil
@@ -192,7 +193,7 @@ func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
 		id := pkg.ID{
 			Type:     pkg.Go,
 			Name:     Unvendor(gopkg.ImportPath),
-			Revision: revision,
+			Revision: revision.Resolved.Revision,
 			Location: "", // TODO: fill this field with something useful?
 		}
 
@@ -209,7 +210,7 @@ func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
 				Resolved: pkg.ID{
 					Type:     pkg.Go,
 					Name:     name,
-					Revision: revision,
+					Revision: revision.Resolved.Revision,
 					Location: "",
 				},
 			})
@@ -233,7 +234,7 @@ func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
 		imports = append(imports, pkg.ID{
 			Type:     pkg.Go,
 			Name:     name,
-			Revision: revision,
+			Revision: revision.Resolved.Revision,
 			Location: "",
 		})
 	}
@@ -252,19 +253,19 @@ func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
 //   2. The package is internal.
 //   3. The package is within the project.
 //
-func GetRevision(project Project, resolver Resolver, gopkg gocmd.Package) (string, error) {
+func GetRevision(project Project, resolver Resolver, gopkg gocmd.Package) (pkg.Import, error) {
 	log.Logger.Debugf("GetRevision: %#v", gopkg)
 	name := Unvendor(gopkg.ImportPath)
 	revision, err := resolver.Resolve(name)
 	if err == errutil.ErrNoRevisionForPackage {
 		log.Logger.Debugf("Could not find revision for package %#v", name)
 		if gopkg.IsStdLib || gopkg.IsInternal || strings.HasPrefix(gopkg.Dir, project.Dir) {
 			log.Logger.Debugf("Skipping package: %#v", gopkg)
-			return "", nil
+			return pkg.Import{}, nil
 		}
 	}
 	if err != nil {
-		return "", err
+		return pkg.Import{}, err
 	}
 	return revision, nil
 }

analyzers/golang/resolve.go

@@ -4,6 +4,8 @@ import (
 	"errors"
 
 	"github.com/fossas/fossa-cli/buildtools/dep"
+	"github.com/fossas/fossa-cli/buildtools/godep"
+	"github.com/fossas/fossa-cli/pkg"
 )
 
 var (
@@ -13,7 +15,7 @@ var (
 // A Resolver provides a single method for resolving the revision of a Go
 // package.
 type Resolver interface {
-	Resolve(importpath string) (string, error)
+	Resolve(importpath string) (pkg.Import, error)
 }
 
 func NewResolver(resolver, dir string) (Resolver, error) {
@@ -25,7 +27,7 @@ func NewResolver(resolver, dir string) (Resolver, error) {
 	case "glide":
 		return nil, errors.New("not yet implemented")
 	case "godep":
-		return nil, errors.New("not yet implemented")
+		return godep.New(dir)
 	case "govendor":
 		return nil, errors.New("not yet implemented")
 	case "vndr":

api/fossa/normalize.go

@@ -29,8 +29,8 @@ type Build struct {
 
 type Dependency struct {
 	// Location
-	Locator string `json:"locator"`
-	Imports []string
+	Locator string   `json:"locator"`
+	Imports []string `json:"imports,omitempty"`
 
 	// Metadata
 	Data *json.RawMessage `json:"data,omitempty"`

buildtools/dep/dep.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/fossas/fossa-cli/errutil"
 	"github.com/fossas/fossa-cli/files"
+	"github.com/fossas/fossa-cli/pkg"
 )
 
 // ErrNoLockfile is returned if a dep manifest is found without an accompanying lockfile.
@@ -21,11 +22,22 @@ type Project struct {
 	Version  string
 }
 
-// A Lockfile contains the contents of a dep lockfile.
+// A Lockfile contains the contents of a dep lockfile. Lockfiles are resolvers.
 type Lockfile struct {
 	Projects []Project
 
-	normalized map[string]string // A normalized map of package import paths to revisions.
+	normalized map[string]pkg.Import // A normalized map of package import paths to revisions.
+}
+
+// Resolve returns the revision of an imported Go package contained within the
+// lockfile. If the package is not found, errutil.ErrNoRevisionForPackage is
+// returned.
+func (l Lockfile) Resolve(importpath string) (pkg.Import, error) {
+	rev, ok := l.normalized[importpath]
+	if !ok {
+		return pkg.Import{}, errutil.ErrNoRevisionForPackage
+	}
+	return rev, nil
 }
 
 // New constructs a golang.Resolver
@@ -41,27 +53,24 @@ func New(dirname string) (Lockfile, error) {
 	if err != nil {
 		return Lockfile{}, err
 	}
-	normalized := make(map[string]string)
+	normalized := make(map[string]pkg.Import)
 	for _, project := range lockfile.Projects {
-		for _, pkg := range project.Packages {
-			normalized[path.Join(project.Name, pkg)] = project.Revision
+		for _, pk := range project.Packages {
+			importpath := path.Join(project.Name, pk)
+			normalized[importpath] = pkg.Import{
+				Target: project.Version,
+				Resolved: pkg.ID{
+					Type:     pkg.Go,
+					Name:     importpath,
+					Revision: project.Revision,
+					Location: "",
+				},
+			}
 		}
 	}
-	return Lockfile{
-		Projects:   lockfile.Projects,
-		normalized: normalized,
-	}, nil
-}
 
-// Resolve returns the revision of an imported Go package contained within the
-// lockfile. If the package is not found, errutil.ErrNoRevisionForPackage is
-// returned.
-func (l Lockfile) Resolve(importpath string) (string, error) {
-	rev, ok := l.normalized[importpath]
-	if !ok {
-		return "", errutil.ErrNoRevisionForPackage
-	}
-	return rev, nil
+	lockfile.normalized = normalized
+	return lockfile, nil
 }
 
 // UsedIn checks whether dep is used correctly within a project folder.

buildtools/gocmd/go.go

@@ -40,6 +40,7 @@ type Go struct {
 // GoListOutput is a subset of the output of `go list`. See `go help list` for
 // details.
 type GoListOutput struct {
+	Name       string
 	ImportPath string
 	Dir        string
 	Standard   bool
@@ -80,7 +81,7 @@ func (g *Go) List(pkgs []string) ([]Package, error) {
 			"GOARCH": g.Arch,
 		},
 	})
-	if err != nil {
+	if err != nil && stdout == "" {
 		if strings.Index(stderr, "build constraints exclude all Go files") != -1 {
 			// TODO: add better documentation around this error, and rename it to be
 			// more useful.
@@ -99,7 +100,7 @@ func (g *Go) List(pkgs []string) ([]Package, error) {
 	var ret []Package
 	for _, pkg := range output {
 		p := Package{
-			Name:       Name(pkg.ImportPath),
+			Name:       pkg.Name,
 			ImportPath: pkg.ImportPath,
 			Dir:        pkg.Dir,
 			IsInternal: strings.Index(pkg.ImportPath, "internal") != -1,

buildtools/godep/godep.go

@@ -2,9 +2,12 @@
 package godep
 
 import (
+	"errors"
 	"path/filepath"
 
+	"github.com/fossas/fossa-cli/errutil"
 	"github.com/fossas/fossa-cli/files"
+	"github.com/fossas/fossa-cli/pkg"
 )
 
 // A Package is a single imported package within a godep project.
@@ -20,6 +23,45 @@ type Lockfile struct {
 	GoVersion    string
 	GodepVersion string
 	Deps         []Package
+
+	normalized map[string]pkg.Import
+}
+
+func (l Lockfile) Resolve(importpath string) (pkg.Import, error) {
+	rev, ok := l.normalized[importpath]
+	if !ok {
+		return pkg.Import{}, errutil.ErrNoRevisionForPackage
+	}
+	return rev, nil
+}
+
+func New(dirname string) (Lockfile, error) {
+	ok, err := UsedIn(dirname)
+	if err != nil {
+		return Lockfile{}, err
+	}
+	if !ok {
+		return Lockfile{}, errors.New("directory does not use godep")
+	}
+	lockfile, err := ReadRaw(filepath.Join(dirname, "Godeps", "Godeps.json"))
+	if err != nil {
+		return Lockfile{}, err
+	}
+	normalized := make(map[string]pkg.Import)
+	for _, project := range lockfile.Deps {
+		normalized[project.ImportPath] = pkg.Import{
+			Target: project.Comment,
+			Resolved: pkg.ID{
+				Type:     pkg.Go,
+				Name:     project.ImportPath,
+				Revision: project.Rev,
+				Location: "",
+			},
+		}
+	}
+
+	lockfile.normalized = normalized
+	return lockfile, nil
 }
 
 // UsedIn checks whether godep is used correctly within a project folder.
@@ -41,3 +83,13 @@ func ReadFile(filename string) ([]Package, error) {
 	}
 	return lockfile.Deps, nil
 }
+
+// ReadRaw reads a raw Lockfile from a Godeps/Godeps.json file.
+func ReadRaw(filename string) (Lockfile, error) {
+	var lockfile Lockfile
+	err := files.ReadJSON(&lockfile, filename)
+	if err != nil {
+		return Lockfile{}, err
+	}
+	return lockfile, nil
+}

cmd/fossa/cmd/init/init.go

@@ -70,21 +70,30 @@ func Do(overwrite, includeAll bool) ([]module.Module, error) {
 		discovered = append(discovered, modules...)
 	}
 
-	// Filter suspicious modules.
+	// Filter noisy modules (docs, examples, etc.).
 	if includeAll {
 		return discovered, nil
 	}
 	var filtered []module.Module
 	for _, d := range discovered {
-		matched, err := regexp.MatchString("(docs?/|test|example|vendor/|node_modules/|.srclib-cache/|spec/|Godeps/|.git/|bower_components/|third_party/)", d.Dir)
+		log.Logger.Debugf("Discovered: %#v", d)
+
+		// Match name regexp.
+		matched, err := regexp.MatchString("(docs?/|test|examples?|vendor/|node_modules/|.srclib-cache/|spec/|Godeps/|.git/|bower_components/|third_party/)", d.Dir)
 		if err != nil {
 			return nil, err
 		}
 		if matched {
 			log.Logger.Warningf("Filtering out suspicious module: %s (%s)", d.Name, d.BuildTarget)
-		} else {
-			filtered = append(filtered, d)
+			continue
+		}
+
+		// For Go, filter out non-executable packages.
+		if d.Type == pkg.Go && !d.IsExecutable {
+			continue
 		}
+
+		filtered = append(filtered, d)
 	}
 	return filtered, nil
 }