Skip to content

Commit

Permalink
WIP: Go option-based analyzers
Browse files Browse the repository at this point in the history
  • Loading branch information
elldritch committed Jun 8, 2018
1 parent f943789 commit fae5186
Show file tree
Hide file tree
Showing 73 changed files with 396 additions and 118 deletions.
106 changes: 64 additions & 42 deletions analyzers/golang/golang.go
Original file line number Diff line number Diff line change
@@ -1,24 +1,29 @@
// Package golang implements the analyzer for Go.
//
// Strategies:
// - default
// - use specific tool, read specific manifest at weird location (anki)
// This package is implemented by externally calling both the `go` tool and any
// external build tools.
//
// options:
// - allow unresolved
// - allow unresolved prefix (hashi govendor)
// FAQ
//
// Why not use `go/build`, or a library like `KyleBanks/depth`?
//
// The `go` tool's interface is incredibly stable over different releases, but
// the internals are not. Using these libraries causes crashes when analyzing
// code that is compiled using a different version of Go. (This was how the
// analyzer was originally implemented.)
package golang

import (
"os"

"github.com/mitchellh/mapstructure"
"github.com/pkg/errors"

"github.com/fossas/fossa-cli/buildtools/gocmd"
"github.com/fossas/fossa-cli/exec"
"github.com/fossas/fossa-cli/log"
"github.com/fossas/fossa-cli/module"
"github.com/fossas/fossa-cli/pkg"
"github.com/pkg/errors"
)

// An Analyzer contains structs used in the analysis of Go packages.
Expand All @@ -31,14 +36,14 @@ type Analyzer struct {

// Options set analyzer options for Go modules.
type Options struct {
BuildOS string `mapstructure:"os"` // The target build OS (for build tags).
BuildArch string `mapstructure:"arch"` // The target build architecture (for build tags).
Strategy string `mapstructure:"strategy"` // See the Go analyzer documentation.
Strategies []string `mapstructure:"strategies"` // Fallback strategies to try in succession.
LockfilePath string `mapstructure:"lockfile"` // For non-standard lockfile locations.
AllowUnresolved bool `mapstructure:"allow-unresolved"` // Allow unresolved revisions.
AllowUnresolvedPrefix string `mapstructure:"allow-unresolved-prefix"` // If set, restricts unresolved revisions to only those that match the prefix.
SkipImportTracing bool `mapstructure:"skip-tracing"` // If true, skips dependency tracing.
BuildOS string `mapstructure:"os"` // Target build OS (for build tags).
BuildArch string `mapstructure:"arch"` // Target build architecture (for build tags).
Strategy string `mapstructure:"strategy"` // See the Go analyzer documentation.
LockfilePath string `mapstructure:"lockfile"` // For non-standard lockfile locations with strategies `manifest:*`.
AllowUnresolved bool `mapstructure:"allow-unresolved"` // Allow unresolved revisions.
AllowUnresolvedPrefix string `mapstructure:"allow-unresolved-prefix"` // If set, allows unresolved revisions for packages whose import path's prefix matches.
SkipImportTracing bool `mapstructure:"skip-tracing"` // If true, skips dependency tracing.
SkipProject bool `mapstructure:"skip-project"` // If true, skips project detection.
}

// New constructs an Analyzer.
Expand Down Expand Up @@ -86,18 +91,18 @@ func (a *Analyzer) Discover(dir string) ([]module.Module, error) {
}

// Clean runs `go clean $PKG`.
func (a *Analyzer) Clean(p module.Module) error {
return a.Go.Clean([]string{p.BuildTarget})
func (a *Analyzer) Clean(m module.Module) error {
return a.Go.Clean([]string{m.BuildTarget})
}

// Build runs `go build $PKG`.
func (a *Analyzer) Build(p module.Module) error {
return a.Go.Build([]string{p.BuildTarget})
func (a *Analyzer) Build(m module.Module) error {
return a.Go.Build([]string{m.BuildTarget})
}

// IsBuilt runs `go list $PKG` and checks for errors.
func (a *Analyzer) IsBuilt(p module.Module) (bool, error) {
pkg, err := a.Go.ListOne(p.BuildTarget)
func (a *Analyzer) IsBuilt(m module.Module) (bool, error) {
pkg, err := a.Go.ListOne(m.BuildTarget)
if err != nil {
return false, err
}
Expand All @@ -106,42 +111,59 @@ func (a *Analyzer) IsBuilt(p module.Module) (bool, error) {

// Analyze builds a dependency graph using go list and then looks up revisions
// using tool-specific lockfiles.
func (a *Analyzer) Analyze(p module.Module) (module.Module, error) {
func (a *Analyzer) Analyze(m module.Module) (module.Module, error) {
// Get Go project.
project, err := a.GetProject(m.BuildTarget)
if err != nil {
return m, err
}
log.Logger.Debugf("Go project: %#v", project)

// Read lockfiles to get revisions.
var lockfile Resolver
switch a.Options.Strategy {
// Read revisions from a tool manifest at a specified location.
case "manifest:dep":
return m, errors.New("not yet implemented")
case "manifest:gdm":
return m, errors.New("not yet implemented")
case "manifest:glide":
return m, errors.New("not yet implemented")
case "manifest:godep":
return p, errors.New("not yet implemented")
return m, errors.New("not yet implemented")
case "manifest:govendor":
return p, errors.New("not yet implemented")
case "manifest:dep":
return p, errors.New("not yet implemented")
return m, errors.New("not yet implemented")
case "manifest:vndr":
return p, errors.New("not yet implemented")
case "manifest:glide":
return p, errors.New("not yet implemented")
return m, errors.New("not yet implemented")

// Resolve revisions by traversing the local $GOPATH and calling the package's
// VCS.
case "gopath-vcs":
return p, errors.New("not yet implemented")
return m, errors.New("not yet implemented")

// Read revisions from an auto-detected tool manifest.
default:
return p, errors.New("not yet implemented")
lockfile, err = NewResolver(project.Tool, project.Manifest)
if err != nil {
return m, err
}
}

// Use `go list` to get imports and deps of module.
gopkg, err := a.Go.ListOne(p.BuildTarget)
if err != nil {
return module.Module{}, err
}
var imports []pkg.ID
for _, i := range gopkg.Imports {
imports = append(imports, pkg.ID{})
}
_ = lockfile

// // Use `go list` to get imports and deps of module.
// gopkg, err := a.Go.ListOne(m.BuildTarget)
// if err != nil {
// return module.Module{}, err
// }
// var imports []pkg.ID
// for _, i := range gopkg.Imports {
// imports = append(imports, pkg.ID{})
// }

// // Use `go list` to get imports of deps of module.

// Use `go list` to get imports of deps of module.
// // If import tracing fails, warn and upload the lockfile.

return p, nil
return m, nil
}
139 changes: 121 additions & 18 deletions analyzers/golang/project.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,126 @@
package golang

// The project of a package is the nearest ancestor with a supported lockfile.
// Two packages in the same project can import each other without needing a
// revision in the lockfile, because these are ostensibly internal.
// Conceptually, anything in the same VCS project (and not in a vendor folder) should be an "internal" import
// because they'll all be versioned together.
// Looking upwards for a lockfile is a subset of this: it's never possible to
// have a lockfile "in between" directories that are not repositories (you'll
// never have github.com/hashicorp/Gopkg.lock), but your internal import might
// be higher up than the closest lockfile.
// alternatively: search upwards for .git or other VCS marker?
// A vendored package always in a different project than the one vendoring it.
func GetProjectFolder(pkg string) (string, error) {
return "", nil
import (
"github.com/fossas/fossa-cli/files"
)

// A Project is a single folder that forms a coherent "project" for a developer
// and is versioned as a single unit. It may contain multiple Go packages.
type Project struct {
Tool string // Name of the dependency management tool used by the project, if any.
Manifest string // Absolute path to the tool's manifest file for this project, if any.
Internal string // Absolute path of the first-party code folder, if any.
}

// i know the location of every package on disk.
// every package that goes up to the same VCS metadata folder _and_ is not in a
// vendor folder is in the same project.
// GetProject gets the project containing any Go package.
//
// This function searches upwards from the Go package's directory, looking for
// lockfiles of supported dependency management tools. If none are found, it
// fails.
//
// The rationale for this design is that the packages in a "project" are
// versioned together. There are two reasonable ways to capture the notion of
// "versioned together":
//
// 1. The nearest lockfile. The nearest lockfile to the package probably locks
// the dependencies of the package.
// 2. The nearest VCS repository. The nearest VCS repository probably contains
// the current "project" being worked on. The only common exception to this
// is monorepos, in which case all the contents of the repository are
// probably internal, so allowing packages within the repository to be
// unresolved is probably acceptable.
//
// There are a couple issues with both of these:
//
// 1. The nearest lockfile is not guaranteed to exist. When it does, it's not
// guaranteed to be _the_ semantic lockfile for the package -- this is
// merely a very common convention, not a requirement.
// 2. The package is not guaranteed to be in a VCS repository. When it is, the
// repository might be extremely weird. One example of this is a repository
// containing the entire $GOPATH (which is a reasonable convention that
// some early adopters of Go used).
//
// fall back to using lockfile location for project discovery _iff_ no VCS is
// found (e.g. new project without git init)
// This function tries its best to mitigate both of these issues:
//
// 1. The nearest lockfile is used for resolving versions. This is a very
// strong convention.
// 2. The nearest VCS repository is used for determining allowed unresolved
// import paths. This is also a very strong convention.
//
// Both of these assumptions can be overridden by the user.
func (a *Analyzer) GetProject(pkg string) (Project, error) {
// Get the directory.
dir, err := a.Dir(pkg)
if err != nil {
return Project{}, err
}

// Find the nearest lockfile.
var toolName string
manifestDir, err := files.WalkUp(dir, func(d string) (bool, error) {
either := &resultExists{}
either.Find("godep", d, "Godeps", "Godeps.json")
either.Find("govendor", d, "vendor", "vendor.json")
either.Find("dep", d, "Gopkg.toml")
either.Find("vndr", d, "vendor.conf")
either.Find("glide", d, "glide.yaml")
either.Find("gdm", d, "Godeps")
if either.err != nil {
return false, either.err
}
if either.which != "" {
toolName = either.which
}
return either.which != "", nil
})

// Find the nearest VCS repository.
repoRoot, err := files.WalkUp(dir, func(d string) (bool, error) {
either := &resultExists{}
either.FindFolder("git", d, ".git")
either.FindFolder("svn", d, ".svn")
either.FindFolder("hg", d, ".hg")
either.FindFolder("bzr", d, ".bzr")
if either.err != nil {
return false, either.err
}
return either.which != "", nil
})
if err != nil {
return Project{}, err
}

return Project{
Tool: toolName,
Manifest: manifestDir,
Internal: repoRoot,
}, nil
}

// This is a monomorphic Either monad. I miss Haskell.
type resultExists struct {
which string
err error
}

func (r *resultExists) Bind(which string, find func(pathElems ...string) (bool, error), pathElems ...string) {
if r.err != nil {
return
}

ok, err := find(pathElems...)
if err != nil {
r.err = err
}
if ok {
r.which = which
}
}

func (r *resultExists) Find(which string, pathElems ...string) {
r.Bind(which, files.Exists, pathElems...)
}

func (r *resultExists) FindFolder(which string, pathElems ...string) {
r.Bind(which, files.ExistsFolder, pathElems...)
}
39 changes: 30 additions & 9 deletions analyzers/golang/resolve.go
Original file line number Diff line number Diff line change
@@ -1,15 +1,36 @@
package golang

import (
"github.com/fossas/fossa-cli/module"
"errors"

"github.com/fossas/fossa-cli/pkg"
)

var (
ErrResolverNotFound = errors.New("unrecognized Go resolver")
)

func Resolve(a Analyzer, p module.Module) {
// exec.Run(exec.Cmd{
// Name: a.GoCmd,
// Argv: []string{"list", "-json", p.BuildTarget},
// })
// golang.List()
// dep.UsedIn()
// dep.Read()
// A Resolver provides a single method for resolving the revision of a Go
// package.
type Resolver interface {
Resolve(importpath string) pkg.ID
}

func NewResolver(resolver, dir string) (Resolver, error) {
switch resolver {
case "dep":
return nil, errors.New("not yet implemented")
case "gdm":
return nil, errors.New("not yet implemented")
case "glide":
return nil, errors.New("not yet implemented")
case "godep":
return nil, errors.New("not yet implemented")
case "govendor":
return nil, errors.New("not yet implemented")
case "vndr":
return nil, errors.New("not yet implemented")
default:
return nil, ErrResolverNotFound
}
}
10 changes: 10 additions & 0 deletions analyzers/golang/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
package golang

// Dir returns the absolute path to a Go package.
func (a *Analyzer) Dir(importpath string) (string, error) {
pkg, err := a.Go.ListOne(importpath)
if err != nil {
return "", err
}
return pkg.Dir, nil
}
10 changes: 10 additions & 0 deletions buildtools/dep/dep.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"path/filepath"

"github.com/fossas/fossa-cli/files"
"github.com/fossas/fossa-cli/pkg"
)

// ErrNoLockfile is returned if a dep manifest is found without an accompanying lockfile.
Expand All @@ -24,6 +25,15 @@ type Lockfile struct {
Projects []Project
}

// New constructs a golang.Resolver
func New(dirname string) (Lockfile, error) {
return Lockfile{}, errors.New("not implemented")
}

func (l Lockfile) Resolve(importpath string) (pkg.ID, error) {
return pkg.ID{}, errors.New("not implemented")
}

// UsedIn checks whether dep is used correctly within a project folder.
func UsedIn(dirname string) (bool, error) {
// Check whether there exists a manifest.
Expand Down

0 comments on commit fae5186

Please sign in to comment.