Skip to content

Commit

Permalink
Implement the report command. (#171)
Browse files Browse the repository at this point in the history
* Changes the -o / --output option to be a file `fossa analyze -o deps.json` for the original behavior you would do `fossa analyze -o -`
* Split the report types into sub commands `fossa report licenses -o NOTICE.txt` || `fossa report dependencies --output deps.json`
* Added the -t / --template option to process the output via a template file first. `fossa report dependencies --output deps.txt --template deps.tmpl`

Default template for licenses report is
```
# 3rd-Party Software License Notice
Generated by fossa-cli (https://github.com/fossas/fossa-cli).
This software includes the following software and licenses:
{{range $license, $deps := .}}
========================================================================
{{$license}}
========================================================================
The following software have components provided under the terms of this license:
{{range $i, $dep := $deps}}
- {{$dep.Project.Title}} (from {{$dep.Project.URL}})
{{- end}}
{{end}}
```
which will produce the same output as before.

Since `fossa analyze -o deps.json` && `fossa report dependencies --output deps.json` should output the same info should analyze still have the `-o` or should people just use `fossa report dependencies`?
  • Loading branch information
djgilcrease authored and elldritch committed Jul 11, 2018
1 parent d938632 commit fcec296
Show file tree
Hide file tree
Showing 15 changed files with 521 additions and 231 deletions.
6 changes: 6 additions & 0 deletions Gopkg.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

10 changes: 10 additions & 0 deletions api/fossa/fossa.go
Original file line number Diff line number Diff line change
Expand Up @@ -57,3 +57,13 @@ func Post(endpoint string, body []byte) (res string, statusCode int, err error)
u := mustParse(endpoint)
return api.Post(u, apiKey, body)
}

func Get(endpoint string) (res string, statusCode int, err error) {
u := mustParse(endpoint)
return api.Get(u, apiKey, nil)
}

func GetJSON(endpoint string, v interface{}) (statusCode int, err error) {
u := mustParse(endpoint)
return api.GetJSON(u, apiKey, nil, v)
}
11 changes: 9 additions & 2 deletions api/fossa/locator.go
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,13 @@ func (l Locator) String() string {
return "git+" + NormalizeGitURL(l.Project) + "$" + l.Revision
}

func (l Locator) QueryString() string {
if l.Fetcher == "go" {
l.Fetcher = "git"
}
return l.String()
}

func NormalizeGitURL(project string) string {
// Remove fetcher prefix (in case project is derived from splitting a locator on '$')
noFetcherPrefix := strings.TrimPrefix(project, "git+")
Expand Down Expand Up @@ -69,8 +76,8 @@ func ReadImportPath(s ImportPathString) ImportPath {
return out
}

func LocatorOf(id pkg.ID) Locator {
return Locator{
func LocatorOf(id pkg.ID) *Locator {
return &Locator{
Fetcher: LocatorType(id.Type),
Project: id.Name,
Revision: id.Revision,
Expand Down
113 changes: 113 additions & 0 deletions api/fossa/revisions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,113 @@
package fossa

import (
"net/url"

"github.com/fossas/fossa-cli/pkg"
)

type License struct {
ID int64 `json:"id"`
LicenseID string `json:"licenseId"`
RevisionID string `json:"revisionId"`
LicenseGroupID int64 `json:"licenseGroupId"`
Ignored bool `json:"ignored"`
Title string `json:"title"`
URL string `json:"url"`
FullText string `json:"text"`
Copyright string `json:"copyright"`
}

type Revision struct {
Locator *Locator `json:"loc"`
Licenses Licenses `json:"licenses"`
Project *Project `json:"project"`
}

type Project struct {
Title string `json:"title"`
URL string `json:"url"`
Public bool `json:"public"`
Authors []string `json:"authors"`
}

type Licenses = []*License
type Revisions = []*Revision

func FetchRevisionForPackage(p pkg.Package) (rev *Revision, err error) {
locator := LocatorOf(p.ID)
ep := serverURL
ep, _ = ep.Parse("/api/revisions/" + url.PathEscape(locator.QueryString()))
_, err = GetJSON(ep.String(), &rev)

if err != nil {
return rev, err
}

if rev.Locator == nil {
rev.Locator = locator
}
if len(rev.Licenses) == 0 {
rev.Licenses = append(rev.Licenses, &License{
LicenseID: "UNKNOWN",
})
}

if rev.Project == nil {
rev.Project = &Project{
Title: rev.Locator.Project,
URL: "UNKNOWN",
}
}
return rev, err
}

func FetchRevisionForDeps(deps map[pkg.ID]pkg.Package) (revs Revisions, err error) {
pkgs := make([]string, 0, len(deps))
for pkgID := range deps {
pkgs = append(pkgs, LocatorOf(pkgID).QueryString())
}

// Split pkgs into chunks of 20 for performance reasons
chunks := make([][]string, 0)
chunkSize := 20
for i := 0; i < len(pkgs); i += chunkSize {
end := i + chunkSize

if end > len(pkgs) {
end = len(pkgs)
}

chunks = append(chunks, pkgs[i:end])
}

ch := make(chan Revisions, len(chunks)) // buffered
for _, chunk := range chunks {
qs := url.Values{}
for _, q := range chunk {
qs.Add("locator", q)
}
ep := serverURL
if ep, err = ep.Parse("/api/revisions?" + qs.Encode()); err != nil {
return revs, err
}

go func(url string) {
var ret Revisions
if _, err := GetJSON(ep.String(), &ret); err != nil {
close(ch)
}
ch <- ret
}(ep.String())
}

revs = make(Revisions, 0)
for range chunks {
select {
case ret := <-ch:
revs = append(revs, ret...)
}
}

return revs, err
}
23 changes: 16 additions & 7 deletions api/fossa/upload.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,6 @@ package fossa

import (
"encoding/json"
"fmt"
"net/url"

"github.com/pkg/errors"
Expand Down Expand Up @@ -46,19 +45,29 @@ func Upload(fetcher, project, revision, title, branch string, data []SourceUnit)
}
log.Logger.Debugf("Uploading data from %#v modules: %#v", len(data), string(payload))

endpoint := "/api/builds/custom?locator=" + url.QueryEscape(Locator{Fetcher: fetcher, Project: project, Revision: revision}.String()) + "&v=" + version.ShortString()
locator := Locator{Fetcher: fetcher, Project: project, Revision: revision}

q := url.Values{}
q.Add("locator", locator.QueryString())
q.Add("v", version.ShortString())

if fetcher == "custom" {
endpoint += fmt.Sprintf("&managedBuild=true&title=%s", url.QueryEscape(title))
q.Add("managedBuild", "true")
q.Add("title", title)
}
if branch != "" {
endpoint += fmt.Sprintf("&branch=%s", url.QueryEscape(branch))
q.Add("branch", branch)
}
if revision != "" {
endpoint += fmt.Sprintf("&revision=%s", url.QueryEscape(revision))
q.Add("revision", revision)
}
var ep *url.URL
if ep, err = url.Parse("/api/builds/custom?" + q.Encode()); err != nil {
log.Logger.Fatal("Failed to generate upload uri")
}
log.Logger.Debugf("Sending build data to %#v", endpoint)
log.Logger.Debugf("Sending build data to %#v", ep.String())

res, statusCode, err := Post(endpoint, payload)
res, statusCode, err := Post(ep.String(), payload)
log.Logger.Debugf("Response: %#v", res)

if statusCode == 428 {
Expand Down
3 changes: 3 additions & 0 deletions buildtools/bower/bower.go
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,9 @@ func (b *Bower) List() (Package, error) {
Argv: []string{"list", "--json"},
Dir: b.Config.CWD,
})
if err != nil {
return Package{}, err
}
var pkg Package
err = json.Unmarshal([]byte(stdout), &pkg)
if err != nil {
Expand Down
63 changes: 41 additions & 22 deletions cmd/fossa/cmd/analyze/analyze.go
Original file line number Diff line number Diff line change
@@ -1,8 +1,8 @@
package analyze

import (
"encoding/json"
"fmt"
"text/template"

"github.com/urfave/cli"

Expand All @@ -15,18 +15,12 @@ import (
"github.com/fossas/fossa-cli/module"
)

var (
Output = "output"
)

var Cmd = cli.Command{
Name: "analyze",
Usage: "Analyze built dependencies",
Action: Run,
ArgsUsage: "MODULE",
Flags: flags.WithGlobalFlags(flags.WithAPIFlags(flags.WithModulesFlags([]cli.Flag{
cli.BoolFlag{Name: flags.Short(Output), Usage: "show analysis output instead of uploading to FOSSA"},
}))),
Flags: flags.WithGlobalFlags(flags.WithAPIFlags(flags.WithModulesFlags(flags.WithAnalysisTemplateFlags([]cli.Flag{})))),
}

var _ cli.ActionFunc = Run
Expand All @@ -45,7 +39,40 @@ func Run(ctx *cli.Context) error {
log.Logger.Fatal("No modules specified.")
}

var analyzed []module.Module
analyzed, err := Modules(modules)
if err != nil {
log.Logger.Fatalf("Could not analyze modules: %s", err.Error())
return err
}

normalized, err := fossa.Normalize(analyzed)
if err != nil {
log.Logger.Fatalf("Could not normalize output: %s", err.Error())
return err
}

if ctx.String(flags.ShowOutput) != "" || ctx.String(flags.Template) != "" {
var tmpl *template.Template
if ctx.String(flags.Template) != "" {
tmpl, err = template.ParseFiles(ctx.String(flags.Template))
if err != nil {
log.Logger.Fatalf("Could not parse template data: %s", err.Error())
}

if ctx.String(flags.ShowOutput) == "" {
err = ctx.Set(flags.ShowOutput, "-")
if err != nil {
log.Logger.Fatalf("Could not set default output to STDOUT", err.Error())
}
}
}
return cmdutil.OutputData(ctx.String(flags.ShowOutput), tmpl, normalized)
}

return uploadAnalysis(normalized)
}

func Modules(modules []module.Module) (analyzed []module.Module, err error) {
defer log.StopSpinner()
for i, m := range modules {
log.ShowSpinner(fmt.Sprintf("Analyzing module (%d/%d): %s", i+1, len(modules), m.Name))
Expand All @@ -69,26 +96,18 @@ func Run(ctx *cli.Context) error {
}
log.StopSpinner()

normalized, err := fossa.Normalize(analyzed)
if err != nil {
log.Logger.Fatalf("Could not normalize output: %s", err.Error())
}
if ctx.Bool(Output) {
out, err := json.Marshal(normalized)
if err != nil {
log.Logger.Fatalf("Could not marshal output: %s", err.Error())
}
log.Printf("%s", string(out))
return nil
}
return analyzed, err
}

func uploadAnalysis(normalized []fossa.SourceUnit) error {
fossa.MustInit(config.Endpoint(), config.APIKey())
log.ShowSpinner("Uploading analysis...")
locator, err := fossa.Upload(config.Fetcher(), config.Project(), config.Revision(), config.Title(), config.Branch(), normalized)
log.StopSpinner()
if err != nil {
log.Logger.Fatalf("Error during upload: %s", err.Error())
return err
}
log.StopSpinner()
log.Printf(cmdutil.FmtReportURL(locator))
return nil
}
39 changes: 39 additions & 0 deletions cmd/fossa/cmd/report/dependencies.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
package report

import (
"text/template"

"github.com/fossas/fossa-cli/cmd/fossa/cmdutil"
"github.com/fossas/fossa-cli/cmd/fossa/flags"
"github.com/fossas/fossa-cli/log"
"github.com/fossas/fossa-cli/pkg"
"github.com/urfave/cli"
)

var dependenciesCmd = cli.Command{
Name: "dependencies",
Usage: "Generate dependencies report",
Flags: flags.WithGlobalFlags(flags.WithAPIFlags(flags.WithModulesFlags(flags.WithReportTemplateFlags([]cli.Flag{})))),
Before: prepareReportCtx,
Action: generateDependencies,
}

func generateDependencies(ctx *cli.Context) (err error) {
pkgs := make([]pkg.Package, 0)
for _, module := range analyzed {
for _, pkg := range module.Deps {
pkgs = append(pkgs, pkg)
}
}

var tmpl *template.Template

if ctx.String(flags.Template) != "" {
tmpl, err = template.ParseFiles(ctx.String(flags.Template))
if err != nil {
log.Logger.Fatalf("Could not parse template data: %s", err.Error())
}
}

return cmdutil.OutputData(ctx.String(flags.ShowOutput), tmpl, pkgs)
}

0 comments on commit fcec296

Please sign in to comment.