feat(report): Implement reports using revisions API instead of depend…

…encies

cmd/fossa/main.go

@@ -141,6 +141,7 @@ func main() {
 				cli.StringFlag{Name: "r, revision", Usage: revisionUsage},
 				cli.StringFlag{Name: "e, endpoint", Usage: endpointUsage},
 				cli.BoolFlag{Name: "debug", Usage: debugUsage},
+				cli.StringFlag{Name: "t, type", Usage: "the type of report to generate (either \"dependencies\" or \"licenses\""},
 			},
 		},
 	}

cmd/fossa/report.go

@@ -5,11 +5,15 @@ import (
 	"fmt"
 	"net/http"
 	"net/url"
+	"strings"
+	"time"
 
+	"github.com/briandowns/spinner"
 	logging "github.com/op/go-logging"
 	"github.com/urfave/cli"
 
 	"github.com/fossas/fossa-cli/config"
+	"github.com/fossas/fossa-cli/module"
 )
 
 var reportLogger = logging.MustGetLogger("report")
@@ -33,37 +37,79 @@ type licenseResponse struct {
 	FullText string
 }
 
-func reportCmd(c *cli.Context) {
-	conf, err := config.New(c)
+func getRevisions(apiURL string, apiKey string, locators []string) ([]dependencyResponse, error) {
+	qs := strings.Join(locators, "&")
+
+	res, err := makeAPIRequest(http.MethodGet, apiURL+"?"+qs, apiKey, nil)
 	if err != nil {
-		reportLogger.Fatalf("Could not load configuration: %s", err.Error())
+		return nil, fmt.Errorf("Could not get licenses from FOSSA API: %s", err.Error())
 	}
+	var deps []dependencyResponse
+	err = json.Unmarshal(res, &deps)
+	if err != nil {
+		return nil, fmt.Errorf("Could not parse API response: %s", err.Error())
+	}
+	return deps, nil
+}
 
-	server, err := url.Parse(conf.Endpoint)
+func reportLicenses(s *spinner.Spinner, endpoint, apiKey string, a analysis) {
+	server, err := url.Parse(endpoint)
 	if err != nil {
 		reportLogger.Fatalf("Invalid FOSSA endpoint: %s", err.Error())
 	}
-	api, err := server.Parse(fmt.Sprintf("/api/revisions/%s/dependencies", url.PathEscape(config.MakeLocator(conf.Project, conf.Revision))))
+	api, err := server.Parse(fmt.Sprintf("/api/revisions"))
 	if err != nil {
 		reportLogger.Fatalf("Invalid API endpoint: %s", err.Error())
 	}
-	params := url.Values{}
-	params.Add("mediated", "true")
-	api.RawQuery = params.Encode()
 
-	res, err := makeAPIRequest(http.MethodGet, api.String(), conf.APIKey, nil)
-	if err != nil {
-		reportLogger.Fatalf("Could not get licenses from FOSSA API: %s", err.Error())
+	s.Suffix = " Loading licenses..."
+	s.Start()
+	total := 0
+	for _, deps := range a {
+		for _ = range deps {
+			total++
+		}
 	}
+	var locators []string
+	var responses []dependencyResponse
+	for _, deps := range a {
+		for _, dep := range deps {
+			if dep.Revision() != "" {
+				locator := string(module.MakeLocator(dep))
+				if dep.Fetcher() == "go" {
+					locator = config.MakeLocator("git", dep.Package(), dep.Revision())
+				}
+				locators = append(locators, fmt.Sprintf("locator[%d]=%s", len(locators), url.QueryEscape(locator)))
 
-	var deps []dependencyResponse
-	err = json.Unmarshal(res, &deps)
-	if err != nil {
-		reportLogger.Fatalf("Could not parse API response: %s", err.Error())
+				// We batch these in groups of 20 for pagination/performance purposes.
+				// TODO: do this in parallel.
+				if len(locators) == 20 {
+					responsePage, err := getRevisions(api.String(), apiKey, locators)
+					if err != nil {
+						s.Stop()
+						reportLogger.Fatalf("Could load licenses: %s", err.Error())
+					}
+					responses = append(responses, responsePage...)
+					locators = []string{}
+					s.Stop()
+					s.Suffix = fmt.Sprintf(" Loading licenses (%d/%d done)...", len(responses), total)
+					s.Restart()
+				}
+			}
+		}
 	}
+	if len(locators) > 0 {
+		responsePage, err := getRevisions(api.String(), apiKey, locators)
+		if err != nil {
+			s.Stop()
+			reportLogger.Fatalf("Could load licenses: %s", err.Error())
+		}
+		responses = append(responses, responsePage...)
+	}
+	s.Stop()
 
 	depsByLicense := make(map[licenseResponse][]dependencyResponse)
-	for _, dep := range deps {
+	for _, dep := range responses {
 		for _, license := range dep.Licenses {
 			depsByLicense[license] = append(depsByLicense[license], dep)
 		}
@@ -85,3 +131,37 @@ The following software have components provided under the terms of this license:
 	}
 	fmt.Println()
 }
+
+func reportCmd(c *cli.Context) {
+	conf, err := config.New(c)
+	if err != nil {
+		reportLogger.Fatalf("Could not load configuration: %s", err.Error())
+	}
+
+	s := spinner.New(spinner.CharSets[11], 100*time.Millisecond)
+
+	s.Suffix = " Analyzing modules..."
+	s.Start()
+	analysis, err := doAnalyze(conf.Modules, conf.AnalyzeCmd.AllowUnresolved)
+	s.Stop()
+	if err != nil {
+		reportLogger.Fatalf("Could not complete analysis (is the project built?): %s", err.Error())
+	}
+
+	switch conf.ReportCmd.Type {
+	case "licenses":
+		reportLicenses(s, conf.Endpoint, conf.APIKey, analysis)
+	case "dependencies":
+		outMap := make(map[string][]module.Dependency)
+		for key, deps := range analysis {
+			outMap[key.module.Name] = deps
+		}
+		out, err := json.Marshal(outMap)
+		if err != nil {
+			reportLogger.Fatalf("Could not marshal analysis: %s", err.Error())
+		}
+		fmt.Println(string(out))
+	default:
+		reportLogger.Fatalf("Report type is not recognized (supported types are \"dependencies\" or \"licenses\": %s", conf.ReportCmd.Type)
+	}
+}

config/config.go

@@ -12,33 +12,38 @@ import (
 
 var configLogger = logging.MustGetLogger("config")
 
-// DefaultConfig specifies the config for the default cmd
+// DefaultConfig specifies the config for the default command
 type DefaultConfig struct {
 	Build bool
 }
 
-// AnalyzeConfig specifies the config for the analyze cmd
+// AnalyzeConfig specifies the config for the analyze command
 type AnalyzeConfig struct {
 	Output          bool
 	AllowUnresolved bool
 }
 
-// BuildConfig specifies the config for the build cmd
+// BuildConfig specifies the config for the build command
 type BuildConfig struct {
 	Force bool
 }
 
-// TestConfig specifies the config for the test cmd
+// TestConfig specifies the config for the test command
 type TestConfig struct {
 	Timeout time.Duration
 }
 
-// UploadConfig specifies the config for the upload cmd
+// UploadConfig specifies the config for the upload command
 type UploadConfig struct {
 	Locators bool
 	Data     string
 }
 
+// ReportConfig specifies the config for the report command
+type ReportConfig struct {
+	Type string // Either "dependencies" or "licenses"
+}
+
 // CLIConfig specifies the config available to the cli
 type CLIConfig struct {
 	APIKey   string
@@ -54,6 +59,7 @@ type CLIConfig struct {
 	BuildCmd   BuildConfig
 	TestCmd    TestConfig
 	UploadCmd  UploadConfig
+	ReportCmd  ReportConfig
 
 	ConfigFilePath string
 }
@@ -142,6 +148,10 @@ func New(c *cli.Context) (CLIConfig, error) {
 			Locators: c.Bool("locators"),
 			Data:     c.String("data"),
 		},
+
+		ReportCmd: ReportConfig{
+			Type: c.String("type"),
+		},
 	}
 
 	// Load configuration file and set overrides.