refactor: misc changes to import

- skip statements that would contain neither commits nor affected artifact
- invoke new backend endpoint
- change order of fields in statements (notes brought towards the top)

Author: copernico

kaybee/VERSION

@@ -1 +1 @@
-0.6.7
+0.6.8

kaybee/internal/model/statement.go

@@ -13,20 +13,20 @@ import (
 	"path/filepath"
 	"reflect"
 	"regexp"
+	"strings"
 
 	"github.com/google/uuid"
 	"gopkg.in/yaml.v2"
-	// "strings"
 )
 
 // Statement represents a vulnerability statement
 type Statement struct {
 	ID                uuid.UUID  `yaml:"-" json:"-"`
 	VulnerabilityID   string     `yaml:"vulnerability_id" json:"vulnerability_id"`
-	Aliases           []Alias    `yaml:"aliases" json:"aliases"`
-	Fixes             []Fix      `yaml:"fixes" json:"-"`
-	AffectedArtifacts []Artifact `yaml:"artifacts" json:"affected_artifacts"`
-	Notes             []Note     `yaml:"notes" json:"notes"`
+	Aliases           []Alias    `yaml:"-" json:"-"`
+	Notes             []Note     `yaml:"notes,omitempty" json:"notes"`
+	Fixes             []Fix      `yaml:"fixes,omitempty" json:"-"`
+	AffectedArtifacts []Artifact `yaml:"artifacts,omitempty" json:"affected_artifacts"`
 	Metadata          Metadata   `yaml:"-" json:"-"`
 }
 
@@ -76,7 +76,7 @@ type Artifact struct {
 // A Note represents a description that accompanies a statement; it can have a
 // set of links and a free-text comment. Neither are mandatory.
 type Note struct {
-	Links []string `json:"links"`
+	Links []string `yaml:"links" json:"links"`
 	Text  string   `json:"text"`
 	hash  string
 }
@@ -178,6 +178,13 @@ func (s *Statement) ToFile(path string) error {
 		os.MkdirAll(targetDir, 0750)
 	}
 
+	// strip slashes from the end of repository URLs
+	for i := range s.Fixes {
+		for j := range s.Fixes[i].Commits {
+			s.Fixes[i].Commits[j].RepositoryURL = strings.TrimRight(s.Fixes[i].Commits[j].RepositoryURL, "/")
+		}
+	}
+
 	dest := filepath.Join(targetDir, "statement.yaml")
 	// fmt.Print("\nSaving statement to file", dest)
 	data, _ := yaml.Marshal(s)

kaybee/internal/tasks/data/default_config.yaml

@@ -1,5 +1,7 @@
 apiVersion: "v1"
 
+# List of sources from which statements are pulled
+# Ranks: smaller value means higher rank (higher priority)
 sources:
   - repo: https://github.com/sap/project-kb
     branch: vulnerability-data

kaybee/internal/tasks/import.go

@@ -94,7 +94,7 @@ func (t *ImportTask) Execute() (success bool) {
 
 	fmt.Printf("Importing vulnerability data from %s (using %d workers)\n", t.backend, t.concurrency)
 
-	importers, err := NewImporterPool(t.backend, t.concurrency, t.limit, nil)
+	importers, err := NewImporterPool(t.backend, t.concurrency, t.limit, nil, t.verbose)
 	if err != nil {
 		log.Fatalln("Could not create importers pool")
 	}
@@ -125,11 +125,12 @@ type Importer struct {
 	Client      *http.Client
 	Filter      map[string][]*regexp.Regexp
 	ProgressBar *progressbar.ProgressBar
+	Verbose     bool
 }
 
 // NewImporterPool instantiates a pool of Exporters, each taking care of fetching vulnerability
 // data for a subset of the overall set of vulnerabilities stored in the Steady backend.
-func NewImporterPool(backend string, concurrent int, limit int, filter map[string][]*regexp.Regexp) (*ImporterPool, error) {
+func NewImporterPool(backend string, concurrent int, limit int, filter map[string][]*regexp.Regexp, verbose bool) (*ImporterPool, error) {
 
 	pool := &ImporterPool{}
 	bugs, err := fetchVulnerabilityIDs(backend)
@@ -171,6 +172,7 @@ func NewImporterPool(backend string, concurrent int, limit int, filter map[strin
 			Filter:      filter,
 			Statements:  make(map[string]model.Statement),
 			ProgressBar: bar,
+			Verbose:     verbose,
 		})
 	}
 
@@ -220,7 +222,8 @@ func (f *Importer) Run() error {
 
 		// Fetch affected artifacts data
 		var affectedLibs []SteadyAffectedLib
-		resp2, err := f.Client.Get(f.Backend + BugsEndpoint + "/" + b.VulnerabilityID + "/affectedLibIds?onlyWellKnown=true")
+		resp2, err := f.Client.Get(f.Backend + BugsEndpoint + "/" + b.VulnerabilityID + "/affectedLibIds?onlyWellKnown=true&resolved=true")
+		// resp2, err := f.Client.Get(f.Backend + BugsEndpoint + "/" + b.VulnerabilityID + "/affectedLibIds?onlyWellKnown=true")
 		if err != nil {
 			return err
 		}
@@ -233,17 +236,32 @@ func (f *Importer) Run() error {
 			return err
 		}
 		f.ProgressBar.Add(1)
+		// fmt.Println("Fetching " + b.VulnerabilityID)
 		s := b.ToStatement()
 		for _, al := range affectedLibs {
-			// if al.Source == "MANUAL" || al.Source == "AST_EQUALITY" {
 			if al.Source == "MANUAL" {
-				s.AffectedArtifacts = append(s.AffectedArtifacts, al.toAffectedArtifact())
+				aa := al.toAffectedArtifact()
+				aa.Reason = "Reviewed manually"
+				s.AffectedArtifacts = append(s.AffectedArtifacts, aa)
+			} else if al.Source == "AST_EQUALITY" {
+				aa := al.toAffectedArtifact()
+				aa.Reason = "Assessed with Eclipse Steady (AST_EQUALITY)"
+				s.AffectedArtifacts = append(s.AffectedArtifacts, aa)
 			}
 		}
 		// fmt.Printf("%+v", affectedLibs)
+
+		// Skip statements that would not contain neither commits nor affected artifacts
+		if len(s.Fixes) == 0 && len(s.AffectedArtifacts) == 0 {
+			if f.Verbose {
+				fmt.Printf("\nStatement for %s would not contain fixes nor affected artifacts, skipping.\n", s.VulnerabilityID)
+			}
+			continue
+		}
 		if !model.Matches(s, f.Filter) {
 			f.Statements[b.VulnerabilityID] = *s
 		}
+
 	}
 	return nil
 }

kaybee/internal/tasks/merge.go

@@ -95,14 +95,15 @@ func (t *MergeTask) Execute() (success bool) {
 		}
 	}
 
+	fmt.Printf("\n")
 	if t.verbose {
-		fmt.Printf("\nReconciling statements...\n")
+		fmt.Printf("Reconciling statements...\n")
 	}
 
 	// TODO adjust terminology: reduce, merge, reconcile....
 	mergedStatements, mergeLog, err := t.policy.Reduce(statementsToMerge)
 	if err != nil {
-		fmt.Printf("\nCould not merge: %v", err)
+		fmt.Printf("Could not merge: %v", err)
 	}
 
 	// fmt.Printf("Merged:\n%v", mergedStatements)
@@ -115,11 +116,8 @@ func (t *MergeTask) Execute() (success bool) {
 		}
 		st[0].ToFile(".kaybee/merged/")
 	}
-	if t.verbose {
-		fmt.Printf("The merge operation on %d statements from %d sources resulted in %d statements.\n", inputStatementCount, len(t.sources), len(mergedStatements))
-	} else {
-		fmt.Printf("Merged %d sources (%d statements): yielded %d statements.\n", len(t.sources), inputStatementCount, len(mergedStatements))
-	}
+
+	fmt.Printf("Merged %d sources (%d statements): yielded %d statements.\n", len(t.sources), inputStatementCount, len(mergedStatements))
 
 	os.MkdirAll(".kaybee/merged/", os.ModePerm)
 	mergeLog.Dump(".kaybee/merged/")