Skip to content

Commit

Permalink
Improve marshal performance by using pointers within CycloneDX Vulner…
Browse files Browse the repository at this point in the history 
…ability data structures (#65)

* Improve performance by using pointers within CycloneDX Vulnerability structs

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Migrate Vuln. struct members to pointers and update marhsal routines

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

* Introduce const for JSON indent. spacing and set to conventional defaults

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>

---------

Signed-off-by: Matt Rutkowski <mrutkows@us.ibm.com>
  • Loading branch information
@mrutkows
mrutkows committed Nov 14, 2023
1 parent a30779e commit e421ef1
Show file tree
Hide file tree
Showing 13 changed files with 404 additions and 267 deletions.
11 changes: 1 addition & 10 deletions cmd/query_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,7 @@ func innerQuery(t *testing.T, filename string, queryRequest *common.QueryRequest
}

// This will print results ONLY if --quiet mode is `false`
printResult(result)
printMarshaledResultOnlyIfNotQuiet(result)
return
}

Expand Down Expand Up @@ -122,15 +122,6 @@ func VerifySelectedFieldsInJsonMap(t *testing.T, keys []string, results interfac
return
}

func printResult(iResult interface{}) {
if !*TestLogQuiet {
// Format results in JSON
fResult, _ := utils.MarshalAnyToFormattedJsonString(iResult)
// Output the JSON data directly to stdout (not subject to log-level)
fmt.Printf("%s\n", fResult)
}
}

// ----------------------------------------
// Command flag tests
// ----------------------------------------
Expand Down
10 changes: 10 additions & 0 deletions cmd/root_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -283,3 +283,13 @@ func bufferContainsValues(buffer bytes.Buffer, values ...string) bool {
}
return true
}

// TODO: find a better way using some log package feature
func printMarshaledResultOnlyIfNotQuiet(iResult interface{}) {
if !*TestLogQuiet {
// Format results in JSON
fResult, _ := utils.MarshalAnyToFormattedJsonString(iResult)
// Output the JSON data directly to stdout (not subject to log-level)
fmt.Printf("%s\n", fResult)
}
}
16 changes: 12 additions & 4 deletions cmd/trim.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,14 @@ const (
FLAG_TRIM_MAP_KEYS = "keys"
)

// TODO: make flag configurable:
// NOTE: 4-space indent is accepted convention:
// https://docs.openstack.org/doc-contrib-guide/json-conv.html
const (
TRIM_OUTPUT_PREFIX = ""
TRIM_OUTPUT_INDENT = " "
)

// flag help (translate)
const (
FLAG_TRIM_OUTPUT_FORMAT_HELP = "format output using the specified type"
Expand Down Expand Up @@ -207,12 +215,12 @@ func Trim(writer io.Writer, persistentFlags utils.PersistentCommandFlags, trimFl
getLogger().Infof("Outputting listing (`%s` format)...", format)
switch format {
case FORMAT_JSON:
err = document.EncodeAsFormattedJSON(writer, "", " ")
err = document.EncodeAsFormattedJSON(writer, TRIM_OUTPUT_PREFIX, TRIM_OUTPUT_INDENT)
default:
// Default to Text output for anything else (set as flag default)
getLogger().Warningf("Stats not supported for `%s` format; defaulting to `%s` format...",
format, FORMAT_TEXT)
err = document.EncodeAsFormattedJSON(writer, "", " ")
getLogger().Warningf("Trim not supported for `%s` format; defaulting to `%s` format...",
format, FORMAT_JSON)
err = document.EncodeAsFormattedJSON(writer, TRIM_OUTPUT_PREFIX, TRIM_OUTPUT_INDENT)
}

return
Expand Down
22 changes: 22 additions & 0 deletions cmd/trim_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ const (
TEST_TRIM_CDX_1_4_ENCODED_CHARS = "test/trim/trim-cdx-1-4-sample-encoded-chars.sbom.json"
TEST_TRIM_CDX_1_4_SAMPLE_XXL_1 = "test/trim/trim-cdx-1-4-sample-xxl-1.sbom.json"
TEST_TRIM_CDX_1_5_SAMPLE_SMALL_COMPS_ONLY = "test/trim/trim-cdx-1-5-sample-small-components-only.sbom.json"
TEST_TRIM_CDX_1_4_SAMPLE_VEX = "test/trim/trim-cdx-1-4-sample-vex.json"
TEST_TRIM_CDX_1_5_SAMPLE_MEDIUM_1 = "test/trim/trim-cdx-1-5-sample-medium-1.sbom.json"
)

Expand Down Expand Up @@ -352,3 +353,24 @@ func TestTrimCdx15FooFromTools(t *testing.T) {
t.Error(fmt.Errorf("invalid trim result: string not found: %s", TEST_STRING_1))
}
}

func TestTrimCdx14SourceFromVulnerabilities(t *testing.T) {
ti := NewTrimTestInfoBasic(TEST_TRIM_CDX_1_4_SAMPLE_VEX, nil)
ti.Keys = append(ti.Keys, "source")
ti.FromPaths = []string{"vulnerabilities"}
ti.TestOutputVariantName = utils.GetCallerFunctionName(2)
ti.OutputFile = ti.CreateTemporaryFilename(TEST_TRIM_CDX_1_4_SAMPLE_VEX)

buffer, _, err := innerTestTrim(t, ti)
s := buffer.String()
if err != nil {
getLogger().Debugf("result: %s", s)
t.Error(err)
}

// Assure JSON map does not contain the trimmed key(s)
err = VerifyTrimOutputFileResult(t, ti, ti.Keys, ti.FromPaths[0])
if err != nil {
t.Error(err)
}
}
145 changes: 1 addition & 144 deletions cmd/vulnerability.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ import (
"encoding/csv"
"fmt"
"io"
"reflect"
"sort"
"strings"
"text/tabwriter"
Expand Down Expand Up @@ -260,156 +259,14 @@ func loadDocumentVulnerabilities(document *schema.BOM, whereFilters []common.Whe
// Hash all components found in the (root).components[] (+ "nested" components)
pVulnerabilities := document.GetCdxVulnerabilities()
if pVulnerabilities != nil && len(*pVulnerabilities) > 0 {
if err = hashVulnerabilities(document, *pVulnerabilities, whereFilters); err != nil {
if err = document.HashVulnerabilities(*pVulnerabilities, whereFilters); err != nil {
return
}
}

return
}

// We need to hash our own informational structure around the CDX data in order
// to simplify --where queries to command line users
func hashVulnerabilities(bom *schema.BOM, vulnerabilities []schema.CDXVulnerability, whereFilters []common.WhereFilter) (err error) {
getLogger().Enter()
defer getLogger().Exit(err)

for _, cdxVulnerability := range vulnerabilities {
_, err = hashVulnerability(bom, cdxVulnerability, whereFilters)
if err != nil {
return
}
}
return
}

// Hash a CDX Component and recursively those of any "nested" components
// TODO we should WARN if version is not a valid semver (e.g., examples/cyclonedx/BOM/laravel-7.12.0/bom.1.3.json)
func hashVulnerability(bom *schema.BOM, cdxVulnerability schema.CDXVulnerability, whereFilters []common.WhereFilter) (vi *schema.VulnerabilityInfo, err error) {
getLogger().Enter()
defer getLogger().Exit(err)
var vulnInfo schema.VulnerabilityInfo
vi = &vulnInfo

if reflect.DeepEqual(cdxVulnerability, schema.CDXVulnerability{}) {
err = getLogger().Errorf("invalid vulnerability info: missing or empty : %v ", cdxVulnerability)
return
}

if cdxVulnerability.Id == "" {
getLogger().Warningf("vulnerability missing required value `id` : %v ", cdxVulnerability)
}

if cdxVulnerability.Published == "" {
getLogger().Warningf("vulnerability (`%s`) missing `published` date", cdxVulnerability.Id)
}

if cdxVulnerability.Created == "" {
getLogger().Warningf("vulnerability (`%s`) missing `created` date", cdxVulnerability.Id)
}

if len(cdxVulnerability.Ratings) == 0 {
getLogger().Warningf("vulnerability (`%s`) missing `ratings`", cdxVulnerability.Id)
}

// hash any component w/o a license using special key name
vulnInfo.Vulnerability = cdxVulnerability
if cdxVulnerability.BOMRef != nil {
vulnInfo.BOMRef = cdxVulnerability.BOMRef.String()
}
vulnInfo.Id = cdxVulnerability.Id

// Truncate dates from 2023-02-02T00:00:00.000Z to 2023-02-02
// Note: if validation errors are found by the "truncate" function,
// it will emit an error and return the original (failing) value
dateTime, _ := utils.TruncateTimeStampISO8601Date(cdxVulnerability.Created)
vulnInfo.Created = dateTime

dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Published)
vulnInfo.Published = dateTime

dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Updated)
vulnInfo.Updated = dateTime

dateTime, _ = utils.TruncateTimeStampISO8601Date(cdxVulnerability.Rejected)
vulnInfo.Rejected = dateTime

vulnInfo.Description = cdxVulnerability.Description

// Source object: retrieve report fields from nested objects
if cdxVulnerability.Source != nil {
source := *cdxVulnerability.Source
vulnInfo.Source = source
vulnInfo.SourceName = source.Name
vulnInfo.SourceUrl = source.Url
}

// TODO: replace empty Analysis values with "UNDEFINED"
vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
if vulnInfo.AnalysisState == "" {
vulnInfo.AnalysisState = schema.VULN_ANALYSIS_STATE_EMPTY
}

vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
if vulnInfo.AnalysisJustification == "" {
vulnInfo.AnalysisJustification = schema.VULN_ANALYSIS_STATE_EMPTY
}
vulnInfo.AnalysisResponse = cdxVulnerability.Analysis.Response
if len(vulnInfo.AnalysisResponse) == 0 {
vulnInfo.AnalysisResponse = []string{schema.VULN_ANALYSIS_STATE_EMPTY}
}

// Convert []int to []string for --where filter
// TODO see if we can eliminate this conversion and handle while preparing report data
// as this SHOULD appear there as []interface{}
if len(cdxVulnerability.Cwes) > 0 {
vulnInfo.CweIds = strings.Fields(strings.Trim(fmt.Sprint(cdxVulnerability.Cwes), "[]"))
}

// CVSS Score Qualitative Rating
// 0.0 None
// 0.1 – 3.9 Low
// 4.0 – 6.9 Medium
// 7.0 – 8.9 High
// 9.0 – 10.0 Critical

// TODO: if summary report, see if more than one severity can be shown without clogging up column data
numRatings := len(cdxVulnerability.Ratings)
if numRatings > 0 {
//var sourceMatch int
for _, rating := range cdxVulnerability.Ratings {
// defer to same source as the top-level vuln. declares
fSeverity := fmt.Sprintf("%s: %v (%s)", rating.Method, rating.Score, rating.Severity)
// give listing priority to ratings that matches top-level vuln. reporting source
if rating.Source.Name == cdxVulnerability.Source.Name {
// prepend to slice
vulnInfo.CvssSeverity = append([]string{fSeverity}, vulnInfo.CvssSeverity...)
continue
}
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, fSeverity)
}

} else {
// Set first entry to empty value (i.e., "none")
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, schema.VULN_RATING_EMPTY)
}

var match bool = true
if len(whereFilters) > 0 {
mapVulnInfo, _ := utils.MarshalStructToJsonMap(vulnInfo)
match, _ = whereFilterMatch(mapVulnInfo, whereFilters)
}

if match {
bom.VulnerabilityMap.Put(vulnInfo.Id, vulnInfo)

getLogger().Tracef("Put: %s (`%s`), `%s`)",
vulnInfo.Id, vulnInfo.Description, vulnInfo.BOMRef)
}

return
}

// NOTE: This list is NOT de-duplicated
// TODO: Add a --no-title flag to skip title output
func DisplayVulnListText(bom *schema.BOM, output io.Writer, flags utils.VulnerabilityCommandFlags) {
Expand Down
4 changes: 2 additions & 2 deletions cmd/vulnerability_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -232,7 +232,6 @@ func TestVulnListCdx13JSON(t *testing.T) {
testInfo.ResultExpectedLineCount = 185
result, _, _ := innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
getLogger().Debugf("result:\n%s", result.String())
//fmt.Printf("result:\n%s", result.String())
}

// -------------------------------------------
Expand All @@ -254,7 +253,8 @@ func TestVulnListTextCdx14WhereClauseAndResultsByIdStartsWith(t *testing.T) {
nil)
testInfo.ResultLineContainsValues = TEST_OUTPUT_CONTAINS
testInfo.ResultLineContainsValuesAtLineNum = 2
innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
result, _, _ := innerTestVulnList(t, testInfo, VULN_TEST_DEFAULT_FLAGS)
getLogger().Debugf("result:\n%s", result.String())
}

func TestVulnListTextCdx14WhereClauseDescContains(t *testing.T) {
Expand Down
36 changes: 21 additions & 15 deletions schema/bom_hash.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -346,7 +346,7 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
getLogger().Warningf("vulnerability (`%s`) missing `created` date", cdxVulnerability.Id)
}

if len(cdxVulnerability.Ratings) == 0 {
if cdxVulnerability.Ratings == nil || len(*cdxVulnerability.Ratings) == 0 {
getLogger().Warningf("vulnerability (`%s`) missing `ratings`", cdxVulnerability.Id)
}

Expand Down Expand Up @@ -383,24 +383,32 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
}

// TODO: replace empty Analysis values with "UNDEFINED"
vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
if vulnInfo.AnalysisState == "" {
vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
}
if cdxVulnerability.Analysis != nil {
vulnInfo.AnalysisState = cdxVulnerability.Analysis.State
if vulnInfo.AnalysisState == "" {
vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
}

vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
if vulnInfo.AnalysisJustification == "" {
vulnInfo.AnalysisJustification = VULN_ANALYSIS_STATE_EMPTY
}

vulnInfo.AnalysisJustification = cdxVulnerability.Analysis.Justification
if vulnInfo.AnalysisJustification == "" {
vulnInfo.AnalysisResponse = *cdxVulnerability.Analysis.Response
if len(vulnInfo.AnalysisResponse) == 0 {
vulnInfo.AnalysisResponse = []string{VULN_ANALYSIS_STATE_EMPTY}
}
} else {
vulnInfo.AnalysisState = VULN_ANALYSIS_STATE_EMPTY
vulnInfo.AnalysisJustification = VULN_ANALYSIS_STATE_EMPTY
}
vulnInfo.AnalysisResponse = cdxVulnerability.Analysis.Response
if len(vulnInfo.AnalysisResponse) == 0 {
vulnInfo.AnalysisResponse = []string{VULN_ANALYSIS_STATE_EMPTY}
}

// Convert []int to []string for --where filter
// TODO see if we can eliminate this conversion and handle while preparing report data
// as this SHOULD appear there as []interface{}
if len(cdxVulnerability.Cwes) > 0 {
if cdxVulnerability.Cwes != nil && len(*cdxVulnerability.Cwes) > 0 {
// strip off slice/array brackets
vulnInfo.CweIds = strings.Fields(strings.Trim(fmt.Sprint(cdxVulnerability.Cwes), "[]"))
}

Expand All @@ -412,10 +420,9 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
// 9.0 – 10.0 Critical

// TODO: if summary report, see if more than one severity can be shown without clogging up column data
numRatings := len(cdxVulnerability.Ratings)
if numRatings > 0 {
if cdxVulnerability.Ratings != nil && len(*cdxVulnerability.Ratings) > 0 {
//var sourceMatch int
for _, rating := range cdxVulnerability.Ratings {
for _, rating := range *cdxVulnerability.Ratings {
// defer to same source as the top-level vuln. declares
fSeverity := fmt.Sprintf("%s: %v (%s)", rating.Method, rating.Score, rating.Severity)
// give listing priority to ratings that matches top-level vuln. reporting source
Expand All @@ -426,7 +433,6 @@ func (bom *BOM) HashVulnerability(cdxVulnerability CDXVulnerability, whereFilter
}
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, fSeverity)
}

} else {
// Set first entry to empty value (i.e., "none")
vulnInfo.CvssSeverity = append(vulnInfo.CvssSeverity, VULN_RATING_EMPTY)
Expand Down
12 changes: 6 additions & 6 deletions schema/cyclonedx.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,8 +59,8 @@ type CDXBom struct {
// v1.3: added "licenses", "properties"
// v1.5: added "lifecycles"
type CDXMetadata struct {
Timestamp string `json:"timestamp,omitempty"`
Tools interface{} `json:"tools,omitempty"` // v1.2: added.v1.5: "tools" is now an interface{}
Timestamp string `json:"timestamp,omitempty" scvs:"bom:core:timestamp"` // urn:owasp:scvs:bom:core:timestamp
Tools interface{} `json:"tools,omitempty"` // v1.2: added.v1.5: "tools" is now an interface{}
Authors *[]CDXOrganizationalContact `json:"authors,omitempty"`
Component *CDXComponent `json:"component,omitempty"`
Manufacturer *CDXOrganizationalEntity `json:"manufacturer,omitempty"`
Expand Down Expand Up @@ -96,10 +96,10 @@ type CDXComponent struct {
Hashes *[]CDXHash `json:"hashes,omitempty"`
Licenses *[]CDXLicenseChoice `json:"licenses,omitempty"`
Copyright string `json:"copyright,omitempty"`
Cpe string `json:"cpe,omitempty"` // See: https://nvd.nist.gov/products/cpe
Purl string `json:"purl,omitempty"` // See: https://github.com/package-url/purl-spec
Swid *CDXSwid `json:"swid,omitempty"` // See: https://www.iso.org/standard/65666.html
Pedigree *CDXPedigree `json:"pedigree,omitempty"` // anon. type
Cpe string `json:"cpe,omitempty"` // See: https://nvd.nist.gov/products/cpe
Purl string `json:"purl,omitempty" scvs:"bom:resource:identifiers:purl"` // See: https://github.com/package-url/purl-spec
Swid *CDXSwid `json:"swid,omitempty"` // See: https://www.iso.org/standard/65666.html
Pedigree *CDXPedigree `json:"pedigree,omitempty"` // anon. type
ExternalReferences *[]CDXExternalReference `json:"externalReferences,omitempty"`
Components *[]CDXComponent `json:"components,omitempty"`
Evidence *CDXComponentEvidence `json:"evidence,omitempty"` // v1.3: added
Expand Down
Loading

0 comments on commit e421ef1

Please sign in to comment.