Store vulnerability report from scanner into a relational format

Convert vulnerability report JSON obtained  from scanner into a relational format describe in:goharbor/community#145

Signed-off-by: prahaladdarkin <prahaladd@vmware.com>

.gitignore

@@ -1,5 +1,6 @@
-!/contrib/helm/harbor
 
+!/contrib/helm/harbor
+*.code-workspace
 make/harbor.yml
 make/docker-compose.yml
 make/common/config/*

make/migrations/postgresql/0050_2.2.0_schema.up.sql

@@ -469,3 +469,44 @@ BEGIN
     update properties set v = cast(duration_in_days as text)  WHERE k = 'robot_token_duration';
   END IF;
 END $$;
+
+/*
+Common vulnerability reporting schema.
+Github proposal link : https://github.com/goharbor/community/pull/145
+*/
+
+-- --------------------------------------------------
+--  Table Structure for `main.VulnerabilityRecord`
+-- --------------------------------------------------
+CREATE TABLE IF NOT EXISTS "vulnerability_record" (
+    "id" serial NOT NULL PRIMARY KEY,
+    "cve_id" text NOT NULL DEFAULT '' ,
+    "registration_uuid" text NOT NULL DEFAULT '',
+    "package" text NOT NULL DEFAULT '' ,
+    "package_version" text NOT NULL DEFAULT '' ,
+    "package_type" text NOT NULL DEFAULT '' ,
+    "severity" text NOT NULL DEFAULT '' ,
+    "fixed_version" text,
+    "urls" text,
+    "cvss_score_v3" double precision,
+    "cvss_score_v2" double precision,
+    "cvss_vector_v3" text,
+    "cvss_vector_v2" text,
+    "description" text,
+    "cwe_ids" text,
+    "vendor_attributes" json,
+    UNIQUE ("cve_id", "registration_uuid", "package", "package_version"),
+    CONSTRAINT fk_registration_uuid FOREIGN  KEY(registration_uuid) REFERENCES scanner_registration(uuid) ON DELETE CASCADE
+);
+
+-- --------------------------------------------------
+--  Table Structure for `main.ReportVulnerabilityRecord`
+-- --------------------------------------------------
+CREATE TABLE IF NOT EXISTS "report_vulnerability_record" (
+    "id" serial NOT NULL PRIMARY KEY,
+    "report_uuid" text NOT NULL DEFAULT '' ,
+    "vuln_record_id" bigint NOT NULL DEFAULT 0 ,
+    UNIQUE ("report_uuid", "vuln_record_id"),
+    CONSTRAINT fk_vuln_record_id FOREIGN  KEY(vuln_record_id) REFERENCES vulnerability_record(id) ON DELETE CASCADE,
+    CONSTRAINT fk_report_uuid FOREIGN  KEY(report_uuid) REFERENCES scan_report(uuid) ON DELETE CASCADE
+);

src/controller/event/handler/webhook/scan/scan.go

@@ -131,7 +131,7 @@ func constructScanImagePayload(event *event.ScanImageEvent, project *models.Proj
 	// If the report is still not ready in the total time, then failed at then
 	for i := 0; i < 10; i++ {
 		// First check in case it is ready
-		if re, err := scan.DefaultController.GetReport(ctx, art, []string{v1.MimeTypeNativeReport}); err == nil {
+		if re, err := scan.DefaultController.GetReport(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}); err == nil {
 			if len(re) > 0 && len(re[0].Report) > 0 {
 				break
 			}
@@ -143,7 +143,7 @@ func constructScanImagePayload(event *event.ScanImageEvent, project *models.Proj
 	}
 
 	// Add scan overview
-	summaries, err := scan.DefaultController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport})
+	summaries, err := scan.DefaultController.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport})
 	if err != nil {
 		return nil, errors.Wrap(err, "construct scan payload")
 	}

src/controller/p2p/preheat/enforcer.go

@@ -477,7 +477,7 @@ func (de *defaultEnforcer) startTask(ctx context.Context, executionID int64, can
 // getVulnerabilitySev gets the severity code value for the given artifact with allowlist option set
 func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Project, art *artifact.Artifact) (uint, error) {
 	al := p.CVEAllowlist.CVESet()
-	r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport}, report.WithCVEAllowlist(&al))
+	r, err := de.scanCtl.GetSummary(ctx, art, []string{v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport}, report.WithCVEAllowlist(&al))
 	if err != nil {
 		if errors.IsNotFoundErr(err) {
 			// no vulnerability report
@@ -487,11 +487,17 @@ func (de *defaultEnforcer) getVulnerabilitySev(ctx context.Context, p *models.Pr
 		return defaultSeverityCode, errors.Wrap(err, "get vulnerability severity")
 	}
 
-	// Severity is based on the native report format.
+	// Severity is based on the native report format or the generic vulnerability report format.
 	// In case no supported report format, treat as same to the no report scenario
 	sum, ok := r[v1.MimeTypeNativeReport]
 	if !ok {
-		return defaultSeverityCode, nil
+		// check if a report with MimeTypeGenericVulnerabilityReport is present.
+		// return the default severity code only if it does not exist
+		sum, ok = r[v1.MimeTypeGenericVulnerabilityReport]
+		if !ok {
+			return defaultSeverityCode, nil
+		}
+
 	}
 
 	sm, ok := sum.(*vuln.NativeReportSummary)

src/controller/scan/base_controller.go

@@ -37,6 +37,7 @@ import (
 	sca "github.com/goharbor/harbor/src/pkg/scan"
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scan"
 	"github.com/goharbor/harbor/src/pkg/scan/dao/scanner"
+	"github.com/goharbor/harbor/src/pkg/scan/postprocessors"
 	"github.com/goharbor/harbor/src/pkg/scan/report"
 	v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1"
 	"github.com/goharbor/harbor/src/pkg/scan/vuln"
@@ -86,6 +87,8 @@ type basicController struct {
 
 	execMgr task.ExecutionManager
 	taskMgr task.Manager
+	// Converter for V1 report to V2 report
+	reportConverter postprocessors.NativeScanReportConverter
 }
 
 // NewController news a scan API controller
@@ -125,6 +128,8 @@ func NewController() Controller {
 
 		execMgr: task.ExecMgr,
 		taskMgr: task.Mgr,
+		// Get the scan V1 to V2 report converters
+		reportConverter: postprocessors.NewNativeToRelationalSchemaConverter(),
 	}
 }
 
@@ -397,8 +402,8 @@ func (bc *basicController) GetReport(ctx context.Context, artifact *ar.Artifact,
 	mimes := make([]string, 0)
 	mimes = append(mimes, mimeTypes...)
 	if len(mimes) == 0 {
-		// Retrieve native as default
-		mimes = append(mimes, v1.MimeTypeNativeReport)
+		// Retrieve native  and the new generic format as default
+		mimes = append(mimes, v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport)
 	}
 
 	// Get current scanner settings
@@ -593,10 +598,19 @@ func (bc *basicController) UpdateReport(ctx context.Context, report *sca.CheckIn
 		return errors.New("no report found to update data")
 	}
 
-	if err := bc.manager.UpdateReportData(ctx, rpl[0].UUID, report.RawReport); err != nil {
+	log.Infof("Converting report ID %s to  the new V2 schema", rpl[0].UUID)
+	_, reportData, err := bc.reportConverter.ToRelationalSchema(ctx, rpl[0].UUID, rpl[0].RegistrationUUID, rpl[0].Digest, report.RawReport)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to convert vulnerability data to new schema for report UUID : %s", rpl[0].UUID)
+	}
+	// update the original report with the new summarized report with all vulnerability data removed.
+	// this is required since the top level layers relay on the vuln.Report struct that
+	// contains additional metadata within the report which if stored in the new columns within the scan_report table
+	// would be redundant
+	if err := bc.manager.UpdateReportData(ctx, rpl[0].UUID, reportData); err != nil {
 		return errors.Wrap(err, "scan controller: handle job hook")
 	}
-
+	log.Infof("Converted report ID %s to the new V2 schema", rpl[0].UUID)
 	return nil
 }
 
@@ -605,7 +619,6 @@ func (bc *basicController) DeleteReports(ctx context.Context, digests ...string)
 	if err := bc.manager.DeleteByDigests(ctx, digests...); err != nil {
 		return errors.Wrap(err, "scan controller: delete reports")
 	}
-
 	return nil
 }
 
@@ -835,6 +848,11 @@ func (bc *basicController) assembleReports(ctx context.Context, reports ...*scan
 		} else {
 			report.Status = job.ErrorStatus.String()
 		}
+		completeReport, err := bc.reportConverter.FromRelationalSchema(ctx, report.UUID, report.Digest, report.Report)
+		if err != nil {
+			return err
+		}
+		report.Report = completeReport
 	}
 
 	return nil

src/controller/scan/base_controller_test.go

@@ -42,6 +42,7 @@ import (
 	scannertesting "github.com/goharbor/harbor/src/testing/controller/scanner"
 	ormtesting "github.com/goharbor/harbor/src/testing/lib/orm"
 	"github.com/goharbor/harbor/src/testing/mock"
+	postprocessorstesting "github.com/goharbor/harbor/src/testing/pkg/scan/postprocessors"
 	reporttesting "github.com/goharbor/harbor/src/testing/pkg/scan/report"
 	tasktesting "github.com/goharbor/harbor/src/testing/pkg/task"
 	"github.com/stretchr/testify/assert"
@@ -60,11 +61,12 @@ type ControllerTestSuite struct {
 	artifact     *artifact.Artifact
 	rawReport    string
 
-	execMgr   *tasktesting.ExecutionManager
-	taskMgr   *tasktesting.Manager
-	reportMgr *reporttesting.Manager
-	ar        artifact.Controller
-	c         Controller
+	execMgr         *tasktesting.ExecutionManager
+	taskMgr         *tasktesting.Manager
+	reportMgr       *reporttesting.Manager
+	ar              artifact.Controller
+	c               Controller
+	reportConverter *postprocessorstesting.ScanReportV1ToV2Converter
 }
 
 // TestController is the entry point of ControllerTestSuite.
@@ -277,8 +279,9 @@ func (suite *ControllerTestSuite) SetupSuite() {
 		cloneCtx: func(ctx context.Context) context.Context { return ctx },
 		makeCtx:  func() context.Context { return context.TODO() },
 
-		execMgr: suite.execMgr,
-		taskMgr: suite.taskMgr,
+		execMgr:         suite.execMgr,
+		taskMgr:         suite.taskMgr,
+		reportConverter: &postprocessorstesting.ScanReportV1ToV2Converter{},
 	}
 }
 

src/controller/scanner/base_controller.go

@@ -296,17 +296,17 @@ func (bc *basicController) Ping(registration *scanner.Registration) (*v1.Scanner
 			return nil, errors.Errorf("missing %s in consumes_mime_types", v1.MimeTypeDockerArtifact)
 		}
 
-		// v1.MimeTypeNativeReport is required
+		// either of v1.MimeTypeNativeReport OR v1.MimeTypeGenericVulnerabilityReport is required
 		found = false
 		for _, pm := range ca.ProducesMimeTypes {
-			if pm == v1.MimeTypeNativeReport {
+			if pm == v1.MimeTypeNativeReport || pm == v1.MimeTypeGenericVulnerabilityReport {
 				found = true
 				break
 			}
 		}
 
 		if !found {
-			return nil, errors.Errorf("missing %s in produces_mime_types", v1.MimeTypeNativeReport)
+			return nil, errors.Errorf("missing %s or %s in produces_mime_types", v1.MimeTypeNativeReport, v1.MimeTypeGenericVulnerabilityReport)
 		}
 	}
 

src/controller/scanner/base_controller_test.go

@@ -260,6 +260,43 @@ func (suite *ControllerTestSuite) TestPing() {
 	suite.NotNil(meta)
 }
 
+// TestPingWithGenericMimeType tests ping for scanners supporting MIME type MimeTypeGenericVulnerabilityReport
+func (suite *ControllerTestSuite) TestPingWithGenericMimeType() {
+	m := &v1.ScannerAdapterMetadata{
+		Scanner: &v1.Scanner{
+			Name:    "Trivy",
+			Vendor:  "Harbor",
+			Version: "0.1.0",
+		},
+		Capabilities: []*v1.ScannerCapability{{
+			ConsumesMimeTypes: []string{
+				v1.MimeTypeOCIArtifact,
+				v1.MimeTypeDockerArtifact,
+			},
+			ProducesMimeTypes: []string{
+				v1.MimeTypeGenericVulnerabilityReport,
+				v1.MimeTypeRawReport,
+			},
+		}},
+		Properties: v1.ScannerProperties{
+			"extra": "testing",
+		},
+	}
+	mc := &v1testing.Client{}
+	mc.On("GetMetadata").Return(m, nil)
+
+	mcp := &v1testing.ClientPool{}
+	mocktesting.OnAnything(mcp, "Get").Return(mc, nil)
+	suite.c = &basicController{
+		manager:    suite.mMgr,
+		proMetaMgr: suite.mMeta,
+		clientPool: mcp,
+	}
+	meta, err := suite.c.Ping(suite.sample)
+	require.NoError(suite.T(), err)
+	suite.NotNil(meta)
+}
+
 // TestGetMetadata ...
 func (suite *ControllerTestSuite) TestGetMetadata() {
 	suite.sample.UUID = "uuid"

src/pkg/notifier/handler/notification/slack_handler.go

@@ -97,7 +97,7 @@ func (s *SlackHandler) process(event *model.HookEvent) error {
 	// Create a slackJob to send message to slack
 	j.Name = job.SlackJob
 
-	// Convert payload to slack format
+	// ToRelationalSchema payload to slack format
 	payload, err := s.convert(event.Payload)
 	if err != nil {
 		return fmt.Errorf("convert payload to slack body failed: %v", err)

src/pkg/p2p/preheat/models/policy/policy.go

@@ -214,7 +214,7 @@ func decodeFilters(filterStr string) ([]*Filter, error) {
 		return nil, err
 	}
 
-	// Convert value type
+	// ToRelationalSchema value type
 	// TODO: remove switch after UI bug #12579 fixed
 	for _, f := range filters {
 		if f.Type == FilterTypeVulnerability {

src/pkg/scan/dao/scan/model.go

@@ -43,3 +43,66 @@ func (r *Report) TableUnique() [][]string {
 		{"digest", "registration_uuid", "mime_type"},
 	}
 }
+
+// VulnerabilityRecord of an individual vulnerability. Identifies an  individual vulnerability item in the scan.
+// Since multiple scanners could be registered with the projects, each scanner
+// would have it's own definition for the same CVE ID. Hence a CVE ID is qualified along
+// with the ID of the scanner that owns the CVE record definition.
+// The scanner ID would be the same as the RegistrationUUID field of Report.
+// Identified by the `cve_id` and `registration_uuid`.
+// Relates to the image using the `digest` and to the report using the `report UUID` field
+type VulnerabilityRecord struct {
+	ID               int64    `orm:"pk;auto;column(id)"`
+	CVEID            string   `orm:"column(cve_id)"`
+	RegistrationUUID string   `orm:"column(registration_uuid)"`
+	Package          string   `orm:"column(package)"`
+	PackageVersion   string   `orm:"column(package_version)"`
+	PackageType      string   `orm:"column(package_type)"`
+	Severity         string   `orm:"column(severity)"`
+	Fix              string   `orm:"column(fixed_version);null"`
+	URLs             string   `orm:"column(urls);null"`
+	CVE3Score        *float64 `orm:"column(cvss_score_v3);null"`
+	CVE2Score        *float64 `orm:"column(cvss_score_v2);null"`
+	CVSS3Vector      string   `orm:"column(cvss_vector_v3);null"` // e.g. CVSS:3.0/AV:L/AC:L/PR:L/UI:N/S:U/C:H/I:N/A:N
+	CVSS2Vector      string   `orm:"column(cvss_vector_v2);null"` // e.g. AV:L/AC:M/Au:N/C:P/I:N/A:N
+	Description      string   `orm:"column(description);null"`
+	CWEIds           string   `orm:"column(cwe_ids);null"` // e.g. CWE-476,CWE-123,CWE-234
+	VendorAttributes string   `orm:"column(vendor_attributes);type(json);null"`
+}
+
+// ReportVulnerabilityRecord is relation table required to optimize data storage for both the
+// vulnerability records and the scan report.
+// identified by composite key (ID, Report)
+// Since each scan report has a separate UUID, the composite key
+// would ensure that the immutability of the historical scan reports is guaranteed.
+// It is sufficient to store the int64 VulnerabilityRecord Id since the vulnerability records
+// are uniquely identified in the table based on the ScannerID and the CVEID
+type ReportVulnerabilityRecord struct {
+	ID           int64  `orm:"pk;auto;column(id)"`
+	Report       string `orm:"column(report_uuid);"`
+	VulnRecordID int64  `orm:"column(vuln_record_id);"`
+}
+
+// TableName for VulnerabilityRecord
+func (vr *VulnerabilityRecord) TableName() string {
+	return "vulnerability_record"
+}
+
+// TableUnique for VulnerabilityRecord
+func (vr *VulnerabilityRecord) TableUnique() [][]string {
+	return [][]string{
+		{"cve_id", "registration_uuid", "package", "package_version", "digest"},
+	}
+}
+
+// TableName for ReportVulnerabilityRecord
+func (rvr *ReportVulnerabilityRecord) TableName() string {
+	return "report_vulnerability_record"
+}
+
+// TableUnique for ReportVulnerabilityRecord
+func (rvr *ReportVulnerabilityRecord) TableUnique() [][]string {
+	return [][]string{
+		{"report_uuid", "vuln_record_id"},
+	}
+}