From e7c751194cb91b0c939bf8eed9b8a63c4ccd7e07 Mon Sep 17 00:00:00 2001 From: stonezdj Date: Tue, 30 Apr 2024 15:03:36 +0800 Subject: [PATCH] Add additional_link for sbom fixes #20346 Signed-off-by: stonezdj --- src/controller/artifact/model.go | 13 +++ src/controller/scan/base_controller.go | 56 +++-------- src/pkg/scan/handler.go | 43 ++++++-- src/pkg/scan/job.go | 20 ++-- src/pkg/scan/sbom/sbom.go | 67 +++++++++++-- src/pkg/scan/vulnerability/vul.go | 106 ++++++++++++++++++-- src/server/v2.0/handler/assembler/report.go | 6 ++ 7 files changed, 235 insertions(+), 76 deletions(-) diff --git a/src/controller/artifact/model.go b/src/controller/artifact/model.go index b1bb377954c..a11df2c5509 100644 --- a/src/controller/artifact/model.go +++ b/src/controller/artifact/model.go @@ -80,6 +80,19 @@ func (artifact *Artifact) SetAdditionLink(addition, version string) { artifact.AdditionLinks[addition] = &AdditionLink{HREF: href, Absolute: false} } +func (artifact *Artifact) SetSBOMAdditionLink(sbomDgst string, version string) { + if artifact.AdditionLinks == nil { + artifact.AdditionLinks = make(map[string]*AdditionLink) + } + addition := "sbom" + projectName, repo := utils.ParseRepository(artifact.RepositoryName) + // encode slash as %252F + repo = repository.Encode(repo) + href := fmt.Sprintf("/api/%s/projects/%s/repositories/%s/artifacts/%s/additions/%s", version, projectName, repo, sbomDgst, addition) + + artifact.AdditionLinks[addition] = &AdditionLink{HREF: href, Absolute: false} +} + // AdditionLink is a link via that the addition can be fetched type AdditionLink struct { HREF string `json:"href"` diff --git a/src/controller/scan/base_controller.go b/src/controller/scan/base_controller.go index c70221a0fa5..0f263c2ecd8 100644 --- a/src/controller/scan/base_controller.go +++ b/src/controller/scan/base_controller.go @@ -184,43 +184,12 @@ func NewController() Controller { // There are two scenarios when artifact is scannable: // 1. The scanner has capability for the artifact directly, eg the artifact is docker image. // 2. The artifact is image index and the scanner has capability for any artifact which is referenced by the artifact. -func (bc *basicController) collectScanningArtifacts(ctx context.Context, r *scanner.Registration, artifact *ar.Artifact) ([]*ar.Artifact, bool, error) { - var ( - scannable bool - artifacts []*ar.Artifact - ) - - walkFn := func(a *ar.Artifact) error { - ok, err := bc.isAccessory(ctx, a) - if err != nil { - return err - } - if ok { - return nil - } - - supported := hasCapability(r, a) - - if !supported && a.IsImageIndex() { - // image index not supported by the scanner, so continue to walk its children - return nil - } - - artifacts = append(artifacts, a) - - if supported { - scannable = true - return ar.ErrSkip // this artifact supported by the scanner, skip to walk its children - } - - return nil +func (bc *basicController) collectScanningArtifacts(ctx context.Context, r *scanner.Registration, artifact *ar.Artifact, scanType string) ([]*ar.Artifact, bool, error) { + handler := sca.GetScanHandler(scanType) + if handler == nil || handler.PreCheckerImpl == nil { + return nil, false, fmt.Errorf("failed to get scan handler, type is %v", scanType) } - - if err := bc.ar.Walk(ctx, artifact, walkFn, nil); err != nil { - return nil, false, err - } - - return artifacts, scannable, nil + return handler.PreCheckerImpl.CollectScanningArtifacts(ctx, r, artifact) } // Scan ... @@ -244,16 +213,17 @@ func (bc *basicController) Scan(ctx context.Context, artifact *ar.Artifact, opti return errors.PreconditionFailedError(nil).WithMessage("scanner %s is deactivated", r.Name) } - artifacts, scannable, err := bc.collectScanningArtifacts(ctx, r, artifact) - if err != nil { - return err - } // Parse options opts, err := parseOptions(options...) if err != nil { return errors.Wrap(err, "scan controller: scan") } + artifacts, scannable, err := bc.collectScanningArtifacts(ctx, r, artifact, opts.GetScanType()) + if err != nil { + return err + } + if !scannable { if opts.FromEvent { // skip to return err for event related scan @@ -637,7 +607,7 @@ func (bc *basicController) GetReport(ctx context.Context, artifact *ar.Artifact, return nil, errors.NotFoundError(nil).WithMessage("no scanner registration configured for project: %d", artifact.ProjectID) } - artifacts, scannable, err := bc.collectScanningArtifacts(ctx, r, artifact) + artifacts, scannable, err := bc.collectScanningArtifacts(ctx, r, artifact, v1.ScanTypeVulnerability) if err != nil { return nil, err } @@ -789,7 +759,7 @@ func (bc *basicController) GetScanLog(ctx context.Context, artifact *ar.Artifact return nil, err } - artifacts, _, err := bc.collectScanningArtifacts(ctx, r, artifact) + artifacts, _, err := bc.collectScanningArtifacts(ctx, r, artifact, v1.ScanTypeVulnerability) if err != nil { return nil, err } @@ -1049,7 +1019,7 @@ func (bc *basicController) launchScanJob(ctx context.Context, param *launchScanJ if handler == nil { return fmt.Errorf("failed to get scan handler, type is %v", param.Type) } - robot, err := bc.makeRobotAccount(ctx, param.Artifact.ProjectID, param.Artifact.RepositoryName, param.Registration, handler.RequiredPermissions()) + robot, err := bc.makeRobotAccount(ctx, param.Artifact.ProjectID, param.Artifact.RepositoryName, param.Registration, handler.RobotHandlerImpl.RequiredPermissions()) if err != nil { return errors.Wrap(err, "scan controller: launch scan job") } diff --git a/src/pkg/scan/handler.go b/src/pkg/scan/handler.go index f402107c70f..9643cc564d3 100644 --- a/src/pkg/scan/handler.go +++ b/src/pkg/scan/handler.go @@ -15,8 +15,11 @@ package scan import ( + "context" "time" + ar "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/controller/scanner" "github.com/goharbor/harbor/src/jobservice/job" "github.com/goharbor/harbor/src/pkg/permission/types" "github.com/goharbor/harbor/src/pkg/robot/model" @@ -24,28 +27,50 @@ import ( v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) -var handlerRegistry = map[string]Handler{} +var handlerRegistry = map[string]*ScanHandler{} // RegisterScanHanlder register scanner handler -func RegisterScanHanlder(requestType string, handler Handler) { +func RegisterScanHanlder(requestType string, handler *ScanHandler) { handlerRegistry[requestType] = handler } // GetScanHandler get the handler -func GetScanHandler(requestType string) Handler { +func GetScanHandler(requestType string) *ScanHandler { return handlerRegistry[requestType] } -// Handler handler for scan job, it could be implement by different scan type, such as vulnerability, sbom -type Handler interface { - // RequestProducesMineTypes returns the produces mime types +// ScanHandler hold all interface related to scan +type ScanHandler struct { + RequestHandlerImpl RequestHandler + ResponseHandlerImpl ResponseHandler + RobotHandlerImpl RobotHandler + PreCheckerImpl PreChecker + PostScanListenerImpl PostScanListener +} + +type RequestHandler interface { RequestProducesMineTypes() []string - // RequiredPermissions defines the permission used by the scan robot account - RequiredPermissions() []*types.Policy // RequestParameters defines the parameters for scan request RequestParameters() map[string]interface{} +} + +type ResponseHandler interface { // ReportURLParameter defines the parameters for scan report ReportURLParameter(sr *v1.ScanRequest) (string, error) - // PostScan defines the operation after scan +} + +type RobotHandler interface { + RequiredPermissions() []*types.Policy +} + +type PostScanListener interface { PostScan(ctx job.Context, sr *v1.ScanRequest, rp *scan.Report, rawReport string, startTime time.Time, robot *model.Robot) (string, error) } + +// PreChecker defines the interface to collect scannable artifact +type PreChecker interface { + // CollectScanningArtifacts collect scannable artifact and check current artifact is scanable + CollectScanningArtifacts(ctx context.Context, r *scanner.Registration, artifact *ar.Artifact) ([]*ar.Artifact, bool, error) + // // HasCapability check if the current scanner can scan the artifact + // HasCapability(r *models.Registration, a *ar.Artifact) bool +} diff --git a/src/pkg/scan/job.go b/src/pkg/scan/job.go index 171e0c30742..4f87e2beebf 100644 --- a/src/pkg/scan/job.go +++ b/src/pkg/scan/job.go @@ -242,8 +242,11 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { } myLogger.Debugf("check scan report for mime %s at %s", m, t.Format("2006/01/02 15:04:05")) - - reportURLParameter, err := handler.ReportURLParameter(req) + if handler.ResponseHandlerImpl == nil { + errs[i] = errors.Wrap(err, "failed to get response handler") + return + } + reportURLParameter, err := handler.ResponseHandlerImpl.ReportURLParameter(req) if err != nil { errs[i] = errors.Wrap(err, "scan job: get report url") return @@ -303,8 +306,10 @@ func (j *Job) Run(ctx job.Context, params job.Parameters) error { return err } myLogger.Debugf("Converting report ID %s to the new V2 schema", rp.UUID) - - reportData, err := handler.PostScan(ctx, req, rp, rawReports[i], startTime, robotAccount) + if handler.PostScanListenerImpl == nil { + return fmt.Errorf("failed to get post scanner listener implementation") + } + reportData, err := handler.PostScanListenerImpl.PostScan(ctx, req, rp, rawReports[i], startTime, robotAccount) if err != nil { myLogger.Errorf("Failed to convert vulnerability data to new schema for report %s, error %v", rp.UUID, err) return err @@ -381,11 +386,12 @@ func ExtractScanReq(params job.Parameters) (*v1.ScanRequest, error) { reqType = req.RequestType[0].Type } handler := GetScanHandler(reqType) - if handler == nil { + requestHandler := handler.RequestHandlerImpl + if handler == nil || requestHandler == nil { return nil, errors.Errorf("failed to get scan handler, request type %v", reqType) } - req.RequestType[0].ProducesMimeTypes = handler.RequestProducesMineTypes() - req.RequestType[0].Parameters = handler.RequestParameters() + req.RequestType[0].ProducesMimeTypes = requestHandler.RequestProducesMineTypes() + req.RequestType[0].Parameters = requestHandler.RequestParameters() } return req, nil } diff --git a/src/pkg/scan/sbom/sbom.go b/src/pkg/scan/sbom/sbom.go index f8e6d2e43e8..415e11c9656 100644 --- a/src/pkg/scan/sbom/sbom.go +++ b/src/pkg/scan/sbom/sbom.go @@ -23,6 +23,8 @@ import ( "time" "github.com/goharbor/harbor/src/common" + ar "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/controller/scanner" "github.com/goharbor/harbor/src/lib/config" scanModel "github.com/goharbor/harbor/src/pkg/scan/dao/scan" sbom "github.com/goharbor/harbor/src/pkg/scan/sbom/model" @@ -43,7 +45,13 @@ const ( ) func init() { - scan.RegisterScanHanlder(v1.ScanTypeSbom, &scanHandler{GenAccessoryFunc: scan.GenAccessoryArt, RegistryServer: registryFQDN}) + scan.RegisterScanHanlder(v1.ScanTypeSbom, &scan.ScanHandler{ + PreCheckerImpl: &preChecker{}, + RequestHandlerImpl: &requestHandler{}, + RobotHandlerImpl: &robotHandler{}, + ResponseHandlerImpl: &responseHandler{}, + PostScanListenerImpl: &postScanListener{GenAccessoryFunc: scan.GenAccessoryArt, RegistryServer: registryFQDN}, + }) } // ScanHandler defines the Handler to generate sbom @@ -52,23 +60,32 @@ type scanHandler struct { RegistryServer func(ctx context.Context) string } +type requestHandler struct { +} + // RequestProducesMineTypes defines the mine types produced by the scan handler -func (v *scanHandler) RequestProducesMineTypes() []string { +func (p *requestHandler) RequestProducesMineTypes() []string { return []string{v1.MimeTypeSBOMReport} } // RequestParameters defines the parameters for scan request -func (v *scanHandler) RequestParameters() map[string]interface{} { +func (p *requestHandler) RequestParameters() map[string]interface{} { return map[string]interface{}{"sbom_media_types": []string{sbomMediaTypeSpdx}} } +type responseHandler struct { +} + // ReportURLParameter defines the parameters for scan report url -func (v *scanHandler) ReportURLParameter(_ *v1.ScanRequest) (string, error) { +func (v *responseHandler) ReportURLParameter(_ *v1.ScanRequest) (string, error) { return fmt.Sprintf("sbom_media_type=%s", url.QueryEscape(sbomMediaTypeSpdx)), nil } +type robotHandler struct { +} + // RequiredPermissions defines the permission used by the scan robot account -func (v *scanHandler) RequiredPermissions() []*types.Policy { +func (r *robotHandler) RequiredPermissions() []*types.Policy { return []*types.Policy{ { Resource: rbac.ResourceRepository, @@ -85,8 +102,13 @@ func (v *scanHandler) RequiredPermissions() []*types.Policy { } } +type postScanListener struct { + RegistryServer func(ctx context.Context) string + GenAccessoryFunc func(scanRep v1.ScanRequest, sbomContent []byte, labels map[string]string, mediaType string, robot *model.Robot) (string, error) +} + // PostScan defines task specific operations after the scan is complete -func (v *scanHandler) PostScan(ctx job.Context, sr *v1.ScanRequest, _ *scanModel.Report, rawReport string, startTime time.Time, robot *model.Robot) (string, error) { +func (p *postScanListener) PostScan(ctx job.Context, sr *v1.ScanRequest, _ *scanModel.Report, rawReport string, startTime time.Time, robot *model.Robot) (string, error) { sbomContent, s, err := retrieveSBOMContent(rawReport) if err != nil { return "", err @@ -96,22 +118,22 @@ func (v *scanHandler) PostScan(ctx job.Context, sr *v1.ScanRequest, _ *scanModel Artifact: sr.Artifact, } // the registry server url is core by default, need to replace it with real registry server url - scanReq.Registry.URL = v.RegistryServer(ctx.SystemContext()) + scanReq.Registry.URL = p.RegistryServer(ctx.SystemContext()) if len(scanReq.Registry.URL) == 0 { return "", fmt.Errorf("empty registry server") } myLogger := ctx.GetLogger() myLogger.Debugf("Pushing accessory artifact to %s/%s", scanReq.Registry.URL, scanReq.Artifact.Repository) - dgst, err := v.GenAccessoryFunc(scanReq, sbomContent, v.annotations(), sbomMimeType, robot) + dgst, err := p.GenAccessoryFunc(scanReq, sbomContent, p.annotations(), sbomMimeType, robot) if err != nil { myLogger.Errorf("error when create accessory from image %v", err) return "", err } - return v.generateReport(startTime, sr.Artifact.Repository, dgst, "Success", s) + return p.generateReport(startTime, sr.Artifact.Repository, dgst, "Success", s) } // annotations defines the annotations for the accessory artifact -func (v *scanHandler) annotations() map[string]string { +func (p *postScanListener) annotations() map[string]string { t := time.Now().Format(time.RFC3339) return map[string]string{ "created": t, @@ -121,7 +143,7 @@ func (v *scanHandler) annotations() map[string]string { } } -func (v *scanHandler) generateReport(startTime time.Time, repository, digest, status string, scanner *v1.Scanner) (string, error) { +func (p *postScanListener) generateReport(startTime time.Time, repository, digest, status string, scanner *v1.Scanner) (string, error) { summary := sbom.Summary{} endTime := time.Now() summary[sbom.StartTime] = startTime @@ -163,3 +185,26 @@ func retrieveSBOMContent(rawReport string) ([]byte, *v1.Scanner, error) { } return sbomContent, rpt.Scanner, nil } + +type preChecker struct { +} + +func (p *preChecker) CollectScanningArtifacts(ctx context.Context, r *scanner.Registration, artifact *ar.Artifact) ([]*ar.Artifact, bool, error) { + // only check the current artifact, do not visit the child artifact or accessory + if r.HasCapability(artifact.Type) { + return []*ar.Artifact{artifact}, true, nil + } + return nil, false, nil +} + +// func (p *preChecker) HasCapability(r *models.Registration, a *ar.Artifact) bool { +// // use allowlist here because currently only docker image is supported by the scanner +// // https://github.com/goharbor/pluggable-scanner-spec/issues/2 +// allowlist := []string{image.ArtifactTypeImage} +// for _, t := range allowlist { +// if a.Type == t { +// return r.HasCapability(a.ManifestMediaType) +// } +// } +// return false +// } diff --git a/src/pkg/scan/vulnerability/vul.go b/src/pkg/scan/vulnerability/vul.go index 2e9194c4a07..f9305ec1baa 100644 --- a/src/pkg/scan/vulnerability/vul.go +++ b/src/pkg/scan/vulnerability/vul.go @@ -15,38 +15,57 @@ package vulnerability import ( + "context" "time" "github.com/goharbor/harbor/src/common/rbac" + ar "github.com/goharbor/harbor/src/controller/artifact" + "github.com/goharbor/harbor/src/controller/artifact/processor/image" + "github.com/goharbor/harbor/src/controller/scanner" "github.com/goharbor/harbor/src/jobservice/job" + "github.com/goharbor/harbor/src/lib/q" + "github.com/goharbor/harbor/src/pkg/accessory" "github.com/goharbor/harbor/src/pkg/permission/types" "github.com/goharbor/harbor/src/pkg/robot/model" scanJob "github.com/goharbor/harbor/src/pkg/scan" "github.com/goharbor/harbor/src/pkg/scan/dao/scan" + models "github.com/goharbor/harbor/src/pkg/scan/dao/scanner" "github.com/goharbor/harbor/src/pkg/scan/postprocessors" v1 "github.com/goharbor/harbor/src/pkg/scan/rest/v1" ) func init() { - scanJob.RegisterScanHanlder(v1.ScanTypeVulnerability, &ScanHandler{}) + scanJob.RegisterScanHanlder(v1.ScanTypeVulnerability, &scanJob.ScanHandler{ + RequestHandlerImpl: &requestHandler{}, + RobotHandlerImpl: &robotHandler{}, + ResponseHandlerImpl: &responseHandler{}, + PostScanListenerImpl: &postScanListener{}, + PreCheckerImpl: &preChecker{acc: accessory.Mgr, ar: ar.Ctl}, + }) } // ScanHandler defines the handler for scan vulnerability type ScanHandler struct { } +type requestHandler struct { +} + // RequestProducesMineTypes returns the produces mime types -func (v *ScanHandler) RequestProducesMineTypes() []string { +func (r *requestHandler) RequestProducesMineTypes() []string { return []string{v1.MimeTypeGenericVulnerabilityReport} } // RequestParameters defines the parameters for scan request -func (v *ScanHandler) RequestParameters() map[string]interface{} { +func (r *requestHandler) RequestParameters() map[string]interface{} { return nil } +type robotHandler struct { +} + // RequiredPermissions defines the permission used by the scan robot account -func (v *ScanHandler) RequiredPermissions() []*types.Policy { +func (r *robotHandler) RequiredPermissions() []*types.Policy { return []*types.Policy{ { Resource: rbac.ResourceRepository, @@ -59,14 +78,89 @@ func (v *ScanHandler) RequiredPermissions() []*types.Policy { } } +type responseHandler struct { +} + // ReportURLParameter vulnerability doesn't require any scan report parameters -func (v *ScanHandler) ReportURLParameter(_ *v1.ScanRequest) (string, error) { +func (r *responseHandler) ReportURLParameter(_ *v1.ScanRequest) (string, error) { return "", nil } +type postScanListener struct { +} + // PostScan ... -func (v *ScanHandler) PostScan(ctx job.Context, _ *v1.ScanRequest, origRp *scan.Report, rawReport string, _ time.Time, _ *model.Robot) (string, error) { +func (p *postScanListener) PostScan(ctx job.Context, _ *v1.ScanRequest, origRp *scan.Report, rawReport string, _ time.Time, _ *model.Robot) (string, error) { // use a new ormer here to use the short db connection _, refreshedReport, err := postprocessors.Converter.ToRelationalSchema(ctx.SystemContext(), origRp.UUID, origRp.RegistrationUUID, origRp.Digest, rawReport) return refreshedReport, err } + +type preChecker struct { + // Accessory manager + acc accessory.Manager + // Artifact controller + ar ar.Controller +} + +func (p *preChecker) CollectScanningArtifacts(ctx context.Context, r *scanner.Registration, artifact *ar.Artifact) ([]*ar.Artifact, bool, error) { + var ( + scannable bool + artifacts []*ar.Artifact + ) + + walkFn := func(a *ar.Artifact) error { + ok, err := p.isAccessory(ctx, a) + if err != nil { + return err + } + if ok { + return nil + } + + supported := p.hasCapability(r, a) + + if !supported && a.IsImageIndex() { + // image index not supported by the scanner, so continue to walk its children + return nil + } + + artifacts = append(artifacts, a) + + if supported { + scannable = true + return ar.ErrSkip // this artifact supported by the scanner, skip to walk its children + } + + return nil + } + + if err := p.ar.Walk(ctx, artifact, walkFn, nil); err != nil { + return nil, false, err + } + + return artifacts, scannable, nil +} + +func (p *preChecker) isAccessory(ctx context.Context, art *ar.Artifact) (bool, error) { + ac, err := p.acc.List(ctx, q.New(q.KeyWords{"ArtifactID": art.Artifact.ID, "digest": art.Artifact.Digest})) + if err != nil { + return false, err + } + if len(ac) > 0 { + return true, nil + } + return false, nil +} + +func (p *preChecker) hasCapability(r *models.Registration, a *ar.Artifact) bool { + // use allowlist here because currently only docker image is supported by the scanner + // https://github.com/goharbor/pluggable-scanner-spec/issues/2 + allowlist := []string{image.ArtifactTypeImage} + for _, t := range allowlist { + if a.Type == t { + return r.HasCapability(a.ManifestMediaType) + } + } + return false +} diff --git a/src/server/v2.0/handler/assembler/report.go b/src/server/v2.0/handler/assembler/report.go index ff11e2b8f13..03f3806197e 100644 --- a/src/server/v2.0/handler/assembler/report.go +++ b/src/server/v2.0/handler/assembler/report.go @@ -27,6 +27,7 @@ import ( const ( vulnerabilitiesAddition = "vulnerabilities" + sbomAddition = "sbom" ) // NewScanReportAssembler returns vul assembler @@ -99,6 +100,11 @@ func (assembler *ScanReportAssembler) Assemble(ctx context.Context) error { sbomModel.ReportID: overview[sbomModel.ReportID], sbomModel.Scanner: overview[sbomModel.Scanner], } + + if sbomDgst, ok := overview[sbomModel.SBOMDigest].(string); ok { + // set additional link for sbom digest + artifact.SetSBOMAdditionLink(sbomDgst, version) + } } } }