Skip to content

Commit

Permalink
refactor: define a new struct for scan targets (#5397)
Browse files Browse the repository at this point in the history
  • Loading branch information
@knqyf263
knqyf263 committed Oct 20, 2023
1 parent 6040d9f commit f2a12f5
Show file tree
Hide file tree
Showing 6 changed files with 179 additions and 140 deletions.
2 changes: 1 addition & 1 deletion pkg/fanal/types/artifact.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -264,7 +264,7 @@ type BlobInfo struct {
CustomResources []CustomResource `json:",omitempty"`
}

// ArtifactDetail is generated by applying blobs
// ArtifactDetail represents the analysis result.
type ArtifactDetail struct {
OS OS `json:",omitempty"`
Repository *Repository `json:",omitempty"`
Expand Down
35 changes: 16 additions & 19 deletions pkg/scanner/langpkg/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ var (
)

type Scanner interface {
Packages(detail ftypes.ArtifactDetail, options types.ScanOptions) types.Results
Scan(detail ftypes.ArtifactDetail, options types.ScanOptions) (types.Results, error)
Packages(target types.ScanTarget, options types.ScanOptions) types.Results
Scan(target types.ScanTarget, options types.ScanOptions) (types.Results, error)
}

type scanner struct{}
Expand All @@ -33,20 +33,15 @@ func NewScanner() Scanner {
return &scanner{}
}

func (s *scanner) Packages(detail ftypes.ArtifactDetail, _ types.ScanOptions) types.Results {
func (s *scanner) Packages(target types.ScanTarget, _ types.ScanOptions) types.Results {
var results types.Results
for _, app := range detail.Applications {
for _, app := range target.Applications {
if len(app.Libraries) == 0 {
continue
}
target := app.FilePath
if t, ok := PkgTargets[app.Type]; ok && target == "" {
// When the file path is empty, we will overwrite it with the pre-defined value.
target = t
}

results = append(results, types.Result{
Target: target,
Target: targetName(app.Type, app.FilePath),
Class: types.ClassLangPkg,
Type: app.Type,
Packages: app.Libraries,
Expand All @@ -55,8 +50,8 @@ func (s *scanner) Packages(detail ftypes.ArtifactDetail, _ types.ScanOptions) ty
return results
}

func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types.Results, error) {
apps := detail.Applications
func (s *scanner) Scan(target types.ScanTarget, _ types.ScanOptions) (types.Results, error) {
apps := target.Applications
log.Logger.Infof("Number of language-specific files: %d", len(apps))
if len(apps) == 0 {
return nil, nil
Expand All @@ -83,14 +78,8 @@ func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types
continue
}

target := app.FilePath
if t, ok := PkgTargets[app.Type]; ok && target == "" {
// When the file path is empty, we will overwrite it with the pre-defined value.
target = t
}

results = append(results, types.Result{
Target: target,
Target: targetName(app.Type, app.FilePath),
Vulnerabilities: vulns,
Class: types.ClassLangPkg,
Type: app.Type,
Expand All @@ -101,3 +90,11 @@ func (s *scanner) Scan(detail ftypes.ArtifactDetail, _ types.ScanOptions) (types
})
return results, nil
}

func targetName(appType ftypes.LangType, filePath string) string {
if t, ok := PkgTargets[appType]; ok && filePath == "" {
// When the file path is empty, we will overwrite it with the pre-defined value.
return t
}
return filePath
}
174 changes: 112 additions & 62 deletions pkg/scanner/local/scan.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -57,24 +57,25 @@ func NewScanner(a applier.Applier, osPkgScanner ospkg.Scanner, langPkgScanner la
}

// Scan scans the artifact and return results.
func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys []string, options types.ScanOptions) (types.Results, ftypes.OS, error) {
artifactDetail, err := s.applier.ApplyLayers(artifactKey, blobKeys)
func (s Scanner) Scan(ctx context.Context, targetName, artifactKey string, blobKeys []string, options types.ScanOptions) (
types.Results, ftypes.OS, error) {
detail, err := s.applier.ApplyLayers(artifactKey, blobKeys)
switch {
case errors.Is(err, analyzer.ErrUnknownOS):
log.Logger.Debug("OS is not detected.")

// Packages may contain OS-independent binary information even though OS is not detected.
if len(artifactDetail.Packages) != 0 {
artifactDetail.OS = ftypes.OS{Family: "none"}
if len(detail.Packages) != 0 {
detail.OS = ftypes.OS{Family: "none"}
}

// If OS is not detected and repositories are detected, we'll try to use repositories as OS.
if artifactDetail.Repository != nil {
log.Logger.Debugf("Package repository: %s %s", artifactDetail.Repository.Family, artifactDetail.Repository.Release)
log.Logger.Debugf("Assuming OS is %s %s.", artifactDetail.Repository.Family, artifactDetail.Repository.Release)
artifactDetail.OS = ftypes.OS{
Family: artifactDetail.Repository.Family,
Name: artifactDetail.Repository.Release,
if detail.Repository != nil {
log.Logger.Debugf("Package repository: %s %s", detail.Repository.Family, detail.Repository.Release)
log.Logger.Debugf("Assuming OS is %s %s.", detail.Repository.Family, detail.Repository.Release)
detail.OS = ftypes.OS{
Family: detail.Repository.Family,
Name: detail.Repository.Release,
}
}
case errors.Is(err, analyzer.ErrNoPkgsDetected):
Expand All @@ -84,29 +85,46 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys
return nil, ftypes.OS{}, xerrors.Errorf("failed to apply layers: %w", err)
}

target := types.ScanTarget{
Name: targetName,
OS: detail.OS,
Repository: detail.Repository,
Packages: mergePkgs(detail.Packages, detail.ImageConfig.Packages, options),
Applications: detail.Applications,
Misconfigurations: mergeMisconfigurations(targetName, detail),
Secrets: mergeSecrets(targetName, detail),
Licenses: detail.Licenses,
CustomResources: detail.CustomResources,
}

return s.ScanTarget(ctx, target, options)
}

func (s Scanner) ScanTarget(ctx context.Context, target types.ScanTarget, options types.ScanOptions) (types.Results, ftypes.OS, error) {
var eosl bool
var results, pkgResults types.Results
var err error

// By default, we need to remove dev dependencies from the result
// IncludeDevDeps option allows you not to remove them
excludeDevDeps(artifactDetail.Applications, options.IncludeDevDeps)
excludeDevDeps(target.Applications, options.IncludeDevDeps)

// Fill OS packages and language-specific packages
if options.ListAllPackages {
if res := s.osPkgScanner.Packages(target, artifactDetail, options); len(res.Packages) != 0 {
if res := s.osPkgScanner.Packages(target, options); len(res.Packages) != 0 {
pkgResults = append(pkgResults, res)
}
pkgResults = append(pkgResults, s.langPkgScanner.Packages(artifactDetail, options)...)
pkgResults = append(pkgResults, s.langPkgScanner.Packages(target, options)...)
}

// Scan packages for vulnerabilities
if options.Scanners.Enabled(types.VulnerabilityScanner) {
var vulnResults types.Results
vulnResults, eosl, err = s.scanVulnerabilities(target, artifactDetail, options)
vulnResults, eosl, err = s.scanVulnerabilities(target, options)
if err != nil {
return nil, ftypes.OS{}, xerrors.Errorf("failed to detect vulnerabilities: %w", err)
}
artifactDetail.OS.Eosl = eosl
target.OS.Eosl = eosl

// Merge package results into vulnerability results
mergedResults := s.fillPkgsInVulns(pkgResults, vulnResults)
Expand All @@ -117,45 +135,20 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys
results = append(results, pkgResults...)
}

// Scan IaC config files
if ShouldScanMisconfigOrRbac(options.Scanners) {
configResults := s.MisconfsToResults(artifactDetail.Misconfigurations)
results = append(results, configResults...)
}
// Store misconfigurations
results = append(results, s.misconfsToResults(target.Misconfigurations, options)...)

// Scan secrets
if options.Scanners.Enabled(types.SecretScanner) {
secretResults := s.secretsToResults(artifactDetail.Secrets)
results = append(results, secretResults...)
}
// Store secrets
results = append(results, s.secretsToResults(target.Secrets, options)...)

// Scan licenses
if options.Scanners.Enabled(types.LicenseScanner) {
licenseResults := s.scanLicenses(artifactDetail, options.LicenseCategories)
results = append(results, licenseResults...)
}

// Scan misconfigurations on container image config
if options.ImageConfigScanners.Enabled(types.MisconfigScanner) {
if im := artifactDetail.ImageConfig.Misconfiguration; im != nil {
im.FilePath = target // Set the target name to the file path as container image config is not a real file.
results = append(results, s.MisconfsToResults([]ftypes.Misconfiguration{*im})...)
}
}

// Scan secrets on container image config
if options.ImageConfigScanners.Enabled(types.SecretScanner) {
if is := artifactDetail.ImageConfig.Secret; is != nil {
is.FilePath = target // Set the target name to the file path as container image config is not a real file.
results = append(results, s.secretsToResults([]ftypes.Secret{*is})...)
}
}
results = append(results, s.scanLicenses(target, options)...)

// For WASM plugins and custom analyzers
if len(artifactDetail.CustomResources) != 0 {
if len(target.CustomResources) != 0 {
results = append(results, types.Result{
Class: types.ClassCustom,
CustomResources: artifactDetail.CustomResources,
CustomResources: target.CustomResources,
})
}

Expand All @@ -170,16 +163,16 @@ func (s Scanner) Scan(ctx context.Context, target, artifactKey string, blobKeys
return nil, ftypes.OS{}, xerrors.Errorf("post scan error: %w", err)
}

return results, artifactDetail.OS, nil
return results, target.OS, nil
}

func (s Scanner) scanVulnerabilities(target string, detail ftypes.ArtifactDetail, options types.ScanOptions) (
func (s Scanner) scanVulnerabilities(target types.ScanTarget, options types.ScanOptions) (
types.Results, bool, error) {
var eosl bool
var results types.Results

if slices.Contains(options.VulnType, types.VulnTypeOS) {
vuln, detectedEOSL, err := s.osPkgScanner.Scan(target, detail, options)
vuln, detectedEOSL, err := s.osPkgScanner.Scan(target, options)
if err != nil {
return nil, false, xerrors.Errorf("unable to scan OS packages: %w", err)
} else if vuln.Target != "" {
Expand All @@ -189,7 +182,7 @@ func (s Scanner) scanVulnerabilities(target string, detail ftypes.ArtifactDetail
}

if slices.Contains(options.VulnType, types.VulnTypeLibrary) {
vulns, err := s.langPkgScanner.Scan(detail, options)
vulns, err := s.langPkgScanner.Scan(target, options)
if err != nil {
return nil, false, xerrors.Errorf("failed to scan application libraries: %w", err)
}
Expand Down Expand Up @@ -217,6 +210,14 @@ func (s Scanner) fillPkgsInVulns(pkgResults, vulnResults types.Results) types.Re
return results
}

func (s Scanner) misconfsToResults(misconfs []ftypes.Misconfiguration, options types.ScanOptions) types.Results {
if !ShouldScanMisconfigOrRbac(options.Scanners) {
return nil
}

return s.MisconfsToResults(misconfs)
}

// MisconfsToResults is exported for trivy-plugin-aqua purposes only
func (s Scanner) MisconfsToResults(misconfs []ftypes.Misconfiguration) types.Results {
log.Logger.Infof("Detected config files: %d", len(misconfs))
Expand Down Expand Up @@ -254,7 +255,11 @@ func (s Scanner) MisconfsToResults(misconfs []ftypes.Misconfiguration) types.Res
return results
}

func (s Scanner) secretsToResults(secrets []ftypes.Secret) types.Results {
func (s Scanner) secretsToResults(secrets []ftypes.Secret, options types.ScanOptions) types.Results {
if !options.Scanners.Enabled(types.SecretScanner) {
return nil
}

var results types.Results
for _, secret := range secrets {
log.Logger.Debugf("Secret file: %s", secret.FilePath)
Expand All @@ -268,15 +273,17 @@ func (s Scanner) secretsToResults(secrets []ftypes.Secret) types.Results {
return results
}

func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail,
categories map[ftypes.LicenseCategory][]string) types.Results {
scanner := licensing.NewScanner(categories)
func (s Scanner) scanLicenses(target types.ScanTarget, options types.ScanOptions) types.Results {
if !options.Scanners.Enabled(types.LicenseScanner) {
return nil
}

var results types.Results
scanner := licensing.NewScanner(options.LicenseCategories)

// License - OS packages
var osPkgLicenses []types.DetectedLicense
for _, pkg := range detail.Packages {
for _, pkg := range target.Packages {
for _, license := range pkg.Licenses {
category, severity := scanner.Scan(license)
osPkgLicenses = append(osPkgLicenses, types.DetectedLicense{
Expand All @@ -296,7 +303,7 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail,
})

// License - language-specific packages
for _, app := range detail.Applications {
for _, app := range target.Applications {
var langLicenses []types.DetectedLicense
for _, lib := range app.Libraries {
for _, license := range lib.Licenses {
Expand All @@ -311,21 +318,21 @@ func (s Scanner) scanLicenses(detail ftypes.ArtifactDetail,
}
}

target := app.FilePath
if t, ok := langpkg.PkgTargets[app.Type]; ok && target == "" {
targetName := app.FilePath
if t, ok := langpkg.PkgTargets[app.Type]; ok && targetName == "" {
// When the file path is empty, we will overwrite it with the pre-defined value.
target = t
targetName = t
}
results = append(results, types.Result{
Target: target,
Target: targetName,
Class: types.ClassLicense,
Licenses: langLicenses,
})
}

// License - file header or license file
var fileLicenses []types.DetectedLicense
for _, license := range detail.Licenses {
for _, license := range target.Licenses {
for _, finding := range license.Findings {
category, severity := scanner.Scan(finding.Name)
fileLicenses = append(fileLicenses, types.DetectedLicense{
Expand Down Expand Up @@ -420,3 +427,46 @@ func excludeDevDeps(apps []ftypes.Application, include bool) {
})
}
}

func mergePkgs(pkgs, pkgsFromCommands []ftypes.Package, options types.ScanOptions) []ftypes.Package {
if !options.ScanRemovedPackages || len(pkgsFromCommands) == 0 {
return pkgs
}

// pkg has priority over pkgsFromCommands
uniqPkgs := make(map[string]struct{})
for _, pkg := range pkgs {
uniqPkgs[pkg.Name] = struct{}{}
}
for _, pkg := range pkgsFromCommands {
if _, ok := uniqPkgs[pkg.Name]; ok {
continue
}
pkgs = append(pkgs, pkg)
}
return pkgs
}

// mergeMisconfigurations merges misconfigurations on container image config
func mergeMisconfigurations(targetName string, detail ftypes.ArtifactDetail) []ftypes.Misconfiguration {
if detail.ImageConfig.Misconfiguration == nil {
return detail.Misconfigurations
}

// Append misconfigurations on container image config
misconf := detail.ImageConfig.Misconfiguration
misconf.FilePath = targetName // Set the target name to the file path as container image config is not a real file.
return append(detail.Misconfigurations, *misconf)
}

// mergeSecrets merges secrets on container image config.
func mergeSecrets(targetName string, detail ftypes.ArtifactDetail) []ftypes.Secret {
if detail.ImageConfig.Secret == nil {
return detail.Secrets
}

// Append secrets on container image config
secret := detail.ImageConfig.Secret
secret.FilePath = targetName // Set the target name to the file path as container image config is not a real file.
return append(detail.Secrets, *secret)
}
Loading

0 comments on commit f2a12f5

Please sign in to comment.