From f14bed453246a880854ff2ee8cdd1a483e4ebc4c Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Thu, 30 Mar 2023 05:53:24 +0300 Subject: [PATCH] feat: add auth support for downloading OCI artifacts (#3915) --- docs/docs/vulnerability/db.md | 38 +++++++ docs/docs/vulnerability/examples/db.md | 4 +- mkdocs.yml | 1 + pkg/commands/app.go | 2 +- pkg/commands/artifact/run.go | 15 +-- pkg/commands/operation/operation.go | 9 +- pkg/commands/server/run.go | 7 +- pkg/db/db.go | 31 +++--- pkg/db/db_test.go | 12 ++- pkg/db/mock_operation.go | 6 +- pkg/fanal/artifact/artifact.go | 2 +- pkg/fanal/artifact/image/remote_sbom.go | 7 +- pkg/flag/options.go | 12 +++ pkg/javadb/client.go | 6 +- pkg/module/command.go | 7 +- pkg/oci/artifact.go | 130 ++++++++++++++---------- pkg/oci/artifact_test.go | 87 ++++------------ pkg/policy/policy.go | 17 ++-- pkg/policy/policy_test.go | 12 +-- pkg/remote/remote.go | 52 ++++++++-- pkg/rpc/server/listen.go | 33 +++--- pkg/rpc/server/listen_test.go | 5 +- 22 files changed, 280 insertions(+), 215 deletions(-) create mode 100644 docs/docs/vulnerability/db.md diff --git a/docs/docs/vulnerability/db.md b/docs/docs/vulnerability/db.md new file mode 100644 index 00000000000..eb7b39baea5 --- /dev/null +++ b/docs/docs/vulnerability/db.md @@ -0,0 +1,38 @@ +# Database +Trivy uses two types of databases for vulnerability detection: + +- Vulnerability Database +- Java Index Database + +This page provides detailed information about these databases. + +## Vulnerability Database +Trivy utilizes a database containing vulnerability information. +This database is built every six hours on [GitHub](https://github.com/aquasecurity/trivy-db) and is distributed via [GitHub Container registry (GHCR)](https://ghcr.io/aquasecurity/trivy-db). +The database is cached and updated as needed. +As Trivy updates the database automatically during execution, users don't need to be concerned about it. + +For CLI flags related to the database, please refer to [this page](./examples/db.md). + +### Private Hosting +If you host the database on your own OCI registry, you can specify a different repository with the `--db-repository` flag. +The default is `ghcr.io/aquasecurity/trivy-db`. + +```shell +$ trivy image --db-repository YOUR_REPO YOUR_IMAGE +``` + +If authentication is required, it can be configured in the same way as for private images. +Please refer to [the documentation](../advanced/private-registries/index.md) for more details. + +## Java Index Database +This database is only downloaded when scanning JAR files so that Trivy can identify the groupId, artifactId, and version of JAR files. +It is built once a day on [GitHub](https://github.com/aquasecurity/trivy-java-db) and distributed via [GitHub Container registry (GHCR)](https://ghcr.io/aquasecurity/trivy-java-db). +Like the vulnerability database, it is automatically downloaded and updated when needed, so users don't need to worry about it. + +### Private Hosting +If you host the database on your own OCI registry, you can specify a different repository with the `--java-db-repository` flag. +The default is `ghcr.io/aquasecurity/trivy-java-db`. + +If authentication is required, you need to run `docker login YOUR_REGISTRY`. +Currently, specifying a username and password is not supported. \ No newline at end of file diff --git a/docs/docs/vulnerability/examples/db.md b/docs/docs/vulnerability/examples/db.md index f2619041deb..ae9820a755e 100644 --- a/docs/docs/vulnerability/examples/db.md +++ b/docs/docs/vulnerability/examples/db.md @@ -1,9 +1,7 @@ # Vulnerability DB ## Skip update of vulnerability DB -`Trivy` downloads its vulnerability database every 12 hours when it starts operating. -This is usually fast, as the size of the DB is only 10~30MB. -But if you want to skip even that, use the `--skip-db-update` option. +If you want to skip downloading the vulnerability database, use the `--skip-db-update` option. ``` $ trivy image --skip-db-update python:3.4-alpine3.9 diff --git a/mkdocs.yml b/mkdocs.yml index bf4ebfd2c83..1e4c363f9e1 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -52,6 +52,7 @@ nav: - OS Packages: docs/vulnerability/detection/os.md - Language-specific Packages: docs/vulnerability/detection/language.md - Data Sources: docs/vulnerability/detection/data-source.md + - Database: docs/vulnerability/db.md - Examples: - Vulnerability Filtering: docs/vulnerability/examples/filter.md - Report Formats: docs/vulnerability/examples/report.md diff --git a/pkg/commands/app.go b/pkg/commands/app.go index f978deb05c4..93789060e38 100644 --- a/pkg/commands/app.go +++ b/pkg/commands/app.go @@ -759,7 +759,7 @@ func NewModuleCommand(globalFlags *flag.GlobalFlagGroup) *cobra.Command { if err != nil { return xerrors.Errorf("flag error: %w", err) } - return module.Install(cmd.Context(), opts.ModuleDir, repo, opts.Quiet, opts.Insecure) + return module.Install(cmd.Context(), opts.ModuleDir, repo, opts.Quiet, opts.Remote()) }, }, &cobra.Command{ diff --git a/pkg/commands/artifact/run.go b/pkg/commands/artifact/run.go index c685a97b461..df2e8ca41f5 100644 --- a/pkg/commands/artifact/run.go +++ b/pkg/commands/artifact/run.go @@ -19,7 +19,6 @@ import ( "github.com/aquasecurity/trivy/pkg/fanal/analyzer/config" "github.com/aquasecurity/trivy/pkg/fanal/artifact" "github.com/aquasecurity/trivy/pkg/fanal/cache" - ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/flag" "github.com/aquasecurity/trivy/pkg/javadb" "github.com/aquasecurity/trivy/pkg/log" @@ -315,7 +314,7 @@ func (r *runner) initDB(opts flag.Options) error { // download the database file noProgress := opts.Quiet || opts.NoProgress - if err := operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.Insecure, opts.SkipDBUpdate); err != nil { + if err := operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, noProgress, opts.SkipDBUpdate, opts.Remote()); err != nil { return err } @@ -610,13 +609,7 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi } } - remoteOption := ftypes.RemoteOptions{ - Credentials: opts.Credentials, - RegistryToken: opts.RegistryToken, - Insecure: opts.Insecure, - Platform: opts.Platform, - AWSRegion: opts.AWSOptions.Region, - } + remoteOpts := opts.Remote() return ScannerConfig{ Target: target, @@ -643,8 +636,8 @@ func initScannerConfig(opts flag.Options, cacheClient cache.Cache) (ScannerConfi Slow: opts.Slow, AWSRegion: opts.Region, - // For container registries - RemoteOptions: remoteOption, + // For OCI registries + RemoteOptions: remoteOpts, // For misconfiguration scanning MisconfScannerOption: configScannerOptions, diff --git a/pkg/commands/operation/operation.go b/pkg/commands/operation/operation.go index 022e296ad5b..20e41b9e204 100644 --- a/pkg/commands/operation/operation.go +++ b/pkg/commands/operation/operation.go @@ -15,6 +15,7 @@ import ( "github.com/aquasecurity/trivy-db/pkg/metadata" "github.com/aquasecurity/trivy/pkg/db" "github.com/aquasecurity/trivy/pkg/fanal/cache" + "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/flag" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/policy" @@ -101,8 +102,8 @@ func (c Cache) ClearArtifacts() error { } // DownloadDB downloads the DB -func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, insecure, skipUpdate bool) error { - client := db.NewClient(cacheDir, quiet, insecure, db.WithDBRepository(dbRepository)) +func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, skipUpdate bool, opt types.RemoteOptions) error { + client := db.NewClient(cacheDir, quiet, db.WithDBRepository(dbRepository)) ctx := context.Background() needsUpdate, err := client.NeedsUpdate(appVersion, skipUpdate) if err != nil { @@ -113,7 +114,7 @@ func DownloadDB(appVersion, cacheDir, dbRepository string, quiet, insecure, skip log.Logger.Info("Need to update DB") log.Logger.Infof("DB Repository: %s", dbRepository) log.Logger.Info("Downloading DB...") - if err = client.Download(ctx, cacheDir); err != nil { + if err = client.Download(ctx, cacheDir, opt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } } @@ -145,7 +146,7 @@ func InitBuiltinPolicies(ctx context.Context, cacheDir string, quiet, skipUpdate needsUpdate := false if !skipUpdate { - needsUpdate, err = client.NeedsUpdate() + needsUpdate, err = client.NeedsUpdate(ctx) if err != nil { return nil, xerrors.Errorf("unable to check if built-in policies need to be updated: %w", err) } diff --git a/pkg/commands/server/run.go b/pkg/commands/server/run.go index 881be8902c8..16e92b41770 100644 --- a/pkg/commands/server/run.go +++ b/pkg/commands/server/run.go @@ -35,7 +35,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) { // download the database file if err = operation.DownloadDB(opts.AppVersion, opts.CacheDir, opts.DBRepository, - true, opts.Insecure, opts.SkipDBUpdate); err != nil { + true, opts.SkipDBUpdate, opts.Remote()); err != nil { return err } @@ -57,6 +57,7 @@ func Run(ctx context.Context, opts flag.Options) (err error) { } m.Register() - server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader, opts.DBRepository) - return server.ListenAndServe(cache, opts.Insecure, opts.SkipDBUpdate) + server := rpcServer.NewServer(opts.AppVersion, opts.Listen, opts.CacheDir, opts.Token, opts.TokenHeader, + opts.DBRepository, opts.Remote()) + return server.ListenAndServe(cache, opts.SkipDBUpdate) } diff --git a/pkg/db/db.go b/pkg/db/db.go index 4bf9c394925..bf96f49a9cd 100644 --- a/pkg/db/db.go +++ b/pkg/db/db.go @@ -12,6 +12,7 @@ import ( "github.com/aquasecurity/trivy-db/pkg/db" "github.com/aquasecurity/trivy-db/pkg/metadata" + "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/oci" ) @@ -24,7 +25,7 @@ const ( // Operation defines the DB operations type Operation interface { NeedsUpdate(cliVersion string, skip bool) (need bool, err error) - Download(ctx context.Context, dst string) (err error) + Download(ctx context.Context, dst string, opt types.RemoteOptions) (err error) } type options struct { @@ -61,14 +62,13 @@ func WithClock(clock clock.Clock) Option { type Client struct { *options - cacheDir string - metadata metadata.Client - quiet bool - insecureSkipTLSVerify bool + cacheDir string + metadata metadata.Client + quiet bool } // NewClient is the factory method for DB client -func NewClient(cacheDir string, quiet, insecure bool, opts ...Option) *Client { +func NewClient(cacheDir string, quiet bool, opts ...Option) *Client { o := &options{ clock: clock.RealClock{}, dbRepository: defaultDBRepository, @@ -79,11 +79,10 @@ func NewClient(cacheDir string, quiet, insecure bool, opts ...Option) *Client { } return &Client{ - options: o, - cacheDir: cacheDir, - metadata: metadata.NewClient(cacheDir), - quiet: quiet, - insecureSkipTLSVerify: insecure, // insecure skip for download DB + options: o, + cacheDir: cacheDir, + metadata: metadata.NewClient(cacheDir), + quiet: quiet, } } @@ -144,18 +143,18 @@ func (c *Client) isNewDB(meta metadata.Metadata) bool { } // Download downloads the DB file -func (c *Client) Download(ctx context.Context, dst string) error { +func (c *Client) Download(ctx context.Context, dst string, opt types.RemoteOptions) error { // Remove the metadata file under the cache directory before downloading DB if err := c.metadata.Delete(); err != nil { log.Logger.Debug("no metadata file") } - art, err := c.initOCIArtifact() + art, err := c.initOCIArtifact(opt) if err != nil { return xerrors.Errorf("OCI artifact error: %w", err) } - if err = art.Download(ctx, db.Dir(dst)); err != nil { + if err = art.Download(ctx, db.Dir(dst), oci.DownloadOption{MediaType: dbMediaType}); err != nil { return xerrors.Errorf("database download error: %w", err) } @@ -184,13 +183,13 @@ func (c *Client) updateDownloadedAt(dst string) error { return nil } -func (c *Client) initOCIArtifact() (*oci.Artifact, error) { +func (c *Client) initOCIArtifact(opt types.RemoteOptions) (*oci.Artifact, error) { if c.artifact != nil { return c.artifact, nil } repo := fmt.Sprintf("%s:%d", c.dbRepository, db.SchemaVersion) - art, err := oci.NewArtifact(repo, dbMediaType, "", c.quiet, c.insecureSkipTLSVerify) + art, err := oci.NewArtifact(repo, c.quiet, opt) if err != nil { var terr *transport.Error if errors.As(err, &terr) { diff --git a/pkg/db/db_test.go b/pkg/db/db_test.go index ee3c09d51e8..873eba00d72 100644 --- a/pkg/db/db_test.go +++ b/pkg/db/db_test.go @@ -18,6 +18,7 @@ import ( tdb "github.com/aquasecurity/trivy-db/pkg/db" "github.com/aquasecurity/trivy-db/pkg/metadata" "github.com/aquasecurity/trivy/pkg/db" + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/oci" ) @@ -152,7 +153,7 @@ func TestClient_NeedsUpdate(t *testing.T) { require.NoError(t, err) } - client := db.NewClient(cacheDir, true, false, db.WithClock(tt.clock)) + client := db.NewClient(cacheDir, true, db.WithClock(tt.clock)) needsUpdate, err := client.NeedsUpdate("test", tt.skip) switch { @@ -218,11 +219,14 @@ func TestClient_Download(t *testing.T) { }, nil) // Mock OCI artifact - art, err := oci.NewArtifact("db", mediaType, "", true, false, oci.WithImage(img)) + opt := ftypes.RemoteOptions{ + Insecure: false, + } + art, err := oci.NewArtifact("db", true, opt, oci.WithImage(img)) require.NoError(t, err) - client := db.NewClient(cacheDir, true, false, db.WithOCIArtifact(art), db.WithClock(timeDownloadedAt)) - err = client.Download(context.Background(), cacheDir) + client := db.NewClient(cacheDir, true, db.WithOCIArtifact(art), db.WithClock(timeDownloadedAt)) + err = client.Download(context.Background(), cacheDir, opt) if tt.wantErr != "" { require.Error(t, err) assert.Contains(t, err.Error(), tt.wantErr) diff --git a/pkg/db/mock_operation.go b/pkg/db/mock_operation.go index 5b8f828b942..be87a776cb6 100644 --- a/pkg/db/mock_operation.go +++ b/pkg/db/mock_operation.go @@ -6,6 +6,8 @@ import ( context "context" mock "github.com/stretchr/testify/mock" + + "github.com/aquasecurity/trivy/pkg/fanal/types" ) // MockOperation is an autogenerated mock type for the Operation type @@ -51,8 +53,8 @@ func (_m *MockOperation) ApplyDownloadExpectations(expectations []OperationDownl } // Download provides a mock function with given fields: ctx, dst -func (_m *MockOperation) Download(ctx context.Context, dst string) error { - ret := _m.Called(ctx, dst) +func (_m *MockOperation) Download(ctx context.Context, dst string, opt types.RemoteOptions) error { + ret := _m.Called(ctx, dst, opt) var r0 error if rf, ok := ret.Get(0).(func(context.Context, string) error); ok { diff --git a/pkg/fanal/artifact/artifact.go b/pkg/fanal/artifact/artifact.go index 3d37424ad36..3b211edefec 100644 --- a/pkg/fanal/artifact/artifact.go +++ b/pkg/fanal/artifact/artifact.go @@ -28,7 +28,7 @@ type Option struct { Slow bool // Lower CPU and memory AWSRegion string - // For container registries + // For OCI registries types.RemoteOptions MisconfScannerOption misconf.ScannerOption diff --git a/pkg/fanal/artifact/image/remote_sbom.go b/pkg/fanal/artifact/image/remote_sbom.go index 285cc73a77c..5a69bcf93a5 100644 --- a/pkg/fanal/artifact/image/remote_sbom.go +++ b/pkg/fanal/artifact/image/remote_sbom.go @@ -81,7 +81,7 @@ func (a Artifact) inspectOCIReferrerSBOM(ctx context.Context) (ftypes.ArtifactRe func (a Artifact) parseReferrer(ctx context.Context, repo string, desc v1.Descriptor) (ftypes.ArtifactReference, error) { const fileName string = "referrer.sbom" repoName := fmt.Sprintf("%s@%s", repo, desc.Digest) - referrer, err := oci.NewArtifact(repoName, desc.ArtifactType, fileName, true, a.artifactOption.Insecure) + referrer, err := oci.NewArtifact(repoName, true, a.artifactOption.RemoteOptions) if err != nil { return ftypes.ArtifactReference{}, xerrors.Errorf("OCI error: %w", err) } @@ -93,7 +93,10 @@ func (a Artifact) parseReferrer(ctx context.Context, repo string, desc v1.Descri defer os.RemoveAll(tmpDir) // Download SBOM to local filesystem - if err = referrer.Download(ctx, tmpDir); err != nil { + if err = referrer.Download(ctx, tmpDir, oci.DownloadOption{ + MediaType: desc.ArtifactType, + Filename: fileName, + }); err != nil { return ftypes.ArtifactReference{}, xerrors.Errorf("SBOM download error: %w", err) } diff --git a/pkg/flag/options.go b/pkg/flag/options.go index 26aa878db37..2818a8d2ab5 100644 --- a/pkg/flag/options.go +++ b/pkg/flag/options.go @@ -15,6 +15,7 @@ import ( "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/fanal/analyzer" + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/report" ) @@ -121,6 +122,17 @@ func (o *Options) Align() { } } +// Remote returns options for OCI registries +func (o *Options) Remote() ftypes.RemoteOptions { + return ftypes.RemoteOptions{ + Credentials: o.Credentials, + RegistryToken: o.RegistryToken, + Insecure: o.Insecure, + Platform: o.Platform, + AWSRegion: o.AWSOptions.Region, + } +} + func addFlag(cmd *cobra.Command, flag *Flag) { if flag == nil || flag.Name == "" { return diff --git a/pkg/javadb/client.go b/pkg/javadb/client.go index 5a757833a26..9fe3f59ffed 100644 --- a/pkg/javadb/client.go +++ b/pkg/javadb/client.go @@ -14,6 +14,7 @@ import ( "github.com/aquasecurity/go-dep-parser/pkg/java/jar" "github.com/aquasecurity/trivy-java-db/pkg/db" "github.com/aquasecurity/trivy-java-db/pkg/types" + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/oci" ) @@ -51,11 +52,12 @@ func (u *Updater) Update() error { log.Logger.Infof("Java DB Repository: %s", u.repo) log.Logger.Info("Downloading the Java DB...") + // TODO: support remote options var a *oci.Artifact - if a, err = oci.NewArtifact(u.repo, mediaType, "", u.quiet, u.insecure); err != nil { + if a, err = oci.NewArtifact(u.repo, u.quiet, ftypes.RemoteOptions{}); err != nil { return xerrors.Errorf("oci error: %w", err) } - if err = a.Download(context.Background(), dbDir); err != nil { + if err = a.Download(context.Background(), dbDir, oci.DownloadOption{MediaType: mediaType}); err != nil { return xerrors.Errorf("DB download error: %w", err) } diff --git a/pkg/module/command.go b/pkg/module/command.go index f446b357bdf..f8f9bfe62df 100644 --- a/pkg/module/command.go +++ b/pkg/module/command.go @@ -8,6 +8,7 @@ import ( "github.com/google/go-containerregistry/pkg/name" "golang.org/x/xerrors" + "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/oci" ) @@ -15,14 +16,14 @@ import ( const mediaType = "application/vnd.module.wasm.content.layer.v1+wasm" // Install installs a module -func Install(ctx context.Context, dir, repo string, quiet, insecure bool) error { +func Install(ctx context.Context, dir, repo string, quiet bool, opt types.RemoteOptions) error { ref, err := name.ParseReference(repo) if err != nil { return xerrors.Errorf("repository parse error: %w", err) } log.Logger.Infof("Installing the module from %s...", repo) - artifact, err := oci.NewArtifact(repo, mediaType, "", quiet, insecure) + artifact, err := oci.NewArtifact(repo, quiet, opt) if err != nil { return xerrors.Errorf("module initialize error: %w", err) } @@ -30,7 +31,7 @@ func Install(ctx context.Context, dir, repo string, quiet, insecure bool) error dst := filepath.Join(dir, ref.Context().Name()) log.Logger.Debugf("Installing the module to %s...", dst) - if err = artifact.Download(ctx, dst); err != nil { + if err = artifact.Download(ctx, dst, oci.DownloadOption{MediaType: mediaType}); err != nil { return xerrors.Errorf("module download error: %w", err) } diff --git a/pkg/oci/artifact.go b/pkg/oci/artifact.go index 31baf48b01f..6fbd3dd8af4 100644 --- a/pkg/oci/artifact.go +++ b/pkg/oci/artifact.go @@ -2,20 +2,19 @@ package oci import ( "context" - "crypto/tls" "io" - "net/http" "os" "path/filepath" + "sync" "github.com/cheggaaa/pb/v3" - "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" "golang.org/x/xerrors" "github.com/aquasecurity/trivy/pkg/downloader" + "github.com/aquasecurity/trivy/pkg/fanal/types" + "github.com/aquasecurity/trivy/pkg/remote" ) const ( @@ -35,79 +34,95 @@ var SupportedSBOMArtifactTypes = []string{ SPDXArtifactType, } -type options struct { - img v1.Image -} - // Option is a functional option -type Option func(*options) +type Option func(*Artifact) // WithImage takes an OCI v1 Image func WithImage(img v1.Image) Option { - return func(opts *options) { - opts.img = img + return func(a *Artifact) { + a.image = img } } // Artifact is used to download artifacts such as vulnerability database and policies from OCI registries. type Artifact struct { - fileName string - image v1.Image - layer v1.Layer // Take the first layer as OCI artifact - quiet bool + m sync.Mutex + repository string + quiet bool + + // For OCI registries + types.RemoteOptions + + image v1.Image // For testing } // NewArtifact returns a new artifact -func NewArtifact(repo, mediaType, fileName string, quiet, insecure bool, opts ...Option) (*Artifact, error) { - o := &options{} +func NewArtifact(repo string, quiet bool, remoteOpt types.RemoteOptions, opts ...Option) (*Artifact, error) { + art := &Artifact{ + repository: repo, + quiet: quiet, + RemoteOptions: remoteOpt, + } - for _, opt := range opts { - opt(o) + for _, o := range opts { + o(art) } + return art, nil +} - if o.img == nil { - ref, err := name.ParseReference(repo) - if err != nil { - return nil, xerrors.Errorf("repository name error (%s): %w", repo, err) - } +func (a *Artifact) populate(ctx context.Context, opt types.RemoteOptions) error { + if a.image != nil { + return nil + } - remoteOpts := []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)} - if insecure { - t := &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: true}, - Proxy: http.ProxyFromEnvironment, - } - remoteOpts = append(remoteOpts, remote.WithTransport(t)) - } + a.m.Lock() + defer a.m.Unlock() - o.img, err = remote.Image(ref, remoteOpts...) - if err != nil { - return nil, xerrors.Errorf("OCI repository error: %w", err) - } + ref, err := name.ParseReference(a.repository) + if err != nil { + return xerrors.Errorf("repository name error (%s): %w", a.repository, err) } - layers, err := o.img.Layers() + a.image, err = remote.Image(ctx, ref, opt) if err != nil { - return nil, xerrors.Errorf("OCI layer error: %w", err) + return xerrors.Errorf("OCI repository error: %w", err) } + return nil +} - manifest, err := o.img.Manifest() +type DownloadOption struct { + MediaType string // Accept any media type if not specified + Filename string // Use the annotation if not specified +} + +func (a *Artifact) Download(ctx context.Context, dir string, opt DownloadOption) error { + if err := a.populate(ctx, a.RemoteOptions); err != nil { + return err + } + + layers, err := a.image.Layers() + if err != nil { + return xerrors.Errorf("OCI layer error: %w", err) + } + + manifest, err := a.image.Manifest() if err != nil { - return nil, xerrors.Errorf("OCI manifest error: %w", err) + return xerrors.Errorf("OCI manifest error: %w", err) } // A single layer is only supported now. if len(layers) != 1 || len(manifest.Layers) != 1 { - return nil, xerrors.Errorf("OCI artifact must be a single layer") + return xerrors.Errorf("OCI artifact must be a single layer") } // Take the first layer layer := layers[0] // Take the file name of the first layer if not specified + fileName := opt.Filename if fileName == "" { if v, ok := manifest.Layers[0].Annotations[titleAnnotation]; !ok { - return nil, xerrors.Errorf("annotation %s is missing", titleAnnotation) + return xerrors.Errorf("annotation %s is missing", titleAnnotation) } else { fileName = v } @@ -115,26 +130,25 @@ func NewArtifact(repo, mediaType, fileName string, quiet, insecure bool, opts .. layerMediaType, err := layer.MediaType() if err != nil { - return nil, xerrors.Errorf("media type error: %w", err) - } else if mediaType != string(layerMediaType) { - return nil, xerrors.Errorf("unacceptable media type: %s", string(layerMediaType)) + return xerrors.Errorf("media type error: %w", err) + } else if opt.MediaType != "" && opt.MediaType != string(layerMediaType) { + return xerrors.Errorf("unacceptable media type: %s", string(layerMediaType)) + } + + if err = a.download(ctx, layer, fileName, dir); err != nil { + return xerrors.Errorf("oci download error: %w", err) } - return &Artifact{ - fileName: fileName, - image: o.img, - layer: layer, - quiet: quiet, - }, nil + return nil } -func (a Artifact) Download(ctx context.Context, dir string) error { - size, err := a.layer.Size() +func (a *Artifact) download(ctx context.Context, layer v1.Layer, fileName, dir string) error { + size, err := layer.Size() if err != nil { return xerrors.Errorf("size error: %w", err) } - rc, err := a.layer.Compressed() + rc, err := layer.Compressed() if err != nil { return xerrors.Errorf("failed to fetch the layer: %w", err) } @@ -154,7 +168,7 @@ func (a Artifact) Download(ctx context.Context, dir string) error { return xerrors.Errorf("failed to create a temp dir: %w", err) } - f, err := os.Create(filepath.Join(tempDir, a.fileName)) + f, err := os.Create(filepath.Join(tempDir, fileName)) if err != nil { return xerrors.Errorf("failed to create a temp file: %w", err) } @@ -176,7 +190,11 @@ func (a Artifact) Download(ctx context.Context, dir string) error { return nil } -func (a Artifact) Digest() (string, error) { +func (a *Artifact) Digest(ctx context.Context) (string, error) { + if err := a.populate(ctx, a.RemoteOptions); err != nil { + return "", err + } + digest, err := a.image.Digest() if err != nil { return "", xerrors.Errorf("digest error: %w", err) diff --git a/pkg/oci/artifact_test.go b/pkg/oci/artifact_test.go index b1d5bc08eea..df1f9fa7b99 100644 --- a/pkg/oci/artifact_test.go +++ b/pkg/oci/artifact_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/oci" "github.com/aquasecurity/trivy/pkg/utils/fsutils" ) @@ -26,10 +27,13 @@ func (f fakeLayer) MediaType() (types.MediaType, error) { return "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip", nil } -func TestNewArtifact(t *testing.T) { +func TestArtifact_Download(t *testing.T) { layer, err := tarball.LayerFromFile("testdata/test.tar.gz") require.NoError(t, err) + txtLayer, err := tarball.LayerFromFile("testdata/test.txt") + require.NoError(t, err) + flayer := fakeLayer{layer} type layersReturns struct { @@ -38,16 +42,20 @@ func TestNewArtifact(t *testing.T) { } tests := []struct { name string + input string mediaType string layersReturns layersReturns + want string wantErr string }{ { name: "happy path", + input: "testdata/test.tar.gz", mediaType: "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip", layersReturns: layersReturns{ layers: []v1.Layer{flayer}, }, + want: "Hello, world", }, { name: "sad: two layers", @@ -68,6 +76,14 @@ func TestNewArtifact(t *testing.T) { }, wantErr: "OCI layer error", }, + { + name: "invalid gzip", + input: "testdata/test.txt", + layersReturns: layersReturns{ + layers: []v1.Layer{txtLayer}, + }, + wantErr: "unexpected EOF", + }, { name: "sad: media type doesn't match", mediaType: "unknown", @@ -102,73 +118,14 @@ func TestNewArtifact(t *testing.T) { }, }, nil) - _, err = oci.NewArtifact("repo", tt.mediaType, "", true, false, oci.WithImage(img)) - if tt.wantErr != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) - return - } - require.NoError(t, err) - }) - } -} - -func TestArtifact_Download(t *testing.T) { - tests := []struct { - name string - input string - want string - wantErr string - }{ - { - name: "happy path", - input: "testdata/test.tar.gz", - want: "Hello, world", - }, - { - name: "invalid gzip", - input: "testdata/test.txt", - wantErr: "unexpected EOF", - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - tempDir := t.TempDir() - fsutils.SetCacheDir(tempDir) - - // Mock layer - layer, err := tarball.LayerFromFile(tt.input) - require.NoError(t, err) - flayer := fakeLayer{layer} - - // Mock image - img := new(fakei.FakeImage) - img.LayersReturns([]v1.Layer{flayer}, nil) - img.ManifestReturns(&v1.Manifest{ - Layers: []v1.Descriptor{ - { - MediaType: "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip", - Size: 100, - Digest: v1.Hash{ - Algorithm: "sha256", - Hex: "cba33656188782852f58993f45b68bfb8577f64cdcf02a604e3fc2afbeb5f2d8", - }, - Annotations: map[string]string{ - "org.opencontainers.image.title": "bundle.tar.gz", - }, - }, - }, - }, nil) - - mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip" - artifact, err := oci.NewArtifact("repo", mediaType, "", true, false, oci.WithImage(img)) + artifact, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img)) require.NoError(t, err) - err = artifact.Download(context.Background(), tempDir) + err = artifact.Download(context.Background(), tempDir, oci.DownloadOption{ + MediaType: tt.mediaType, + }) if tt.wantErr != "" { - require.Error(t, err) - assert.Contains(t, err.Error(), tt.wantErr) + assert.ErrorContains(t, err, tt.wantErr) return } require.NoError(t, err) diff --git a/pkg/policy/policy.go b/pkg/policy/policy.go index 89707a454ea..711c95cff3b 100644 --- a/pkg/policy/policy.go +++ b/pkg/policy/policy.go @@ -9,12 +9,12 @@ import ( "time" "github.com/open-policy-agent/opa/bundle" + "golang.org/x/xerrors" + "k8s.io/utils/clock" + "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/oci" - - "golang.org/x/xerrors" - "k8s.io/utils/clock" ) const ( @@ -51,7 +51,6 @@ type Client struct { *options policyDir string quiet bool - insecure bool } // Metadata holds default policy metadata @@ -80,7 +79,7 @@ func NewClient(cacheDir string, quiet bool, opts ...Option) (*Client, error) { func (c *Client) populateOCIArtifact() error { if c.artifact == nil { repo := fmt.Sprintf("%s:%d", bundleRepository, bundleVersion) - art, err := oci.NewArtifact(repo, policyMediaType, "", c.quiet, c.insecure) + art, err := oci.NewArtifact(repo, c.quiet, types.RemoteOptions{}) if err != nil { return xerrors.Errorf("OCI artifact error: %w", err) } @@ -96,11 +95,11 @@ func (c *Client) DownloadBuiltinPolicies(ctx context.Context) error { } dst := c.contentDir() - if err := c.artifact.Download(ctx, dst); err != nil { + if err := c.artifact.Download(ctx, dst, oci.DownloadOption{MediaType: policyMediaType}); err != nil { return xerrors.Errorf("download error: %w", err) } - digest, err := c.artifact.Digest() + digest, err := c.artifact.Digest(ctx) if err != nil { return xerrors.Errorf("digest error: %w", err) } @@ -142,7 +141,7 @@ func (c *Client) LoadBuiltinPolicies() ([]string, error) { } // NeedsUpdate returns if the default policy should be updated -func (c *Client) NeedsUpdate() (bool, error) { +func (c *Client) NeedsUpdate(ctx context.Context) (bool, error) { meta, err := c.GetMetadata() if err != nil { return true, nil @@ -157,7 +156,7 @@ func (c *Client) NeedsUpdate() (bool, error) { return false, xerrors.Errorf("OPA bundle error: %w", err) } - digest, err := c.artifact.Digest() + digest, err := c.artifact.Digest(ctx) if err != nil { return false, xerrors.Errorf("digest error: %w", err) } diff --git a/pkg/policy/policy_test.go b/pkg/policy/policy_test.go index afd9d79f3b3..3a55557aa95 100644 --- a/pkg/policy/policy_test.go +++ b/pkg/policy/policy_test.go @@ -19,6 +19,7 @@ import ( "k8s.io/utils/clock" fake "k8s.io/utils/clock/testing" + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/oci" "github.com/aquasecurity/trivy/pkg/policy" ) @@ -115,8 +116,7 @@ func TestClient_LoadBuiltinPolicies(t *testing.T) { }, nil) // Mock OCI artifact - mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip" - art, err := oci.NewArtifact("repo", mediaType, "", true, true, oci.WithImage(img)) + art, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img)) require.NoError(t, err) c, err := policy.NewClient(tt.cacheDir, true, policy.WithOCIArtifact(art)) @@ -257,15 +257,14 @@ func TestClient_NeedsUpdate(t *testing.T) { require.NoError(t, err) } - mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip" - art, err := oci.NewArtifact("repo", mediaType, "", true, true, oci.WithImage(img)) + art, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img)) require.NoError(t, err) c, err := policy.NewClient(tmpDir, true, policy.WithOCIArtifact(art), policy.WithClock(tt.clock)) require.NoError(t, err) // Assert results - got, err := c.NeedsUpdate() + got, err := c.NeedsUpdate(context.Background()) assert.Equal(t, tt.wantErr, err != nil) assert.Equal(t, tt.want, got) }) @@ -362,8 +361,7 @@ func TestClient_DownloadBuiltinPolicies(t *testing.T) { }, nil) // Mock OCI artifact - mediaType := "application/vnd.cncf.openpolicyagent.layer.v1.tar+gzip" - art, err := oci.NewArtifact("repo", mediaType, "", true, true, oci.WithImage(img)) + art, err := oci.NewArtifact("repo", true, ftypes.RemoteOptions{}, oci.WithImage(img)) require.NoError(t, err) c, err := policy.NewClient(tempDir, true, policy.WithClock(tt.clock), policy.WithOCIArtifact(art)) diff --git a/pkg/remote/remote.go b/pkg/remote/remote.go index 068d888ae1d..f7ea9932772 100644 --- a/pkg/remote/remote.go +++ b/pkg/remote/remote.go @@ -27,12 +27,15 @@ type Descriptor = remote.Descriptor // Get is a wrapper of google/go-containerregistry/pkg/v1/remote.Get // so that it can try multiple authentication methods. func Get(ctx context.Context, ref name.Reference, option types.RemoteOptions) (*Descriptor, error) { - remoteOpts := []remote.Option{remote.WithTransport(transport(option.Insecure))} + transport := httpTransport(option.Insecure) var errs error + // Try each authentication method until it succeeds for _, authOpt := range authOptions(ctx, ref, option) { - // Try each authentication method until it succeeds - remoteOpts = append(remoteOpts, authOpt) + remoteOpts := []remote.Option{ + remote.WithTransport(transport), + authOpt, + } if option.Platform != "" { s, err := parsePlatform(ref, option.Platform, remoteOpts) @@ -58,13 +61,42 @@ func Get(ctx context.Context, ref name.Reference, option types.RemoteOptions) (* return nil, errs } +// Image is a wrapper of google/go-containerregistry/pkg/v1/remote.Image +// so that it can try multiple authentication methods. +func Image(ctx context.Context, ref name.Reference, option types.RemoteOptions) (v1.Image, error) { + transport := httpTransport(option.Insecure) + + var errs error + // Try each authentication method until it succeeds + for _, authOpt := range authOptions(ctx, ref, option) { + remoteOpts := []remote.Option{ + remote.WithTransport(transport), + authOpt, + } + index, err := remote.Image(ref, remoteOpts...) + if err != nil { + errs = multierror.Append(errs, err) + continue + } + return index, nil + } + + // No authentication succeeded + return nil, errs +} + +// Referrers is a wrapper of google/go-containerregistry/pkg/v1/remote.Referrers +// so that it can try multiple authentication methods. func Referrers(ctx context.Context, d name.Digest, option types.RemoteOptions) (*v1.IndexManifest, error) { - remoteOpts := []remote.Option{remote.WithTransport(transport(option.Insecure))} + transport := httpTransport(option.Insecure) var errs error + // Try each authentication method until it succeeds for _, authOpt := range authOptions(ctx, d, option) { - // Try each authentication method until it succeeds - remoteOpts = append(remoteOpts, authOpt) + remoteOpts := []remote.Option{ + remote.WithTransport(transport), + authOpt, + } index, err := remote.Referrers(d, remoteOpts...) if err != nil { errs = multierror.Append(errs, err) @@ -77,7 +109,7 @@ func Referrers(ctx context.Context, d name.Digest, option types.RemoteOptions) ( return nil, errs } -func transport(insecure bool) *http.Transport { +func httpTransport(insecure bool) *http.Transport { d := &net.Dialer{ Timeout: 10 * time.Minute, } @@ -105,13 +137,13 @@ func authOptions(ctx context.Context, ref name.Reference, option types.RemoteOpt } switch { - case len(opts) > 0: - return opts case option.RegistryToken != "": bearer := authn.Bearer{Token: option.RegistryToken} return []remote.Option{remote.WithAuth(&bearer)} default: - return []remote.Option{remote.WithAuthFromKeychain(authn.DefaultKeychain)} + // Use the keychain anyway at the end + opts = append(opts, remote.WithAuthFromKeychain(authn.DefaultKeychain)) + return opts } } diff --git a/pkg/rpc/server/listen.go b/pkg/rpc/server/listen.go index af631090952..a6de3d41ef5 100644 --- a/pkg/rpc/server/listen.go +++ b/pkg/rpc/server/listen.go @@ -16,6 +16,7 @@ import ( dbFile "github.com/aquasecurity/trivy/pkg/db" dbc "github.com/aquasecurity/trivy/pkg/db" "github.com/aquasecurity/trivy/pkg/fanal/cache" + "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/log" "github.com/aquasecurity/trivy/pkg/utils/fsutils" rpcCache "github.com/aquasecurity/trivy/rpc/cache" @@ -32,31 +33,35 @@ type Server struct { token string tokenHeader string dbRepository string + + // For OCI registries + types.RemoteOptions } // NewServer returns an instance of Server -func NewServer(appVersion, addr, cacheDir, token, tokenHeader, dbRepository string) Server { +func NewServer(appVersion, addr, cacheDir, token, tokenHeader, dbRepository string, opt types.RemoteOptions) Server { return Server{ - appVersion: appVersion, - addr: addr, - cacheDir: cacheDir, - token: token, - tokenHeader: tokenHeader, - dbRepository: dbRepository, + appVersion: appVersion, + addr: addr, + cacheDir: cacheDir, + token: token, + tokenHeader: tokenHeader, + dbRepository: dbRepository, + RemoteOptions: opt, } } // ListenAndServe starts Trivy server -func (s Server) ListenAndServe(serverCache cache.Cache, insecure, skipDBUpdate bool) error { +func (s Server) ListenAndServe(serverCache cache.Cache, skipDBUpdate bool) error { requestWg := &sync.WaitGroup{} dbUpdateWg := &sync.WaitGroup{} go func() { - worker := newDBWorker(dbc.NewClient(s.cacheDir, true, insecure, dbc.WithDBRepository(s.dbRepository))) + worker := newDBWorker(dbc.NewClient(s.cacheDir, true, dbc.WithDBRepository(s.dbRepository))) ctx := context.Background() for { time.Sleep(updateInterval) - if err := worker.update(ctx, s.appVersion, s.cacheDir, skipDBUpdate, dbUpdateWg, requestWg); err != nil { + if err := worker.update(ctx, s.appVersion, s.cacheDir, skipDBUpdate, dbUpdateWg, requestWg, s.RemoteOptions); err != nil { log.Logger.Errorf("%+v\n", err) } } @@ -121,7 +126,7 @@ func newDBWorker(dbClient dbFile.Operation) dbWorker { } func (w dbWorker) update(ctx context.Context, appVersion, cacheDir string, - skipDBUpdate bool, dbUpdateWg, requestWg *sync.WaitGroup) error { + skipDBUpdate bool, dbUpdateWg, requestWg *sync.WaitGroup, opt types.RemoteOptions) error { log.Logger.Debug("Check for DB update...") needsUpdate, err := w.dbClient.NeedsUpdate(appVersion, skipDBUpdate) if err != nil { @@ -131,20 +136,20 @@ func (w dbWorker) update(ctx context.Context, appVersion, cacheDir string, } log.Logger.Info("Updating DB...") - if err = w.hotUpdate(ctx, cacheDir, dbUpdateWg, requestWg); err != nil { + if err = w.hotUpdate(ctx, cacheDir, dbUpdateWg, requestWg, opt); err != nil { return xerrors.Errorf("failed DB hot update: %w", err) } return nil } -func (w dbWorker) hotUpdate(ctx context.Context, cacheDir string, dbUpdateWg, requestWg *sync.WaitGroup) error { +func (w dbWorker) hotUpdate(ctx context.Context, cacheDir string, dbUpdateWg, requestWg *sync.WaitGroup, opt types.RemoteOptions) error { tmpDir, err := os.MkdirTemp("", "db") if err != nil { return xerrors.Errorf("failed to create a temp dir: %w", err) } defer os.RemoveAll(tmpDir) - if err = w.dbClient.Download(ctx, tmpDir); err != nil { + if err = w.dbClient.Download(ctx, tmpDir, opt); err != nil { return xerrors.Errorf("failed to download vulnerability DB: %w", err) } diff --git a/pkg/rpc/server/listen_test.go b/pkg/rpc/server/listen_test.go index 0245332aa07..823dfe0016b 100644 --- a/pkg/rpc/server/listen_test.go +++ b/pkg/rpc/server/listen_test.go @@ -19,6 +19,7 @@ import ( "github.com/aquasecurity/trivy-db/pkg/metadata" dbFile "github.com/aquasecurity/trivy/pkg/db" "github.com/aquasecurity/trivy/pkg/fanal/cache" + ftypes "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/aquasecurity/trivy/pkg/utils/fsutils" rpcCache "github.com/aquasecurity/trivy/rpc/cache" ) @@ -140,7 +141,7 @@ func Test_dbWorker_update(t *testing.T) { defer func() { _ = db.Close() }() if tt.download.call { - mockDBClient.On("Download", mock.Anything, mock.Anything).Run( + mockDBClient.On("Download", mock.Anything, mock.Anything, mock.Anything).Run( func(args mock.Arguments) { // fake download: copy testdata/new.db to tmpDir/db/trivy.db tmpDir := args.String(1) @@ -160,7 +161,7 @@ func Test_dbWorker_update(t *testing.T) { var dbUpdateWg, requestWg sync.WaitGroup err := w.update(context.Background(), tt.args.appVersion, cacheDir, - tt.needsUpdate.input.skip, &dbUpdateWg, &requestWg) + tt.needsUpdate.input.skip, &dbUpdateWg, &requestWg, ftypes.RemoteOptions{}) if tt.wantErr != "" { require.NotNil(t, err, tt.name) assert.Contains(t, err.Error(), tt.wantErr, tt.name)