From e6a67941ab57ae114ba716bc77dce9c89b7cb4a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Mon, 8 May 2023 17:03:00 +0300 Subject: [PATCH 1/7] Implement index manifest collection for the containerd hostfs image type --- Dockerfile.imgcollector.tilt | 5 ++ Dockerfile.mockapi | 4 +- Tiltfile | 4 +- castai/imagemeta_types.go | 7 +- cmd/imgcollector/collector/collector.go | 10 ++- cmd/imgcollector/image/blob.go | 3 +- cmd/imgcollector/image/daemon.go | 8 +-- cmd/imgcollector/image/daemon/image.go | 1 + .../image/hostfs/containerd_image.go | 72 ++++++++++++++----- .../image/hostfs/containerd_image_test.go | 20 ++++++ cmd/imgcollector/image/hostfs/image.go | 1 + cmd/imgcollector/image/image.go | 8 +++ cmd/imgcollector/image/remote.go | 2 +- 13 files changed, 113 insertions(+), 32 deletions(-) create mode 100644 Dockerfile.imgcollector.tilt diff --git a/Dockerfile.imgcollector.tilt b/Dockerfile.imgcollector.tilt new file mode 100644 index 00000000..cae15e41 --- /dev/null +++ b/Dockerfile.imgcollector.tilt @@ -0,0 +1,5 @@ +FROM alpine:3.17 + +COPY ./bin/castai-imgcollector /usr/local/bin/castai-imgcollector + +CMD ["/usr/local/bin/castai-imgcollector"] diff --git a/Dockerfile.mockapi b/Dockerfile.mockapi index 9735cb7c..8274dfe8 100644 --- a/Dockerfile.mockapi +++ b/Dockerfile.mockapi @@ -1,3 +1,5 @@ -FROM gcr.io/distroless/static-debian11 +FROM alpine:3.17 + COPY ./bin/mockapi /usr/local/bin/mockapi + CMD ["/usr/local/bin/mockapi"] diff --git a/Tiltfile b/Tiltfile index aba78c50..a5291160 100644 --- a/Tiltfile +++ b/Tiltfile @@ -1,5 +1,5 @@ if config.tilt_subcommand == "down": - fail("consider using `kubectl delete ns castai-sec") + fail("consider using `kubectl delete ns kvisor") load('ext://restart_process', 'docker_build_with_restart') load('ext://namespace', 'namespace_create') @@ -44,7 +44,7 @@ local_resource( local_resource( 'imgcollector-docker-build', - 'docker build -t localhost:5000/kvisor-imgcollector . -f Dockerfile.imgcollector && docker push localhost:5000/kvisor-imgcollector', + 'docker build -t localhost:5000/kvisor-imgcollector . -f Dockerfile.imgcollector.tilt && docker push localhost:5000/kvisor-imgcollector', deps=[ './bin/castai-imgcollector' ], diff --git a/castai/imagemeta_types.go b/castai/imagemeta_types.go index c7b47426..4ca9b2a0 100644 --- a/castai/imagemeta_types.go +++ b/castai/imagemeta_types.go @@ -11,8 +11,11 @@ type ImageMetadata struct { ResourceIDs []string `json:"resourceIDs,omitempty"` BlobsInfo []types.BlobInfo `json:"blobsInfo,omitempty"` ConfigFile *v1.ConfigFile `json:"configFile,omitempty"` - Manifest *v1.Manifest `json:"manifest,omitempty"` - OsInfo *OsInfo `json:"osInfo,omitempty"` + // Manifest specification can be found here: https://github.com/opencontainers/image-spec/blob/main/manifest.md + Manifest *v1.Manifest `json:"manifest,omitempty"` + // Index specification can be found here: https://github.com/opencontainers/image-spec/blob/main/image-index.md + Index *v1.IndexManifest `json:"index,omitempty"` + OsInfo *OsInfo `json:"osInfo,omitempty"` } // nolint:musttag diff --git a/cmd/imgcollector/collector/collector.go b/cmd/imgcollector/collector/collector.go index a6ee705e..ca94e152 100644 --- a/cmd/imgcollector/collector/collector.go +++ b/cmd/imgcollector/collector/collector.go @@ -104,7 +104,12 @@ func (c *Collector) Collect(ctx context.Context) error { if err != nil { return err } - + + index, err := img.Index() + if err != nil { + return fmt.Errorf("extract index: %w", err) + } + manifest, err := img.Manifest() if err != nil { return fmt.Errorf("extract manifest: %w", err) @@ -117,6 +122,7 @@ func (c *Collector) Collect(ctx context.Context) error { BlobsInfo: arRef.BlobsInfo, ConfigFile: arRef.ConfigFile, Manifest: manifest, + Index: index, OsInfo: &castai.OsInfo{ ArtifactInfo: arRef.ArtifactInfo, OS: arRef.OsInfo, @@ -128,7 +134,7 @@ func (c *Collector) Collect(ctx context.Context) error { return nil } -func (c *Collector) getImage(ctx context.Context) (image.Image, func(), error) { +func (c *Collector) getImage(ctx context.Context) (image.ImageWithIndex, func(), error) { imgRef, err := name.ParseReference(c.cfg.ImageName) if err != nil { return nil, nil, err diff --git a/cmd/imgcollector/image/blob.go b/cmd/imgcollector/image/blob.go index 3a830755..0e4f8850 100644 --- a/cmd/imgcollector/image/blob.go +++ b/cmd/imgcollector/image/blob.go @@ -1,11 +1,10 @@ package image import ( - "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/castai/kvisor/cmd/imgcollector/image/hostfs" ) -func NewFromContainerdHostFS(imageID string, config hostfs.ContainerdHostFSConfig) (types.Image, func(), error) { +func NewFromContainerdHostFS(imageID string, config hostfs.ContainerdHostFSConfig) (ImageWithIndex, func(), error) { hash, err := hostfs.NewImageHash(imageID) if err != nil { return nil, nil, err diff --git a/cmd/imgcollector/image/daemon.go b/cmd/imgcollector/image/daemon.go index b26c004a..e41d0b67 100644 --- a/cmd/imgcollector/image/daemon.go +++ b/cmd/imgcollector/image/daemon.go @@ -9,9 +9,7 @@ import ( "github.com/castai/kvisor/cmd/imgcollector/image/daemon" ) -type Image = types.Image - -func NewFromContainerdDaemon(ctx context.Context, imageName string) (types.Image, func(), error) { +func NewFromContainerdDaemon(ctx context.Context, imageName string) (ImageWithIndex, func(), error) { img, cleanup, err := daemon.ContainerdImage(ctx, imageName) if err != nil { return nil, nil, err @@ -22,7 +20,7 @@ func NewFromContainerdDaemon(ctx context.Context, imageName string) (types.Image }, cleanup, nil } -func NewFromDockerDaemon(imageName string, ref name.Reference) (types.Image, func(), error) { +func NewFromDockerDaemon(imageName string, ref name.Reference) (ImageWithIndex, func(), error) { img, cleanup, err := daemon.DockerImage(ref) if err != nil { return nil, nil, err @@ -33,7 +31,7 @@ func NewFromDockerDaemon(imageName string, ref name.Reference) (types.Image, fun }, cleanup, nil } -func NewFromDockerDaemonTarFile(imageName, localTarPath string, ref name.Reference) (types.Image, func(), error) { +func NewFromDockerDaemonTarFile(imageName, localTarPath string, ref name.Reference) (ImageWithIndex, func(), error) { img, cleanup, err := daemon.DockerTarImage(ref, localTarPath) if err != nil { return nil, nil, err diff --git a/cmd/imgcollector/image/daemon/image.go b/cmd/imgcollector/image/daemon/image.go index 8ab53b26..d74e443f 100644 --- a/cmd/imgcollector/image/daemon/image.go +++ b/cmd/imgcollector/image/daemon/image.go @@ -26,6 +26,7 @@ type Image interface { v1.Image RepoTags() []string RepoDigests() []string + Index() (*v1.IndexManifest, error) } var mu sync.Mutex diff --git a/cmd/imgcollector/image/hostfs/containerd_image.go b/cmd/imgcollector/image/hostfs/containerd_image.go index b8ab4ee5..9f7d3982 100644 --- a/cmd/imgcollector/image/hostfs/containerd_image.go +++ b/cmd/imgcollector/image/hostfs/containerd_image.go @@ -42,6 +42,11 @@ func NewContainerdImage(hash v1.Hash, cfg ContainerdHostFSConfig) (Image, error) configBytes: configBytes, contentDir: cfg.ContentDir, } + + index, err := manifestReader.resolveIndex() + if err == nil { + img.index = index + } return img, nil } @@ -74,6 +79,16 @@ type manifestOrIndex struct { Manifests []v1.Descriptor `json:"manifests"` } +func readManifest(atPath string, into *manifestOrIndex) error { + fileBytes, err := os.ReadFile(atPath) + if err != nil { + return err + } + + return json.Unmarshal(fileBytes, into) +} + +// manifest part of the sum type func (mi *manifestOrIndex) manifest() *v1.Manifest { return &v1.Manifest{ SchemaVersion: mi.SchemaVersion, @@ -84,22 +99,40 @@ func (mi *manifestOrIndex) manifest() *v1.Manifest { } } -func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { - // Try to find manifest file. In most cases image id digest will point to manifest or index. +// index part of the sum type +func (mi *manifestOrIndex) index() *v1.IndexManifest { + return &v1.IndexManifest{ + SchemaVersion: mi.SchemaVersion, + MediaType: mi.MediaType, + Manifests: mi.Manifests, + Annotations: mi.Annotations, + } +} + +func (h *containerdManifestReader) resolveDigest() (*manifestOrIndex, error) { var mi manifestOrIndex - readManifest := func(manifestPath string) error { - var err error - fileBytes, err := os.ReadFile(manifestPath) - if err != nil { - return err - } - err = json.Unmarshal(fileBytes, &mi) - if err != nil { - return err - } - return nil + if err := readManifest(path.Join(h.cfg.ContentDir, blobs, h.imgHash.Algorithm, h.imgHash.Hex), &mi); err != nil { + return nil, err + } + return &mi, nil +} + +func (h *containerdManifestReader) resolveIndex() (*v1.IndexManifest, error) { + mi, err := h.resolveDigest() + if err != nil { + return nil, err } - if err := readManifest(path.Join(h.cfg.ContentDir, blobs, h.imgHash.Algorithm, h.imgHash.Hex)); err != nil { + + if len(mi.Manifests) == 0 { + return nil, fmt.Errorf("not an index manifest") + } + + return mi.index(), nil +} + +func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { + mi, err := h.resolveDigest() + if err != nil { return nil, err } @@ -111,7 +144,7 @@ func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { if err != nil { return nil, fmt.Errorf("searching manifest path: %w", err) } - if err := readManifest(manifestPath); err != nil { + if err := readManifest(manifestPath, mi); err != nil { return nil, err } } @@ -124,7 +157,7 @@ func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { if len(mi.Manifests) > 0 { for _, m := range mi.Manifests { if matchingPlatform(h.cfg.Platform, *m.Platform) { - if err := readManifest(path.Join(h.cfg.ContentDir, blobs, m.Digest.Algorithm, m.Digest.Hex)); err != nil { + if err := readManifest(path.Join(h.cfg.ContentDir, blobs, m.Digest.Algorithm, m.Digest.Hex), mi); err != nil { return nil, err } if len(mi.Layers) == 0 { @@ -184,13 +217,14 @@ func (h *containerdManifestReader) searchManifestPath() (string, error) { return "", err } if manifestPath == "" { - return "", errors.New("manifest not find by searching in blobs content") + return "", errors.New("manifest not found by searching in blobs content") } return manifestPath, nil } type containerdBlobImage struct { manifest *v1.Manifest + index *v1.IndexManifest config *v1.ConfigFile configBytes []byte imgHash v1.Hash @@ -223,6 +257,10 @@ func (b *containerdBlobImage) Manifest() (*v1.Manifest, error) { return b.manifest, nil } +func (b *containerdBlobImage) Index() (*v1.IndexManifest, error) { + return b.index, nil +} + func (b *containerdBlobImage) RawConfigFile() ([]byte, error) { return b.configBytes, nil } diff --git a/cmd/imgcollector/image/hostfs/containerd_image_test.go b/cmd/imgcollector/image/hostfs/containerd_image_test.go index 77fb8a52..9a77617d 100644 --- a/cmd/imgcollector/image/hostfs/containerd_image_test.go +++ b/cmd/imgcollector/image/hostfs/containerd_image_test.go @@ -59,3 +59,23 @@ func TestContainerdImage(t *testing.T) { }) } } + +func TestContainerdImageWithIndex(t *testing.T) { + r := require.New(t) + img, err := NewContainerdImage(v1.Hash{ + Algorithm: "sha256", + Hex: "211a3be9e15e1e4ccd75220aa776d92e06235552351464db2daf043bd30a0ac0", + }, + ContainerdHostFSConfig{ + Platform: v1.Platform{ + Architecture: "amd64", + OS: "linux", + }, + ContentDir: "./testdata/containerd_content", + }, + ) + r.NoError(err) + index, err := img.Index() + r.NoError(err) + r.Len(index.Manifests, 2) +} diff --git a/cmd/imgcollector/image/hostfs/image.go b/cmd/imgcollector/image/hostfs/image.go index 35413bce..877ed872 100644 --- a/cmd/imgcollector/image/hostfs/image.go +++ b/cmd/imgcollector/image/hostfs/image.go @@ -10,6 +10,7 @@ type Image interface { v1.Image RepoTags() []string RepoDigests() []string + Index() (*v1.IndexManifest, error) } // NewImageHash returns image hash from string in format: diff --git a/cmd/imgcollector/image/image.go b/cmd/imgcollector/image/image.go index 51171121..9f2b9ac2 100644 --- a/cmd/imgcollector/image/image.go +++ b/cmd/imgcollector/image/image.go @@ -3,9 +3,17 @@ package image import ( "fmt" + "github.com/aquasecurity/trivy/pkg/fanal/types" v1 "github.com/google/go-containerregistry/pkg/v1" ) +type Image = types.Image + +type ImageWithIndex interface { + Image + Index() (*v1.IndexManifest, error) +} + func ID(img v1.Image) (string, error) { h, err := img.ConfigName() if err != nil { diff --git a/cmd/imgcollector/image/remote.go b/cmd/imgcollector/image/remote.go index 64998a0f..9a7cda6d 100644 --- a/cmd/imgcollector/image/remote.go +++ b/cmd/imgcollector/image/remote.go @@ -37,7 +37,7 @@ type DockerOption struct { NonSSL bool `yaml:"non_ssl"` } -func NewFromRemote(ctx context.Context, imageName string, option DockerOption) (types.Image, error) { +func NewFromRemote(ctx context.Context, imageName string, option DockerOption) (ImageWithIndex, error) { var nameOpts []name.Option if option.NonSSL { nameOpts = append(nameOpts, name.Insecure) From 12d3021935d09b369c446bf6f476056e2f40bdeb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Tue, 9 May 2023 11:45:44 +0300 Subject: [PATCH 2/7] Reuse return value of resolveDigest instead of calling it multiple times --- .../image/hostfs/containerd_image.go | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/cmd/imgcollector/image/hostfs/containerd_image.go b/cmd/imgcollector/image/hostfs/containerd_image.go index 9f7d3982..b1b1ed68 100644 --- a/cmd/imgcollector/image/hostfs/containerd_image.go +++ b/cmd/imgcollector/image/hostfs/containerd_image.go @@ -43,7 +43,12 @@ func NewContainerdImage(hash v1.Hash, cfg ContainerdHostFSConfig) (Image, error) contentDir: cfg.ContentDir, } - index, err := manifestReader.resolveIndex() + mi, err := manifestReader.resolveDigest() + if err != nil { + return nil, fmt.Errorf("resolving digest: %w", err) + } + + index, err := manifestReader.resolveIndex(mi) if err == nil { img.index = index } @@ -117,17 +122,12 @@ func (h *containerdManifestReader) resolveDigest() (*manifestOrIndex, error) { return &mi, nil } -func (h *containerdManifestReader) resolveIndex() (*v1.IndexManifest, error) { - mi, err := h.resolveDigest() - if err != nil { - return nil, err - } - - if len(mi.Manifests) == 0 { +func (h *containerdManifestReader) resolveIndex(from *manifestOrIndex) (*v1.IndexManifest, error) { + if len(from.Manifests) == 0 { return nil, fmt.Errorf("not an index manifest") } - return mi.index(), nil + return from.index(), nil } func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { From 04263ff6cc1a9b4c514bdfaa0589d9484754e7fc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Tue, 9 May 2023 14:25:19 +0300 Subject: [PATCH 3/7] Return a not implemented error for the rest of 3 image types. --- cmd/imgcollector/image/daemon.go | 1 - cmd/imgcollector/image/daemon/image.go | 7 +++++++ cmd/imgcollector/image/remote.go | 9 ++++++++- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/cmd/imgcollector/image/daemon.go b/cmd/imgcollector/image/daemon.go index e41d0b67..9ae5140b 100644 --- a/cmd/imgcollector/image/daemon.go +++ b/cmd/imgcollector/image/daemon.go @@ -3,7 +3,6 @@ package image import ( "context" - "github.com/aquasecurity/trivy/pkg/fanal/types" "github.com/google/go-containerregistry/pkg/name" "github.com/castai/kvisor/cmd/imgcollector/image/daemon" diff --git a/cmd/imgcollector/image/daemon/image.go b/cmd/imgcollector/image/daemon/image.go index d74e443f..b52de479 100644 --- a/cmd/imgcollector/image/daemon/image.go +++ b/cmd/imgcollector/image/daemon/image.go @@ -8,6 +8,7 @@ package daemon import ( "context" + "errors" "fmt" "io" "os" @@ -22,6 +23,8 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" ) +var errNotImplemented = errors.New("not implemented") + type Image interface { v1.Image RepoTags() []string @@ -100,6 +103,10 @@ func (img *image) Manifest() (*v1.Manifest, error) { return img.Image.Manifest() } +func (img *image) Index() (*v1.IndexManifest, error) { + return nil, errNotImplemented +} + func (img *image) ConfigFile() (*v1.ConfigFile, error) { if len(img.inspect.RootFS.Layers) == 0 { // Podman doesn't return RootFS... diff --git a/cmd/imgcollector/image/remote.go b/cmd/imgcollector/image/remote.go index 9a7cda6d..b9b6fe4d 100644 --- a/cmd/imgcollector/image/remote.go +++ b/cmd/imgcollector/image/remote.go @@ -3,6 +3,7 @@ package image import ( "context" "crypto/tls" + "errors" "fmt" "net/http" "strings" @@ -15,6 +16,8 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" ) +var errNotImplemented = errors.New("not implemented") + type DockerOption struct { // Auth UserName string `yaml:"user_name"` @@ -65,7 +68,7 @@ func NewFromRemote(ctx context.Context, imageName string, option DockerOption) ( return img, nil } -func tryRemote(ctx context.Context, imageName string, ref name.Reference, option types.DockerOption) (types.Image, error) { +func tryRemote(ctx context.Context, imageName string, ref name.Reference, option types.DockerOption) (ImageWithIndex, error) { var remoteOpts []remote.Option if option.InsecureSkipTLSVerify { t := &http.Transport{ @@ -138,6 +141,10 @@ func (img remoteImage) RepoDigests() []string { return []string{repoDigest} } +func (img remoteImage) Index() (*v1.IndexManifest, error) { + return nil, errNotImplemented +} + type implicitReference struct { ref name.Reference } From 770f14731c4a183d2cd4ed8f12b31da95cadeaff Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Tue, 9 May 2023 16:20:09 +0300 Subject: [PATCH 4/7] Log not implemented error for the Index method, but continue execution --- cmd/imgcollector/collector/collector.go | 18 ++++++++++-------- cmd/imgcollector/image/image.go | 3 +++ 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/imgcollector/collector/collector.go b/cmd/imgcollector/collector/collector.go index ca94e152..58fd3ce7 100644 --- a/cmd/imgcollector/collector/collector.go +++ b/cmd/imgcollector/collector/collector.go @@ -105,29 +105,31 @@ func (c *Collector) Collect(ctx context.Context) error { return err } - index, err := img.Index() - if err != nil { - return fmt.Errorf("extract index: %w", err) - } - manifest, err := img.Manifest() if err != nil { return fmt.Errorf("extract manifest: %w", err) } - if err := c.client.SendImageMetadata(ctx, &castai.ImageMetadata{ + metadata := &castai.ImageMetadata{ ImageName: c.cfg.ImageName, ImageID: c.cfg.ImageID, ResourceIDs: strings.Split(c.cfg.ResourceIDs, ","), BlobsInfo: arRef.BlobsInfo, ConfigFile: arRef.ConfigFile, Manifest: manifest, - Index: index, OsInfo: &castai.OsInfo{ ArtifactInfo: arRef.ArtifactInfo, OS: arRef.OsInfo, }, - }); err != nil { + } + + if index, err := img.Index(); err != nil { + c.log.Debugf("extract index: %s", err) + } else { + metadata.Index = index + } + + if err := c.client.SendImageMetadata(ctx, metadata); err != nil { return err } diff --git a/cmd/imgcollector/image/image.go b/cmd/imgcollector/image/image.go index 9f2b9ac2..4f6b09bd 100644 --- a/cmd/imgcollector/image/image.go +++ b/cmd/imgcollector/image/image.go @@ -1,6 +1,7 @@ package image import ( + "errors" "fmt" "github.com/aquasecurity/trivy/pkg/fanal/types" @@ -14,6 +15,8 @@ type ImageWithIndex interface { Index() (*v1.IndexManifest, error) } +var ErrNotImplemented = errors.New("not implemented") + func ID(img v1.Image) (string, error) { h, err := img.ConfigName() if err != nil { From dfa9e3c6adc2d78d96d7a0867e2b792b264f2302 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Wed, 10 May 2023 09:21:51 +0300 Subject: [PATCH 5/7] Remove unnecessary error return values --- cmd/imgcollector/collector/collector.go | 4 +--- cmd/imgcollector/image/daemon/image.go | 9 +++------ cmd/imgcollector/image/hostfs/containerd_image.go | 13 ++++++------- cmd/imgcollector/image/hostfs/image.go | 2 +- cmd/imgcollector/image/image.go | 2 +- cmd/imgcollector/image/remote.go | 7 ++----- 6 files changed, 14 insertions(+), 23 deletions(-) diff --git a/cmd/imgcollector/collector/collector.go b/cmd/imgcollector/collector/collector.go index 58fd3ce7..fae05c95 100644 --- a/cmd/imgcollector/collector/collector.go +++ b/cmd/imgcollector/collector/collector.go @@ -123,9 +123,7 @@ func (c *Collector) Collect(ctx context.Context) error { }, } - if index, err := img.Index(); err != nil { - c.log.Debugf("extract index: %s", err) - } else { + if index := img.Index(); index != nil { metadata.Index = index } diff --git a/cmd/imgcollector/image/daemon/image.go b/cmd/imgcollector/image/daemon/image.go index b52de479..c0240d07 100644 --- a/cmd/imgcollector/image/daemon/image.go +++ b/cmd/imgcollector/image/daemon/image.go @@ -8,7 +8,6 @@ package daemon import ( "context" - "errors" "fmt" "io" "os" @@ -23,13 +22,11 @@ import ( "github.com/google/go-containerregistry/pkg/v1/tarball" ) -var errNotImplemented = errors.New("not implemented") - type Image interface { v1.Image RepoTags() []string RepoDigests() []string - Index() (*v1.IndexManifest, error) + Index() *v1.IndexManifest } var mu sync.Mutex @@ -103,8 +100,8 @@ func (img *image) Manifest() (*v1.Manifest, error) { return img.Image.Manifest() } -func (img *image) Index() (*v1.IndexManifest, error) { - return nil, errNotImplemented +func (img *image) Index() *v1.IndexManifest { + return nil } func (img *image) ConfigFile() (*v1.ConfigFile, error) { diff --git a/cmd/imgcollector/image/hostfs/containerd_image.go b/cmd/imgcollector/image/hostfs/containerd_image.go index b1b1ed68..857c099d 100644 --- a/cmd/imgcollector/image/hostfs/containerd_image.go +++ b/cmd/imgcollector/image/hostfs/containerd_image.go @@ -48,8 +48,7 @@ func NewContainerdImage(hash v1.Hash, cfg ContainerdHostFSConfig) (Image, error) return nil, fmt.Errorf("resolving digest: %w", err) } - index, err := manifestReader.resolveIndex(mi) - if err == nil { + if index := manifestReader.resolveIndex(mi); index != nil { img.index = index } return img, nil @@ -122,12 +121,12 @@ func (h *containerdManifestReader) resolveDigest() (*manifestOrIndex, error) { return &mi, nil } -func (h *containerdManifestReader) resolveIndex(from *manifestOrIndex) (*v1.IndexManifest, error) { +func (h *containerdManifestReader) resolveIndex(from *manifestOrIndex) *v1.IndexManifest { if len(from.Manifests) == 0 { - return nil, fmt.Errorf("not an index manifest") + return nil } - return from.index(), nil + return from.index() } func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { @@ -257,8 +256,8 @@ func (b *containerdBlobImage) Manifest() (*v1.Manifest, error) { return b.manifest, nil } -func (b *containerdBlobImage) Index() (*v1.IndexManifest, error) { - return b.index, nil +func (b *containerdBlobImage) Index() *v1.IndexManifest { + return b.index } func (b *containerdBlobImage) RawConfigFile() ([]byte, error) { diff --git a/cmd/imgcollector/image/hostfs/image.go b/cmd/imgcollector/image/hostfs/image.go index 877ed872..27f312b3 100644 --- a/cmd/imgcollector/image/hostfs/image.go +++ b/cmd/imgcollector/image/hostfs/image.go @@ -10,7 +10,7 @@ type Image interface { v1.Image RepoTags() []string RepoDigests() []string - Index() (*v1.IndexManifest, error) + Index() *v1.IndexManifest } // NewImageHash returns image hash from string in format: diff --git a/cmd/imgcollector/image/image.go b/cmd/imgcollector/image/image.go index 4f6b09bd..dfef4332 100644 --- a/cmd/imgcollector/image/image.go +++ b/cmd/imgcollector/image/image.go @@ -12,7 +12,7 @@ type Image = types.Image type ImageWithIndex interface { Image - Index() (*v1.IndexManifest, error) + Index() *v1.IndexManifest } var ErrNotImplemented = errors.New("not implemented") diff --git a/cmd/imgcollector/image/remote.go b/cmd/imgcollector/image/remote.go index b9b6fe4d..960cc1b4 100644 --- a/cmd/imgcollector/image/remote.go +++ b/cmd/imgcollector/image/remote.go @@ -3,7 +3,6 @@ package image import ( "context" "crypto/tls" - "errors" "fmt" "net/http" "strings" @@ -16,8 +15,6 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" ) -var errNotImplemented = errors.New("not implemented") - type DockerOption struct { // Auth UserName string `yaml:"user_name"` @@ -141,8 +138,8 @@ func (img remoteImage) RepoDigests() []string { return []string{repoDigest} } -func (img remoteImage) Index() (*v1.IndexManifest, error) { - return nil, errNotImplemented +func (img remoteImage) Index() *v1.IndexManifest { + return nil } type implicitReference struct { From 90af5d8ec5a7a2e4310e153f6be7148f564dd192 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Wed, 10 May 2023 10:23:54 +0300 Subject: [PATCH 6/7] Remove redundant methods --- .../image/hostfs/containerd_image.go | 94 +++++++++---------- cmd/imgcollector/image/image.go | 3 - 2 files changed, 43 insertions(+), 54 deletions(-) diff --git a/cmd/imgcollector/image/hostfs/containerd_image.go b/cmd/imgcollector/image/hostfs/containerd_image.go index 857c099d..313af5d5 100644 --- a/cmd/imgcollector/image/hostfs/containerd_image.go +++ b/cmd/imgcollector/image/hostfs/containerd_image.go @@ -25,43 +25,36 @@ const ( ) func NewContainerdImage(hash v1.Hash, cfg ContainerdHostFSConfig) (Image, error) { - manifestReader := newContainerdManifestReader(hash, cfg) - manifest, err := manifestReader.resolveManifest() + metadataReader := newContainerdMetadataReader(hash, cfg) + metadata, err := metadataReader.readMetadata() if err != nil { return nil, fmt.Errorf("resolving manifest: %w", err) } - config, configBytes, err := manifestReader.readConfig(manifest.Config.Digest.String()) + config, configBytes, err := metadataReader.readConfig(metadata.Manifest.Config.Digest.String()) if err != nil { return nil, fmt.Errorf("reading config file: %w", err) } - img := &containerdBlobImage{ - manifest: manifest, + return &containerdBlobImage{ + manifest: metadata.Manifest, + index: metadata.Index, config: config, configBytes: configBytes, contentDir: cfg.ContentDir, - } - - mi, err := manifestReader.resolveDigest() - if err != nil { - return nil, fmt.Errorf("resolving digest: %w", err) - } - - if index := manifestReader.resolveIndex(mi); index != nil { - img.index = index - } - return img, nil + }, nil } -func newContainerdManifestReader(hash v1.Hash, cfg ContainerdHostFSConfig) *containerdManifestReader { - return &containerdManifestReader{ +func newContainerdMetadataReader(hash v1.Hash, cfg ContainerdHostFSConfig) *containerdMetadataReader { + return &containerdMetadataReader{ imgHash: hash, cfg: cfg, } } -type containerdManifestReader struct { +// containerdMetadataReader is used to follow image references as described here: +// https://github.com/google/go-containerregistry/blob/main/images/ociimage.jpeg +type containerdMetadataReader struct { cfg ContainerdHostFSConfig imgHash v1.Hash } @@ -71,6 +64,11 @@ type ContainerdHostFSConfig struct { ContentDir string } +type containerdMetadata struct { + Index *v1.IndexManifest + Manifest *v1.Manifest +} + type manifestOrIndex struct { SchemaVersion int64 `json:"schemaVersion"` MediaType types.MediaType `json:"mediaType,omitempty"` @@ -113,65 +111,59 @@ func (mi *manifestOrIndex) index() *v1.IndexManifest { } } -func (h *containerdManifestReader) resolveDigest() (*manifestOrIndex, error) { - var mi manifestOrIndex - if err := readManifest(path.Join(h.cfg.ContentDir, blobs, h.imgHash.Algorithm, h.imgHash.Hex), &mi); err != nil { - return nil, err - } - return &mi, nil -} - -func (h *containerdManifestReader) resolveIndex(from *manifestOrIndex) *v1.IndexManifest { - if len(from.Manifests) == 0 { - return nil - } - - return from.index() -} - -func (h *containerdManifestReader) resolveManifest() (*v1.Manifest, error) { - mi, err := h.resolveDigest() - if err != nil { +func (h *containerdMetadataReader) readMetadata() (*containerdMetadata, error) { + var ( + metadata containerdMetadata + manOrIdx manifestOrIndex + ) + if err := readManifest( + path.Join(h.cfg.ContentDir, blobs, h.imgHash.Algorithm, h.imgHash.Hex), &manOrIdx, + ); err != nil { return nil, err } // This case indicates that image id digest points to config file. // In such case we need to find manifest by iterating all files and searching for // config file digest hash inside files content. - if len(mi.Layers) == 0 && len(mi.Manifests) == 0 { + if len(manOrIdx.Layers) == 0 && len(manOrIdx.Manifests) == 0 { manifestPath, err := h.searchManifestPath() if err != nil { return nil, fmt.Errorf("searching manifest path: %w", err) } - if err := readManifest(manifestPath, mi); err != nil { + if err := readManifest(manifestPath, &manOrIdx); err != nil { return nil, err } } - if len(mi.Layers) > 0 { - return mi.manifest(), nil + if len(manOrIdx.Layers) > 0 { + metadata.Manifest = manOrIdx.manifest() + return &metadata, nil } // Search manifest from index manifest. - if len(mi.Manifests) > 0 { - for _, m := range mi.Manifests { - if matchingPlatform(h.cfg.Platform, *m.Platform) { - if err := readManifest(path.Join(h.cfg.ContentDir, blobs, m.Digest.Algorithm, m.Digest.Hex), mi); err != nil { + if len(manOrIdx.Manifests) > 0 { + metadata.Index = manOrIdx.index() + for _, manifest := range manOrIdx.Manifests { + if matchingPlatform(h.cfg.Platform, *manifest.Platform) { + if err := readManifest( + path.Join(h.cfg.ContentDir, blobs, manifest.Digest.Algorithm, manifest.Digest.Hex), &manOrIdx, + ); err != nil { return nil, err } - if len(mi.Layers) == 0 { + if len(manOrIdx.Layers) == 0 { return nil, errors.New("invalid manifest, no layers") } - return mi.manifest(), nil + metadata.Manifest = manOrIdx.manifest() + return &metadata, nil } } return nil, fmt.Errorf("manifest not found for platform: %s %s", h.cfg.Platform.Architecture, h.cfg.Platform.OS) } - return nil, fmt.Errorf("unrecognised manifest mediatype %q", string(mi.MediaType)) + return nil, fmt.Errorf("unrecognised manifest mediatype %q", string(manOrIdx.MediaType)) } -func (h *containerdManifestReader) readConfig(configID string) (*v1.ConfigFile, []byte, error) { +func (h *containerdMetadataReader) readConfig(configID string) (*v1.ConfigFile, []byte, error) { p := strings.Split(configID, ":") if len(p) < 2 { return nil, nil, fmt.Errorf("invalid configID: %s", configID) @@ -192,7 +184,7 @@ func (h *containerdManifestReader) readConfig(configID string) (*v1.ConfigFile, return &cfg, configBytes, nil } -func (h *containerdManifestReader) searchManifestPath() (string, error) { +func (h *containerdMetadataReader) searchManifestPath() (string, error) { root := path.Join(h.cfg.ContentDir, blobs, h.imgHash.Algorithm) var manifestPath string digestBytes := []byte(h.imgHash.Hex) diff --git a/cmd/imgcollector/image/image.go b/cmd/imgcollector/image/image.go index dfef4332..542edee0 100644 --- a/cmd/imgcollector/image/image.go +++ b/cmd/imgcollector/image/image.go @@ -1,7 +1,6 @@ package image import ( - "errors" "fmt" "github.com/aquasecurity/trivy/pkg/fanal/types" @@ -15,8 +14,6 @@ type ImageWithIndex interface { Index() *v1.IndexManifest } -var ErrNotImplemented = errors.New("not implemented") - func ID(img v1.Image) (string, error) { h, err := img.ConfigName() if err != nil { From 211787a0e5deb5138c5c400fa038009e8b4c2359 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Domas=20Tama=C5=A1auskas?= Date: Wed, 10 May 2023 10:52:06 +0300 Subject: [PATCH 7/7] Update the tests --- cmd/imgcollector/image/hostfs/containerd_image_test.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/imgcollector/image/hostfs/containerd_image_test.go b/cmd/imgcollector/image/hostfs/containerd_image_test.go index 9a77617d..0ab99efd 100644 --- a/cmd/imgcollector/image/hostfs/containerd_image_test.go +++ b/cmd/imgcollector/image/hostfs/containerd_image_test.go @@ -75,7 +75,13 @@ func TestContainerdImageWithIndex(t *testing.T) { }, ) r.NoError(err) - index, err := img.Index() - r.NoError(err) + + index := img.Index() + r.NotNil(index) r.Len(index.Manifests, 2) + + manifest, err := img.Manifest() + r.NoError(err) + r.NotNil(manifest) + r.Len(manifest.Layers, 2) }