From 4cb09675bd9b8e9c30f8aae5862857a0db838bf6 Mon Sep 17 00:00:00 2001 From: Ben Oukhanov Date: Wed, 28 Feb 2024 09:18:55 +0000 Subject: [PATCH 1/2] feat: build and push image index Push an image index to have multiple image manifests, instead of single image manifest when needed. Jira-Url: https://issues.redhat.com/browse/CNV-38597 Signed-off-by: Ben Oukhanov --- artifacts/centos/centos.go | 1 + artifacts/centos/centos_test.go | 12 +- artifacts/centosstream/centos-stream.go | 1 + artifacts/centosstream/centos-stream_test.go | 2 + artifacts/fedora/fedora.go | 21 ++-- artifacts/fedora/fedora_test.go | 40 +++--- artifacts/ubuntu/ubuntu.go | 7 +- artifacts/ubuntu/ubuntu_test.go | 5 +- cmd/medius/common/registry.go | 98 ++++++++------- cmd/medius/docs/docs.go | 8 +- cmd/medius/images/common.go | 5 +- cmd/medius/images/promote.go | 5 +- cmd/medius/images/push.go | 125 +++++++++++++++---- cmd/medius/images/verify.go | 5 +- pkg/api/artifact.go | 4 +- pkg/build/build.go | 34 ++++- pkg/repository/repository.go | 18 ++- 17 files changed, 272 insertions(+), 119 deletions(-) diff --git a/artifacts/centos/centos.go b/artifacts/centos/centos.go index f6f45020..74985120 100644 --- a/artifacts/centos/centos.go +++ b/artifacts/centos/centos.go @@ -64,6 +64,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) { SHA256Sum: checksum, DownloadURL: baseURL + candidate, AdditionalUniqueTags: getAdditionalTags(c.Version, c.Variant, candidate), + ImageArchitecture: "amd64", }, nil } diff --git a/artifacts/centos/centos_test.go b/artifacts/centos/centos_test.go index fd8c01f1..66c4a54f 100644 --- a/artifacts/centos/centos_test.go +++ b/artifacts/centos/centos_test.go @@ -28,6 +28,7 @@ var _ = Describe("Centos", func() { DownloadURL: "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2", Compression: "", AdditionalUniqueTags: []string{"8.4.2105-20210603.0", "8.4.2105"}, + ImageArchitecture: "amd64", }, map[string]string{ "TEST_ENV_VAR": "test-value", @@ -50,6 +51,7 @@ var _ = Describe("Centos", func() { DownloadURL: "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2", Compression: "", AdditionalUniqueTags: []string{"8.3.2011-20201204.2", "8.3.2011"}, + ImageArchitecture: "amd64", }, map[string]string{ "TEST_ENV_VAR": "test-value", @@ -68,8 +70,9 @@ var _ = Describe("Centos", func() { ), Entry("centos:7-2009", "7-2009", "testdata/centos7.checksum", &api.ArtifactDetails{ - SHA256Sum: "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3", - DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2", + SHA256Sum: "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3", + DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2", + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", @@ -90,8 +93,9 @@ var _ = Describe("Centos", func() { ), Entry("centos:7-1809", "7-1809", "testdata/centos7.checksum", &api.ArtifactDetails{ - SHA256Sum: "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2", - DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2", + SHA256Sum: "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2", + DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2", + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", diff --git a/artifacts/centosstream/centos-stream.go b/artifacts/centosstream/centos-stream.go index 34a3ed81..494d276f 100644 --- a/artifacts/centosstream/centos-stream.go +++ b/artifacts/centosstream/centos-stream.go @@ -89,6 +89,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) { SHA256Sum: checksum, DownloadURL: baseURL + candidate, AdditionalUniqueTags: additionalTags, + ImageArchitecture: "amd64", }, nil } diff --git a/artifacts/centosstream/centos-stream_test.go b/artifacts/centosstream/centos-stream_test.go index a1decf7f..03cb9ef3 100644 --- a/artifacts/centosstream/centos-stream_test.go +++ b/artifacts/centosstream/centos-stream_test.go @@ -28,6 +28,7 @@ var _ = Describe("CentosStream", func() { SHA256Sum: "8e22e67687b81e38c7212fc30c47cb24cbc4935c0f2459ed139f498397d1e7cd", DownloadURL: "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210603.0.x86_64.qcow2", AdditionalUniqueTags: []string{"8-20210603.0"}, + ImageArchitecture: "amd64", }, &docs.UserData{ Username: "centos", @@ -54,6 +55,7 @@ var _ = Describe("CentosStream", func() { SHA256Sum: "bcebdc00511d6e18782732570056cfbc7cba318302748bfc8f66be9c0db68142", DownloadURL: "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20211222.0.x86_64.qcow2", AdditionalUniqueTags: []string{"9-20211222.0"}, + ImageArchitecture: "amd64", }, &docs.UserData{ Username: "cloud-user", diff --git a/artifacts/fedora/fedora.go b/artifacts/fedora/fedora.go index 36992bc0..9549fbbb 100644 --- a/artifacts/fedora/fedora.go +++ b/artifacts/fedora/fedora.go @@ -77,6 +77,7 @@ func (f *fedora) Inspect() (*api.ArtifactDetails, error) { SHA256Sum: release.Sha256, DownloadURL: release.Link, AdditionalUniqueTags: []string{additionalTag}, + ImageArchitecture: "amd64", }, nil } } @@ -105,23 +106,25 @@ func (f *fedora) Tests() []api.ArtifactTest { } } -func (f *fedoraGatherer) Gather() ([]api.Artifact, error) { +func (f *fedoraGatherer) Gather() ([][]api.Artifact, error) { releases, err := getReleases(f.getter) if err != nil { return nil, fmt.Errorf("error getting releases: %v", err) } - artifacts := []api.Artifact{} + artifacts := [][]api.Artifact{} for i, release := range releases { if f.releaseMatches(&releases[i]) { artifacts = append(artifacts, - New( - release.Version, - map[string]string{ - common.DefaultInstancetypeEnv: "u1.small", - common.DefaultPreferenceEnv: "fedora", - }, - ), + []api.Artifact{ + New( + release.Version, + map[string]string{ + common.DefaultInstancetypeEnv: "u1.small", + common.DefaultPreferenceEnv: "fedora", + }, + ), + }, ) } } diff --git a/artifacts/fedora/fedora_test.go b/artifacts/fedora/fedora_test.go index 6a55d352..29898df1 100644 --- a/artifacts/fedora/fedora_test.go +++ b/artifacts/fedora/fedora_test.go @@ -28,6 +28,7 @@ var _ = Describe("Fedora", func() { SHA256Sum: "fe84502779b3477284a8d4c86731f642ca10dd3984d2b5eccdf82630a9ca2de6", DownloadURL: "https://download.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2", //nolint:lll AdditionalUniqueTags: []string{"35-1.2"}, + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", @@ -51,6 +52,7 @@ var _ = Describe("Fedora", func() { SHA256Sum: "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea", DownloadURL: "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", //nolint:lll AdditionalUniqueTags: []string{"34-1.2"}, + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", @@ -72,25 +74,29 @@ var _ = Describe("Fedora", func() { ) It("Gather should be able to parse releases files", func() { - artifacts := []api.Artifact{ - &fedora{ - Version: "36", - Arch: "x86_64", - Variant: "Cloud", - getter: &http.HTTPGetter{}, - EnvVariables: map[string]string{ - common.DefaultInstancetypeEnv: "u1.small", - common.DefaultPreferenceEnv: "fedora", + artifacts := [][]api.Artifact{ + { + &fedora{ + Version: "36", + Arch: "x86_64", + Variant: "Cloud", + getter: &http.HTTPGetter{}, + EnvVariables: map[string]string{ + common.DefaultInstancetypeEnv: "u1.small", + common.DefaultPreferenceEnv: "fedora", + }, }, }, - &fedora{ - Version: "35", - Arch: "x86_64", - Variant: "Cloud", - getter: &http.HTTPGetter{}, - EnvVariables: map[string]string{ - common.DefaultInstancetypeEnv: "u1.small", - common.DefaultPreferenceEnv: "fedora", + { + &fedora{ + Version: "35", + Arch: "x86_64", + Variant: "Cloud", + getter: &http.HTTPGetter{}, + EnvVariables: map[string]string{ + common.DefaultInstancetypeEnv: "u1.small", + common.DefaultPreferenceEnv: "fedora", + }, }, }, } diff --git a/artifacts/ubuntu/ubuntu.go b/artifacts/ubuntu/ubuntu.go index 67c01bd3..6c51f02a 100644 --- a/artifacts/ubuntu/ubuntu.go +++ b/artifacts/ubuntu/ubuntu.go @@ -52,9 +52,10 @@ func (u *ubuntu) Inspect() (*api.ArtifactDetails, error) { } if checksum, exists := checksums[u.Variant]; exists { return &api.ArtifactDetails{ - SHA256Sum: checksum, - DownloadURL: baseURL + u.Variant, - Compression: u.Compression, + SHA256Sum: checksum, + DownloadURL: baseURL + u.Variant, + Compression: u.Compression, + ImageArchitecture: "amd64", }, nil } return nil, fmt.Errorf("file %q does not exist in the SHA256SUMS file: %v", u.Variant, err) diff --git a/artifacts/ubuntu/ubuntu_test.go b/artifacts/ubuntu/ubuntu_test.go index cd97be07..24f64539 100644 --- a/artifacts/ubuntu/ubuntu_test.go +++ b/artifacts/ubuntu/ubuntu_test.go @@ -24,8 +24,9 @@ var _ = Describe("Ubuntu", func() { }, Entry("ubuntu:22.04", "22.04", "testdata/SHA256SUM", &api.ArtifactDetails{ - SHA256Sum: "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd", - DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img", + SHA256Sum: "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd", + DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img", + ImageArchitecture: "amd64", }, map[string]string{ common.DefaultInstancetypeEnv: "u1.small", diff --git a/cmd/medius/common/registry.go b/cmd/medius/common/registry.go index 101a77cf..ea469fae 100644 --- a/cmd/medius/common/registry.go +++ b/cmd/medius/common/registry.go @@ -16,7 +16,7 @@ import ( ) type Entry struct { - Artifact api.Artifact + Artifacts []api.Artifact UseForDocs bool UseForLatest bool SkipWhenNotFocused bool @@ -24,69 +24,73 @@ type Entry struct { var staticRegistry = []Entry{ { - Artifact: centos.New("8.4", nil), + Artifacts: []api.Artifact{ + centos.New("8.4", nil), + }, UseForDocs: false, }, { - Artifact: centos.New( - "7-2009", - defaultEnvVariables("u1.small", "centos.7"), - ), + Artifacts: []api.Artifact{ + centos.New("7-2009", defaultEnvVariables("u1.small", "centos.7")), + }, UseForDocs: true, }, { - Artifact: centosstream.New( - "9", - &docs.UserData{ - Username: "cloud-user", - }, - defaultEnvVariables("u1.small", "centos.stream9"), - ), + Artifacts: []api.Artifact{ + centosstream.New("9", &docs.UserData{Username: "cloud-user"}, defaultEnvVariables("u1.small", "centos.stream9")), + }, UseForDocs: true, }, { - Artifact: centosstream.New( - "8", - &docs.UserData{ - Username: "centos", - }, - defaultEnvVariables("u1.small", "centos.stream8"), - ), + Artifacts: []api.Artifact{ + centosstream.New("8", &docs.UserData{Username: "centos"}, defaultEnvVariables("u1.small", "centos.stream8")), + }, UseForDocs: false, }, { - Artifact: ubuntu.New( - "22.04", - defaultEnvVariables("u1.small", "ubuntu"), - ), + Artifacts: []api.Artifact{ + ubuntu.New("22.04", defaultEnvVariables("u1.small", "ubuntu")), + }, UseForDocs: true, }, { - Artifact: ubuntu.New( - "20.04", - defaultEnvVariables("u1.small", "ubuntu"), - ), + Artifacts: []api.Artifact{ + ubuntu.New("20.04", defaultEnvVariables("u1.small", "ubuntu")), + }, UseForDocs: false, }, { - Artifact: ubuntu.New( - "18.04", - defaultEnvVariables("u1.small", "ubuntu"), - ), + Artifacts: []api.Artifact{ + ubuntu.New("18.04", defaultEnvVariables("u1.small", "ubuntu")), + }, UseForDocs: false, }, // for testing only { - Artifact: generic.New( - &api.ArtifactDetails{ - SHA256Sum: "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1", - DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img", - }, - &api.Metadata{ - Name: "cirros", - Version: "6.1", - }, - ), + Artifacts: []api.Artifact{ + generic.New( + &api.ArtifactDetails{ + SHA256Sum: "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1", + DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img", + ImageArchitecture: "amd64", + }, + &api.Metadata{ + Name: "cirros", + Version: "6.1", + }, + ), + generic.New( + &api.ArtifactDetails{ + SHA256Sum: "db9420c481c11dee17860aa46fb1a3efa05fa4fb152726d6344e24da03cb0ccf", + DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img", + ImageArchitecture: "arm64", + }, + &api.Metadata{ + Name: "cirros", + Version: "6.1", + }, + ), + }, SkipWhenNotFocused: true, UseForDocs: false, }, @@ -100,7 +104,7 @@ func gatherArtifacts(registry *[]Entry, gatherers []api.ArtifactsGatherer) { } else { for i := range artifacts { *registry = append(*registry, Entry{ - Artifact: artifacts[i], + Artifacts: artifacts[i], UseForDocs: i == 0, UseForLatest: i == 0, }) @@ -131,12 +135,16 @@ func ShouldSkip(focus string, entry *Entry) bool { return entry.SkipWhenNotFocused } + if len(entry.Artifacts) == 0 { + return true + } + focusSplit := strings.Split(focus, ":") wildcardFocus := len(focusSplit) == 2 && focusSplit[1] == "*" if wildcardFocus { - return focusSplit[0] != entry.Artifact.Metadata().Name + return focusSplit[0] != entry.Artifacts[0].Metadata().Name } - return focus != entry.Artifact.Metadata().Describe() + return focus != entry.Artifacts[0].Metadata().Describe() } diff --git a/cmd/medius/docs/docs.go b/cmd/medius/docs/docs.go index 0c41d63d..62b6e3e3 100644 --- a/cmd/medius/docs/docs.go +++ b/cmd/medius/docs/docs.go @@ -60,10 +60,12 @@ func run(options *common.Options) error { } focusMatched = true - log := common.Logger(p.Artifact) - name := p.Artifact.Metadata().Name - description, err := createDescription(p.Artifact, options.PublishDocsOptions.Registry) + artifact := p.Artifacts[0] + log := common.Logger(artifact) + name := artifact.Metadata().Name + + description, err := createDescription(artifact, options.PublishDocsOptions.Registry) if err != nil { success = false log.Errorf("error marshaling example for %q: %v", name, err) diff --git a/cmd/medius/images/common.go b/cmd/medius/images/common.go index 41b5844f..c137d57c 100644 --- a/cmd/medius/images/common.go +++ b/cmd/medius/images/common.go @@ -44,15 +44,16 @@ func spawnWorkers(ctx context.Context, o *common.Options, go func() { defer wg.Done() for e := range jobChan { + artifact := e.Artifacts[0] result, workerErr := fn(e) if result != nil { resultsChan <- workerResult{ - Key: e.Artifact.Metadata().Describe(), + Key: artifact.Metadata().Describe(), Value: *result, } } if workerErr != nil && !errors.Is(workerErr, context.Canceled) { - common.Logger(e.Artifact).Error(workerErr) + common.Logger(artifact).Error(workerErr) errChan <- workerErr } if errors.Is(ctx.Err(), context.Canceled) { diff --git a/cmd/medius/images/promote.go b/cmd/medius/images/promote.go index 91848a17..06d85883 100644 --- a/cmd/medius/images/promote.go +++ b/cmd/medius/images/promote.go @@ -29,7 +29,8 @@ func NewPromoteImagesCommand(options *common.Options) *cobra.Command { } focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) { - description := e.Artifact.Metadata().Describe() + artifact := e.Artifacts[0] + description := artifact.Metadata().Describe() r, ok := results[description] if !ok { return nil, nil @@ -42,7 +43,7 @@ func NewPromoteImagesCommand(options *common.Options) *cobra.Command { } errString := "" - err := promoteArtifact(cmd.Context(), e.Artifact, r.Tags, options) + err := promoteArtifact(cmd.Context(), artifact, r.Tags, options) if err != nil { errString = err.Error() } diff --git a/cmd/medius/images/push.go b/cmd/medius/images/push.go index 00939719..36455d34 100644 --- a/cmd/medius/images/push.go +++ b/cmd/medius/images/push.go @@ -46,10 +46,11 @@ func NewPublishImagesCommand(options *common.Options) *cobra.Command { focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) { errString := "" + artifact := e.Artifacts[0] b := buildAndPublish{ Ctx: cmd.Context(), - Log: common.Logger(e.Artifact), + Log: common.Logger(artifact), Options: options, Repo: &repository.RepositoryImpl{}, Getter: &http.HTTPGetter{}, @@ -107,18 +108,17 @@ func NewPublishImagesCommand(options *common.Options) *cobra.Command { } func (b *buildAndPublish) Do(entry *common.Entry, timestamp time.Time) ([]string, error) { - metadata := entry.Artifact.Metadata() - artifactInfo, err := entry.Artifact.Inspect() + metadata := entry.Artifacts[0].Metadata() + artifactInfo, err := entry.Artifacts[0].Inspect() if err != nil { return nil, fmt.Errorf("error introspecting artifact %q: %v", metadata.Describe(), err) } - b.Log.Infof("Remote artifact checksum: %q", artifactInfo.SHA256Sum) - imageSha, err := b.getImageSha(metadata.Describe()) + rebuildNeeded, err := b.rebuildNeeded(entry) if err != nil { return nil, err } - if imageSha == artifactInfo.SHA256Sum && !b.Options.PublishImagesOptions.ForceBuild { + if !rebuildNeeded && !b.Options.PublishImagesOptions.ForceBuild { b.Log.Info("Nothing to do.") return nil, nil } @@ -126,26 +126,26 @@ func (b *buildAndPublish) Do(entry *common.Entry, timestamp time.Time) ([]string return nil, b.Ctx.Err() } - b.Log.Infof("Rebuild needed, downloading %q ...", artifactInfo.DownloadURL) - file, err := b.getArtifact(artifactInfo) + images, artifacts, err := b.buildImages(entry) if err != nil { return nil, err } - defer os.Remove(file) - - b.Log.Info("Building containerdisk ...") - containerDisk, err := build.ContainerDisk(file, build.ContainerDiskConfig(artifactInfo.SHA256Sum, metadata.EnvVariables)) - if err != nil { - return nil, fmt.Errorf("error creating the containerdisk : %v", err) - } - if errors.Is(b.Ctx.Err(), context.Canceled) { - return nil, b.Ctx.Err() - } + defer cleanupArtifacts(artifacts) names := prepareTags(timestamp, b.Options.PublishImagesOptions.TargetRegistry, entry, artifactInfo) for _, name := range names { - if err := b.pushImage(containerDisk, name); err != nil { - return nil, err + if len(images) > 1 { + containerDiskIndex, err := build.ContainerDiskIndex(images) + if err != nil { + return nil, fmt.Errorf("error creating the containerdisk index : %v", err) + } + if err := b.pushImageIndex(containerDiskIndex, name); err != nil { + return nil, err + } + } else if len(images) == 1 { + if err := b.pushImage(images[0], name); err != nil { + return nil, err + } } if errors.Is(b.Ctx.Err(), context.Canceled) { return nil, b.Ctx.Err() @@ -155,9 +155,9 @@ func (b *buildAndPublish) Do(entry *common.Entry, timestamp time.Time) ([]string return prepareTags(timestamp, "", entry, artifactInfo), nil } -func (b *buildAndPublish) getImageSha(description string) (imageSha string, err error) { +func (b *buildAndPublish) getImageSha(description, arch string) (imageSha string, err error) { imageName := path.Join(b.Options.PublishImagesOptions.SourceRegistry, description) - imageInfo, err := b.Repo.ImageMetadata(imageName, b.Options.AllowInsecureRegistry) + imageInfo, err := b.Repo.ImageMetadata(imageName, arch, b.Options.AllowInsecureRegistry) if err != nil { err = b.handleMetadataError(imageName, err) } else { @@ -247,6 +247,65 @@ func (b *buildAndPublish) readArtifact(artifactReader http.ReadCloserWithChecksu return file.Name(), nil } +func (b *buildAndPublish) buildImages(entry *common.Entry) ([]v1.Image, []string, error) { + var images []v1.Image + var artifacts []string + + for i := range entry.Artifacts { + metadata := entry.Artifacts[i].Metadata() + artifactInfo, err := entry.Artifacts[i].Inspect() + if err != nil { + return nil, nil, fmt.Errorf("error introspecting artifact %q: %v", metadata.Describe(), err) + } + + b.Log.Infof("Rebuild needed, downloading %q ...", artifactInfo.DownloadURL) + file, err := b.getArtifact(artifactInfo) + if err != nil { + return nil, nil, err + } + artifacts = append(artifacts, file) + + b.Log.Info("Building containerdisk ...") + image, err := build.ContainerDisk(file, + artifactInfo.ImageArchitecture, + build.ContainerDiskConfig(artifactInfo.SHA256Sum, metadata.EnvVariables)) + if err != nil { + return nil, nil, fmt.Errorf("error creating the containerdisk : %v", err) + } + if errors.Is(b.Ctx.Err(), context.Canceled) { + return nil, nil, b.Ctx.Err() + } + images = append(images, image) + } + + return images, artifacts, nil +} + +func (b *buildAndPublish) rebuildNeeded(entry *common.Entry) (bool, error) { + if len(entry.Artifacts) == 0 { + err := errors.New("entry has no artifacts to check for rebuild") + b.Log.Error(err) + return false, err + } + + for i := range entry.Artifacts { + metadata := entry.Artifacts[i].Metadata() + artifactInfo, err := entry.Artifacts[i].Inspect() + if err != nil { + return false, fmt.Errorf("error introspecting artifact %q: %v", metadata.Describe(), err) + } + imageSha, err := b.getImageSha(metadata.Describe(), artifactInfo.ImageArchitecture) + if err != nil { + return false, err + } + if imageSha != artifactInfo.SHA256Sum { + return true, nil + } + } + + return false, nil +} + func (b *buildAndPublish) pushImage(containerDisk v1.Image, name string) error { if !b.Options.DryRun { b.Log.Infof("Pushing %s", name) @@ -261,8 +320,22 @@ func (b *buildAndPublish) pushImage(containerDisk v1.Image, name string) error { return nil } +func (b *buildAndPublish) pushImageIndex(containerDiskIndex v1.ImageIndex, name string) error { + if !b.Options.DryRun { + b.Log.Infof("Pushing %s image index", name) + if err := b.Repo.PushImageIndex(b.Ctx, containerDiskIndex, name); err != nil { + b.Log.WithError(err).Error("Failed to push image image") + return err + } + } else { + b.Log.Infof("Dry run enabled, not pushing %s image index", name) + } + + return nil +} + func prepareTags(timestamp time.Time, registry string, entry *common.Entry, artifactDetails *api.ArtifactDetails) []string { - metadata := entry.Artifact.Metadata() + metadata := entry.Artifacts[0].Metadata() imageName := path.Join(registry, metadata.Describe()) names := []string{fmt.Sprintf("%s-%s", imageName, timestamp.Format("0601021504"))} @@ -281,3 +354,9 @@ func prepareTags(timestamp time.Time, registry string, entry *common.Entry, arti return names } + +func cleanupArtifacts(artifacts []string) { + for _, file := range artifacts { + os.Remove(file) + } +} diff --git a/cmd/medius/images/verify.go b/cmd/medius/images/verify.go index 58df0570..2ed0499e 100644 --- a/cmd/medius/images/verify.go +++ b/cmd/medius/images/verify.go @@ -48,7 +48,8 @@ func NewVerifyImagesCommand(options *common.Options) *cobra.Command { } focusMatched, resultsChan, workerErr := spawnWorkers(cmd.Context(), options, func(e *common.Entry) (*api.ArtifactResult, error) { - description := e.Artifact.Metadata().Describe() + artifact := e.Artifacts[0] + description := artifact.Metadata().Describe() r, ok := results[description] if !ok { return nil, nil @@ -61,7 +62,7 @@ func NewVerifyImagesCommand(options *common.Options) *cobra.Command { } errString := "" - err := verifyArtifact(cmd.Context(), e.Artifact, r, options, client) + err := verifyArtifact(cmd.Context(), artifact, r, options, client) if err != nil { errString = err.Error() } diff --git a/pkg/api/artifact.go b/pkg/api/artifact.go index 961ec1ba..fb124ae0 100644 --- a/pkg/api/artifact.go +++ b/pkg/api/artifact.go @@ -32,6 +32,8 @@ type ArtifactDetails struct { SHA256Sum string // DownloadURL points to the target image. DownloadURL string + // ImageArchitecture is the target architecture of the image. + ImageArchitecture string // Compression describes the compression format of the downloaded image. // Supported are "" (none), "gzip" and "xz". Compression string @@ -70,5 +72,5 @@ type Artifact interface { type ArtifactsGatherer interface { // Gather must return a sorted list of dynamically gathered artifacts. // Artifacts have to be sorted in descending order with the latest release coming first. - Gather() ([]Artifact, error) + Gather() ([][]Artifact, error) } diff --git a/pkg/build/build.go b/pkg/build/build.go index ce93299c..4f221194 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -6,12 +6,13 @@ import ( v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/empty" "github.com/google/go-containerregistry/pkg/v1/mutate" + "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/tarball" ) const ( - LabelShaSum = "shasum" - ImageArchitecture = "amd64" + LabelShaSum = "shasum" + ImageOS = "linux" ) func ContainerDiskConfig(checksum string, envVariables map[string]string) v1.Config { @@ -27,7 +28,7 @@ func ContainerDiskConfig(checksum string, envVariables map[string]string) v1.Con return v1.Config{Labels: labels, Env: env} } -func ContainerDisk(imgPath string, config v1.Config) (v1.Image, error) { +func ContainerDisk(imgPath, imgArch string, config v1.Config) (v1.Image, error) { img := empty.Image layer, err := tarball.LayerFromOpener(StreamLayerOpener(imgPath)) if err != nil { @@ -45,7 +46,8 @@ func ContainerDisk(imgPath string, config v1.Config) (v1.Image, error) { } // Modify the config file - cf.Architecture = ImageArchitecture + cf.Architecture = imgArch + cf.OS = ImageOS cf.Config = config img, err = mutate.ConfigFile(img, cf) @@ -55,3 +57,27 @@ func ContainerDisk(imgPath string, config v1.Config) (v1.Image, error) { return img, nil } + +func ContainerDiskIndex(images []v1.Image) (v1.ImageIndex, error) { + var indexAddendum []mutate.IndexAddendum + + for _, image := range images { + configFile, err := image.ConfigFile() + if err != nil { + return nil, err + } + + descriptor, err := partial.Descriptor(image) + if err != nil { + return nil, err + } + descriptor.Platform = configFile.Platform() + + indexAddendum = append(indexAddendum, mutate.IndexAddendum{ + Add: image, + Descriptor: *descriptor, + }) + } + + return mutate.AppendManifests(empty.Index, indexAddendum...), nil +} diff --git a/pkg/repository/repository.go b/pkg/repository/repository.go index d1b2c63f..6914505d 100644 --- a/pkg/repository/repository.go +++ b/pkg/repository/repository.go @@ -12,7 +12,9 @@ import ( "github.com/docker/distribution/registry/api/errcode" v2 "github.com/docker/distribution/registry/api/v2" "github.com/google/go-containerregistry/pkg/crane" + crname "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/pkg/errors" ) @@ -28,17 +30,20 @@ type ImageInfo struct { } type Repository interface { - ImageMetadata(imgRef string, insecure bool) (*ImageInfo, error) + ImageMetadata(imgRef, arch string, insecure bool) (*ImageInfo, error) PushImage(ctx context.Context, img v1.Image, imgRef string) error + PushImageIndex(ctx context.Context, img v1.ImageIndex, imgRef string) error CopyImage(ctx context.Context, srcRef, dstRef string, insecure bool) error } type RepositoryImpl struct { } -func (r RepositoryImpl) ImageMetadata(imgRef string, insecure bool) (imageInfo *ImageInfo, retErr error) { +func (r RepositoryImpl) ImageMetadata(imgRef, arch string, insecure bool) (imageInfo *ImageInfo, retErr error) { sys := &types.SystemContext{ OCIInsecureSkipTLSVerify: insecure, + ArchitectureChoice: arch, + OSChoice: "linux", } if insecure { sys.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue @@ -82,6 +87,15 @@ func (r RepositoryImpl) PushImage(ctx context.Context, img v1.Image, imgRef stri return crane.Push(img, imgRef, crane.WithContext(ctx)) } +func (r RepositoryImpl) PushImageIndex(ctx context.Context, imageIndex v1.ImageIndex, imageRef string) error { + ref, err := crname.ParseReference(imageRef) + if err != nil { + return err + } + + return remote.WriteIndex(ref, imageIndex, crane.GetOptions(crane.WithContext(ctx)).Remote...) +} + func (r RepositoryImpl) CopyImage(ctx context.Context, srcRef, dstRef string, insecure bool) error { options := []crane.Option{ crane.WithContext(ctx), From 7e7f181a046b362bd09f424926f2a8846e3148b9 Mon Sep 17 00:00:00 2001 From: Ben Oukhanov Date: Thu, 28 Mar 2024 11:51:29 +0000 Subject: [PATCH 2/2] fix: use Docker image index and manifests Empty image sets Docker manifest schema and not OCI. Image manifests included in OCI image index, and to keep consistency all of them should have Docker media types. Signed-off-by: Ben Oukhanov --- pkg/build/build.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkg/build/build.go b/pkg/build/build.go index 4f221194..297bda25 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -8,6 +8,7 @@ import ( "github.com/google/go-containerregistry/pkg/v1/mutate" "github.com/google/go-containerregistry/pkg/v1/partial" "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" ) const ( @@ -29,12 +30,12 @@ func ContainerDiskConfig(checksum string, envVariables map[string]string) v1.Con } func ContainerDisk(imgPath, imgArch string, config v1.Config) (v1.Image, error) { - img := empty.Image layer, err := tarball.LayerFromOpener(StreamLayerOpener(imgPath)) if err != nil { return nil, fmt.Errorf("error creating an image layer from disk: %v", err) } + img := mutate.MediaType(empty.Image, types.DockerManifestSchema2) img, err = mutate.AppendLayers(img, layer) if err != nil { return nil, fmt.Errorf("error appending the image layer: %v", err) @@ -79,5 +80,6 @@ func ContainerDiskIndex(images []v1.Image) (v1.ImageIndex, error) { }) } - return mutate.AppendManifests(empty.Index, indexAddendum...), nil + idx := mutate.IndexMediaType(empty.Index, types.DockerManifestList) + return mutate.AppendManifests(idx, indexAddendum...), nil }