Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat: build and push image index #81

Merged
merged 2 commits into from
Mar 28, 2024
Merged

feat: build and push image index #81

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
4cb0967
feat: build and push image index
codingben Feb 28, 2024
7e7f181
fix: use Docker image index and manifests
codingben Mar 28, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions artifacts/centos/centos.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) {
SHA256Sum: checksum,
DownloadURL: baseURL + candidate,
AdditionalUniqueTags: getAdditionalTags(c.Version, c.Variant, candidate),
ImageArchitecture: "amd64",
}, nil
}

Expand Down
12 changes: 8 additions & 4 deletions artifacts/centos/centos_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ var _ = Describe("Centos", func() {
DownloadURL: "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.4.2105-20210603.0.x86_64.qcow2",
Compression: "",
AdditionalUniqueTags: []string{"8.4.2105-20210603.0", "8.4.2105"},
ImageArchitecture: "amd64",
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I would export this as variable

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It could be a const, yes.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks, I'll take care of it in another PR where we'll add also arm64 images.

},
map[string]string{
"TEST_ENV_VAR": "test-value",
Expand All @@ -50,6 +51,7 @@ var _ = Describe("Centos", func() {
DownloadURL: "https://cloud.centos.org/centos/8/x86_64/images/CentOS-8-GenericCloud-8.3.2011-20201204.2.x86_64.qcow2",
Compression: "",
AdditionalUniqueTags: []string{"8.3.2011-20201204.2", "8.3.2011"},
ImageArchitecture: "amd64",
},
map[string]string{
"TEST_ENV_VAR": "test-value",
Expand All @@ -68,8 +70,9 @@ var _ = Describe("Centos", func() {
),
Entry("centos:7-2009", "7-2009", "testdata/centos7.checksum",
&api.ArtifactDetails{
SHA256Sum: "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3",
DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2",
SHA256Sum: "e38bab0475cc6d004d2e17015969c659e5a308111851b0e2715e84646035bdd3",
DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-2009.qcow2",
ImageArchitecture: "amd64",
},
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
Expand All @@ -90,8 +93,9 @@ var _ = Describe("Centos", func() {
),
Entry("centos:7-1809", "7-1809", "testdata/centos7.checksum",
&api.ArtifactDetails{
SHA256Sum: "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2",
DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2",
SHA256Sum: "42c062df8a8c36991ec0282009dd52ac488461a3f7ee114fc21a765bfc2671c2",
DownloadURL: "https://cloud.centos.org/centos/7/images/CentOS-7-x86_64-GenericCloud-1809.qcow2",
ImageArchitecture: "amd64",
},
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
Expand Down
1 change: 1 addition & 0 deletions artifacts/centosstream/centos-stream.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,7 @@ func (c *centos) Inspect() (*api.ArtifactDetails, error) {
SHA256Sum: checksum,
DownloadURL: baseURL + candidate,
AdditionalUniqueTags: additionalTags,
ImageArchitecture: "amd64",
}, nil
}

Expand Down
2 changes: 2 additions & 0 deletions artifacts/centosstream/centos-stream_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ var _ = Describe("CentosStream", func() {
SHA256Sum: "8e22e67687b81e38c7212fc30c47cb24cbc4935c0f2459ed139f498397d1e7cd",
DownloadURL: "https://cloud.centos.org/centos/8-stream/x86_64/images/CentOS-Stream-GenericCloud-8-20210603.0.x86_64.qcow2",
AdditionalUniqueTags: []string{"8-20210603.0"},
ImageArchitecture: "amd64",
},
&docs.UserData{
Username: "centos",
Expand All @@ -54,6 +55,7 @@ var _ = Describe("CentosStream", func() {
SHA256Sum: "bcebdc00511d6e18782732570056cfbc7cba318302748bfc8f66be9c0db68142",
DownloadURL: "https://cloud.centos.org/centos/9-stream/x86_64/images/CentOS-Stream-GenericCloud-9-20211222.0.x86_64.qcow2",
AdditionalUniqueTags: []string{"9-20211222.0"},
ImageArchitecture: "amd64",
},
&docs.UserData{
Username: "cloud-user",
Expand Down
21 changes: 12 additions & 9 deletions artifacts/fedora/fedora.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -77,6 +77,7 @@ func (f *fedora) Inspect() (*api.ArtifactDetails, error) {
SHA256Sum: release.Sha256,
DownloadURL: release.Link,
AdditionalUniqueTags: []string{additionalTag},
ImageArchitecture: "amd64",
}, nil
}
}
Expand Down Expand Up @@ -105,23 +106,25 @@ func (f *fedora) Tests() []api.ArtifactTest {
}
}

func (f *fedoraGatherer) Gather() ([]api.Artifact, error) {
func (f *fedoraGatherer) Gather() ([][]api.Artifact, error) {
releases, err := getReleases(f.getter)
if err != nil {
return nil, fmt.Errorf("error getting releases: %v", err)
}

artifacts := []api.Artifact{}
artifacts := [][]api.Artifact{}
for i, release := range releases {
if f.releaseMatches(&releases[i]) {
artifacts = append(artifacts,
New(
release.Version,
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
common.DefaultPreferenceEnv: "fedora",
},
),
[]api.Artifact{
New(
release.Version,
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
common.DefaultPreferenceEnv: "fedora",
},
),
},
)
}
}
Expand Down
40 changes: 23 additions & 17 deletions artifacts/fedora/fedora_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ var _ = Describe("Fedora", func() {
SHA256Sum: "fe84502779b3477284a8d4c86731f642ca10dd3984d2b5eccdf82630a9ca2de6",
DownloadURL: "https://download.fedoraproject.org/pub/fedora/linux/releases/35/Cloud/x86_64/images/Fedora-Cloud-Base-35-1.2.x86_64.qcow2", //nolint:lll
AdditionalUniqueTags: []string{"35-1.2"},
ImageArchitecture: "amd64",
},
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
Expand All @@ -51,6 +52,7 @@ var _ = Describe("Fedora", func() {
SHA256Sum: "b9b621b26725ba95442d9a56cbaa054784e0779a9522ec6eafff07c6e6f717ea",
DownloadURL: "https://download.fedoraproject.org/pub/fedora/linux/releases/34/Cloud/x86_64/images/Fedora-Cloud-Base-34-1.2.x86_64.qcow2", //nolint:lll
AdditionalUniqueTags: []string{"34-1.2"},
ImageArchitecture: "amd64",
},
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
Expand All @@ -72,25 +74,29 @@ var _ = Describe("Fedora", func() {
)

It("Gather should be able to parse releases files", func() {
artifacts := []api.Artifact{
&fedora{
Version: "36",
Arch: "x86_64",
Variant: "Cloud",
getter: &http.HTTPGetter{},
EnvVariables: map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
common.DefaultPreferenceEnv: "fedora",
artifacts := [][]api.Artifact{
{
&fedora{
Version: "36",
Arch: "x86_64",
Variant: "Cloud",
getter: &http.HTTPGetter{},
EnvVariables: map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
common.DefaultPreferenceEnv: "fedora",
},
},
},
&fedora{
Version: "35",
Arch: "x86_64",
Variant: "Cloud",
getter: &http.HTTPGetter{},
EnvVariables: map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
common.DefaultPreferenceEnv: "fedora",
{
&fedora{
Version: "35",
Arch: "x86_64",
Variant: "Cloud",
getter: &http.HTTPGetter{},
EnvVariables: map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
common.DefaultPreferenceEnv: "fedora",
},
},
},
}
Expand Down
7 changes: 4 additions & 3 deletions artifacts/ubuntu/ubuntu.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,9 +52,10 @@ func (u *ubuntu) Inspect() (*api.ArtifactDetails, error) {
}
if checksum, exists := checksums[u.Variant]; exists {
return &api.ArtifactDetails{
SHA256Sum: checksum,
DownloadURL: baseURL + u.Variant,
Compression: u.Compression,
SHA256Sum: checksum,
DownloadURL: baseURL + u.Variant,
Compression: u.Compression,
ImageArchitecture: "amd64",
}, nil
}
return nil, fmt.Errorf("file %q does not exist in the SHA256SUMS file: %v", u.Variant, err)
Expand Down
5 changes: 3 additions & 2 deletions artifacts/ubuntu/ubuntu_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,9 @@ var _ = Describe("Ubuntu", func() {
},
Entry("ubuntu:22.04", "22.04", "testdata/SHA256SUM",
&api.ArtifactDetails{
SHA256Sum: "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd",
DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img",
SHA256Sum: "de5e632e17b8965f2baf4ea6d2b824788e154d9a65df4fd419ec4019898e15cd",
DownloadURL: "https://cloud-images.ubuntu.com/releases/22.04/release/ubuntu-22.04-server-cloudimg-amd64.img",
ImageArchitecture: "amd64",
},
map[string]string{
common.DefaultInstancetypeEnv: "u1.small",
Expand Down
98 changes: 53 additions & 45 deletions cmd/medius/common/registry.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,77 +16,81 @@ import (
)

type Entry struct {
Artifact api.Artifact
Artifacts []api.Artifact
UseForDocs bool
UseForLatest bool
SkipWhenNotFocused bool
}

var staticRegistry = []Entry{
{
Artifact: centos.New("8.4", nil),
Artifacts: []api.Artifact{
centos.New("8.4", nil),
},
UseForDocs: false,
},
{
Artifact: centos.New(
"7-2009",
defaultEnvVariables("u1.small", "centos.7"),
),
Artifacts: []api.Artifact{
centos.New("7-2009", defaultEnvVariables("u1.small", "centos.7")),
},
UseForDocs: true,
},
{
Artifact: centosstream.New(
"9",
&docs.UserData{
Username: "cloud-user",
},
defaultEnvVariables("u1.small", "centos.stream9"),
),
Artifacts: []api.Artifact{
centosstream.New("9", &docs.UserData{Username: "cloud-user"}, defaultEnvVariables("u1.small", "centos.stream9")),
},
UseForDocs: true,
},
{
Artifact: centosstream.New(
"8",
&docs.UserData{
Username: "centos",
},
defaultEnvVariables("u1.small", "centos.stream8"),
),
Artifacts: []api.Artifact{
centosstream.New("8", &docs.UserData{Username: "centos"}, defaultEnvVariables("u1.small", "centos.stream8")),
},
UseForDocs: false,
},
{
Artifact: ubuntu.New(
"22.04",
defaultEnvVariables("u1.small", "ubuntu"),
),
Artifacts: []api.Artifact{
ubuntu.New("22.04", defaultEnvVariables("u1.small", "ubuntu")),
},
UseForDocs: true,
},
{
Artifact: ubuntu.New(
"20.04",
defaultEnvVariables("u1.small", "ubuntu"),
),
Artifacts: []api.Artifact{
ubuntu.New("20.04", defaultEnvVariables("u1.small", "ubuntu")),
},
UseForDocs: false,
},
{
Artifact: ubuntu.New(
"18.04",
defaultEnvVariables("u1.small", "ubuntu"),
),
Artifacts: []api.Artifact{
ubuntu.New("18.04", defaultEnvVariables("u1.small", "ubuntu")),
},
UseForDocs: false,
},
// for testing only
{
Artifact: generic.New(
&api.ArtifactDetails{
SHA256Sum: "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1",
DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img",
},
&api.Metadata{
Name: "cirros",
Version: "6.1",
},
),
Artifacts: []api.Artifact{
codingben marked this conversation as resolved.
Show resolved Hide resolved
generic.New(
&api.ArtifactDetails{
SHA256Sum: "cc704ab14342c1c8a8d91b66a7fc611d921c8b8f1aaf4695f9d6463d913fa8d1",
DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-x86_64-disk.img",
ImageArchitecture: "amd64",
},
&api.Metadata{
Name: "cirros",
Version: "6.1",
},
),
generic.New(
&api.ArtifactDetails{
SHA256Sum: "db9420c481c11dee17860aa46fb1a3efa05fa4fb152726d6344e24da03cb0ccf",
DownloadURL: "https://download.cirros-cloud.net/0.6.1/cirros-0.6.1-aarch64-disk.img",
ImageArchitecture: "arm64",
},
&api.Metadata{
Name: "cirros",
Version: "6.1",
},
),
},
SkipWhenNotFocused: true,
UseForDocs: false,
},
Expand All @@ -100,7 +104,7 @@ func gatherArtifacts(registry *[]Entry, gatherers []api.ArtifactsGatherer) {
} else {
for i := range artifacts {
*registry = append(*registry, Entry{
Artifact: artifacts[i],
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

That still looks wrong. The gatherer needs to return artifacts with multiple architectures and this loop needs to append them to the registry. (Artifacts: artifacts[i])

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think you reviewed old revision. This is how it looks like today:

func gatherArtifacts(registry *[]Entry, gatherers []api.ArtifactsGatherer) {
	for _, gatherer := range gatherers {
		artifacts, err := gatherer.Gather()
		if err != nil {
			logrus.Warn("Failed to gather artifacts", err)
		} else {
			for i := range artifacts {
				*registry = append(*registry, Entry{
					Artifacts:    []api.Artifact{artifacts[0][i]},
					UseForDocs:   i == 0,
					UseForLatest: i == 0,
				})
			}
		}
	}
}

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I reviewed the "left" side. The new is still wrong, since like this it probably run out of range. i is a range over the outer slice and you use it to access the inner slice. The artifacts returned by the gatherer should be [][]api.Artifact, then you can set each outer slice item as []api.Artifact in the loop below with artifacts[i]. IMO the gatherer should construct the multi-arch Artifact already. Like this you end up with a separate artifact for each architecture and version.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah got it, thanks Felix! :)

Artifacts: artifacts[i],
UseForDocs: i == 0,
UseForLatest: i == 0,
})
Expand Down Expand Up @@ -131,12 +135,16 @@ func ShouldSkip(focus string, entry *Entry) bool {
return entry.SkipWhenNotFocused
}

if len(entry.Artifacts) == 0 {
return true
}

focusSplit := strings.Split(focus, ":")
wildcardFocus := len(focusSplit) == 2 && focusSplit[1] == "*"

if wildcardFocus {
return focusSplit[0] != entry.Artifact.Metadata().Name
return focusSplit[0] != entry.Artifacts[0].Metadata().Name
}

return focus != entry.Artifact.Metadata().Describe()
return focus != entry.Artifacts[0].Metadata().Describe()
}
8 changes: 5 additions & 3 deletions cmd/medius/docs/docs.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,12 @@ func run(options *common.Options) error {
}

focusMatched = true
log := common.Logger(p.Artifact)
name := p.Artifact.Metadata().Name

description, err := createDescription(p.Artifact, options.PublishDocsOptions.Registry)
artifact := p.Artifacts[0]
log := common.Logger(artifact)
name := artifact.Metadata().Name

description, err := createDescription(artifact, options.PublishDocsOptions.Registry)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should add the supported architectures into the description. Maybe in a follow up.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, let's take care of docs later.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please create an issue for that.

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Done: #83

if err != nil {
success = false
log.Errorf("error marshaling example for %q: %v", name, err)
Expand Down
5 changes: 3 additions & 2 deletions cmd/medius/images/common.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,15 +44,16 @@ func spawnWorkers(ctx context.Context, o *common.Options,
go func() {
defer wg.Done()
for e := range jobChan {
artifact := e.Artifacts[0]
result, workerErr := fn(e)
if result != nil {
resultsChan <- workerResult{
Key: e.Artifact.Metadata().Describe(),
Key: artifact.Metadata().Describe(),
Value: *result,
}
}
if workerErr != nil && !errors.Is(workerErr, context.Canceled) {
common.Logger(e.Artifact).Error(workerErr)
common.Logger(artifact).Error(workerErr)
errChan <- workerErr
}
if errors.Is(ctx.Err(), context.Canceled) {
Expand Down
Loading