Finalise interface between composer and image definitions

cmd/gen-manifests/main.go

@@ -21,6 +21,9 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/distroregistry"
 	"github.com/osbuild/osbuild-composer/internal/dnfjson"
+	"github.com/osbuild/osbuild-composer/internal/manifest"
+	"github.com/osbuild/osbuild-composer/internal/ostree"
+	"github.com/osbuild/osbuild-composer/internal/rhsm/facts"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 )
 
@@ -120,17 +123,22 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
 
 	options := distro.ImageOptions{Size: 0}
 	if cr.OSTree != nil {
-		options.OSTree = distro.OSTreeImageOptions{
+		options.OSTree = &ostree.ImageOptions{
 			URL:           cr.OSTree.URL,
 			ImageRef:      cr.OSTree.Ref,
 			FetchChecksum: cr.OSTree.Parent,
 			RHSM:          cr.OSTree.RHSM,
 		}
+	} else {
+		// use default OSTreeRef for image type
+		options.OSTree = &ostree.ImageOptions{
+			ImageRef: imgType.OSTreeRef(),
+		}
 	}
 
 	// add RHSM fact to detect changes
-	options.Facts = &distro.FactsImageOptions{
-		ApiType: "test-manifest",
+	options.Facts = &facts.ImageOptions{
+		APIType: facts.TEST_APITYPE,
 	}
 
 	job := func(msgq chan string) (err error) {
@@ -148,17 +156,13 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
 			bp = blueprint.Blueprint(*cr.Blueprint)
 		}
 
-		containerSpecs, err := resolveContainers(bp.Containers, archName)
+		manifest, _, err := imgType.Manifest(&bp, options, repos, seedArg)
 		if err != nil {
-			return fmt.Errorf("[%s] container resolution failed: %s", filename, err.Error())
-		}
-
-		if options.OSTree.ImageRef == "" {
-			// use default OSTreeRef for image type
-			options.OSTree.ImageRef = imgType.OSTreeRef()
+			err = fmt.Errorf("[%s] failed: %s", filename, err)
+			return
 		}
 
-		packageSpecs, err := depsolve(cacheDir, imgType, bp, options, repos, distribution, archName)
+		packageSpecs, err := depsolve(cacheDir, manifest.Content.PackageSets, distribution, archName)
 		if err != nil {
 			err = fmt.Errorf("[%s] depsolve failed: %s", filename, err.Error())
 			return
@@ -167,11 +171,21 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
 			err = fmt.Errorf("[%s] nil package specs", filename)
 			return
 		}
-		manifest, _, err := imgType.Manifest(cr.Blueprint.Customizations, options, repos, packageSpecs, containerSpecs, seedArg)
+
+		if cr.Blueprint != nil {
+			bp = blueprint.Blueprint(*cr.Blueprint)
+		}
+
+		containerSpecs, err := resolvePipelineContainers(manifest.Content.Containers, archName)
 		if err != nil {
-			err = fmt.Errorf("[%s] failed: %s", filename, err)
-			return
+			return fmt.Errorf("[%s] container resolution failed: %s", filename, err.Error())
+		}
+
+		mf, err := manifest.Serialize(packageSpecs, containerSpecs)
+		if err != nil {
+			return fmt.Errorf("[%s] manifest serialization failed: %s", filename, err.Error())
 		}
+
 		request := composeRequest{
 			Distro:       distribution.Name(),
 			Arch:         archName,
@@ -181,7 +195,7 @@ func makeManifestJob(name string, imgType distro.ImageType, cr composeRequest, d
 			Blueprint:    cr.Blueprint,
 			OSTree:       cr.OSTree,
 		}
-		err = save(manifest, packageSpecs, containerSpecs, request, path, filename)
+		err = save(mf, packageSpecs, containerSpecs, request, path, filename)
 		return
 	}
 	return job
@@ -243,20 +257,31 @@ func readRepos() DistroArchRepoMap {
 	return darm
 }
 
-func resolveContainers(containers []blueprint.Container, archName string) ([]container.Spec, error) {
+func resolveContainers(containers []container.SourceSpec, archName string) ([]container.Spec, error) {
 	resolver := container.NewResolver(archName)
 
 	for _, c := range containers {
-		resolver.Add(c.Source, c.Name, c.TLSVerify)
+		resolver.Add(c)
 	}
 
 	return resolver.Finish()
 }
 
-func depsolve(cacheDir string, imageType distro.ImageType, bp blueprint.Blueprint, options distro.ImageOptions, repos []rpmmd.RepoConfig, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) {
+func resolvePipelineContainers(containerSources map[string][]container.SourceSpec, archName string) (map[string][]container.Spec, error) {
+	containerSpecs := make(map[string][]container.Spec, len(containerSources))
+	for plName, sourceSpecs := range containerSources {
+		specs, err := resolveContainers(sourceSpecs, archName)
+		if err != nil {
+			return nil, err
+		}
+		containerSpecs[plName] = specs
+	}
+	return containerSpecs, nil
+}
+
+func depsolve(cacheDir string, packageSets map[string][]rpmmd.PackageSet, d distro.Distro, arch string) (map[string][]rpmmd.PackageSpec, error) {
 	solver := dnfjson.NewSolver(d.ModulePlatformID(), d.Releasever(), arch, d.Name(), cacheDir)
 	solver.SetDNFJSONPath("./dnf-json")
-	packageSets := imageType.PackageSets(bp, options, repos)
 	depsolvedSets := make(map[string][]rpmmd.PackageSpec)
 	for name, pkgSet := range packageSets {
 		res, err := solver.Depsolve(pkgSet)
@@ -268,15 +293,15 @@ func depsolve(cacheDir string, imageType distro.ImageType, bp blueprint.Blueprin
 	return depsolvedSets, nil
 }
 
-func save(manifest distro.Manifest, pkgs map[string][]rpmmd.PackageSpec, containers []container.Spec, cr composeRequest, path, filename string) error {
+func save(ms manifest.OSBuildManifest, pkgs map[string][]rpmmd.PackageSpec, containers map[string][]container.Spec, cr composeRequest, path, filename string) error {
 	data := struct {
 		ComposeRequest composeRequest                 `json:"compose-request"`
-		Manifest       distro.Manifest                `json:"manifest"`
+		Manifest       manifest.OSBuildManifest       `json:"manifest"`
 		RPMMD          map[string][]rpmmd.PackageSpec `json:"rpmmd"`
-		Containers     []container.Spec               `json:"containers,omitempty"`
+		Containers     map[string][]container.Spec    `json:"containers,omitempty"`
 		NoImageInfo    bool                           `json:"no-image-info"`
 	}{
-		cr, manifest, pkgs, containers, true,
+		cr, ms, pkgs, containers, true,
 	}
 	b, err := json.MarshalIndent(data, "", "  ")
 	if err != nil {

cmd/osbuild-dnf-json-tests/main_test.go

@@ -14,6 +14,8 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	rhel "github.com/osbuild/osbuild-composer/internal/distro/rhel8"
 	"github.com/osbuild/osbuild-composer/internal/dnfjson"
+	"github.com/osbuild/osbuild-composer/internal/ostree"
+	"github.com/osbuild/osbuild-composer/internal/platform"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
 )
 
@@ -43,18 +45,28 @@ func TestCrossArchDepsolve(t *testing.T) {
 				t.Run(imgTypeStr, func(t *testing.T) {
 					imgType, err := arch.GetImageType(imgTypeStr)
 					require.NoError(t, err)
-
-					packages := imgType.PackageSets(blueprint.Blueprint{},
+					// set up bare minimum args for image type
+					var customizations *blueprint.Customizations
+					if imgTypeStr == "edge-simplified-installer" {
+						customizations = &blueprint.Customizations{
+							InstallationDevice: "/dev/null",
+						}
+					}
+					manifest, _, err := imgType.Manifest(
+						&blueprint.Blueprint{
+							Customizations: customizations,
+						},
 						distro.ImageOptions{
-							OSTree: distro.OSTreeImageOptions{
+							OSTree: &ostree.ImageOptions{
 								URL:           "foo",
 								ImageRef:      "bar",
 								FetchChecksum: "baz",
 							},
 						},
-						repos[archStr])
+						repos[archStr], 0)
+					assert.NoError(t, err)
 
-					for _, set := range packages {
+					for _, set := range manifest.Content.PackageSets {
 						_, err = solver.Depsolve(set)
 						assert.NoError(t, err)
 					}
@@ -75,21 +87,23 @@ func TestDepsolvePackageSets(t *testing.T) {
 
 	// Set up temporary directory for rpm/dnf cache
 	dir := t.TempDir()
-	solver := dnfjson.NewSolver(cs9.ModulePlatformID(), cs9.Releasever(), distro.X86_64ArchName, cs9.Name(), dir)
+	solver := dnfjson.NewSolver(cs9.ModulePlatformID(), cs9.Releasever(), platform.ARCH_X86_64.String(), cs9.Name(), dir)
 
 	repos, err := rpmmd.LoadRepositories([]string{repoDir}, cs9.Name())
 	require.NoErrorf(t, err, "Failed to LoadRepositories %v", cs9.Name())
-	x86Repos, ok := repos[distro.X86_64ArchName]
-	require.Truef(t, ok, "failed to get %q repos for %q", distro.X86_64ArchName, cs9.Name())
+	x86Repos, ok := repos[platform.ARCH_X86_64.String()]
+	require.Truef(t, ok, "failed to get %q repos for %q", platform.ARCH_X86_64.String(), cs9.Name())
 
-	x86Arch, err := cs9.GetArch(distro.X86_64ArchName)
-	require.Nilf(t, err, "failed to get %q arch of %q distro", distro.X86_64ArchName, cs9.Name())
+	x86Arch, err := cs9.GetArch(platform.ARCH_X86_64.String())
+	require.Nilf(t, err, "failed to get %q arch of %q distro", platform.ARCH_X86_64.String(), cs9.Name())
 
 	qcow2ImageTypeName := "qcow2"
 	qcow2Image, err := x86Arch.GetImageType(qcow2ImageTypeName)
-	require.Nilf(t, err, "failed to get %q image type of %q/%q distro/arch", qcow2ImageTypeName, cs9.Name(), distro.X86_64ArchName)
+	require.Nilf(t, err, "failed to get %q image type of %q/%q distro/arch", qcow2ImageTypeName, cs9.Name(), platform.ARCH_X86_64.String())
 
-	imagePkgSets := qcow2Image.PackageSets(blueprint.Blueprint{Packages: []blueprint.Package{{Name: "bind"}}}, distro.ImageOptions{}, x86Repos)
+	manifestSource, _, err := qcow2Image.Manifest(&blueprint.Blueprint{Packages: []blueprint.Package{{Name: "bind"}}}, distro.ImageOptions{}, x86Repos, 0)
+	require.Nilf(t, err, "failed to initialise manifest for %q image type of %q/%q distro/arch", qcow2ImageTypeName, cs9.Name(), platform.ARCH_X86_64.String())
+	imagePkgSets := manifestSource.Content.PackageSets
 
 	gotPackageSpecsSets := make(map[string][]rpmmd.PackageSpec, len(imagePkgSets))
 	for name, pkgSet := range imagePkgSets {

cmd/osbuild-image-tests/main_test.go

@@ -29,7 +29,7 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/boot/openstacktest"
 	"github.com/osbuild/osbuild-composer/internal/boot/vmwaretest"
 	"github.com/osbuild/osbuild-composer/internal/common"
-	"github.com/osbuild/osbuild-composer/internal/distro"
+	"github.com/osbuild/osbuild-composer/internal/platform"
 	"github.com/osbuild/osbuild-composer/internal/test"
 )
 
@@ -410,7 +410,7 @@ func testBootUsingOpenStack(t *testing.T, imagePath string) {
 	currentArch := common.CurrentArch()
 
 	// skip on aarch64 because we don't have aarch64 openstack or kvm machines
-	if currentArch == distro.Aarch64ArchName {
+	if currentArch == platform.ARCH_AARCH64.String() {
 		t.Skip("Openstack boot test is skipped on aarch64.")
 		// if no credentials are given, fall back to qemu
 	} else if (creds == gophercloud.AuthOptions{}) {

cmd/osbuild-package-sets/main.go

@@ -11,6 +11,7 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/distroregistry"
+	"github.com/osbuild/osbuild-composer/internal/ostree"
 )
 
 func main() {
@@ -47,12 +48,16 @@ func main() {
 
 	encoder := json.NewEncoder(os.Stdout)
 	encoder.SetIndent("", "  ")
-	pkgset := image.PackageSets(blueprint.Blueprint{}, distro.ImageOptions{
-		OSTree: distro.OSTreeImageOptions{
+	options := distro.ImageOptions{
+		OSTree: &ostree.ImageOptions{
 			URL:           "foo",
 			ImageRef:      "bar",
 			FetchChecksum: "baz",
 		},
-	}, nil)
-	_ = encoder.Encode(pkgset)
+	}
+	manifest, _, err := image.Manifest(&blueprint.Blueprint{}, options, nil, 0)
+	if err != nil {
+		panic(err)
+	}
+	_ = encoder.Encode(manifest.Content.PackageSets)
 }

cmd/osbuild-pipeline/main.go

@@ -13,6 +13,7 @@ import (
 	"github.com/osbuild/osbuild-composer/internal/distro"
 	"github.com/osbuild/osbuild-composer/internal/distroregistry"
 	"github.com/osbuild/osbuild-composer/internal/dnfjson"
+	"github.com/osbuild/osbuild-composer/internal/ostree"
 
 	"github.com/osbuild/osbuild-composer/internal/blueprint"
 	"github.com/osbuild/osbuild-composer/internal/rpmmd"
@@ -64,15 +65,15 @@ func findDnfJsonBin() string {
 	panic(fmt.Sprintf("could not find 'dnf-json' in any of the known paths: %+v", locations))
 }
 
-func resolveContainers(bp blueprint.Blueprint, archName string) ([]container.Spec, error) {
-	if len(bp.Containers) == 0 {
+func resolveContainers(sourceSpecs []container.SourceSpec, archName string) ([]container.Spec, error) {
+	if len(sourceSpecs) == 0 {
 		return nil, nil
 	}
 
 	resolver := container.NewResolver(archName)
 
-	for _, c := range bp.Containers {
-		resolver.Add(c.Source, c.Name, c.TLSVerify)
+	for _, c := range sourceSpecs {
+		resolver.Add(c)
 	}
 
 	return resolver.Finish()
@@ -179,7 +180,7 @@ func main() {
 
 	options := distro.ImageOptions{
 		Size: imageType.Size(0),
-		OSTree: distro.OSTreeImageOptions{
+		OSTree: &ostree.ImageOptions{
 			ImageRef:      composeRequest.OSTree.Ref,
 			FetchChecksum: composeRequest.OSTree.Parent,
 			URL:           composeRequest.OSTree.URL,
@@ -199,46 +200,47 @@ func main() {
 	// let the cache grow to fit much more repository metadata than we usually allow
 	solver.SetMaxCacheSize(3 * 1024 * 1024 * 1024)
 
-	packageSets := imageType.PackageSets(composeRequest.Blueprint, options, repos)
-	depsolvedSets := make(map[string][]rpmmd.PackageSpec)
+	manifest, _, err := imageType.Manifest(&composeRequest.Blueprint, options, repos, seedArg)
+	if err != nil {
+		panic(err.Error())
+	}
 
-	for name, pkgSet := range packageSets {
+	depsolvedSets := make(map[string][]rpmmd.PackageSpec)
+	for name, pkgSet := range manifest.Content.PackageSets {
 		res, err := solver.Depsolve(pkgSet)
 		if err != nil {
 			panic("Could not depsolve: " + err.Error())
 		}
 		depsolvedSets[name] = res
 	}
 
+	containers := make(map[string][]container.Spec, len(manifest.Content.Containers))
+	for name, sourceSpecs := range manifest.Content.Containers {
+		containerSpecs, err := resolveContainers(sourceSpecs, arch.Name())
+		if err != nil {
+			panic("Could not resolve containers: " + err.Error())
+		}
+		containers[name] = containerSpecs
+	}
+
 	var bytes []byte
 	if rpmmdArg {
 		bytes, err = json.Marshal(depsolvedSets)
 		if err != nil {
 			panic(err)
 		}
 	} else {
-
-		containerSpecs, err := resolveContainers(composeRequest.Blueprint, arch.Name())
-		if err != nil {
-			panic("Could not resolve containers: " + err.Error())
-		}
-
 		if composeRequest.OSTree.Ref == "" {
 			// use default OSTreeRef for image type
 			composeRequest.OSTree.Ref = imageType.OSTreeRef()
 		}
 
-		manifest, _, err := imageType.Manifest(composeRequest.Blueprint.Customizations,
-			options,
-			repos,
-			depsolvedSets,
-			containerSpecs,
-			seedArg)
+		ms, err := manifest.Serialize(depsolvedSets, containers)
 		if err != nil {
 			panic(err.Error())
 		}
 
-		bytes, err = json.Marshal(manifest)
+		bytes, err = json.Marshal(ms)
 		if err != nil {
 			panic(err)
 		}

cmd/osbuild-playground/playground.go

@@ -48,7 +48,7 @@ func RunPlayground(img image.ImageKind, d distro.Distro, arch distro.Arch, repos
 		fmt.Fprintf(os.Stderr, "could not clean dnf cache: %s", err.Error())
 	}
 
-	bytes, err := manifest.Serialize(packageSpecs)
+	bytes, err := manifest.Serialize(packageSpecs, nil)
 	if err != nil {
 		panic("failed to serialize manifest: " + err.Error())
 	}