[9.1] distro/rhel9: edge images default to LVM

.gitlab-ci.yml

@@ -379,7 +379,10 @@ OSTree simplified installer:
           - rhos-01/rhel-8.7-nightly-x86_64
           - rhos-01/centos-stream-8-x86_64
           - rhos-01/rhel-9.1-nightly-x86_64
-          - rhos-01/centos-stream-9-x86_64
+          # this is disabled because of two BZs in cs9
+          # rhbz#2124735 and rhbz#2108646 preventing 
+          # the system to install and boot properly
+          # - rhos-01/centos-stream-9-x86_64
 
 OSTree raw image:
   stage: test

Schutzfile

@@ -180,34 +180,34 @@
           {
             "title": "RHEL-8-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-x86_64-baseos-n8.7-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-x86_64-baseos-n8.7-20220912"
           },
           {
             "title": "RHEL-8-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-x86_64-appstream-n8.7-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-x86_64-appstream-n8.7-20220912"
           },
           {
             "title": "RHEL-8-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-x86_64-crb-n8.7-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-x86_64-crb-n8.7-20220912"
           }
         ],
         "aarch64": [
           {
             "title": "RHEL-8-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-aarch64-baseos-n8.7-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-aarch64-baseos-n8.7-20220912"
           },
           {
             "title": "RHEL-8-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-aarch64-appstream-n8.7-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-aarch64-appstream-n8.7-20220912"
           },
           {
             "title": "RHEL-8-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-aarch64-crb-n8.7-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el8/el8-aarch64-crb-n8.7-20220912"
           }
         ]
       }
@@ -233,34 +233,34 @@
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-baseos-n9.1-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-baseos-n9.1-20220913"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-appstream-n9.1-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-appstream-n9.1-20220913"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-crb-n9.1-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-x86_64-crb-n9.1-20220913"
           }
         ],
         "aarch64": [
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-BaseOS",
             "name": "baseos",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-baseos-n9.1-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-baseos-n9.1-20220913"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-AppStream",
             "name": "appstream",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-appstream-n9.1-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-appstream-n9.1-20220913"
           },
           {
             "title": "RHEL-9-RPMREPO-NIGHTLY-CRB",
             "name": "crb",
-            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-crb-n9.1-20220715"
+            "baseurl": "https://rpmrepo.osbuild.org/v2/mirror/rhvpn/el9/el9-aarch64-crb-n9.1-20220913"
           }
         ]
       }

internal/distro/rhel9/package_sets.go

@@ -946,6 +946,7 @@ func edgeCommitPackageSet(t *imageType) rpmmd.PackageSet {
 			"ima-evm-utils",
 			"audit",
 			"podman",
+			"containernetworking-plugins", // required for cni networks but not a hard dependency of podman >= 4.2.0 (rhbz#2123210)
 			"container-selinux",
 			"skopeo",
 			"criu",

internal/distro/rhel9/partition_tables.go

@@ -221,13 +221,23 @@ var edgeBasePartitionTables = distro.BasePartitionTableMap{
 						Policy:           "{}",
 						RemovePassphrase: true,
 					},
-					Payload: &disk.Filesystem{
-						Type:         "xfs",
-						Label:        "root",
-						Mountpoint:   "/",
-						FSTabOptions: "defaults",
-						FSTabFreq:    0,
-						FSTabPassNo:  0,
+					Payload: &disk.LVMVolumeGroup{
+						Name:        "rootvg",
+						Description: "built with lvm2 and osbuild",
+						LogicalVolumes: []disk.LVMLogicalVolume{
+							{
+								Size: 9 * 1024 * 1024 * 1024, // 9 GB
+								Name: "rootlv",
+								Payload: &disk.Filesystem{
+									Type:         "xfs",
+									Label:        "root",
+									Mountpoint:   "/",
+									FSTabOptions: "defaults",
+									FSTabFreq:    0,
+									FSTabPassNo:  0,
+								},
+							},
+						},
 					},
 				},
 			},
@@ -281,13 +291,23 @@ var edgeBasePartitionTables = distro.BasePartitionTableMap{
 						Policy:           "{}",
 						RemovePassphrase: true,
 					},
-					Payload: &disk.Filesystem{
-						Type:         "xfs",
-						Label:        "root",
-						Mountpoint:   "/",
-						FSTabOptions: "defaults",
-						FSTabFreq:    0,
-						FSTabPassNo:  0,
+					Payload: &disk.LVMVolumeGroup{
+						Name:        "rootvg",
+						Description: "built with lvm2 and osbuild",
+						LogicalVolumes: []disk.LVMLogicalVolume{
+							{
+								Size: 9 * 1024 * 1024 * 1024, // 9 GB
+								Name: "rootlv",
+								Payload: &disk.Filesystem{
+									Type:         "xfs",
+									Label:        "root",
+									Mountpoint:   "/",
+									FSTabOptions: "defaults",
+									FSTabFreq:    0,
+									FSTabPassNo:  0,
+								},
+							},
+						},
 					},
 				},
 			},

internal/osbuild/common_test.go

@@ -2,7 +2,8 @@ package osbuild
 
 import "github.com/osbuild/osbuild-composer/internal/disk"
 
-// This is a copy of `internal/disk/disk_test.go`:
+// This is a copy of `internal/disk/disk_test.go`
+// (but ours has one more entry: "luks+lvm+clevisBind"):
 var testPartitionTables = map[string]disk.PartitionTable{
 
 	"plain": {
@@ -202,6 +203,83 @@ var testPartitionTables = map[string]disk.PartitionTable{
 		},
 	},
 
+	"luks+lvm+clevisBind": {
+		UUID: "D209C89E-EA5E-4FBD-B161-B461CCE297E0",
+		Type: "gpt",
+		Partitions: []disk.Partition{
+			{
+				Size:     1048576, // 1MB
+				Bootable: true,
+				Type:     disk.BIOSBootPartitionGUID,
+				UUID:     disk.BIOSBootPartitionUUID,
+			},
+			{
+				Size: 209715200, // 200 MB
+				Type: disk.EFISystemPartitionGUID,
+				UUID: disk.EFISystemPartitionUUID,
+				Payload: &disk.Filesystem{
+					Type:         "vfat",
+					UUID:         disk.EFIFilesystemUUID,
+					Mountpoint:   "/boot/efi",
+					Label:        "EFI-SYSTEM",
+					FSTabOptions: "defaults,uid=0,gid=0,umask=077,shortname=winnt",
+					FSTabFreq:    0,
+					FSTabPassNo:  2,
+				},
+			},
+			{
+				Size: 1024000, // 500 MB
+				Type: disk.FilesystemDataGUID,
+				UUID: disk.FilesystemDataUUID,
+				Payload: &disk.Filesystem{
+					Type:         "xfs",
+					Mountpoint:   "/boot",
+					Label:        "boot",
+					FSTabOptions: "defaults",
+					FSTabFreq:    0,
+					FSTabPassNo:  0,
+				},
+			},
+			{
+				Type: disk.FilesystemDataGUID,
+				UUID: disk.RootPartitionUUID,
+				Payload: &disk.LUKSContainer{
+					Label:      "crypt_root",
+					Cipher:     "cipher_null",
+					Passphrase: "osbuild",
+					PBKDF: disk.Argon2id{
+						Memory:      32,
+						Iterations:  4,
+						Parallelism: 1,
+					},
+					Clevis: &disk.ClevisBind{
+						Pin:              "null",
+						Policy:           "{}",
+						RemovePassphrase: true,
+					},
+					Payload: &disk.LVMVolumeGroup{
+						Name:        "rootvg",
+						Description: "built with lvm2 and osbuild",
+						LogicalVolumes: []disk.LVMLogicalVolume{
+							{
+								Size: 9 * 1024 * 1024 * 1024, // 9 GB
+								Name: "rootlv",
+								Payload: &disk.Filesystem{
+									Type:         "xfs",
+									Label:        "root",
+									Mountpoint:   "/",
+									FSTabOptions: "defaults",
+									FSTabFreq:    0,
+									FSTabPassNo:  0,
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+	},
+
 	"btrfs": {
 		UUID: "D209C89E-EA5E-4FBD-B161-B461CCE297E0",
 		Type: "gpt",

internal/osbuild/device.go

@@ -97,6 +97,7 @@ func GenDeviceCreationStages(pt *disk.PartitionTable, filename string) []*Stage
 
 func GenDeviceFinishStages(pt *disk.PartitionTable, filename string) []*Stage {
 	stages := make([]*Stage, 0)
+	removeKeyStages := make([]*Stage, 0)
 
 	genStages := func(e disk.Entity, path []disk.Entity) error {
 
@@ -111,7 +112,7 @@ func GenDeviceFinishStages(pt *disk.PartitionTable, filename string) []*Stage {
 
 			if ent.Clevis != nil {
 				if ent.Clevis.RemovePassphrase {
-					stages = append(stages, NewLUKS2RemoveKeyStage(&LUKS2RemoveKeyStageOptions{
+					removeKeyStages = append(removeKeyStages, NewLUKS2RemoveKeyStage(&LUKS2RemoveKeyStageOptions{
 						Passphrase: ent.Passphrase,
 					}, stageDevices))
 				}
@@ -138,6 +139,10 @@ func GenDeviceFinishStages(pt *disk.PartitionTable, filename string) []*Stage {
 	}
 
 	_ = pt.ForEachEntity(genStages)
+	// Ensure that "org.osbuild.luks2.remove-key" stages are done after
+	// "org.osbuild.lvm2.metadata" stages, we cannot open a device if its
+	// password has changed
+	stages = append(stages, removeKeyStages...)
 	return stages
 }
 

internal/osbuild/device_test.go

@@ -114,3 +114,28 @@ func TestGenDeviceFinishStages(t *testing.T) {
 	assert.True(ok, "Need LVM2MetadataStageOptions for org.osbuild.lvm2.metadata")
 	assert.Equal("root", opts.VGName)
 }
+
+func TestGenDeviceFinishStagesOrderWithLVMClevisBind(t *testing.T) {
+	assert := assert.New(t)
+
+	// math/rand is good enough in this case
+	/* #nosec G404 */
+	rng := rand.New(rand.NewSource(13))
+
+	luks_lvm := testPartitionTables["luks+lvm+clevisBind"]
+
+	pt, err := disk.NewPartitionTable(&luks_lvm, []blueprint.FilesystemCustomization{}, 0, false, rng)
+	assert.NoError(err)
+
+	stages := GenDeviceFinishStages(pt, "image.raw")
+
+	// we should have two stages
+	assert.Equal(2, len(stages))
+	lvm := stages[0]
+	luks := stages[1]
+
+	// the first one should be "org.osbuild.lvm2.metadata"
+	assert.Equal("org.osbuild.lvm2.metadata", lvm.Type)
+	// followed by "org.osbuild.luks2.remove-key"
+	assert.Equal("org.osbuild.luks2.remove-key", luks.Type)
+}

osbuild-composer.spec

@@ -251,9 +251,6 @@ install -m 0644 -vp test/data/koji/*                               %{buildroot}%
 install -m 0755 -vd                                                %{buildroot}%{_datadir}/tests/osbuild-composer/x509
 install -m 0644 -vp test/data/x509/*                               %{buildroot}%{_datadir}/tests/osbuild-composer/x509/
 
-install -m 0755 -vd                                                %{buildroot}%{_datadir}/tests/osbuild-composer/openshift
-install -m 0644 -vp test/data/openshift/*                          %{buildroot}%{_datadir}/tests/osbuild-composer/openshift/
-
 install -m 0755 -vd                                                %{buildroot}%{_datadir}/tests/osbuild-composer/schemas
 install -m 0644 -vp pkg/jobqueue/dbjobqueue/schemas/*              %{buildroot}%{_datadir}/tests/osbuild-composer/schemas/
 

test/cases/diff-manifests.sh

@@ -31,12 +31,18 @@ basebranch=$(curl  \
     -H 'Accept: application/vnd.github.v3+json' \
     "https://api.github.com/repos/osbuild/osbuild-composer/pulls/${prnum}" | jq -r ".base.ref")
 
-greenprint "Fetching origin/${basebranch}"
-git fetch origin "${basebranch}"
+greenprint "Adding upstream GitHub remote"
+# distro version branches aren't synced to GitLab, so we will need to fetch
+# them from GitHub directly
+git remote add gh https://github.com/osbuild/osbuild-composer
+git remote show gh
+
+greenprint "Fetching gh/${basebranch}"
+git fetch gh "${basebranch}"
 
 greenprint "Getting revision IDs for HEAD and merge-base"
 head=$(git rev-parse HEAD)
-mergebase=$(git merge-base HEAD "origin/${basebranch}")
+mergebase=$(git merge-base HEAD "gh/${basebranch}")
 
 if [[ "${head}" == "${mergebase}" ]]; then
     greenprint "HEAD and merge-base are the same"