diff --git a/docker/build-common-util.sh b/docker/build-common-util.sh index 394e9cad6f..1114b5c320 100644 --- a/docker/build-common-util.sh +++ b/docker/build-common-util.sh @@ -38,7 +38,6 @@ EOF fi # lower compression levels in a PR build, to save build time if [ -z ${CI_ARCHIVER_MODE+x} ]; then - echo "ZIP_COMPRESSION_LEVEL ?= \"-1\"" >> conf/auto.conf echo "XZ_COMPRESSION_LEVEL ?= \"-0\"" >> conf/auto.conf fi } diff --git a/docker/post-build.sh b/docker/post-build.sh index 09c81d00c7..f63b0baadd 100755 --- a/docker/post-build.sh +++ b/docker/post-build.sh @@ -28,6 +28,11 @@ set -u # create auto.conf using functions in build-common-util.sh auto_conf_common +# post-build testing builds images but only .wic is sufficient +# (default in IMAGE_FSTYPES). We skip compression and bmap formats +# to optimize testing time +echo "REFKIT_VM_IMAGE_TYPES = \"\"" >> conf/auto.conf + export BUILD_ID=${CI_BUILD_ID} export BB_ENV_EXTRAWHITE="$BB_ENV_EXTRAWHITE BUILD_ID" diff --git a/docker/tester-exec.sh b/docker/tester-exec.sh index 91985fc217..e5948b1a7c 100755 --- a/docker/tester-exec.sh +++ b/docker/tester-exec.sh @@ -77,13 +77,8 @@ testimg() { wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.bmap wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.xz -O - | unxz - > ${FILENAME} if [ ! -s ${FILENAME} ]; then - wget ${_WGET_OPTS} ${CI_BUILD_URL}/images/${MACHINE}/${FILENAME}.zip - if [ -s ${FILENAME}.zip ]; then - unzip ${FILENAME}.zip - else - echo "ERROR: No file ${FILENAME}.xz or ${FILENAME}.zip found, can not continue." + echo "ERROR: No file ${FILENAME}.xz, can not continue." exit 1 - fi fi set -e diff --git a/meta-refkit-core/classes/image-dsk.bbclass b/meta-refkit-core/classes/image-dsk.bbclass index 7eae234d24..82d5765ab0 100644 --- a/meta-refkit-core/classes/image-dsk.bbclass +++ b/meta-refkit-core/classes/image-dsk.bbclass @@ -5,25 +5,6 @@ # it with the efi stub obtained from systemd-boot. # The layout of the image is described in a separate, customizable json file. -# A layout files is built accordingly to the following example: -# { -# "gpt_initial_offset_mb": 3, <- Space allocated for the 1st GPT -# "gpt_tail_padding_mb": 3, <- Space allocated for the 2nd GPT -# "primary_uefi_boot_partition": { <-- Name of the entry in the dictionary -# "name": "primary_uefi", <-- Name of the partition in the GPT (MAX 16 ch) -# "uuid": 0, <-- UUID of the partition, 0 means random -# "size_mb": 30, <-- Size of the partition in MB -# "source": "${S}/hdd/boot", <-- Directory containing the root for the partition -# "filesystem": "vfat", <-- Filesystem for the partition -# "type": "ef00" <-- Type of the partition, to be used in the GPT -# }, -# [...] Iterate partitions as needed -# } -# -# The main rootfs partition is a special case and must be named "rootfs". -# This is required to identify it and pass its Partition UUID to the kernel, for booting. - - # By default, the full image is meant to fit into 4*10^9 bytes, i.e. # "4GB" regardless whether 1000 or 1024 is used as base. 64M are reserved # for potential partitioning overhead. @@ -49,16 +30,6 @@ do_uefiapp[depends] += " \ ${INITRD_IMAGE}:do_image_complete \ " -IMAGE_DEPENDS_dsk += " \ - gptfdisk-native:do_populate_sysroot \ - parted-native:do_populate_sysroot \ - mtools-native:do_populate_sysroot \ - dosfstools-native:do_populate_sysroot \ - dosfstools-native:do_populate_sysroot \ - python-native:do_populate_sysroot \ - bmap-tools-native:do_populate_sysroot \ - " - # Always ensure that the INITRD_IMAGE gets added to the initramfs .cpio. # This needs to be done even when the actual .dsk image format is inactive, # because the .cpio file gets copied into the rootfs, and that rootfs @@ -77,35 +48,7 @@ REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE ?= "deadbeef-dead-beef-dead-beefdeadbeef" PARTITION_TYPE_EFI = "EF00" PARTITION_TYPE_EFI_BACKUP = "2700" -DSK_IMAGE_LAYOUT ??= ' \ -{ \ - "gpt_initial_offset_mb": 3, \ - "gpt_tail_padding_mb": 3, \ - "partition_01_primary_uefi_boot": { \ - "name": "primary_uefi", \ - "uuid": 0, \ - "size_mb": ${REFKIT_VFAT_MB}, \ - "source": "${IMAGE_ROOTFS}/boot/", \ - "filesystem": "vfat", \ - "type": "${PARTITION_TYPE_EFI}" \ - }, \ - "partition_02_secondary_uefi_boot": { \ - "name": "secondary_uefi", \ - "uuid": 0, \ - "size_mb": ${REFKIT_VFAT_MB}, \ - "source": "${IMAGE_ROOTFS}/boot/", \ - "filesystem": "vfat", \ - "type": "${PARTITION_TYPE_EFI_BACKUP}" \ - }, \ - "partition_03_rootfs": { \ - "name": "rootfs", \ - "uuid": "${REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE}", \ - "size_mb": 3700, \ - "source": "${IMAGE_ROOTFS}", \ - "filesystem": "ext4", \ - "type": "8300" \ - } \ -}' +DSK_IMAGE_LAYOUT ??= '{}' inherit deploy @@ -140,9 +83,10 @@ export PART_%(pnum)d_FS=%(filesystem)s partition_table = json.loads(layout) full_image_size_mb = partition_table["gpt_initial_offset_mb"] + \ - partition_table["gpt_tail_padding_mb"] + partition_table["gpt_tail_padding_mb"] \ + if partition_table else 0 - rootfs_type = None + rootfs_type = '' pnum = 0 for key in sorted(partition_table.keys()): if not isinstance(partition_table[key], dict): @@ -154,7 +98,7 @@ export PART_%(pnum)d_FS=%(filesystem)s partition_table[key]['uuid'] = str(uuid.uuid4()) # Store these for the creation of the UEFI binary if partition_table[key]['name'] == 'rootfs': - rootfs_type = partition_table[key]['filesystem'] + rootfs_type = 'rootfstype=%s ' % partition_table[key]['filesystem'] int_part_uuid = d.getVar('INT_STORAGE_ROOTFS_PARTUUID_VALUE', True) else: int_part_uuid = partition_table[key]["uuid"] @@ -168,8 +112,8 @@ export PART_%(pnum)d_FS=%(filesystem)s } pnum = pnum + 1 - assert rootfs_type is not None - partition_data += "export PART_COUNT=%d\n" % pnum + if pnum > 0: + partition_data += "export PART_COUNT=%d\n" % pnum if os.path.exists(d.expand('${B}/initrd')): os.remove(d.expand('${B}/initrd')) @@ -188,7 +132,7 @@ export PART_%(pnum)d_FS=%(filesystem)s def generate_app(partuuid, cmdline, suffix): with open(d.expand('${B}/cmdline' + suffix + '.txt'), 'w') as f: - f.write(d.expand('${APPEND} root=PARTUUID=%s rootfstype=%s %s' % \ + f.write(d.expand('${APPEND} root=PARTUUID=%s %s%s' % \ (partuuid, rootfs_type, cmdline))) check_call(d.expand('objcopy ' + '--add-section .osrel=${B}/machine.txt ' + @@ -209,9 +153,10 @@ export PART_%(pnum)d_FS=%(filesystem)s generate_app(d.getVar('REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE', True), "installer", "") generate_app(d.getVar('INT_STORAGE_ROOTFS_PARTUUID_VALUE', True), "", "_internal_storage") - with open(d.expand('${B}/emmc-partitions-data'), 'w') as emmc_part_data: - emmc_part_data.write(partition_data) - shutil.copyfile(d.expand('${B}/emmc-partitions-data'), d.expand('${DEPLOYDIR}/emmc-partitions-data')) + if partition_data: + with open(d.expand('${B}/emmc-partitions-data'), 'w') as emmc_part_data: + emmc_part_data.write(partition_data) + shutil.copyfile(d.expand('${B}/emmc-partitions-data'), d.expand('${DEPLOYDIR}/emmc-partitions-data')) # The RMC database is deployed unconditionally but not read if the BIOS is in SecureBoot mode. # XXX: However, the check for SecureBoot is not present. The bug is tracked in @@ -280,21 +225,3 @@ do_rootfs[depends] += '${@bb.utils.contains('IMAGE_FEATURES','secureboot','sbsig ROOTFS_POSTPROCESS_COMMAND += " ${@bb.utils.contains('IMAGE_FEATURES','secureboot','uefiapp_sign;','',d)} " ROOTFS_POSTPROCESS_COMMAND += " uefiapp_deploy; " - -# All variables explicitly passed to image-dsk.py. -IMAGE_DSK_VARIABLES = " \ - APPEND \ - IMGDEPLOYDIR \ - DSK_IMAGE_LAYOUT \ - IMAGE_LINK_NAME \ - IMAGE_NAME \ - IMAGE_ROOTFS \ - ROOTFS_TYPE \ - REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE \ - PARTITION_TYPE_EFI \ - PARTITION_TYPE_EFI_BACKUP \ - S \ -" - -IMAGE_CMD_dsk = "${PYTHON} ${IMAGE_DSK_BASE}/lib/image-dsk.py ${@' '.join(["'%s=%s'" % (x, d.getVar(x, True) or '') for x in d.getVar('IMAGE_DSK_VARIABLES', True).split()])}" -IMAGE_CMD_dsk[vardeps] = "${IMAGE_DSK_VARIABLES}" diff --git a/meta-refkit-core/classes/image_types_extra.bbclass b/meta-refkit-core/classes/image_types_extra.bbclass deleted file mode 100644 index 499cfeddad..0000000000 --- a/meta-refkit-core/classes/image_types_extra.bbclass +++ /dev/null @@ -1,10 +0,0 @@ -# IoT Reference OS Kit custom conversion types - -CONVERSIONTYPES_append = " vdi" -CONVERSION_CMD_vdi = "qemu-img convert -O vdi ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.vdi" -CONVERSION_DEPENDS_vdi = "qemu-native" - -CONVERSIONTYPES_append = " zip" -ZIP_COMPRESSION_LEVEL ?= "-9" -CONVERSION_CMD_zip = "zip ${ZIP_COMPRESSION_LEVEL} ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}.zip ${IMAGE_NAME}${IMAGE_NAME_SUFFIX}.${type}" -CONVERSION_DEPENDS_zip = "zip-native" diff --git a/meta-refkit-core/classes/refkit-image.bbclass b/meta-refkit-core/classes/refkit-image.bbclass index da59c0bd67..7b354070be 100644 --- a/meta-refkit-core/classes/refkit-image.bbclass +++ b/meta-refkit-core/classes/refkit-image.bbclass @@ -212,10 +212,8 @@ NOISO = "1" REFKIT_QB_DRIVE_TYPE = "/dev/vd" QB_DRIVE_TYPE = "${REFKIT_QB_DRIVE_TYPE}" -inherit image_types_extra - # Replace the default "live" (aka HDDIMG) images with whole-disk images -REFKIT_VM_IMAGE_TYPES ?= "" +REFKIT_VM_IMAGE_TYPES ??= "" IMAGE_FSTYPES_append = " ${REFKIT_VM_IMAGE_TYPES}" # unconditionally set in x86-base.inc so we just remove it to avoid diff --git a/meta-refkit-core/lib/image-dsk.py b/meta-refkit-core/lib/image-dsk.py deleted file mode 100644 index 2bddc5f9da..0000000000 --- a/meta-refkit-core/lib/image-dsk.py +++ /dev/null @@ -1,180 +0,0 @@ -#!/usr/bin/env python -# -# Image creation class for GPT multi-partioned disk images for EFI systems. -# -# Copyright (C) 2015-2016 Intel Corporation -# Licensed under the MIT license - -import json -import os -import sys -import shutil -from re import sub -from glob import glob -from uuid import uuid4 -from subprocess import check_call -from bmaptools import Filemap - -VARS = dict([x.split('=', 1) for x in sys.argv[1:]]) - - -def lookup_var(varname, location=None): - """Look up a variable in the parameters or the environment.""" - if varname in VARS: - return VARS[varname] - if varname in os.environ: - return os.environ[varname] - exit("image-dsk.py: variable %s%s not passed by image-dsk.bbclass," - " add it to the parameters." % - (varname, ("used in " + location) if location else "")) - - -def symlink(src, dst): - """Helper for symlink housekeeping.""" - if os.path.exists(dst): - os.remove(dst) - os.symlink(src, dst) - - -def populate_rawcopy(src, dst): - """Argument src is a raw partition generated elsewhere.""" - shutil.copyfile(src, dst) - - -def populate_vfat(src, dst): - """Create and populate a FAT partition, out of a root directory .""" - check_call(['mkdosfs', dst]) - """this requires shell expansion for wildcard - check_call(['mcopy', '-i', dst, '-s', src + '/*', '::/'])""" - os.system('mcopy -i ' + dst + ' -s ' + src + '/* ::/') - - -def populate_ext4(src, dst): - """Create and populate an ext4 partition out of a root directory .""" - check_call(['mkfs.ext4', '-F', dst] + (['-d', src] if src else [])) - - -def expand_vars(arg_string, location=None): - """Expand variables in arg_string.""" - return sub(r'\$\{([^}]+)\}', lambda x: lookup_var(x.group(1), location), - arg_string) - - -def truncate_mib(fname, fsize): - """Create sparse file with requested size (in MiB).""" - with open(fname, "a+b") as fobj: - fobj.truncate(int(fsize) * 1024 * 1024) - - -def sparse_copy(src_fname, dst_fname, offset_mib=0): - """Efficiently copy sparse file to or into another file.""" - filemap = Filemap.filemap(src_fname) - try: - dst_file = open(dst_fname, 'r+b') - except IOError: - dst_file = open(dst_fname, 'wb') - - for first, last in filemap.get_mapped_ranges(0, filemap.blocks_cnt): - start = first * filemap.block_size - end = (last + 1) * filemap.block_size - - filemap._f_image.seek(start, os.SEEK_SET) - dst_file.seek((offset_mib * 1024 * 1024) + start, os.SEEK_SET) - - chunk_size = 1024 * 1024 - to_read = end - start - read = 0 - - while read < to_read: - if read + chunk_size > to_read: - chunk_size = to_read - read - chunk = filemap._f_image.read(chunk_size) - dst_file.write(chunk) - read += chunk_size - dst_file.close() - - -def do_dsk_image(): - """Entry point for generating the disk image.""" - # Load the descripton of the disk layout. - partition_table = json.loads(expand_vars('${DSK_IMAGE_LAYOUT}')) - - # Before adding up the size of each partition, add the size of the GPT - full_image_size_mb = partition_table["gpt_initial_offset_mb"] + \ - partition_table["gpt_tail_padding_mb"] - - # The rootfs is special, because its PARTUUID must be aligned with - # the kernel command line, to allow pivot-rooting. - for key in sorted(partition_table.iterkeys()): - if not isinstance(partition_table[key], dict): - continue - # Calculate the total image size. - full_image_size_mb += partition_table[key]["size_mb"] - # Generate randomized uuids, if required (uuid == 0) - # Otherwise leave whatever was set in the configuration file. - if str(partition_table[key]['uuid']) == '0': - partition_table[key]['uuid'] = str(uuid4()).lower() - # Store these for the creation of the UEFI binary - if partition_table[key]['name'] == 'rootfs': - # The rootfs partuuid is not randomized, because it is required - # by the command line embedded in the efi-combo-binary - # and it might be even preferrable to fix it to specific values - # for each product. - # Default to lower case, to avoid issues from camelcase. - partition_table[key]["uuid"] = \ - expand_vars("${REMOVABLE_MEDIA_ROOTFS_PARTUUID_VALUE}").lower() - - # Save to disk the layout with the PARTUUIDs used, to facilitate the - # job of accessing programmatically individual partitions. - disk_layout_file = \ - os.path.join(expand_vars("${IMGDEPLOYDIR}"), - expand_vars('${IMAGE_NAME}-disk-layout.json')) - disk_layout_file_link = \ - os.path.join(expand_vars("${IMGDEPLOYDIR}"), - expand_vars('${IMAGE_LINK_NAME}-disk-layout.json')) - with open(disk_layout_file, 'w') as disk_layout: - json.dump(obj=partition_table, fp=disk_layout, - indent=4, separators=(',', ': ')) - symlink(expand_vars('${IMAGE_NAME}-disk-layout.json'), - disk_layout_file_link) - - # First step in creating the full disk image: loop file + GPT partition. - full_image_name = \ - os.path.join(expand_vars("${IMGDEPLOYDIR}"), - expand_vars('${IMAGE_NAME}.dsk')) - truncate_mib(full_image_name, full_image_size_mb) - check_call(['sgdisk', '-o', full_image_name]) - - partition_start_mb = partition_table["gpt_initial_offset_mb"] - for key in sorted(partition_table.iterkeys()): - if not isinstance(partition_table[key], dict): - continue - # Generate even more auxiliary variable - partition_logical_name = str(partition_table[key]["name"]) - partition_size_mb = partition_table[key]["size_mb"] - partition_name = expand_vars("${IMAGE_NAME}") + '.' + \ - partition_table[key]["name"] + ".part" - partition_type = expand_vars(partition_table[key]["type"]) - full_partition_name = \ - os.path.join(expand_vars("${IMGDEPLOYDIR}"), partition_name) - # Create the temporary loop file for hostong the partition. - truncate_mib(full_partition_name, partition_size_mb) - # Populate the partition accordingly to its parameters. - eval('populate_' + str(partition_table[key]["filesystem"]) + - '("' + expand_vars(partition_table[key]["source"]) + '", "' + - full_partition_name + '")') - # Allocate space for the partition in the image loop file. - check_call(['sgdisk', '-c=0:' + partition_logical_name, - '-n=0:' + str(partition_start_mb) + 'M:+' + - str(partition_size_mb) + 'M', - '-t=0:' + partition_type, - '-u=0:' + str(partition_table[key]["uuid"]), - full_image_name]) - sparse_copy(full_partition_name, full_image_name, partition_start_mb) - # Remove the partition, now that it exists in the disk image. - if os.path.exists(full_partition_name): - os.remove(full_partition_name) - partition_start_mb += partition_table[key]["size_mb"] - -if __name__ == "__main__": - do_dsk_image() diff --git a/meta-refkit/conf/distro/include/refkit-ci.inc b/meta-refkit/conf/distro/include/refkit-ci.inc index cd48d70ef7..c44b6eba23 100644 --- a/meta-refkit/conf/distro/include/refkit-ci.inc +++ b/meta-refkit/conf/distro/include/refkit-ci.inc @@ -44,7 +44,8 @@ require conf/distro/include/refkit-development.inc # If we can access /dev/kvm, then let's use it. QEMU_USE_KVM = "${@ 'intel-corei7-64 intel-core2-32 qemux86 qemux86-64' if os.access('/dev/kvm', os.R_OK|os.W_OK) else '' }" -REFKIT_VM_IMAGE_TYPES = "wic.xz wic.zip wic.bmap wic.xz.sha256sum" +XZ_INTEGRITY_CHECK = "sha256" +REFKIT_VM_IMAGE_TYPES ?= "wic.xz wic.bmap" # # pre/post-build oe-selftests started by CI builder, whitespace-separated. diff --git a/meta-refkit/conf/layer.conf b/meta-refkit/conf/layer.conf index 4a7b638e1f..9ba5d9ecda 100644 --- a/meta-refkit/conf/layer.conf +++ b/meta-refkit/conf/layer.conf @@ -11,15 +11,9 @@ BBFILES += "${LAYERDIR}/recipes-*/*/*.bb \ ${LAYERDIR}/fixes/*/recipes-*/*/*/*.bbappend \ " -# Set a variable to get to the top of the metadata location -IMAGE_DSK_BASE := '${LAYERDIR}' - # Set a variable to get to the top of the IoT Reference OS Kit metadata location. META_REFKIT_BASE := '${LAYERDIR}' -# Used by image-dsk.bbclass to find lib/image-dsk.py. -IMAGE_DSK_BASE ?= '${META_REFKIT_BASE}' - # Required by stateless.bbclass to find patches needed only when # compiling stateless. STATELESS_PATCHES_BASE ?= "${META_REFKIT_BASE}/conf/distro/stateless-patches"