[TEST-BUILD] Basic flatpak support (no flatpak-session yet), reworked on top of latest intel-iot-refkit

meta-refkit-core/classes/flatpak-config.bbclass

@@ -0,0 +1,36 @@
+# We expect to find our scripts here, in the scripts subdirectory.
+FLATPAKBASE = "${META_REFKIT_CORE_BASE}"
+
+# This is a per-build per-image primary bare-user flatpak repository.
+FLATPAK_REPO = "${WORKDIR}/flatpak.bare-user"
+
+# This is an archive-z2 repository where we export our builds for testing.
+# This can be exposed over HTTP for consumption by flatpak. Among other
+# things, this can be used to pull in the generated BaseSdk and BasePlatform
+# repository branches to a development host for building flatpak applications
+# against the corresponding flatpak-enabled image. Set this to empty if you
+# don't want to automatically publish to such a repository.
+FLATPAK_EXPORT ?= "${TMPDIR}/flatpak.archive-z2"
+
+# We use the domain and the (canonical) branch together with ${MACHINE} to
+# construct the full flatpak REFs of our base and SDK runtimes. The full REF
+# is considered the canonical branch and is constructed as:
+#
+#     runtime/${FLATPAK_DOMAIN}.Base{Platform,Sdk}/${MACHINE}/${FLATPAK_BRANCH}
+#
+# Optionally we publish builds as two additional branches:
+#
+#   - an optional rolling 'latest' corresponding to the last build
+#   - an optional rolling 'build' tagged with the ${BUILD_ID}
+#
+# Setting the corresponding variables for the optional branches to empty
+# disables publishing/creating those branches.
+FLATPAK_DOMAIN ?= "example.org"
+FLATPAK_BASE   ?= "${@d.getVar('PN').split('-flatpak-')[0]}"
+FLATPAK_BRANCH ?= "${DISTRO}/${FLATPAK_BASE}/${DISTRO_VERSION}"
+FLATPAK_LATEST ?= "${DISTRO}/${FLATPAK_BASE}/latest"
+FLATPAK_BUILD  ?= "${DISTRO}/${FLATPAK_BASE}/build/${BUILD_ID}"
+
+# This is the GPG key id of our repository signing key. If you set this to
+# empty, signing is disabled altogether.
+FLATPAK_GPGID ?= "refkit-signing@key"

meta-refkit-core/classes/flatpak-image-variants.bbclass

@@ -0,0 +1,39 @@
+IMAGE_FEATURES[validitems] += " \
+    flatpak \
+    tools-sdk \
+    dev-pkgs \
+    tools-debug \
+    tools-profile \
+"
+
+FEATURE_PACKAGES_flatpak = " \
+    packagegroup-flatpak \
+"
+
+#
+# Define two flatpak-related image variants.
+#
+# - flatpak runtime image variant 'flatpak-runtime':
+#     This variant corresponds to a flatpak BasePlatform runtime. In
+#     addition to the content of its base image, this variant has the
+#     necessary runtime bits for flatpak. Using this image on a device
+#     enables one to pull in, update and run applications as flatpaks
+#     from flatpak remotes/repositories.
+#
+# - flatpak SDK image variant 'flatpak-sdk':
+#     This variant corresponds to a flatpak BaseSdk runtime. It has the
+#     necessary bits for compiling applications and publishing them as
+#     flatpaks in flatpak repositories.
+#
+# When building these images variants, a flatpak repository will also be
+# populated with the contents of these images. This repository can be used
+# to flatpak-install the runtime and SDK runtimes on a development machine
+# for generating flatpaks for the flatpak-runtime image variant.
+
+# 'flatpak-runtime' variant (runtime image for a device)
+IMAGE_VARIANT[flatpak-runtime] = "flatpak"
+
+# 'flatpak-sdk' variant (SDK image for a development host)
+IMAGE_VARIANT[flatpak-sdk] = "flatpak tools-develop tools-debug dev-pkgs"
+
+BBCLASSEXTEND += "imagevariant:flatpak-runtime imagevariant:flatpak-sdk"

meta-refkit-core/classes/flatpak-repository.bbclass

@@ -0,0 +1,132 @@
+# Check we have the necessary distro features enabled.
+inherit distro_features_check
+REQUIRED_DISTRO_FEATURES_append = " usrmerge systemd pam"
+
+inherit flatpak-config
+
+REFKIT_SIGNING_KEYS += "${FLATPAK_GPGID}"
+inherit refkit-signing-keys
+
+#
+# Create and populate a primary flatpak repository from/for an image.
+#
+fakeroot do_flatpak_populate_repository () {
+   echo "Flatpak repository population:"
+   echo "  * FLATPAKBASE:    ${FLATPAKBASE}"
+   echo "  * IMAGE_BASENAME: ${IMAGE_BASENAME}"
+
+   # Bail out early if flatpak is not enabled for this image.
+   case ${IMAGE_BASENAME} in
+       *-flatpak-runtime) RUNTIME_TYPE=BasePlatform;;
+       *-flatpak-sdk)     RUNTIME_TYPE=BaseSdk;;
+       *)
+          echo "${IMAGE_BASENAME} is not a flatpak-enabled image..."
+          return 0
+          ;;
+   esac
+
+   echo "${IMAGE_BASENAME} is a flatpak $RUNTIME_TYPE image"
+
+   if [ -n "${FLATPAK_GPGID}" ]; then
+       GPG_SIGN="--gpg-home ${REFKIT_SIGNING_GPGDIR} \
+                 --gpg-id ${FLATPAK_GPGID}"
+   else
+       GPG_SIGN=""
+   fi
+
+   _base="runtime/${FLATPAK_DOMAIN}.$RUNTIME_TYPE/${MACHINE}"
+   _t=""
+   for _b in ${FLATPAK_BRANCH} ${FLATPAK_LATEST} ${FLATPAK_BUILD}; do
+       BRANCHES="$BRANCHES$_t$_base/$_b"
+       _t=","
+   done
+
+   echo "Using flatpak branches $BRANCHES for ${IMAGE_ROOTFS}..."
+
+   # Generate/populate flatpak/OSTree repository
+   ${FLATPAKBASE}/scripts/flatpak-populate-repo.sh \
+       --repo-path ${FLATPAK_REPO} \
+       --repo-mode bare-user \
+       $GPG_SIGN \
+       --branches "$BRANCHES" \
+       --image-sysroot ${IMAGE_ROOTFS} \
+       --tmp-dir ${TMPDIR}
+}
+
+do_flatpak_populate_repository[depends] += " \
+    ostree-native:do_populate_sysroot \
+    flatpak-native:do_populate_sysroot \
+    gnupg1-native:do_populate_sysroot \
+"
+
+do_flatpak_populate_repository[vardeps] += " \
+    FLATPAK_REPO \
+    FLATPAK_EXPORT \
+    FLATPAK_DOMAIN \
+    FLATPAK_BRANCH \
+    FLATPAK_LATEST \
+    FLATPAK_BUILD \
+    FLATPAK_GPGID \
+"
+
+#
+# Export an image (well the bare-user repo, really) to an archive-z2 repo.
+#
+fakeroot do_flatpak_export_repository () {
+   # Bail out early if no export repository is defined.
+   if [ -z "${FLATPAK_EXPORT}" ]; then
+       echo "Flatpak repository for export not specified, skip export..."
+       return 0
+   fi
+
+   # Bail out early if flatpak is not enabled for this image.
+   case ${IMAGE_BASENAME} in
+       *-flatpak-runtime) RUNTIME_TYPE=BasePlatform;;
+       *-flatpak-sdk)     RUNTIME_TYPE=BaseSdk;;
+       *)
+          echo "${IMAGE_BASENAME} is not a flatpak-enabled image..."
+          return 0
+          ;;
+   esac
+
+   echo "${IMAGE_BASENAME} is a flatpak $RUNTIME_TYPE image"
+
+   if [ -n "${FLATPAK_GPGID}" ]; then
+       GPG_SIGN="--gpg-home ${REFKIT_SIGNING_GPGDIR} \
+                 --gpg-id ${FLATPAK_GPGID}"
+   else
+       GPG_SIGN=""
+   fi
+
+   # Export to archive-z2 flatpak/OSTree repository
+   ${FLATPAKBASE}/scripts/flatpak-populate-repo.sh \
+       --repo-path ${FLATPAK_REPO} \
+       --repo-export ${FLATPAK_EXPORT} \
+       $GPG_SIGN \
+       --tmp-dir ${TMPDIR}
+}
+
+do_flatpak_export_repository[depends] += " \
+    ostree-native:do_populate_sysroot \
+    flatpak-native:do_populate_sysroot \
+    gnupg1-native:do_populate_sysroot \
+"
+
+do_flatpak_export_repository[vardeps] += " \
+    FLATPAK_REPO \
+    FLATPAK_EXPORT \
+    FLATPAK_DOMAIN \
+    FLATPAK_BRANCH \
+    FLATPAK_LATEST \
+    FLATPAK_BUILD \
+    FLATPAK_GPGID \
+"
+
+addtask flatpak_populate_repository \
+    after do_rootfs \
+    before do_image_complete
+
+addtask flatpak_export_repository \
+    after do_flatpak_populate_repository \
+    before do_image_complete
+

meta-refkit-core/classes/refkit-image.bbclass

@@ -256,6 +256,12 @@ IMAGE_MODE_VALID = "${@ d.getVar('REFKIT_IMAGE_MODE_VALID') or '' }"
 # variants of variants.
 inherit image-mode-variants
 
+# Enable flatpak image variant and repository generation.
+inherit ${@bb.utils.contains('DISTRO_FEATURES', 'flatpak', \
+              'flatpak-image-variants', '', d)}
+inherit ${@bb.utils.contains('DISTRO_FEATURES', 'flatpak', \
+              'flatpak-repository', '', d)}
+
 BUILD_ID ?= "${DATETIME}"
 # Do not re-trigger builds just because ${DATETIME} changed.
 BUILD_ID[vardepsexclude] += "DATETIME"
@@ -523,3 +529,43 @@ EOF
     fi
 }
 ROOTFS_POSTPROCESS_COMMAND += "refkit_image_system_serialgetty; "
+
+#  Prepare the symlinks required for merged /usr at the time of rootfs creation.
+
+# The links created in rootfs are:
+#/bin --> /usr/sbin
+#/sbin --> /usr/sbin
+#/lib --> /usr/lib
+#/lib64 --> /usr/lib64
+
+# We cannot make these symlinks as part of 'base-files' or some other package.
+# Because at rootfs creation, installation of the package(say kernel) that
+# depends on these root folders/links fails, if package manager installs this
+# package prior to base-files.
+
+# These symbolic links in top level folder should present as long as
+#   - kerenl tools use /lib/{module,firmware}
+#   - shell scripts uses
+#upstream commit waiting for review: 
+# http://lists.openembedded.org/pipermail/openembedded-core/2017-February/133151.html
+create_merged_usr_symlinks() {
+    install -m 0755 -d ${IMAGE_ROOTFS}/${base_bindir}
+    install -m 0755 -d ${IMAGE_ROOTFS}/${base_sbindir}
+    install -m 0755 -d ${IMAGE_ROOTFS}/${base_libdir}
+    lnr ${IMAGE_ROOTFS}${base_bindir} ${IMAGE_ROOTFS}/bin
+    lnr ${IMAGE_ROOTFS}${base_sbindir} ${IMAGE_ROOTFS}/sbin
+    lnr ${IMAGE_ROOTFS}${base_libdir} ${IMAGE_ROOTFS}/${baselib}
+
+    if [ "${nonarch_base_libdir}" != "${base_libdir}" ]; then
+       install -m 0755 -d ${IMAGE_ROOTFS}/${nonarch_base_libdir}
+       lnr ${IMAGE_ROOTFS}${nonarch_base_libdir} ${IMAGE_ROOTFS}/lib
+    fi
+
+    # create base links for multilibs
+    multi_libdirs="${@d.getVar('MULTILIB_VARIANTS')}"
+    for d in $multi_libdirs; do
+        install -m 0755 -d ${IMAGE_ROOTFS}/${exec_prefix}/$d
+        lnr ${IMAGE_ROOTFS}/${exec_prefix}/$d ${IMAGE_ROOTFS}/$d
+    done
+}
+ROOTFS_PREPROCESS_COMMAND += "${@bb.utils.contains('DISTRO_FEATURES', 'usrmerge', 'create_merged_usr_symlinks; ', '',d)}"

meta-refkit-core/classes/refkit-signing-keys.bbclass

@@ -0,0 +1,84 @@
+#
+# This class can be used to generate (or import) a set of signing keys,
+# for whatever purpose the build might need those. Once such purpose is
+# the signing of base OS and flatpak ostree repositories.
+#
+# To make sure all the necessary keys get generated list them in your
+# local.conf (or some other global configuration file) by setting
+# REFKIT_SIGNING_KEYS to necessary key IDs.
+
+# Signing keys to generate, a list of key IDs.
+REFKIT_SIGNING_KEYS    ?= ""
+
+# This is where we put our GPG homedir, export keys to, etc.
+REFKIT_SIGNING_GPGDIR  ?= "${TMPDIR}/gnupg"
+
+# How long we let two parallel key generation tasks clash.
+REFKIT_SIGNING_TIMEOUT ?= "60"
+
+# task to generate/check all requested signing keys
+fakeroot do_generate_signing_keys () {
+    # Bail out early if we have no keys to generate.
+    if [ -z "${REFKIT_SIGNING_KEYS}" -o -z "${REFKIT_SIGNING_GPGDIR}" ]; then
+        echo "No GPG key IDs or directory set, nothing to do..."
+        return 0
+    fi
+
+    # When building several images in parallel (e.g. in CI), we have to
+    # make sure we don't let two tasks start generating the same signing
+    # key into the keyring. While GPG itself seems to semi-gracefully
+    # survive a keyring with duplicate key ids, gpgme (or maybe just ostree,
+    # I did not bother checking it) segfaults in such a case.
+    # Therefore, we have this unholy kludge where we use mkdir(2) as a
+    # lock, and let the task getting there first do the deed, while the
+    # second one just waits for the first to finish (and consequently causes
+    # its own dependent tasks to properly wait for the keys to get generated).
+    # Yuck...
+
+    dir="${REFKIT_SIGNING_GPGDIR}"
+    mkdir -p "${dir%/*}"
+    mkdir "${dir}.lock" || { # Forgive me Thompson&Dijkstra, for I have sinned...
+        slept=0
+        for id in ${REFKIT_SIGNING_KEYS}; do
+            while [ $slept -lt ${REFKIT_SIGNING_TIMEOUT} ]; do
+                if [ ! -e ${dir}/$id.sec ]; then
+                    echo "Waiting for generation of signing key $id..."
+                    sleep 1
+                    let slept=$slept+1
+                else
+                    echo "Got signing key $id..."
+                    break
+                fi
+            done
+        done
+        if [ $slept -ge ${REFKIT_SIGNING_TIMEOUT} ]; then
+            echo "Signing key generation timed out..."
+            return 1
+        else
+            return 0
+        fi
+    }
+
+    dir="${REFKIT_SIGNING_GPGDIR}"
+    for id in ${REFKIT_SIGNING_KEYS}; do
+        pubkey="$dir/$id.pub"
+        seckey="$dir/$id.sec"
+
+        # Generate repository signing GPG keys, if we don't have them yet.
+        echo "Generating/checking signing key $id..."
+
+        ${FLATPAKBASE}/scripts/gpg-keygen.sh \
+            --home $dir \
+            --id $id \
+            --pub $pubkey \
+            --sec $seckey
+    done
+
+    rmdir "${dir}.lock"
+}
+
+do_generate_signing_keys[depends] += " \
+    gnupg1-native:do_populate_sysroot \
+"
+
+addtask generate_signing_keys before do_rootfs

meta-refkit-core/classes/required-systemd.bbclass

@@ -0,0 +1,7 @@
+# Same as systemd.bbclass but should be used by recipes which require
+# systemd (as opposed to just support systemd).
+
+REQUIRED_DISTRO_FEATURES = "systemd"
+inherit distro_features_check
+
+inherit systemd.bbclass

meta-refkit-core/conf/distro/include/enable-refkit-config.inc

@@ -15,3 +15,5 @@ VIRTUAL-RUNTIME_initscripts = ""
 # defaults from OE-core.
 DISTRO_FEATURES_DEFAULT_remove = "${REFKIT_DEFAULT_DISTRO_FEATURES_REMOVE}"
 DISTRO_FEATURES ?= "${DISTRO_FEATURES_DEFAULT} ${DISTRO_FEATURES_LIBC} ${REFKIT_DEFAULT_DISTRO_FEATURES}"
+
+require conf/distro/include/flatpak.inc

meta-refkit-core/conf/distro/include/flatpak.inc

@@ -0,0 +1,8 @@
+REFKIT_DEFAULT_DISTRO_FEATURES += " \
+    usrmerge \
+    pam \
+    flatpak \
+"
+
+# Enable D-Bus session bus support, needed by flatpak.
+PACKAGECONFIG_append_pn-dbus_refkit-config = " user-session"