From e3471650d6177a6310ff703c71e0b6ddbb645ab5 Mon Sep 17 00:00:00 2001
From: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
Date: Thu, 11 Aug 2016 16:21:54 +0200
Subject: [PATCH 1/7] Use downstream packaging in spread tests

This patch changes spread setup phase to build the native package (for
Ubuntu and soon for Debian) in a way that is more consistent with the
actual release process. This should help us release with confidence as
all CI tests will run with real downstream packaging applied to real
upstream tarball and build in an isolated manner with sbuild.

This also paves the way towards the removal of the debian/ packaging
from the upstream repository. This will be done in a separate pull
request to avoid clutter.

The spread prepare stage was moved to .spread-prepare.sh so that it can
be shell-checked more easily.

The debian-8 spread target was removed because currently the downstream
packaging is not compatible. Debian maintainers were notified to the
issue so that we should be able to re-enable Debian (sid) support soon.

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
---
 .spread-data/apt-keys/README.md      |   4 +
 .spread-data/apt-keys/sbuild-key.pub | Bin 0 -> 427 bytes
 .spread-data/apt-keys/sbuild-key.sec | Bin 0 -> 759 bytes
 .spread-prepare.sh                   | 226 +++++++++++++++++++++++++++
 Makefile.am                          |   7 +
 spread.yaml                          |  33 +---
 6 files changed, 240 insertions(+), 30 deletions(-)
 create mode 100644 .spread-data/apt-keys/README.md
 create mode 100644 .spread-data/apt-keys/sbuild-key.pub
 create mode 100644 .spread-data/apt-keys/sbuild-key.sec
 create mode 100755 .spread-prepare.sh

diff --git a/.spread-data/apt-keys/README.md b/.spread-data/apt-keys/README.md
new file mode 100644
index 0000000..e496ac7
--- /dev/null
+++ b/.spread-data/apt-keys/README.md
@@ -0,0 +1,4 @@
+This directory contains keys used by the sbuild program to sign the temporary
+archive. Those keys are kept in the tree as ephemeral test virtual machines do
+not have sufficient entropy to generate keys by themselves in reasonable amount
+of time.
diff --git a/.spread-data/apt-keys/sbuild-key.pub b/.spread-data/apt-keys/sbuild-key.pub
new file mode 100644
index 0000000000000000000000000000000000000000..34d8b57690b7e8133b73a799e4bd470e7c47d8f6
GIT binary patch
literal 427
zcmV;c0aX5&jRaN%NFV_O0JxNLP0VZb%PQy4fPWY1qHz=2-OFRY>?qI^dQplJc^zEW
zlyRWPtL6IdhsvnpS3$?APa};lq4O_Ap2#WX{ICdgZ<>h{OB+QR{8T$<)it#RQ;UDv
zo1&Xvq_!6MnVPkhDzPZp-u_v+iG@fsOst&m@-QZ{HYLBpm|BMB0RRyJ00FdMQ(|>#
zY-Au)X=iR_av&&EVs&Y3WFSIyX>4R5L}hSgZe(R{V|gG!a${&|c4Z(-WqBzeJYsce
zY-D6DbZ>8Lb1h_Lc4cfpY-w|Jb1q?QX>W9BE@Wk5X<=?IZ*pfoh`0n30RjLb1p-zC
zNFV|mF9r(<2nPcK1{DYb2?`4Y76JnS0v-VZ7k~f?2@nT;U_;3Udp>|o1OSx3R@!!G
zeO2yv*U_X3=WZ+PDC*k<$x^81WV$O(@N(X!KIC@!eb=xSnKhFGzZ}c<S7{R<>=yL0
zrd@Jf1&OKj`DLF)gj<}2xF~|^ufn$emLArzJ30U+BH$|lghxX(opymUmyCa05JsK*
VihyKrl%!aG4JdUEL$`+VD?>j~u}}a2

literal 0
HcmV?d00001

diff --git a/.spread-data/apt-keys/sbuild-key.sec b/.spread-data/apt-keys/sbuild-key.sec
new file mode 100644
index 0000000000000000000000000000000000000000..cdadd277936c5971b14b3e4006a9e202c0408c60
GIT binary patch
literal 759
zcmV<T0to$;0oVjq1V|tO1OT{{a!t%@^vf#e(13pz>7sEH+TF`zzw9W`6M9jK5_uh5
z*OYOf$E)S~?}y5$;#Wb(sZS$~FQM}<MV`nh<@~S+bZ?r86H6OK8vIl{XVo>e1yhTE
z+MA-AU!=Ac`I(xvmMXC*+1~zHxrv2HG)%0V@A5DvvNk2Z!kAiy<^cc^0RRC21N;o<
z(laL6Ac08}a>QLeh=(IpVovfd+y^k$RhmwdvlVqrUBzPweH4R?zfCm?2xC?C)*N!J
zr#LZI>x}))QYUeTN7*H>_wA$Z!mrR5Ym!s!Pz__GhS~vAuldxk0qCXNQ`7|L(}~1%
z?$J&%)ffS5L6|LI<kkWo8Yb!uPjLbO#wm8kL!bY7l&-PszNG4;35_uO%37w~_kKPM
zBr-2N=SU`7Z;5P$MMigXM<b$z+!`hbjju9OCMsU{jErd+0s!uGBL2r@%0n_q<dOgx
z#O469Q?<cLD6sZ)WY~lK-n0n$@{^uImxAp2sW>X*f>Q&JG+$Cu4q^ly+uF#lxhVnv
zU2hs`Jn!Ui>z0s`V=50{4nki!=B-RCbJFh#OZz5>PKLlT!5vD$`J{|^)OJMoliKLL
zmu#8!wAyKlBNJN*o=vo1Q(|>#Y-Au)X=iR_av&&EVs&Y3WFSIyX>4R5L}hSgZe(R{
zV|gG!a${&|c4Z(-WqBzeJYsceY-D6DbZ>8Lb1h_Lc4cfpY-w|Jb1q?QX>W9BE@Wk5
zX<=?IZ*pfoh`0n30RjLb1p-zCNFV|mF9r(<2nPcK1{DYb2?`4Y76JnS0v-VZ7k~f?
z2@nT;U_;3Udp>|o1OSx3R@!!GeO2yv*U_X3=WZ+PDC*k<$x^81WV$O(@N(X!KIC@!
zeb=xSnKhFGzZ}c<S7{R<>=yL0rd@Jf1&OKj`DLF)gj<}2xF~|^ufn$emLArzJ30U+
pBH$|lghxX(opymUmyCa05JsK*ihyKrl%!aG4JdUEL$`+VD?^GfP~ZRn

literal 0
HcmV?d00001

diff --git a/.spread-prepare.sh b/.spread-prepare.sh
new file mode 100755
index 0000000..4ac8ca0
--- /dev/null
+++ b/.spread-prepare.sh
@@ -0,0 +1,226 @@
+#!/bin/sh
+# This script is started by spread to prepare the execution environment
+set -x
+set -u
+set -e
+
+# Sanity check, are we in the top-level directory of the tree?
+test -f configure.ac || ( echo 'this script must be executed from the top-level of the tree' && exit 1)
+
+# Record where the top level directory is
+top_dir=$(pwd)
+
+# Record the current distribution release data to know what to do
+release_ID="$( . /etc/os-release && echo "${ID:-linux}" )"
+release_VERSION_ID="$( . /etc/os-release && echo "${VERSION_ID:-}" )"
+
+
+# Create source distribution tarball and place it in the top-level directory.
+create_dist_tarball() {
+    # Load the version number from a dedicated file
+    local pkg_version=
+    pkg_version="$(cat "$top_dir/VERSION")"
+
+    # Ensure that build system is up-to-date and ready
+    autoreconf -i
+    # XXX: This fixes somewhat odd error when configure below (in an empty directory) fails with:
+    # configure: error: source directory already configured; run "make distclean" there first
+    test -f Makefile && make distclean
+
+    # Create a scratch space to run configure
+    scratch_dir="$(mktemp -d)"
+    trap 'rm -rf "$scratch_dir"' EXIT
+
+    # Configure the project in a scratch directory
+    cd "$scratch_dir"
+    "$top_dir/configure" --prefix=/usr
+
+    # Create the distribution tarball
+    make dist
+
+    # Ensure we got the tarball we were expecting to see
+    test -f "snap-confine-$pkg_version.tar.gz"
+
+    # Move it to the top-level directory
+    mv "snap-confine-$pkg_version.tar.gz" "$top_dir/"
+}
+
+
+# build Ubuntu binary package and place it in $top_dir
+# $1 = /etc/os-release ID field
+# $2 = /etc/os-release VERSION_ID field (possibly empty!)
+build_debian_or_ubuntu_package() {
+    local pkg_version=
+    local distro_codename=
+    local distro_archive=
+    local distro_packaging_git=
+    local distro_packaging_git_branch=
+    local sbuild_args=
+    
+    pkg_version="$(cat "$top_dir/VERSION")"
+
+    case "$1" in
+        ubuntu)
+            distro_archive=http://archive.ubuntu.com/ubuntu
+            distro_packaging_git=https://git.launchpad.net/snap-confine
+            case "$2" in
+                14.04)
+                    distro_codename=trusty
+                    distro_packaging_git_branch=14.04
+                    ;;
+                16.04)
+                    distro_codename=xenial
+                    distro_packaging_git_branch=16.04
+                    ;;
+                16.10)
+                    distro_codename=yakkety
+                    distro_packaging_git_branch=16.10
+                    ;;
+                *)
+                    echo "unsupported Ubuntu VERSION_ID: $2"
+                    exit 1
+                    ;;
+            esac
+            # NOTE: universe has to be enabled as it is not enabled by default
+            sbuild_args="--extra-repository=deb http://archive.ubuntu.com/ubuntu/ ${distro_codename} universe"
+            ;;
+        debian)
+            distro_archive=http://ftp.debian.org/debian
+            # NOTE: Debian packaging needs to be updated. I sent a mail to the
+            # debian maintainer with instructions on what needs to happen and
+            # how it fits into the CI system.
+            #
+            # For now all builds on debian will fail as they still contains
+            # debian/patches that are now applied upstream.
+            distro_packaging_git=git://anonscm.debian.org/collab-maint/snap-confine.git
+            case "$2" in
+                '')  # sid
+                    distro_codename=sid
+                    distro_packaging_git_branch=debian
+                    ;;
+                *)
+                    echo "unsupported Debian VERSION_ID: $2"
+                    exit 1
+                    ;;
+            esac
+            ;;
+        *)
+            echo "unsupported distribution ID: $1"
+            exit 1
+            ;;
+    esac
+
+    # Ensure that we have a sbuild chroot ready
+    if ! schroot -l | grep "chroot:${distro_codename}-.*-sbuild"; then
+        sbuild-createchroot \
+            --include=eatmydata \
+            "--make-sbuild-tarball=/var/lib/sbuild/${distro_codename}-amd64.tar.gz" \
+            "$distro_codename" "$(mktemp -d)" \
+            "$distro_archive"
+    fi
+
+    # Create a scratch space 
+    scratch_dir="$(mktemp -d)"
+    trap 'rm -rf "$scratch_dir"' EXIT
+
+    # Do everything in the scratch directory
+    cd "$scratch_dir"
+
+    # Fetch the current Ubuntu packaging for the appropriate release
+    git clone -b "$distro_packaging_git_branch" "$distro_packaging_git" distro-packaging
+
+    # Install all the build dependencies declared by the package.
+    apt build-dep -y ./distro-packaging/
+
+    # Generate a new upstream tarball from the current state of the tree
+    ( cd "$top_dir" && create_dist_tarball )
+
+    # Prepare the .orig tarball and unpackaged source tree
+    cp "$top_dir/snap-confine-$pkg_version.tar.gz" "snap-confine_$pkg_version.orig.tar.gz"
+    tar -zxf "snap-confine_$pkg_version.orig.tar.gz" 
+
+    # Apply the debian directory from downstream packaging to form a complete source package
+    mv "distro-packaging/debian" "snap-confine-$pkg_version/debian"
+    rm -rf distro-packaging
+
+    # Add an automatically-generated changelog entry
+    # The --controlmaint takes the maintainer details from debian/control
+    ( cd "snap-confine-$pkg_version" && dch --controlmaint --newversion "${pkg_version}-1" "Automatic CI build")
+
+    # Build an unsigned source package
+    ( cd "snap-confine-$pkg_version" && dpkg-buildpackage -uc -us -S )
+
+    # Copy source package files to the top-level directory (this helps for
+    # interactive debugging since the package is available right there)
+    cp ./*.dsc ./*.debian.tar.* ./*.orig.tar.gz "$top_dir/"
+
+    # Build a binary package in a clean chroot.
+    # NOTE: nocheck is because the package still includes old unit tests that
+    # are deeply integrated into how ubuntu apparmor denials are logged. This
+    # should be removed once those test are migrated to spread testes.
+    DEB_BUILD_OPTIONS=nocheck sbuild \
+        --arch-all \
+        --dist="$distro_codename" \
+        --batch \
+        "$sbuild_args" \
+        "snap-confine_${pkg_version}-1.dsc"
+
+    # Copy all binary packages to the top-level directory
+    cp ./*.deb "$top_dir/"
+}
+
+
+# Apply tweaks
+case "$release_ID" in
+    ubuntu)
+        # apt update is hanging on security.ubuntu.com with IPv6.
+        sysctl -w net.ipv6.conf.all.disable_ipv6=1
+        trap "sysctl -w net.ipv6.conf.all.disable_ipv6=0" EXIT
+        ;;
+esac
+
+
+# Install all the build dependencies
+case "$release_ID" in
+    ubuntu|debian)
+        apt-get update
+        # On Debian and derivatives we need the following things:
+        # - sbuild -- to build the binary package with extra hygiene
+        # - devscripts -- to modify the changelog automatically
+        # - git -- to clone native downstream packaging
+        apt-get install --quiet -y sbuild devscripts git
+        # XXX: Taken from https://wiki.debian.org/sbuild
+        mkdir -p /root/.gnupg
+        # NOTE: We cannot use sbuild-update --keygen as virtual machines lack
+        # the necessary entropy to generate keys before the spread timeout
+        # kicks in. Instead we just copy pre-made, insecure keys from the
+        # source repository.
+        mkdir -p /var/lib/sbuild/apt-keys/
+        cp -a .spread-data/apt-keys/* /var/lib/sbuild/apt-keys/
+        sbuild-adduser "$LOGNAME"
+        ;;
+    *)
+        echo "unsupported distribution: $release_ID"
+        echo "patch spread-prepare to teach it about how to install build dependencies"
+        exit 1
+        ;;
+esac
+
+# Build and install the native package using downstream packaging and the fresh upstream tarball
+case "$release_ID" in
+    ubuntu|debian)
+        build_debian_or_ubuntu_package "$release_ID" "$release_VERSION_ID"
+        # Install the freshly-built packages
+        dpkg -i snap-confine_*.deb || apt-get -f install -y
+        dpkg -i ubuntu-core-launcher_*.deb || apt-get -f install -y
+        # Install snapd (testes require it)
+        apt-get install -y snapd
+        ;;
+    *)
+        echo "unsupported distribution: $release_ID"
+        exit 1
+        ;;
+esac
+
+# Install the core snap
+snap list | grep -q ubuntu-core || snap install ubuntu-core
diff --git a/Makefile.am b/Makefile.am
index 9e7b987..f319a7a 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -1,2 +1,9 @@
 SUBDIRS = src tests docs
 EXTRA_DIST = PORTING README.md VERSION
+
+.PHONY: check
+check: check-syntax
+
+.PHONY: check-syntax
+check-syntax:
+	shellcheck --format=gcc .spread-prepare.sh
diff --git a/spread.yaml b/spread.yaml
index 17643b9..fb75c88 100644
--- a/spread.yaml
+++ b/spread.yaml
@@ -10,45 +10,18 @@ backends:
         systems:
             - ubuntu-16.04-64-grub
             # - ubuntu-16.04-32-grub
-            - debian-8
 
 path: /remote/path/
 
 exclude:
     - .git
+    - debian
+    - autom4te.cache
 
 prepare: |
     echo "Spread is running as $(id)"
     [ "$REUSE_PROJECT" != 1 ] || exit 0
-    release_ID="$( . /etc/os-release && echo "${ID:-linux}" )"
-    case $release_ID in
-        ubuntu)
-            # apt update is hanging on security.ubuntu.com with IPv6.
-            sysctl -w net.ipv6.conf.all.disable_ipv6=1
-            trap "sysctl -w net.ipv6.conf.all.disable_ipv6=0" EXIT
-            ;;
-        debian)
-            echo "deb http://ftp.de.debian.org/debian sid main" > /etc/apt/sources.list.d/snappy.list
-            ;;
-    esac
-    case $release_ID in
-        ubuntu|debian)
-            apt-get update
-            apt-get install --quiet -y fakeroot
-            # Build a local copy of snap-confine
-            apt-get install --quiet -y autoconf automake autotools-dev debhelper dh-apparmor dh-autoreconf indent libapparmor-dev libseccomp-dev libudev-dev pkg-config shellcheck udev python3-docutils libglib2.0-dev
-            test -d /home/test || adduser --quiet --disabled-password --gecos '' test
-            chown test.test -R ..
-            sudo -i -u test /bin/sh -c "cd $PWD && DEB_BUILD_OPTIONS=nocheck dpkg-buildpackage -tc -b -Zgzip"
-            dpkg -i ../snap-confine_*.deb || apt-get -f install -y
-            dpkg -i ../ubuntu-core-launcher_*.deb || apt-get -f install -y
-            rm -f ../snap-confine_*.deb ../ubuntu-core-launcher_*.deb
-            # Install snapd (testes require it)
-            apt-get install -y snapd
-            ;;
-    esac
-    # Install the core snap
-    snap list | grep -q ubuntu-core || snap install ubuntu-core
+    ./.spread-prepare.sh
 
 suites:
     spread-tests/:

From 1c15cc2f1d7d4b0bee990e2cf2a6af1ae7a94cc5 Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Tue, 16 Aug 2016 16:19:36 +0200
Subject: [PATCH 2/7] source distro specific data

---
 spread-tests/distros/Debian.                  |  2 +
 spread-tests/distros/Debian.common            |  8 +++
 spread-tests/distros/Ubuntu.14.04             |  2 +
 spread-tests/distros/Ubuntu.16.04             |  2 +
 spread-tests/distros/Ubuntu.16.10             |  2 +
 spread-tests/distros/Ubuntu.common            |  2 +
 .../spread-prepare.sh                         | 69 ++-----------------
 spread.yaml                                   |  2 +-
 8 files changed, 25 insertions(+), 64 deletions(-)
 create mode 100644 spread-tests/distros/Debian.
 create mode 100644 spread-tests/distros/Debian.common
 create mode 100644 spread-tests/distros/Ubuntu.14.04
 create mode 100644 spread-tests/distros/Ubuntu.16.04
 create mode 100644 spread-tests/distros/Ubuntu.16.10
 create mode 100644 spread-tests/distros/Ubuntu.common
 rename .spread-prepare.sh => spread-tests/spread-prepare.sh (71%)

diff --git a/spread-tests/distros/Debian. b/spread-tests/distros/Debian.
new file mode 100644
index 0000000..4a5a9eb
--- /dev/null
+++ b/spread-tests/distros/Debian.
@@ -0,0 +1,2 @@
+distro_codename=sid
+distro_packaging_git_branch=debian
diff --git a/spread-tests/distros/Debian.common b/spread-tests/distros/Debian.common
new file mode 100644
index 0000000..de33716
--- /dev/null
+++ b/spread-tests/distros/Debian.common
@@ -0,0 +1,8 @@
+distro_archive=http://ftp.debian.org/debian
+# NOTE: Debian packaging needs to be updated. I sent a mail to the
+# debian maintainer with instructions on what needs to happen and
+# how it fits into the CI system.
+#
+# For now all builds on debian will fail as they still contains
+# debian/patches that are now applied upstream.
+distro_packaging_git=git://anonscm.debian.org/collab-maint/snap-confine.git
diff --git a/spread-tests/distros/Ubuntu.14.04 b/spread-tests/distros/Ubuntu.14.04
new file mode 100644
index 0000000..8471d57
--- /dev/null
+++ b/spread-tests/distros/Ubuntu.14.04
@@ -0,0 +1,2 @@
+distro_codename=trusty
+distro_packaging_git_branch=14.04
diff --git a/spread-tests/distros/Ubuntu.16.04 b/spread-tests/distros/Ubuntu.16.04
new file mode 100644
index 0000000..4e89a35
--- /dev/null
+++ b/spread-tests/distros/Ubuntu.16.04
@@ -0,0 +1,2 @@
+distro_codename=xenial
+distro_packaging_git_branch=16.04
diff --git a/spread-tests/distros/Ubuntu.16.10 b/spread-tests/distros/Ubuntu.16.10
new file mode 100644
index 0000000..374ea2e
--- /dev/null
+++ b/spread-tests/distros/Ubuntu.16.10
@@ -0,0 +1,2 @@
+distro_codename=yakkety
+distro_packaging_git_branch=16.10
diff --git a/spread-tests/distros/Ubuntu.common b/spread-tests/distros/Ubuntu.common
new file mode 100644
index 0000000..b177d58
--- /dev/null
+++ b/spread-tests/distros/Ubuntu.common
@@ -0,0 +1,2 @@
+distro_archive=http://archive.ubuntu.com/ubuntu
+distro_packaging_git=https://git.launchpad.net/snap-confine
diff --git a/.spread-prepare.sh b/spread-tests/spread-prepare.sh
similarity index 71%
rename from .spread-prepare.sh
rename to spread-tests/spread-prepare.sh
index 4ac8ca0..bbb1bd7 100755
--- a/.spread-prepare.sh
+++ b/spread-tests/spread-prepare.sh
@@ -45,70 +45,13 @@ create_dist_tarball() {
     mv "snap-confine-$pkg_version.tar.gz" "$top_dir/"
 }
 
+build_debian_or_ubuntu_package() { 
+    # FIXME: error handling and friendly message about how to
+    #        add new distro specific bits etc
+    # source the distro specific vars
+    . $top_dir/spread-tests/distros/$release_ID.common
+    . $top_dir/spread-tests/distros/$release_ID.$release_VERSION_ID
 
-# build Ubuntu binary package and place it in $top_dir
-# $1 = /etc/os-release ID field
-# $2 = /etc/os-release VERSION_ID field (possibly empty!)
-build_debian_or_ubuntu_package() {
-    local pkg_version=
-    local distro_codename=
-    local distro_archive=
-    local distro_packaging_git=
-    local distro_packaging_git_branch=
-    local sbuild_args=
-    
-    pkg_version="$(cat "$top_dir/VERSION")"
-
-    case "$1" in
-        ubuntu)
-            distro_archive=http://archive.ubuntu.com/ubuntu
-            distro_packaging_git=https://git.launchpad.net/snap-confine
-            case "$2" in
-                14.04)
-                    distro_codename=trusty
-                    distro_packaging_git_branch=14.04
-                    ;;
-                16.04)
-                    distro_codename=xenial
-                    distro_packaging_git_branch=16.04
-                    ;;
-                16.10)
-                    distro_codename=yakkety
-                    distro_packaging_git_branch=16.10
-                    ;;
-                *)
-                    echo "unsupported Ubuntu VERSION_ID: $2"
-                    exit 1
-                    ;;
-            esac
-            # NOTE: universe has to be enabled as it is not enabled by default
-            sbuild_args="--extra-repository=deb http://archive.ubuntu.com/ubuntu/ ${distro_codename} universe"
-            ;;
-        debian)
-            distro_archive=http://ftp.debian.org/debian
-            # NOTE: Debian packaging needs to be updated. I sent a mail to the
-            # debian maintainer with instructions on what needs to happen and
-            # how it fits into the CI system.
-            #
-            # For now all builds on debian will fail as they still contains
-            # debian/patches that are now applied upstream.
-            distro_packaging_git=git://anonscm.debian.org/collab-maint/snap-confine.git
-            case "$2" in
-                '')  # sid
-                    distro_codename=sid
-                    distro_packaging_git_branch=debian
-                    ;;
-                *)
-                    echo "unsupported Debian VERSION_ID: $2"
-                    exit 1
-                    ;;
-            esac
-            ;;
-        *)
-            echo "unsupported distribution ID: $1"
-            exit 1
-            ;;
-    esac
 
     # Ensure that we have a sbuild chroot ready
     if ! schroot -l | grep "chroot:${distro_codename}-.*-sbuild"; then
diff --git a/spread.yaml b/spread.yaml
index fb75c88..c3a3510 100644
--- a/spread.yaml
+++ b/spread.yaml
@@ -21,7 +21,7 @@ exclude:
 prepare: |
     echo "Spread is running as $(id)"
     [ "$REUSE_PROJECT" != 1 ] || exit 0
-    ./.spread-prepare.sh
+    ./spread-tests/spread-prepare.sh
 
 suites:
     spread-tests/:

From c20cc2dfd56e4d546f54e35c9fb9f679d101504a Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Tue, 16 Aug 2016 16:34:09 +0200
Subject: [PATCH 3/7] move .spread-data to spread-tests/data

---
 .../data}/apt-keys/README.md                        |   0
 .../data}/apt-keys/sbuild-key.pub                   | Bin
 .../data}/apt-keys/sbuild-key.sec                   | Bin
 spread-tests/spread-prepare.sh                      |   2 +-
 4 files changed, 1 insertion(+), 1 deletion(-)
 rename {.spread-data => spread-tests/data}/apt-keys/README.md (100%)
 rename {.spread-data => spread-tests/data}/apt-keys/sbuild-key.pub (100%)
 rename {.spread-data => spread-tests/data}/apt-keys/sbuild-key.sec (100%)

diff --git a/.spread-data/apt-keys/README.md b/spread-tests/data/apt-keys/README.md
similarity index 100%
rename from .spread-data/apt-keys/README.md
rename to spread-tests/data/apt-keys/README.md
diff --git a/.spread-data/apt-keys/sbuild-key.pub b/spread-tests/data/apt-keys/sbuild-key.pub
similarity index 100%
rename from .spread-data/apt-keys/sbuild-key.pub
rename to spread-tests/data/apt-keys/sbuild-key.pub
diff --git a/.spread-data/apt-keys/sbuild-key.sec b/spread-tests/data/apt-keys/sbuild-key.sec
similarity index 100%
rename from .spread-data/apt-keys/sbuild-key.sec
rename to spread-tests/data/apt-keys/sbuild-key.sec
diff --git a/spread-tests/spread-prepare.sh b/spread-tests/spread-prepare.sh
index bbb1bd7..5db0ac7 100755
--- a/spread-tests/spread-prepare.sh
+++ b/spread-tests/spread-prepare.sh
@@ -139,7 +139,7 @@ case "$release_ID" in
         # kicks in. Instead we just copy pre-made, insecure keys from the
         # source repository.
         mkdir -p /var/lib/sbuild/apt-keys/
-        cp -a .spread-data/apt-keys/* /var/lib/sbuild/apt-keys/
+        cp -a $top_level/spread-tests/data/apt-keys/* /var/lib/sbuild/apt-keys/
         sbuild-adduser "$LOGNAME"
         ;;
     *)

From aeeb1b0969fea9e3830e6cbdabe0252303a762b5 Mon Sep 17 00:00:00 2001
From: Michael Vogt <mvo@ubuntu.com>
Date: Tue, 16 Aug 2016 16:48:19 +0200
Subject: [PATCH 4/7] make spread happy again

---
 spread-tests/distros/{Debian. => debian.}     |  0
 .../distros/{Debian.common => debian.common}  |  0
 .../distros/{Ubuntu.14.04 => ubuntu.14.04}    |  0
 .../distros/{Ubuntu.16.04 => ubuntu.16.04}    |  0
 .../distros/{Ubuntu.16.10 => ubuntu.16.10}    |  0
 .../distros/{Ubuntu.common => ubuntu.common}  |  1 +
 spread-tests/{ => main}/cgroup-used/task.yaml |  0
 .../hostfs-created-on-demand/task.yaml        |  0
 .../media-visible-in-devmode/task.yaml        |  0
 .../task.yaml                                 |  0
 .../mount-profiles-bin-snap-source/task.yaml  |  0
 .../mount-profiles-missing-dst/task.yaml      |  0
 .../mount-profiles-missing-src/task.yaml      |  0
 .../mount-profiles-mount-tmpfs/task.yaml      |  0
 .../mount-profiles-ro-mount/task.yaml         |  0
 .../mount-profiles-rw-mount/task.yaml         |  0
 .../{ => main}/mount-usr-src/task.yaml        |  0
 .../{ => main}/test-snap-runs/task.yaml       |  0
 .../ubuntu-core-launcher-exists/task.yaml     |  0
 spread-tests/{ => main}/unit-tests/task.yaml  |  0
 .../user-data-dir-created/task.yaml           |  0
 spread-tests/spread-prepare.sh                | 31 ++++++++++---------
 spread.yaml                                   |  2 +-
 23 files changed, 19 insertions(+), 15 deletions(-)
 rename spread-tests/distros/{Debian. => debian.} (100%)
 rename spread-tests/distros/{Debian.common => debian.common} (100%)
 rename spread-tests/distros/{Ubuntu.14.04 => ubuntu.14.04} (100%)
 rename spread-tests/distros/{Ubuntu.16.04 => ubuntu.16.04} (100%)
 rename spread-tests/distros/{Ubuntu.16.10 => ubuntu.16.10} (100%)
 rename spread-tests/distros/{Ubuntu.common => ubuntu.common} (52%)
 rename spread-tests/{ => main}/cgroup-used/task.yaml (100%)
 rename spread-tests/{ => main}/hostfs-created-on-demand/task.yaml (100%)
 rename spread-tests/{ => main}/media-visible-in-devmode/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-bin-snap-destination/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-bin-snap-source/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-missing-dst/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-missing-src/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-mount-tmpfs/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-ro-mount/task.yaml (100%)
 rename spread-tests/{ => main}/mount-profiles-rw-mount/task.yaml (100%)
 rename spread-tests/{ => main}/mount-usr-src/task.yaml (100%)
 rename spread-tests/{ => main}/test-snap-runs/task.yaml (100%)
 rename spread-tests/{ => main}/ubuntu-core-launcher-exists/task.yaml (100%)
 rename spread-tests/{ => main}/unit-tests/task.yaml (100%)
 rename spread-tests/{ => main}/user-data-dir-created/task.yaml (100%)

diff --git a/spread-tests/distros/Debian. b/spread-tests/distros/debian.
similarity index 100%
rename from spread-tests/distros/Debian.
rename to spread-tests/distros/debian.
diff --git a/spread-tests/distros/Debian.common b/spread-tests/distros/debian.common
similarity index 100%
rename from spread-tests/distros/Debian.common
rename to spread-tests/distros/debian.common
diff --git a/spread-tests/distros/Ubuntu.14.04 b/spread-tests/distros/ubuntu.14.04
similarity index 100%
rename from spread-tests/distros/Ubuntu.14.04
rename to spread-tests/distros/ubuntu.14.04
diff --git a/spread-tests/distros/Ubuntu.16.04 b/spread-tests/distros/ubuntu.16.04
similarity index 100%
rename from spread-tests/distros/Ubuntu.16.04
rename to spread-tests/distros/ubuntu.16.04
diff --git a/spread-tests/distros/Ubuntu.16.10 b/spread-tests/distros/ubuntu.16.10
similarity index 100%
rename from spread-tests/distros/Ubuntu.16.10
rename to spread-tests/distros/ubuntu.16.10
diff --git a/spread-tests/distros/Ubuntu.common b/spread-tests/distros/ubuntu.common
similarity index 52%
rename from spread-tests/distros/Ubuntu.common
rename to spread-tests/distros/ubuntu.common
index b177d58..9acd710 100644
--- a/spread-tests/distros/Ubuntu.common
+++ b/spread-tests/distros/ubuntu.common
@@ -1,2 +1,3 @@
 distro_archive=http://archive.ubuntu.com/ubuntu
 distro_packaging_git=https://git.launchpad.net/snap-confine
+sbuild_args="--extra-repository=deb http://archive.ubuntu.com/ubuntu/ ${distro_codename} universe" 
\ No newline at end of file
diff --git a/spread-tests/cgroup-used/task.yaml b/spread-tests/main/cgroup-used/task.yaml
similarity index 100%
rename from spread-tests/cgroup-used/task.yaml
rename to spread-tests/main/cgroup-used/task.yaml
diff --git a/spread-tests/hostfs-created-on-demand/task.yaml b/spread-tests/main/hostfs-created-on-demand/task.yaml
similarity index 100%
rename from spread-tests/hostfs-created-on-demand/task.yaml
rename to spread-tests/main/hostfs-created-on-demand/task.yaml
diff --git a/spread-tests/media-visible-in-devmode/task.yaml b/spread-tests/main/media-visible-in-devmode/task.yaml
similarity index 100%
rename from spread-tests/media-visible-in-devmode/task.yaml
rename to spread-tests/main/media-visible-in-devmode/task.yaml
diff --git a/spread-tests/mount-profiles-bin-snap-destination/task.yaml b/spread-tests/main/mount-profiles-bin-snap-destination/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-bin-snap-destination/task.yaml
rename to spread-tests/main/mount-profiles-bin-snap-destination/task.yaml
diff --git a/spread-tests/mount-profiles-bin-snap-source/task.yaml b/spread-tests/main/mount-profiles-bin-snap-source/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-bin-snap-source/task.yaml
rename to spread-tests/main/mount-profiles-bin-snap-source/task.yaml
diff --git a/spread-tests/mount-profiles-missing-dst/task.yaml b/spread-tests/main/mount-profiles-missing-dst/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-missing-dst/task.yaml
rename to spread-tests/main/mount-profiles-missing-dst/task.yaml
diff --git a/spread-tests/mount-profiles-missing-src/task.yaml b/spread-tests/main/mount-profiles-missing-src/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-missing-src/task.yaml
rename to spread-tests/main/mount-profiles-missing-src/task.yaml
diff --git a/spread-tests/mount-profiles-mount-tmpfs/task.yaml b/spread-tests/main/mount-profiles-mount-tmpfs/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-mount-tmpfs/task.yaml
rename to spread-tests/main/mount-profiles-mount-tmpfs/task.yaml
diff --git a/spread-tests/mount-profiles-ro-mount/task.yaml b/spread-tests/main/mount-profiles-ro-mount/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-ro-mount/task.yaml
rename to spread-tests/main/mount-profiles-ro-mount/task.yaml
diff --git a/spread-tests/mount-profiles-rw-mount/task.yaml b/spread-tests/main/mount-profiles-rw-mount/task.yaml
similarity index 100%
rename from spread-tests/mount-profiles-rw-mount/task.yaml
rename to spread-tests/main/mount-profiles-rw-mount/task.yaml
diff --git a/spread-tests/mount-usr-src/task.yaml b/spread-tests/main/mount-usr-src/task.yaml
similarity index 100%
rename from spread-tests/mount-usr-src/task.yaml
rename to spread-tests/main/mount-usr-src/task.yaml
diff --git a/spread-tests/test-snap-runs/task.yaml b/spread-tests/main/test-snap-runs/task.yaml
similarity index 100%
rename from spread-tests/test-snap-runs/task.yaml
rename to spread-tests/main/test-snap-runs/task.yaml
diff --git a/spread-tests/ubuntu-core-launcher-exists/task.yaml b/spread-tests/main/ubuntu-core-launcher-exists/task.yaml
similarity index 100%
rename from spread-tests/ubuntu-core-launcher-exists/task.yaml
rename to spread-tests/main/ubuntu-core-launcher-exists/task.yaml
diff --git a/spread-tests/unit-tests/task.yaml b/spread-tests/main/unit-tests/task.yaml
similarity index 100%
rename from spread-tests/unit-tests/task.yaml
rename to spread-tests/main/unit-tests/task.yaml
diff --git a/spread-tests/user-data-dir-created/task.yaml b/spread-tests/main/user-data-dir-created/task.yaml
similarity index 100%
rename from spread-tests/user-data-dir-created/task.yaml
rename to spread-tests/main/user-data-dir-created/task.yaml
diff --git a/spread-tests/spread-prepare.sh b/spread-tests/spread-prepare.sh
index 5db0ac7..142c781 100755
--- a/spread-tests/spread-prepare.sh
+++ b/spread-tests/spread-prepare.sh
@@ -45,22 +45,16 @@ create_dist_tarball() {
     mv "snap-confine-$pkg_version.tar.gz" "$top_dir/"
 }
 
-build_debian_or_ubuntu_package() { 
+build_debian_or_ubuntu_package() {
+    local pkg_version
+    local sbuild_args=""
+    pkg_version="$(cat "$top_dir/VERSION")"
+    
     # FIXME: error handling and friendly message about how to
     #        add new distro specific bits etc
     # source the distro specific vars
-    . $top_dir/spread-tests/distros/$release_ID.common
-    . $top_dir/spread-tests/distros/$release_ID.$release_VERSION_ID
-
-
-    # Ensure that we have a sbuild chroot ready
-    if ! schroot -l | grep "chroot:${distro_codename}-.*-sbuild"; then
-        sbuild-createchroot \
-            --include=eatmydata \
-            "--make-sbuild-tarball=/var/lib/sbuild/${distro_codename}-amd64.tar.gz" \
-            "$distro_codename" "$(mktemp -d)" \
-            "$distro_archive"
-    fi
+    . "$top_dir/spread-tests/distros/$release_ID.$release_VERSION_ID"
+    . "$top_dir/spread-tests/distros/$release_ID.common"
 
     # Create a scratch space 
     scratch_dir="$(mktemp -d)"
@@ -97,6 +91,15 @@ build_debian_or_ubuntu_package() {
     # interactive debugging since the package is available right there)
     cp ./*.dsc ./*.debian.tar.* ./*.orig.tar.gz "$top_dir/"
 
+    # Ensure that we have a sbuild chroot ready
+    if ! schroot -l | grep "chroot:${distro_codename}-.*-sbuild"; then
+        sbuild-createchroot \
+            --include=eatmydata \
+            "--make-sbuild-tarball=/var/lib/sbuild/${distro_codename}-amd64.tar.gz" \
+            "$distro_codename" "$(mktemp -d)" \
+            "$distro_archive"
+    fi
+
     # Build a binary package in a clean chroot.
     # NOTE: nocheck is because the package still includes old unit tests that
     # are deeply integrated into how ubuntu apparmor denials are logged. This
@@ -139,7 +142,7 @@ case "$release_ID" in
         # kicks in. Instead we just copy pre-made, insecure keys from the
         # source repository.
         mkdir -p /var/lib/sbuild/apt-keys/
-        cp -a $top_level/spread-tests/data/apt-keys/* /var/lib/sbuild/apt-keys/
+        cp -a "$top_dir/spread-tests/data/apt-keys/"* /var/lib/sbuild/apt-keys/
         sbuild-adduser "$LOGNAME"
         ;;
     *)
diff --git a/spread.yaml b/spread.yaml
index c3a3510..bb20980 100644
--- a/spread.yaml
+++ b/spread.yaml
@@ -24,7 +24,7 @@ prepare: |
     ./spread-tests/spread-prepare.sh
 
 suites:
-    spread-tests/:
+    spread-tests/main/:
         summary: Full-system tests for snap-confine
     spread-tests/regression/:
         summary: Regression tests for past bug-fixes

From 84bc3170fbefe20754e1547fadb3738bd4a3cb1c Mon Sep 17 00:00:00 2001
From: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
Date: Tue, 16 Aug 2016 18:04:52 +0200
Subject: [PATCH 5/7] Abbreviate shell 'set' statements

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
---
 spread-tests/spread-prepare.sh | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/spread-tests/spread-prepare.sh b/spread-tests/spread-prepare.sh
index 142c781..c5c403a 100755
--- a/spread-tests/spread-prepare.sh
+++ b/spread-tests/spread-prepare.sh
@@ -1,8 +1,6 @@
 #!/bin/sh
 # This script is started by spread to prepare the execution environment
-set -x
-set -u
-set -e
+set -xue
 
 # Sanity check, are we in the top-level directory of the tree?
 test -f configure.ac || ( echo 'this script must be executed from the top-level of the tree' && exit 1)

From 505c3b1bec54b05914809e515d2d7e55908de2dd Mon Sep 17 00:00:00 2001
From: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
Date: Tue, 16 Aug 2016 18:11:19 +0200
Subject: [PATCH 6/7] Split-off release.sh from spread-prepare.sh

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
---
 spread-tests/release.sh        | 41 ++++++++++++++++++++++++++++++++++
 spread-tests/spread-prepare.sh | 32 +-------------------------
 2 files changed, 42 insertions(+), 31 deletions(-)
 create mode 100755 spread-tests/release.sh

diff --git a/spread-tests/release.sh b/spread-tests/release.sh
new file mode 100755
index 0000000..83f743c
--- /dev/null
+++ b/spread-tests/release.sh
@@ -0,0 +1,41 @@
+#!/bin/sh
+# This script creates a new release tarball
+set -xue
+
+# Sanity check, are we in the top-level directory of the tree?
+test -f configure.ac || ( echo 'this script must be executed from the top-level of the tree' && exit 1)
+
+# Record where the top level directory is
+top_dir=$(pwd)
+
+# Create source distribution tarball and place it in the top-level directory.
+create_dist_tarball() {
+    # Load the version number from a dedicated file
+    local pkg_version=
+    pkg_version="$(cat "$top_dir/VERSION")"
+
+    # Ensure that build system is up-to-date and ready
+    autoreconf -i
+    # XXX: This fixes somewhat odd error when configure below (in an empty directory) fails with:
+    # configure: error: source directory already configured; run "make distclean" there first
+    test -f Makefile && make distclean
+
+    # Create a scratch space to run configure
+    scratch_dir="$(mktemp -d)"
+    trap 'rm -rf "$scratch_dir"' EXIT
+
+    # Configure the project in a scratch directory
+    cd "$scratch_dir"
+    "$top_dir/configure" --prefix=/usr
+
+    # Create the distribution tarball
+    make dist
+
+    # Ensure we got the tarball we were expecting to see
+    test -f "snap-confine-$pkg_version.tar.gz"
+
+    # Move it to the top-level directory
+    mv "snap-confine-$pkg_version.tar.gz" "$top_dir/"
+}
+
+create_dist_tarball
diff --git a/spread-tests/spread-prepare.sh b/spread-tests/spread-prepare.sh
index c5c403a..0de0e83 100755
--- a/spread-tests/spread-prepare.sh
+++ b/spread-tests/spread-prepare.sh
@@ -13,36 +13,6 @@ release_ID="$( . /etc/os-release && echo "${ID:-linux}" )"
 release_VERSION_ID="$( . /etc/os-release && echo "${VERSION_ID:-}" )"
 
 
-# Create source distribution tarball and place it in the top-level directory.
-create_dist_tarball() {
-    # Load the version number from a dedicated file
-    local pkg_version=
-    pkg_version="$(cat "$top_dir/VERSION")"
-
-    # Ensure that build system is up-to-date and ready
-    autoreconf -i
-    # XXX: This fixes somewhat odd error when configure below (in an empty directory) fails with:
-    # configure: error: source directory already configured; run "make distclean" there first
-    test -f Makefile && make distclean
-
-    # Create a scratch space to run configure
-    scratch_dir="$(mktemp -d)"
-    trap 'rm -rf "$scratch_dir"' EXIT
-
-    # Configure the project in a scratch directory
-    cd "$scratch_dir"
-    "$top_dir/configure" --prefix=/usr
-
-    # Create the distribution tarball
-    make dist
-
-    # Ensure we got the tarball we were expecting to see
-    test -f "snap-confine-$pkg_version.tar.gz"
-
-    # Move it to the top-level directory
-    mv "snap-confine-$pkg_version.tar.gz" "$top_dir/"
-}
-
 build_debian_or_ubuntu_package() {
     local pkg_version
     local sbuild_args=""
@@ -68,7 +38,7 @@ build_debian_or_ubuntu_package() {
     apt build-dep -y ./distro-packaging/
 
     # Generate a new upstream tarball from the current state of the tree
-    ( cd "$top_dir" && create_dist_tarball )
+    ( cd "$top_dir" && spread-tests/release.sh )
 
     # Prepare the .orig tarball and unpackaged source tree
     cp "$top_dir/snap-confine-$pkg_version.tar.gz" "snap-confine_$pkg_version.orig.tar.gz"

From 422ad2d43ce93f594ab7e4d05394cece597fac41 Mon Sep 17 00:00:00 2001
From: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
Date: Tue, 16 Aug 2016 20:03:03 +0200
Subject: [PATCH 7/7] Improve error handling in spread-prepare.sh

Signed-off-by: Zygmunt Krynicki <zygmunt.krynicki@canonical.com>
---
 spread-tests/spread-prepare.sh | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/spread-tests/spread-prepare.sh b/spread-tests/spread-prepare.sh
index 0de0e83..0a54f93 100755
--- a/spread-tests/spread-prepare.sh
+++ b/spread-tests/spread-prepare.sh
@@ -15,15 +15,30 @@ release_VERSION_ID="$( . /etc/os-release && echo "${VERSION_ID:-}" )"
 
 build_debian_or_ubuntu_package() {
     local pkg_version
+    local distro_packaging_git_branch
+    local distro_packaging_git
+    local distro_archive
+    local distro_codename
     local sbuild_args=""
     pkg_version="$(cat "$top_dir/VERSION")"
     
-    # FIXME: error handling and friendly message about how to
-    #        add new distro specific bits etc
+    if [ ! -f "$top_dir/spread-tests/distros/$release_ID.$release_VERSION_ID" ] || \
+       [ ! -f "$top_dir/spread-tests/distros/$release_ID.common" ]; then
+        echo "Distribution: $release_ID (release $release_VERSION_ID) is not supported"
+        echo "please read this script and create new files in spread-test/distros"
+        exit 1
+    fi
+
     # source the distro specific vars
     . "$top_dir/spread-tests/distros/$release_ID.$release_VERSION_ID"
     . "$top_dir/spread-tests/distros/$release_ID.common"
 
+    # sanity check, ensure that essential variables were defined
+    test -n "$distro_packaging_git_branch"
+    test -n "$distro_packaging_git"
+    test -n "$distro_archive"
+    test -n "$distro_codename"
+
     # Create a scratch space 
     scratch_dir="$(mktemp -d)"
     trap 'rm -rf "$scratch_dir"' EXIT