Changed the implementation of Containers.migration to match the 'lxc … (

#319)

Implements migration from one remote to another.  Note that it doesn't necessarily support
clustered servers; this is a pre-clustering migration between remotes function.

* Changed the implementation of Containers.migration to match the 'lxc move <container_name> <remote_name>:' behaviour
* added tests that should also catch the started container and the exception

Signed-off-by: gabrik <gabriele.baldoni@gmail.com>

CONTRIBUTORS.rst

@@ -34,6 +34,7 @@ These are the contributors to pylxd according to the Github repository.
  chrismacnaughton Chris MacNaughton
  ppkt             Karol Werner
  mrtc0            Kohei Morita
+ gabrik           Gabriele Baldoni
  felix-engelmann  Felix Engelmann
  ===============  ==================================
 

integration/test_containers.py

@@ -12,7 +12,6 @@
 #    License for the specific language governing permissions and limitations
 #    under the License.
 from pylxd import exceptions
-
 from integration.testing import IntegrationTestCase
 
 
@@ -204,3 +203,66 @@ def test_publish(self):
         self.assertIn(
             image.fingerprint,
             [i.fingerprint for i in self.client.images.all()])
+
+    # COMMENT gabrik - 29/08/2018:
+    # This test is commented because CRIU does NOT work
+    # in LXD inside LXD
+
+    def test_migrate_running(self):
+        """A running container is migrated."""
+        from pylxd.client import Client
+        first_host = 'https://10.0.3.111:8443/'
+        second_host = 'https://10.0.3.222:8443/'
+
+        client1 = Client(endpoint=first_host, verify=False)
+        client1.authenticate('password')
+
+        client2 = Client(endpoint=second_host, verify=False)
+        client2.authenticate('password')
+        an_container = \
+            client1.containers.get(self.container.name)
+        an_container.start(wait=True)
+        an_container.sync()
+        an_migrated_container = \
+            an_container.migrate(client2, wait=True)
+
+        self.assertEqual(an_container.name,
+                         an_migrated_container.name)
+        self.assertEqual(client2,
+                         an_migrated_container.client)
+
+    def test_migrate_local_client(self):
+        """Raise ValueError, cannot migrate from local connection"""
+        from pylxd.client import Client
+
+        second_host = 'https://10.0.3.222:8443/'
+        client2 =\
+            Client(endpoint=second_host, verify=False)
+        client2.authenticate('password')
+
+        self.assertRaises(ValueError,
+                          self.container.migrate, client2)
+
+    def test_migrate_stopped(self):
+        """A stopped container is migrated."""
+        from pylxd.client import Client
+
+        first_host = 'https://10.0.3.111:8443/'
+        second_host = 'https://10.0.3.222:8443/'
+
+        client1 = \
+            Client(endpoint=first_host, verify=False)
+        client1.authenticate('password')
+
+        client2 = \
+            Client(endpoint=second_host, verify=False)
+        client2.authenticate('password')
+        an_container = \
+            client1.containers.get(self.container.name)
+        an_migrated_container = \
+            an_container.migrate(client2, wait=True)
+
+        self.assertEqual(an_container.name,
+                         an_migrated_container.name)
+        self.assertEqual(client2,
+                         an_migrated_container.client)

migration/busybox.py

@@ -0,0 +1,154 @@
+# This code is stolen directly from lxd-images, for expediency's sake.
+import atexit
+import hashlib
+import io
+import json
+import os
+import shutil
+import subprocess
+import tarfile
+import tempfile
+import uuid
+
+
+def find_on_path(command):
+    """Is command on the executable search path?"""
+
+    if 'PATH' not in os.environ:
+        return False
+    path = os.environ['PATH']
+    for element in path.split(os.pathsep):
+        if not element:
+            continue
+        filename = os.path.join(element, command)
+        if os.path.isfile(filename) and os.access(filename, os.X_OK):
+            return True
+    return False
+
+
+class Busybox(object):
+    workdir = None
+
+    def __init__(self):
+        # Create our workdir
+        self.workdir = tempfile.mkdtemp()
+
+    def cleanup(self):
+        if self.workdir:
+            shutil.rmtree(self.workdir)
+
+    def create_tarball(self, split=False):
+        xz = "pxz" if find_on_path("pxz") else "xz"
+
+        destination_tar = os.path.join(self.workdir, "busybox.tar")
+        target_tarball = tarfile.open(destination_tar, "w:")
+
+        if split:
+            destination_tar_rootfs = os.path.join(self.workdir,
+                                                  "busybox.rootfs.tar")
+            target_tarball_rootfs = tarfile.open(destination_tar_rootfs, "w:")
+
+        metadata = {'architecture': os.uname()[4],
+                    'creation_date': int(os.stat("/bin/busybox").st_ctime),
+                    'properties': {
+                        'os': "Busybox",
+                        'architecture': os.uname()[4],
+                        'description': "Busybox %s" % os.uname()[4],
+                        'name': "busybox-%s" % os.uname()[4],
+                        # Don't overwrite actual busybox images.
+                        'obfuscate': str(uuid.uuid4()), },
+                    }
+
+        # Add busybox
+        with open("/bin/busybox", "rb") as fd:
+            busybox_file = tarfile.TarInfo()
+            busybox_file.size = os.stat("/bin/busybox").st_size
+            busybox_file.mode = 0o755
+            if split:
+                busybox_file.name = "bin/busybox"
+                target_tarball_rootfs.addfile(busybox_file, fd)
+            else:
+                busybox_file.name = "rootfs/bin/busybox"
+                target_tarball.addfile(busybox_file, fd)
+
+        # Add symlinks
+        busybox = subprocess.Popen(["/bin/busybox", "--list-full"],
+                                   stdout=subprocess.PIPE,
+                                   universal_newlines=True)
+        busybox.wait()
+
+        for path in busybox.stdout.read().split("\n"):
+            if not path.strip():
+                continue
+
+            symlink_file = tarfile.TarInfo()
+            symlink_file.type = tarfile.SYMTYPE
+            symlink_file.linkname = "/bin/busybox"
+            if split:
+                symlink_file.name = "%s" % path.strip()
+                target_tarball_rootfs.addfile(symlink_file)
+            else:
+                symlink_file.name = "rootfs/%s" % path.strip()
+                target_tarball.addfile(symlink_file)
+
+        # Add directories
+        for path in ("dev", "mnt", "proc", "root", "sys", "tmp"):
+            directory_file = tarfile.TarInfo()
+            directory_file.type = tarfile.DIRTYPE
+            if split:
+                directory_file.name = "%s" % path
+                target_tarball_rootfs.addfile(directory_file)
+            else:
+                directory_file.name = "rootfs/%s" % path
+                target_tarball.addfile(directory_file)
+
+        # Add the metadata file
+        metadata_yaml = json.dumps(metadata, sort_keys=True,
+                                   indent=4, separators=(',', ': '),
+                                   ensure_ascii=False).encode('utf-8') + b"\n"
+
+        metadata_file = tarfile.TarInfo()
+        metadata_file.size = len(metadata_yaml)
+        metadata_file.name = "metadata.yaml"
+        target_tarball.addfile(metadata_file,
+                               io.BytesIO(metadata_yaml))
+
+        # Add an /etc/inittab; this is to work around:
+        # http://lists.busybox.net/pipermail/busybox/2015-November/083618.html
+        # Basically, since there are some hardcoded defaults that misbehave, we
+        # just pass an empty inittab so those aren't applied, and then busybox
+        # doesn't spin forever.
+        inittab = tarfile.TarInfo()
+        inittab.size = 1
+        inittab.name = "/rootfs/etc/inittab"
+        target_tarball.addfile(inittab, io.BytesIO(b"\n"))
+
+        target_tarball.close()
+        if split:
+            target_tarball_rootfs.close()
+
+        # Compress the tarball
+        r = subprocess.call([xz, "-9", destination_tar])
+        if r:
+            raise Exception("Failed to compress: %s" % destination_tar)
+
+        if split:
+            r = subprocess.call([xz, "-9", destination_tar_rootfs])
+            if r:
+                raise Exception("Failed to compress: %s" %
+                                destination_tar_rootfs)
+            return destination_tar + ".xz", destination_tar_rootfs + ".xz"
+        else:
+            return destination_tar + ".xz"
+
+
+def create_busybox_image():
+    busybox = Busybox()
+    atexit.register(busybox.cleanup)
+
+    path = busybox.create_tarball()
+
+    with open(path, "rb") as fd:
+        fingerprint = hashlib.sha256(fd.read()).hexdigest()
+
+    return path, fingerprint

migration/run_migration_integration_tests-18-04

@@ -0,0 +1,86 @@
+#!/bin/sh
+
+#NOTE: gabrik (24 Aug 2018) - this test create two containers for testing the migration
+
+NETWORK_NAME=pylxd0
+CONTAINER_IMAGE=ubuntu:18.04
+CONTAINERONE_NAME=pylxd-`uuidgen | cut -d"-" -f1`
+CONTAINERTWO_NAME=pylxd-`uuidgen | cut -d"-" -f1`
+PROFILE_NAME=pylxd
+
+
+#This create a network for testing the migration
+lxc network create $NETWORK_NAME ipv6.address=none ipv4.address=10.0.3.1/24 ipv4.nat=true
+
+#This create the profile used by the containers to be attached in the same network
+lxc profile copy default $PROFILE_NAME
+lxc profile device remove $PROFILE_NAME eth0
+lxc network attach-profile $NETWORK_NAME $PROFILE_NAME eth0
+
+# This creates a privileged container, because I was bumping into situations where it
+# seemed that we had maxed out user namespaces (I haven't checked it out, but it's likely
+# a bug in LXD).
+
+#First container
+lxc init $CONTAINER_IMAGE $CONTAINERONE_NAME -p $PROFILE_NAME -c security.nesting=true -c security.privileged=true
+lxc config set $CONTAINERONE_NAME raw.lxc "lxc.mount.auto = proc:rw sys:rw"
+lxc start $CONTAINERONE_NAME
+sleep 5  # Wait for the network to come up
+lxc exec $CONTAINERONE_NAME -- apt-get update
+lxc exec $CONTAINERONE_NAME -- apt-get install -y tox python3-dev libssl-dev libffi-dev build-essential criu
+lxc exec $CONTAINERONE_NAME -- lxc config set core.trust_password password
+lxc exec $CONTAINERONE_NAME -- lxc config set core.https_address [::]
+lxc exec $CONTAINERONE_NAME -- mkdir -p /root/.config/lxc
+openssl genrsa 1024 > ./$CONTAINERONE_NAME.key
+lxc file push ./$CONTAINERONE_NAME.key $CONTAINERONE_NAME/root/.config/lxc/client.key
+rm ./$CONTAINERONE_NAME.key
+lxc exec $CONTAINERONE_NAME -- chmod 400 /root/.config/lxc/client.key
+lxc exec $CONTAINERONE_NAME -- openssl req -new -x509 -nodes -sha1 -days 365 \
+	-key /root/.config/lxc/client.key -out /root/.config/lxc/client.crt \
+	-subj="/C=UK/ST=London/L=London/O=OrgName/OU=Test/CN=example.com"
+
+# create a default dir storage pool for bionic
+lxc exec $CONTAINERONE_NAME -- lxc storage create default dir
+lxc exec $CONTAINERONE_NAME -- lxc profile device add default root disk path=/ pool=default
+
+lxc exec $CONTAINERONE_NAME -- sudo shutdown -r now
+sleep 5  # Wait for the network to come up
+lxc exec $CONTAINERONE_NAME -- sudo ifconfig eth0 10.0.3.111 netmask 255.255.255.0
+lxc exec $CONTAINERONE_NAME -- route add default gw 10.0.3.1 eth0
+
+#Second container
+lxc init $CONTAINER_IMAGE $CONTAINERTWO_NAME -p $PROFILE_NAME -c security.nesting=true -c security.privileged=true
+lxc config set $CONTAINERTWO_NAME raw.lxc "lxc.mount.auto = proc:rw sys:rw"
+lxc start $CONTAINERTWO_NAME
+sleep 5  # Wait for the network to come up
+lxc exec $CONTAINERTWO_NAME -- apt-get update
+lxc exec $CONTAINERTWO_NAME -- apt-get install -y tox python3-dev libssl-dev libffi-dev build-essential criu
+lxc exec $CONTAINERTWO_NAME -- lxc config set core.trust_password password
+lxc exec $CONTAINERTWO_NAME -- lxc config set core.https_address [::]:8443
+lxc exec $CONTAINERONE_NAME -- mkdir -p /root/.config/lxc
+openssl genrsa 1024 > ./$CONTAINERTWO_NAME.key
+lxc file push ./$CONTAINERTWO_NAME.key $CONTAINERTWO_NAME/root/.config/lxc/client.key
+rm ./$CONTAINERTWO_NAME.key
+lxc exec $CONTAINERTWO_NAME -- chmod 400 /root/.config/lxc/client.key
+lxc exec $CONTAINERTWO_NAME -- openssl req -new -x509 -nodes -sha1 -days 365 \
+	-key /root/.config/lxc/client.key -out /root/.config/lxc/client.crt \
+	-subj="/C=UK/ST=London/L=London/O=OrgName/OU=Test/CN=example.com"
+
+# create a default dir storage pool for bionic
+lxc exec $CONTAINERTWO_NAME -- lxc storage create default dir
+lxc exec $CONTAINERTWO_NAME -- lxc profile device add default root disk path=/ pool=default
+lxc exec $CONTAINERTWO_NAME -- sudo shutdown -r now
+sleep 5  # Wait for the network to come up
+lxc exec $CONTAINERTWO_NAME -- sudo ifconfig eth0 10.0.3.222 netmask 255.255.255.0
+lxc exec $CONTAINERTWO_NAME -- route add default gw 10.0.3.1 eth0
+
+lxc exec $CONTAINERONE_NAME -- mkdir -p /opt/pylxd
+# NOTE: rockstar (13 Sep 2016) - --recursive is not supported in lxd <2.1, so
+# until we have pervasive support for that, we'll do this tar hack.
+tar cf - * .git | lxc exec $CONTAINERONE_NAME -- tar xf - -C /opt/pylxd
+lxc exec $CONTAINERONE_NAME -- /bin/sh -c "cd /opt/pylxd && tox -emigration"
+
+lxc delete --force $CONTAINERONE_NAME
+lxc delete --force $CONTAINERTWO_NAME
+lxc profile delete $PROFILE_NAME
+lxc network delete $NETWORK_NAME