Merge branch '1.0.x' into 'master'

reprounzip-docker/reprounzip_docker.py

@@ -29,7 +29,7 @@
 from reprounzip.unpackers.common import COMPAT_OK, COMPAT_MAYBE, \
     UsageError, CantFindInstaller, composite_action, target_must_exist, \
     make_unique_name, shell_escape, select_installer, busybox_url, \
-    rpzsudo_binary, FileUploader, FileDownloader, get_runs, \
+    rpzsudo_binary, rpztar_url, FileUploader, FileDownloader, get_runs, \
     add_environment_options, parse_environment_args, interruptible_call, \
     metadata_read, metadata_write, metadata_initial_iofiles, \
     metadata_update_run, parse_ports
@@ -180,10 +180,16 @@ def docker_setup_create(args):
                     shutil.copyfileobj(f_in, f_out)
             dockerfile.write('COPY rpzsudo /rpzsudo\n\n')
 
+            # Installs rpztar
+            download_file(rpztar_url(arch),
+                          target / 'rpztar',
+                          'rpztar-%s' % arch)
+            dockerfile.write('COPY rpztar /rpztar\n\n')
+
             dockerfile.write('COPY data.tgz /reprozip_data.tgz\n\n')
             dockerfile.write('COPY rpz-files.list /rpz-files.list\n')
             dockerfile.write('RUN \\\n'
-                             '    chmod +x /busybox /rpzsudo && \\\n')
+                             '    chmod +x /busybox /rpzsudo /rpztar && \\\n')
 
             if args.install_pkgs:
                 # Install every package through package manager
@@ -242,19 +248,14 @@ def docker_setup_create(args):
                     else:
                         logger.info("Missing file %s", path)
             rpz_pack.close()
-            # FIXME : for some reason we need reversed() here, I'm not sure why
-            # Need to read more of tar's docs.
-            # TAR bug: --no-overwrite-dir removes --keep-old-files
             with (target / 'rpz-files.list').open('wb') as filelist:
-                for p in reversed(pathlist):
-                    filelist.write(join_root(rpz_pack.data_prefix, p).path)
+                for p in pathlist:
+                    filelist.write(join_root(PosixPath(''), p).path)
                     filelist.write(b'\0')
             dockerfile.write(
                 '    cd / && '
-                '(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
-                '--numeric-owner --strip=1 --null -T /rpz-files.list || '
-                '/busybox echo "TAR reports errors, this might or might '
-                'not prevent the execution to run")\n')
+                + '/rpztar /reprozip_data.tgz /rpz-files.list\n',
+            )
 
             # Setup entry point
             dockerfile.write(

reprounzip-vagrant/reprounzip_vagrant/__init__.py

@@ -27,6 +27,7 @@
 from reprounzip.unpackers.common import COMPAT_OK, COMPAT_MAYBE, COMPAT_NO, \
     CantFindInstaller, composite_action, target_must_exist, \
     make_unique_name, shell_escape, select_installer, busybox_url, join_root, \
+    rpztar_url, \
     FileUploader, FileDownloader, get_runs, add_environment_options, \
     fixup_environment, metadata_read, metadata_write, \
     metadata_initial_iofiles, metadata_update_run, parse_ports
@@ -155,23 +156,28 @@ def machine_setup(target):
 
     if use_chroot:
         # Mount directories
+        logger.debug("Mounting directories")
         ssh = paramiko.SSHClient()
         ssh.set_missing_host_key_policy(IgnoreMissingKey())
         ssh.connect(**info)
         chan = ssh.get_transport().open_session()
         chan.exec_command(
             '/usr/bin/sudo /bin/sh -c %s' % shell_escape(
-                'if ! grep -q "/experimentroot/dev " /proc/mounts; then '
+                'if ! grep -q "/experimentroot " /etc/mtab; then '
+                'mount -o bind /.experimentdata /experimentroot; '
+                'fi; '
+                'if ! grep -q "/experimentroot/dev " /etc/mtab; then '
                 'mount -o rbind /dev /experimentroot/dev; '
                 'fi; '
-                'if ! grep -q "/experimentroot/proc " /proc/mounts; then '
+                'if ! grep -q "/experimentroot/proc " /etc/mtab; then '
                 'mount -t proc none /experimentroot/proc; '
                 'fi'))
         if chan.recv_exit_status() != 0:
             logger.critical("Couldn't mount directories in chroot")
             sys.exit(1)
         if gui:
             # Mount X11 socket
+            logger.debug("Mounting X11 socket")
             chan = ssh.get_transport().open_session()
             chan.exec_command(
                 '/usr/bin/sudo /bin/sh -c %s' % shell_escape(
@@ -185,6 +191,8 @@ def machine_setup(target):
                 logger.critical("Couldn't mount X11 sockets in chroot")
                 sys.exit(1)
         ssh.close()
+    else:
+        logger.debug("NOT mounting directories")
 
     return info
 
@@ -276,45 +284,58 @@ def vagrant_setup_create(args):
         # Writes setup script
         logger.info("Writing setup script %s...", target / 'setup.sh')
         with (target / 'setup.sh').open('w', encoding='utf-8',
-                                        newline='\n') as fp:
-            fp.write('#!/bin/sh\n\nset -e\n\n')
+                                        newline='\n') as script:
+            script.write('#!/bin/sh\n\nset -e\n\n')
             if packages:
                 # Updates package sources
                 update_script = installer.update_script()
                 if update_script:
-                    fp.write(update_script)
-                fp.write('\n')
+                    script.write(update_script)
+                script.write('\n')
                 # Installs necessary packages
-                fp.write(installer.install_script(packages))
-                fp.write('\n')
+                script.write(installer.install_script(packages))
+                script.write('\n')
                 # TODO : Compare package versions (painful because of sh)
 
+            # Copies rpztar
+            if not use_chroot:
+                arch = runs[0]['architecture']
+                download_file(rpztar_url(arch),
+                              target / 'rpztar',
+                              'rpztar-%s' % arch)
+                script.write(r'''
+cp /vagrant/rpztar /usr/local/bin/rpztar
+chmod +x /usr/local/bin/rpztar
+''')
+
             # Untar
             if use_chroot:
-                fp.write('\n'
-                         'mkdir /experimentroot; cd /experimentroot\n')
-                fp.write('tar zpxf /vagrant/data.tgz --numeric-owner '
-                         '--strip=1 %s\n' % rpz_pack.data_prefix)
+                script.write('\n'
+                             'mkdir /experimentroot\n'
+                             'mkdir /.experimentdata; cd /.experimentdata\n')
+                script.write('tar zpxf /vagrant/data.tgz --numeric-owner '
+                             '--strip=1 %s\n' % rpz_pack.data_prefix)
                 if mount_bind:
-                    fp.write('\n'
-                             'mkdir -p /experimentroot/dev\n'
-                             'mkdir -p /experimentroot/proc\n')
+                    script.write('\n'
+                                 'mkdir -p /.experimentdata/dev\n'
+                                 'mkdir -p /.experimentdata/proc\n')
 
                 for pkg in packages:
-                    fp.write('\n# Copies files from package %s\n' % pkg.name)
+                    script.write('\n# Copies files from package %s\n'
+                                 % pkg.name)
                     for f in pkg.files:
                         f = f.path
                         dest = join_root(PosixPath('/experimentroot'), f)
-                        fp.write('mkdir -p %s\n' %
-                                 shell_escape(str(f.parent)))
-                        fp.write('cp -L %s %s\n' % (
-                                 shell_escape(str(f)),
-                                 shell_escape(str(dest))))
-                fp.write(
+                        script.write('mkdir -p %s\n' %
+                                     shell_escape(str(f.parent)))
+                        script.write('cp -L %s %s\n' % (
+                                     shell_escape(str(f)),
+                                     shell_escape(str(dest))))
+                script.write(
                     '\n'
-                    'cp /etc/resolv.conf /experimentroot/etc/resolv.conf\n')
+                    'cp /etc/resolv.conf /.experimentdata/etc/resolv.conf\n')
             else:
-                fp.write('\ncd /\n')
+                script.write('\ncd /\n')
                 paths = set()
                 pathlist = []
                 # Adds intermediate directories, and checks for existence in
@@ -337,33 +358,26 @@ def vagrant_setup_create(args):
                             pathlist.append(path)
                         else:
                             logger.info("Missing file %s", path)
-                # FIXME : for some reason we need reversed() here, I'm not sure
-                # why. Need to read more of tar's docs.
-                # TAR bug: --no-overwrite-dir removes --keep-old-files
-                # TAR bug: there is no way to make --keep-old-files not report
-                # an error if an existing file is encountered. --skip-old-files
-                # was introduced too recently. Instead, we just ignore the exit
-                # status
-                with (target / 'rpz-files.list').open('wb') as lfp:
-                    for p in reversed(pathlist):
-                        lfp.write(join_root(rpz_pack.data_prefix, p).path)
-                        lfp.write(b'\0')
-                fp.write('tar zpxf /vagrant/data.tgz --keep-old-files '
-                         '--numeric-owner --strip=1 '
-                         '--null -T /vagrant/rpz-files.list || /bin/true\n')
+                with (target / 'rpz-files.list').open('wb') as filelist:
+                    for p in pathlist:
+                        filelist.write(join_root(PosixPath(''), p).path)
+                        filelist.write(b'\0')
+                script.write('/usr/local/bin/rpztar '
+                             '/vagrant/data.tgz '
+                             '/vagrant/rpz-files.list\n')
 
             # Copies busybox
             if use_chroot:
                 arch = runs[0]['architecture']
                 download_file(busybox_url(arch),
                               target / 'busybox',
                               'busybox-%s' % arch)
-                fp.write(r'''
-cp /vagrant/busybox /experimentroot/busybox
-chmod +x /experimentroot/busybox
-mkdir -p /experimentroot/bin
-[ -e /experimentroot/bin/sh ] || \
-    ln -s /busybox /experimentroot/bin/sh
+                script.write(r'''
+cp /vagrant/busybox /.experimentdata/busybox
+chmod +x /.experimentdata/busybox
+mkdir -p /.experimentdata/bin
+[ -e /.experimentdata/bin/sh ] || \
+    ln -s /busybox /.experimentdata/bin/sh
 ''')
 
         # Copies pack
@@ -398,33 +412,33 @@ def write_vagrantfile(target, unpacked_info):
 
     logger.info("Writing %s...", target / 'Vagrantfile')
     with (target / 'Vagrantfile').open('w', encoding='utf-8',
-                                       newline='\n') as fp:
+                                       newline='\n') as vgfile:
         # Vagrant header and version
-        fp.write(
+        vgfile.write(
             '# -*- mode: ruby -*-\n'
             '# vi: set ft=ruby\n\n'
             'VAGRANTFILE_API_VERSION = "2"\n\n'
             'Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|\n')
         # Selects which box to install
-        fp.write('  config.vm.box = "%s"\n' % box)
+        vgfile.write('  config.vm.box = "%s"\n' % box)
         # Run the setup script on the virtual machine
-        fp.write('  config.vm.provision "shell", path: "setup.sh"\n')
+        vgfile.write('  config.vm.provision "shell", path: "setup.sh"\n')
 
         # Memory size
         if memory is not None or gui:
-            fp.write('  config.vm.provider "virtualbox" do |v|\n')
+            vgfile.write('  config.vm.provider "virtualbox" do |v|\n')
             if memory is not None:
-                fp.write('    v.memory = %d\n' % memory)
+                vgfile.write('    v.memory = %d\n' % memory)
             if gui:
-                fp.write('    v.gui = true\n')
-            fp.write('  end\n')
+                vgfile.write('    v.gui = true\n')
+            vgfile.write('  end\n')
 
         # Port forwarding
         for port in ports:
-            fp.write('  config.vm.network "forwarded_port", host: '
-                     '%s, guest: %s, protocol: "%s"\n' % port)
+            vgfile.write('  config.vm.network "forwarded_port", host: '
+                         '%s, guest: %s, protocol: "%s"\n' % port)
 
-        fp.write('end\n')
+        vgfile.write('end\n')
 
 
 @target_must_exist
@@ -695,7 +709,7 @@ def download(self, remote_path, local_path):
 
         # Move file to final destination
         try:
-            ltemp.rename(local_path)
+            ltemp.move(local_path)
         except OSError as e:
             logger.critical("Couldn't download output file: %s\n%s",
                             remote_path, str(e))

reprounzip/reprounzip/parameters.py

@@ -93,6 +93,12 @@ def get_parameter(section):
         "x86_64": "https://s3.amazonaws.com/reprozip-files/busybox-x86_64",
         "i686": "https://s3.amazonaws.com/reprozip-files/busybox-i686"
     },
+    "rpztar_url": {
+        "x86_64": "https://github.com/remram44/rpztar/releases/download/"
+                  "v1/rpztar-x86_64",
+        "i686": "https://github.com/remram44/rpztar/releases/download/"
+                "v1/rpztar-i686"
+    },
     "docker_images": {
         "default": {
             "distribution": "debian",

reprounzip/reprounzip/unpackers/common/__init__.py

@@ -13,6 +13,7 @@
     COMPAT_OK, COMPAT_NO, COMPAT_MAYBE, \
     composite_action, target_must_exist, unique_names, \
     make_unique_name, shell_escape, load_config, busybox_url, rpzsudo_binary, \
+    rpztar_url, \
     FileUploader, FileDownloader, get_runs, add_environment_options, \
     parse_environment_args, fixup_environment, interruptible_call, \
     metadata_read, metadata_write, metadata_initial_iofiles, \
@@ -26,7 +27,7 @@
            'UsageError', 'CantFindInstaller',
            'composite_action', 'target_must_exist', 'unique_names',
            'make_unique_name', 'shell_escape', 'load_config', 'busybox_url',
-           'rpzsudo_binary',
+           'rpzsudo_binary', 'rpztar_url',
            'join_root', 'FileUploader', 'FileDownloader', 'get_runs',
            'add_environment_options', 'parse_environment_args',
            'fixup_environment', 'interruptible_call', 'metadata_read',

reprounzip/reprounzip/unpackers/common/misc.py

@@ -135,6 +135,12 @@ def rpzsudo_binary(arch):
     )
 
 
+def rpztar_url(arch):
+    """Gets the correct URL for the rpztar binary given the architecture.
+    """
+    return get_parameter('rpztar_url')[arch]
+
+
 class FileUploader(object):
     """Common logic for 'upload' commands.
     """

tests/functional.py

@@ -407,8 +407,8 @@ def check_simple(args, stream, infile=1):
             # Get output file
             check_call(rpuz + ['vagrant', 'download',
                                (tests / 'vagrant/simplevagrantchroot').path,
-                               'arg2:voutput1.txt'])
-            with Path('voutput1.txt').open(encoding='utf-8') as fp:
+                               'arg2:voutput3.txt'])
+            with Path('voutput3.txt').open(encoding='utf-8') as fp:
                 assert fp.read().strip() == '42'
             # Replace input file via path
             check_call(rpuz + ['vagrant', 'upload',
@@ -476,8 +476,8 @@ def check_simple(args, stream, infile=1):
             # Get output file
             check_call(rpuz + ['vagrant', 'download',
                                (tests / 'vagrant/simplevagrant').path,
-                               'arg2:voutput1.txt'])
-            with Path('voutput1.txt').open(encoding='utf-8') as fp:
+                               'arg2:woutput3.txt'])
+            with Path('woutput3.txt').open(encoding='utf-8') as fp:
                 assert fp.read().strip() == '42'
             # Destroy
             check_call(rpuz + ['vagrant', 'destroy',