Merge pull request #374 from VIDA-NYU/docker-tar

Develop custom 'tar' replacement to merge bundle with existing system

reprounzip-docker/reprounzip/unpackers/docker.py

@@ -30,6 +30,7 @@
 from reprounzip.unpackers.common import COMPAT_OK, COMPAT_MAYBE, \
     UsageError, CantFindInstaller, composite_action, target_must_exist, \
     make_unique_name, shell_escape, select_installer, busybox_url, sudo_url, \
+    rpztar_url, \
     FileUploader, FileDownloader, get_runs, add_environment_options, \
     fixup_environment, interruptible_call, metadata_read, metadata_write, \
     metadata_initial_iofiles, metadata_update_run, parse_ports
@@ -182,10 +183,16 @@ def docker_setup_create(args):
                           'rpzsudo-%s' % arch)
             dockerfile.write('COPY rpzsudo /rpzsudo\n\n')
 
+            # Installs rpztar
+            download_file(rpztar_url(arch),
+                          target / 'rpztar',
+                          'rpztar-%s' % arch)
+            dockerfile.write('COPY rpztar /rpztar\n\n')
+
             dockerfile.write('COPY data.tgz /reprozip_data.tgz\n\n')
             dockerfile.write('COPY rpz-files.list /rpz-files.list\n')
             dockerfile.write('RUN \\\n'
-                             '    chmod +x /busybox /rpzsudo && \\\n')
+                             '    chmod +x /busybox /rpzsudo /rpztar && \\\n')
 
             if args.install_pkgs:
                 # Install every package through package manager
@@ -244,19 +251,14 @@ def docker_setup_create(args):
                     else:
                         logger.info("Missing file %s", path)
             rpz_pack.close()
-            # FIXME : for some reason we need reversed() here, I'm not sure why
-            # Need to read more of tar's docs.
-            # TAR bug: --no-overwrite-dir removes --keep-old-files
             with (target / 'rpz-files.list').open('wb') as filelist:
-                for p in reversed(pathlist):
-                    filelist.write(join_root(rpz_pack.data_prefix, p).path)
+                for p in pathlist:
+                    filelist.write(join_root(PosixPath(''), p).path)
                     filelist.write(b'\0')
             dockerfile.write(
                 '    cd / && '
-                '(tar zpxf /reprozip_data.tgz -U --recursive-unlink '
-                '--numeric-owner --strip=1 --null -T /rpz-files.list || '
-                '/busybox echo "TAR reports errors, this might or might '
-                'not prevent the execution to run")\n')
+                + '/rpztar /reprozip_data.tgz /rpz-files.list\n',
+            )
 
         # Meta-data for reprounzip
         write_dict(target, metadata_initial_iofiles(config))

reprounzip-vagrant/reprounzip/unpackers/vagrant/__init__.py

@@ -29,6 +29,7 @@
 from reprounzip.unpackers.common import COMPAT_OK, COMPAT_MAYBE, COMPAT_NO, \
     CantFindInstaller, composite_action, target_must_exist, \
     make_unique_name, shell_escape, select_installer, busybox_url, join_root, \
+    rpztar_url, \
     FileUploader, FileDownloader, get_runs, add_environment_options, \
     fixup_environment, metadata_read, metadata_write, \
     metadata_initial_iofiles, metadata_update_run, parse_ports
@@ -297,6 +298,17 @@ def vagrant_setup_create(args):
                 script.write('\n')
                 # TODO : Compare package versions (painful because of sh)
 
+            # Copies rpztar
+            if not use_chroot:
+                arch = runs[0]['architecture']
+                download_file(rpztar_url(arch),
+                              target / 'rpztar',
+                              'rpztar-%s' % arch)
+                script.write(r'''
+cp /vagrant/rpztar /usr/local/bin/rpztar
+chmod +x /usr/local/bin/rpztar
+''')
+
             # Untar
             if use_chroot:
                 script.write('\n'
@@ -346,21 +358,13 @@ def vagrant_setup_create(args):
                             pathlist.append(path)
                         else:
                             logger.info("Missing file %s", path)
-                # FIXME : for some reason we need reversed() here, I'm not sure
-                # why. Need to read more of tar's docs.
-                # TAR bug: --no-overwrite-dir removes --keep-old-files
-                # TAR bug: there is no way to make --keep-old-files not report
-                # an error if an existing file is encountered. --skip-old-files
-                # was introduced too recently. Instead, we just ignore the exit
-                # status
                 with (target / 'rpz-files.list').open('wb') as filelist:
-                    for p in reversed(pathlist):
+                    for p in pathlist:
                         filelist.write(join_root(PosixPath(''), p).path)
                         filelist.write(b'\0')
-                script.write('tar zpxf /vagrant/data.tgz --keep-old-files '
-                             '--numeric-owner --strip=1 '
-                             '--null -T /vagrant/rpz-files.list '
-                             '|| /bin/true\n')
+                script.write('/usr/local/bin/rpztar '
+                             '/vagrant/data.tgz '
+                             '/vagrant/rpz-files.list\n')
 
             # Copies busybox
             if use_chroot:

reprounzip/reprounzip/parameters.py

@@ -100,6 +100,12 @@ def get_parameter(section):
         "i686": "https://github.com/remram44/static-sudo/releases/download/"
                 "current/rpzsudo-i686"
     },
+    "rpztar_url": {
+        "x86_64": "https://github.com/remram44/rpztar/releases/download/"
+                  "v1/rpztar-x86_64",
+        "i686": "https://github.com/remram44/rpztar/releases/download/"
+                "v1/rpztar-i686"
+    },
     "docker_images": {
         "default": "debian",
         "images": {

reprounzip/reprounzip/unpackers/common/__init__.py

@@ -15,6 +15,7 @@
     COMPAT_OK, COMPAT_NO, COMPAT_MAYBE, \
     composite_action, target_must_exist, unique_names, \
     make_unique_name, shell_escape, load_config, busybox_url, sudo_url, \
+    rpztar_url, \
     FileUploader, FileDownloader, get_runs, add_environment_options, \
     fixup_environment, interruptible_call, \
     metadata_read, metadata_write, metadata_initial_iofiles, \
@@ -28,7 +29,7 @@
            'UsageError', 'CantFindInstaller',
            'composite_action', 'target_must_exist', 'unique_names',
            'make_unique_name', 'shell_escape', 'load_config', 'busybox_url',
-           'sudo_url',
+           'sudo_url', 'rpztar_url',
            'join_root', 'FileUploader', 'FileDownloader', 'get_runs',
            'add_environment_options', 'fixup_environment',
            'interruptible_call', 'metadata_read', 'metadata_write',

reprounzip/reprounzip/unpackers/common/misc.py

@@ -135,6 +135,12 @@ def sudo_url(arch):
     return get_parameter('rpzsudo_url')[arch]
 
 
+def rpztar_url(arch):
+    """Gets the correct URL for the rpztar binary given the architecture.
+    """
+    return get_parameter('rpztar_url')[arch]
+
+
 class FileUploader(object):
     """Common logic for 'upload' commands.
     """