Reprounzip unpacker containerexec

reprounzip-containerexec/LICENSE.txt

@@ -0,0 +1,28 @@
+Copyright (C) 2014-2017, New York University
+Copyright (C) 2017 Dirk Beyer
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its contributors
+   may be used to endorse or promote products derived from this software
+   without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
+ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

reprounzip-containerexec/MANIFEST.in

@@ -0,0 +1,2 @@
+include README.rst
+include LICENSE.txt

reprounzip-containerexec/README.rst

@@ -0,0 +1,31 @@
+ReproZip
+========
+
+`ReproZip <https://www.reprozip.org/>`__ is a tool aimed at simplifying the process of creating reproducible
+experiments from command-line executions, a frequently-used common denominator
+in computational science. It tracks operating system calls and creates a package
+that contains all the binaries, files and dependencies required to run a given
+command on the author's computational environment (packing step).
+A reviewer can then extract the experiment in his environment to reproduce the results (unpacking step).
+
+reprounzip-containerexec
+------------------------
+
+This is the component responsible for the unpacking step.
+It uses Linux kernel namespaces to create a container similarly to Docker.
+Contrary to Docker, however, it works without installation of additional software
+and without root access.
+It is based on the tool ``containerexec``, which is part of `BenchExec <https://github.com/sosy-lab/benchexec/>`_.
+Please refer to the `documentation <https://github.com/sosy-lab/benchexec/blob/master/doc/container.md>`_
+for the system requirements.
+
+
+Additional Information
+----------------------
+
+For more information about ReproZip,
+please refer to its `website <https://www.reprozip.org/>`_,
+as well as to the `documentation <https://reprozip.readthedocs.io/>`_.
+
+For more information about BenchExec,
+please refer to its `website <https://github.com/sosy-lab/benchexec/>`_.

reprounzip-containerexec/reprounzip/__init__.py

@@ -0,0 +1,5 @@
+try:  # pragma: no cover
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:  # pragma: no cover
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

reprounzip-containerexec/reprounzip/unpackers/__init__.py

@@ -0,0 +1,5 @@
+try:  # pragma: no cover
+    __import__('pkg_resources').declare_namespace(__name__)
+except ImportError:  # pragma: no cover
+    from pkgutil import extend_path
+    __path__ = extend_path(__path__, __name__)

reprounzip-containerexec/reprounzip/unpackers/containerexec.py

@@ -0,0 +1,215 @@
+# Copyright (C) 2014-2017 New York University
+# Copyright (C) 2017 Dirk Beyer
+# This file is part of ReproZip which is released under the Revised BSD License
+# See file LICENSE for full license details.
+
+"""Containerexec plugin for reprounzip."""
+
+# prepare for Python 3
+from __future__ import \
+    absolute_import, division, print_function, unicode_literals
+
+import argparse
+import logging
+import signal
+from rpaths import Path
+import sys
+
+from benchexec import BenchExecException, containerexecutor
+from reprounzip import signals
+from reprounzip.common import load_config as load_config_file
+from reprounzip.unpackers.common import target_must_exist, shell_escape, \
+    get_runs, add_environment_options, fixup_environment, metadata_read, \
+    metadata_write, metadata_initial_iofiles, metadata_update_run
+from reprounzip.unpackers.default import chroot_create, download, \
+    test_linux_same_arch, upload
+from reprounzip.utils import stderr, rmtree_fixed
+
+TYPE_ = 'containerexec'
+
+
+def containerexec_create(args):
+    """Unpacks the experiment in a specified folder so it can be run with
+    containerexec.
+
+    The files in the rpz-file (pack) are unpacked to the target location, and
+    system files are also copied if they are not already available in the pack.
+    Busybox will be also installed in case /bin/sh wasn't packed.
+    """
+    chroot_create(args)
+
+    # Rewrite the meta-data for reprounzip with a specific type-name
+    # of the containerexec unpacker
+    target = Path(args.target[0])
+    config = load_config_file(target / 'config.yml', True)
+    metadata_write(target, metadata_initial_iofiles(config), TYPE_)
+
+
+@target_must_exist
+def containerexec_run(args):
+    """Runs the experiment in a container environment that is partially isolated
+    from the host.
+
+    The process is isolated from other processes on the same system,
+    in a similar way as for example Docker isolates applications
+    (using operating-level system virtualization).
+    For further informations, see also
+    https://github.com/sosy-lab/benchexec/blob/master/doc/container.md
+    """
+    if args is None:
+        args = sys.args
+
+    logging.info('Received arguments: %s', args)
+
+    target = Path(args.target[0])
+    unpacked_info = metadata_read(target, TYPE_)
+    cmdline = args.cmdline
+
+    # Loads config
+    config = load_config_file(target / 'config.yml', True)
+    runs = config.runs
+
+    selected_runs = get_runs(runs, args.run, cmdline)
+
+    root_dir = target / b"root"
+    root_dir = str(root_dir.resolve())
+
+    if args.x11 and not any('DISPLAY' in s for s in args.pass_env):
+        args.pass_env.append('DISPLAY')
+
+    signals.pre_run(target=target)
+
+    # Each run is executed in its own executor process.
+    for run_number in selected_runs:
+        run = runs[run_number]
+
+        working_dir = shell_escape(run['workingdir'])
+        env = fixup_environment(run['environ'], args)
+
+        uid = run['uid']
+        gid = run['gid']
+
+        # FIXME : Use exec -a or something if binary != argv[0]
+        if cmdline is None:
+            argv = [run['binary']] + run['argv'][1:]
+        else:
+            argv = cmdline
+
+        executor = containerexecutor.ContainerExecutor(uid=uid, gid=gid,
+                                                       network_access=True)
+
+        # ensure that process gets killed on interrupt/kill signal
+        def signal_handler_kill(signum, frame):
+            executor.stop()
+        signal.signal(signal.SIGTERM, signal_handler_kill)
+        signal.signal(signal.SIGINT, signal_handler_kill)
+
+        # actual run execution
+        try:
+            result = executor.execute_run(argv, workingDir=working_dir,
+                                          rootDir=root_dir, environ=env)
+        except (BenchExecException, OSError) as e:
+            sys.exit("Cannot execute process: {0}.".format(e))
+
+    stderr.write(
+        "\n*** Command finished, status: %d\n" % result.value or result.signal)
+    signals.post_run(target=target, retcode=result.value)
+
+    # Update input file status
+    metadata_update_run(config, unpacked_info, selected_runs)
+    metadata_write(target, unpacked_info, TYPE_)
+
+
+@target_must_exist
+def containerexec_destroy(args):
+    """Destroys the directory.
+    """
+    target = Path(args.target[0])
+
+    logging.info("Removing directory %s...", target)
+    signals.pre_destroy(target=target)
+    rmtree_fixed(target)
+    signals.post_destroy(target=target)
+
+
+def setup(parser, **kwargs):
+    """Unpacks the files in a directory and runs the experiment in a container
+    that is partially isolated from the host. ContainerExec is part of
+    BenchExec: https://github.com/sosy-lab/benchexec/
+
+    setup           creates the directory (needs the pack filename)
+    upload          replaces input files in the directory
+                    (without arguments, lists input files)
+    run             runs the experiment in a container
+    download        gets output files from the machine
+                    (without arguments, lists output files)
+    destroy         removes the unpacked directory
+
+    For example:
+
+        $ reprounzip containerexec setup mypackage.rpz path
+        $ reprounzip containerexec upload path/ input:/home/.../input.txt
+        $ reprounzip containerexec run path/
+        $ reprounzip containerexec download path/ results:/home/.../results.txt
+        $ reprounzip containerexec destroy path
+
+    Upload specifications are either:
+      :input_id             restores the original input file from the pack
+      filename:input_id     replaces the input file with the specified local
+                            file
+
+    Download specifications are either:
+      output_id:            print the output file to stdout
+      output_id:filename    extracts the output file to the corresponding local
+                            path
+    """
+
+    subparsers = parser.add_subparsers(title="actions",
+                                       metavar='', help=argparse.SUPPRESS)
+
+    def add_opt_general(opts):
+        opts.add_argument('target', nargs=1, help="Experiment directory")
+
+    # setup
+    parser_setup = subparsers.add_parser('setup')
+    parser_setup.add_argument('pack', nargs=1, help="Pack to extract")
+    add_opt_general(parser_setup)
+    parser_setup.set_defaults(func=containerexec_create, restore_owner=False)
+
+    # upload
+    parser_upload = subparsers.add_parser('upload')
+    add_opt_general(parser_upload)
+    parser_upload.add_argument('file', nargs=argparse.ZERO_OR_MORE,
+                               help="<path>:<input_file_name")
+    parser_upload.set_defaults(func=upload, type=TYPE_,
+                               restore_owner=False)
+
+    # run
+    parser_run = subparsers.add_parser('run')
+    add_opt_general(parser_run)
+    parser_run.add_argument('run', default=None, nargs=argparse.OPTIONAL)
+    parser_run.add_argument('--cmdline', nargs=argparse.REMAINDER,
+                            help="Command line to run")
+    parser_run.add_argument('--enable-x11', action='store_true', default=False,
+                            dest='x11',
+                            help="Enable X11 support (needs an X server on "
+                                 "the host)")
+    add_environment_options(parser_run)
+    parser_run.set_defaults(func=containerexec_run)
+
+    # download
+    parser_download = subparsers.add_parser('download')
+    add_opt_general(parser_download)
+    parser_download.add_argument('file', nargs=argparse.ZERO_OR_MORE,
+                                 help="<output_file_name>[:<path>]")
+    parser_download.add_argument('--all', action='store_true',
+                                 help="Download all output files to the "
+                                      "current directory")
+    parser_download.set_defaults(func=download, type=TYPE_)
+
+    # destroy
+    parser_destroy = subparsers.add_parser('destroy')
+    add_opt_general(parser_destroy)
+    parser_destroy.set_defaults(func=containerexec_destroy)
+
+    return {'test_compatibility': test_linux_same_arch}

reprounzip-containerexec/setup.cfg

@@ -0,0 +1,2 @@
+[bdist_wheel]
+universal = 1

reprounzip-containerexec/setup.py

@@ -0,0 +1,46 @@
+import io
+import os
+import setuptools
+
+# pip workaround
+os.chdir(os.path.abspath(os.path.dirname(__file__)))
+
+with io.open('README.rst', encoding='utf-8') as fp:
+    long_description = fp.read()
+setuptools.setup(
+    name = 'reprounzip-containerexec',
+    version = '1.0',
+    author = 'Dirk Beyer',
+    description = "An unpacker for reprozip using the container technology of BenchExec",
+    long_description = long_description,
+    url = 'https://www.reprozip.org',
+    license = 'BSD-3-Clause',
+    keywords = [
+        'reprozip', 'reprounzip', 'reproducibility', 'provenance',
+        'benchexec', 'containerexec', 'container'],
+      classifiers=[
+          'Development Status :: 5 - Production/Stable',
+          'Intended Audience :: Science/Research',
+          'License :: OSI Approved :: BSD License',
+        'Operating System :: POSIX :: Linux',
+          'Programming Language :: Python :: 2.7',
+          'Programming Language :: Python :: 3',
+          'Programming Language :: Python :: 3.4',
+          'Programming Language :: Python :: 3.5',
+          'Topic :: Scientific/Engineering',
+        'Topic :: System :: Archiving'],
+    platforms = ['Linux'],
+
+    packages = ['reprounzip', 'reprounzip.unpackers'],
+    entry_points = {
+        'reprounzip.unpackers': [
+            'containerexec = reprounzip.unpackers.containerexec:setup']
+        },
+    namespace_packages = ['reprounzip', 'reprounzip.unpackers'],
+    install_requires = [
+        'reprounzip>=1.0.8',
+        'rpaths>=0.8',
+        'benchexec>=1.11',
+        ],
+    zip_safe = True,
+    )