Skip to content

Commit

Permalink
Merge branch 'docker-entrypoint' into 'master'
Browse files Browse the repository at this point in the history
  • Loading branch information
@remram44
remram44 committed May 18, 2017
2 parents 78c3fd6 + 6afd0bf commit a45ab72
Show file tree
Hide file tree
Showing 6 changed files with 206 additions and 49 deletions.
164 changes: 138 additions & 26 deletions reprounzip-docker/reprounzip/unpackers/docker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,10 +30,10 @@
UsageError, CantFindInstaller, composite_action, target_must_exist, \
make_unique_name, shell_escape, select_installer, busybox_url, sudo_url, \
FileUploader, FileDownloader, get_runs, add_environment_options, \
fixup_environment, interruptible_call, metadata_read, metadata_write, \
metadata_initial_iofiles, metadata_update_run, parse_ports
parse_environment_args, interruptible_call, metadata_read, \
metadata_write, metadata_initial_iofiles, metadata_update_run, parse_ports
from reprounzip.unpackers.common.x11 import X11Handler, LocalForwarder
from reprounzip.utils import unicode_, iteritems, stderr, join_root, \
from reprounzip.utils import unicode_, irange, iteritems, stderr, join_root, \
download_file


Expand Down Expand Up @@ -249,6 +249,113 @@ def docker_setup_create(args):
'/busybox echo "TAR reports errors, this might or might '
'not prevent the execution to run")\n')

# Setup entry point
fp.write('COPY rpz_entrypoint.sh /rpz_entrypoint.sh\n'
'ENTRYPOINT ["/busybox", "sh", "/rpz_entrypoint.sh"]\n')

# Write entry point script
logging.info("Writing %s...", target / 'rpz_entrypoint.sh')
with (target / 'rpz_entrypoint.sh').open('w', encoding='utf-8',
newline='\n') as fp:
# The entrypoint gets some arguments from the run command
# By default, it just does all the runs
# "run N" executes the run with that number
# "cmd STR" sets a replacement command-line for the next run
# "do STR" executes a command as-is
fp.write(
'#!/bin/sh\n'
'\n'
'COMMAND=\n'
'ENVVARS=\n'
'\n'
'if [ $# = 0 ]; then\n'
' exec /busybox sh /rpz_entrypoint.sh')
for nb in irange(len(runs)):
fp.write(' run %d' % nb)
fp.write(
'\n'
'fi\n'
'\n'
'while [ $# != 0 ]; do\n'
' case "$1" in\n'
' help)\n'
' echo "Image built from reprounzip-docker" >&2\n'
' echo "Usage: docker run <image> [cmd word [word '
'...]] [run <R>]" >&2\n'
' echo " \\`cmd ...\\` changes the command for '
'the next \\`run\\` option" >&2\n'
' echo " \\`run <name|number>\\` runs the '
'specified run" >&2\n'
' echo "By default, all the runs are executed." '
'>&2\n'
' echo "The runs in this image are:" >&2\n')
for run in runs:
fp.write(
' echo " {name}: {cmdline}" >&2\n'.format(
name=run['id'],
cmdline=' '.join(shell_escape(a)
for a in run['argv'])))
fp.write(
' exit 0\n'
' ;;\n'
' do)\n'
' shift\n'
' $1\n'
' ;;\n'
' env)\n'
' shift\n'
' ENVVARS="$1"\n'
' ;;\n'
' cmd)\n'
' shift\n'
' COMMAND="$1"\n'
' ;;\n'
' run)\n'
' shift\n'
' case "$1" in\n')
for i, run in enumerate(runs):
cmdline = ' '.join([run['binary']] + run['argv'][1:])
fp.write(
' {name})\n'
' RUNCOMMAND={cmd}\n'
' RUNWD={wd}\n'
' RUNENV={env}\n'
' RUNUID={uid}\n'
' RUNGID={gid}\n'
' ;;\n'.format(
name='%s|%d' % (run['id'], i),
cmd=shell_escape(cmdline),
wd=shell_escape(run['workingdir']),
env=shell_escape(' '.join(
'%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(run['environ']))),
uid=run.get('uid', 1000),
gid=run.get('gid', 1000)))
fp.write(
' *)\n'
' echo "RPZ: Unknown run $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' if [ -n "$COMMAND" ]; then\n'
' RUNCOMMAND="$COMMAND"\n'
' COMMAND=\n'
' fi\n'
' export RUNWD; export RUNENV; export ENVVARS; '
'export RUNCOMMAND\n'
' /rpzsudo "#$RUNUID" "#$RUNGID" /busybox sh -c '
'"cd \\"\\$RUNWD\\" && /busybox env -i $RUNENV $ENVVARS '
'$RUNCOMMAND"\n'
' ENVVARS=\n'
' ;;\n'
' *)\n'
' echo "RPZ: Unknown option $1" >&2\n'
' exit 1\n'
' ;;\n'
' esac\n'
' shift\n'
'done\n')

# Meta-data for reprounzip
write_dict(target, metadata_initial_iofiles(config))

Expand Down Expand Up @@ -462,30 +569,35 @@ def docker_run(args):
else:
x11 = X11Handler(False, ('local', hostname), args.x11_display)

cmds = []
cmd = []
for run_number in selected_runs:
run = runs[run_number]
cmd = 'cd %s && ' % shell_escape(run['workingdir'])
cmd += '/busybox env -i '
environ = x11.fix_env(run['environ'])
environ = fixup_environment(environ, args)
cmd += ' '.join('%s=%s' % (shell_escape(k), shell_escape(v))
for k, v in iteritems(environ))
cmd += ' '
env_set, env_unset = x11.env_fixes(run['environ'])
a_env_set, a_env_unset = parse_environment_args(args)
env_set.update(a_env_set)
env_unset.extend(a_env_unset)
if env_set or env_unset:
cmd.append('env')
env = []
for k in env_unset:
env.append('-u')
env.append(shell_escape(k))
for k, v in iteritems(env_set):
env.append('%s=%s' % (shell_escape(k), shell_escape(v)))
cmd.append(' '.join(env))
# FIXME : Use exec -a or something if binary != argv[0]
if cmdline is None:
argv = [run['binary']] + run['argv'][1:]
else:
argv = cmdline
cmd += ' '.join(shell_escape(a) for a in argv)
uid = run.get('uid', 1000)
gid = run.get('gid', 1000)
cmd = '/rpzsudo \'#%d\' \'#%d\' /busybox sh -c %s' % (
uid, gid,
shell_escape(cmd))
cmds.append(cmd)
cmds = x11.init_cmds + cmds
cmds = ' && '.join(cmds)
if cmdline is not None:
cmd.append('cmd')
cmd.append(' '.join(shell_escape(a) for a in cmdline))
cmd.append('run')
cmd.append('%d' % run_number)
cmd = list(chain.from_iterable([['do', shell_escape(c)]
for c in x11.init_cmds] +
[cmd]))
if logging.getLogger().isEnabledFor(logging.DEBUG):
logging.debug("Passing arguments to Docker image:")
for c in cmd:
logging.debug(c)

signals.pre_run(target=target)

Expand All @@ -503,7 +615,7 @@ def docker_run(args):
'-d', '-t'] +
port_options +
args.docker_option +
[image, '/busybox', 'sh', '-c', cmds])
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
Expand All @@ -518,7 +630,7 @@ def docker_run(args):
'-i', '-t'] +
port_options +
args.docker_option +
[image, '/busybox', 'sh', '-c', cmds])
[image] + cmd)
if retcode != 0:
logging.critical("docker run failed with code %d", retcode)
subprocess.call(['docker', 'rm', '-f', container])
Expand Down
9 changes: 3 additions & 6 deletions reprounzip-vagrant/reprounzip/unpackers/vagrant/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -441,13 +441,10 @@ class LocalX11Handler(BaseX11Handler):
init_cmds = []

@staticmethod
def fix_env(env):
"""Sets ``$XAUTHORITY`` and ``$DISPLAY`` in the environment.
def env_fixes(env):
"""Sets ``DISPLAY`` and remove ``$XAUTHORITY`` in the environment.
"""
new_env = dict(env)
new_env.pop('XAUTHORITY', None)
new_env['DISPLAY'] = ':0'
return new_env
return {'DISPLAY': ':0'}, ['XAUTHORITY']


@target_must_exist
Expand Down
8 changes: 4 additions & 4 deletions reprounzip/reprounzip/unpackers/common/__init__.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
composite_action, target_must_exist, unique_names, \
make_unique_name, shell_escape, load_config, busybox_url, sudo_url, \
FileUploader, FileDownloader, get_runs, add_environment_options, \
fixup_environment, interruptible_call, \
parse_environment_args, fixup_environment, interruptible_call, \
metadata_read, metadata_write, metadata_initial_iofiles, \
metadata_update_run, parse_ports
from reprounzip.unpackers.common.packages import THIS_DISTRIBUTION, \
Expand All @@ -30,7 +30,7 @@
'make_unique_name', 'shell_escape', 'load_config', 'busybox_url',
'sudo_url',
'join_root', 'FileUploader', 'FileDownloader', 'get_runs',
'add_environment_options', 'fixup_environment',
'interruptible_call', 'metadata_read', 'metadata_write',
'metadata_initial_iofiles', 'metadata_update_run',
'add_environment_options', 'parse_environment_args',
'fixup_environment', 'interruptible_call', 'metadata_read',
'metadata_write', 'metadata_initial_iofiles', 'metadata_update_run',
'parse_ports']
24 changes: 18 additions & 6 deletions reprounzip/reprounzip/unpackers/common/misc.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -417,23 +417,35 @@ def add_environment_options(parser):
"multiple times)")


def fixup_environment(environ, args):
def parse_environment_args(args):
if not (args.pass_env or args.set_env):
return environ
environ = dict(environ)
return {}, []

env_set = {}
env_unset = []

regexes = [re.compile(pattern + '$') for pattern in args.pass_env]
for var in os.environ:
if any(regex.match(var) for regex in regexes):
environ[var] = os.environ[var]
env_set[var] = os.environ[var]

for var in args.set_env:
if '=' in var:
var, value = var.split('=', 1)
environ[var] = value
env_set[var] = value
else:
environ.pop(var, None)
env_unset.append(var)

return env_set, env_unset


def fixup_environment(environ, args):
env_set, env_unset = parse_environment_args(args)
if env_set or env_unset:
environ = dict(environ)
environ.update(env_set)
for k in env_unset:
environ.pop(k, None)
return environ


Expand Down
27 changes: 20 additions & 7 deletions reprounzip/reprounzip/unpackers/common/x11.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,20 @@ class BaseX11Handler(object):
commands, and `port_forward` which describes the reverse port tunnels from
the experiment to the local X server.
"""
def fix_env(self, env):
"""Apply env_fixes to get a new environment dict.
"""
env_set, env_unset = self.env_fixes(env)
if env_set or env_unset:
env = dict(env)
env.update(env_set)
for k in env_unset:
env.pop(k, None)
return env

@staticmethod
def env_fixes(env):
return {}, []


class X11Handler(BaseX11Handler):
Expand Down Expand Up @@ -292,18 +306,17 @@ def connect(src_addr):

return [(6000 + self.display, connect)]

def fix_env(self, env):
def env_fixes(self, env):
"""Sets ``$XAUTHORITY`` and ``$DISPLAY`` in the environment.
"""
if not self.enabled:
return env
new_env = dict(env)
new_env['XAUTHORITY'] = str(self.xauth)
return {}, []
env_set = {'XAUTHORITY': str(self.xauth)}
if self.target[0] == 'local':
new_env['DISPLAY'] = '127.0.0.1:%d' % self.display
env_set['DISPLAY'] = '127.0.0.1:%d' % self.display
elif self.target[0] == 'internet':
new_env['DISPLAY'] = '%s:%d' % (self.target[1], self.display)
return new_env
env_set['DISPLAY'] = '%s:%d' % (self.target[1], self.display)
return env_set, []

@property
def init_cmds(self):
Expand Down
23 changes: 23 additions & 0 deletions tests/test_reprounzip.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -134,6 +134,10 @@ def __init__(self, pass_env, set_env):

old_environ, os.environ = os.environ, outer_env
try:
self.assertEqual(
reprounzip.unpackers.common.parse_environment_args(
FakeArgs([], [])),
({}, []))
self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
Expand All @@ -144,6 +148,13 @@ def __init__(self, pass_env, set_env):
'SHARED': 'sharedvalue',
})

self.assertEqual(
reprounzip.unpackers.common.parse_environment_args(
FakeArgs(['COMMON', 'INONLY', 'OUTONLY', 'EMPTY'], [])),
({'OUTONLY': 'outvalue',
'COMMON': 'commonvalueout',
'EMPTY': ''},
[]))
self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
Expand All @@ -156,6 +167,14 @@ def __init__(self, pass_env, set_env):
'EMPTY': '',
})

self.assertEqual(
reprounzip.unpackers.common.parse_environment_args(
FakeArgs(['OUTONLY'],
['SHARED=surprise', 'COMMON=', 'INONLY'])),
({'OUTONLY': 'outvalue',
'COMMON': '',
'SHARED': 'surprise'},
['INONLY']))
self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
Expand All @@ -167,6 +186,10 @@ def __init__(self, pass_env, set_env):
'SHARED': 'surprise',
})

self.assertEqual(
reprounzip.unpackers.common.parse_environment_args(
FakeArgs(['.*Y$'], [])),
({'OUTONLY': 'outvalue', 'EMPTY': ''}, []))
self.assertEqual(
reprounzip.unpackers.common.fixup_environment(
inner_env,
Expand Down

0 comments on commit a45ab72

Please sign in to comment.