Skip to content

Commit

Permalink
Make buildkit optional, use command-line flag (#376)
Browse files Browse the repository at this point in the history
  • Loading branch information
@remram44
remram44 committed Jun 24, 2021
1 parent d7dfb24 commit c6cc92d
Show file tree
Hide file tree
Showing 2 changed files with 93 additions and 61 deletions.
48 changes: 39 additions & 9 deletions reprounzip-docker/reprounzip_docker.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -166,7 +166,8 @@ def docker_setup_create(args):
logger.info("Writing %s...", target / 'Dockerfile')
with (target / 'Dockerfile').open('w', encoding='utf-8',
newline='\n') as dockerfile:
dockerfile.write('# syntax=docker/dockerfile:1.2\n\n')
if args.use_buildkit:
dockerfile.write('# syntax=docker/dockerfile:1.2\n\n')

dockerfile.write('FROM %s\n\n' % base_image)

Expand All @@ -188,8 +189,18 @@ def docker_setup_create(args):
'rpztar-%s' % arch)
dockerfile.write('COPY rpztar /rpztar\n\n')

dockerfile.write('RUN --mount=type=bind,target=/rpzcontext \\\n'
' chmod +x /busybox /rpzsudo /rpztar && \\\n')
if args.use_buildkit:
dockerfile.write(
'RUN --mount=type=bind,target=/rpz \\\n'
' chmod +x /busybox /rpzsudo /rpztar && \\\n'
)
else:
dockerfile.write('COPY data.tgz /rpz-data.tgz\n\n')
dockerfile.write('COPY files.list /rpz-files.list\n')
dockerfile.write(
'RUN \\\n'
' chmod +x /busybox /rpzsudo /rpztar && \\\n'
)

if args.install_pkgs:
# Install every package through package manager
Expand Down Expand Up @@ -252,10 +263,15 @@ def docker_setup_create(args):
for p in pathlist:
filelist.write(join_root(PosixPath(''), p).path)
filelist.write(b'\0')
dockerfile.write(
' cd / && '
+ '/rpztar /rpzcontext/data.tgz /rpzcontext/files.list\n',
)

if args.use_buildkit:
dockerfile.write(
' cd / && /rpztar /rpz/data.tgz /rpz/files.list\n',
)
else:
dockerfile.write(
' cd / && /rpztar /rpz-data.tgz /rpz-files.list\n',
)

# Setup entry point
dockerfile.write(
Expand Down Expand Up @@ -393,10 +409,13 @@ def docker_setup_build(args):

logger.info("Calling 'docker build'...")
try:
env = None
if args.use_buildkit:
env = dict(os.environ, DOCKER_BUILDKIT='1')
retcode = subprocess.call(args.docker_cmd.split() + ['build', '-t'] +
args.docker_option + [image, '.'],
cwd=target.path,
env=dict(os.environ, DOCKER_BUILDKIT='1'))
env=env)
except OSError:
logger.critical("docker executable not found")
sys.exit(1)
Expand Down Expand Up @@ -955,6 +974,14 @@ def setup(parser, **kwargs):
def add_opt_general(opts):
opts.add_argument('target', nargs=1, help="Experiment directory")

def add_opt_setup(opts):
opts.add_argument('--use-buildkit', action='store_true',
default=True, dest='use_buildkit',
help=argparse.SUPPRESS)
opts.add_argument('--dont-use-buildkit', action='store_false',
default=True, dest='use_buildkit',
help=argparse.SUPPRESS)

# Common between setup and setup/create
def add_opt_setup_create(opts):
opts.add_argument('pack', nargs=1, help="Pack to extract")
Expand Down Expand Up @@ -987,19 +1014,22 @@ def add_raw_docker_option(opts):

parser_setup_create = subparsers.add_parser('setup/create')
add_opt_setup_create(parser_setup_create)
add_opt_setup(parser_setup_create)
add_opt_general(parser_setup_create)
parser_setup_create.set_defaults(func=docker_setup_create)

# setup/build
parser_setup_build = subparsers.add_parser('setup/build')
add_opt_general(parser_setup_build)
add_opt_setup_build(parser_setup_build)
add_opt_setup(parser_setup_build)
add_opt_general(parser_setup_build)
parser_setup_build.set_defaults(func=docker_setup_build)

# setup
parser_setup = subparsers.add_parser('setup')
add_opt_setup_create(parser_setup)
add_opt_setup_build(parser_setup)
add_opt_setup(parser_setup)
add_opt_general(parser_setup)
parser_setup.set_defaults(func=docker_setup)

Expand Down
106 changes: 54 additions & 52 deletions tests/functional.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -489,58 +489,60 @@ def check_simple(args, stream, infile=1):
if (tests / 'vagrant/simplevagrant').exists():
(tests / 'vagrant/simplevagrant').rmtree()

# Unpack Docker
check_call(rpuz + ['docker', 'setup/create', 'simple.rpz', 'simpledocker'])
print("\nDocker project set up in simpledocker", file=sys.stderr)
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', 'simpledocker'])
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'%s:bind.bin' % (Path.cwd() / 'simple')])
assert same_files('simple.orig', 'bind.bin')
# Replace input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', 'simpledocker'])
check_call(rpuz + ['showfiles', 'simpledocker'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', 'simpledocker',
':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', 'simpledocker',
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + ['docker', 'upload', 'simpledocker',
'%s:%s' % (tests / 'simple_input2.txt',
tests / 'simple_input.txt')])
# Run again
check_simple(rpuz + ['docker', 'run', 'simpledocker'], 'out', 2)
# Destroy
check_call(rpuz + ['docker', 'destroy', 'simpledocker'])
elif interactive:
print("Test and press enter", file=sys.stderr)
sys.stdin.readline()
finally:
if Path('simpledocker').exists():
Path('simpledocker').rmtree()
# Unpack Docker, both with and without buildkit
for arg, sufx in (('--dont-use-buildkit', ''), ('--use-buildkit', '-bk')):
path = 'simpledocker' + sufx
check_call(rpuz + ['docker', 'setup/create',
arg, 'simple.rpz', path])
print("\nDocker project set up in %s" % path, file=sys.stderr)
try:
if run_docker:
check_call(rpuz + ['docker', 'setup/build', arg, path])
check_simple(rpuz + ['docker', 'run', path], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', path,
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Get random file
check_call(rpuz + ['docker', 'download', path,
'%s:bind.bin' % (Path.cwd() / 'simple')])
assert same_files('simple.orig', 'bind.bin')
# Replace input file
check_call(rpuz + ['docker', 'upload', path,
'%s:arg1' % (tests / 'simple_input2.txt')])
check_call(rpuz + ['docker', 'upload', path])
check_call(rpuz + ['showfiles', path])
# Run again
check_simple(rpuz + ['docker', 'run', path], 'out', 2)
# Get output file
check_call(rpuz + ['docker', 'download', path,
'arg2:doutput2.txt'])
with Path('doutput2.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '36'
# Reset input file
check_call(rpuz + ['docker', 'upload', path, ':arg1'])
# Run again
check_simple(rpuz + ['docker', 'run', path], 'out')
# Get output file
check_call(rpuz + ['docker', 'download', path,
'arg2:doutput1.txt'])
with Path('doutput1.txt').open(encoding='utf-8') as fp:
assert fp.read().strip() == '42'
# Replace input file via path
check_call(rpuz + ['docker', 'upload', path,
'%s:%s' % (tests / 'simple_input2.txt',
tests / 'simple_input.txt')])
# Run again
check_simple(rpuz + ['docker', 'run', path], 'out', 2)
# Destroy
check_call(rpuz + ['docker', 'destroy', path])
elif interactive:
print("Test and press enter", file=sys.stderr)
sys.stdin.readline()
finally:
if Path(path).exists():
Path(path).rmtree()

# ########################################
# 'threads' program: testrun
Expand Down

0 comments on commit c6cc92d

Please sign in to comment.