Skip to content

Commit

Permalink
Try fix
Browse files Browse the repository at this point in the history
  • Loading branch information
@pitrou
pitrou committed Apr 2, 2024
1 parent 6d3c491 commit 44460ef
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 68 deletions.
79 changes: 26 additions & 53 deletions dev/archery/archery/docker/cli.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -46,10 +46,19 @@ def _execute(self, *args, **kwargs):
callback=validate_arrow_sources,
help="Specify Arrow source directory.")
@click.option('--dry-run/--execute', default=False,
help="Display the docker-compose commands instead of executing "
"them.")
help="Display the docker commands instead of executing them.")
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--using-docker-buildx', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_BUILDX',
help="Use buildx with docker CLI directly for building instead "
"of calling docker-compose or the plain docker build "
"command. This option makes the build cache reusable "
"across hosts.")
@click.pass_context
def docker(ctx, src, dry_run):
def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
"""
Interact with docker-compose based builds.
"""
Expand All @@ -64,7 +73,10 @@ def docker(ctx, src, dry_run):

# take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
# environment variables to keep the usage similar to docker-compose
using_docker_cli |= using_docker_buildx
compose = DockerCompose(config_path, params=os.environ,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx,
debug=ctx.obj.get('debug', False))
if dry_run:
_mock_compose_calls(compose)
Expand All @@ -83,24 +95,19 @@ def check_config(obj):

@docker.command('pull')
@click.argument('image')
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for pulling instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--pull-leaf/--no-leaf', default=True,
help="Whether to pull leaf images too.")
@click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
help="Whether to ignore pull failures.")
@click.pass_obj
def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
ignore_pull_failures):
def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
"""
Execute docker-compose pull.
"""
compose = obj['compose']

try:
compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
compose.pull(image, pull_leaf=pull_leaf,
ignore_pull_failures=ignore_pull_failures)
except UndefinedImage as e:
raise click.ClickException(
Expand All @@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
@click.argument('image')
@click.option('--force-pull/--no-pull', default=True,
help="Whether to force pull the image and its ancestor images")
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--using-docker-buildx', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_BUILDX',
help="Use buildx with docker CLI directly for building instead "
"of calling docker-compose or the plain docker build "
"command. This option makes the build cache reusable "
"across hosts.")
@click.option('--use-cache/--no-cache', default=True,
help="Whether to use cache when building the image and its "
"ancestor images")
Expand All @@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
"passed as the argument. To disable caching for both the "
"image and its ancestors use --no-cache option.")
@click.pass_obj
def docker_build(obj, image, *, force_pull, using_docker_cli,
using_docker_buildx, use_cache, use_leaf_cache):
def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
"""
Execute docker-compose builds.
"""
compose = obj['compose']

using_docker_cli |= using_docker_buildx
try:
if force_pull:
compose.pull(image, pull_leaf=use_leaf_cache,
using_docker=using_docker_cli)
compose.pull(image, pull_leaf=use_leaf_cache)
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx,
pull_parents=force_pull)
except UndefinedImage as e:
raise click.ClickException(
Expand All @@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
help="Whether to force build the image and its ancestor images")
@click.option('--build-only', default=False, is_flag=True,
help="Pull and/or build the image, but do not run it")
@click.option('--using-docker-cli', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_CLI',
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.option('--using-docker-buildx', default=False, is_flag=True,
envvar='ARCHERY_USE_DOCKER_BUILDX',
help="Use buildx with docker CLI directly for building instead "
"of calling docker-compose or the plain docker build "
"command. This option makes the build cache reusable "
"across hosts.")
@click.option('--use-cache/--no-cache', default=True,
help="Whether to use cache when building the image and its "
"ancestor images")
Expand All @@ -191,16 +173,16 @@ def docker_build(obj, image, *, force_pull, using_docker_cli,
"image and its ancestors use --no-cache option.")
@click.option('--resource-limit', default=None,
help="A CPU/memory limit preset to mimic CI environments like "
"GitHub Actions. Implies --using-docker-cli. Note that "
"GitHub Actions. Mandates --using-docker-cli. Note that "
"exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely "
"required, unless Docker is configured with cgroups v2 "
"(else Docker will silently ignore the limits).")
@click.option('--volume', '-v', multiple=True,
help="Set volume within the container")
@click.pass_obj
def docker_run(obj, image, command, *, env, user, force_pull, force_build,
build_only, using_docker_cli, using_docker_buildx, use_cache,
use_leaf_cache, resource_limit, volume):
build_only, use_cache, use_leaf_cache, resource_limit,
volume):
"""
Execute docker-compose builds.
Expand Down Expand Up @@ -234,26 +216,21 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
archery docker run ubuntu-cpp bash
"""
compose = obj['compose']
using_docker_cli |= using_docker_buildx

env = dict(kv.split('=', 1) for kv in env)
try:
if force_pull:
compose.pull(image, pull_leaf=use_leaf_cache,
using_docker=using_docker_cli)
compose.pull(image, pull_leaf=use_leaf_cache)
if force_build:
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache,
using_docker=using_docker_cli,
using_buildx=using_docker_buildx)
use_leaf_cache=use_leaf_cache)
if build_only:
return
compose.run(
image,
command=command,
env=env,
user=user,
using_docker=using_docker_cli,
resource_limit=resource_limit,
volumes=volume
)
Expand All @@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build,
@click.option('--password', '-p', required=False,
envvar='ARCHERY_DOCKER_PASSWORD',
help='Docker repository password')
@click.option('--using-docker-cli', default=False, is_flag=True,
help="Use docker CLI directly for building instead of calling "
"docker-compose. This may help to reuse cached layers.")
@click.pass_obj
def docker_compose_push(obj, image, user, password, using_docker_cli):
def docker_compose_push(obj, image, user, password):
"""Push the generated docker-compose image."""
compose = obj['compose']
compose.push(image, user=user, password=password,
using_docker=using_docker_cli)
compose.push(image, user=user, password=password)


@docker.command('images')
Expand Down
40 changes: 25 additions & 15 deletions dev/archery/archery/docker/core.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,11 @@ class UndefinedImage(Exception):

class ComposeConfig:

def __init__(self, config_path, dotenv_path, compose_bin, params=None):
def __init__(self, config_path, dotenv_path, compose_bin,
using_docker=False, using_buildx=False,
params=None):
self.using_docker = using_docker
self.using_buildx = using_buildx
config_path = _ensure_path(config_path)
if dotenv_path:
dotenv_path = _ensure_path(dotenv_path)
Expand Down Expand Up @@ -122,8 +126,13 @@ def _read_config(self, config_path, compose_bin):
)

# trigger docker-compose's own validation
compose = Command('docker-compose')
args = ['--file', str(config_path), 'config']
if self.using_docker:
compose = Docker()
args = ['compose']
else:
compose = Command('docker-compose')
args = []
args += ['--file', str(config_path), 'config']
result = compose.run(*args, env=self.env, check=False,
stderr=subprocess.PIPE, stdout=subprocess.PIPE)

Expand Down Expand Up @@ -164,10 +173,12 @@ def __init__(self, docker_bin=None):
class DockerCompose(Command):

def __init__(self, config_path, dotenv_path=None, compose_bin=None,
params=None, debug=False):
using_docker=False, using_buildx=False, params=None,
debug=False):
compose_bin = default_bin(compose_bin, 'docker-compose')
self.config = ComposeConfig(config_path, dotenv_path, compose_bin,
params)
params=params, using_docker=using_docker,
using_buildx=using_buildx)
self.bin = compose_bin
self.debug = debug
self.pull_memory = set()
Expand Down Expand Up @@ -215,14 +226,13 @@ def _execute_docker(self, *args, **kwargs):
)
)

def pull(self, service_name, pull_leaf=True, using_docker=False,
ignore_pull_failures=True):
def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True):
def _pull(service):
args = ['pull']
if service['image'] in self.pull_memory:
return

if using_docker:
if self.config.using_docker:
try:
self._execute_docker(*args, service['image'])
except Exception as e:
Expand All @@ -245,7 +255,7 @@ def _pull(service):
_pull(service)

def build(self, service_name, use_cache=True, use_leaf_cache=True,
using_docker=False, using_buildx=False, pull_parents=True):
pull_parents=True):
def _build(service, use_cache):
if 'build' not in service:
# nothing to do
Expand Down Expand Up @@ -273,7 +283,7 @@ def _build(service, use_cache):
if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1':
args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1'])

if using_buildx:
if self.config.using_buildx:
for k, v in service['build'].get('args', {}).items():
args.extend(['--build-arg', '{}={}'.format(k, v)])

Expand All @@ -295,7 +305,7 @@ def _build(service, use_cache):
service['build'].get('context', '.')
])
self._execute_docker("buildx", "build", *args)
elif using_docker:
elif self.config.using_docker:
# better for caching
if self.debug:
args.append("--progress=plain")
Expand All @@ -322,7 +332,7 @@ def _build(service, use_cache):
_build(service, use_cache=use_cache and use_leaf_cache)

def run(self, service_name, command=None, *, env=None, volumes=None,
user=None, using_docker=False, resource_limit=None):
user=None, resource_limit=None):
service = self.config.get(service_name)

args = []
Expand All @@ -337,7 +347,7 @@ def run(self, service_name, command=None, *, env=None, volumes=None,
for volume in volumes:
args.extend(['--volume', volume])

if using_docker or service['need_gpu'] or resource_limit:
if self.config.using_docker or service['need_gpu'] or resource_limit:
# use gpus, requires docker>=19.03
if service['need_gpu']:
args.extend(['--gpus', 'all'])
Expand Down Expand Up @@ -399,9 +409,9 @@ def run(self, service_name, command=None, *, env=None, volumes=None,
args.append(command)
self._execute_compose('run', '--rm', *args)

def push(self, service_name, user=None, password=None, using_docker=False):
def push(self, service_name, user=None, password=None):
def _push(service):
if using_docker:
if self.config.using_docker:
return self._execute_docker('push', service['image'])
else:
return self._execute_compose('push', service['name'])
Expand Down

0 comments on commit 44460ef

Please sign in to comment.