From ce36d7b8aef6214f7fe46e87eafd550448202e8c Mon Sep 17 00:00:00 2001 From: Antoine Pitrou Date: Tue, 2 Apr 2024 17:25:00 +0200 Subject: [PATCH] Try fix --- .github/workflows/cpp.yml | 3 +- .github/workflows/java.yml | 3 +- .github/workflows/python.yml | 3 +- .github/workflows/ruby.yml | 3 +- dev/archery/archery/docker/cli.py | 79 ++++++++++-------------------- dev/archery/archery/docker/core.py | 40 +++++++++------ 6 files changed, 55 insertions(+), 76 deletions(-) diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml index dc82d589ce456..13e80eb2f81f3 100644 --- a/.github/workflows/cpp.yml +++ b/.github/workflows/cpp.yml @@ -53,6 +53,7 @@ permissions: contents: read env: + ARCHERY_USE_DOCKER_CLI: 1 ARROW_ENABLE_TIMING_TESTS: OFF DOCKER_VOLUME_PREFIX: ".docker/" @@ -140,8 +141,6 @@ jobs: sudo apt install -y --no-install-recommends python3 python3-dev python3-pip - name: Setup Archery run: python3 -m pip install -e dev/archery[docker] - - name: Find docker-compose - run: which -a docker-compose - name: Execute Docker Build env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml index 17d5b9b8eab1d..d31760424267c 100644 --- a/.github/workflows/java.yml +++ b/.github/workflows/java.yml @@ -45,6 +45,7 @@ permissions: contents: read env: + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: @@ -80,8 +81,6 @@ jobs: python-version: 3.8 - name: Setup Archery run: pip install -e dev/archery[docker] - - name: Find docker-compose - run: which -a docker-compose - name: Execute Docker Build env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml index b7cf7eec59216..3d88caa557eb9 100644 --- a/.github/workflows/python.yml +++ b/.github/workflows/python.yml @@ -41,6 +41,7 @@ permissions: contents: read env: + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: @@ -105,8 +106,6 @@ jobs: python-version: 3.8 - name: Setup Archery run: pip install -e dev/archery[docker] - - name: Find docker-compose - run: which -a docker-compose - name: Execute Docker Build env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml index 2196a181edb84..3d5e8996c8dc2 100644 --- a/.github/workflows/ruby.yml +++ b/.github/workflows/ruby.yml @@ -53,6 +53,7 @@ permissions: contents: read env: + ARCHERY_USE_DOCKER_CLI: 1 DOCKER_VOLUME_PREFIX: ".docker/" jobs: @@ -87,8 +88,6 @@ jobs: python-version: 3.8 - name: Setup Archery run: pip install -e dev/archery[docker] - - name: Find docker-compose - run: which -a docker-compose - name: Execute Docker Build env: ARCHERY_DOCKER_USER: ${{ secrets.DOCKERHUB_USER }} diff --git a/dev/archery/archery/docker/cli.py b/dev/archery/archery/docker/cli.py index 20d9a16138bac..e6baf0ca1f002 100644 --- a/dev/archery/archery/docker/cli.py +++ b/dev/archery/archery/docker/cli.py @@ -46,10 +46,19 @@ def _execute(self, *args, **kwargs): callback=validate_arrow_sources, help="Specify Arrow source directory.") @click.option('--dry-run/--execute', default=False, - help="Display the docker-compose commands instead of executing " - "them.") + help="Display the docker commands instead of executing them.") +@click.option('--using-docker-cli', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_CLI', + help="Use docker CLI directly for building instead of calling " + "docker-compose. This may help to reuse cached layers.") +@click.option('--using-docker-buildx', default=False, is_flag=True, + envvar='ARCHERY_USE_DOCKER_BUILDX', + help="Use buildx with docker CLI directly for building instead " + "of calling docker-compose or the plain docker build " + "command. This option makes the build cache reusable " + "across hosts.") @click.pass_context -def docker(ctx, src, dry_run): +def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx): """ Interact with docker-compose based builds. """ @@ -64,7 +73,10 @@ def docker(ctx, src, dry_run): # take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the # environment variables to keep the usage similar to docker-compose + using_docker_cli |= using_docker_buildx compose = DockerCompose(config_path, params=os.environ, + using_docker=using_docker_cli, + using_buildx=using_docker_buildx, debug=ctx.obj.get('debug', False)) if dry_run: _mock_compose_calls(compose) @@ -83,24 +95,19 @@ def check_config(obj): @docker.command('pull') @click.argument('image') -@click.option('--using-docker-cli', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_CLI', - help="Use docker CLI directly for pulling instead of calling " - "docker-compose. This may help to reuse cached layers.") @click.option('--pull-leaf/--no-leaf', default=True, help="Whether to pull leaf images too.") @click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True, help="Whether to ignore pull failures.") @click.pass_obj -def docker_pull(obj, image, *, using_docker_cli, pull_leaf, - ignore_pull_failures): +def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures): """ Execute docker-compose pull. """ compose = obj['compose'] try: - compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli, + compose.pull(image, pull_leaf=pull_leaf, ignore_pull_failures=ignore_pull_failures) except UndefinedImage as e: raise click.ClickException( @@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf, @click.argument('image') @click.option('--force-pull/--no-pull', default=True, help="Whether to force pull the image and its ancestor images") -@click.option('--using-docker-cli', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_CLI', - help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") -@click.option('--using-docker-buildx', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_BUILDX', - help="Use buildx with docker CLI directly for building instead " - "of calling docker-compose or the plain docker build " - "command. This option makes the build cache reusable " - "across hosts.") @click.option('--use-cache/--no-cache', default=True, help="Whether to use cache when building the image and its " "ancestor images") @@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf, "passed as the argument. To disable caching for both the " "image and its ancestors use --no-cache option.") @click.pass_obj -def docker_build(obj, image, *, force_pull, using_docker_cli, - using_docker_buildx, use_cache, use_leaf_cache): +def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache): """ Execute docker-compose builds. """ compose = obj['compose'] - using_docker_cli |= using_docker_buildx try: if force_pull: - compose.pull(image, pull_leaf=use_leaf_cache, - using_docker=using_docker_cli) + compose.pull(image, pull_leaf=use_leaf_cache) compose.build(image, use_cache=use_cache, use_leaf_cache=use_leaf_cache, - using_docker=using_docker_cli, - using_buildx=using_docker_buildx, pull_parents=force_pull) except UndefinedImage as e: raise click.ClickException( @@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, help="Whether to force build the image and its ancestor images") @click.option('--build-only', default=False, is_flag=True, help="Pull and/or build the image, but do not run it") -@click.option('--using-docker-cli', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_CLI', - help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") -@click.option('--using-docker-buildx', default=False, is_flag=True, - envvar='ARCHERY_USE_DOCKER_BUILDX', - help="Use buildx with docker CLI directly for building instead " - "of calling docker-compose or the plain docker build " - "command. This option makes the build cache reusable " - "across hosts.") @click.option('--use-cache/--no-cache', default=True, help="Whether to use cache when building the image and its " "ancestor images") @@ -191,7 +173,7 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, "image and its ancestors use --no-cache option.") @click.option('--resource-limit', default=None, help="A CPU/memory limit preset to mimic CI environments like " - "GitHub Actions. Implies --using-docker-cli. Note that " + "GitHub Actions. Mandates --using-docker-cli. Note that " "exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely " "required, unless Docker is configured with cgroups v2 " "(else Docker will silently ignore the limits).") @@ -199,8 +181,8 @@ def docker_build(obj, image, *, force_pull, using_docker_cli, help="Set volume within the container") @click.pass_obj def docker_run(obj, image, command, *, env, user, force_pull, force_build, - build_only, using_docker_cli, using_docker_buildx, use_cache, - use_leaf_cache, resource_limit, volume): + build_only, use_cache, use_leaf_cache, resource_limit, + volume): """ Execute docker-compose builds. @@ -234,18 +216,14 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, archery docker run ubuntu-cpp bash """ compose = obj['compose'] - using_docker_cli |= using_docker_buildx env = dict(kv.split('=', 1) for kv in env) try: if force_pull: - compose.pull(image, pull_leaf=use_leaf_cache, - using_docker=using_docker_cli) + compose.pull(image, pull_leaf=use_leaf_cache) if force_build: compose.build(image, use_cache=use_cache, - use_leaf_cache=use_leaf_cache, - using_docker=using_docker_cli, - using_buildx=using_docker_buildx) + use_leaf_cache=use_leaf_cache) if build_only: return compose.run( @@ -253,7 +231,6 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, command=command, env=env, user=user, - using_docker=using_docker_cli, resource_limit=resource_limit, volumes=volume ) @@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user, force_pull, force_build, @click.option('--password', '-p', required=False, envvar='ARCHERY_DOCKER_PASSWORD', help='Docker repository password') -@click.option('--using-docker-cli', default=False, is_flag=True, - help="Use docker CLI directly for building instead of calling " - "docker-compose. This may help to reuse cached layers.") @click.pass_obj -def docker_compose_push(obj, image, user, password, using_docker_cli): +def docker_compose_push(obj, image, user, password): """Push the generated docker-compose image.""" compose = obj['compose'] - compose.push(image, user=user, password=password, - using_docker=using_docker_cli) + compose.push(image, user=user, password=password) @docker.command('images') diff --git a/dev/archery/archery/docker/core.py b/dev/archery/archery/docker/core.py index 184d9808759b8..5c739a3750399 100644 --- a/dev/archery/archery/docker/core.py +++ b/dev/archery/archery/docker/core.py @@ -58,7 +58,11 @@ class UndefinedImage(Exception): class ComposeConfig: - def __init__(self, config_path, dotenv_path, compose_bin, params=None): + def __init__(self, config_path, dotenv_path, compose_bin, + using_docker=False, using_buildx=False, + params=None): + self.using_docker = using_docker + self.using_buildx = using_buildx config_path = _ensure_path(config_path) if dotenv_path: dotenv_path = _ensure_path(dotenv_path) @@ -122,8 +126,13 @@ def _read_config(self, config_path, compose_bin): ) # trigger docker-compose's own validation - compose = Command('docker-compose') - args = ['--file', str(config_path), 'config'] + if self.using_docker: + compose = Docker() + args = ['compose'] + else: + compose = Command('docker-compose') + args = [] + args += ['--file', str(config_path), 'config'] result = compose.run(*args, env=self.env, check=False, stderr=subprocess.PIPE, stdout=subprocess.PIPE) @@ -164,10 +173,12 @@ def __init__(self, docker_bin=None): class DockerCompose(Command): def __init__(self, config_path, dotenv_path=None, compose_bin=None, - params=None, debug=False): + using_docker=False, using_buildx=False, params=None, + debug=False): compose_bin = default_bin(compose_bin, 'docker-compose') self.config = ComposeConfig(config_path, dotenv_path, compose_bin, - params) + params=params, using_docker=using_docker, + using_buildx=using_buildx) self.bin = compose_bin self.debug = debug self.pull_memory = set() @@ -215,14 +226,13 @@ def _execute_docker(self, *args, **kwargs): ) ) - def pull(self, service_name, pull_leaf=True, using_docker=False, - ignore_pull_failures=True): + def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True): def _pull(service): args = ['pull'] if service['image'] in self.pull_memory: return - if using_docker: + if self.config.using_docker: try: self._execute_docker(*args, service['image']) except Exception as e: @@ -245,7 +255,7 @@ def _pull(service): _pull(service) def build(self, service_name, use_cache=True, use_leaf_cache=True, - using_docker=False, using_buildx=False, pull_parents=True): + pull_parents=True): def _build(service, use_cache): if 'build' not in service: # nothing to do @@ -273,7 +283,7 @@ def _build(service, use_cache): if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1': args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1']) - if using_buildx: + if self.config.using_buildx: for k, v in service['build'].get('args', {}).items(): args.extend(['--build-arg', '{}={}'.format(k, v)]) @@ -295,7 +305,7 @@ def _build(service, use_cache): service['build'].get('context', '.') ]) self._execute_docker("buildx", "build", *args) - elif using_docker: + elif self.config.using_docker: # better for caching if self.debug: args.append("--progress=plain") @@ -322,7 +332,7 @@ def _build(service, use_cache): _build(service, use_cache=use_cache and use_leaf_cache) def run(self, service_name, command=None, *, env=None, volumes=None, - user=None, using_docker=False, resource_limit=None): + user=None, resource_limit=None): service = self.config.get(service_name) args = [] @@ -337,7 +347,7 @@ def run(self, service_name, command=None, *, env=None, volumes=None, for volume in volumes: args.extend(['--volume', volume]) - if using_docker or service['need_gpu'] or resource_limit: + if self.config.using_docker or service['need_gpu'] or resource_limit: # use gpus, requires docker>=19.03 if service['need_gpu']: args.extend(['--gpus', 'all']) @@ -399,9 +409,9 @@ def run(self, service_name, command=None, *, env=None, volumes=None, args.append(command) self._execute_compose('run', '--rm', *args) - def push(self, service_name, user=None, password=None, using_docker=False): + def push(self, service_name, user=None, password=None): def _push(service): - if using_docker: + if self.config.using_docker: return self._execute_docker('push', service['image']) else: return self._execute_compose('push', service['name'])