diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index d5b2e186eea..fe16deaf019 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -33,6 +33,7 @@ def create(conan_api, parser, *args): parser.add_argument("-bt", "--build-test", action="append", help="Same as '--build' but only for the test_package requires. By default" " if not specified it will take the '--build' value if specified") + raw_args = args[0] args = parser.parse_args(*args) if args.test_missing and args.test_folder == "": @@ -62,6 +63,25 @@ def create(conan_api, parser, *args): lockfile = conan_api.lockfile.update_lockfile_export(lockfile, conanfile, ref, is_build) print_profiles(profile_host, profile_build) + if profile_host.runner and not os.environ.get("CONAN_RUNNER_ENVIRONMENT"): + from conan.internal.runner.docker import DockerRunner + from conan.internal.runner.ssh import SSHRunner + from conan.internal.runner.wsl import WSLRunner + try: + runner_type = profile_host.runner['type'].lower() + except KeyError: + raise ConanException(f"Invalid runner configuration. 'type' must be defined") + runner_instances_map = { + 'docker': DockerRunner, + # 'ssh': SSHRunner, + # 'wsl': WSLRunner, + } + try: + runner_instance = runner_instances_map[runner_type] + except KeyError: + raise ConanException(f"Invalid runner type '{runner_type}'. Allowed values: {', '.join(runner_instances_map.keys())}") + return runner_instance(conan_api, 'create', profile_host, profile_build, args, raw_args).run() + if args.build is not None and args.build_test is None: args.build_test = args.build diff --git a/conan/internal/runner/__init__.py b/conan/internal/runner/__init__.py new file mode 100644 index 00000000000..8ee22e17661 --- /dev/null +++ b/conan/internal/runner/__init__.py @@ -0,0 +1,6 @@ +class RunnerException(Exception): + def __init__(self, *args, **kwargs): + self.command = kwargs.pop("command", None) + self.stdout_log = kwargs.pop("stdout_log", None) + self.stderr_log = kwargs.pop("stderr_log", None) + super(RunnerException, self).__init__(*args, **kwargs) \ No newline at end of file diff --git a/conan/internal/runner/docker.py b/conan/internal/runner/docker.py new file mode 100644 index 00000000000..ebc6374e37d --- /dev/null +++ b/conan/internal/runner/docker.py @@ -0,0 +1,273 @@ +from collections import namedtuple +import os +import json +import platform +import shutil +import yaml +from conan.api.model import ListPattern +from conan.api.output import Color, ConanOutput +from conan.api.conan_api import ConfigAPI +from conan.cli import make_abs_path +from conan.internal.runner import RunnerException +from conans.client.profile_loader import ProfileLoader +from conans.errors import ConanException +from conans.model.version import Version + + +def config_parser(file_path): + Build = namedtuple('Build', ['dockerfile', 'build_context', 'build_args', 'cache_from']) + Run = namedtuple('Run', ['name', 'environment', 'user', 'privileged', 'cap_add', 'security_opt', 'volumes']) + Conf = namedtuple('Conf', ['image', 'build', 'run']) + if file_path: + def _instans_or_error(value, obj): + if value and (not isinstance(value, obj)): + raise ConanException(f"docker runner configfile syntax error: {value} must be a {obj.__name__}") + return value + with open(file_path, 'r') as f: + runnerfile = yaml.safe_load(f) + return Conf( + image=_instans_or_error(runnerfile.get('image'), str), + build=Build( + dockerfile=_instans_or_error(runnerfile.get('build', {}).get('dockerfile'), str), + build_context=_instans_or_error(runnerfile.get('build', {}).get('build_context'), str), + build_args=_instans_or_error(runnerfile.get('build', {}).get('build_args'), dict), + cache_from=_instans_or_error(runnerfile.get('build', {}).get('cacheFrom'), list), + ), + run=Run( + name=_instans_or_error(runnerfile.get('run', {}).get('name'), str), + environment=_instans_or_error(runnerfile.get('run', {}).get('containerEnv'), dict), + user=_instans_or_error(runnerfile.get('run', {}).get('containerUser'), str), + privileged=_instans_or_error(runnerfile.get('run', {}).get('privileged'), bool), + cap_add=_instans_or_error(runnerfile.get('run', {}).get('capAdd'), list), + security_opt=_instans_or_error(runnerfile.get('run', {}).get('securityOpt'), list), + volumes=_instans_or_error(runnerfile.get('run', {}).get('mounts'), dict), + ) + ) + else: + return Conf( + image=None, + build=Build(dockerfile=None, build_context=None, build_args=None, cache_from=None), + run=Run(name=None, environment=None, user=None, privileged=None, cap_add=None, security_opt=None, volumes=None) + ) + + +def _docker_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) + + +class DockerRunner: + def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): + import docker + import docker.api.build + try: + self.docker_client = docker.from_env() + self.docker_api = docker.APIClient() + docker.api.build.process_dockerfile = lambda dockerfile, path: ('Dockerfile', dockerfile) + except: + raise ConanException("Docker Client failed to initialize." + "\n - Check if docker is installed and running" + "\n - Run 'pip install pip install conan[runners]'") + self.conan_api = conan_api + self.build_profile = build_profile + self.args = args + self.abs_host_path = make_abs_path(args.path) + if args.format: + raise ConanException("format argument is forbidden if running in a docker runner") + + # Runner config + self.abs_runner_home_path = os.path.join(self.abs_host_path, '.conanrunner') + self.abs_docker_path = os.path.join('/root/conanrunner', os.path.basename(self.abs_host_path)).replace("\\","/") + + # Update conan command and some paths to run inside the container + raw_args[raw_args.index(args.path)] = self.abs_docker_path + self.profiles = [] + if self.args.profile_build and self.args.profile_host: + profile_list = set(self.args.profile_build + self.args.profile_host) + else: + profile_list = self.args.profile_host or self.args.profile_build + + # Update the profile paths + for i, raw_arg in enumerate(raw_args): + for i, raw_profile in enumerate(profile_list): + _profile = ProfileLoader.get_profile_path(os.path.join(ConfigAPI(self.conan_api).home(), 'profiles'), raw_profile, os.getcwd()) + _name = f'{os.path.basename(_profile)}_{i}' + if raw_profile in raw_arg: + raw_args[raw_args.index(raw_arg)] = raw_arg.replace(raw_profile, os.path.join(self.abs_docker_path, '.conanrunner/profiles', _name)) + self.profiles.append([_profile, os.path.join(self.abs_runner_home_path, 'profiles', _name)]) + + self.command = ' '.join([f'conan {command}'] + [f'"{raw_arg}"' if ' ' in raw_arg else raw_arg for raw_arg in raw_args] + ['-f json > create.json']) + + # Container config + # https://containers.dev/implementors/json_reference/ + self.configfile = config_parser(host_profile.runner.get('configfile')) + self.dockerfile = host_profile.runner.get('dockerfile') or self.configfile.build.dockerfile + self.docker_build_context = host_profile.runner.get('build_context') or self.configfile.build.build_context + self.image = host_profile.runner.get('image') or self.configfile.image + if not (self.dockerfile or self.image): + raise ConanException("'dockerfile' or docker image name is needed") + self.image = self.image or 'conan-runner-default' + self.name = self.configfile.image or f'conan-runner-{host_profile.runner.get("suffix", "docker")}' + self.remove = str(host_profile.runner.get('remove', 'false')).lower() == 'true' + self.cache = str(host_profile.runner.get('cache', 'clean')) + self.container = None + + def run(self): + """ + run conan inside a Docker continer + """ + if self.dockerfile: + _docker_info(f'Building the Docker image: {self.image}') + self.build_image() + volumes, environment = self.create_runner_environment() + error = False + try: + if self.docker_client.containers.list(all=True, filters={'name': self.name}): + _docker_info('Starting the docker container') + self.container = self.docker_client.containers.get(self.name) + self.container.start() + else: + if self.configfile.run.environment: + environment.update(self.configfile.run.environment) + if self.configfile.run.volumes: + volumes.update(self.configfile.run.volumes) + _docker_info('Creating the docker container') + self.container = self.docker_client.containers.run( + self.image, + "/bin/bash -c 'while true; do sleep 30; done;'", + name=self.name, + volumes=volumes, + environment=environment, + user=self.configfile.run.user, + privileged=self.configfile.run.privileged, + cap_add=self.configfile.run.cap_add, + security_opt=self.configfile.run.security_opt, + detach=True, + auto_remove=False) + _docker_info(f'Container {self.name} running') + except Exception as e: + raise ConanException(f'Imposible to run the container "{self.name}" with image "{self.image}"' + f'\n\n{str(e)}') + try: + self.init_container() + self.run_command(self.command) + self.update_local_cache() + except ConanException as e: + error = True + raise e + except RunnerException as e: + error = True + raise ConanException(f'"{e.command}" inside docker fail' + f'\n\nLast command output: {str(e.stdout_log)}') + finally: + if self.container: + error_prefix = 'ERROR: ' if error else '' + _docker_info(f'{error_prefix}Stopping container', error) + self.container.stop() + if self.remove: + _docker_info(f'{error_prefix}Removing container', error) + self.container.remove() + + def build_image(self): + dockerfile_file_path = self.dockerfile + if os.path.isdir(self.dockerfile): + dockerfile_file_path = os.path.join(self.dockerfile, 'Dockerfile') + with open(dockerfile_file_path) as f: + build_path = self.docker_build_context or os.path.dirname(dockerfile_file_path) + ConanOutput().highlight(f"Dockerfile path: '{dockerfile_file_path}'") + ConanOutput().highlight(f"Docker build context: '{build_path}'\n") + docker_build_logs = self.docker_api.build( + path=build_path, + dockerfile=f.read(), + tag=self.image, + buildargs=self.configfile.build.build_args, + cache_from=self.configfile.build.cache_from, + ) + for chunk in docker_build_logs: + for line in chunk.decode("utf-8").split('\r\n'): + if line: + stream = json.loads(line).get('stream') + if stream: + ConanOutput().status(stream.strip()) + + def run_command(self, command, log=True): + if log: + _docker_info(f'Running in container: "{command}"') + exec_instance = self.docker_api.exec_create(self.container.id, f"/bin/bash -c '{command}'", tty=True) + exec_output = self.docker_api.exec_start(exec_instance['Id'], tty=True, stream=True, demux=True,) + stderr_log, stdout_log = '', '' + try: + for (stdout_out, stderr_out) in exec_output: + if stdout_out is not None: + stdout_log += stdout_out.decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().status(stdout_out.decode('utf-8', errors='ignore').strip()) + if stderr_out is not None: + stderr_log += stderr_out.decode('utf-8', errors='ignore').strip() + if log: + ConanOutput().status(stderr_out.decode('utf-8', errors='ignore').strip()) + except Exception as e: + if platform.system() == 'Windows': + import pywintypes + if isinstance(e, pywintypes.error): + pass + else: + raise e + exit_metadata = self.docker_api.exec_inspect(exec_instance['Id']) + if exit_metadata['Running'] or exit_metadata['ExitCode'] > 0: + raise RunnerException(command=command, stdout_log=stdout_log, stderr_log=stderr_log) + return stdout_log, stderr_log + + def create_runner_environment(self): + shutil.rmtree(self.abs_runner_home_path, ignore_errors=True) + volumes = {self.abs_host_path: {'bind': self.abs_docker_path, 'mode': 'rw'}} + environment = {'CONAN_RUNNER_ENVIRONMENT': '1'} + if self.cache == 'shared': + volumes[ConfigAPI(self.conan_api).home()] = {'bind': '/root/.conan2', 'mode': 'rw'} + if self.cache in ['clean', 'copy']: + os.mkdir(self.abs_runner_home_path) + os.mkdir(os.path.join(self.abs_runner_home_path, 'profiles')) + + # Copy all conan config files to docker workspace + for file_name in ['global.conf', 'settings.yml', 'remotes.json']: + src_file = os.path.join(ConfigAPI(self.conan_api).home(), file_name) + if os.path.exists(src_file): + shutil.copy(src_file, os.path.join(self.abs_runner_home_path, file_name)) + + # Copy all profiles to docker workspace + for current_path, new_path in self.profiles: + shutil.copy(current_path, new_path) + + if self.cache == 'copy': + tgz_path = os.path.join(self.abs_runner_home_path, 'local_cache_save.tgz') + _docker_info(f'Save host cache in: {tgz_path}') + self.conan_api.cache.save(self.conan_api.list.select(ListPattern("*:*")), tgz_path) + return volumes, environment + + def init_container(self): + min_conan_version = '2.1' + stdout, _ = self.run_command('conan --version', log=True) + docker_conan_version = str(stdout.split('Conan version ')[1].replace('\n', '').replace('\r', '')) # Remove all characters and color + if Version(docker_conan_version) <= Version(min_conan_version): + ConanOutput().status(f'ERROR: conan version inside the container must be greater than {min_conan_version}', fg=Color.BRIGHT_RED) + raise ConanException( f'conan version inside the container must be greater than {min_conan_version}') + if self.cache != 'shared': + self.run_command('mkdir -p ${HOME}/.conan2/profiles', log=False) + self.run_command('cp -r "'+self.abs_docker_path+'/.conanrunner/profiles/." ${HOME}/.conan2/profiles/.', log=False) + for file_name in ['global.conf', 'settings.yml', 'remotes.json']: + if os.path.exists( os.path.join(self.abs_runner_home_path, file_name)): + self.run_command('cp "'+self.abs_docker_path+'/.conanrunner/'+file_name+'" ${HOME}/.conan2/'+file_name, log=False) + if self.cache in ['copy']: + self.run_command('conan cache restore "'+self.abs_docker_path+'/.conanrunner/local_cache_save.tgz"') + + def update_local_cache(self): + if self.cache != 'shared': + self.run_command('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json', log=False) + self.run_command('conan cache save --list=pkglist.json --file "'+self.abs_docker_path+'"/.conanrunner/docker_cache_save.tgz') + tgz_path = os.path.join(self.abs_runner_home_path, 'docker_cache_save.tgz') + _docker_info(f'Restore host cache from: {tgz_path}') + package_list = self.conan_api.cache.restore(tgz_path) diff --git a/conan/internal/runner/ssh.py b/conan/internal/runner/ssh.py new file mode 100644 index 00000000000..b47f0124739 --- /dev/null +++ b/conan/internal/runner/ssh.py @@ -0,0 +1,273 @@ +from pathlib import Path +import pathlib +import tempfile +from conan.api.conan_api import ConfigAPI +from conan.api.output import Color, ConanOutput +from conans.errors import ConanException +from conans.client.profile_loader import ProfileLoader + +import os +from io import BytesIO +import sys + +def ssh_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) + +class SSHRunner: + + def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): + from paramiko.config import SSHConfig + from paramiko.client import SSHClient + self.conan_api = conan_api + self.command = command + self.host_profile = host_profile + self.build_profile = build_profile + self.remote_host_profile = None + self.remote_build_profile = None + self.remote_python_command = None + self.remote_create_dir = None + self.remote_is_windows = None + self.args = args + self.raw_args = raw_args + self.ssh_config = None + self.remote_workspace = None + self.remote_conan = None + self.remote_conan_home = None + if host_profile.runner.get('use_ssh_config', False): + ssh_config_file = Path.home() / ".ssh" / "config" + ssh_config = SSHConfig.from_file(open(ssh_config_file)) + + hostname = host_profile.runner.get("host") # TODO: this one is required + if ssh_config and ssh_config.lookup(hostname): + hostname = ssh_config.lookup(hostname)['hostname'] + + self.client = SSHClient() + self.client.load_system_host_keys() + self.client.connect(hostname) + + + def run(self, use_cache=True): + ssh_info('Got to SSHRunner.run(), doing nothing') + + self.ensure_runner_environment() + self.copy_working_conanfile_path() + + raw_args = self.raw_args + raw_args[raw_args.index(self.args.path)] = self.remote_create_dir + raw_args = " ".join(raw_args) + + _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + remote_json_output = _Path(self.remote_create_dir).joinpath("conan_create.json").as_posix() + command = f"{self.remote_conan} create {raw_args} --format json > {remote_json_output}" + + ssh_info(f"Remote command: {command}") + + stdout, _ = self._run_command(command) + first_line = True + while not stdout.channel.exit_status_ready(): + line = stdout.channel.recv(1024) + if first_line and self.remote_is_windows: + # Avoid clearing and moving the cursor when the remote server is Windows + # https://github.com/PowerShell/Win32-OpenSSH/issues/1738#issuecomment-789434169 + line = line.replace(b"\x1b[2J\x1b[m\x1b[H",b"") + sys.stdout.buffer.write(line) + sys.stdout.buffer.flush() + first_line = False + + if stdout.channel.recv_exit_status() == 0: + self.update_local_cache(remote_json_output) + + # self.client.close() + def ensure_runner_environment(self): + has_python3_command = False + python_is_python3 = False + + _, _stdout, _stderr = self.client.exec_command("python3 --version") + has_python3_command = _stdout.channel.recv_exit_status() == 0 + if not has_python3_command: + _, _stdout, _stderr = self.client.exec_command("python --version") + if _stdout.channel.recv_exit_status() == 0 and "Python 3" in _stdout.read().decode(): + python_is_python3 = True + + python_command = "python" if python_is_python3 else "python3" + self.remote_python_command = python_command + + if not has_python3_command and not python_is_python3: + raise ConanException("Unable to locate working Python 3 executable in remote SSH environment") + + # Determine if remote host is Windows + _, _stdout, _ = self.client.exec_command(f'{python_command} -c "import os; print(os.name)"') + if _stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to determine remote OS type") + is_windows = _stdout.read().decode().strip() == "nt" + self.remote_is_windows = is_windows + + # Get remote user home folder + _, _stdout, _ = self.client.exec_command(f'{python_command} -c "from pathlib import Path; print(Path.home())"') + if _stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to determine remote home user folder") + home_folder = _stdout.read().decode().strip() + + # Expected remote paths + remote_folder = Path(home_folder) / ".conan2remote" + remote_folder = remote_folder.as_posix().replace("\\", "/") + self.remote_workspace = remote_folder + remote_conan_home = Path(home_folder) / ".conan2remote" / "conanhome" + remote_conan_home = remote_conan_home.as_posix().replace("\\", "/") + self.remote_conan_home = remote_conan_home + ssh_info(f"Remote workfolder: {remote_folder}") + + # Ensure remote folders exist + for folder in [remote_folder, remote_conan_home]: + _, _stdout, _stderr = self.client.exec_command(f"""{python_command} -c "import os; os.makedirs('{folder}', exist_ok=True)""") + if _stdout.channel.recv_exit_status() != 0: + ssh_info(f"Error creating remote folder: {_stderr.read().decode()}") + raise ConanException(f"Unable to create remote workfolder at {folder}") + + conan_venv = remote_folder + "/venv" + if is_windows: + conan_cmd = remote_folder + "/venv/Scripts/conan.exe" + else: + conan_cmd = remote_folder + "/venv/bin/conan" + + ssh_info(f"Expected remote conan home: {remote_conan_home}") + ssh_info(f"Expected remote conan command: {conan_cmd}") + + # Check if remote Conan executable exists, otherwise invoke pip inside venv + sftp = self.client.open_sftp() + try: + sftp.stat(conan_cmd) + has_remote_conan = True + except FileNotFoundError: + has_remote_conan = False + finally: + sftp.close() + + if not has_remote_conan: + _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m venv {conan_venv}") + if _stdout.channel.recv_exit_status() != 0: + ssh_info(f"Unable to create remote venv: {_stderr.read().decode().strip()}") + + if is_windows: + python_command = remote_folder + "/venv" + "/Scripts" + "/python.exe" + else: + python_command = remote_folder + "/venv" + "/bin" + "/python" + + _, _stdout, _stderr = self.client.exec_command(f"{python_command} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper") + if _stdout.channel.recv_exit_status() != 0: + # Note: this may fail on windows + ssh_info(f"Unable to install conan in venv: {_stderr.read().decode().strip()}") + + remote_env = { + 'CONAN_HOME': remote_conan_home, + 'CONAN_RUNNER_ENVIRONMENT': "1" + } + if is_windows: + # Wrapper script with environment variables preset + env_lines = "\n".join([f"set {k}={v}" for k,v in remote_env.items()]) + conan_bat_contents = f"""@echo off\n{env_lines}\n{conan_cmd} %*\n""" + conan_bat = remote_folder + "/conan.bat" + try: + sftp = self.client.open_sftp() + sftp.putfo(BytesIO(conan_bat_contents.encode()), conan_bat) + except: + raise ConanException("unable to set up Conan remote script") + finally: + sftp.close() + + self.remote_conan = conan_bat + _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} config home") + ssh_info(f"Remote conan config home returned: {_stdout.read().decode().strip()}") + _, _stdout, _stderr = self.client.exec_command(f"{self.remote_conan} profile detect --force") + self._copy_profiles() + + + def _copy_profiles(self): + sftp = self.client.open_sftp() + + # TODO: very questionable choices here + try: + profiles = { + self.args.profile_host[0]: self.host_profile.dumps(), + self.args.profile_build[0]: self.build_profile.dumps() + } + + for name, contents in profiles.items(): + dest_filename = self.remote_conan_home + f"/profiles/{name}" + sftp.putfo(BytesIO(contents.encode()), dest_filename) + except: + raise ConanException("Unable to copy profiles to remote") + finally: + sftp.close() + + def copy_working_conanfile_path(self): + resolved_path = Path(self.args.path).resolve() + if resolved_path.is_file(): + resolved_path = resolved_path.parent + + if not resolved_path.is_dir(): + return ConanException("Error determining conanfile directory") + + # Create temporary destination directory + temp_dir_create_cmd = f"""{self.remote_python_command} -c "import tempfile; print(tempfile.mkdtemp(dir='{self.remote_workspace}'))""" + _, _stdout, _ = self.client.exec_command(temp_dir_create_cmd) + if _stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to create remote temporary directory") + self.remote_create_dir = _stdout.read().decode().strip().replace("\\", '/') + + # Copy current folder to destination using sftp + _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + sftp = self.client.open_sftp() + for root, dirs, files in os.walk(resolved_path.as_posix()): + relative_root = Path(root).relative_to(resolved_path) + for dir in dirs: + dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(dir).as_posix() + sftp.mkdir(dst) + for file in files: + orig = os.path.join(root, file) + dst = _Path(self.remote_create_dir).joinpath(relative_root).joinpath(file).as_posix() + sftp.put(orig, dst) + sftp.close() + + def _run_command(self, command): + ''' Run a command in an SSH session. + When requesting a pseudo-terminal from the server, + ensure we pass width and height that matches the current + terminal + ''' + channel = self.client.get_transport().open_session() + if sys.stdout.isatty(): + width, height = os.get_terminal_size() + channel.get_pty(width=width, height=height) + + channel.exec_command(command) + + stdout = channel.makefile("r") + stderr = channel.makefile("r") + return stdout, stderr + + def update_local_cache(self, json_result): + # ('conan list --graph=create.json --graph-binaries=build --format=json > pkglist.json' + _Path = pathlib.PureWindowsPath if self.remote_is_windows else pathlib.PurePath + pkg_list_json = _Path(self.remote_create_dir).joinpath("pkg_list.json").as_posix() + pkg_list_command = f"{self.remote_conan} list --graph={json_result} --graph-binaries=build --format=json > {pkg_list_json}" + _, stdout, _ = self.client.exec_command(pkg_list_command) + if stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to generate remote package list") + + conan_cache_tgz = _Path(self.remote_create_dir).joinpath("cache.tgz").as_posix() + cache_save_command = f"{self.remote_conan} cache save --list {pkg_list_json} --file {conan_cache_tgz}" + _, stdout, _ = self.client.exec_command(cache_save_command) + if stdout.channel.recv_exit_status() != 0: + raise ConanException("Unable to save remote conan cache state") + + sftp = self.client.open_sftp() + with tempfile.TemporaryDirectory() as tmp: + local_cache_tgz = os.path.join(tmp, 'cache.tgz') + sftp.get(conan_cache_tgz, local_cache_tgz) + package_list = self.conan_api.cache.restore(local_cache_tgz) diff --git a/conan/internal/runner/wsl.py b/conan/internal/runner/wsl.py new file mode 100644 index 00000000000..3e4744b6225 --- /dev/null +++ b/conan/internal/runner/wsl.py @@ -0,0 +1,144 @@ +from pathlib import PurePosixPath, PureWindowsPath, Path +from conan.api.output import Color, ConanOutput +from conans.errors import ConanException +from conans.util.runners import conan_run +from conans.client.subsystems import subsystem_path +from conan.tools.files import save +from io import StringIO +import tempfile +import os + +def wsl_info(msg, error=False): + fg=Color.BRIGHT_MAGENTA + if error: + fg=Color.BRIGHT_RED + ConanOutput().status('\n┌'+'─'*(2+len(msg))+'┐', fg=fg) + ConanOutput().status(f'| {msg} |', fg=fg) + ConanOutput().status('└'+'─'*(2+len(msg))+'┘\n', fg=fg) + + +class WSLRunner: + def __init__(self, conan_api, command, host_profile, build_profile, args, raw_args): + self.conan_api = conan_api + self.command = command + self.host_profile = host_profile + self.build_profile = build_profile + self.remote_host_profile = None + self.remote_build_profile = None + self.remote_python_command = None + self.remote_conan = None + self.remote_conan_home = None + self.args = args + self.raw_args = raw_args + + # to pass to wsl.exe (optional, otherwise run with defaults) + distro = host_profile.runner.get("distribution", None) + user = host_profile.runner.get("user", None) + + self.shared_cache = host_profile.runner.get("shared_cache", False) + if self.shared_cache: + storage_path = Path(conan_api.config.home()) / 'p' # TODO: there's an API for this!! + self.remote_conan_cache = subsystem_path("wsl", storage_path.as_posix()) + + def run(self): + self.ensure_runner_environment() + + raw_args = self.raw_args + current_path = Path(self.args.path).resolve() + current_path_wsl = subsystem_path("wsl", current_path.as_posix()) + + raw_args[raw_args.index(self.args.path)] = current_path_wsl + raw_args = " ".join(raw_args) + + with tempfile.TemporaryDirectory() as tmp_dir: + if not self.shared_cache: + create_json = PureWindowsPath(tmp_dir).joinpath("create.json").as_posix() + raw_args += f" --format=json > {create_json}" + tmp_dir_wsl = subsystem_path("wsl", tmp_dir) + command = f"wsl.exe --cd {tmp_dir_wsl} -- CONAN_RUNNER_ENVIRONMENT=1 CONAN_HOME={self.remote_conan_home} {self.remote_conan} create {raw_args}" + rc = conan_run(command) + if rc == 0 and not self.shared_cache: + create_json_wsl = subsystem_path("wsl", create_json) + pkglist_json = PureWindowsPath(tmp_dir).joinpath("pkglist.json").as_posix() + pkglist_json_wsl = subsystem_path("wsl", pkglist_json) + + saved_cache = PureWindowsPath(tmp_dir).joinpath("saved_cache.tgz").as_posix() + saved_cache_wsl = subsystem_path("wsl", saved_cache) + conan_run(f"wsl.exe --cd {tmp_dir_wsl} -- CONAN_RUNNER_ENVIRONMENT=1 CONAN_HOME={self.remote_conan_home} {self.remote_conan} list --graph={create_json_wsl} --format=json > {pkglist_json}") + conan_run(f"wsl.exe --cd {tmp_dir_wsl} -- CONAN_RUNNER_ENVIRONMENT=1 CONAN_HOME={self.remote_conan_home} {self.remote_conan} cache save --list={pkglist_json_wsl} --file {saved_cache_wsl}") + self.conan_api.cache.restore(saved_cache) + else: + pass + #print(command) + + def ensure_runner_environment(self): + stdout = StringIO() + stderr = StringIO() + + ret = conan_run('wsl.exe echo $HOME', stdout=stdout) + if ret == 0: + remote_home = PurePosixPath(stdout.getvalue().strip()) + stdout = StringIO() + + remote_conan = remote_home / ".conan2remote" / "venv" / "bin" / "conan" + self.remote_conan = remote_conan.as_posix() + + wsl_info(self.remote_conan) + + conan_home = remote_home / ".conan2remote" / "conan_home" + self.remote_conan_home = conan_home + + has_conan = conan_run(f"wsl.exe CONAN_HOME={conan_home.as_posix()} {remote_conan} --version", stdout=stdout, stderr=stderr) == 0 + + if not has_conan: + wsl_info("Bootstrapping Conan in remote") + conan_run(f"wsl.exe mkdir -p {remote_home}/.conan2remote") + venv = remote_home / ".conan2remote"/ "venv" + python = venv / "bin" / "python" + self.remote_python_command = python + conan_run(f"wsl.exe python3 -m venv {venv.as_posix()}") + conan_run(f"wsl.exe {python} -m pip install pip wheel --upgrade") + conan_run(f"wsl.exe {python} -m pip install git+https://github.com/conan-io/conan@feature/docker_wrapper") + conan_run(f"wsl.exe CONAN_HOME={conan_home.as_posix()} {remote_conan} --version", stdout=stdout) + + remote_conan_version = stdout.getvalue().strip() + wsl_info(f"Remote conan version: {remote_conan_version}") + stdout = StringIO() + stderr = StringIO() + + # If this command succeeds, great - if not because it already exists, ignore + conan_run(f"wsl.exe CONAN_HOME={conan_home.as_posix()} {remote_conan} profile detect", stdout=stdout, stderr=stderr) + + + conf_content = f"core.cache:storage_path={self.remote_conan_cache}\n" if self.shared_cache else "" + with tempfile.TemporaryDirectory() as tmp: + global_conf = os.path.join(tmp, "global.conf") + save(None, path=global_conf, content=conf_content) + global_conf_wsl = subsystem_path("wsl", global_conf) + remote_global_conf = self.remote_conan_home.joinpath("global.conf") + conan_run(f"wsl.exe cp {global_conf_wsl} {remote_global_conf}") + + self._copy_profiles() + + def _copy_profiles(self): + # TODO: questionable choices, may fail + + # Note: see the use of \\wsl$\\, we could place the files + # directly. We would need to work out the exact distro name first + profiles = { + self.args.profile_host[0]: self.host_profile.dumps(), + self.args.profile_build[0]: self.build_profile.dumps() + } + + with tempfile.TemporaryDirectory() as tmp: + # path = os.path.join(tmp, 'something') + for name, contents in profiles.items(): + outfile = os.path.join(tmp, name) + save(None, path=outfile, content=contents) + outfile_wsl = subsystem_path("wsl", outfile) + remote_profile = self.remote_conan_home.joinpath("profiles").as_posix() + "/" + + # This works but copies the file with executable attribute + conan_run(f"wsl.exe cp {outfile_wsl} {remote_profile}") + + diff --git a/conans/client/profile_loader.py b/conans/client/profile_loader.py index fc83a734416..6b1ba23e81a 100644 --- a/conans/client/profile_loader.py +++ b/conans/client/profile_loader.py @@ -230,7 +230,8 @@ def get_profile(profile_text, base_profile=None): "platform_requires", "platform_tool_requires", "settings", "options", "conf", "buildenv", "runenv", - "replace_requires", "replace_tool_requires"]) + "replace_requires", "replace_tool_requires", + "runner"]) # Parse doc sections into Conan model, Settings, Options, etc settings, package_settings = _ProfileValueParser._parse_settings(doc) @@ -302,8 +303,21 @@ def load_replace(doc_replace_requires): base_profile.buildenv.update_profile_env(buildenv) if runenv is not None: base_profile.runenv.update_profile_env(runenv) + + + runner = _ProfileValueParser._parse_key_value(doc.runner) if doc.runner else {} + base_profile.runner.update(runner) return base_profile + @staticmethod + def _parse_key_value(raw_info): + result = OrderedDict() + for br_line in raw_info.splitlines(): + tokens = br_line.split("=", 1) + pattern, req_list = tokens + result[pattern.strip()] = req_list.strip() + return result + @staticmethod def _parse_tool_requires(doc): result = OrderedDict() diff --git a/conans/model/profile.py b/conans/model/profile.py index 7654ed9acf3..6160afd84d4 100644 --- a/conans/model/profile.py +++ b/conans/model/profile.py @@ -24,6 +24,7 @@ def __init__(self): self.conf = ConfDefinition() self.buildenv = ProfileEnvironment() self.runenv = ProfileEnvironment() + self.runner = {} # Cached processed values self.processed_settings = None # Settings with values, and smart completion @@ -124,6 +125,7 @@ def compose_profile(self, other): self.replace_requires.update(other.replace_requires) self.replace_tool_requires.update(other.replace_tool_requires) + self.runner.update(other.runner) current_platform_tool_requires = {r.name: r for r in self.platform_tool_requires} current_platform_tool_requires.update({r.name: r for r in other.platform_tool_requires}) diff --git a/conans/requirements_dev.txt b/conans/requirements_dev.txt index decec3b2ef5..c81ee6c559c 100644 --- a/conans/requirements_dev.txt +++ b/conans/requirements_dev.txt @@ -6,3 +6,5 @@ WebTest>=2.0.18, <2.1.0 bottle PyJWT pluginbase +docker>=5.0.0, <6.0.0 +setuptools \ No newline at end of file diff --git a/conans/requirements_runner.txt b/conans/requirements_runner.txt new file mode 100644 index 00000000000..ac5f6f5065e --- /dev/null +++ b/conans/requirements_runner.txt @@ -0,0 +1,2 @@ +paramiko +docker>=5.0.0, <6.0.0 diff --git a/conans/test/integration/command/dockerfiles/Dockerfile b/conans/test/integration/command/dockerfiles/Dockerfile new file mode 100644 index 00000000000..068de9a2efd --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile @@ -0,0 +1,11 @@ +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_args b/conans/test/integration/command/dockerfiles/Dockerfile_args new file mode 100644 index 00000000000..67685e1f6fc --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_args @@ -0,0 +1,12 @@ +ARG BASE_IMAGE +FROM $BASE_IMAGE +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_ninja b/conans/test/integration/command/dockerfiles/Dockerfile_ninja new file mode 100644 index 00000000000..acd643d2da6 --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_ninja @@ -0,0 +1,12 @@ +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + ninja-build \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_profile_detect b/conans/test/integration/command/dockerfiles/Dockerfile_profile_detect new file mode 100644 index 00000000000..76f2b20a5de --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_profile_detect @@ -0,0 +1,12 @@ +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . +RUN conan profile detect \ No newline at end of file diff --git a/conans/test/integration/command/dockerfiles/Dockerfile_test b/conans/test/integration/command/dockerfiles/Dockerfile_test new file mode 100644 index 00000000000..068de9a2efd --- /dev/null +++ b/conans/test/integration/command/dockerfiles/Dockerfile_test @@ -0,0 +1,11 @@ +FROM ubuntu:22.04 +RUN apt-get update \ + && DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \ + build-essential \ + cmake \ + python3 \ + python3-pip \ + python3-venv \ + && rm -rf /var/lib/apt/lists/* +COPY . /root/conan-io +RUN cd /root/conan-io && pip install -e . \ No newline at end of file diff --git a/conans/test/integration/command/runner_test.py b/conans/test/integration/command/runner_test.py new file mode 100644 index 00000000000..5341d73092a --- /dev/null +++ b/conans/test/integration/command/runner_test.py @@ -0,0 +1,437 @@ +import textwrap +import os +import pytest +import docker +from conans.test.utils.tools import TestClient +from conans.test.assets.cmake import gen_cmakelists +from conans.test.assets.sources import gen_function_h, gen_function_cpp + + +def docker_skip(test_image=None): + try: + docker_client = docker.from_env() + if test_image: + docker_client.images.pull(test_image) + except docker.errors.DockerException: + return True + except docker.errors.ImageNotFound: + return True + except docker.errors.APIError: + return True + return False + + +def conan_base_path(): + import conans + return os.path.dirname(os.path.dirname(conans.__file__)) + + +def dockerfile_path(name=None): + path = os.path.join(os.path.dirname(__file__), "dockerfiles") + if name: + path = os.path.join(path, name) + return path + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_dockerfile_folder_path(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + + profile_host_copy = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=copy + remove=True + """) + + profile_host_clean = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=clean + remove=True + """) + + client.save({"host_copy": profile_host_copy, "host_clean": profile_host_clean, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run("create . -pr:h host_copy -pr:b build") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + client.run("create . -pr:h host_clean -pr:b build") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_dockerfile_file_path(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path("Dockerfile_test")} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=copy + remove=True + """) + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run("create . -pr:h host -pr:b build") + print(client.out) + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +@pytest.mark.parametrize("build_type,shared", [("Release", False), ("Debug", True)]) +@pytest.mark.tool("ninja") +def test_create_docker_runner_with_ninja(build_type, shared): + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.cmake import CMake, CMakeToolchain + + class Library(ConanFile): + name = "hello" + version = "1.0" + settings = 'os', 'arch', 'compiler', 'build_type' + exports_sources = 'hello.h', '*.cpp', 'CMakeLists.txt' + options = {'shared': [True, False]} + default_options = {'shared': False} + + def generate(self): + tc = CMakeToolchain(self, generator="Ninja") + tc.generate() + + def build(self): + cmake = CMake(self) + cmake.configure() + cmake.build() + self.run(os.sep.join([".", "myapp"])) + + def package(self): + cmake = CMake(self) + cmake.install() + """) + + client = TestClient(path_with_spaces=False) + client.save({'conanfile.py': conanfile, + "CMakeLists.txt": gen_cmakelists(libsources=["hello.cpp"], + appsources=["main.cpp"], + install=True), + "hello.h": gen_function_h(name="hello"), + "hello.cpp": gen_function_cpp(name="hello", includes=["hello"]), + "main.cpp": gen_function_cpp(name="main", includes=["hello"], + calls=["hello"])}) + profile = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + image=conan-runner-ninja-test + dockerfile={dockerfile_path("Dockerfile_ninja")} + build_context={conan_base_path()} + cache=copy + remove=True + """) + client.save({"profile": profile}) + settings = "-s os=Linux -s arch=x86_64 -s build_type={} -o hello/*:shared={}".format(build_type, shared) + # create should also work + client.run("create . --name=hello --version=1.0 {} -pr:h=profile -pr:b=profile".format(settings)) + assert 'cmake -G "Ninja"' in client.out + assert "main: {}!".format(build_type) in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_profile_abs_path(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path("Dockerfile_test")} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=copy + remove=True + """) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_profile_abs_path_from_configfile(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + configfile = textwrap.dedent(f""" + image: conan-runner-default-test + build: + dockerfile: {dockerfile_path("Dockerfile_test")} + build_context: {conan_base_path()} + """) + client.save({"configfile.yaml": configfile}) + + + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + configfile={os.path.join(client.current_folder, 'configfile.yaml')} + cache=copy + remove=True + """) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_profile_abs_path_from_configfile_with_args(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + configfile = textwrap.dedent(f""" + image: conan-runner-default-test-with-args + build: + dockerfile: {dockerfile_path("Dockerfile_args")} + build_context: {conan_base_path()} + build_args: + BASE_IMAGE: ubuntu:22.04 + run: + name: my-conan-runner-container-with-args + """) + client.save({"configfile.yaml": configfile}) + + + profile_build = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + """) + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + configfile={os.path.join(client.current_folder, 'configfile.yaml')} + cache=copy + remove=True + """) + + client.save({"host": profile_host, "build": profile_build}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run(f"create . -pr:h '{os.path.join(client.current_folder, 'host')}' -pr:b '{os.path.join(client.current_folder, 'build')}'") + print(client.out) + assert "test/integration/command/dockerfiles/Dockerfile_args" in client.out + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_default_build_profile(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path("Dockerfile_profile_detect")} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=clean + remove=True + """) + + client.save({"host_clean": profile_host}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + client.run("create . -pr:h host_clean") + + assert "Restore: pkg/0.2" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe" in client.out + assert "Restore: pkg/0.2:8631cf963dbbb4d7a378a64a6fd1dc57558bc2fe metadata" in client.out + assert "Removing container" in client.out + + +@pytest.mark.skipif(docker_skip('ubuntu:22.04'), reason="Only docker running") +# @pytest.mark.xfail(reason="conan inside docker optional test") +def test_create_docker_runner_default_build_profile_error(): + """ + Tests the ``conan create . `` + """ + client = TestClient() + + profile_host = textwrap.dedent(f"""\ + [settings] + arch=x86_64 + build_type=Release + compiler=gcc + compiler.cppstd=gnu17 + compiler.libcxx=libstdc++11 + compiler.version=11 + os=Linux + [runner] + type=docker + dockerfile={dockerfile_path()} + build_context={conan_base_path()} + image=conan-runner-default-test + cache=clean + remove=True + """) + + client.save({"host_clean": profile_host}) + client.run("new cmake_lib -d name=pkg -d version=0.2") + with pytest.raises(Exception, match="ERROR: The default build profile '/root/.conan2/profiles/default' doesn't exist.") as exception: + client.run("create . -pr:h host_clean") \ No newline at end of file diff --git a/setup.py b/setup.py index 1864af173a4..f61b72ae21f 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,7 @@ def generate_long_description_file(): project_requirements = get_requires("conans/requirements.txt") dev_requirements = get_requires("conans/requirements_dev.txt") +runners_requirements = get_requires("conans/requirements_runner.txt") excluded_server_packages = ["conans.server*"] exclude = excluded_test_packages + excluded_server_packages @@ -112,10 +113,11 @@ def generate_long_description_file(): # List additional groups of dependencies here (e.g. development # dependencies). You can install these using the following syntax, # for example: - # $ pip install -e .[dev,test] + # $ pip install -e .[dev,test,runners] extras_require={ 'dev': dev_requirements, 'test': dev_requirements, + 'runners': runners_requirements }, # If there are data files included in your packages that need to be