From e9f823a12478a97ae0104a838721f7d1aa60c396 Mon Sep 17 00:00:00 2001 From: Florian de Gaulejac Date: Wed, 3 Jan 2024 13:32:53 +0100 Subject: [PATCH 1/5] Add built in runtime deployer --- conan/internal/deploy.py | 69 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 68 insertions(+), 1 deletion(-) diff --git a/conan/internal/deploy.py b/conan/internal/deploy.py index 36ca53ef6a6..d80990a36ab 100644 --- a/conan/internal/deploy.py +++ b/conan/internal/deploy.py @@ -32,7 +32,8 @@ def _load(path): if os.path.isfile(cache_path): return _load(cache_path) builtin_deploy = {"full_deploy.py": full_deploy, - "direct_deploy.py": direct_deploy}.get(d) + "direct_deploy.py": direct_deploy, + "runtime_deploy.py": runtime_deploy}.get(d) if builtin_deploy is not None: return builtin_deploy raise ConanException(f"Cannot find deployer '{d}'") @@ -80,6 +81,72 @@ def full_deploy(graph, output_folder): _deploy_single(dep, conanfile, output_folder, folder_name) +def runtime_deploy(graph, output_folder): + """ + Deploy all the shared libraries and the executables of the dependencies in a flat directory. + """ + conanfile = graph.root.conanfile + conanfile.output.info(f"Deploying the runtime...") + for _, dep in conanfile.dependencies.items(): + conanfile.output.verbose(f"Searching for shared libraries and executables in {dep.ref}...") + if dep.package_folder is None: + conanfile.output.verbose(f"{dep.ref} does not have any package folder") + continue + if not dep.cpp_info.bindirs and not dep.cpp_info.libdirs: + conanfile.output.verbose(f"{dep.ref} does not have any bin or lib directory") + continue + + for bindir_name in dep.cpp_info.bindirs: + bindir_path = os.path.join(dep.package_folder, bindir_name) + if not os.path.isdir(bindir_path): + conanfile.output.verbose(f"{bindir_path} does not exist") + continue + file_count = _flatten_directory(dep, conanfile, bindir_path, output_folder) + conanfile.output.info(f"Copied {file_count} files from {dep.ref}, directory named {bindir_name}") + + for libdir_name in dep.cpp_info.libdirs: + libdir_path = os.path.join(dep.package_folder, libdir_name) + if not os.path.isdir(libdir_path): + conanfile.output.verbose(f"{libdir_path} does not exist") + continue + file_count = _flatten_directory(dep, conanfile, libdir_path, output_folder, [".dll", ".dylib",".so"]) + conanfile.output.info(f"Copied {file_count} files from {dep.ref}, directory named {libdir_name}") + conanfile.output.info(f"Runtime deployed!") + + +def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter = None): + """ + Copy all the files from the source directory in a flat output directory. + An optional string, named extension_filter, can be set to copy only the files with the listed extensions. + """ + if not os.path.exists(output_dir): + os.makedirs(output_dir) + + file_count = 0 + symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) + for src_dirpath, _, src_filenames in os.walk(src_dir, followlinks=symlinks): + for src_filename in src_filenames: + if extension_filter: + for extension in extension_filter: + if not src_filename.endswith(extension): + continue + src_filepath = os.path.join(src_dirpath, src_filename) + dest_filepath = os.path.join(output_dir, src_filename) + if os.path.exists(dest_filepath): + conanfile.output.verbose(f"{src_filename} already exists and will be overwritten") + try: + file_count += 1 + shutil.copy2(src_filepath, dest_filepath, follow_symlinks=symlinks) + conanfile.output.verbose(f"Copied {src_filename} into {output_dir}") + except Exception as e: + if "WinError 1314" in str(e): + ConanOutput().error("runtime_deploy: Symlinks in Windows require admin privileges " + "or 'Developer mode = ON'", error_type="exception") + raise ConanException(f"runtime_deploy: The copy of '{dep}' files failed: {e}.\nYou can " + f"use 'tools.deployer:symlinks' conf to disable symlinks") + return file_count + + def _deploy_single(dep, conanfile, output_folder, folder_name): new_folder = os.path.join(output_folder, folder_name) rmdir(new_folder) From de6f47ac2da590b0b51925c3fd51fdd6c7348dba Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 16 Jan 2024 23:46:53 +0100 Subject: [PATCH 2/5] test + minor styling --- conan/internal/deploy.py | 42 +++++++++---------- .../functional/command/test_install_deploy.py | 25 +++++++++++ 2 files changed, 44 insertions(+), 23 deletions(-) diff --git a/conan/internal/deploy.py b/conan/internal/deploy.py index d80990a36ab..4131b605e6e 100644 --- a/conan/internal/deploy.py +++ b/conan/internal/deploy.py @@ -87,6 +87,7 @@ def runtime_deploy(graph, output_folder): """ conanfile = graph.root.conanfile conanfile.output.info(f"Deploying the runtime...") + mkdir(output_folder) for _, dep in conanfile.dependencies.items(): conanfile.output.verbose(f"Searching for shared libraries and executables in {dep.ref}...") if dep.package_folder is None: @@ -96,44 +97,39 @@ def runtime_deploy(graph, output_folder): conanfile.output.verbose(f"{dep.ref} does not have any bin or lib directory") continue - for bindir_name in dep.cpp_info.bindirs: - bindir_path = os.path.join(dep.package_folder, bindir_name) - if not os.path.isdir(bindir_path): - conanfile.output.verbose(f"{bindir_path} does not exist") + for bindir in dep.cpp_info.bindirs: + if not os.path.isdir(bindir): + conanfile.output.warning(f"{bindir} does not exist") continue - file_count = _flatten_directory(dep, conanfile, bindir_path, output_folder) - conanfile.output.info(f"Copied {file_count} files from {dep.ref}, directory named {bindir_name}") + count = _flatten_directory(dep, conanfile, bindir, output_folder) + conanfile.output.info(f"Copied {count} files from {dep.ref}, directory named {bindir}") - for libdir_name in dep.cpp_info.libdirs: - libdir_path = os.path.join(dep.package_folder, libdir_name) - if not os.path.isdir(libdir_path): - conanfile.output.verbose(f"{libdir_path} does not exist") + for libdir in dep.cpp_info.libdirs: + if not os.path.isdir(libdir): + conanfile.output.warning(f"{libdir} does not exist") continue - file_count = _flatten_directory(dep, conanfile, libdir_path, output_folder, [".dll", ".dylib",".so"]) - conanfile.output.info(f"Copied {file_count} files from {dep.ref}, directory named {libdir_name}") + count = _flatten_directory(dep, conanfile, libdir, output_folder, [".dylib", ".so"]) + conanfile.output.info(f"Copied {count} files from {dep.ref}, directory named {libdir}") conanfile.output.info(f"Runtime deployed!") -def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter = None): +def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter=None): """ Copy all the files from the source directory in a flat output directory. - An optional string, named extension_filter, can be set to copy only the files with the listed extensions. + An optional string, named extension_filter, can be set to copy only the files with + the listed extensions. """ - if not os.path.exists(output_dir): - os.makedirs(output_dir) - file_count = 0 symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) for src_dirpath, _, src_filenames in os.walk(src_dir, followlinks=symlinks): for src_filename in src_filenames: - if extension_filter: - for extension in extension_filter: - if not src_filename.endswith(extension): - continue + if extension_filter and not any(src_filename.endswith(ext) for ext in extension_filter): + continue + src_filepath = os.path.join(src_dirpath, src_filename) dest_filepath = os.path.join(output_dir, src_filename) if os.path.exists(dest_filepath): - conanfile.output.verbose(f"{src_filename} already exists and will be overwritten") + conanfile.output.warning(f"{src_filename} already exists and will be overwritten") try: file_count += 1 shutil.copy2(src_filepath, dest_filepath, follow_symlinks=symlinks) @@ -143,7 +139,7 @@ def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter = N ConanOutput().error("runtime_deploy: Symlinks in Windows require admin privileges " "or 'Developer mode = ON'", error_type="exception") raise ConanException(f"runtime_deploy: The copy of '{dep}' files failed: {e}.\nYou can " - f"use 'tools.deployer:symlinks' conf to disable symlinks") + f"use 'tools.deployer:symlinks' conf to disable symlinks") return file_count diff --git a/conans/test/functional/command/test_install_deploy.py b/conans/test/functional/command/test_install_deploy.py index f9fd31d42af..0f9c4e86e59 100644 --- a/conans/test/functional/command/test_install_deploy.py +++ b/conans/test/functional/command/test_install_deploy.py @@ -435,3 +435,28 @@ def package_info(self): env = c.load("conanbuildenv-release-x86_64.sh") assert f'export MYPATH="{some_abs_path}/mypath"' in env + + +class TestRuntimeDeployer: + def test_runtime_deploy(self): + c = TestClient() + conanfile = textwrap.dedent(""" + from conan import ConanFile + from conan.tools.files import copy + class Pkg(ConanFile): + def package(self): + copy(self, "*.so", src=self.build_folder, dst=self.package_folder) + copy(self, "*.dll", src=self.build_folder, dst=self.package_folder) + """) + c.save({"pkga/conanfile.py": conanfile, + "pkga/lib/pkga.so": "", + "pkga/bin/pkga.dll": "", + "pkgb/conanfile.py": conanfile, + "pkgb/lib/pkgb.so": ""}) + c.run("export-pkg pkga --name=pkga --version=1.0") + c.run("export-pkg pkgb --name=pkgb --version=1.0") + c.run("install --requires=pkga/1.0 --requires=pkgb/1.0 --deployer=runtime_deploy " + "--deployer-folder=myruntime -vvv") + + expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"]) + assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected From 21083059160fa1a57033dc3f0bcb85d2eac86a67 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 3 Apr 2024 00:32:47 +0200 Subject: [PATCH 3/5] wip --- conan/internal/deploy.py | 31 +++++++++++++++---------------- 1 file changed, 15 insertions(+), 16 deletions(-) diff --git a/conan/internal/deploy.py b/conan/internal/deploy.py index dba30abb9cd..9c215f36182 100644 --- a/conan/internal/deploy.py +++ b/conan/internal/deploy.py @@ -93,31 +93,30 @@ def runtime_deploy(graph, output_folder): Deploy all the shared libraries and the executables of the dependencies in a flat directory. """ conanfile = graph.root.conanfile - conanfile.output.info(f"Deploying the runtime...") + output = ConanOutput(scope="runtime_deploy") + output.info(f"Deploying dependencies runtime to folder: {output_folder}") + output.warning("This deployer is experimental and subject to change. " + "Please give feedback at https://github.com/conan-io/conan/issues") mkdir(output_folder) - for _, dep in conanfile.dependencies.items(): - conanfile.output.verbose(f"Searching for shared libraries and executables in {dep.ref}...") + for _, dep in conanfile.dependencies.host.items(): if dep.package_folder is None: - conanfile.output.verbose(f"{dep.ref} does not have any package folder") + output.warning(f"{dep.ref} does not have any package folder, skipping binary") continue - if not dep.cpp_info.bindirs and not dep.cpp_info.libdirs: - conanfile.output.verbose(f"{dep.ref} does not have any bin or lib directory") - continue - + count = 0 for bindir in dep.cpp_info.bindirs: if not os.path.isdir(bindir): - conanfile.output.warning(f"{bindir} does not exist") + output.warning(f"{dep.ref} {bindir} does not exist") continue - count = _flatten_directory(dep, conanfile, bindir, output_folder) - conanfile.output.info(f"Copied {count} files from {dep.ref}, directory named {bindir}") + count += _flatten_directory(dep, conanfile, bindir, output_folder) for libdir in dep.cpp_info.libdirs: if not os.path.isdir(libdir): - conanfile.output.warning(f"{libdir} does not exist") + output.warning(f"{dep.ref} {libdir} does not exist") continue - count = _flatten_directory(dep, conanfile, libdir, output_folder, [".dylib", ".so"]) - conanfile.output.info(f"Copied {count} files from {dep.ref}, directory named {libdir}") - conanfile.output.info(f"Runtime deployed!") + count += _flatten_directory(dep, conanfile, libdir, output_folder, [".dylib", ".so"]) + + output.info(f"Copied {count} files from {dep.ref}") + conanfile.output.success(f"Runtime deployed to folder: {output_folder}") def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter=None): @@ -140,7 +139,7 @@ def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter=Non try: file_count += 1 shutil.copy2(src_filepath, dest_filepath, follow_symlinks=symlinks) - conanfile.output.verbose(f"Copied {src_filename} into {output_dir}") + conanfile.output.verbose(f"Copied {src_filepath} into {output_dir}") except Exception as e: if "WinError 1314" in str(e): ConanOutput().error("runtime_deploy: Symlinks in Windows require admin privileges " From 744741a60a5da92a62929ed1c6662649cbfa073f Mon Sep 17 00:00:00 2001 From: memsharded Date: Mon, 10 Jun 2024 23:40:26 +0200 Subject: [PATCH 4/5] review, add components support --- conan/internal/deploy.py | 29 ++++++++++++------- .../functional/command/test_install_deploy.py | 29 +++++++++++++++++++ 2 files changed, 48 insertions(+), 10 deletions(-) diff --git a/conan/internal/deploy.py b/conan/internal/deploy.py index 9c215f36182..d5f04693d20 100644 --- a/conan/internal/deploy.py +++ b/conan/internal/deploy.py @@ -1,3 +1,4 @@ +import filecmp import os import shutil @@ -98,35 +99,38 @@ def runtime_deploy(graph, output_folder): output.warning("This deployer is experimental and subject to change. " "Please give feedback at https://github.com/conan-io/conan/issues") mkdir(output_folder) + symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) for _, dep in conanfile.dependencies.host.items(): if dep.package_folder is None: output.warning(f"{dep.ref} does not have any package folder, skipping binary") continue count = 0 - for bindir in dep.cpp_info.bindirs: + cpp_info = dep.cpp_info.aggregated_components() + for bindir in cpp_info.bindirs: if not os.path.isdir(bindir): output.warning(f"{dep.ref} {bindir} does not exist") continue - count += _flatten_directory(dep, conanfile, bindir, output_folder) + count += _flatten_directory(dep, bindir, output_folder, symlinks) - for libdir in dep.cpp_info.libdirs: + for libdir in cpp_info.libdirs: if not os.path.isdir(libdir): output.warning(f"{dep.ref} {libdir} does not exist") continue - count += _flatten_directory(dep, conanfile, libdir, output_folder, [".dylib", ".so"]) + count += _flatten_directory(dep, libdir, output_folder, symlinks, [".dylib", ".so"]) output.info(f"Copied {count} files from {dep.ref}") conanfile.output.success(f"Runtime deployed to folder: {output_folder}") -def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter=None): +def _flatten_directory(dep, src_dir, output_dir, symlinks, extension_filter=None): """ Copy all the files from the source directory in a flat output directory. An optional string, named extension_filter, can be set to copy only the files with the listed extensions. """ file_count = 0 - symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True) + + output = ConanOutput(scope="runtime_deploy") for src_dirpath, _, src_filenames in os.walk(src_dir, followlinks=symlinks): for src_filename in src_filenames: if extension_filter and not any(src_filename.endswith(ext) for ext in extension_filter): @@ -135,16 +139,21 @@ def _flatten_directory(dep, conanfile, src_dir, output_dir, extension_filter=Non src_filepath = os.path.join(src_dirpath, src_filename) dest_filepath = os.path.join(output_dir, src_filename) if os.path.exists(dest_filepath): - conanfile.output.warning(f"{src_filename} already exists and will be overwritten") + if filecmp.cmp(src_filepath, dest_filepath): # Be efficient, do not copy + output.verbose(f"{dest_filepath} exists with same contents, skipping copy") + continue + else: + output.warning(f"{dest_filepath} exists and will be overwritten") + try: file_count += 1 shutil.copy2(src_filepath, dest_filepath, follow_symlinks=symlinks) - conanfile.output.verbose(f"Copied {src_filepath} into {output_dir}") + output.verbose(f"Copied {src_filepath} into {output_dir}") except Exception as e: if "WinError 1314" in str(e): - ConanOutput().error("runtime_deploy: Symlinks in Windows require admin privileges " + ConanOutput().error("runtime_deploy: Windows symlinks require admin privileges " "or 'Developer mode = ON'", error_type="exception") - raise ConanException(f"runtime_deploy: The copy of '{dep}' files failed: {e}.\nYou can " + raise ConanException(f"runtime_deploy: Copy of '{dep}' files failed: {e}.\nYou can " f"use 'tools.deployer:symlinks' conf to disable symlinks") return file_count diff --git a/test/functional/command/test_install_deploy.py b/test/functional/command/test_install_deploy.py index 8be0e9b4d19..1967e4b4836 100644 --- a/test/functional/command/test_install_deploy.py +++ b/test/functional/command/test_install_deploy.py @@ -460,3 +460,32 @@ def package(self): expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"]) assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected + + def test_runtime_deploy_components(self): + c = TestClient() + conanfile = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import copy + class Pkg(ConanFile): + def package(self): + copy(self, "*.so", src=self.build_folder, + dst=os.path.join(self.package_folder, "a")) + copy(self, "*.dll", src=self.build_folder, + dst=os.path.join(self.package_folder, "b")) + def package_info(self): + self.cpp_info.components["a"].libdirs = ["a"] + self.cpp_info.components["b"].bindirs = ["b"] + """) + c.save({"pkga/conanfile.py": conanfile, + "pkga/lib/pkga.so": "", + "pkga/bin/pkga.dll": "", + "pkgb/conanfile.py": conanfile, + "pkgb/lib/pkgb.so": ""}) + c.run("export-pkg pkga --name=pkga --version=1.0") + c.run("export-pkg pkgb --name=pkgb --version=1.0") + c.run("install --requires=pkga/1.0 --requires=pkgb/1.0 --deployer=runtime_deploy " + "--deployer-folder=myruntime -vvv") + + expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"]) + assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected From d244a5dc5d2f66c93e9df0a20f8bc43f0ab46cf5 Mon Sep 17 00:00:00 2001 From: memsharded Date: Wed, 12 Jun 2024 10:15:41 +0200 Subject: [PATCH 5/5] docstrings --- conan/cli/commands/build.py | 3 ++- conan/cli/commands/graph.py | 5 ++++- conan/cli/commands/install.py | 3 ++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/conan/cli/commands/build.py b/conan/cli/commands/build.py index 2212e5f79ac..bf1543a8681 100644 --- a/conan/cli/commands/build.py +++ b/conan/cli/commands/build.py @@ -23,7 +23,8 @@ def build(conan_api, parser, *args): parser.add_argument("-of", "--output-folder", help='The root output folder for generated and build files') parser.add_argument("-d", "--deployer", action="append", - help='Deploy using the provided deployer to the output folder') + help="Deploy using the provided deployer to the output folder. " + "Built-in deployers: 'full_deploy', 'direct_deploy', 'runtime_deploy'") parser.add_argument("--deployer-folder", help="Deployer output folder, base build folder by default if not set") parser.add_argument("--build-require", action='store_true', default=False, diff --git a/conan/cli/commands/graph.py b/conan/cli/commands/graph.py index c5fb5f168bd..85d7cb510ec 100644 --- a/conan/cli/commands/graph.py +++ b/conan/cli/commands/graph.py @@ -175,7 +175,10 @@ def graph_info(conan_api, parser, subparser, *args): subparser.add_argument("--package-filter", action="append", help='Print information only for packages that match the patterns') subparser.add_argument("-d", "--deployer", action="append", - help='Deploy using the provided deployer to the output folder') + help="Deploy using the provided deployer to the output folder. " + "Built-in deployers: 'full_deploy', 'direct_deploy'. Deployers " + "will only deploy recipes, as 'conan graph info' do not retrieve " + "binaries") subparser.add_argument("-df", "--deployer-folder", help="Deployer output folder, base build folder by default if not set") subparser.add_argument("--build-require", action='store_true', default=False, diff --git a/conan/cli/commands/install.py b/conan/cli/commands/install.py index cad3168a543..703e022ea0f 100644 --- a/conan/cli/commands/install.py +++ b/conan/cli/commands/install.py @@ -30,7 +30,8 @@ def install(conan_api, parser, *args): parser.add_argument("-of", "--output-folder", help='The root output folder for generated and build files') parser.add_argument("-d", "--deployer", action="append", - help='Deploy using the provided deployer to the output folder') + help="Deploy using the provided deployer to the output folder. " + "Built-in deployers: 'full_deploy', 'direct_deploy', 'runtime_deploy'") parser.add_argument("--deployer-folder", help="Deployer output folder, base build folder by default if not set") parser.add_argument("--deployer-package", action="append",