Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add built in runtime deployer #15382

Merged
merged 9 commits into from
Jun 12, 2024
3 changes: 2 additions & 1 deletion conan/cli/commands/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,8 @@ def build(conan_api, parser, *args):
parser.add_argument("-of", "--output-folder",
help='The root output folder for generated and build files')
parser.add_argument("-d", "--deployer", action="append",
help='Deploy using the provided deployer to the output folder')
help="Deploy using the provided deployer to the output folder. "
"Built-in deployers: 'full_deploy', 'direct_deploy', 'runtime_deploy'")
parser.add_argument("--deployer-folder",
help="Deployer output folder, base build folder by default if not set")
parser.add_argument("--build-require", action='store_true', default=False,
Expand Down
5 changes: 4 additions & 1 deletion conan/cli/commands/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -175,7 +175,10 @@ def graph_info(conan_api, parser, subparser, *args):
subparser.add_argument("--package-filter", action="append",
help='Print information only for packages that match the patterns')
subparser.add_argument("-d", "--deployer", action="append",
help='Deploy using the provided deployer to the output folder')
help="Deploy using the provided deployer to the output folder. "
"Built-in deployers: 'full_deploy', 'direct_deploy'. Deployers "
"will only deploy recipes, as 'conan graph info' do not retrieve "
"binaries")
subparser.add_argument("-df", "--deployer-folder",
help="Deployer output folder, base build folder by default if not set")
subparser.add_argument("--build-require", action='store_true', default=False,
Expand Down
3 changes: 2 additions & 1 deletion conan/cli/commands/install.py
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ def install(conan_api, parser, *args):
parser.add_argument("-of", "--output-folder",
help='The root output folder for generated and build files')
parser.add_argument("-d", "--deployer", action="append",
help='Deploy using the provided deployer to the output folder')
help="Deploy using the provided deployer to the output folder. "
"Built-in deployers: 'full_deploy', 'direct_deploy', 'runtime_deploy'")
parser.add_argument("--deployer-folder",
help="Deployer output folder, base build folder by default if not set")
parser.add_argument("--deployer-package", action="append",
Expand Down
73 changes: 72 additions & 1 deletion conan/internal/deploy.py
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
import filecmp
import os
import shutil

Expand Down Expand Up @@ -31,7 +32,8 @@ def _load(path):
if os.path.isfile(cache_path):
return _load(cache_path)
builtin_deploy = {"full_deploy.py": full_deploy,
"direct_deploy.py": direct_deploy}.get(d)
"direct_deploy.py": direct_deploy,
"runtime_deploy.py": runtime_deploy}.get(d)
if builtin_deploy is not None:
return builtin_deploy
raise ConanException(f"Cannot find deployer '{d}'")
Expand Down Expand Up @@ -87,6 +89,75 @@ def full_deploy(graph, output_folder):
_deploy_single(dep, conanfile, output_folder, folder_name)


def runtime_deploy(graph, output_folder):
"""
Deploy all the shared libraries and the executables of the dependencies in a flat directory.
"""
conanfile = graph.root.conanfile
output = ConanOutput(scope="runtime_deploy")
output.info(f"Deploying dependencies runtime to folder: {output_folder}")
output.warning("This deployer is experimental and subject to change. "
"Please give feedback at https://github.com/conan-io/conan/issues")
mkdir(output_folder)
symlinks = conanfile.conf.get("tools.deployer:symlinks", check_type=bool, default=True)
for _, dep in conanfile.dependencies.host.items():
if dep.package_folder is None:
output.warning(f"{dep.ref} does not have any package folder, skipping binary")
continue
count = 0
cpp_info = dep.cpp_info.aggregated_components()
for bindir in cpp_info.bindirs:
if not os.path.isdir(bindir):
output.warning(f"{dep.ref} {bindir} does not exist")
continue
count += _flatten_directory(dep, bindir, output_folder, symlinks)

for libdir in cpp_info.libdirs:
if not os.path.isdir(libdir):
output.warning(f"{dep.ref} {libdir} does not exist")
continue
count += _flatten_directory(dep, libdir, output_folder, symlinks, [".dylib", ".so"])

output.info(f"Copied {count} files from {dep.ref}")
conanfile.output.success(f"Runtime deployed to folder: {output_folder}")


def _flatten_directory(dep, src_dir, output_dir, symlinks, extension_filter=None):
"""
Copy all the files from the source directory in a flat output directory.
An optional string, named extension_filter, can be set to copy only the files with
the listed extensions.
"""
file_count = 0

output = ConanOutput(scope="runtime_deploy")
for src_dirpath, _, src_filenames in os.walk(src_dir, followlinks=symlinks):
for src_filename in src_filenames:
if extension_filter and not any(src_filename.endswith(ext) for ext in extension_filter):
continue

src_filepath = os.path.join(src_dirpath, src_filename)
dest_filepath = os.path.join(output_dir, src_filename)
if os.path.exists(dest_filepath):
if filecmp.cmp(src_filepath, dest_filepath): # Be efficient, do not copy
output.verbose(f"{dest_filepath} exists with same contents, skipping copy")
continue
else:
output.warning(f"{dest_filepath} exists and will be overwritten")

try:
file_count += 1
shutil.copy2(src_filepath, dest_filepath, follow_symlinks=symlinks)
output.verbose(f"Copied {src_filepath} into {output_dir}")
except Exception as e:
if "WinError 1314" in str(e):
ConanOutput().error("runtime_deploy: Windows symlinks require admin privileges "
"or 'Developer mode = ON'", error_type="exception")
raise ConanException(f"runtime_deploy: Copy of '{dep}' files failed: {e}.\nYou can "
f"use 'tools.deployer:symlinks' conf to disable symlinks")
return file_count


def _deploy_single(dep, conanfile, output_folder, folder_name):
new_folder = os.path.join(output_folder, folder_name)
rmdir(new_folder)
Expand Down
54 changes: 54 additions & 0 deletions test/functional/command/test_install_deploy.py
Original file line number Diff line number Diff line change
Expand Up @@ -435,3 +435,57 @@ def package_info(self):

env = c.load("conanbuildenv-release-x86_64.sh")
assert f'export MYPATH="{some_abs_path}/mypath"' in env


class TestRuntimeDeployer:
def test_runtime_deploy(self):
c = TestClient()
conanfile = textwrap.dedent("""
from conan import ConanFile
from conan.tools.files import copy
class Pkg(ConanFile):
def package(self):
copy(self, "*.so", src=self.build_folder, dst=self.package_folder)
copy(self, "*.dll", src=self.build_folder, dst=self.package_folder)
""")
c.save({"pkga/conanfile.py": conanfile,
"pkga/lib/pkga.so": "",
"pkga/bin/pkga.dll": "",
"pkgb/conanfile.py": conanfile,
"pkgb/lib/pkgb.so": ""})
c.run("export-pkg pkga --name=pkga --version=1.0")
c.run("export-pkg pkgb --name=pkgb --version=1.0")
c.run("install --requires=pkga/1.0 --requires=pkgb/1.0 --deployer=runtime_deploy "
"--deployer-folder=myruntime -vvv")

expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"])
assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected

def test_runtime_deploy_components(self):
c = TestClient()
conanfile = textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import copy
class Pkg(ConanFile):
def package(self):
copy(self, "*.so", src=self.build_folder,
dst=os.path.join(self.package_folder, "a"))
copy(self, "*.dll", src=self.build_folder,
dst=os.path.join(self.package_folder, "b"))
def package_info(self):
self.cpp_info.components["a"].libdirs = ["a"]
self.cpp_info.components["b"].bindirs = ["b"]
""")
c.save({"pkga/conanfile.py": conanfile,
"pkga/lib/pkga.so": "",
"pkga/bin/pkga.dll": "",
"pkgb/conanfile.py": conanfile,
"pkgb/lib/pkgb.so": ""})
c.run("export-pkg pkga --name=pkga --version=1.0")
c.run("export-pkg pkgb --name=pkgb --version=1.0")
c.run("install --requires=pkga/1.0 --requires=pkgb/1.0 --deployer=runtime_deploy "
"--deployer-folder=myruntime -vvv")

expected = sorted(["pkga.so", "pkgb.so", "pkga.dll"])
assert sorted(os.listdir(os.path.join(c.current_folder, "myruntime"))) == expected