diff --git a/conan/cli/commands/create.py b/conan/cli/commands/create.py index fe16deaf019..13fa6d337e5 100644 --- a/conan/cli/commands/create.py +++ b/conan/cli/commands/create.py @@ -95,6 +95,9 @@ def create(conan_api, parser, *args): else: requires = [ref] if not is_build else None tool_requires = [ref] if is_build else None + if conanfile.vendor: # Automatically allow repackaging for conan create + pr = profile_build if is_build else profile_host + pr.conf.update("&:tools.graph:vendor", "build") deps_graph = conan_api.graph.load_graph_requires(requires, tool_requires, profile_host=profile_host, profile_build=profile_build, diff --git a/conan/cli/formatters/graph/info_graph_html.py b/conan/cli/formatters/graph/info_graph_html.py index 4179698501a..67db93e2ff9 100644 --- a/conan/cli/formatters/graph/info_graph_html.py +++ b/conan/cli/formatters/graph/info_graph_html.py @@ -144,6 +144,11 @@ if (node.recipe == "Platform") { font.background = "Violet"; } + if (node.vendor) { + borderColor = "Red"; + shapeProperties = {borderDashes: [3,5]}; + borderWidth = 2; + } nodes.push({ id: node_id, font: font, @@ -225,6 +230,14 @@ font: {size: 35, color: "white"}, color: {border: "SkyBlue", background: "Black"} }); + counter++; + + legend_nodes.push({x: x + counter*step, y: y, shape: "box", + label: "vendor", font: {size: 35}, + color: {border: "Red"}, + shapeProperties: {borderDashes: [3,5]}, + borderWidth: 2 + }); return {nodes: new vis.DataSet(legend_nodes)}; } let error = document.getElementById("error"); diff --git a/conans/client/graph/compute_pid.py b/conans/client/graph/compute_pid.py index 60842cefae9..cbfff70991f 100644 --- a/conans/client/graph/compute_pid.py +++ b/conans/client/graph/compute_pid.py @@ -38,6 +38,9 @@ def compute_package_id(node, new_config, config_version): else: data[require] = req_info + if conanfile.vendor: # Make the package_id fully independent of dependencies versions + data, build_data = OrderedDict(), OrderedDict() # TODO, cleaner, now minimal diff + reqs_info = RequirementsInfo(data) build_requires_info = RequirementsInfo(build_data) python_requires = PythonRequiresInfo(python_requires, python_mode) diff --git a/conans/client/graph/graph.py b/conans/client/graph/graph.py index 453cbf851eb..74a7f0273e9 100644 --- a/conans/client/graph/graph.py +++ b/conans/client/graph/graph.py @@ -108,6 +108,8 @@ def propagate_downstream(self, require, node, src_node=None): self.transitive_deps.pop(require, None) self.transitive_deps[require] = TransitiveRequirement(require, node) + if self.conanfile.vendor: + return # Check if need to propagate downstream if not self.dependants: return @@ -154,6 +156,8 @@ def check_downstream_exists(self, require): # Check if need to propagate downstream # Then propagate downstream + if self.conanfile.vendor: + return result # Seems the algrithm depth-first, would only have 1 dependant at most to propagate down # at any given time if not self.dependants: diff --git a/conans/client/graph/graph_binaries.py b/conans/client/graph/graph_binaries.py index c343ccd2ba1..fc1980d992d 100644 --- a/conans/client/graph/graph_binaries.py +++ b/conans/client/graph/graph_binaries.py @@ -181,6 +181,15 @@ def _evaluate_node(self, node, build_mode, remotes, update): node.build_allowed = True node.binary = BINARY_BUILD if not node.cant_build else BINARY_INVALID + if node.binary == BINARY_BUILD: + conanfile = node.conanfile + if conanfile.vendor and not conanfile.conf.get("tools.graph:vendor", choices=("build",)): + node.conanfile.info.invalid = f"The package '{conanfile.ref}' is a vendoring one, " \ + f"needs to be built from source, but it " \ + "didn't enable 'tools.graph:vendor=build' to compute " \ + "its dependencies" + node.binary = BINARY_INVALID + def _process_node(self, node, build_mode, remotes, update): # Check that this same reference hasn't already been checked if self._evaluate_is_cached(node): diff --git a/conans/client/graph/graph_builder.py b/conans/client/graph/graph_builder.py index d248d25bba4..41793dee9e6 100644 --- a/conans/client/graph/graph_builder.py +++ b/conans/client/graph/graph_builder.py @@ -5,7 +5,7 @@ from conan.internal.cache.conan_reference_layout import BasicLayout from conans.client.conanfile.configure import run_configure_method from conans.client.graph.graph import DepsGraph, Node, CONTEXT_HOST, \ - CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL + CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE from conans.client.graph.graph import RECIPE_PLATFORM from conans.client.graph.graph_error import GraphLoopError, GraphConflictError, GraphMissingError, \ GraphRuntimeError, GraphError @@ -52,7 +52,10 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None): continue new_node = self._expand_require(require, node, dep_graph, profile_host, profile_build, graph_lock) - if new_node: + if new_node and (not new_node.conanfile.vendor + or new_node.recipe == RECIPE_EDITABLE or + new_node.conanfile.conf.get("tools.graph:vendor", + choices=("build",))): self._initialize_requires(new_node, dep_graph, graph_lock, profile_build, profile_host) open_requires.extendleft((r, new_node) @@ -386,6 +389,7 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr @staticmethod def _compute_down_options(node, require, new_ref): # The consumer "up_options" are the options that come from downstream to this node + visible = require.visible and not node.conanfile.vendor if require.options is not None: # If the consumer has specified "requires(options=xxx)", we need to use it # It will have less priority than downstream consumers @@ -395,11 +399,11 @@ def _compute_down_options(node, require, new_ref): # options["dep"].opt=value only propagate to visible and host dependencies # we will evaluate if necessary a potential "build_options", but recall that it is # now possible to do "self.build_requires(..., options={k:v})" to specify it - if require.visible: + if visible: # Only visible requirements in the host context propagate options from downstream down_options.update_options(node.conanfile.up_options) else: - if require.visible: + if visible: down_options = node.conanfile.up_options elif not require.build: # for requires in "host", like test_requires, pass myoptions down_options = node.conanfile.private_up_options diff --git a/conans/model/conan_file.py b/conans/model/conan_file.py index 7fcdbf8ac23..a174b260b1c 100644 --- a/conans/model/conan_file.py +++ b/conans/model/conan_file.py @@ -47,8 +47,8 @@ class ConanFile: default_options = None default_build_options = None package_type = None + vendor = False languages = [] - implements = [] provides = None @@ -170,6 +170,7 @@ def serialize(self): result["label"] = self.display_name if self.info is not None: result["info"] = self.info.serialize() + result["vendor"] = self.vendor return result @property diff --git a/conans/model/conf.py b/conans/model/conf.py index e110378455e..49a27c3ff17 100644 --- a/conans/model/conf.py +++ b/conans/model/conf.py @@ -83,6 +83,7 @@ "tools.files.download:retry": "Number of retries in case of failure when downloading", "tools.files.download:retry_wait": "Seconds to wait between download attempts", "tools.files.download:verify": "If set, overrides recipes on whether to perform SSL verification for their downloaded files. Only recommended to be set while testing", + "tools.graph:vendor": "(Experimental) If 'build', enables the computation of dependencies of vendoring packages to build them", "tools.graph:skip_binaries": "Allow the graph to skip binaries not needed in the current configuration (True by default)", "tools.gnu:make_program": "Indicate path to make program", "tools.gnu:define_libcxx11_abi": "Force definition of GLIBCXX_USE_CXX11_ABI=1 for libstdc++11", diff --git a/test/integration/command/info/info_test.py b/test/integration/command/info/info_test.py index be8b53186f1..e2bde2a0acb 100644 --- a/test/integration/command/info/info_test.py +++ b/test/integration/command/info/info_test.py @@ -4,7 +4,7 @@ from conan.cli.exit_codes import ERROR_GENERAL from conans.model.recipe_ref import RecipeReference -from conan.test.utils.tools import TestClient, GenConanfile, TurboTestClient +from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, GenConanfile, TurboTestClient class TestBasicCliOutput: @@ -425,3 +425,27 @@ class Pkg(ConanFile): assert "pkg/0.1@user" in c.out c.run("graph info . --channel=channel") assert "pkg/0.1@user/channel" in c.out + + +def test_graph_info_bundle(): + c = TestClient(light=True) + c.save({"subfolder/conanfile.py": GenConanfile("liba", "1.0")}) + c.run("create ./subfolder") + conanfile = textwrap.dedent(""" + from conan import ConanFile + class RepackageRecipe(ConanFile): + name = "lib" + version = "1.0" + def requirements(self): + self.requires("liba/1.0") + vendor = True + """) + c.save({"conanfile.py": conanfile}) + c.run("create .") + c.save({"conanfile.py": GenConanfile("consumer", "1.0").with_requires("lib/1.0")}) + + c.run("graph info . --build='lib*'") + c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Invalid")}) + + c.run("graph info . -c tools.graph:vendor=build --build='lib*'") + c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Build")}) diff --git a/test/integration/command_v2/test_inspect.py b/test/integration/command_v2/test_inspect.py index 8de7c470ac1..cd744580017 100644 --- a/test/integration/command_v2/test_inspect.py +++ b/test/integration/command_v2/test_inspect.py @@ -22,7 +22,9 @@ def test_basic_inspect(): " shared: ['True', 'False']", 'package_type: None', 'requires: []', - 'revision_mode: hash'] + 'revision_mode: hash', + 'vendor: False' + ] def test_options_description(): @@ -92,6 +94,7 @@ def test_normal_inspect(): 'package_type: None', 'requires: []', 'revision_mode: hash', + 'vendor: False', 'version: 1.0'] @@ -137,7 +140,8 @@ class Pkg(ConanFile): "False, 'test': False, 'force': False, 'direct': True, 'build': " "False, 'transitive_headers': None, 'transitive_libs': None, 'headers': " "True, 'package_id_mode': None, 'visible': True}]", - 'revision_mode: hash'] == tc.out.splitlines() + 'revision_mode: hash', + 'vendor: False'] == tc.out.splitlines() def test_pythonrequires_remote(): diff --git a/test/integration/test_package_vendor.py b/test/integration/test_package_vendor.py new file mode 100644 index 00000000000..562ae4cd386 --- /dev/null +++ b/test/integration/test_package_vendor.py @@ -0,0 +1,140 @@ +import os +import textwrap + +from conan.test.assets.genconanfile import GenConanfile +from conan.test.utils.tools import TestClient + + +def test_package_vendor(): + c = TestClient() + app = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import copy, save + + class App(ConanFile): + name = "app" + version = "0.1" + package_type = "application" + vendor = True + requires = "pkga/0.1" + def package(self): + copy(self, "*", src=self.dependencies["pkga"].package_folder, + dst=self.package_folder) + save(self, os.path.join(self.package_folder, "app.exe"), "app") + """) + + c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library") + .with_package_file("pkga.dll", "dll"), + "app/conanfile.py": app + }) + c.run("create pkga") + c.run("create app") # -c tools.graph:vendor=build will be automatic + assert "app/0.1: package(): Packaged 1 '.dll' file: pkga.dll" in c.out + + # we can safely remove pkga + c.run("remove pkg* -c") + c.run("list app:*") + assert "pkga" not in c.out # The binary doesn't depend on pkga + c.run("install --requires=app/0.1 --deployer=full_deploy") + assert "pkga" not in c.out + assert c.load("full_deploy/host/app/0.1/app.exe") == "app" + assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll" + + # we can create a modified pkga + c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library") + .with_package_file("pkga.dll", "newdll")}) + c.run("create pkga") + # still using the re-packaged one + c.run("install --requires=app/0.1 --deployer=full_deploy") + assert "pkga" not in c.out + assert c.load("full_deploy/host/app/0.1/app.exe") == "app" + assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll" + + # but we can force the expansion, still not the rebuild + c.run("install --requires=app/0.1 --deployer=full_deploy -c tools.graph:vendor=build") + assert "pkga" in c.out + assert c.load("full_deploy/host/app/0.1/app.exe") == "app" + assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll" + + # and finally we can force the expansion and the rebuild + c.run("install --requires=app/0.1 --build=app* --deployer=full_deploy " + "-c tools.graph:vendor=build") + assert "pkga" in c.out + assert c.load("full_deploy/host/app/0.1/app.exe") == "app" + assert c.load("full_deploy/host/app/0.1/pkga.dll") == "newdll" + # This shoulnd't happen, no visibility over transitive dependencies of app + assert not os.path.exists(os.path.join(c.current_folder, "full_deploy", "host", "pkga")) + + # lets remove the binary + c.run("remove app:* -c") + c.run("install --requires=app/0.1", assert_error=True) + assert "Missing binary" in c.out + c.run("install --requires=app/0.1 --build=missing", assert_error=True) + assert "app/0.1: Invalid: The package 'app/0.1' is a vendoring one, needs to be built " \ + "from source, but it didn't enable 'tools.graph:vendor=build'" in c.out + + c.run("install --requires=app/0.1 --build=missing -c tools.graph:vendor=build") + assert "pkga" in c.out # it works + + +def test_package_vendor_editable(): + c = TestClient() + pkgb = textwrap.dedent(""" + import os + from conan import ConanFile + from conan.tools.files import copy, save + + class App(ConanFile): + name = "pkgb" + version = "0.1" + package_type = "shared-library" + vendor = True + requires = "pkga/0.1" + def layout(self): + self.folders.build = "build" + self.cpp.build.bindirs = ["build"] + def generate(self): + copy(self, "*", src=self.dependencies["pkga"].package_folder, + dst=self.build_folder) + def build(self): + save(self, os.path.join(self.build_folder, "pkgb.dll"), "dll") + """) + + c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library") + .with_package_file("bin/pkga.dll", "d"), + "pkgb/conanfile.py": pkgb, + "app/conanfile.py": GenConanfile("app", "0.1").with_settings("os") + .with_requires("pkgb/0.1") + }) + c.run("create pkga") + c.run("editable add pkgb") + c.run("install app -s os=Linux") + assert "pkga" in c.out + # The environment file of "app" doesn't have any visibility of the "pkga" paths + envfile_app = c.load("app/conanrunenv.sh") + assert "pkga" not in envfile_app + # But the environment file needed to build "pkgb" has visibility over the "pkga" paths + envfile_pkgb = c.load("pkgb/conanrunenv.sh") + assert "pkga" in envfile_pkgb + + +def test_vendor_dont_propagate_options(): + c = TestClient() + app = GenConanfile("app", "0.1").with_requires("pkga/0.1").with_class_attribute("vendor=True") + c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_shared_option(False), + "app/conanfile.py": app, + "consumer/conanfile.txt": "[requires]\napp/0.1", + "consumer_shared/conanfile.txt": "[requires]\napp/0.1\n[options]\n*:shared=True" + }) + c.run("create pkga") + c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Build")}) + c.run("create app") + c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"), + "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}) + c.run("install consumer --build=app/* -c tools.graph:vendor=build") + c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"), + "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")}) + c.run("install consumer_shared --build=app/* -c tools.graph:vendor=build") + c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"), + "app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})