Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Package vendor new feature #16073

Merged
merged 21 commits into from
Jun 5, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
3 changes: 3 additions & 0 deletions conan/cli/commands/create.py
Original file line number Diff line number Diff line change
Expand Up @@ -95,6 +95,9 @@ def create(conan_api, parser, *args):
else:
requires = [ref] if not is_build else None
tool_requires = [ref] if is_build else None
if conanfile.vendor: # Automatically allow repackaging for conan create
pr = profile_build if is_build else profile_host
pr.conf.update("&:tools.graph:vendor", "build")
deps_graph = conan_api.graph.load_graph_requires(requires, tool_requires,
profile_host=profile_host,
profile_build=profile_build,
Expand Down
13 changes: 13 additions & 0 deletions conan/cli/formatters/graph/info_graph_html.py
Original file line number Diff line number Diff line change
Expand Up @@ -144,6 +144,11 @@
if (node.recipe == "Platform") {
font.background = "Violet";
}
if (node.vendor) {
borderColor = "Red";
shapeProperties = {borderDashes: [3,5]};
borderWidth = 2;
}
nodes.push({
id: node_id,
font: font,
Expand Down Expand Up @@ -225,6 +230,14 @@
font: {size: 35, color: "white"},
color: {border: "SkyBlue", background: "Black"}
});
counter++;

legend_nodes.push({x: x + counter*step, y: y, shape: "box",
label: "vendor", font: {size: 35},
color: {border: "Red"},
shapeProperties: {borderDashes: [3,5]},
borderWidth: 2
});
return {nodes: new vis.DataSet(legend_nodes)};
}
let error = document.getElementById("error");
Expand Down
3 changes: 3 additions & 0 deletions conans/client/graph/compute_pid.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ def compute_package_id(node, new_config, config_version):
else:
data[require] = req_info

if conanfile.vendor: # Make the package_id fully independent of dependencies versions
data, build_data = OrderedDict(), OrderedDict() # TODO, cleaner, now minimal diff

reqs_info = RequirementsInfo(data)
build_requires_info = RequirementsInfo(build_data)
python_requires = PythonRequiresInfo(python_requires, python_mode)
Expand Down
4 changes: 4 additions & 0 deletions conans/client/graph/graph.py
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,8 @@ def propagate_downstream(self, require, node, src_node=None):
self.transitive_deps.pop(require, None)
self.transitive_deps[require] = TransitiveRequirement(require, node)

if self.conanfile.vendor:
return
# Check if need to propagate downstream
if not self.dependants:
return
Expand Down Expand Up @@ -154,6 +156,8 @@ def check_downstream_exists(self, require):
# Check if need to propagate downstream
# Then propagate downstream

if self.conanfile.vendor:
return result
# Seems the algrithm depth-first, would only have 1 dependant at most to propagate down
# at any given time
if not self.dependants:
Expand Down
9 changes: 9 additions & 0 deletions conans/client/graph/graph_binaries.py
Original file line number Diff line number Diff line change
Expand Up @@ -181,6 +181,15 @@ def _evaluate_node(self, node, build_mode, remotes, update):
node.build_allowed = True
node.binary = BINARY_BUILD if not node.cant_build else BINARY_INVALID

if node.binary == BINARY_BUILD:
conanfile = node.conanfile
if conanfile.vendor and not conanfile.conf.get("tools.graph:vendor", choices=("build",)):
node.conanfile.info.invalid = f"The package '{conanfile.ref}' is a vendoring one, " \
f"needs to be built from source, but it " \
"didn't enable 'tools.graph:vendor=build' to compute " \
memsharded marked this conversation as resolved.
Show resolved Hide resolved
"its dependencies"
node.binary = BINARY_INVALID

def _process_node(self, node, build_mode, remotes, update):
# Check that this same reference hasn't already been checked
if self._evaluate_is_cached(node):
Expand Down
12 changes: 8 additions & 4 deletions conans/client/graph/graph_builder.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
from conan.internal.cache.conan_reference_layout import BasicLayout
from conans.client.conanfile.configure import run_configure_method
from conans.client.graph.graph import DepsGraph, Node, CONTEXT_HOST, \
CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL
CONTEXT_BUILD, TransitiveRequirement, RECIPE_VIRTUAL, RECIPE_EDITABLE
from conans.client.graph.graph import RECIPE_PLATFORM
from conans.client.graph.graph_error import GraphLoopError, GraphConflictError, GraphMissingError, \
GraphRuntimeError, GraphError
Expand Down Expand Up @@ -52,7 +52,10 @@ def load_graph(self, root_node, profile_host, profile_build, graph_lock=None):
continue
new_node = self._expand_require(require, node, dep_graph, profile_host,
profile_build, graph_lock)
if new_node:
if new_node and (not new_node.conanfile.vendor
or new_node.recipe == RECIPE_EDITABLE or
new_node.conanfile.conf.get("tools.graph:vendor",
choices=("build",))):
self._initialize_requires(new_node, dep_graph, graph_lock, profile_build,
profile_host)
open_requires.extendleft((r, new_node)
Expand Down Expand Up @@ -386,6 +389,7 @@ def _create_new_node(self, node, require, graph, profile_host, profile_build, gr
@staticmethod
def _compute_down_options(node, require, new_ref):
# The consumer "up_options" are the options that come from downstream to this node
visible = require.visible and not node.conanfile.vendor
if require.options is not None:
# If the consumer has specified "requires(options=xxx)", we need to use it
# It will have less priority than downstream consumers
Expand All @@ -395,11 +399,11 @@ def _compute_down_options(node, require, new_ref):
# options["dep"].opt=value only propagate to visible and host dependencies
# we will evaluate if necessary a potential "build_options", but recall that it is
# now possible to do "self.build_requires(..., options={k:v})" to specify it
if require.visible:
if visible:
# Only visible requirements in the host context propagate options from downstream
down_options.update_options(node.conanfile.up_options)
else:
if require.visible:
if visible:
down_options = node.conanfile.up_options
elif not require.build: # for requires in "host", like test_requires, pass myoptions
down_options = node.conanfile.private_up_options
Expand Down
3 changes: 2 additions & 1 deletion conans/model/conan_file.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,8 +47,8 @@ class ConanFile:
default_options = None
default_build_options = None
package_type = None
vendor = False
languages = []

implements = []

provides = None
Expand Down Expand Up @@ -170,6 +170,7 @@ def serialize(self):
result["label"] = self.display_name
if self.info is not None:
result["info"] = self.info.serialize()
result["vendor"] = self.vendor
return result

@property
Expand Down
1 change: 1 addition & 0 deletions conans/model/conf.py
Original file line number Diff line number Diff line change
Expand Up @@ -83,6 +83,7 @@
"tools.files.download:retry": "Number of retries in case of failure when downloading",
"tools.files.download:retry_wait": "Seconds to wait between download attempts",
"tools.files.download:verify": "If set, overrides recipes on whether to perform SSL verification for their downloaded files. Only recommended to be set while testing",
"tools.graph:vendor": "(Experimental) If 'build', enables the computation of dependencies of vendoring packages to build them",
"tools.graph:skip_binaries": "Allow the graph to skip binaries not needed in the current configuration (True by default)",
"tools.gnu:make_program": "Indicate path to make program",
"tools.gnu:define_libcxx11_abi": "Force definition of GLIBCXX_USE_CXX11_ABI=1 for libstdc++11",
Expand Down
26 changes: 25 additions & 1 deletion test/integration/command/info/info_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@

from conan.cli.exit_codes import ERROR_GENERAL
from conans.model.recipe_ref import RecipeReference
from conan.test.utils.tools import TestClient, GenConanfile, TurboTestClient
from conan.test.utils.tools import NO_SETTINGS_PACKAGE_ID, TestClient, GenConanfile, TurboTestClient


class TestBasicCliOutput:
Expand Down Expand Up @@ -425,3 +425,27 @@ class Pkg(ConanFile):
assert "pkg/0.1@user" in c.out
c.run("graph info . --channel=channel")
assert "pkg/0.1@user/channel" in c.out


def test_graph_info_bundle():
c = TestClient(light=True)
c.save({"subfolder/conanfile.py": GenConanfile("liba", "1.0")})
c.run("create ./subfolder")
conanfile = textwrap.dedent("""
from conan import ConanFile
class RepackageRecipe(ConanFile):
name = "lib"
version = "1.0"
def requirements(self):
self.requires("liba/1.0")
vendor = True
""")
c.save({"conanfile.py": conanfile})
c.run("create .")
c.save({"conanfile.py": GenConanfile("consumer", "1.0").with_requires("lib/1.0")})

c.run("graph info . --build='lib*'")
c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Invalid")})

c.run("graph info . -c tools.graph:vendor=build --build='lib*'")
c.assert_listed_binary({"lib/1.0": (NO_SETTINGS_PACKAGE_ID, "Build")})
8 changes: 6 additions & 2 deletions test/integration/command_v2/test_inspect.py
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,9 @@ def test_basic_inspect():
" shared: ['True', 'False']",
'package_type: None',
'requires: []',
'revision_mode: hash']
'revision_mode: hash',
'vendor: False'
]


def test_options_description():
Expand Down Expand Up @@ -92,6 +94,7 @@ def test_normal_inspect():
'package_type: None',
'requires: []',
'revision_mode: hash',
'vendor: False',
'version: 1.0']


Expand Down Expand Up @@ -137,7 +140,8 @@ class Pkg(ConanFile):
"False, 'test': False, 'force': False, 'direct': True, 'build': "
"False, 'transitive_headers': None, 'transitive_libs': None, 'headers': "
"True, 'package_id_mode': None, 'visible': True}]",
'revision_mode: hash'] == tc.out.splitlines()
'revision_mode: hash',
'vendor: False'] == tc.out.splitlines()


def test_pythonrequires_remote():
Expand Down
140 changes: 140 additions & 0 deletions test/integration/test_package_vendor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,140 @@
import os
import textwrap

from conan.test.assets.genconanfile import GenConanfile
from conan.test.utils.tools import TestClient


def test_package_vendor():
c = TestClient()
app = textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import copy, save

class App(ConanFile):
name = "app"
version = "0.1"
package_type = "application"
vendor = True
requires = "pkga/0.1"
def package(self):
copy(self, "*", src=self.dependencies["pkga"].package_folder,
dst=self.package_folder)
save(self, os.path.join(self.package_folder, "app.exe"), "app")
""")

c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
.with_package_file("pkga.dll", "dll"),
"app/conanfile.py": app
})
c.run("create pkga")
c.run("create app") # -c tools.graph:vendor=build will be automatic
assert "app/0.1: package(): Packaged 1 '.dll' file: pkga.dll" in c.out

# we can safely remove pkga
c.run("remove pkg* -c")
c.run("list app:*")
assert "pkga" not in c.out # The binary doesn't depend on pkga
c.run("install --requires=app/0.1 --deployer=full_deploy")
assert "pkga" not in c.out
assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"

# we can create a modified pkga
c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
.with_package_file("pkga.dll", "newdll")})
c.run("create pkga")
# still using the re-packaged one
c.run("install --requires=app/0.1 --deployer=full_deploy")
assert "pkga" not in c.out
assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"

# but we can force the expansion, still not the rebuild
c.run("install --requires=app/0.1 --deployer=full_deploy -c tools.graph:vendor=build")
assert "pkga" in c.out
assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
assert c.load("full_deploy/host/app/0.1/pkga.dll") == "dll"

# and finally we can force the expansion and the rebuild
c.run("install --requires=app/0.1 --build=app* --deployer=full_deploy "
"-c tools.graph:vendor=build")
assert "pkga" in c.out
assert c.load("full_deploy/host/app/0.1/app.exe") == "app"
assert c.load("full_deploy/host/app/0.1/pkga.dll") == "newdll"
# This shoulnd't happen, no visibility over transitive dependencies of app
assert not os.path.exists(os.path.join(c.current_folder, "full_deploy", "host", "pkga"))

# lets remove the binary
c.run("remove app:* -c")
c.run("install --requires=app/0.1", assert_error=True)
assert "Missing binary" in c.out
c.run("install --requires=app/0.1 --build=missing", assert_error=True)
assert "app/0.1: Invalid: The package 'app/0.1' is a vendoring one, needs to be built " \
"from source, but it didn't enable 'tools.graph:vendor=build'" in c.out

c.run("install --requires=app/0.1 --build=missing -c tools.graph:vendor=build")
assert "pkga" in c.out # it works


def test_package_vendor_editable():
c = TestClient()
pkgb = textwrap.dedent("""
import os
from conan import ConanFile
from conan.tools.files import copy, save

class App(ConanFile):
name = "pkgb"
version = "0.1"
package_type = "shared-library"
vendor = True
requires = "pkga/0.1"
def layout(self):
self.folders.build = "build"
self.cpp.build.bindirs = ["build"]
def generate(self):
copy(self, "*", src=self.dependencies["pkga"].package_folder,
dst=self.build_folder)
def build(self):
save(self, os.path.join(self.build_folder, "pkgb.dll"), "dll")
""")

c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_package_type("shared-library")
.with_package_file("bin/pkga.dll", "d"),
"pkgb/conanfile.py": pkgb,
"app/conanfile.py": GenConanfile("app", "0.1").with_settings("os")
.with_requires("pkgb/0.1")
})
c.run("create pkga")
c.run("editable add pkgb")
c.run("install app -s os=Linux")
assert "pkga" in c.out
# The environment file of "app" doesn't have any visibility of the "pkga" paths
envfile_app = c.load("app/conanrunenv.sh")
assert "pkga" not in envfile_app
# But the environment file needed to build "pkgb" has visibility over the "pkga" paths
envfile_pkgb = c.load("pkgb/conanrunenv.sh")
assert "pkga" in envfile_pkgb


def test_vendor_dont_propagate_options():
c = TestClient()
app = GenConanfile("app", "0.1").with_requires("pkga/0.1").with_class_attribute("vendor=True")
c.save({"pkga/conanfile.py": GenConanfile("pkga", "0.1").with_shared_option(False),
"app/conanfile.py": app,
"consumer/conanfile.txt": "[requires]\napp/0.1",
"consumer_shared/conanfile.txt": "[requires]\napp/0.1\n[options]\n*:shared=True"
})
c.run("create pkga")
c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Build")})
c.run("create app")
c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
"app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})
c.run("install consumer --build=app/* -c tools.graph:vendor=build")
c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
"app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})
c.run("install consumer_shared --build=app/* -c tools.graph:vendor=build")
c.assert_listed_binary({"pkga/0.1": ("55c609fe8808aa5308134cb5989d23d3caffccf2", "Cache"),
"app/0.1": ("da39a3ee5e6b4b0d3255bfef95601890afd80709", "Build")})