New lockfiles iteration. Including version-only lockfiles.

conans/build_info/build_info.py

@@ -11,7 +11,6 @@
 from conans.errors import AuthenticationException, RequestErrorException, ConanException
 from conans.model.graph_lock import LOCKFILE_VERSION
 from conans.model.ref import ConanFileReference
-from conans.model.version import Version
 from conans.paths import ARTIFACTS_PROPERTIES_PUT_PREFIX
 from conans.paths import get_conan_user_home
 from conans.util.files import save
@@ -49,31 +48,27 @@ def __init__(self, output, build_info_file, lockfile, user=None, password=None,
         self._output = output
         self._conan_cache = ClientCache(os.path.join(get_conan_user_home(), ".conan"), output)
 
-    def parse_pref(self, pref):
-        ref = ConanFileReference.loads(pref, validate=False)
-        rrev = ref.revision.split("#")[0].split(":")[0]
-        pid = ref.revision.split("#")[0].split(":")[1]
-        prev = "" if len(ref.revision.split("#")) == 1 else ref.revision.split("#")[1]
+    def parse_ref(self, ref):
+        ref = ConanFileReference.loads(ref, validate=False)
+        rrev = ref.revision
         return {
             "name": ref.name,
             "version": ref.version,
             "user": ref.user,
             "channel": ref.channel,
             "rrev": rrev,
-            "pid": pid,
-            "prev": prev
         }
 
-    def _get_reference(self, pref):
-        r = self.parse_pref(pref)
+    def _get_reference(self, ref):
+        r = self.parse_ref(ref)
         if r.get("user") and r.get("channel"):
             return "{name}/{version}@{user}/{channel}".format(**r)
         else:
             return "{name}/{version}".format(**r)
 
-    def _get_package_reference(self, pref):
-        r = self.parse_pref(pref)
-        return "{reference}:{pid}".format(reference=self._get_reference(pref), **r)
+    def _get_package_reference(self, ref, pid):
+        r = self.parse_ref(ref)
+        return "{reference}:{pid}".format(reference=self._get_reference(ref), pid=pid)
 
     def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_format="{}",
                                 package_id=None):
@@ -112,86 +107,104 @@ def _get_metadata_artifacts(self, metadata, request_path, use_id=False, name_for
                                                   "id": "conan_sources.tgz" if use_id else None}
         return set([Artifact(k, **v) for k, v in ret.items()])
 
-    def _get_recipe_artifacts(self, pref, is_dependency):
-        r = self.parse_pref(pref)
+    def _get_recipe_artifacts(self, ref, is_dependency):
+        r = self.parse_ref(ref)
         if r.get("user") and r.get("channel"):
             ref = "{name}/{version}@{user}/{channel}#{rrev}".format(**r)
         else:
             ref = "{name}/{version}#{rrev}".format(**r)
         reference = ConanFileReference.loads(ref)
         package_layout = self._conan_cache.package_layout(reference)
         metadata = package_layout.load_metadata()
-        name_format = "{} :: {{}}".format(self._get_reference(pref)) if is_dependency else "{}"
+        name_format = "{} :: {{}}".format(self._get_reference(ref)) if is_dependency else "{}"
         if r.get("user") and r.get("channel"):
             url = "{user}/{name}/{version}/{channel}/{rrev}/export".format(**r)
         else:
             url = "_/{name}/{version}/_/{rrev}/export".format(**r)
 
-        return self._get_metadata_artifacts(metadata, url, name_format=name_format, use_id=is_dependency)
+        return self._get_metadata_artifacts(metadata, url, name_format=name_format,
+                                            use_id=is_dependency)
 
-    def _get_package_artifacts(self, pref, is_dependency):
-        r = self.parse_pref(pref)
+    def _get_package_artifacts(self, ref, pid, prev, is_dependency):
+        r = self.parse_ref(ref)
         if r.get("user") and r.get("channel"):
             ref = "{name}/{version}@{user}/{channel}#{rrev}".format(**r)
         else:
             ref = "{name}/{version}#{rrev}".format(**r)
         reference = ConanFileReference.loads(ref)
         package_layout = self._conan_cache.package_layout(reference)
         metadata = package_layout.load_metadata()
-        name_format = "{} :: {{}}".format(self._get_package_reference(pref)) if is_dependency else "{}"
+        if is_dependency:
+            name_format = "{} :: {{}}".format(self._get_package_reference(ref, pid))
+        else:
+            name_format = "{}"
         if r.get("user") and r.get("channel"):
-            url = "{user}/{name}/{version}/{channel}/{rrev}/package/{pid}/{prev}".format(**r)
+            url = "{user}/{name}/{version}/{channel}/{rrev}/package/{pid}/{prev}"
         else:
-            url = "_/{name}/{version}/_/{rrev}/package/{pid}/{prev}".format(**r)
-        arts = self._get_metadata_artifacts(metadata, url, name_format=name_format, use_id=is_dependency,
-                                            package_id=r["pid"])
+            url = "_/{name}/{version}/_/{rrev}/package/{pid}/{prev}"
+        url = url.format(pid=pid, prev=prev, **r)
+        arts = self._get_metadata_artifacts(metadata, url, name_format=name_format,
+                                            use_id=is_dependency, package_id=pid)
         return arts
 
     def process_lockfile(self):
         modules = defaultdict(lambda: {"id": None, "artifacts": set(), "dependencies": set()})
 
-        def _gather_deps(node_uid, contents, func):
-            node_content = contents["graph_lock"]["nodes"].get(node_uid)
-            artifacts = func(node_content["pref"], is_dependency=True)
-            for id_node in node_content.get("requires", []):
-                artifacts.update(_gather_deps(id_node, contents, func))
-            for id_node in node_content.get("build_requires", []):
-                artifacts.update(_gather_deps(id_node, contents, func))
+        def _gather_transitive_recipes(nid, contents):
+            n = contents["graph_lock"]["nodes"][nid]
+            artifacts = self._get_recipe_artifacts(n["ref"], is_dependency=True)
+            for id_node in n.get("requires", []):
+                artifacts.update(_gather_transitive_recipes(id_node, contents))
+            for id_node in n.get("build_requires", []):
+                artifacts.update(_gather_transitive_recipes(id_node, contents))
+            return artifacts
+
+        def _gather_transitive_packages(nid, contents):
+            n = contents["graph_lock"]["nodes"][nid]
+            artifacts = self._get_package_artifacts(n["ref"], n["package_id"], n["prev"],
+                                                    is_dependency=True)
+            for id_node in n.get("requires", []):
+                artifacts.update(_gather_transitive_packages(id_node, contents))
+            for id_node in n.get("build_requires", []):
+                artifacts.update(_gather_transitive_packages(id_node, contents))
             return artifacts
 
         with open(self._lockfile) as json_data:
             data = json.load(json_data)
 
-        version = Version(data["version"])
-        if version < LOCKFILE_VERSION:
-            raise ConanException("This lockfile was created with a previous incompatible version "
+        version = data["version"]
+        if version != LOCKFILE_VERSION:
+            raise ConanException("This lockfile was created with an incompatible version "
                                  "of Conan. Please update all your Conan clients")
 
         # Gather modules, their artifacts and recursively all required artifacts
         for _, node in data["graph_lock"]["nodes"].items():
-            pref = node["pref"]
+            ref = node["ref"]
+            pid = node.get("package_id")
+            prev = node.get("prev")
             if node.get("modified"):  # Work only on generated nodes
                 # Create module for the recipe reference
-                recipe_key = self._get_reference(pref)
+                recipe_key = self._get_reference(ref)
                 modules[recipe_key]["id"] = recipe_key
                 modules[recipe_key]["artifacts"].update(
-                    self._get_recipe_artifacts(pref, is_dependency=False))
+                    self._get_recipe_artifacts(ref, is_dependency=False))
                 # TODO: what about `python_requires`?
-                # TODO: can we associate any properties to the recipe? Profile/options may be different per lockfile
+                # TODO: can we associate any properties to the recipe? Profile/options may
+                # TODO: be different per lockfile
 
                 # Create module for the package_id
-                package_key = self._get_package_reference(pref)
+                package_key = self._get_package_reference(ref, pid)
                 modules[package_key]["id"] = package_key
                 modules[package_key]["artifacts"].update(
-                    self._get_package_artifacts(pref, is_dependency=False))
+                    self._get_package_artifacts(ref, pid, prev, is_dependency=False))
 
                 # Recurse requires
                 node_ids = node.get("requires", []) + node.get("build_requires", [])
                 for node_id in node_ids:
-                    modules[recipe_key]["dependencies"].update(
-                        _gather_deps(node_id, data, self._get_recipe_artifacts))
-                    modules[package_key]["dependencies"].update(
-                        _gather_deps(node_id, data, self._get_package_artifacts))
+                    modules[recipe_key]["dependencies"].update(_gather_transitive_recipes(node_id,
+                                                                                          data))
+                    modules[package_key]["dependencies"].update(_gather_transitive_packages(node_id,
+                                                                                            data))
 
                 # TODO: Is the recipe a 'dependency' of the package
 

conans/client/cmd/create.py

@@ -33,14 +33,48 @@ def create(app, ref, graph_info, remotes, update, build_modes,
     test_conanfile_path = _get_test_conanfile_path(test_folder, conanfile_path)
 
     if test_conanfile_path:
-        install_build_and_test(app, test_conanfile_path, ref, graph_info, remotes, update,
-                               build_modes=build_modes,
-                               manifest_folder=manifest_folder,
-                               manifest_verify=manifest_verify,
-                               manifest_interactive=manifest_interactive,
-                               keep_build=keep_build,
-                               test_build_folder=test_build_folder,
-                               recorder=recorder)
+        if graph_info.graph_lock:
+            # If we have a lockfile, then we are first going to make sure the lockfile is used
+            # correctly to build the package in the cache, and only later will try to run
+            # test_package
+            out = app.out
+            out.info("Installing and building %s" % repr(ref))
+            deps_install(app=app,
+                         ref_or_path=ref,
+                         create_reference=ref,
+                         install_folder=None,  # Not output anything
+                         manifest_folder=manifest_folder,
+                         manifest_verify=manifest_verify,
+                         manifest_interactive=manifest_interactive,
+                         remotes=remotes,
+                         graph_info=graph_info,
+                         build_modes=build_modes,
+                         update=update,
+                         keep_build=keep_build,
+                         recorder=recorder)
+            out.info("Executing test_package %s" % repr(ref))
+            try:
+                graph_info.graph_lock.relax()
+                # FIXME: It needs to clear the cache, otherwise it fails
+                app.binaries_analyzer._evaluated = {}
+                # FIXME: Forcing now not building test dependencies, binaries should be there
+                install_build_and_test(app, test_conanfile_path, ref, graph_info, remotes,
+                                       update, build_modes=None,
+                                       test_build_folder=test_build_folder, recorder=recorder)
+            except Exception as e:
+                raise ConanException("Something failed while testing '%s' test_package after "
+                                     "it was built using the lockfile. Please report this error: %s"
+                                     % (str(ref), str(e)))
+
+        else:
+            install_build_and_test(app, test_conanfile_path, ref, graph_info, remotes, update,
+                                   build_modes=build_modes,
+                                   manifest_folder=manifest_folder,
+                                   manifest_verify=manifest_verify,
+                                   manifest_interactive=manifest_interactive,
+                                   keep_build=keep_build,
+                                   test_build_folder=test_build_folder,
+                                   recorder=recorder)
     else:
         deps_install(app=app,
                      ref_or_path=ref,

conans/client/cmd/export.py

@@ -87,7 +87,7 @@ def cmd_export(app, conanfile_path, name, version, user, channel, keep_source,
 
     # If we receive lock information, python_requires could have been locked
     if graph_lock:
-        node_id = graph_lock.get_node(ref)
+        node_id = graph_lock.get_consumer(ref)
         python_requires = graph_lock.python_requires(node_id)
         # TODO: check that the locked python_requires are different from the loaded ones
         app.range_resolver.clear_output()  # invalidate previous version range output

conans/client/cmd/export_pkg.py

@@ -68,8 +68,8 @@ def _init_conanfile_infos():
 
     packager.update_package_metadata(prev, layout, package_id, full_ref.revision)
     pref = PackageReference(pref.ref, pref.id, prev)
-    if graph_info.graph_lock:
+    if pkg_node.graph_lock_node:
         # after the package has been created we need to update the node PREV
         pkg_node.prev = pref.revision
-        graph_info.graph_lock.update_check_graph(deps_graph, output)
+        pkg_node.graph_lock_node.prev = pref.revision
     recorder.package_exported(pref)