Feature/remove compatibility mode (#4462)

* WIP

* layout of dev

* wip

* Removed versions

* working but not tested

* local

* Local, package failing because of none timestamp in PREV

* WIP

* search tests ok

* Fixing tests and small refactor

* renamed module

* review

* ISO in client, reviewed

* Fixing tests

* Fixing tests

* New tests about metadata being updated

* Review

* Time cleared on export/package

* Fix revisions list

* fixed test

* Unused import

* WIP

* Review

* Fixed test message assert

* Use env var instead of test client

* Cool testing framework and a lot of remove with revisions tests

* Remove tests revisions

* Drafting tests

* Test passing, a lot missing yet

* PENDING MANY TESTS

* some more tests

* Pending many tests

* Added more tests

* No revisions tests passing

* Tests passing, only about 10 to complete

* Finished tests

* Upload package id not reference

* One more test

* SVN test

* IOError

* Fixing tests

* Fixing windows test with export package and package revisions

* Fixed test

* Pending migration

* Migration OK

* Revisioning the revisions pull request

* CONAN_REVISIONS_ENABLED

* Safer migration

* Review

.ci/jenkins/Jenkinsfile

@@ -69,10 +69,10 @@ if(is_pr){ // Read the PR body to
 
 def flavors
 if(test_revisions){
-    flavors = ["enabled_revisions", "disabled_revisions",  "blocked_v2"]
+    flavors = ["disabled_revisions", "enabled_revisions"]
 }
 else{
-    flavors = ["blocked_v2"]
+    flavors = ["disabled_revisions"]
 }
 
 def module = "\"conans.test\""

.ci/jenkins/runner.py

@@ -67,8 +67,7 @@ def run_tests(module_path, pyver, source_folder, tmp_folder, flavor, excluded_ta
     env["PYTHONPATH"] = source_folder
     env["CONAN_LOGGING_LEVEL"] = "50" if platform.system() == "Darwin" else "50"
     env["CHANGE_AUTHOR_DISPLAY_NAME"] = ""
-    env["CONAN_API_V2_BLOCKED"] = "True" if flavor == "blocked_v2" else "False"
-    env["CONAN_CLIENT_REVISIONS_ENABLED"] = "True" if flavor == "enabled_revisions" else "False"
+    env["TESTING_REVISIONS_ENABLED"] = "True" if flavor == "enabled_revisions" else "False"
     # Related with the error: LINK : fatal error LNK1318: Unexpected PDB error; RPC (23) '(0x000006BA)'
     # More info: http://blog.peter-b.co.uk/2017/02/stop-mspdbsrv-from-breaking-ci-build.html
     # Update, this doesn't solve the issue, other issues arise:
@@ -106,7 +105,7 @@ def run(command):
     parser.add_argument('--num_cores', type=int, help='Number of cores to use', default=3)
     parser.add_argument('--exclude_tags', '-e', nargs=1, action=Extender,
                         help='Tags to exclude from testing, e.g.: rest_api')
-    parser.add_argument('--flavor', '-f', help='enabled_revisions, disabled_revisions, blocked_v2')
+    parser.add_argument('--flavor', '-f', help='enabled_revisions, disabled_revisions')
     args = parser.parse_args()
 
     run_tests(args.module, args.pyver, args.source_folder, args.tmp_folder, args.flavor,

conans/client/cache/cache.py

@@ -47,9 +47,9 @@ class ClientCache(SimplePaths):
 
     def __init__(self, base_folder, store_folder, output):
         self.conan_folder = join(base_folder, ".conan")
-        self._conan_config = None
+        self._config = None
         self._output = output
-        self._store_folder = store_folder or self.conan_config.storage_path or self.conan_folder
+        self._store_folder = store_folder or self.config.storage_path or self.conan_folder
         self._no_lock = None
         self.client_cert_path = normpath(join(self.conan_folder, CLIENT_CERT))
         self.client_cert_key_path = normpath(join(self.conan_folder, CLIENT_KEY))
@@ -83,7 +83,7 @@ def cacert_path(self):
 
     def _no_locks(self):
         if self._no_lock is None:
-            self._no_lock = self.conan_config.cache_no_locks
+            self._no_lock = self.config.cache_no_locks
         return self._no_lock
 
     def conanfile_read_lock(self, ref):
@@ -145,13 +145,13 @@ def registry_path(self):
         return reg_json_path
 
     @property
-    def conan_config(self):
-        if not self._conan_config:
+    def config(self):
+        if not self._config:
             if not os.path.exists(self.conan_conf_path):
                 save(self.conan_conf_path, normalize(default_client_conf))
 
-            self._conan_config = ConanClientConfigParser(self.conan_conf_path)
-        return self._conan_config
+            self._config = ConanClientConfigParser(self.conan_conf_path)
+        return self._config
 
     @property
     def localdb(self):
@@ -171,11 +171,11 @@ def settings_path(self):
 
     @property
     def default_profile_path(self):
-        if os.path.isabs(self.conan_config.default_profile):
-            return self.conan_config.default_profile
+        if os.path.isabs(self.config.default_profile):
+            return self.config.default_profile
         else:
             return join(self.conan_folder, PROFILES_FOLDER,
-                        self.conan_config.default_profile)
+                        self.config.default_profile)
 
     @property
     def hooks_path(self):
@@ -295,7 +295,7 @@ def remove_package_locks(self, reference):
         shutil.rmtree(os.path.join(conan_folder, "locks"), ignore_errors=True)
 
     def invalidate(self):
-        self._conan_config = None
+        self._config = None
         self._no_lock = None
 
 

conans/client/cmd/copy.py

@@ -3,8 +3,9 @@
 
 from conans.client.source import complete_recipe_sources
 from conans.errors import ConanException
+from conans.model.package_metadata import PackageMetadata
 from conans.model.ref import ConanFileReference, PackageReference
-from conans.util.files import rmdir
+from conans.util.files import rmdir, save
 
 
 def _prepare_sources(cache, reference, remote_manager, loader):
@@ -39,6 +40,9 @@ def package_copy(src_ref, user_channel, package_ids, paths, user_io, short_paths
     dest_ref = ConanFileReference.loads("%s/%s@%s" % (src_ref.name,
                                                       src_ref.version,
                                                       user_channel))
+    # Generate metadata
+    src_metadata = paths.package_layout(src_ref).load_metadata()
+
     # Copy export
     export_origin = paths.export(src_ref)
     if not os.path.exists(export_origin):
@@ -60,6 +64,7 @@ def package_copy(src_ref, user_channel, package_ids, paths, user_io, short_paths
     user_io.out.info("Copied sources %s to %s" % (str(src_ref), str(dest_ref)))
 
     # Copy packages
+    package_revisions = {}  # To be stored in the metadata
     for package_id in package_ids:
         pref_origin = PackageReference(src_ref, package_id)
         pref_dest = PackageReference(dest_ref, package_id)
@@ -70,5 +75,14 @@ def package_copy(src_ref, user_channel, package_ids, paths, user_io, short_paths
                                                          " Override?" % str(package_id)):
                 continue
             rmdir(package_path_dest)
+        package_revisions[package_id] = (src_metadata.packages[package_id].revision,
+                                         src_metadata.recipe.revision)
         shutil.copytree(package_path_origin, package_path_dest, symlinks=True)
         user_io.out.info("Copied %s to %s" % (str(package_id), str(dest_ref)))
+
+    # Generate the metadata
+    with paths.package_layout(dest_ref).update_metadata() as metadata:
+        metadata.recipe.revision = src_metadata.recipe.revision
+        for package_id, (revision, recipe_revision) in package_revisions.items():
+            metadata.packages[package_id].revision = revision
+            metadata.packages[package_id].recipe_revision = recipe_revision

conans/client/cmd/download.py

@@ -15,9 +15,9 @@ def download(ref, package_ids, remote_name, recipe, remote_manager,
     hook_manager.execute("pre_download", reference=ref, remote=remote)
     # First of all download package recipe
     try:
-        remote_manager.get_recipe(ref, remote)
+        ref = remote_manager.get_recipe(ref, remote)
     except NotFoundException:
-        raise NotFoundException("'%s' not found in remote '%s'" % (str(ref), remote.name))
+        raise NotFoundException("'%s' not found in remote '%s'" % (ref.full_repr(), remote.name))
     registry.refs.set(ref, remote.name)
     conan_file_path = cache.conanfile(ref)
     conanfile = loader.load_class(conan_file_path)
@@ -27,7 +27,7 @@ def download(ref, package_ids, remote_name, recipe, remote_manager,
         complete_recipe_sources(remote_manager, cache, conanfile, ref)
 
         if not package_ids:  # User didnt specify a specific package binary
-            output.info("Getting the complete package list from '%s'..." % str(ref))
+            output.info("Getting the complete package list from '%s'..." % ref.full_repr())
             packages_props = remote_manager.search_packages(remote, ref, None)
             package_ids = list(packages_props.keys())
             if not package_ids:

conans/client/cmd/export.py

@@ -7,24 +7,27 @@
 from conans.client.cmd.export_linter import conan_linter
 from conans.client.file_copier import FileCopier
 from conans.client.output import ScopedOutput
+from conans.client.remover import DiskRemover
+from conans.client.tools import Git, SVN
 from conans.errors import ConanException
 from conans.model.manifest import FileTreeManifest
 from conans.model.scm import SCM, get_scm_data
+from conans.model.scm import detect_repo_type
 from conans.paths import CONANFILE
-from conans.search.search import search_recipes
+from conans.search.search import search_recipes, search_packages
 from conans.util.files import is_dirty, load, mkdir, rmdir, save, set_dirty, remove
 from conans.util.log import logger
 
 
-def export_alias(reference, target_reference, cache):
+def export_alias(reference, target_reference, cache, output):
     if reference.name != target_reference.name:
         raise ConanException("An alias can only be defined to a package with the same name")
     conanfile = """
 from conans import ConanFile
 
 class AliasConanfile(ConanFile):
     alias = "%s"
-""" % str(target_reference)
+""" % target_reference.full_repr()
 
     export_path = cache.export(reference)
     mkdir(export_path)
@@ -33,6 +36,9 @@ class AliasConanfile(ConanFile):
     digest = FileTreeManifest.create(export_path)
     digest.save(export_path)
 
+    # Create the metadata for the alias
+    _update_revision_in_metadata(cache, output, None, reference, digest)
+
 
 def cmd_export(conanfile_path, conanfile, ref, keep_source, output, cache, hook_manager):
     """ Export the recipe
@@ -50,7 +56,7 @@ def cmd_export(conanfile_path, conanfile, ref, keep_source, output, cache, hook_
     # Maybe a platform check could be added, but depends on disk partition
     reference = str(ref)
     refs = search_recipes(cache, reference, ignorecase=True)
-    if refs and ref not in refs:
+    if refs and ref not in [r.copy_clear_rev() for r in refs]:
         raise ConanException("Cannot export package with same name but different case\n"
                              "You exported '%s' but already existing '%s'"
                              % (reference, " ".join(str(s) for s in refs)))
@@ -73,10 +79,8 @@ def cmd_export(conanfile_path, conanfile, ref, keep_source, output, cache, hook_
         export_source(conanfile, origin_folder, package_layout.export_sources())
         shutil.copy2(conanfile_path, package_layout.conanfile())
 
-        scm_data, captured_revision = _capture_export_scm_data(conanfile,
-                                                               os.path.dirname(conanfile_path),
-                                                               package_layout.export(),
-                                                               output, cache, ref)
+        _capture_export_scm_data(conanfile, os.path.dirname(conanfile_path),
+                                 package_layout.export(), output, cache, ref)
 
         # Execute post-export hook before computing the digest
         hook_manager.execute("post_export", conanfile=conanfile, reference=ref,
@@ -94,10 +98,7 @@ def cmd_export(conanfile_path, conanfile, ref, keep_source, output, cache, hook_
         digest.save(package_layout.export())
 
     # Compute the revision for the recipe
-    revision = scm_data.revision if scm_data and captured_revision else digest.summary_hash
-    with package_layout.update_metadata() as metadata:
-        # Note that there is no time set, the time will come from the remote
-        metadata.recipe.revision = revision
+    _update_revision_in_metadata(cache, output, os.path.dirname(conanfile_path), ref, digest)
 
     # FIXME: Conan 2.0 Clear the registry entry if the recipe has changed
     source_folder = package_layout.source()
@@ -115,6 +116,19 @@ def cmd_export(conanfile_path, conanfile, ref, keep_source, output, cache, hook_
             output.warn(str(e))
             set_dirty(source_folder)
 
+    # When revisions enabled, remove the packages not matching the revision
+    if cache.config.revisions_enabled:
+        packages = search_packages(cache, ref, query=None)
+        metadata = cache.package_layout(ref).load_metadata()
+        recipe_revision = metadata.recipe.revision
+        to_remove = [pid for pid in packages if
+                     metadata.packages.get(pid) and
+                     metadata.packages.get(pid).recipe_revision != recipe_revision]
+        if to_remove:
+            output.info("Removing the local binary packages from different recipe revisions")
+            remover = DiskRemover(cache)
+            remover.remove_packages(ref, ids_filter=to_remove)
+
 
 def _capture_export_scm_data(conanfile, conanfile_dir, destination_folder, output, paths, ref):
 
@@ -124,11 +138,9 @@ def _capture_export_scm_data(conanfile, conanfile_dir, destination_folder, outpu
 
     scm_data = get_scm_data(conanfile)
     if not scm_data:
-        return None, False
+        return
 
     # Resolve SCMData in the user workspace (someone may want to access CVS or import some py)
-    captured_revision = scm_data.capture_revision
-
     scm = SCM(scm_data, conanfile_dir, output)
     captured = scm_data.capture_origin or scm_data.capture_revision
 
@@ -155,7 +167,7 @@ def _capture_export_scm_data(conanfile, conanfile_dir, destination_folder, outpu
         if src_path:
             save(scm_src_file, os.path.normpath(src_path).replace("\\", "/"))
 
-    return scm_data, captured_revision
+    return scm_data
 
 
 def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
@@ -208,6 +220,31 @@ def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
     save(conanfile_path, content)
 
 
+def _detect_scm_revision(path):
+    repo_type = detect_repo_type(path)
+    if not repo_type:
+        return None, None
+
+    repo_obj = {"git": Git, "svn": SVN}.get(repo_type)(path)
+    return repo_obj.get_revision(), repo_type
+
+
+def _update_revision_in_metadata(cache, output, path, ref, digest):
+
+    scm_revision_detected, repo_type = _detect_scm_revision(path)
+    revision = scm_revision_detected or digest.summary_hash
+    if cache.config.revisions_enabled:
+        if scm_revision_detected:
+            output.info("Using {} commit as the recipe"
+                        " revision: {} ".format(repo_type, revision))
+        else:
+            output.info("Using the exported files summary hash as the recipe"
+                        " revision: {} ".format(revision))
+    with cache.package_layout(ref).update_metadata() as metadata:
+        metadata.recipe.revision = revision
+        metadata.recipe.time = None
+
+
 def _recreate_folders(destination_folder, destination_src_folder):
     try:
         if os.path.exists(destination_folder):

conans/client/cmd/export_pkg.py

@@ -3,6 +3,7 @@
 from conans.client import packager
 from conans.client.graph.graph_manager import load_deps_info
 from conans.errors import ConanException
+from conans.model.manifest import FileTreeManifest
 from conans.model.ref import PackageReference
 from conans.util.files import rmdir
 
@@ -46,4 +47,9 @@ def export_pkg(cache, graph_manager, hook_manager, recorder, output,
         packager.create_package(conanfile, package_id, source_folder, build_folder,
                                 dest_package_folder, install_folder, hook_manager, conan_file_path,
                                 ref, local=True)
+        with cache.package_layout(ref).update_metadata() as metadata:
+            readed_manifest = FileTreeManifest.load(dest_package_folder)
+            metadata.packages[package_id].revision = readed_manifest.summary_hash
+            metadata.packages[package_id].recipe_revision = metadata.recipe.revision
+
     recorder.package_exported(pref)

conans/client/cmd/search.py

@@ -1,7 +1,8 @@
 from collections import OrderedDict, namedtuple
 
-from conans.errors import NotFoundException
-from conans.search.search import filter_outdated, search_packages, search_recipes
+from conans.errors import NotFoundException, ConanException
+from conans.search.search import (filter_outdated, search_packages, search_recipes,
+                                  filter_by_revision)
 
 
 class Search(object):
@@ -49,6 +50,10 @@ def search_packages(self, ref=None, remote_name=None, query=None, outdated=False
         if not remote_name:
             return self._search_packages_in_local(ref, query, outdated)
 
+        if ref.revision and not self._cache.config.revisions_enabled:
+            raise ConanException("Revisions not enabled in the client, specify a "
+                                 "reference without revision")
+
         if remote_name == 'all':
             return self._search_packages_in_all(ref, query, outdated)
 
@@ -57,13 +62,19 @@ def search_packages(self, ref=None, remote_name=None, query=None, outdated=False
     def _search_packages_in_local(self, ref=None, query=None, outdated=False):
         packages_props = search_packages(self._cache, ref, query)
         ordered_packages = OrderedDict(sorted(packages_props.items()))
+
         try:
             recipe_hash = self._cache.package_layout(ref).load_manifest().summary_hash
         except IOError:  # It could not exist in local
             recipe_hash = None
 
-        if outdated and recipe_hash:
+        if outdated:
             ordered_packages = filter_outdated(ordered_packages, recipe_hash)
+        elif self._cache.config.revisions_enabled:
+            # With revisions, by default filter the packages not belonging to the recipe
+            # unless outdated is specified.
+            metadata = self._cache.package_layout(ref).load_metadata()
+            ordered_packages = filter_by_revision(metadata, ordered_packages)
 
         references = OrderedDict()
         references[None] = self.remote_ref(ordered_packages, recipe_hash)