Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Binary registry #3726

Merged
merged 19 commits into from Oct 18, 2018
@@ -119,7 +119,7 @@ def registry(self):
else:
self._output.warn("Remotes registry file missing, "
"creating default one in %s" % reg_json_path)
save(reg_json_path, dump_registry(default_remotes, {}))
save(reg_json_path, dump_registry(default_remotes, {}, {}))
return reg_json_path

@property
@@ -1,5 +1,6 @@
import ast
import os

import shutil

from conans.client.cmd.export_linter import conan_linter
@@ -10,10 +11,9 @@
from conans.model.manifest import FileTreeManifest
from conans.model.scm import SCM
from conans.paths import CONAN_MANIFEST, CONANFILE
from conans.search.search import search_recipes
from conans.util.files import save, rmdir, is_dirty, set_dirty, mkdir, load
from conans.util.log import logger
from conans.search.search import search_recipes
from conans.client.plugin_manager import PluginManager


def export_alias(reference, target_reference, client_cache):
@@ -35,7 +35,7 @@ class AliasConanfile(ConanFile):


def cmd_export(conanfile_path, conanfile, reference, keep_source, output, client_cache,
plugin_manager):
plugin_manager, registry):
""" Export the recipe
param conanfile_path: the original source directory of the user containing a
conanfile.py
@@ -56,7 +56,7 @@ def cmd_export(conanfile_path, conanfile, reference, keep_source, output, client

with client_cache.conanfile_write_lock(reference):
_export_conanfile(conanfile_path, conanfile.output, client_cache, conanfile, reference,
keep_source)
keep_source, registry)
conanfile_cache_path = client_cache.conanfile(reference)
plugin_manager.execute("post_export", conanfile=conanfile, conanfile_path=conanfile_cache_path,
reference=reference)
@@ -130,7 +130,7 @@ def _replace_scm_data_in_conanfile(conanfile_path, scm_data):
save(conanfile_path, content)


def _export_conanfile(conanfile_path, output, paths, conanfile, conan_ref, keep_source):
def _export_conanfile(conanfile_path, output, paths, conanfile, conan_ref, keep_source, registry):

exports_folder = paths.export(conan_ref)
exports_source_folder = paths.export_sources(conan_ref, conanfile.short_paths)
@@ -153,6 +153,8 @@ def _export_conanfile(conanfile_path, output, paths, conanfile, conan_ref, keep_
modified_recipe = True
digest.save(exports_folder)

# FIXME: Conan 2.0 Clear the registry entry if the recipe has changed

source = paths.source(conan_ref, conanfile.short_paths)
remove = False
if is_dirty(source):
@@ -1,11 +1,10 @@
import time

from conans import load
from conans.client.source import complete_recipe_sources
from conans.errors import ConanException, NotFoundException
from conans.model.ref import PackageReference, ConanFileReference
from conans.util.log import logger
from conans.client.source import complete_recipe_sources
from conans.search.search import search_recipes, search_packages
from conans.util.log import logger


def _is_a_reference(ref):
@@ -82,27 +81,28 @@ def _upload(self, conan_file, conan_ref, packages_ids, retry, retry_wait,
integrity_check, policy, remote_name, recorder):
"""Uploads the recipes and binaries identified by conan_ref"""

defined_remote = self._registry.refs.get(conan_ref)
default_remote = self._registry.remotes.default
cur_recipe_remote = self._registry.refs.get(conan_ref)
if remote_name: # If remote_name is given, use it
upload_remote = self._registry.remotes.get(remote_name)
elif defined_remote: # Else, if the package had defined a remote, use it
upload_remote = defined_remote
else: # Or use the default otherwise
upload_remote = self._registry.remotes.default
recipe_remote = self._registry.remotes.get(remote_name)
else:
recipe_remote = cur_recipe_remote or default_remote

conanfile_path = self._client_cache.conanfile(conan_ref)
# FIXME: I think it makes no sense to specify a remote to "pre_upload"
# FIXME: because the recipe can have one and the package a different one

This comment has been minimized.

Copy link
@memsharded

memsharded Oct 15, 2018

Contributor

It depends on what will be the final model. If one upload of a package can only upload to a given remote, or is able to upload different binaries to different remotes, in the same command. To discuss.

self._plugin_manager.execute("pre_upload", conanfile_path=conanfile_path,
reference=conan_ref, remote=upload_remote)
reference=conan_ref, remote=recipe_remote)

if policy != UPLOAD_POLICY_FORCE:
remote_manifest = self._check_recipe_date(conan_ref, upload_remote)
remote_manifest = self._check_recipe_date(conan_ref, recipe_remote)
else:
remote_manifest = None

self._user_io.out.info("Uploading %s to remote '%s'" % (str(conan_ref), upload_remote.name))
self._upload_recipe(conan_ref, retry, retry_wait, policy, upload_remote, remote_manifest)
self._user_io.out.info("Uploading %s to remote '%s'" % (str(conan_ref), recipe_remote.name))
self._upload_recipe(conan_ref, retry, retry_wait, policy, recipe_remote, remote_manifest)

recorder.add_recipe(str(conan_ref), upload_remote.name, upload_remote.url)
recorder.add_recipe(str(conan_ref), recipe_remote.name, recipe_remote.url)

if packages_ids:
# Can't use build_policy_always here because it's not loaded (only load_class)
@@ -111,18 +111,32 @@ def _upload(self, conan_file, conan_ref, packages_ids, retry, retry_wait,
"no packages can be uploaded")
total = len(packages_ids)
for index, package_id in enumerate(packages_ids):
ret_upload_package = self._upload_package(PackageReference(conan_ref, package_id),
pref = PackageReference(conan_ref, package_id)
cur_package_remote = self._registry.prefs.get(pref)

This comment has been minimized.

Copy link
@memsharded

memsharded Oct 15, 2018

Contributor

Not sure about this, this is changing current behavior, and it will be again changed in conan 2.0. Probably remove this.

This comment has been minimized.

Copy link
@lasote

lasote Oct 16, 2018

Author Contributor

Why is changing in Conan 2.0?

This comment has been minimized.

Copy link
@lasote

lasote Oct 16, 2018

Author Contributor

Anyway, I will change it to choose the specified remote or the default one. Ok?


if remote_name: # If remote_name is given, use it
p_remote = self._registry.remotes.get(remote_name)
else:
p_remote = cur_package_remote or default_remote

ret_upload_package = self._upload_package(pref,
index + 1, total, retry, retry_wait,
integrity_check,
policy, upload_remote)
policy, p_remote)

if (not cur_package_remote or not remote_name) and policy != UPLOAD_POLICY_SKIP:
self._registry.prefs.set(pref, p_remote.name)

This comment has been minimized.

Copy link
@jgsogo

jgsogo Oct 11, 2018

Member

I would want to know why it is conditioned to not cur_package_remote or not remote_name... At first, I expected the registry to be updated unconditionally if the package was uploaded.

This comment has been minimized.

Copy link
@lasote

lasote Oct 16, 2018

Author Contributor

This is an internal decision (probably arbitrary), that if you are installing from R1 but uploading to R2, you want the updates still from R1, otherwise you need to change the reference manually.


if ret_upload_package:
recorder.add_package(str(conan_ref), package_id)

if not defined_remote and policy != UPLOAD_POLICY_SKIP:
self._registry.refs.set(conan_ref, upload_remote.name)

if (not cur_recipe_remote or not remote_name) and policy != UPLOAD_POLICY_SKIP:
self._registry.refs.set(conan_ref, recipe_remote.name)

This comment has been minimized.

Copy link
@jgsogo

jgsogo Oct 11, 2018

Member

I would think that the registry should be updated just after uploading the recipe (line 103)

And also, I don't know why it is conditioned on not cur_recipe_remote or not remote_name (same comment as the one regarding packages).

This comment has been minimized.

Copy link
@lasote

lasote Oct 16, 2018

Author Contributor

Agree about moving the registry update.
I think the condition can be simplified to not cur_recipe_remote, the same for packages. Let me check. The idea is: If something has a registry entry, then an upload won't change it.


# FIXME: I think it makes no sense to specify a remote to "post_upload"
# FIXME: because the recipe can have one and the package a different one
self._plugin_manager.execute("post_upload", conanfile_path=conanfile_path,
reference=conan_ref, remote=upload_remote)
reference=conan_ref, remote=recipe_remote)

def _upload_recipe(self, conan_reference, retry, retry_wait, policy, remote, remote_manifest):
conan_file_path = self._client_cache.conanfile(conan_reference)
@@ -139,7 +153,8 @@ def _upload_package(self, package_ref, index=1, total=1, retry=None, retry_wait=
integrity_check=False, policy=None, remote=None):
"""Uploads the package identified by package_id"""

msg = ("Uploading package %d/%d: %s" % (index, total, str(package_ref.package_id)))
msg = ("Uploading package %d/%d: %s to '%s'" % (index, total, str(package_ref.package_id),
remote.name))
t1 = time.time()
self._user_io.out.info(msg)

@@ -1148,9 +1148,29 @@ def remote(self, *args):
"a package recipe")
parser_pupd.add_argument('reference', help='Package recipe reference')
parser_pupd.add_argument('remote', help='Name of the remote')

list_pref = subparsers.add_parser('list_pref', help='List the package binaries and '
'its associated remotes')
list_pref.add_argument('reference', help='Package recipe reference')

add_pref = subparsers.add_parser('add_pref',
help="Associate a package reference to a remote")
add_pref.add_argument('package_reference', help='Binary package reference')
add_pref.add_argument('remote', help='Name of the remote')

remove_pref = subparsers.add_parser('remove_pref', help="Dissociate a package's reference "
"and its remote")
remove_pref.add_argument('package_reference', help='Binary package reference')

update_pref = subparsers.add_parser('update_pref', help="Update the remote associated with "
"a binary package")
update_pref.add_argument('package_reference', help='Bianary package reference')
update_pref.add_argument('remote', help='Name of the remote')
This conversation was marked as resolved by jgsogo

This comment has been minimized.

Copy link
@jgsogo

jgsogo Oct 11, 2018

Member

I wonder why the user would need to edit the registry file 😕

This comment has been minimized.

Copy link
@memsharded

memsharded Oct 15, 2018

Contributor

Letting users do whatever they want. However, with the new usage of remote info planned for conan 2.0, it could be that the commands to edit the registry are not necessary anymore.

This comment has been minimized.

Copy link
@lasote

lasote Oct 16, 2018

Author Contributor

I think they still will need to change some references, see my previous comment about the upload not updating the references:

This is an internal decision (probably arbitrary), that if you are installing from R1 but
uploading to R2, you want the updates still from R1, otherwise you need to change the
reference manually.

You always will need to be able to change the references no mater where they are, right?


args = parser.parse_args(*args)

reference = args.reference if hasattr(args, 'reference') else None
package_reference = args.package_reference if hasattr(args, 'package_reference') else None

verify_ssl = get_bool_from_text(args.verify_ssl) if hasattr(args, 'verify_ssl') else False

@@ -1178,6 +1198,15 @@ def remote(self, *args):
return self._conan.remote_remove_ref(reference)
elif args.subcommand == "update_ref":
return self._conan.remote_update_ref(reference, remote_name)
elif args.subcommand == "list_pref":
refs = self._conan.remote_list_pref(reference)
self._outputer.remote_ref_list(refs)
elif args.subcommand == "add_pref":
return self._conan.remote_add_pref(package_reference, remote_name)
elif args.subcommand == "remove_pref":
return self._conan.remote_remove_pref(package_reference)
elif args.subcommand == "update_pref":
return self._conan.remote_update_pref(package_reference, remote_name)

def profile(self, *args):
""" Lists profiles in the '.conan/profiles' folder, or shows profile details.
@@ -29,7 +29,7 @@
from conans.client.cmd.test import PackageTester
from conans.client.userio import UserIO
from conans.errors import ConanException
from conans.model.ref import ConanFileReference
from conans.model.ref import ConanFileReference, PackageReference
from conans.model.version import Version
from conans.paths import get_conan_user_home, CONANINFO, BUILD_INFO
from conans.util.env_reader import get_env
@@ -346,7 +346,7 @@ def create(self, conanfile_path, name=None, version=None, user=None, channel=Non
# Forcing an export!
if not not_export:
cmd_export(conanfile_path, conanfile, reference, keep_source, self._user_io.out,
self._client_cache, self._plugin_manager)
self._client_cache, self._plugin_manager, self._registry)

if build_modes is None: # Not specified, force build the tested library
build_modes = [conanfile.name]
@@ -413,7 +413,7 @@ def export_pkg(self, conanfile_path, name, channel, source_folder=None, build_fo

reference, conanfile = self._loader.load_export(conanfile_path, name, version, user, channel)
cmd_export(conanfile_path, conanfile, reference, False, self._user_io.out,
self._client_cache, self._plugin_manager)
self._client_cache, self._plugin_manager, self._registry)

recorder = ActionRecorder()
manager = self._init_manager(recorder)
@@ -677,7 +677,7 @@ def export(self, path, name, version, user, channel, keep_source=False, cwd=None
conanfile_path = _get_conanfile_path(path, cwd, py=True)
reference, conanfile = self._loader.load_export(conanfile_path, name, version, user, channel)
cmd_export(conanfile_path, conanfile, reference, keep_source, self._user_io.out,
self._client_cache, self._plugin_manager)
self._client_cache, self._plugin_manager, self._registry)

@api_method
def remove(self, pattern, query=None, packages=None, builds=None, src=False, force=False,
@@ -827,6 +827,32 @@ def remote_update_ref(self, reference, remote_name):
reference = ConanFileReference.loads(str(reference))
return self._registry.refs.update(reference, remote_name)

@api_method
def remote_list_pref(self, reference):
reference = ConanFileReference.loads(str(reference))

This comment has been minimized.

Copy link
@jgsogo

jgsogo Oct 11, 2018

Member

Add validation, it is an user input: ConanFileReference.loads(str(reference), validate=True).... in the future that argument should be False by default.

ret = {}
tmp = self._registry.prefs.list
for r, remote in tmp.items():
pref = PackageReference.loads(r)
if pref.conan == reference:
ret[r] = remote
return ret

@api_method
def remote_add_pref(self, package_reference, remote_name):
p_reference = PackageReference.loads(str(package_reference))
return self._registry.prefs.set(p_reference, remote_name, check_exists=True)

@api_method
def remote_remove_pref(self, package_reference):
p_reference = PackageReference.loads(str(package_reference))
return self._registry.prefs.remove(p_reference)

@api_method
def remote_update_pref(self, package_reference, remote_name):
p_reference = PackageReference.loads(str(package_reference))
return self._registry.prefs.update(p_reference, remote_name)

@api_method
def profile_list(self):
return cmd_profile_list(self._client_cache.profiles_path, self._user_io.out)
@@ -1,14 +1,14 @@
import os

from conans.util.files import rmdir, is_dirty
from conans.model.ref import PackageReference
from conans.client.output import ScopedOutput
from conans.errors import NotFoundException, NoRemoteAvailable
from conans.model.manifest import FileTreeManifest
from conans.model.info import ConanInfo
from conans.client.graph.graph import (BINARY_BUILD, BINARY_UPDATE, BINARY_CACHE,
BINARY_DOWNLOAD, BINARY_MISSING, BINARY_SKIP,
BINARY_WORKSPACE)
from conans.client.output import ScopedOutput
from conans.errors import NotFoundException, NoRemoteAvailable
from conans.model.info import ConanInfo
from conans.model.manifest import FileTreeManifest
from conans.model.ref import PackageReference
from conans.util.files import rmdir, is_dirty


class GraphBinariesAnalyzer(object):
@@ -80,7 +80,7 @@ def _evaluate_node(self, node, build_mode, update, evaluated_references, remote_
if remote_name:
remote = self._registry.remotes.get(remote_name)
else:
remote = self._registry.refs.get(conan_ref)
remote = self._registry.prefs.get(package_ref)
remotes = self._registry.remotes.list

if os.path.exists(package_folder):
@@ -36,6 +36,7 @@ def _get_recipe(self, reference, check_updates, update, remote_name, recorder):
if not os.path.exists(conanfile_path):
remote, new_ref = self._download_recipe(reference, output, remote_name, recorder)
status = RECIPE_DOWNLOADED
self._registry.refs.set(reference, remote.name)

This comment has been minimized.

Copy link
@memsharded

memsharded Oct 15, 2018

Contributor

This is done already in self._download_recipe. Remove from that function?

return conanfile_path, status, remote, new_ref

remote = self._registry.refs.get(reference)
@@ -1,35 +1,36 @@
import os
import time
import shutil
import platform

import shutil
import time

from conans.client import tools
from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING_BUILD_FOLDER, INSTALL_ERROR_BUILDING,\
from conans.client.file_copier import report_copied_files
from conans.client.generators import write_generators, TXTGenerator
from conans.client.graph.graph import BINARY_SKIP, BINARY_MISSING, \
BINARY_DOWNLOAD, BINARY_UPDATE, BINARY_BUILD, BINARY_CACHE
from conans.client.importer import remove_imports
from conans.client.output import ScopedOutput
from conans.client.packager import create_package
from conans.client.recorder.action_recorder import INSTALL_ERROR_MISSING_BUILD_FOLDER, \
INSTALL_ERROR_BUILDING, \
INSTALL_ERROR_MISSING
from conans.client.source import config_source, complete_recipe_sources
from conans.client.tools.env import pythonpath
from conans.errors import (ConanException, conanfile_exception_formatter,
ConanExceptionInUserConanfileMethod)
from conans.model.build_info import CppInfo
from conans.model.conan_file import get_env_context_manager
from conans.model.env_info import EnvInfo
from conans.model.ref import PackageReference
from conans.model.user_info import UserInfo
from conans.paths import CONANINFO, BUILD_INFO, RUN_LOG_NAME
from conans.util.env_reader import get_env
from conans.util.files import (save, rmdir, mkdir, make_read_only,
set_dirty, clean_dirty, load)
from conans.model.ref import PackageReference
from conans.util.log import logger
from conans.errors import (ConanException, conanfile_exception_formatter,
ConanExceptionInUserConanfileMethod)
from conans.client.packager import create_package
from conans.client.generators import write_generators, TXTGenerator
from conans.model.build_info import CppInfo
from conans.client.output import ScopedOutput
from conans.client.source import config_source, complete_recipe_sources
from conans.util.env_reader import get_env
from conans.client.importer import remove_imports

from conans.util.tracer import log_package_built,\
from conans.util.tracer import log_package_built, \
log_package_got_from_local_cache
from conans.client.tools.env import pythonpath
from conans.client.graph.graph import BINARY_SKIP, BINARY_MISSING,\
BINARY_DOWNLOAD, BINARY_UPDATE, BINARY_BUILD, BINARY_CACHE
from conans.client.file_copier import report_copied_files


def build_id(conan_file):
@@ -315,8 +316,7 @@ def _handle_node_cache(self, node, package_ref, keep_build, processed_package_re
elif node.binary in (BINARY_UPDATE, BINARY_DOWNLOAD):
self._remote_manager.get_package(package_ref, package_folder,
node.binary_remote, output, self._recorder)
if node.binary_remote != node.remote:
self._registry.refs.set(conan_ref, node.binary_remote.name)
self._registry.prefs.set(package_ref, node.binary_remote.name)
elif node.binary == BINARY_CACHE:
output.success('Already installed!')
log_package_got_from_local_cache(package_ref)
@@ -391,6 +391,7 @@ def _build_package(self, node, package_ref, output, keep_build):
else:
# Log build
self._log_built_package(builder.build_folder, package_ref, time.time() - t1)
# FIXME: Conan 2.0 Clear the registry entry (package ref)

def _log_built_package(self, build_folder, package_ref, duration):
log_file = os.path.join(build_folder, RUN_LOG_NAME)
ProTip! Use n and p to navigate between commits in a pull request.
You can’t perform that action at this time.