From ef8328c1ff525ff3f3d49253c1aca8d57bf56395 Mon Sep 17 00:00:00 2001 From: memsharded Date: Thu, 28 Nov 2019 12:39:57 +0100 Subject: [PATCH 01/12] removing pylint from conan codebase. TODO: Move to hooks --- conans/client/cmd/export.py | 2 - conans/client/cmd/export_linter.py | 86 --------- conans/client/conf/__init__.py | 3 - conans/requirements.txt | 5 - .../functional/command/export_linter_test.py | 176 ------------------ 5 files changed, 272 deletions(-) delete mode 100644 conans/client/cmd/export_linter.py delete mode 100644 conans/test/functional/command/export_linter_test.py diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py index 0395c47b6b8..82fe871d753 100644 --- a/conans/client/cmd/export.py +++ b/conans/client/cmd/export.py @@ -4,7 +4,6 @@ import six -from conans.client.cmd.export_linter import conan_linter from conans.client.file_copier import FileCopier from conans.client.output import Color, ScopedOutput from conans.client.remover import DiskRemover @@ -102,7 +101,6 @@ def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") - conan_linter(conanfile_path, output) output = conanfile.output # Get previous digest diff --git a/conans/client/cmd/export_linter.py b/conans/client/cmd/export_linter.py deleted file mode 100644 index 30762c1f338..00000000000 --- a/conans/client/cmd/export_linter.py +++ /dev/null @@ -1,86 +0,0 @@ -import json -import os -import platform -import sys -from subprocess import PIPE, Popen - -from conans import __path__ as root_path -from conans.client.output import Color -from conans.errors import ConanException - - -def conan_linter(conanfile_path, out): - if getattr(sys, 'frozen', False): - out.info("No linter available. Use a pip installed conan for recipe linting") - return - apply_lint = os.environ.get("CONAN_RECIPE_LINTER", True) - if not apply_lint or apply_lint == "False": - return - - dir_path = os.path.dirname(root_path[0]).replace("\\", "/") - dirname = os.path.dirname(conanfile_path).replace("\\", "/") - hook = '--init-hook="import sys;sys.path.extend([\'%s\', \'%s\'])"' % (dirname, dir_path) - - try: - py3_msgs = None - msgs, py3_msgs = _normal_linter(conanfile_path, hook) - except Exception as e: - out.warn("Failed pylint: %s" % e) - else: - if py3_msgs: - out.writeln("Python 3 incompatibilities\n ERROR: %s" - % "\n ERROR: ".join(py3_msgs), - front=Color.BRIGHT_MAGENTA) - if msgs: - out.writeln("Linter warnings\n WARN: %s" % "\n WARN: ".join(msgs), - front=Color.MAGENTA) - pylint_werr = os.environ.get("CONAN_PYLINT_WERR", None) - if pylint_werr and (py3_msgs or msgs): - raise ConanException("Package recipe has linter errors. Please fix them.") - - -def _runner(args): - command = ["pylint", "--output-format=json"] + args - command = " ".join(command) - shell = True if platform.system() != "Windows" else False - proc = Popen(command, shell=shell, bufsize=10, stdout=PIPE, stderr=PIPE) - stdout, _ = proc.communicate() - return json.loads(stdout.decode("utf-8")) if stdout else {} - - -def _normal_linter(conanfile_path, hook): - args = ["--py3k", "--enable=all", "--reports=no", "--disable=no-absolute-import", - "--persistent=no", "--load-plugins=conans.pylint_plugin", - hook, '"%s"' % conanfile_path] - pylintrc = os.environ.get("CONAN_PYLINTRC", None) - if pylintrc: - if not os.path.exists(pylintrc): - raise ConanException("File %s defined by PYLINTRC doesn't exist" % pylintrc) - args.append('--rcfile="%s"' % pylintrc) - - output_json = _runner(args) - - def _accept_message(msg): - symbol = msg.get("symbol") - - if symbol in ("bare-except", "broad-except"): # No exception type(s) specified - return False - if symbol == "import-error" and msg.get("column") > 3: # Import of a conan python package - return False - if symbol == "no-name-in-module" and "python_requires" in msg.get("message"): - return False - - return True - - result = [] - py3msgs = [] - for msg in output_json: - if msg.get("type") in ("warning", "error"): - message_id = msg.get("symbol") - if message_id in ("print-statement", "dict-iter-method"): - py3msgs.append("Py3 incompatibility. Line %s: %s" - % (msg.get("line"), msg.get("message"))) - elif _accept_message(msg): - result.append("Linter. Line %s: %s" % (msg.get("line"), msg.get("message"))) - - return result, py3msgs diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 55ceb97eb91..1582f34bdd7 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -125,7 +125,6 @@ # bash_path = "" # environment CONAN_BASH_PATH (only windows) # recipe_linter = False # environment CONAN_RECIPE_LINTER # read_only_cache = True # environment CONAN_READ_ONLY_CACHE -# pylintrc = path/to/pylintrc_file # environment CONAN_PYLINTRC # cache_no_locks = True # environment CONAN_CACHE_NO_LOCKS # user_home_short = your_path # environment CONAN_USER_HOME_SHORT # use_always_short_paths = False # environment CONAN_USE_ALWAYS_SHORT_PATHS @@ -206,9 +205,7 @@ def env_vars(self): "CONAN_COMPRESSION_LEVEL": self._env_c("general.compression_level", "CONAN_COMPRESSION_LEVEL", "9"), "CONAN_NON_INTERACTIVE": self._env_c("general.non_interactive", "CONAN_NON_INTERACTIVE", "False"), "CONAN_SKIP_BROKEN_SYMLINKS_CHECK": self._env_c("general.skip_broken_symlinks_check", "CONAN_SKIP_BROKEN_SYMLINKS_CHECK", "False"), - "CONAN_PYLINTRC": self._env_c("general.pylintrc", "CONAN_PYLINTRC", None), "CONAN_CACHE_NO_LOCKS": self._env_c("general.cache_no_locks", "CONAN_CACHE_NO_LOCKS", "False"), - "CONAN_PYLINT_WERR": self._env_c("general.pylint_werr", "CONAN_PYLINT_WERR", None), "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_SYSREQUIRES_MODE": self._env_c("general.sysrequires_mode", "CONAN_SYSREQUIRES_MODE", "enabled"), "CONAN_REQUEST_TIMEOUT": self._env_c("general.request_timeout", "CONAN_REQUEST_TIMEOUT", None), diff --git a/conans/requirements.txt b/conans/requirements.txt index bf09a4ad7b6..d0c267b2ef9 100644 --- a/conans/requirements.txt +++ b/conans/requirements.txt @@ -8,14 +8,9 @@ fasteners>=0.14.1 six>=1.10.0,<1.13.0 node-semver==0.6.1 distro>=1.0.2, <1.2.0 -pylint>=1.9.3, <2.0; python_version < '3' -pylint>=2.0,!=2.3.0; python_version >= '3' future>=0.16.0, <0.19.0 pygments>=2.0, <3.0 -astroid>=1.6.5, <2.0; python_version < '3' -astroid>=2.2.0; python_version >= '3' deprecation>=2.0, <2.1 tqdm>=4.28.1, <5 Jinja2>=2.3, <3 python-dateutil>=2.7.0, <3 -typed-ast<1.4; python_version == '3.4' and platform_system=='Windows' diff --git a/conans/test/functional/command/export_linter_test.py b/conans/test/functional/command/export_linter_test.py deleted file mode 100644 index a188cbd0a7e..00000000000 --- a/conans/test/functional/command/export_linter_test.py +++ /dev/null @@ -1,176 +0,0 @@ -import os -import unittest - -import six -from mock import patch - -from conans.client import tools -from conans.paths import CONANFILE, DATA_YML -from conans.test.utils.tools import TestClient - -conanfile = """ -from conans import ConanFile, tools -class TestConan(ConanFile): - name = "Hello" - version = "1.2" - def build(self): - print("HEllo world") - for k, v in {}.iteritems(): - pass - tools.msvc_build_command(self.settings, "path") -""" - - -@patch.dict('os.environ', {"CONAN_RECIPE_LINTER": "True"}) -class ExportLinterTest(unittest.TestCase): - - def test_basic(self): - client = TestClient() - client.save({CONANFILE: conanfile}) - client.run("export . lasote/stable") - self._check_linter(client.out) - - def _check_linter(self, output): - if six.PY2: - self.assertIn("ERROR: Py3 incompatibility. Line 7: print statement used", output) - self.assertIn("ERROR: Py3 incompatibility. Line 8: Calling a dict.iter*() method", - output) - self.assertIn("WARN: Linter. Line 8: Unused variable 'k'", output) - self.assertIn("WARN: Linter. Line 8: Unused variable 'v'", output) - - def test_disable_linter(self): - client = TestClient() - client.save({CONANFILE: conanfile}) - with tools.environment_append({"CONAN_RECIPE_LINTER": "False"}): - client.run("export . lasote/stable") - self.assertNotIn("ERROR: Py3 incompatibility", client.out) - self.assertNotIn("WARN: Linter", client.out) - - def test_custom_rc_linter(self): - client = TestClient() - pylintrc = """[FORMAT] -indent-string=' ' - """ - client.save({CONANFILE: conanfile, - "pylintrc": pylintrc}) - client.run('config set general.pylintrc="%s"' - % os.path.join(client.current_folder, "pylintrc")) - client.run("export . lasote/stable") - self.assertIn("Bad indentation. Found 4 spaces, expected 2", client.out) - - def test_dynamic_fields(self): - client = TestClient() - conanfile_base = """ -from conans import ConanFile -class BaseConan(ConanFile): - name = "baselib" - version = "1.0" -""" - client.save({CONANFILE: conanfile_base}) - client.run("export . conan/stable") - - conanfile2 = """ -from conans import ConanFile, python_requires - -python_requires("baselib/1.0@conan/stable") -class TestConan(ConanFile): - name = "Hello" - version = "1.2" - - def build(self): - self.output.info(self.source_folder) - self.output.info(self.package_folder) - self.output.info(self.build_folder) - self.output.info(self.install_folder) - - def package(self): - self.copy("*") - - def package_id(self): - self.info.header_only() - - def build_id(self): - self.output.info(str(self.info_build)) - - def build_requirements(self): - self.build_requires("baselib/1.0@conan/stable") -""" - client.save({CONANFILE: conanfile2}) - client.run("export . lasote/stable") - self.assertNotIn("Linter", client.out) - # ensure nothing breaks - client.run("install Hello/1.2@lasote/stable --build") - - def test_catch_em_all(self): - client = TestClient() - conanfile_base = """ -from conans import ConanFile -class BaseConan(ConanFile): - name = "baselib" - version = "1.0" - - def source(self): - try: - raise Exception("Pikaaaaa!!") - except: - print("I got pikachu!!") - - try: - raise Exception("Pikaaaaa!!") - except Exception: - print("I got pikachu!!") -""" - client.save({CONANFILE: conanfile_base}) - client.run("export . conan/stable") - self.assertNotIn("Failed pylint", client.out) - self.assertNotIn("Linter", client.out) - - def test_warning_as_errors(self): - client = TestClient() - client.save({CONANFILE: conanfile}) - client.run("config set general.pylint_werr=True") - client.run("export . lasote/stable", assert_error=True) - self._check_linter(client.out) - self.assertIn("ERROR: Package recipe has linter errors. Please fix them", - client.out) - - def export_deploy_test(self): - conanfile = """ -from conans import ConanFile -class BaseConan(ConanFile): - name = "baselib" - version = "1.0" - - def deploy(self): - self.copy_deps("*.dll") -""" - client = TestClient() - client.save({CONANFILE: conanfile}) - client.run("export . conan/stable") - self.assertNotIn("Failed pylint", client.out) - self.assertNotIn("Linter warnings", client.out) - self.assertNotIn("WARN: Linter. Line 8: Instance of 'BaseConan' has no 'copy_deps' member", - client.out) - self.assertNotIn("WARN: Linter. Line 8: self.copy_deps is not callable", - client.out) - - def test_conan_data(self): - client = TestClient() - conanfile = """ -from conans import ConanFile -class ExampleConan(ConanFile): - name = "example" - - def build(self): - print(self.conan_data["sources"][float(self.version)]) -""" - conandata = """sources: - 2.3: - url: "https://github.com/google/cctz/archive/v2.3.tar.gz" - sha256: "8615b20d4e33e02a271c3b93a3b208e3d7d5d66880f5f6208b03426e448f32db" -""" - client.save({DATA_YML: conandata}) - client.save({CONANFILE: conanfile}) - client.run("export . 2.3@conan/stable") - self.assertNotIn("Linter.", client.out) - self.assertNotIn("Instance of 'ExampleConan' has no 'conan_data' member", client.out) From 992a13b6684cb54afd78b68337ff3f7957715086 Mon Sep 17 00:00:00 2001 From: javierg Date: Fri, 29 Nov 2019 17:24:37 +0100 Subject: [PATCH 02/12] add python_requires to 'conans' --- conans/pylint_plugin.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/conans/pylint_plugin.py b/conans/pylint_plugin.py index 01fb3557658..a10a227da43 100644 --- a/conans/pylint_plugin.py +++ b/conans/pylint_plugin.py @@ -45,3 +45,13 @@ def transform_conanfile(node): MANAGER.register_transform( astroid.ClassDef, transform_conanfile, lambda node: node.qname() == "conans.model.conan_file.ConanFile") + + +def _python_requires_member(): + return astroid.parse(""" + from conans.client.graph.python_requires import ConanPythonRequire + python_requires = ConanPythonRequire() + """) + + +astroid.register_module_extender(astroid.MANAGER, "conans", _python_requires_member) From 3e313d909e0c365ebf9376de5c32e21a05834e54 Mon Sep 17 00:00:00 2001 From: Morwenn Date: Mon, 2 Dec 2019 12:53:37 +0100 Subject: [PATCH 03/12] Fix SyntaxWarning in Python 3.8 (#6165) --- conans/client/tools/win.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/conans/client/tools/win.py b/conans/client/tools/win.py index 6c2363b8771..2d32ef621f4 100644 --- a/conans/client/tools/win.py +++ b/conans/client/tools/win.py @@ -280,7 +280,7 @@ def vswhere(all_=False, prerelease=False, products=None, requires=None, version= arguments.append("-requires") arguments.extend(requires) - if len(version) is not 0: + if len(version) != 0: arguments.append("-version") arguments.append(version) @@ -290,7 +290,7 @@ def vswhere(all_=False, prerelease=False, products=None, requires=None, version= if legacy: arguments.append("-legacy") - if len(property_) is not 0: + if len(property_) != 0: arguments.append("-property") arguments.append(property_) From c1fe52bd772cba8021f12aeb12f75bb5ce712740 Mon Sep 17 00:00:00 2001 From: SSE4 Date: Mon, 2 Dec 2019 22:11:55 +0700 Subject: [PATCH 04/12] - generate ConfigVersion.cmake file (#6063) Signed-off-by: SSE4 --- .../generators/cmake_find_package_multi.py | 29 ++++++ .../cmake_find_package_multi_test.py | 89 ++++++++++++++++++- .../unittests/client/generators/cmake_test.py | 3 +- 3 files changed, 119 insertions(+), 2 deletions(-) diff --git a/conans/client/generators/cmake_find_package_multi.py b/conans/client/generators/cmake_find_package_multi.py index 99e781000e2..87a221744a9 100644 --- a/conans/client/generators/cmake_find_package_multi.py +++ b/conans/client/generators/cmake_find_package_multi.py @@ -61,6 +61,33 @@ class CMakeFindPackageMultiGenerator(Generator): $<$:${{{name}_COMPILE_OPTIONS_DEBUG_LIST}}>) """ +# https://gitlab.kitware.com/cmake/cmake/blob/master/Modules/BasicConfigVersion-SameMajorVersion.cmake.in + version_template = """ +set(PACKAGE_VERSION "{version}") + +if(PACKAGE_VERSION VERSION_LESS PACKAGE_FIND_VERSION) + set(PACKAGE_VERSION_COMPATIBLE FALSE) +else() + + if("{version}" MATCHES "^([0-9]+)\\\\.") + set(CVF_VERSION_MAJOR "${{CMAKE_MATCH_1}}") + else() + set(CVF_VERSION_MAJOR "{version}") + endif() + + if(PACKAGE_FIND_VERSION_MAJOR STREQUAL CVF_VERSION_MAJOR) + set(PACKAGE_VERSION_COMPATIBLE TRUE) + else() + set(PACKAGE_VERSION_COMPATIBLE FALSE) + endif() + + if(PACKAGE_FIND_VERSION STREQUAL PACKAGE_VERSION) + set(PACKAGE_VERSION_EXACT TRUE) + endif() + +endif() +""" + @property def filename(self): pass @@ -79,6 +106,8 @@ def content(self): find_lib = target_template.format(name=depname, deps=deps, build_type_suffix=build_type_suffix) ret["{}Target-{}.cmake".format(depname, build_type.lower())] = find_lib + ret["{}ConfigVersion.cmake".format(depname)] = self.version_template.\ + format(version=cpp_info.version) return ret def _build_type_suffix(self, build_type): diff --git a/conans/test/functional/generators/cmake_find_package_multi_test.py b/conans/test/functional/generators/cmake_find_package_multi_test.py index 8420b0352b2..d427a829654 100644 --- a/conans/test/functional/generators/cmake_find_package_multi_test.py +++ b/conans/test/functional/generators/cmake_find_package_multi_test.py @@ -4,6 +4,7 @@ import unittest from nose.plugins.attrib import attr +from parameterized import parameterized from conans.model.ref import ConanFileReference, PackageReference from conans.test.utils.tools import TestClient, NO_SETTINGS_PACKAGE_ID, replace_in_file @@ -229,8 +230,8 @@ def cpp_info_name_test(self): output=client.out) client.run("create .") cmakelists = """ -project(consumer) cmake_minimum_required(VERSION 3.1) +project(consumer) find_package(MYHELLO2) get_target_property(tmp MYHELLO2::MYHELLO2 INTERFACE_LINK_LIBRARIES) @@ -261,3 +262,89 @@ def build(self): "$<$:;>;" "$<$:;>", client.out) + + def no_version_file_test(self): + client = TestClient() + client.run("new hello/1.1 -s") + client.run("create .") + + cmakelists = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.1) + project(consumer) + find_package(hello 1.0 REQUIRED) + message(STATUS "hello found: ${{hello_FOUND}}") + """) + + conanfile = textwrap.dedent(""" + from conans import ConanFile, CMake + + + class Conan(ConanFile): + settings = "build_type" + requires = "hello/1.1" + generators = "cmake_find_package_multi" + + def build(self): + cmake = CMake(self) + cmake.configure() + """) + + client.save({"conanfile.py": conanfile, "CMakeLists.txt": cmakelists}) + client.run("install .") + os.unlink(os.path.join(client.current_folder, "helloConfigVersion.cmake")) + exit_code = client.run("build .", assert_error=True) + self.assertNotEqual(0, exit_code) + + @parameterized.expand([ + ("find_package(hello 1.0)", False, True), + ("find_package(hello 1.1)", False, True), + ("find_package(hello 1.2)", False, False), + ("find_package(hello 1.0 EXACT)", False, False), + ("find_package(hello 1.1 EXACT)", False, True), + ("find_package(hello 1.2 EXACT)", False, False), + ("find_package(hello 0.1)", False, False), + ("find_package(hello 2.0)", False, False), + ("find_package(hello 1.0 REQUIRED)", False, True), + ("find_package(hello 1.1 REQUIRED)", False, True), + ("find_package(hello 1.2 REQUIRED)", True, False), + ("find_package(hello 1.0 EXACT REQUIRED)", True, False), + ("find_package(hello 1.1 EXACT REQUIRED)", False, True), + ("find_package(hello 1.2 EXACT REQUIRED)", True, False), + ("find_package(hello 0.1 REQUIRED)", True, False), + ("find_package(hello 2.0 REQUIRED)", True, False) + ]) + def version_test(self, find_package_string, cmake_fails, package_found): + client = TestClient() + client.run("new hello/1.1 -s") + client.run("create .") + + cmakelists = textwrap.dedent(""" + cmake_minimum_required(VERSION 3.1) + project(consumer) + {find_package_string} + message(STATUS "hello found: ${{hello_FOUND}}") + """).format(find_package_string=find_package_string) + + conanfile = textwrap.dedent(""" + from conans import ConanFile, CMake + + + class Conan(ConanFile): + settings = "build_type" + requires = "hello/1.1" + generators = "cmake_find_package_multi" + + def build(self): + cmake = CMake(self) + cmake.configure() + """) + + client.save({"conanfile.py": conanfile, "CMakeLists.txt": cmakelists}) + client.run("install .") + exit_code = client.run("build .", assert_error=cmake_fails) + if cmake_fails: + self.assertNotEqual(exit_code, 0) + elif package_found: + self.assertIn("hello found: 1", client.out) + else: + self.assertIn("hello found: 0", client.out) diff --git a/conans/test/unittests/client/generators/cmake_test.py b/conans/test/unittests/client/generators/cmake_test.py index 3acbd6e9a8a..4366c5015e4 100644 --- a/conans/test/unittests/client/generators/cmake_test.py +++ b/conans/test/unittests/client/generators/cmake_test.py @@ -419,7 +419,8 @@ def cmake_find_package_multi_test(self): content = generator.content six.assertCountEqual(self, ['MyPkG2Targets.cmake', 'MyPkGConfig.cmake', 'MyPkG2Config.cmake', 'MyPkGTargets.cmake', 'MyPkGTarget-debug.cmake', - 'MyPkG2Target-debug.cmake'], content.keys()) + 'MyPkG2Target-debug.cmake', 'MyPkGConfigVersion.cmake', + 'MyPkG2ConfigVersion.cmake'], content.keys()) self.assertNotIn("my_pkg", content["MyPkGConfig.cmake"]) self.assertNotIn("MY_PKG", content["MyPkGConfig.cmake"]) self.assertNotIn("my_pkg", content["MyPkG2Config.cmake"]) From cc1668d88dc1bdcd0798e172fbc484ab02ecfd5c Mon Sep 17 00:00:00 2001 From: Jintao Yin Date: Tue, 3 Dec 2019 00:01:19 +0800 Subject: [PATCH 05/12] Fixes #5925 Make the first line from find program as the result (#6039) * 3. extra separator in Windows * Improve the error message when failed to connect to remote * Improve the error message when failed to connect to remote * Improve the error message when failed to connect to remote * Revert unwanted changes * Pick the first match which is the best match * Pick the first match which is the best match * make reversed explicit (better that [::-1]) --- .../test/functional/generators/virtualenv_test.py | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/conans/test/functional/generators/virtualenv_test.py b/conans/test/functional/generators/virtualenv_test.py index fa1a3a8322d..06cd53098f4 100644 --- a/conans/test/functional/generators/virtualenv_test.py +++ b/conans/test/functional/generators/virtualenv_test.py @@ -249,27 +249,26 @@ def test_list_variable(self): def test_find_program(self): # If we add the path, we should found the env/executable instead of ori/executable + # Watch out! 'cmd' returns all the paths where the executable is found, so we need to + # take into account the first match (iterate in reverse order) generator = VirtualEnvGenerator(ConanFileMock()) generator.env = {"PATH": [self.env_path], } stdout, environment = self._run_virtualenv(generator) - cpaths = dict(l.split("=", 1) for l in stdout.splitlines() if l.startswith("__conan_")) + cpaths = dict(l.split("=", 1) for l in reversed(stdout.splitlines()) if l.startswith("__conan_")) self.assertEqual(cpaths["__conan_pre_path__"], cpaths["__conan_post_path__"]) self.assertEqual(cpaths["__conan_env_path__"], cpaths["__conan_post_path__"]) - epaths = dict(l.split("=", 1) for l in stdout.splitlines() if l.startswith("__exec_")) + epaths = dict(l.split("=", 1) for l in reversed(stdout.splitlines()) if l.startswith("__exec_")) self.assertEqual(epaths["__exec_pre_path__"], epaths["__exec_post_path__"]) - if self.commands.id == "cmd": # FIXME: This is a bug, it doesn't take into account the new path - self.assertNotEqual(epaths["__exec_env_path__"], os.path.join(self.env_path, self.app)) - else: - self.assertEqual(epaths["__exec_env_path__"], os.path.join(self.env_path, self.app)) + self.assertEqual(epaths["__exec_env_path__"], os.path.join(self.env_path, self.app)) # With any other path, we keep finding the original one generator = VirtualEnvGenerator(ConanFileMock()) generator.env = {"PATH": [os.path.join(self.test_folder, "wrong")], } stdout, environment = self._run_virtualenv(generator) - epaths = dict(l.split("=", 1) for l in stdout.splitlines() if l.startswith("__exec_")) + epaths = dict(l.split("=", 1) for l in reversed(stdout.splitlines()) if l.startswith("__exec_")) self.assertEqual(epaths["__exec_pre_path__"], epaths["__exec_post_path__"]) self.assertEqual(epaths["__exec_env_path__"], epaths["__exec_post_path__"]) From 7042f3929f36b1f8c2ad9c4ad63e7396639f059b Mon Sep 17 00:00:00 2001 From: Carlos Zoido Date: Mon, 2 Dec 2019 17:18:58 +0100 Subject: [PATCH 06/12] Fix output folder for conanworkspace.cmake when rebuilding dependencies (#6060) * Added required = True to subparsers in order to print error message in Py2 and Py3. * sync * basic concurrent upload at reference level with futures * revert changes * add line * Lock buggy urllib3 (#5808) * app simplifying (#5806) * Apply lockfile before updating downstream requires (#5771) * apply graph_lock before looking for overrides * first step: get rid of the warning * cleaner if graph_lock is passed to the function * only update requires upstream if no lockfile is applied * fix tests * Deprecation of CONAN_USERNAME and CONAN_CHANNEL: fix error message (#5756) * if CONAN_USERNAME and CONAN_CHANNEL are deprecated, the error cannot recommend them * update tests accordingly * test client load() file method (#5815) * no user/channel repr without _ (#5817) * no user/channel repr without _ * minor fixes * fix tests * Remove py34 (#5820) * fix upload package id (#5824) * - update macOS, watchOS, tvOS, iOS version numbers (#5823) * Refresh token client support. (#5662) * Refresh token client support. Missing tests. Missing migration * public method * WIP * Refresh almost there * Removed prints * Try migrate * Migration * Add comment * Refresh token flow following RFC recommentations * Refresh ok * review * Remove traces * Refactor capabilities * Removed tmp file * Review * #5819 Show warning message for Python 3.4 (#5829) * #5819 Show warning message for Python 3.4 - Add new warning message for python 3.4 which is no longer supported - Added funcional tests to validate both python 3.4 and 2.x Signed-off-by: Uilian Ries * #5819 Fix broken tests Signed-off-by: Uilian Ries * Add cpp_info.name to cmake and pkg_config generators (#5598) * Add cpp_info.name to cmake generators * Fix unit tests to mimic real behavior * cmake_paths test * add test for cmake generator * Add cmake_find_package test * fix test in py3 * Applied cpp_info.name to pkg_config generator * check different name in pkg_config * sync with develop * save conanworkspace.cmake relative to base folder * add test * siplify test * add context manager to build folder * change indent --- conans/client/installer.py | 58 +++++++++---------- .../functional/workspace/workspace_test.py | 40 ++++++++++++- 2 files changed, 68 insertions(+), 30 deletions(-) diff --git a/conans/client/installer.py b/conans/client/installer.py index 9ebe38b75a0..1f35bdfac08 100644 --- a/conans/client/installer.py +++ b/conans/client/installer.py @@ -191,35 +191,35 @@ def build_package(self, node, keep_build, recorder, remotes): with package_layout.conanfile_read_lock(self._output): _remove_folder_raising(package_folder) mkdir(build_folder) - os.chdir(build_folder) - self._output.info('Building your package in %s' % build_folder) - try: - if getattr(conanfile, 'no_copy_source', False): - conanfile.source_folder = source_folder - else: - conanfile.source_folder = build_folder - - if not skip_build: - with get_env_context_manager(conanfile): - conanfile.build_folder = build_folder - conanfile.package_folder = package_folder - # In local cache, install folder always is build_folder - conanfile.install_folder = build_folder - self._build(conanfile, pref, build_folder) - clean_dirty(build_folder) - - prev = self._package(conanfile, pref, package_layout, conanfile_path, build_folder, - package_folder) - assert prev - node.prev = prev - log_file = os.path.join(build_folder, RUN_LOG_NAME) - log_file = log_file if os.path.exists(log_file) else None - log_package_built(pref, time.time() - t1, log_file) - recorder.package_built(pref) - except ConanException as exc: - recorder.package_install_error(pref, INSTALL_ERROR_BUILDING, - str(exc), remote_name=None) - raise exc + with tools.chdir(build_folder): + self._output.info('Building your package in %s' % build_folder) + try: + if getattr(conanfile, 'no_copy_source', False): + conanfile.source_folder = source_folder + else: + conanfile.source_folder = build_folder + + if not skip_build: + with get_env_context_manager(conanfile): + conanfile.build_folder = build_folder + conanfile.package_folder = package_folder + # In local cache, install folder always is build_folder + conanfile.install_folder = build_folder + self._build(conanfile, pref, build_folder) + clean_dirty(build_folder) + + prev = self._package(conanfile, pref, package_layout, conanfile_path, build_folder, + package_folder) + assert prev + node.prev = prev + log_file = os.path.join(build_folder, RUN_LOG_NAME) + log_file = log_file if os.path.exists(log_file) else None + log_package_built(pref, time.time() - t1, log_file) + recorder.package_built(pref) + except ConanException as exc: + recorder.package_install_error(pref, INSTALL_ERROR_BUILDING, + str(exc), remote_name=None) + raise exc return node.pref diff --git a/conans/test/functional/workspace/workspace_test.py b/conans/test/functional/workspace/workspace_test.py index d01255d1246..45f8742bbd4 100644 --- a/conans/test/functional/workspace/workspace_test.py +++ b/conans/test/functional/workspace/workspace_test.py @@ -12,7 +12,7 @@ from conans.errors import ConanException from conans.model.workspace import Workspace from conans.test.utils.test_files import temp_folder -from conans.test.utils.tools import TestClient +from conans.test.utils.tools import TestClient, GenConanfile from conans.util.files import load, save conanfile_build = """from conans import ConanFile, CMake @@ -1020,6 +1020,44 @@ class Lib(ConanFile): self.assertTrue(os.path.exists(os.path.join(client.current_folder, "ws_install", "conanworkspace.cmake"))) + def test_install_folder_rebuilt_requirements(self): + # https://github.com/conan-io/conan/issues/6046 + client = TestClient() + tool = dedent(""" + from conans import ConanFile + class Tool(ConanFile): + def package_info(self): + self.cpp_info.libs = ["MyToolLib"] + """) + client.save({"conanfile.py": tool}) + client.run("export . Tool/0.1@user/testing") + client.save({"conanfile.py": GenConanfile().with_name("HelloB").with_version("0.1")}, + path=os.path.join(client.current_folder, "B")) + client.save({"conanfile.py": GenConanfile().with_name("HelloA").with_version( + "0.1").with_build_require_plain("Tool/0.1@user/testing").with_require_plain("HelloB/0.1")}, + path=os.path.join(client.current_folder, "A")) + + project = dedent(""" + editables: + HelloB/0.1: + path: B + HelloA/0.1: + path: A + layout: layout + root: HelloA/0.1 + workspace_generator: cmake + """) + layout = dedent(""" + [build_folder] + build + """) + client.save({"conanws.yml": project, + "layout": layout}) + client.run( + "workspace install conanws.yml --install-folder=ws_install --build Tool/0.1@user/testing") + self.assertTrue(os.path.exists(os.path.join(client.current_folder, "ws_install", + "conanworkspace.cmake"))) + def missing_subarguments_test(self): client = TestClient() client.run("workspace", assert_error=True) From 5b9f5a800459f45eabc18e5d5cbd8704292c3af6 Mon Sep 17 00:00:00 2001 From: "Javier G. Sogo" Date: Mon, 2 Dec 2019 17:53:50 +0100 Subject: [PATCH 07/12] Publish 'artifacts.properties' using matrix params (#6014) * draft to test * add missing argument to functions * rename 'put_headers' to 'artifact_properties' * store matrix params as the preformatted string in the router * fallback to empty string * something going wrong: too many cchanges * at least these are not needed * typo * it was only needed to remove the matrix_params for the bottle server * soooo * remove class name * fix minor bug * now with revisions too * make docs with triple double quotes * we need to modify the conan_server to accept matrix_params too * fix tests * add tests with artifacts.properties * quote values of matrix params * we can stack routes * add tests with many different values * remove changes not needed * use server capability to choose the URL to build * do not send headers if matrix_params capability * do not add 'matrix_params' functionality to conan-server * rename variable, 'file' is reserved * remove change uneeded * conan server does not support matrix params at all * use the proper if/else --- conans/__init__.py | 1 + conans/client/rest/client_routes.py | 179 ++++++++++-------- conans/client/rest/rest_client.py | 11 +- conans/client/rest/rest_client_common.py | 3 +- conans/client/rest/rest_client_v1.py | 10 +- conans/client/rest/rest_client_v2.py | 14 +- conans/model/rest_routes.py | 42 ++-- conans/server/rest/controller/v1/conan.py | 3 +- .../controller/v1/file_upload_download.py | 8 +- conans/server/store/disk_adapter.py | 16 +- .../functional/remote/auth_bearer_test.py | 20 +- .../functional/remote/put_properties_test.py | 93 +++++++-- 12 files changed, 249 insertions(+), 151 deletions(-) diff --git a/conans/__init__.py b/conans/__init__.py index 7f0450385be..ac70949b038 100644 --- a/conans/__init__.py +++ b/conans/__init__.py @@ -16,6 +16,7 @@ CHECKSUM_DEPLOY = "checksum_deploy" # Only when v2 REVISIONS = "revisions" # Only when enabled in config, not by default look at server_launcher.py ONLY_V2 = "only_v2" # Remotes and virtuals from Artifactory returns this capability +MATRIX_PARAMS = "matrix_params" OAUTH_TOKEN = "oauth_token" SERVER_CAPABILITIES = [COMPLEX_SEARCH_CAPABILITY, REVISIONS] # Server is always with revisions DEFAULT_REVISION_V1 = "0" diff --git a/conans/client/rest/client_routes.py b/conans/client/rest/client_routes.py index 3159ca49b65..e4c2a5fa1c0 100644 --- a/conans/client/rest/client_routes.py +++ b/conans/client/rest/client_routes.py @@ -1,32 +1,49 @@ -from six.moves.urllib.parse import urlencode +from six.moves.urllib.parse import quote, urlencode from conans.model.ref import ConanFileReference from conans.model.rest_routes import RestRoutes -from conans.paths import CONAN_MANIFEST, CONANINFO +from conans.paths import CONAN_MANIFEST, CONANINFO, ARTIFACTS_PROPERTIES_PUT_PREFIX -def _format_ref(url, ref): +def _format_ref(url, ref, matrix_params=""): url = url.format(name=ref.name, version=ref.version, username=ref.user or "_", - channel=ref.channel or "_", revision=ref.revision) + channel=ref.channel or "_", revision=ref.revision, + matrix_params=matrix_params) return url -def _format_pref(url, pref): +def _format_pref(url, pref, matrix_params=""): ref = pref.ref url = url.format(name=ref.name, version=ref.version, username=ref.user or "_", channel=ref.channel or "_", revision=ref.revision, package_id=pref.id, - p_revision=pref.revision) + p_revision=pref.revision, matrix_params=matrix_params) return url -routes = RestRoutes() +def _remove_put_prefix(artifacts_properties): + len_prefix = len(ARTIFACTS_PROPERTIES_PUT_PREFIX) + for key, value in artifacts_properties.items(): + if key.startswith(ARTIFACTS_PROPERTIES_PUT_PREFIX): + key = key[len_prefix:] + yield key, value class ClientCommonRouter(object): """Search urls shared between v1 and v2""" + def __init__(self, base_url, artifacts_properties, matrix_params): + self.base_url = base_url + self.routes = RestRoutes(matrix_params=matrix_params) + + if artifacts_properties: + props_dict = dict(_remove_put_prefix(artifacts_properties)) + self._matrix_params_str = ";" + ";".join(["{}={}".format(key, quote(value, safe='')) + for key, value in props_dict.items()]) + else: + self._matrix_params_str = "" + def ping(self): - return self.base_url + routes.ping + return self.base_url + self.routes.ping def search(self, pattern, ignorecase): """URL search recipes""" @@ -38,32 +55,42 @@ def search(self, pattern, ignorecase): if not ignorecase: params["ignorecase"] = "False" query = "?%s" % urlencode(params) - return self.base_url + "%s%s" % (routes.common_search, query) + return self.base_url + "%s%s" % (self.routes.common_search, query) def search_packages(self, ref, query=None): """URL search packages for a recipe""" - route = routes.common_search_packages_revision \ - if ref.revision else routes.common_search_packages + route = self.routes.common_search_packages_revision \ + if ref.revision else self.routes.common_search_packages url = _format_ref(route, ref) if query: url += "?%s" % urlencode({"q": query}) return self.base_url + url def oauth_authenticate(self): - return self.base_url + routes.oauth_authenticate + return self.base_url + self.routes.oauth_authenticate def common_authenticate(self): - return self.base_url + routes.common_authenticate + return self.base_url + self.routes.common_authenticate def common_check_credentials(self): - return self.base_url + routes.common_check_credentials + return self.base_url + self.routes.common_check_credentials class ClientV1Router(ClientCommonRouter): """Builds urls for v1""" - def __init__(self, base_url): - self.base_url = "{}/v1/".format(base_url) + def __init__(self, base_url, artifact_properties, matrix_params): + base_url = "{}/v1/".format(base_url) + super(ClientV1Router, self).__init__(base_url, artifact_properties, matrix_params) + + def add_matrix_params(self, urls): + if not self._matrix_params_str: + return urls + + ret = {} + for filename, url in urls.items(): + ret[filename] = url.replace("v1/files/", "v1/files{}/".format(self._matrix_params_str)) + return ret def search_packages(self, ref, query=None): ref = ref.copy_clear_rev() @@ -75,11 +102,11 @@ def remove_recipe(self, ref): def remove_recipe_files(self, ref): """Removes files from the recipe""" - return self.base_url + _format_ref(routes.v1_remove_recipe_files, ref.copy_clear_rev()) + return self.base_url + _format_ref(self.routes.v1_remove_recipe_files, ref.copy_clear_rev()) def remove_packages(self, ref): """Remove files from a package""" - return self.base_url + _format_ref(routes.v1_remove_packages, ref.copy_clear_rev()) + return self.base_url + _format_ref(self.routes.v1_remove_packages, ref.copy_clear_rev()) def recipe_snapshot(self, ref): """get recipe manifest url""" @@ -91,54 +118,58 @@ def package_snapshot(self, pref): def recipe_manifest(self, ref): """get recipe manifest url""" - return self.base_url + _format_ref(routes.v1_recipe_digest, ref.copy_clear_rev()) + return self.base_url + _format_ref(self.routes.v1_recipe_digest, ref.copy_clear_rev()) def package_manifest(self, pref): """get manifest url""" - return self.base_url + _format_pref(routes.v1_package_digest, pref.copy_clear_revs()) + return self.base_url + _format_pref(self.routes.v1_package_digest, pref.copy_clear_revs()) def recipe_download_urls(self, ref): """ urls to download the recipe""" - return self.base_url + _format_ref(routes.v1_recipe_download_urls, - ref.copy_clear_rev()) + return self.base_url + _format_ref(self.routes.v1_recipe_download_urls, ref.copy_clear_rev()) def package_download_urls(self, pref): """ urls to download the package""" pref = pref.copy_with_revs(None, None) - return self.base_url + _format_pref(routes.v1_package_download_urls, pref) + return self.base_url + _format_pref(self.routes.v1_package_download_urls, pref) - def recipe_upload_urls(self, ref): + def recipe_upload_urls(self, ref, add_matrix_params=False): """ urls to upload the recipe""" - return self.base_url + _format_ref(routes.v1_recipe_upload_urls, ref.copy_clear_rev()) + matrix_params = self._matrix_params_str if add_matrix_params else "" + return self.base_url + _format_ref(self.routes.v1_recipe_upload_urls, ref.copy_clear_rev(), + matrix_params=matrix_params) - def package_upload_urls(self, pref): + def package_upload_urls(self, pref, add_matrix_params=False): """ urls to upload the package""" pref = pref.copy_with_revs(None, None) - return self.base_url + _format_pref(routes.v1_package_upload_urls, pref) + matrix_params = self._matrix_params_str if add_matrix_params else "" + return self.base_url + _format_pref(self.routes.v1_package_upload_urls, pref, + matrix_params=matrix_params) - @staticmethod - def _for_recipe(ref): - return _format_ref(routes.recipe, ref.copy_clear_rev()) + def _for_recipe(self, ref): + return _format_ref(self.routes.recipe, ref.copy_clear_rev()) - @staticmethod - def _for_package(pref): + def _for_package(self, pref): pref = pref.copy_with_revs(None, None) - return _format_pref(routes.package, pref) + return _format_pref(self.routes.package, pref) class ClientV2Router(ClientCommonRouter): """Builds urls for v2""" - def __init__(self, base_url): - self.base_url = "{}/v2/".format(base_url) + def __init__(self, base_url, artifacts_properties, matrix_params): + base_url = "{}/v2/".format(base_url) + super(ClientV2Router, self).__init__(base_url, artifacts_properties, matrix_params) - def recipe_file(self, ref, path): + def recipe_file(self, ref, path, add_matrix_params=False): """Recipe file url""" - return self.base_url + self._for_recipe_file(ref, path) + matrix_params = self._matrix_params_str if add_matrix_params else "" + return self.base_url + self._for_recipe_file(ref, path, matrix_params=matrix_params) - def package_file(self, pref, path): + def package_file(self, pref, path, add_matrix_params=False): """Package file url""" - return self.base_url + self._for_package_file(pref, path) + matrix_params = self._matrix_params_str if add_matrix_params else "" + return self.base_url + self._for_package_file(pref, path, matrix_params=matrix_params) def remove_recipe(self, ref): """Remove recipe url""" @@ -146,7 +177,7 @@ def remove_recipe(self, ref): def recipe_revisions(self, ref): """Get revisions for a recipe url""" - return self.base_url + _format_ref(routes.recipe_revisions, ref) + return self.base_url + _format_ref(self.routes.recipe_revisions, ref) def remove_package(self, pref): """Remove package url""" @@ -159,15 +190,15 @@ def remove_all_packages(self, ref): def recipe_manifest(self, ref): """Get the url for getting a conanmanifest.txt from a recipe""" - return self.base_url + self._for_recipe_file(ref, CONAN_MANIFEST) + return self.base_url + self._for_recipe_file(ref, CONAN_MANIFEST, matrix_params=None) def package_manifest(self, pref): """Get the url for getting a conanmanifest.txt from a package""" - return self.base_url + self._for_package_file(pref, CONAN_MANIFEST) + return self.base_url + self._for_package_file(pref, CONAN_MANIFEST, matrix_params=None) def package_info(self, pref): """Get the url for getting a conaninfo.txt from a package""" - return self.base_url + self._for_package_file(pref, CONANINFO) + return self.base_url + self._for_package_file(pref, CONANINFO, matrix_params=None) def recipe_snapshot(self, ref): """get recipe manifest url""" @@ -179,78 +210,72 @@ def package_snapshot(self, pref): def package_revisions(self, pref): """get revisions for a package url""" - return self.base_url + _format_pref(routes.package_revisions, pref) + return self.base_url + _format_pref(self.routes.package_revisions, pref) def package_latest(self, pref): """Get the latest of a package""" assert pref.ref.revision is not None, "Cannot get the latest package without RREV" - return self.base_url + _format_pref(routes.package_revision_latest, pref) + return self.base_url + _format_pref(self.routes.package_revision_latest, pref) def recipe_latest(self, ref): """Get the latest of a recipe""" assert ref.revision is None, "for_recipe_latest shouldn't receive RREV" - return self.base_url + _format_ref(routes.recipe_latest, ref) + return self.base_url + _format_ref(self.routes.recipe_latest, ref) - @staticmethod - def _for_package_file(pref, path): + def _for_package_file(self, pref, path, matrix_params): """url for getting a file from a package, with revisions""" assert pref.ref.revision is not None, "_for_package_file needs RREV" assert pref.revision is not None, "_for_package_file needs PREV" - return ClientV2Router._format_pref_path(routes.package_revision_file, pref, path) + return ClientV2Router._format_pref_path(self.routes.package_revision_file, pref, path, + matrix_params=matrix_params) - @staticmethod - def _for_package_files(pref): + def _for_package_files(self, pref): """url for getting the recipe list""" assert pref.revision is not None, "_for_package_files needs PREV" assert pref.ref.revision is not None, "_for_package_files needs RREV" - return _format_pref(routes.package_revision_files, pref) + return _format_pref(self.routes.package_revision_files, pref) - @staticmethod - def _for_recipe_file(ref, path): + def _for_recipe_file(self, ref, path, matrix_params): """url for a recipe file, with or without revisions""" assert ref.revision is not None, "for_recipe_file needs RREV" - return ClientV2Router._format_ref_path(routes.recipe_revision_file, ref, path) + return ClientV2Router._format_ref_path(self.routes.recipe_revision_file, ref, path, + matrix_params=matrix_params) - @staticmethod - def _for_recipe_files(ref): + def _for_recipe_files(self, ref): """url for getting the recipe list""" assert ref.revision is not None, "for_recipe_files needs RREV" - return _format_ref(routes.recipe_revision_files, ref) + return _format_ref(self.routes.recipe_revision_files, ref) - @staticmethod - def _for_recipe(ref): + def _for_recipe(self, ref): """url for a recipe with or without revisions (without rev, only for delete the root recipe, or v1)""" - return _format_ref(routes.recipe_revision, ref) + return _format_ref(self.routes.recipe_revision, ref) - @staticmethod - def _for_packages(ref): + def _for_packages(self, ref): """url for a recipe with or without revisions""" - return _format_ref(routes.packages_revision, ref) + return _format_ref(self.routes.packages_revision, ref) - @staticmethod - def _for_package(pref): + def _for_package(self, pref): """url for the package with or without revisions""" - return _format_pref(routes.package_revision, pref) + return _format_pref(self.routes.package_revision, pref) - @staticmethod - def _for_recipe_root(ref): - return _format_ref(routes.recipe, ref.copy_clear_rev()) + def _for_recipe_root(self, ref): + return _format_ref(self.routes.recipe, ref.copy_clear_rev()) - @staticmethod - def _for_package_root(pref): + def _for_package_root(self, pref): pref = pref.copy_with_revs(None, None) - return _format_pref(routes.package, pref) + return _format_pref(self.routes.package, pref) @staticmethod - def _format_ref_path(url, ref, path): + def _format_ref_path(url, ref, path, matrix_params): ret = url.format(name=ref.name, version=ref.version, username=ref.user or "_", - channel=ref.channel or "_", revision=ref.revision, path=path) + channel=ref.channel or "_", revision=ref.revision, path=path, + matrix_params=matrix_params or "") return ret @staticmethod - def _format_pref_path(url, pref, path): + def _format_pref_path(url, pref, path, matrix_params): ref = pref.ref return url.format(name=ref.name, version=ref.version, username=ref.user or "_", channel=ref.channel or "_", revision=ref.revision, package_id=pref.id, - p_revision=pref.revision, path=path) + p_revision=pref.revision, path=path, matrix_params=matrix_params or "") diff --git a/conans/client/rest/rest_client.py b/conans/client/rest/rest_client.py index 585046c03c9..0c617a2bb4e 100644 --- a/conans/client/rest/rest_client.py +++ b/conans/client/rest/rest_client.py @@ -1,4 +1,4 @@ -from conans import CHECKSUM_DEPLOY, REVISIONS, ONLY_V2, OAUTH_TOKEN +from conans import CHECKSUM_DEPLOY, REVISIONS, ONLY_V2, OAUTH_TOKEN, MATRIX_PARAMS from conans.client.rest.rest_client_v1 import RestV1Methods from conans.client.rest.rest_client_v2 import RestV2Methods from conans.errors import OnlyV2Available, AuthenticationException @@ -60,14 +60,16 @@ def _capable(self, capability): def _get_api(self): revisions = self._capable(REVISIONS) + matrix_params = self._capable(MATRIX_PARAMS) if self._revisions_enabled and revisions: checksum_deploy = self._capable(CHECKSUM_DEPLOY) return RestV2Methods(self._remote_url, self._token, self._custom_headers, self._output, self._requester, self._verify_ssl, self._artifacts_properties, - checksum_deploy) + checksum_deploy, matrix_params) else: return RestV1Methods(self._remote_url, self._token, self._custom_headers, self._output, - self._requester, self._verify_ssl, self._artifacts_properties) + self._requester, self._verify_ssl, self._artifacts_properties, + matrix_params) def get_recipe_manifest(self, ref): return self._get_api().get_recipe_manifest(ref) @@ -100,8 +102,7 @@ def get_package_path(self, pref, path): return self._get_api().get_package_path(pref, path) def upload_recipe(self, ref, files_to_upload, deleted, retry, retry_wait): - return self._get_api().upload_recipe(ref, files_to_upload, deleted, retry, - retry_wait) + return self._get_api().upload_recipe(ref, files_to_upload, deleted, retry, retry_wait) def upload_package(self, pref, files_to_upload, deleted, retry, retry_wait): return self._get_api().upload_package(pref, files_to_upload, deleted, retry, retry_wait) diff --git a/conans/client/rest/rest_client_common.py b/conans/client/rest/rest_client_common.py index 01b60e5e6b2..85623d72907 100644 --- a/conans/client/rest/rest_client_common.py +++ b/conans/client/rest/rest_client_common.py @@ -62,7 +62,7 @@ def inner(*argc, **argv): class RestCommonMethods(object): def __init__(self, remote_url, token, custom_headers, output, requester, verify_ssl, - artifacts_properties=None): + artifacts_properties=None, matrix_params=False): self.token = token self.remote_url = remote_url @@ -71,6 +71,7 @@ def __init__(self, remote_url, token, custom_headers, output, requester, verify_ self.requester = requester self.verify_ssl = verify_ssl self._artifacts_properties = artifacts_properties + self._matrix_params = matrix_params @property def auth(self): diff --git a/conans/client/rest/rest_client_v1.py b/conans/client/rest/rest_client_v1.py index aa3807156d3..24196c11b34 100644 --- a/conans/client/rest/rest_client_v1.py +++ b/conans/client/rest/rest_client_v1.py @@ -31,7 +31,8 @@ class RestV1Methods(RestCommonMethods): @property def router(self): - return ClientV1Router(self.remote_url.rstrip("/")) + return ClientV1Router(self.remote_url.rstrip("/"), self._artifacts_properties, + self._matrix_params) def _download_files(self, file_urls, quiet=False): """ @@ -123,6 +124,8 @@ def _upload_recipe(self, ref, files_to_upload, retry, retry_wait): file_sizes = {filename.replace("\\", "/"): os.stat(abs_path).st_size for filename, abs_path in files_to_upload.items()} urls = self._get_file_to_url_dict(url, data=file_sizes) + if self._matrix_params: + urls = self.router.add_matrix_params(urls) self._upload_files(urls, files_to_upload, self._output, retry, retry_wait) def _upload_package(self, pref, files_to_upload, retry, retry_wait): @@ -132,6 +135,8 @@ def _upload_package(self, pref, files_to_upload, retry, retry_wait): abs_path in files_to_upload.items()} logger.debug("Requesting upload urls...") urls = self._get_file_to_url_dict(url, data=file_sizes) + if self._matrix_params: + urls = self.router.add_matrix_params(urls) logger.debug("Requesting upload urls...Done!") self._upload_files(urls, files_to_upload, self._output, retry, retry_wait) @@ -146,9 +151,10 @@ def _upload_files(self, file_urls, files, output, retry, retry_wait): output.rewrite_line("Uploading %s" % filename) auth, dedup = self._file_server_capabilities(resource_url) try: + headers = self._artifacts_properties if not self._matrix_params else {} uploader.upload(resource_url, files[filename], auth=auth, dedup=dedup, retry=retry, retry_wait=retry_wait, - headers=self._artifacts_properties) + headers=headers) except Exception as exc: output.error("\nError uploading file: %s, '%s'" % (filename, exc)) failed.append(filename) diff --git a/conans/client/rest/rest_client_v2.py b/conans/client/rest/rest_client_v2.py index dadad0c20e1..263947beaea 100644 --- a/conans/client/rest/rest_client_v2.py +++ b/conans/client/rest/rest_client_v2.py @@ -19,15 +19,16 @@ class RestV2Methods(RestCommonMethods): def __init__(self, remote_url, token, custom_headers, output, requester, verify_ssl, - artifacts_properties=None, checksum_deploy=False): + artifacts_properties=None, checksum_deploy=False, matrix_params=False): super(RestV2Methods, self).__init__(remote_url, token, custom_headers, output, requester, - verify_ssl, artifacts_properties) + verify_ssl, artifacts_properties, matrix_params) self._checksum_deploy = checksum_deploy @property def router(self): - return ClientV2Router(self.remote_url.rstrip("/")) + return ClientV2Router(self.remote_url.rstrip("/"), self._artifacts_properties, + self._matrix_params) def _get_file_list_json(self, url): data = self.get_json(url) @@ -161,11 +162,11 @@ def _list_dir_contents(path, files): def _upload_recipe(self, ref, files_to_upload, retry, retry_wait): # Direct upload the recipe - urls = {fn: self.router.recipe_file(ref, fn) for fn in files_to_upload} + urls = {fn: self.router.recipe_file(ref, fn, add_matrix_params=True) for fn in files_to_upload} self._upload_files(files_to_upload, urls, retry, retry_wait) def _upload_package(self, pref, files_to_upload, retry, retry_wait): - urls = {fn: self.router.package_file(pref, fn) + urls = {fn: self.router.package_file(pref, fn, add_matrix_params=True) for fn in files_to_upload} self._upload_files(files_to_upload, urls, retry, retry_wait) @@ -180,10 +181,11 @@ def _upload_files(self, files, urls, retry, retry_wait): self._output.rewrite_line("Uploading %s" % filename) resource_url = urls[filename] try: + headers = self._artifacts_properties if not self._matrix_params else {} uploader.upload(resource_url, files[filename], auth=self.auth, dedup=self._checksum_deploy, retry=retry, retry_wait=retry_wait, - headers=self._artifacts_properties) + headers=headers) except (AuthenticationException, ForbiddenException): raise except Exception as exc: diff --git a/conans/model/rest_routes.py b/conans/model/rest_routes.py index 2fa7392ffa4..03c58d12953 100644 --- a/conans/model/rest_routes.py +++ b/conans/model/rest_routes.py @@ -1,9 +1,25 @@ -class RestRoutes(object): +class RestRoutesCommon(object): + ping = "ping" + common_search = "conans/search" + common_authenticate = "users/authenticate" + oauth_authenticate = "users/token" + common_check_credentials = "users/check_credentials" + + +class RestRoutes(RestRoutesCommon): + + def __init__(self, matrix_params=False): + if matrix_params: + self.base = 'conans{matrix_params}' + self.files_base = 'files{matrix_params}' + else: + self.base = 'conans' + self.files_base = 'files' @property def recipe(self): - return 'conans/{name}/{version}/{username}/{channel}' + return self.base + '/{name}/{version}/{username}/{channel}' @property def recipe_latest(self): @@ -69,7 +85,7 @@ def package_revision_file(self): # ONLY V1 @property def v1_updown_file(self): - return "files/{path}" + return "%s/{path}" % self.files_base @property def v1_recipe_digest(self): @@ -103,15 +119,6 @@ def v1_remove_recipe_files(self): def v1_remove_packages(self): return "%s/packages/delete" % self.recipe - # COMMON URLS - @property - def ping(self): - return "ping" - - @property - def common_search(self): - return "conans/search" - @property def common_search_packages(self): return "%s/search" % self.recipe @@ -120,14 +127,3 @@ def common_search_packages(self): def common_search_packages_revision(self): return "%s/search" % self.recipe_revision - @property - def common_authenticate(self): - return "users/authenticate" - - @property - def oauth_authenticate(self): - return "users/token" - - @property - def common_check_credentials(self): - return "users/check_credentials" diff --git a/conans/server/rest/controller/v1/conan.py b/conans/server/rest/controller/v1/conan.py index d55ed475d39..6e51d6c7caf 100644 --- a/conans/server/rest/controller/v1/conan.py +++ b/conans/server/rest/controller/v1/conan.py @@ -87,8 +87,7 @@ def get_conanfile_download_urls(name, version, username, channel, auth_user): return urls_norm @app.route(r.v1_package_download_urls, method=["GET"]) - def get_package_download_urls(name, version, username, channel, package_id, - auth_user): + def get_package_download_urls(name, version, username, channel, package_id, auth_user): """ Get a dict with all packages files and the download url for each one """ diff --git a/conans/server/rest/controller/v1/file_upload_download.py b/conans/server/rest/controller/v1/file_upload_download.py index e216d29e5f6..19dcf2d5ec6 100644 --- a/conans/server/rest/controller/v1/file_upload_download.py +++ b/conans/server/rest/controller/v1/file_upload_download.py @@ -44,20 +44,20 @@ class ConanFileUpload(FileUpload): FIXME: Review bottle.FileUpload and analyze possible security or general issues """ @cached_property def filename(self): - ''' Name of the file on the client file system, but normalized to ensure + """ Name of the file on the client file system, but normalized to ensure file system compatibility. An empty filename is returned as 'empty'. Only ASCII letters, digits, dashes, underscores and dots are allowed in the final filename. Accents are removed, if possible. Whitespace is replaced by a single dash. Leading or tailing dots or dashes are removed. The filename is limited to 255 characters. - ''' + """ fname = self.raw_filename if six.PY2: if not isinstance(fname, unicode): fname = fname.decode('utf8', 'ignore') fname = normalize('NFKD', fname).encode('ASCII', 'ignore').decode('ASCII') fname = os.path.basename(fname.replace('\\', os.path.sep)) -# fname = re.sub(r'[^a-zA-Z0-9-_.\s]', '', fname).strip() -# fname = re.sub(r'[-\s]+', '-', fname).strip('.-') + # fname = re.sub(r'[^a-zA-Z0-9-_.\s]', '', fname).strip() + # fname = re.sub(r'[-\s]+', '-', fname).strip('.-') return fname[:255] or 'empty' diff --git a/conans/server/store/disk_adapter.py b/conans/server/store/disk_adapter.py index 88dd495963f..70ee863f452 100644 --- a/conans/server/store/disk_adapter.py +++ b/conans/server/store/disk_adapter.py @@ -8,8 +8,8 @@ class ServerDiskAdapter(object): - '''Manage access to disk files with common methods required - for conan operations''' + """Manage access to disk files with common methods required + for conan operations""" def __init__(self, base_url, base_storage_path, updown_auth_manager): """ :param: base_url Base url for generate urls to download and upload operations""" @@ -21,10 +21,10 @@ def __init__(self, base_url, base_storage_path, updown_auth_manager): # ONLY USED BY APIV1 def get_download_urls(self, paths, user=None): - '''Get the urls for download the specified files using s3 signed request. + """Get the urls for download the specified files using s3 signed request. returns a dict with this structure: {"filepath": "http://..."} - paths is a list of path files ''' + paths is a list of path files """ assert isinstance(paths, list) ret = {} @@ -40,10 +40,10 @@ def get_download_urls(self, paths, user=None): # ONLY USED BY APIV1 def get_upload_urls(self, paths_sizes, user=None): - '''Get the urls for upload the specified files using s3 signed request. + """Get the urls for upload the specified files using s3 signed request. returns a dict with this structure: {"filepath": "http://..."} - paths_sizes is a dict of {path: size_in_bytes} ''' + paths_sizes is a dict of {path: size_in_bytes} """ assert isinstance(paths_sizes, dict) ret = {} for filepath, filesize in paths_sizes.items(): @@ -75,13 +75,13 @@ def get_file_list(self, absolute_path="", files_subset=None): return abs_paths def delete_folder(self, path): - '''Delete folder from disk. Path already contains base dir''' + """Delete folder from disk. Path already contains base dir""" if not path_exists(path, self._store_folder): raise NotFoundException("") rmdir(path) def delete_file(self, path): - '''Delete files from bucket. Path already contains base dir''' + """Delete files from bucket. Path already contains base dir""" if not path_exists(path, self._store_folder): raise NotFoundException("") os.remove(path) diff --git a/conans/test/functional/remote/auth_bearer_test.py b/conans/test/functional/remote/auth_bearer_test.py index 05e9966be5c..62835c00492 100644 --- a/conans/test/functional/remote/auth_bearer_test.py +++ b/conans/test/functional/remote/auth_bearer_test.py @@ -1,9 +1,11 @@ import unittest from bottle import request +from parameterized.parameterized import parameterized from conans.test.utils.tools import TestClient, TestServer from conans.util.env_reader import get_env +from conans.util.files import save conanfile = """ from conans import ConanFile @@ -15,8 +17,8 @@ class OpenSSLConan(ConanFile): class AuthorizationHeaderSpy(object): - ''' Generic plugin to handle Authorization header. Must be extended and implement - some abstract methods in subclasses''' + """ Generic plugin to handle Authorization header. Must be extended and implement + some abstract methods in subclasses""" name = 'authorizationheaderspy' api = 2 @@ -37,9 +39,9 @@ class ReturnHandlerPlugin(object): api = 2 def apply(self, callback, _): - '''Apply plugin''' + """Apply plugin""" def wrapper(*args, **kwargs): - '''Capture possible exceptions to manage the return''' + """Capture possible exceptions to manage the return""" result = callback(*args, **kwargs) if isinstance(result, dict): for k, v in result.items(): @@ -50,11 +52,14 @@ def wrapper(*args, **kwargs): class AuthorizeBearerTest(unittest.TestCase): - def basic_test(self): + @parameterized.expand([(False, ), (True, )]) + def basic_test(self, artifacts_properties): auth = AuthorizationHeaderSpy() server = TestServer(plugins=[auth]) servers = {"default": server} client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) + if artifacts_properties: + save(client.cache.artifacts_properties_path, "key=value") client.save({"conanfile.py": conanfile}) client.run("export . lasote/stable") errors = client.run("upload Hello/0.1@lasote/stable") @@ -83,13 +88,16 @@ def basic_test(self): if auth_type: self.assertIn(auth_type, real_call[1]) + @parameterized.expand([(False,), (True,)]) @unittest.skipIf(get_env("TESTING_REVISIONS_ENABLED", False), "ApiV1 test") - def no_signature_test(self): + def no_signature_test(self, artifacts_properties): auth = AuthorizationHeaderSpy() retur = ReturnHandlerPlugin() server = TestServer(plugins=[auth, retur]) servers = {"default": server} client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) + if artifacts_properties: + save(client.cache.artifacts_properties_path, "key=value") client.save({"conanfile.py": conanfile}) client.run("export . lasote/stable") # Upload will fail, as conan_server is expecting a signed URL diff --git a/conans/test/functional/remote/put_properties_test.py b/conans/test/functional/remote/put_properties_test.py index 3bf43b30eb1..0bb75e30388 100644 --- a/conans/test/functional/remote/put_properties_test.py +++ b/conans/test/functional/remote/put_properties_test.py @@ -1,6 +1,10 @@ import os +import re import unittest +from six.moves.urllib.parse import unquote + +from conans import MATRIX_PARAMS from conans.test.utils.cpp_test_files import cpp_hello_conan_files from conans.test.utils.tools import TestClient, TestRequester, TestServer from conans.util.files import save @@ -8,38 +12,93 @@ class PutPropertiesTest(unittest.TestCase): - def setUp(self): - test_server = TestServer() - self.servers = {"default": test_server} - def create_empty_property_file_test(self): + test_server = TestServer() + servers = {"default": test_server} files = cpp_hello_conan_files("Hello0", "0.1", build=False) - client = TestClient(servers=self.servers, users={"default": [("lasote", "mypass")]}) + client = TestClient(servers=servers, users={"default": [("lasote", "mypass")]}) client.save(files) client.run("export . lasote/stable") props_file = client.cache.artifacts_properties_path self.assertTrue(os.path.exists(props_file)) def put_properties_test(self): - wanted_vars = {"MyHeader1": "MyHeaderValue1;MyHeaderValue2", "Other": "Value"} + test_server = TestServer() + servers = {"default": test_server} + + wanted_vars = {"key0": "value", + "key1": "with space", + "key2": "with/slash", + "key3": "with.dot", + "key4": "with;semicolon", + "key5": "with~virgul", + "key6": "with#hash" + } - class RequesterCheckHeaders(TestRequester): - def put(self, url, **kwargs): + class RequesterCheckArtifactProperties(TestRequester): + def put(self_requester, url, **kwargs): + # Check headers for name, value in wanted_vars.items(): value1 = kwargs["headers"][name] - if value1 != value: - raise Exception() - return super(RequesterCheckHeaders, self).put(url, **kwargs) + self.assertEqual(value1, value) - self.client = TestClient(requester_class=RequesterCheckHeaders, servers=self.servers, - users={"default": [("lasote", "mypass")]}) - _create_property_files(self.client, wanted_vars) + # Check matrix params + self.assertNotIn(';', url) + mp = re.match(r"^[^;\s]+;(?P[^/]+)/.*", url) + self.assertFalse(mp) + + return super(RequesterCheckArtifactProperties, self_requester).put(url, **kwargs) + + client = TestClient(requester_class=RequesterCheckArtifactProperties, + servers=servers, users={"default": [("lasote", "mypass")]}) + _create_property_files(client, wanted_vars) files = cpp_hello_conan_files("Hello0", "0.1", build=False) - self.client.save(files) - self.client.run("export . lasote/stable") - self.client.run("upload Hello0/0.1@lasote/stable -c") + client.save(files) + client.run("export . lasote/stable") + client.run("upload Hello0/0.1@lasote/stable -c") + + def matrix_params_test(self): + test_server = TestServer(server_capabilities=[MATRIX_PARAMS, ]) + servers = {"default": test_server} + + wanted_vars = {"key0": "value", + "key1": "with space", + "key2": "with/slash", + "key3": "with.dot", + "key4": "with;semicolon", + "key5": "with~virgul", + "key6": "with#hash" + } + + class RequesterCheckArtifactProperties(TestRequester): + def put(self_requester, url, **kwargs): + # Check headers + self.assertListEqual(list(kwargs["headers"].keys()), + ["X-Checksum-Sha1", "User-Agent"]) + + # Check matrix params + m = re.match(r"^[^;\s]+;(?P[^/]+)/.*", url) + mp = m.group("matrix_params") + values = [it.split("=") for it in mp.split(';')] + values = {key: unquote(value) for (key, value) in values} + for name, value in wanted_vars.items(): + value1 = values[name] + self.assertEqual(value1, value) + + slice_start, slice_end = m.span(1) + url = url[: slice_start-1] + url[slice_end:] # matrix params are not implemented for conan-server + return super(RequesterCheckArtifactProperties, self_requester).put(url, **kwargs) + + client = TestClient(requester_class=RequesterCheckArtifactProperties, + servers=servers, users={"default": [("lasote", "mypass")]}) + _create_property_files(client, wanted_vars) + + files = cpp_hello_conan_files("Hello0", "0.1", build=False) + client.save(files) + client.run("export . lasote/stable") + client.run("upload Hello0/0.1@lasote/stable -c") def _create_property_files(client, values): From 23488e6bd49be8750d12be4f08f618da7b5a0be3 Mon Sep 17 00:00:00 2001 From: Melanie <58263267+CMelanie@users.noreply.github.com> Date: Tue, 3 Dec 2019 09:06:02 +0100 Subject: [PATCH 08/12] Add res dir to the variables when using cmake_find_package. (issue 3722) (#6166) --- conans/client/generators/cmake_find_package_common.py | 1 + 1 file changed, 1 insertion(+) diff --git a/conans/client/generators/cmake_find_package_common.py b/conans/client/generators/cmake_find_package_common.py index 6c6a0b0a748..8134c70e43b 100644 --- a/conans/client/generators/cmake_find_package_common.py +++ b/conans/client/generators/cmake_find_package_common.py @@ -1,6 +1,7 @@ target_template = """ set({name}_INCLUDE_DIRS{build_type_suffix} {deps.include_paths}) set({name}_INCLUDES{build_type_suffix} {deps.include_paths}) +set({name}_RES_DIRS{build_type_suffix} {deps.res_paths}) set({name}_DEFINITIONS{build_type_suffix} {deps.defines}) set({name}_LINKER_FLAGS{build_type_suffix}_LIST "{deps.sharedlinkflags_list}" "{deps.exelinkflags_list}") set({name}_COMPILE_DEFINITIONS{build_type_suffix} {deps.compile_definitions}) From 48bc42ce116c4fe79d80cb534b66d9967d6cf7db Mon Sep 17 00:00:00 2001 From: James Date: Tue, 3 Dec 2019 09:32:35 +0100 Subject: [PATCH 09/12] skip downloading call to to_file_bytes (#6142) --- conans/client/rest/uploader_downloader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/conans/client/rest/uploader_downloader.py b/conans/client/rest/uploader_downloader.py index 19ee7f2346d..70fe725f8e9 100644 --- a/conans/client/rest/uploader_downloader.py +++ b/conans/client/rest/uploader_downloader.py @@ -3,6 +3,8 @@ import time from copy import copy +import six + from conans.util import progress_bar from conans.client.rest import response_to_str from conans.errors import AuthenticationException, ConanConnectionError, ConanException, \ @@ -173,7 +175,9 @@ def write_chunks(chunks, path): mkdir(os.path.dirname(path)) with open(path, 'wb') as file_handler: for chunk in chunks: - file_handler.write(to_file_bytes(chunk)) + assert ((six.PY3 and isinstance(chunk, bytes)) or + (six.PY2 and isinstance(chunk, str)), "download chunk not binary") + file_handler.write(chunk) downloaded_size += len(chunk) else: ret_data = bytearray() From 3ba3195bf9a5649f343fdcc684f577cdd2cf4d2d Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 3 Dec 2019 11:12:44 +0100 Subject: [PATCH 10/12] warn if env-vars defined --- conans/client/cmd/export.py | 3 ++ conans/client/conf/__init__.py | 2 + conans/test/functional/command/export_test.py | 53 ++++++++++--------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py index 82fe871d753..a76567bab59 100644 --- a/conans/client/cmd/export.py +++ b/conans/client/cmd/export.py @@ -101,6 +101,9 @@ def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") + if os.getenv("CONAN_PYLINT_WERR") or os.getenv("CONAN_PYLINTRC"): + output.warn("CONAN_PYLINT_WERR and CONAN_PYLINTRC no longer have effect. Recipe linting " + "has been moved to a hook. Check https://github.com/conan-io/hooks") output = conanfile.output # Get previous digest diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 1582f34bdd7..9c2afd0e4e3 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -205,7 +205,9 @@ def env_vars(self): "CONAN_COMPRESSION_LEVEL": self._env_c("general.compression_level", "CONAN_COMPRESSION_LEVEL", "9"), "CONAN_NON_INTERACTIVE": self._env_c("general.non_interactive", "CONAN_NON_INTERACTIVE", "False"), "CONAN_SKIP_BROKEN_SYMLINKS_CHECK": self._env_c("general.skip_broken_symlinks_check", "CONAN_SKIP_BROKEN_SYMLINKS_CHECK", "False"), + "CONAN_PYLINTRC": self._env_c("general.pylintrc", "CONAN_PYLINTRC", None), "CONAN_CACHE_NO_LOCKS": self._env_c("general.cache_no_locks", "CONAN_CACHE_NO_LOCKS", "False"), + "CONAN_PYLINT_WERR": self._env_c("general.pylint_werr", "CONAN_PYLINT_WERR", None), "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_SYSREQUIRES_MODE": self._env_c("general.sysrequires_mode", "CONAN_SYSREQUIRES_MODE", "enabled"), "CONAN_REQUEST_TIMEOUT": self._env_c("general.request_timeout", "CONAN_REQUEST_TIMEOUT", None), diff --git a/conans/test/functional/command/export_test.py b/conans/test/functional/command/export_test.py index 6a1d83a9790..b1c50572039 100644 --- a/conans/test/functional/command/export_test.py +++ b/conans/test/functional/command/export_test.py @@ -35,24 +35,15 @@ class TestConan(ConanFile): def export_without_full_reference_test(self): client = TestClient() - client.save({"conanfile.py": """from conans import ConanFile -class MyPkg(ConanFile): - pass -"""}) + client.save({"conanfile.py": GenConanfile()}) client.run("export . lasote/stable", assert_error=True) self.assertIn("conanfile didn't specify name", client.out) - client.save({"conanfile.py": """from conans import ConanFile -class MyPkg(ConanFile): - name="Lib" -"""}) + client.save({"conanfile.py": GenConanfile().with_name("Lib")}) client.run("export . lasote/stable", assert_error=True) self.assertIn("conanfile didn't specify version", client.out) - client.save({"conanfile.py": """from conans import ConanFile -class MyPkg(ConanFile): - pass -"""}) + client.save({"conanfile.py": GenConanfile()}) client.run("export . lib/1.0@lasote/channel") self.assertIn("lib/1.0@lasote/channel: A new conanfile.py version was exported", client.out) @@ -68,14 +59,14 @@ class MyPkg(ConanFile): def test_export_read_only(self): client = TestClient() - conanfile = """ -from conans import ConanFile -class TestConan(ConanFile): - name = "Hello" - version = "1.2" - exports = "file1.txt" - exports_sources = "file2.txt" -""" + conanfile = textwrap.dedent(""" + from conans import ConanFile + class TestConan(ConanFile): + name = "Hello" + version = "1.2" + exports = "file1.txt" + exports_sources = "file2.txt" + """) ref = ConanFileReference.loads("Hello/1.2@lasote/stable") export_path = client.cache.package_layout(ref).export() export_src_path = client.cache.package_layout(ref).export_sources() @@ -86,8 +77,8 @@ class TestConan(ConanFile): client.save(files) mode1 = os.stat(os.path.join(client.current_folder, "file1.txt")).st_mode mode2 = os.stat(os.path.join(client.current_folder, "file2.txt")).st_mode - os.chmod(os.path.join(client.current_folder, "file1.txt"), mode1 &~ stat.S_IWRITE) - os.chmod(os.path.join(client.current_folder, "file2.txt"), mode2 &~ stat.S_IWRITE) + os.chmod(os.path.join(client.current_folder, "file1.txt"), mode1 & ~stat.S_IWRITE) + os.chmod(os.path.join(client.current_folder, "file2.txt"), mode2 & ~stat.S_IWRITE) client.run("export . lasote/stable") self.assertEqual(load(os.path.join(export_path, "file1.txt")), "") @@ -114,8 +105,8 @@ class TestConan(ConanFile): "file1.txt": "", "file2.txt": ""} client.save(files) - os.chmod(os.path.join(client.current_folder, "file1.txt"), mode1 &~ stat.S_IWRITE) - os.chmod(os.path.join(client.current_folder, "file2.txt"), mode2 &~ stat.S_IWRITE) + os.chmod(os.path.join(client.current_folder, "file1.txt"), mode1 & ~stat.S_IWRITE) + os.chmod(os.path.join(client.current_folder, "file2.txt"), mode2 & ~stat.S_IWRITE) client.run("export . lasote/stable") self.assertEqual(load(os.path.join(export_path, "file1.txt")), "") self.assertEqual(load(os.path.join(export_src_path, "file2.txt")), "") @@ -500,3 +491,17 @@ def export_conflict_no_user_channel_test(self): self.assertIn("pkg/0.1: A new conanfile.py version was exported", client.out) client.run('export . Pkg/0.1@', assert_error=True) self.assertIn("ERROR: Cannot export package with same name but different case", client.out) + + def test_warning_as_errors(self): + client = TestClient() + client.save({CONANFILE: GenConanfile()}) + client.run("config set general.pylint_werr=True") + client.run("export . pkg/0.1@user/stable") + self.assertIn("WARN: CONAN_PYLINT_WERR and CONAN_PYLINTRC no longer have effect", + client.out) + client = TestClient() + client.save({CONANFILE: GenConanfile()}) + client.run("config set general.pylintrc=something") + client.run("export . pkg/0.1@user/stable") + self.assertIn("WARN: CONAN_PYLINT_WERR and CONAN_PYLINTRC no longer have effect", + client.out) From 9dc4b1991b9c2988223a1530e372d57509d5a885 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 3 Dec 2019 11:31:46 +0100 Subject: [PATCH 11/12] removed warning and env-vars --- conans/client/cmd/export.py | 3 --- conans/client/conf/__init__.py | 2 -- conans/test/functional/command/export_test.py | 14 -------------- 3 files changed, 19 deletions(-) diff --git a/conans/client/cmd/export.py b/conans/client/cmd/export.py index a76567bab59..82fe871d753 100644 --- a/conans/client/cmd/export.py +++ b/conans/client/cmd/export.py @@ -101,9 +101,6 @@ def cmd_export(app, conanfile_path, name, version, user, channel, keep_source, logger.debug("EXPORT: %s" % conanfile_path) output.highlight("Exporting package recipe") - if os.getenv("CONAN_PYLINT_WERR") or os.getenv("CONAN_PYLINTRC"): - output.warn("CONAN_PYLINT_WERR and CONAN_PYLINTRC no longer have effect. Recipe linting " - "has been moved to a hook. Check https://github.com/conan-io/hooks") output = conanfile.output # Get previous digest diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 9c2afd0e4e3..1582f34bdd7 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -205,9 +205,7 @@ def env_vars(self): "CONAN_COMPRESSION_LEVEL": self._env_c("general.compression_level", "CONAN_COMPRESSION_LEVEL", "9"), "CONAN_NON_INTERACTIVE": self._env_c("general.non_interactive", "CONAN_NON_INTERACTIVE", "False"), "CONAN_SKIP_BROKEN_SYMLINKS_CHECK": self._env_c("general.skip_broken_symlinks_check", "CONAN_SKIP_BROKEN_SYMLINKS_CHECK", "False"), - "CONAN_PYLINTRC": self._env_c("general.pylintrc", "CONAN_PYLINTRC", None), "CONAN_CACHE_NO_LOCKS": self._env_c("general.cache_no_locks", "CONAN_CACHE_NO_LOCKS", "False"), - "CONAN_PYLINT_WERR": self._env_c("general.pylint_werr", "CONAN_PYLINT_WERR", None), "CONAN_SYSREQUIRES_SUDO": self._env_c("general.sysrequires_sudo", "CONAN_SYSREQUIRES_SUDO", "False"), "CONAN_SYSREQUIRES_MODE": self._env_c("general.sysrequires_mode", "CONAN_SYSREQUIRES_MODE", "enabled"), "CONAN_REQUEST_TIMEOUT": self._env_c("general.request_timeout", "CONAN_REQUEST_TIMEOUT", None), diff --git a/conans/test/functional/command/export_test.py b/conans/test/functional/command/export_test.py index b1c50572039..297bb879c97 100644 --- a/conans/test/functional/command/export_test.py +++ b/conans/test/functional/command/export_test.py @@ -491,17 +491,3 @@ def export_conflict_no_user_channel_test(self): self.assertIn("pkg/0.1: A new conanfile.py version was exported", client.out) client.run('export . Pkg/0.1@', assert_error=True) self.assertIn("ERROR: Cannot export package with same name but different case", client.out) - - def test_warning_as_errors(self): - client = TestClient() - client.save({CONANFILE: GenConanfile()}) - client.run("config set general.pylint_werr=True") - client.run("export . pkg/0.1@user/stable") - self.assertIn("WARN: CONAN_PYLINT_WERR and CONAN_PYLINTRC no longer have effect", - client.out) - client = TestClient() - client.save({CONANFILE: GenConanfile()}) - client.run("config set general.pylintrc=something") - client.run("export . pkg/0.1@user/stable") - self.assertIn("WARN: CONAN_PYLINT_WERR and CONAN_PYLINTRC no longer have effect", - client.out) From 26090294ae42b68494353adb9123a02e5e6c67c1 Mon Sep 17 00:00:00 2001 From: memsharded Date: Tue, 3 Dec 2019 11:51:59 +0100 Subject: [PATCH 12/12] remove another env-var --- conans/client/conf/__init__.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/conans/client/conf/__init__.py b/conans/client/conf/__init__.py index 1582f34bdd7..1016efdea5e 100644 --- a/conans/client/conf/__init__.py +++ b/conans/client/conf/__init__.py @@ -123,7 +123,6 @@ # verbose_traceback = False # environment CONAN_VERBOSE_TRACEBACK # error_on_override = False # environment CONAN_ERROR_ON_OVERRIDE # bash_path = "" # environment CONAN_BASH_PATH (only windows) -# recipe_linter = False # environment CONAN_RECIPE_LINTER # read_only_cache = True # environment CONAN_READ_ONLY_CACHE # cache_no_locks = True # environment CONAN_CACHE_NO_LOCKS # user_home_short = your_path # environment CONAN_USER_HOME_SHORT @@ -212,7 +211,6 @@ def env_vars(self): "CONAN_RETRY": self._env_c("general.retry", "CONAN_RETRY", None), "CONAN_RETRY_WAIT": self._env_c("general.retry_wait", "CONAN_RETRY_WAIT", None), "CONAN_VS_INSTALLATION_PREFERENCE": self._env_c("general.vs_installation_preference", "CONAN_VS_INSTALLATION_PREFERENCE", None), - "CONAN_RECIPE_LINTER": self._env_c("general.recipe_linter", "CONAN_RECIPE_LINTER", "True"), "CONAN_CPU_COUNT": self._env_c("general.cpu_count", "CONAN_CPU_COUNT", None), "CONAN_READ_ONLY_CACHE": self._env_c("general.read_only_cache", "CONAN_READ_ONLY_CACHE", None), "CONAN_USER_HOME_SHORT": self._env_c("general.user_home_short", "CONAN_USER_HOME_SHORT", None),