diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index e54a9c7779c..ccbd7c2c945 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -128,8 +128,8 @@ cache: export PYTHONPATH="$IDF_PATH/tools:$IDF_PATH/tools/esp_app_trace:$IDF_PATH/components/partition_table:$IDF_PATH/tools/ci/python_packages:$PYTHONPATH" .setup_tools_and_idf_python_venv: &setup_tools_and_idf_python_venv | - # must use after setup_tools_except_target_test - # otherwise the export.sh won't work properly + # Since the version 3.21 CMake passes source files and include dirs to ninja using absolute paths. + $IDF_PATH/tools/idf_tools.py --non-interactive install cmake # download constraint file for dev if [[ -n "$CI_PYTHON_CONSTRAINT_BRANCH" ]]; then diff --git a/tools/idf_tools.py b/tools/idf_tools.py index bd4827546ad..3b14eab8877 100755 --- a/tools/idf_tools.py +++ b/tools/idf_tools.py @@ -1,7 +1,7 @@ #!/usr/bin/env python # coding=utf-8 # -# SPDX-FileCopyrightText: 2019-2022 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD # # SPDX-License-Identifier: Apache-2.0 # @@ -1678,6 +1678,94 @@ def action_check(args): # type: ignore raise SystemExit(1) +# The following functions are used in process_tool which is a part of the action_export. +def handle_recommended_version_to_use( + tool, + tool_name, + version_to_use, + prefer_system_hint, +): # type: (IDFTool, str, str, str) -> Tuple[list, dict] + tool_export_paths = tool.get_export_paths(version_to_use) + tool_export_vars = tool.get_export_vars(version_to_use) + if tool.version_in_path and tool.version_in_path not in tool.versions: + info('Not using an unsupported version of tool {} found in PATH: {}.'.format( + tool.name, tool.version_in_path) + prefer_system_hint, f=sys.stderr) + return tool_export_paths, tool_export_vars + + +def handle_supported_or_deprecated_version(tool, tool_name): # type: (IDFTool, str) -> None + version_obj: IDFToolVersion = tool.versions[tool.version_in_path] # type: ignore + if version_obj.status == IDFToolVersion.STATUS_SUPPORTED: + info('Using a supported version of tool {} found in PATH: {}.'.format(tool_name, tool.version_in_path), + f=sys.stderr) + info('However the recommended version is {}.'.format(tool.get_recommended_version()), + f=sys.stderr) + elif version_obj.status == IDFToolVersion.STATUS_DEPRECATED: + warn('using a deprecated version of tool {} found in PATH: {}'.format(tool_name, tool.version_in_path)) + + +def handle_missing_versions( + tool, + tool_name, + install_cmd, + prefer_system_hint +): # type: (IDFTool, str, str, str) -> None + fatal('tool {} has no installed versions. Please run \'{}\' to install it.'.format( + tool.name, install_cmd)) + if tool.version_in_path and tool.version_in_path not in tool.versions: + info('An unsupported version of tool {} was found in PATH: {}. '.format(tool_name, tool.version_in_path) + + prefer_system_hint, f=sys.stderr) + + +def process_tool( + tool, + tool_name, + args, + install_cmd, + prefer_system_hint +): # type: (IDFTool, str, argparse.Namespace, str, str) -> Tuple[list, dict, bool] + tool_found: bool = True + tool_export_paths: List[str] = [] + tool_export_vars: Dict[str, str] = {} + + tool.find_installed_versions() + recommended_version_to_use = tool.get_preferred_installed_version() + + if not tool.is_executable and recommended_version_to_use: + tool_export_vars = tool.get_export_vars(recommended_version_to_use) + return tool_export_paths, tool_export_vars, tool_found + + if recommended_version_to_use and not args.prefer_system: + tool_export_paths, tool_export_vars = handle_recommended_version_to_use( + tool, tool_name, recommended_version_to_use, prefer_system_hint + ) + return tool_export_paths, tool_export_vars, tool_found + + if tool.version_in_path: + if tool.version_in_path not in tool.versions: + # unsupported version + if args.prefer_system: # type: ignore + warn('using an unsupported version of tool {} found in PATH: {}'.format( + tool.name, tool.version_in_path)) + return tool_export_paths, tool_export_vars, tool_found + else: + # unsupported version in path + pass + else: + # supported/deprecated version in PATH, use it + handle_supported_or_deprecated_version(tool, tool_name) + return tool_export_paths, tool_export_vars, tool_found + + if not tool.versions_installed: + if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: + handle_missing_versions(tool, tool_name, install_cmd, prefer_system_hint) + tool_found = False + # If a tool found, but it is optional and does not have versions installed, use whatever is in PATH. + return tool_export_paths, tool_export_vars, tool_found + + return tool_export_paths, tool_export_vars, tool_found + + def action_export(args): # type: ignore if args.deactivate and different_idf_detected(): deactivate_statement(args) @@ -1697,58 +1785,10 @@ def action_export(args): # type: ignore for name, tool in tools_info.items(): if tool.get_install_type() == IDFTool.INSTALL_NEVER: continue - tool.find_installed_versions() - version_to_use = tool.get_preferred_installed_version() - - if not tool.is_executable and version_to_use: - tool_export_vars = tool.get_export_vars(version_to_use) - export_vars = {**export_vars, **tool_export_vars} - continue - - if tool.version_in_path: - if tool.version_in_path not in tool.versions: - # unsupported version - if args.prefer_system: # type: ignore - warn('using an unsupported version of tool {} found in PATH: {}'.format( - tool.name, tool.version_in_path)) - continue - else: - # unsupported version in path - pass - else: - # supported/deprecated version in PATH, use it - version_obj = tool.versions[tool.version_in_path] - if version_obj.status == IDFToolVersion.STATUS_SUPPORTED: - info('Using a supported version of tool {} found in PATH: {}.'.format(name, tool.version_in_path), - f=sys.stderr) - info('However the recommended version is {}.'.format(tool.get_recommended_version()), - f=sys.stderr) - elif version_obj.status == IDFToolVersion.STATUS_DEPRECATED: - warn('using a deprecated version of tool {} found in PATH: {}'.format(name, tool.version_in_path)) - continue - - if not tool.versions_installed: - if tool.get_install_type() == IDFTool.INSTALL_ALWAYS: - all_tools_found = False - fatal('tool {} has no installed versions. Please run \'{}\' to install it.'.format( - tool.name, install_cmd)) - if tool.version_in_path and tool.version_in_path not in tool.versions: - info('An unsupported version of tool {} was found in PATH: {}. '.format(name, tool.version_in_path) + - prefer_system_hint, f=sys.stderr) - continue - else: - # tool is optional, and does not have versions installed - # use whatever is available in PATH - continue - - if tool.version_in_path and tool.version_in_path not in tool.versions: - info('Not using an unsupported version of tool {} found in PATH: {}.'.format( - tool.name, tool.version_in_path) + prefer_system_hint, f=sys.stderr) - - export_paths = tool.get_export_paths(version_to_use) - if export_paths: - paths_to_export += export_paths - tool_export_vars = tool.get_export_vars(version_to_use) + tool_export_paths, tool_export_vars, tool_found = process_tool(tool, name, args, install_cmd, prefer_system_hint) + if not tool_found: + all_tools_found = False + paths_to_export += tool_export_paths export_vars = {**export_vars, **tool_export_vars} current_path = os.getenv('PATH') diff --git a/tools/test_idf_tools/test_idf_tools.py b/tools/test_idf_tools/test_idf_tools.py index e034a01b1e6..988efdbbbf4 100755 --- a/tools/test_idf_tools/test_idf_tools.py +++ b/tools/test_idf_tools/test_idf_tools.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# SPDX-FileCopyrightText: 2019-2021 Espressif Systems (Shanghai) CO LTD +# SPDX-FileCopyrightText: 2019-2023 Espressif Systems (Shanghai) CO LTD # SPDX-License-Identifier: Apache-2.0 import json @@ -90,6 +90,10 @@ class TestUsage(unittest.TestCase): @classmethod def setUpClass(cls): + with open(os.path.join(os.getenv('IDF_PATH'), 'tools/tools.json'), 'r') as json_file: + tools_dict = json.load(json_file) + cls.tools_dict = tools_dict + old_tools_dir = os.environ.get('IDF_TOOLS_PATH') or os.path.expanduser(idf_tools.IDF_TOOLS_PATH_DEFAULT) mirror_prefix_map = None @@ -496,6 +500,57 @@ def test_deactivate(self): self.assertTrue(os.path.isfile(deactivate_file), 'File {} was not found. '.format(deactivate_file)) self.assertNotEqual(os.stat(self.idf_env_json).st_size, 0, 'File {} is empty. '.format(deactivate_file)) + def test_export_recommended_version(self): + always_install_and_recommended_tools = [] + for tool in self.tools_dict['tools']: + if tool['install'] != 'always': + continue + for version in tool['versions']: + if version['status'] != 'recommended': + continue + always_install_and_recommended_tools.append({ + 'name': tool['name'], + 'version': version['name'] + }) + self.run_idf_tools_with_action(['install']) + output = self.run_idf_tools_with_action(['export']) + + for tool in always_install_and_recommended_tools: + self.assertIn(f"{tool['name']}/{tool['version']}", output) + + def test_export_recommended_version_cmake(self): + tool_to_test = 'cmake' + tool_version = '' + for tool in self.tools_dict['tools']: + if tool['name'] != tool_to_test: + continue + for version in tool['versions']: + if version['status'] == 'recommended': + tool_version = version['name'] + break + + self.run_idf_tools_with_action(['install']) + self.run_idf_tools_with_action(['install', tool_to_test]) + output = self.run_idf_tools_with_action(['export']) + + self.assertIn(f'{tool_to_test}/{tool_version}', output) + + def test_export_prefer_system_cmake(self): + tool_to_test = 'cmake' + self.run_idf_tools_with_action(['install']) + self.run_idf_tools_with_action(['install', tool_to_test]) + # cmake is installed via apt + output = self.run_idf_tools_with_action(['export', '--prefer-system']) + + self.assertNotIn(tool_to_test, output) + + def test_export_supported_version_cmake(self): + tool_to_test = 'cmake' + self.run_idf_tools_with_action(['install']) + output = self.run_idf_tools_with_action(['export']) + + self.assertNotIn(tool_to_test, output) + class TestMaintainer(unittest.TestCase):