Merge branch 'feature/add_qemu_to_tools-json_v5.1' into 'release/v5.1'

Add QEMU to tools.json (v5.1)

See merge request espressif/esp-idf!26407

.gitlab/ci/host-test.yml

@@ -135,10 +135,10 @@ test_idf_tools:
     entrypoint: [""]  # use system python3. no extra pip package installed
   script:
     # Tools must be downloaded for testing
-    - python3 ${IDF_PATH}/tools/idf_tools.py download
+    - python3 ${IDF_PATH}/tools/idf_tools.py download required qemu-riscv32 qemu-xtensa
     - cd ${IDF_PATH}/tools/test_idf_tools
     - python3 -m pip install jsonschema
-    - python3 ./test_idf_tools.py
+    - python3 ./test_idf_tools.py -v
     - python3 ./test_idf_tools_python_env.py
 
 .test_efuse_table_on_host_template:

docs/en/api-guides/tools/idf-tools-notes.inc

@@ -1,5 +1,7 @@
 .. This file gets included from auto-generated part of idf-tools.rst.
 .. Comments "tool-NAME-notes" act as delimiters.
+..
+.. This is a padding to have the same line numbers as zh_CN version>
 
 
 .. tool-xtensa-esp-elf-gdb-notes
@@ -77,6 +79,18 @@ On Linux and macOS, it is recommended to install ninja using the OS-specific pac
 .. tool-esp-rom-elfs-notes
 
 
+---
+
+.. tool-qemu-xtensa-notes
+
+Some ESP-specific instructions for running QEMU for Xtensa chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32/README.md
+
+---
+
+.. tool-qemu-riscv32-notes
+
+Some ESP-specific instructions for running QEMU for RISC-V chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32c3/README.md
+
 ---
 
 .. tool-idf-python-notes

docs/zh_CN/api-guides/tools/idf-tools-notes.inc

@@ -46,7 +46,7 @@
 
 .. tool-cmake-notes
 
-On Linux and macOS, it is recommended to install CMake using the OS package manager. However, for convenience it is possible to install CMake using idf_tools.py along with the other tools.
+On Linux and macOS, it is recommended to install CMake using the OS-specific package manager (like apt, yum, brew, etc.). However, for convenience it is possible to install CMake using idf_tools.py along with the other tools.
 
 ---
 
@@ -57,7 +57,7 @@ On Linux and macOS, it is recommended to install CMake using the OS package mana
 
 .. tool-ninja-notes
 
-On Linux and macOS, it is recommended to install ninja using the OS package manager. However, for convenience it is possible to install ninja using idf_tools.py along with the other tools.
+On Linux and macOS, it is recommended to install ninja using the OS-specific package manager (like apt, yum, brew, etc.). However, for convenience it is possible to install ninja using idf_tools.py along with the other tools.
 
 ---
 
@@ -79,6 +79,18 @@ On Linux and macOS, it is recommended to install ninja using the OS package mana
 .. tool-esp-rom-elfs-notes
 
 
+---
+
+.. tool-qemu-xtensa-notes
+
+Some ESP-specific instructions for running QEMU for Xtensa chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32/README.md
+
+---
+
+.. tool-qemu-riscv32-notes
+
+Some ESP-specific instructions for running QEMU for RISC-V chips are here: https://github.com/espressif/esp-toolchain-docs/blob/main/qemu/esp32c3/README.md
+
 ---
 
 .. tool-idf-python-notes

tools/docker/Dockerfile

@@ -7,6 +7,7 @@ RUN : \
   && apt-get install -y \
     apt-utils \
     bison \
+    bzip2 \
     ca-certificates \
     ccache \
     check \
@@ -18,7 +19,10 @@ RUN : \
     lcov \
     libbsd-dev \
     libffi-dev \
+    libglib2.0-0 \
     libncurses-dev \
+    libpixman-1-0 \
+    libslirp0 \
     libusb-1.0-0-dev \
     make \
     ninja-build \
@@ -77,6 +81,7 @@ RUN echo IDF_CHECKOUT_REF=$IDF_CHECKOUT_REF IDF_CLONE_BRANCH_OR_TAG=$IDF_CLONE_B
 RUN : \
   && update-ca-certificates --fresh \
   && $IDF_PATH/tools/idf_tools.py --non-interactive install required --targets=${IDF_INSTALL_TARGETS} \
+  && $IDF_PATH/tools/idf_tools.py --non-interactive install qemu* --targets=${IDF_INSTALL_TARGETS} \
   && $IDF_PATH/tools/idf_tools.py --non-interactive install cmake \
   && $IDF_PATH/tools/idf_tools.py --non-interactive install-python-env \
   && rm -rf $IDF_TOOLS_PATH/dist \
@@ -89,35 +94,6 @@ ENV IDF_PYTHON_CHECK_CONSTRAINTS=no
 # Ccache is installed, enable it by default
 ENV IDF_CCACHE_ENABLE=1
 
-# Install QEMU runtime dependencies
-RUN : \
-  && apt-get update && apt-get install -y -q \
-    bzip2 \
-    libglib2.0-0 \
-    libpixman-1-0 \
-    libslirp0 \
-  && rm -rf /var/lib/apt/lists/* \
-  && :
-
-# Install QEMU
-ARG QEMU_VER=develop_8.0.0_20230522
-ARG QEMU_RISCV32_DIST=esp-qemu-riscv32-softmmu-${QEMU_VER}-x86_64-linux-gnu.tar.bz2
-ARG QEMU_RISCV32_SHA256=bc7607720ff3d7e3d39f3e1810b8795f376f4b9cf3783c8f2ed3f7f14ba74717
-ARG QEMU_XTENSA_DIST=esp-qemu-xtensa-softmmu-${QEMU_VER}-x86_64-linux-gnu.tar.bz2
-ARG QEMU_XTENSA_SHA256=a7e5e779fd593cb15f6d197034dc2fb427ed9165a4743e2febc6f6a47dfcc618
-
-RUN bash -c ': \
-  && wget --no-verbose https://github.com/espressif/qemu/releases/download/esp-${QEMU_VER//_/-}/${QEMU_RISCV32_DIST} \
-  && echo "${QEMU_RISCV32_SHA256} *${QEMU_RISCV32_DIST}" | sha256sum --check --strict - \
-  && tar -xf ${QEMU_RISCV32_DIST} -C /opt \
-  && rm ${QEMU_RISCV32_DIST} \
-  && wget --no-verbose https://github.com/espressif/qemu/releases/download/esp-${QEMU_VER//_/-}/${QEMU_XTENSA_DIST} \
-  && echo "${QEMU_XTENSA_SHA256} *${QEMU_XTENSA_DIST}" | sha256sum --check --strict - \
-  && tar -xf ${QEMU_XTENSA_DIST} -C /opt \
-  && rm ${QEMU_XTENSA_DIST} \
-  '
-ENV PATH=/opt/qemu/bin:${PATH}
-
 COPY entrypoint.sh /opt/esp/entrypoint.sh
 ENTRYPOINT [ "/opt/esp/entrypoint.sh" ]
 CMD [ "/bin/bash" ]

tools/idf_tools.py

@@ -33,6 +33,7 @@
 import copy
 import datetime
 import errno
+import fnmatch
 import functools
 import hashlib
 import json
@@ -194,7 +195,16 @@ def get_by_filename(file_name):  # type: (str) -> Optional[str]
         return Platforms.get(found_alias)
 
 
-CURRENT_PLATFORM = Platforms.get(PYTHON_PLATFORM)
+def parse_platform_arg(platform_str):  # type: (str) -> str
+    platform = Platforms.get(platform_str)
+    if platform is None:
+        fatal(f'unknown platform: {platform}')
+        raise SystemExit(1)
+    return platform
+
+
+CURRENT_PLATFORM = parse_platform_arg(PYTHON_PLATFORM)
+
 
 EXPORT_SHELL = 'shell'
 EXPORT_KEY_VALUE = 'key-value'
@@ -387,6 +397,8 @@ def unpack(filename, destination):  # type: (str, str) -> None
         archive_obj = tarfile.open(filename, 'r:gz')  # type: Union[TarFile, ZipFile]
     elif filename.endswith(('.tar.xz')):
         archive_obj = tarfile.open(filename, 'r:xz')
+    elif filename.endswith(('.tar.bz2')):
+        archive_obj = tarfile.open(filename, 'r:bz2')
     elif filename.endswith('zip'):
         archive_obj = ZipFile(filename)
     else:
@@ -714,6 +726,13 @@ def get_install_type(self):  # type: () -> Callable[[str], None]
     def get_supported_targets(self):  # type: ()  -> list[str]
         return self._current_options.supported_targets  # type: ignore
 
+    def is_supported_for_any_of_targets(self, targets):  # type: (list[str]) -> bool
+        """
+        Checks whether the tool is suitable for at least one of the specified targets.
+        """
+        supported_targets = self.get_supported_targets()
+        return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
+
     def compatible_with_platform(self):  # type: () -> bool
         return any([v.compatible_with_platform() for v in self.versions.values()])
 
@@ -1395,24 +1414,66 @@ def get_python_env_path() -> Tuple[str, str, str, str]:
     return idf_python_env_path, idf_python_export_path, virtualenv_python, idf_version
 
 
-def add_and_check_targets(idf_env_obj, targets_str):  # type: (IDFEnv, str) -> list[str]
+def parse_tools_arg(tools_str):  # type: (List[str]) -> List[str]
     """
-    Define targets from targets_str, check that the target names are valid and add them to idf_env_obj
+    Base parsing "tools" argumets: all, required, etc
+    """
+    if not tools_str:
+        return ['required']
+    else:
+        return tools_str
+
+
+def expand_tools_arg(tools_spec, overall_tools, targets):  # type: (list[str], OrderedDict, list[str]) -> list[str]
+    """ Expand list of tools 'tools_spec' in according:
+        - a tool is in the 'overall_tools' list
+        - consider metapackages like "required" and "all"
+        - process wildcards in tool names
+        - a tool supports chips from 'targets'
+    """
+    tools = []
+    # Filtering tools if they are in overall_tools
+    # Processing wildcards if possible
+    for tool_pattern in tools_spec:
+        tools.extend([k for k, _ in overall_tools.items() if fnmatch.fnmatch(k,tool_pattern) and k not in tools])
+
+    # Processing "metapackage"
+    if 'required' in tools_spec:
+        tools.extend([k for k, v in overall_tools.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS and k not in tools])
+
+    elif 'all' in tools_spec:
+        tools.extend([k for k, v in overall_tools.items() if v.get_install_type() != IDFTool.INSTALL_NEVER and k not in tools])
+
+    # Filtering by ESP_targets
+    tools = [k for k in tools if overall_tools[k].is_supported_for_any_of_targets(targets)]
+    return tools
+
+
+def parse_targets_arg(targets_str):  # type: (str) -> List[str]
+    """
+    Parse and check if targets_str is a valid list of targets and return a target list
     """
     targets_from_tools_json = get_all_targets_from_tools_json()
     invalid_targets = []
 
     targets_str = targets_str.lower()
     targets = targets_str.replace('-', '').split(',')
-    if targets != ['all']:
+    if targets == ['all']:
+        return targets_from_tools_json
+    else:
         invalid_targets = [t for t in targets if t not in targets_from_tools_json]
         if invalid_targets:
             warn('Targets: "{}" are not supported. Only allowed options are: {}.'.format(', '.join(invalid_targets), ', '.join(targets_from_tools_json)))
             raise SystemExit(1)
-        idf_env_obj.get_active_idf_record().extend_targets(targets)
-    else:
-        idf_env_obj.get_active_idf_record().extend_targets(targets_from_tools_json)
+        return targets
+
 
+def add_and_check_targets(idf_env_obj, targets_str):  # type: (IDFEnv, str) -> list[str]
+    """
+    Define targets from targets_str, check that the target names are valid and add them to idf_env_obj
+    """
+    targets = parse_targets_arg(targets_str)
+    idf_env_obj.get_active_idf_record().extend_targets(targets)
     return idf_env_obj.get_active_idf_record().targets
 
 
@@ -1789,12 +1850,7 @@ def apply_github_assets_option(idf_download_url):  # type: (str) -> str
 
 
 def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec,
-                                     quiet=False):  # type: (Optional[str], list[str], list[str], bool) -> Tuple[list[str], Dict[str, IDFTool]]
-    selected_platform = Platforms.get(selected_platform)
-    if selected_platform is None:
-        fatal(f'unknown platform: {selected_platform}')
-        raise SystemExit(1)
-
+                                     quiet=False):  # type: (str, list[str], list[str], bool) -> Tuple[list[str], Dict[str, IDFTool]]
     # If this function is not called from action_download, but is used just for detecting active tools, info about downloading is unwanted.
     global global_quiet
     try:
@@ -1806,32 +1862,20 @@ def get_tools_spec_and_platform_info(selected_platform, targets, tools_spec,
             tool_for_platform = tool_obj.copy_for_platform(selected_platform)
             tools_info_for_platform[name] = tool_for_platform
 
-        if not tools_spec or 'required' in tools_spec:
-            # Downloading tools for all ESP_targets required by the operating system.
-            tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
-            # Filtering tools user defined list of ESP_targets
-            if 'all' not in targets:
-                def is_tool_selected(tool):  # type: (IDFTool) -> bool
-                    supported_targets = tool.get_supported_targets()
-                    return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
-                tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
-            info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
-
-        # Downloading tools for all ESP_targets (MacOS, Windows, Linux)
-        elif 'all' in tools_spec:
-            tools_spec = [k for k, v in tools_info_for_platform.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
-            info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
+        tools_spec = expand_tools_arg(tools_spec, tools_info_for_platform, targets)
+        info('Downloading tools for {}: {}'.format(selected_platform, ', '.join(tools_spec)))
     finally:
         global_quiet = old_global_quiet
 
     return tools_spec, tools_info_for_platform
 
 
 def action_download(args):  # type: ignore
-    tools_spec = args.tools
+    tools_spec = parse_tools_arg(args.tools)
+
     targets = []  # type: list[str]
-    # Downloading tools required for defined ESP_targets
-    if 'required' in tools_spec:
+    # Saving IDFEnv::targets for selected ESP_targets if all tools have been specified
+    if 'required' in tools_spec or 'all' in tools_spec:
         idf_env_obj = IDFEnv.get_idf_env()
         targets = add_and_check_targets(idf_env_obj, args.targets)
         try:
@@ -1840,9 +1884,13 @@ def action_download(args):  # type: ignore
             if args.targets in targets:
                 targets.remove(args.targets)
             warn('Downloading tools for targets was not successful with error: {}'.format(err))
+    # Taking into account ESP_targets but not saving them for individual tools (specified list of tools)
+    else:
+        targets = parse_targets_arg(args.targets)
 
-    tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(args.platform, targets, args.tools)
+    platform = parse_platform_arg(args.platform)
 
+    tools_spec, tools_info_for_platform = get_tools_spec_and_platform_info(platform, targets, tools_spec)
     for tool_spec in tools_spec:
         if '@' not in tool_spec:
             tool_name = tool_spec
@@ -1864,18 +1912,17 @@ def action_download(args):  # type: ignore
         tool_spec = '{}@{}'.format(tool_name, tool_version)
 
         info('Downloading {}'.format(tool_spec))
-        _idf_tool_obj = tool_obj.versions[tool_version].get_download_for_platform(args.platform)
+        _idf_tool_obj = tool_obj.versions[tool_version].get_download_for_platform(platform)
         _idf_tool_obj.url = get_idf_download_url_apply_mirrors(args, _idf_tool_obj.url)
 
         tool_obj.download(tool_version)
 
 
 def action_install(args):  # type: ignore
-    tools_info = load_tools_info()
-    tools_spec = args.tools  # type: ignore
+    tools_spec = parse_tools_arg(args.tools)
+
     targets = []  # type: list[str]
-    info('Current system platform: {}'.format(CURRENT_PLATFORM))
-    # No single tool '<tool_name>@<version>' was defined, install whole toolchains
+    # Saving IDFEnv::targets for selected ESP_targets if all tools have been specified
     if 'required' in tools_spec or 'all' in tools_spec:
         idf_env_obj = IDFEnv.get_idf_env()
         targets = add_and_check_targets(idf_env_obj, args.targets)
@@ -1886,23 +1933,14 @@ def action_install(args):  # type: ignore
                 targets.remove(args.targets)
             warn('Installing targets was not successful with error: {}'.format(err))
         info('Selected targets are: {}'.format(', '.join(targets)))
+    # Taking into account ESP_targets but not saving them for individual tools (specified list of tools)
+    else:
+        targets = parse_targets_arg(args.targets)
 
-        # Installing tools for defined ESP_targets
-        if 'required' in tools_spec:
-            tools_spec = [k for k, v in tools_info.items() if v.get_install_type() == IDFTool.INSTALL_ALWAYS]
-            # If only some ESP_targets are defined, filter tools for those
-            if len(get_all_targets_from_tools_json()) != len(targets):
-                def is_tool_selected(tool):  # type: (IDFTool) -> bool
-                    supported_targets = tool.get_supported_targets()
-                    return (any(item in targets for item in supported_targets) or supported_targets == ['all'])
-                tools_spec = [k for k in tools_spec if is_tool_selected(tools_info[k])]
-            info('Installing tools: {}'.format(', '.join(tools_spec)))
-
-        # Installing all available tools for all operating systems (MacOS, Windows, Linux)
-        else:
-            tools_spec = [k for k, v in tools_info.items() if v.get_install_type() != IDFTool.INSTALL_NEVER]
-            info('Installing tools: {}'.format(', '.join(tools_spec)))
-
+    info('Current system platform: {}'.format(CURRENT_PLATFORM))
+    tools_info = load_tools_info()
+    tools_spec = expand_tools_arg(tools_spec, tools_info, targets)
+    info('Installing tools: {}'.format(', '.join(tools_spec)))
     for tool_spec in tools_spec:
         if '@' not in tool_spec:
             tool_name = tool_spec
@@ -2541,6 +2579,7 @@ def main(argv):  # type: (list[str]) -> None
     install.add_argument('tools', metavar='TOOL', nargs='*', default=['required'],
                          help='Tools to install. ' +
                          'To install a specific version use <tool_name>@<version> syntax. ' +
+                         'To install tools by pattern use wildcards in <tool_name_pattern> . ' +
                          'Use empty or \'required\' to install required tools, not optional ones. ' +
                          'Use \'all\' to install all tools, including the optional ones.')
     install.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +
@@ -2551,6 +2590,7 @@ def main(argv):  # type: (list[str]) -> None
     download.add_argument('tools', metavar='TOOL', nargs='*', default=['required'],
                           help='Tools to download. ' +
                           'To download a specific version use <tool_name>@<version> syntax. ' +
+                          'To download tools by pattern use wildcards in <tool_name_pattern> . ' +
                           'Use empty or \'required\' to download required tools, not optional ones. ' +
                           'Use \'all\' to download all tools, including the optional ones.')
     download.add_argument('--targets', default='all', help='A comma separated list of desired chip targets for installing.' +