Merge branch 'fix/idf_monitor_pc_addr_intervals' into 'master'

Tools: IDF Monitor: Determine possible instruction addresses based on ELF segments

Closes IDF-6174

See merge request espressif/esp-idf!21393

docs/en/api-guides/tools/idf-monitor.rst

@@ -61,7 +61,7 @@ For easy interaction with IDF Monitor, use the keyboard shortcuts given in the t
      -
    * - Ctrl+C
      - Interrupt running application
-     - Pauses IDF Monitor and run GDB_ project debugger to debug the application at runtime. This requires :ref:CONFIG_ESP_SYSTEM_GDBSTUB_RUNTIME option to be enabled.
+     - Pauses IDF Monitor and runs GDB_ project debugger to debug the application at runtime. This requires :ref:CONFIG_ESP_SYSTEM_GDBSTUB_RUNTIME option to be enabled.
 
 Any keys pressed, other than ``Ctrl-]`` and ``Ctrl-T``, will be sent through the serial port.
 
@@ -72,13 +72,13 @@ IDF-specific features
 Automatic Address Decoding
 ~~~~~~~~~~~~~~~~~~~~~~~~~~
 
-Whenever ESP-IDF outputs a hexadecimal code address of the form ``0x4_______``, IDF Monitor uses ``addr2line_`` to look up the location in the source code and find the function name.
+Whenever the chip outputs a hexadecimal address that points to executable code, IDF monitor looks up the location in the source code (file name and line number) and prints the location on the next line in yellow.
 
 .. highlight:: none
 
 .. only:: CONFIG_IDF_TARGET_ARCH_XTENSA
 
-  If an ESP-IDF app crashes and panics, a register dump and backtrace is produced, such as the following::
+  If an ESP-IDF app crashes and panics, a register dump and backtrace are produced, such as the following::
 
       Guru Meditation Error of type StoreProhibited occurred on core  0. Exception was unhandled.
       Register dump:

docs/zh_CN/api-guides/tools/idf-monitor.rst

@@ -72,7 +72,7 @@ IDF 监视器是一个串行终端程序，用于收发目标设备串口的串
 自动解码地址
 ~~~~~~~~~~~~~~~~
 
-ESP-IDF 输出形式为 ``0x4_______`` 的十六进制代码地址后，IDF 监视器将使用 ``addr2line_`` 查找该地址在源代码中的位置和对应的函数名。
+每当芯片输出指向可执行代码的十六进制地址时，IDF 监视器将查找该地址在源代码中的位置（文件名和行号），并在下一行用黄色打印出该位置。
 
 .. highlight:: none
 

tools/idf_monitor_base/constants.py

@@ -40,8 +40,8 @@
 # paths to scripts
 PANIC_OUTPUT_DECODE_SCRIPT = os.path.join(os.path.dirname(__file__), '..', 'gdb_panic_server.py')
 
-# regex matches an potential PC value (0x4xxxxxxx)
-MATCH_PCADDR = re.compile(r'0x4[0-9a-f]{7}', re.IGNORECASE)
+# regex matches an potential address
+ADDRESS_RE = re.compile(r'0x[0-9a-f]{8}', re.IGNORECASE)
 
 DEFAULT_TOOLCHAIN_PREFIX = 'xtensa-esp32-elf-'
 

tools/idf_monitor_base/logger.py

@@ -8,8 +8,9 @@
 
 from serial.tools import miniterm  # noqa: F401
 
-from .constants import MATCH_PCADDR
+from .constants import ADDRESS_RE
 from .output_helpers import lookup_pc_address, red_print, yellow_print
+from .pc_address_matcher import PcAddressMatcher
 
 
 class Logger:
@@ -25,6 +26,7 @@ def __init__(self, elf_file, console, timestamps, timestamp_format, pc_address_b
         self._pc_address_buffer = pc_address_buffer
         self.enable_address_decoding = enable_address_decoding
         self.toolchain_prefix = toolchain_prefix
+        self.pc_address_matcher = PcAddressMatcher(self.elf_file) if enable_address_decoding else None
 
     @property
     def pc_address_buffer(self):  # type: () -> bytes
@@ -117,7 +119,9 @@ def handle_possible_pc_address_in_line(self, line):  # type: (bytes) -> None
         self._pc_address_buffer = b''
         if not self.enable_address_decoding:
             return
-        for m in re.finditer(MATCH_PCADDR, line.decode(errors='ignore')):
-            translation = lookup_pc_address(m.group(), self.toolchain_prefix, self.elf_file)
-            if translation:
-                self.print(translation, console_printer=yellow_print)
+        for m in re.finditer(ADDRESS_RE, line.decode(errors='ignore')):
+            num = m.group()
+            if self.pc_address_matcher.is_executable_address(int(num, 16)):
+                translation = lookup_pc_address(num, self.toolchain_prefix, self.elf_file)
+                if translation:
+                    self.print(translation, console_printer=yellow_print)

tools/idf_monitor_base/pc_address_matcher.py

@@ -0,0 +1,50 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Apache-2.0
+
+from elftools.elf.constants import SH_FLAGS
+from elftools.elf.elffile import ELFFile
+
+
+class PcAddressMatcher(object):
+    """
+    Class for detecting potentional addresses which will consequently run through the external addr2line command to
+    indentify and print information about it.
+
+    The input to this class is the path to the ELF file. Addresses of sections with executable flag are stored and
+    used later for lookup.
+    """
+
+    def __init__(self, elf_path):  # type: (str) -> None
+        self.intervals = []
+        try:
+            with open(elf_path, 'rb') as f:
+                elf = ELFFile(f)
+
+                for section in elf.iter_sections():
+                    if section['sh_flags'] & SH_FLAGS.SHF_EXECINSTR:
+                        start = section['sh_addr']
+                        size = section['sh_size']
+                        end = start + size
+                        self.intervals.append((start, end))
+
+        except FileNotFoundError:
+            # ELF file is just an optional argument
+            pass
+
+        # sort them in order to have faster lookup
+        self.intervals = sorted(self.intervals)
+
+    def is_executable_address(self, addr):  # type: (int) -> bool
+        """
+        Returns True/False depending on of "addr" is in one of the ELF sections with executable flag set.
+        """
+
+        for start, end in self.intervals:
+            if start > addr:
+                # The intervals are sorted. This means that loop can end because all remaining intervals starts are
+                # greater than the current start
+                return False
+            if start <= addr < end:
+                return True
+
+        return False

tools/idf_monitor_base/serial_handler.py

@@ -122,7 +122,7 @@ def handle_serial_input(self, data, console_parser, coredump, gdb_helper, line_m
             # It is possible that the incomplete line cuts in half the PC
             # address. A small buffer is kept and will be used the next time
             # handle_possible_pc_address_in_line is invoked to avoid this problem.
-            # MATCH_PCADDR matches 10 character long addresses. Therefore, we
+            # ADDRESS_RE matches 10 character long addresses. Therefore, we
             # keep the last 9 characters.
             self.logger.pc_address_buffer = self._last_line_part[-9:]
             # GDB sequence can be cut in half also. GDB sequence is 7

tools/test_apps/system/monitor_addr_lookup/CMakeLists.txt

@@ -0,0 +1,4 @@
+cmake_minimum_required(VERSION 3.16)
+
+include($ENV{IDF_PATH}/tools/cmake/project.cmake)
+project(monitor_addr_lookup)

tools/test_apps/system/monitor_addr_lookup/README.md

@@ -0,0 +1,2 @@
+| Supported Targets | ESP32 | ESP32-C2 | ESP32-C3 | ESP32-C6 | ESP32-S2 | ESP32-S3 |
+| ----------------- | ----- | -------- | -------- | -------- | -------- | -------- |

tools/test_apps/system/monitor_addr_lookup/main/CMakeLists.txt

@@ -0,0 +1,2 @@
+idf_component_register(SRCS "main.c"
+                    INCLUDE_DIRS "")

tools/test_apps/system/monitor_addr_lookup/main/main.c

@@ -0,0 +1,42 @@
+/*
+ * SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+ *
+ * SPDX-License-Identifier: Unlicense OR CC0-1.0
+ */
+#include <time.h>
+
+#include "freertos/FreeRTOS.h"
+#include "freertos/task.h"
+
+static volatile bool s_initialization_done = false;
+
+static void initialize(void)
+{
+    srand(time(0));
+}
+
+static int get_random_number(void)
+{
+    if (!s_initialization_done) {
+        initialize();
+        s_initialization_done = true;
+    }
+    return rand();
+}
+
+void app_main(void)
+{
+    volatile int number = get_random_number();
+    int *n = malloc(sizeof(int));
+
+    assert(n);
+
+    *n = number;
+
+    printf("app_main is running from 0x%x\n", (int) app_main);
+    printf("Initializer function at 0x%x\n", (int) initialize);
+    printf("Got %d stored at 0x%x and 0x%x from a function from 0x%x\n", *n, (int) n, (int) (&number), (int) get_random_number);
+    printf("This is the end of the report\n");
+
+    free(n);
+}

tools/test_apps/system/monitor_addr_lookup/pytest_monitor_addr_lookup.py

@@ -0,0 +1,44 @@
+# SPDX-FileCopyrightText: 2023 Espressif Systems (Shanghai) CO LTD
+# SPDX-License-Identifier: Unlicense OR CC0-1.0
+import os
+import re
+import sys
+
+import pexpect
+import pytest
+from pytest_embedded import Dut
+
+
+@pytest.mark.generic
+@pytest.mark.supported_targets
+def test_monitor_addr_lookup(config: str, dut: Dut) -> None:
+    # The port needs to be closed because idf_monitor.py will connect to it
+    dut.serial.stop_redirect_thread()
+
+    monitor_py = os.path.join(os.environ['IDF_PATH'], 'tools', 'idf_monitor.py')
+    monitor_cmd = ' '.join([sys.executable, monitor_py, os.path.join(dut.app.binary_path, 'monitor_addr_lookup.elf'),
+                            '--port', str(dut.serial.port)])
+    monitor_log_path = os.path.join(dut.logdir, 'monitor.txt')
+
+    with open(monitor_log_path, 'w') as log, pexpect.spawn(monitor_cmd, logfile=log, timeout=5, encoding='utf-8', codec_errors='ignore') as p:
+        p.expect_exact('main_task: Calling app_main()')
+
+        ADDRESS = '0x[a-f0-9]{8}'
+
+        p.expect(re.compile(r'app_main is running from ({})'.format(ADDRESS)))
+        a = p.match.group(1)
+        p.expect_exact('{}: app_main at'.format(a))
+
+        p.expect(re.compile(r'Initializer function at ({})'.format(ADDRESS)))
+        a = p.match.group(1)
+        p.expect_exact('{}: initialize at'.format(a))
+
+        p.expect(re.compile(r'Got \d+ stored at ({}) and ({}) from a function from ({})'.format(ADDRESS, ADDRESS, ADDRESS)))
+        var1 = p.match.group(1)
+        var2 = p.match.group(2)
+        func = p.match.group(3)
+        match_index = p.expect([str(var1), str(var2), pexpect.TIMEOUT])
+        assert match_index == 2  # should be TIMEOUT because addr2line should not match addresses of variables
+        p.expect_exact('{}: get_random_number at'.format(func))
+
+        p.expect_exact('This is the end of the report')