From 3ca648d1b20fe83c43842d82a23828460283104a Mon Sep 17 00:00:00 2001 From: Yi Lai Date: Wed, 13 May 2026 08:12:47 -0400 Subject: [PATCH 1/3] edac: add more EDAC tests 1. add more EDAC tests to improve coverage 2. enhance scripts to support imh_edac driver validation Signed-off-by: Yi Lai (cherry picked from commit 1680b9afa227fe153ee36e9cf2fca18a7f8a04e2) --- BM/edac/README.md | 24 ++++- BM/edac/intel_edac.sh | 211 +++++++++++++++++++++++++++++++++++++++--- BM/edac/tests | 5 +- 3 files changed, 224 insertions(+), 16 deletions(-) diff --git a/BM/edac/README.md b/BM/edac/README.md index 2ba710a2..6086707e 100644 --- a/BM/edac/README.md +++ b/BM/edac/README.md @@ -1,17 +1,33 @@ # EDAC -EDAC (Error Detection and Correction) framework provides support for detecting and correcting memory errors on systems with ECC (Error-Correcting Code) memory or other error detection mechanism. +EDAC (Error Detection and Correction) framework provides support for detecting and correcting memory errors on systems with ECC (Error-Correcting Code) memory or other error detection mechanisms. -Intel i10nm EDAC driver supports the Intel 10nm series server integrated memory controller. +Intel EDAC drivers supported: +- **i10nm_edac**: ICX, SPR, EMR, GNR, SRF, CWF and other 10nm+ server platforms (CPU family 6) +- **imh_edac**: Diamond Rapids (DMR) and future IMH platforms (CPU family 19) + +The test script auto-detects the platform and selects the appropriate driver. + +## Test Cases + +| Test | Description | +|------|-------------| +| check_edac_bus | Verify EDAC bus exists and has registered devices | +| check_edac_driver | Verify EDAC driver is loaded (attempts modprobe if not) | +| edac_addr_decode | Test address decoding via EDAC debugfs interface | +| edac_mc_index_change | Inject errors and verify multiple MC indices in decode log | +| edac_mc_check_populated | Cross-check decoded MCs against sysfs populated MCs | ## Usage -You can run the cases one by one, e.g. command +Run cases individually: ``` ./intel_edac.sh -t check_edac_bus +./intel_edac.sh -t edac_addr_decode ``` -You also can run the cases together with runtests command, e.g. + +Run all cases with runtests: ``` cd .. diff --git a/BM/edac/intel_edac.sh b/BM/edac/intel_edac.sh index f6cefac6..48258df4 100755 --- a/BM/edac/intel_edac.sh +++ b/BM/edac/intel_edac.sh @@ -1,7 +1,7 @@ #!/bin/bash # SPDX-License-Identifier: GPL-2.0-only # Copyright (c) 2022 Intel Corporation -# Description: Test script for Intel i10nm EDAC driver, which supports 10nm series server +# Description: Test script for Intel EDAC drivers (i10nm_edac and imh_edac) # EDAC: Error Detection and Correction # @Author Yi Lai yi1.lai@intel.com @@ -10,7 +10,6 @@ cd "$(dirname "$0")" 2>/dev/null || exit 1 source ../.env EDAC_BUS="/sys/bus/edac" -EDAC_DRIVER=i10nm_edac : "${CASE_NAME:=""}" @@ -22,25 +21,214 @@ usage() { __EOF } +# Detect CPU family to determine EDAC driver variant and debugfs path +# Family 19 (0x13): IMH - Diamond Rapids (DMR) and future IMH platforms +# Family 6: i10nm - ICX, SPR, EMR, GNR, SRF, CWF and other 10nm+ server platforms +get_edac_debug_path() { + get_cpu_model + case $FML in + 13) + MC="imh" + EDAC_DEBUG_PATH="/sys/kernel/debug/edac/imh_test/addr" + ;; + *) + MC="i10nm" + EDAC_DEBUG_PATH="/sys/kernel/debug/edac/i10nm_test/addr" + ;; + esac + LOG="${PWD}/dmesg.decoding.via.debugfs.${MC}_edac.log" + [[ -f "$LOG" ]] && rm -f "$LOG" +} + +edac_addr_decode() { + local MARKER dmesg_output + + if [[ -w /dev/kmsg ]]; then + MARKER="EDAC_CHECK_MARKER_$(date +%s%N)" + echo "$MARKER" > /dev/kmsg + fi + + echo 0x12345 > "$EDAC_DEBUG_PATH" + + if [[ -w /dev/kmsg ]]; then + dmesg_output=$(dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER") + else + dmesg_output=$(dmesg | tail -n 100) + fi + + if echo "$dmesg_output" | grep -q -e "ADDR 0x12345"; then + test_print_trc "EDAC address decode successfully" + else + die "Failed to decode EDAC address" + fi +} + +edac_test_error_inject() { + local MARKER + local TOLM=2 + local SIZE_KB SIZE_GB tmp_addr_file addr_low addr_high + + SIZE_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) + SIZE_GB=$((SIZE_KB / 1024 / 1024)) + tmp_addr_file="${PWD}/einj_edac.txt" + > "$tmp_addr_file" + + for ((i = 0; i < (SIZE_GB + TOLM) * 4; i += 4)); do + addr_low=$(head -c 32 /dev/urandom | md5sum | head -c 7) + addr_high=$(printf "%x" "$i") + echo "0x${addr_high}${addr_low}" >> "$tmp_addr_file" + done + + if [[ -w /dev/kmsg ]]; then + MARKER="EDAC_INJECT_MARKER_$(date +%s%N)" + echo "$MARKER" > /dev/kmsg + fi + + while read -r addr; do + echo "$addr" > "$EDAC_DEBUG_PATH" + done < "$tmp_addr_file" + + if [[ -w /dev/kmsg ]]; then + dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER" >> "$LOG" + else + dmesg | tail -n 1000 >> "$LOG" + fi + rm -f "$tmp_addr_file" +} + +edac_test_error_inject_iomem() { + local MARKER + local PAGESIZE=4096 + local NUM_TESTADDR=40 + local RANGE_SIZE_THR=500 + local tmp_addr_file iomem_tmp URANDOM + local start_addr end_addr rand_addr test_pfn_base test_pfn test_addr + + tmp_addr_file="${PWD}/einj_iomem.txt" + iomem_tmp="${PWD}/iomem_tmp" + > "$tmp_addr_file" + + URANDOM=$(od -An -N4 -t uL /dev/urandom | tr -d " ") + grep "System RAM" /proc/iomem | cut -d ':' -f1 > "$iomem_tmp" + + while read -r line; do + start_addr=$((16#$(echo "$line" | awk -F '-' '{print $1}'))) + end_addr=$((16#$(echo "$line" | awk -F '-' '{print $2}'))) + # skip address < 1MB + ((start_addr < 0x100000)) && continue + # skip small memory areas (<500MB) + (((end_addr - start_addr) < (RANGE_SIZE_THR * 0x100000))) && continue + + rand_addr=$((start_addr + URANDOM % (end_addr - start_addr))) + if ((rand_addr + NUM_TESTADDR * PAGESIZE > end_addr)); then + rand_addr=$start_addr + fi + test_pfn_base=$((rand_addr / PAGESIZE)) + for ((i = 1; i <= NUM_TESTADDR; i++)); do + test_pfn=$((test_pfn_base + i)) + test_addr=$((test_pfn * PAGESIZE)) + ((test_addr > end_addr)) && break + printf "0x%lx\n" "$test_addr" >> "$tmp_addr_file" + done + done < "$iomem_tmp" + + if [[ -w /dev/kmsg ]]; then + MARKER="EDAC_INJECT_IOMEM_MARKER_$(date +%s%N)" + echo "$MARKER" > /dev/kmsg + fi + + while read -r addr; do + echo "$addr" > "$EDAC_DEBUG_PATH" + done < "$tmp_addr_file" + + if [[ -w /dev/kmsg ]]; then + dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER" >> "$LOG" + else + dmesg | tail -n 1000 >> "$LOG" + fi + rm -f "$tmp_addr_file" "$iomem_tmp" +} + +edac_mc_index_change() { + local mc_count + mc_count=$(grep -oE "EDAC MC[0-9]+:" "$LOG" | sort -u | wc -l) + if ((mc_count > 1)); then + test_print_trc "Decoding log contains $mc_count unique Memory Controller entries" + else + die "Decoding log contains only $mc_count Memory Controller entry" + fi +} + +edac_mc_check_populated() { + local mc_indices sorted_mc_indices populated_indexes sorted_populated + local SYSFS_EDAC_MC_DIR="/sys/devices/system/edac/mc" + + mc_indices=($(grep -oE "EDAC MC[0-9]+:" "$LOG" | sort -u | grep -oE "[0-9]+")) + sorted_mc_indices=($(printf "%s\n" "${mc_indices[@]}" | sort -n)) + + [[ -d "$SYSFS_EDAC_MC_DIR" ]] || die "EDAC mc structure not found under $SYSFS_EDAC_MC_DIR" + + populated_indexes=() + for size_file in "$SYSFS_EDAC_MC_DIR"/mc*/size_mb; do + [[ -r "$size_file" ]] || continue + local size num mc_dir + size=$(cat "$size_file" 2>/dev/null) + if [[ -n "$size" ]] && ((size > 0)); then + mc_dir=$(dirname "$size_file") + num=$(basename "$mc_dir" | tr -d 'mc') + populated_indexes+=("$num") + fi + done + + [[ ${#populated_indexes[@]} -gt 0 ]] || die "No populated memory controllers found" + sorted_populated=($(printf "%s\n" "${populated_indexes[@]}" | sort -n)) + + test_print_trc "Populated MCs from sysfs: ${sorted_populated[*]}" + test_print_trc "Decoded MCs from log: ${sorted_mc_indices[*]}" + + if [[ "${sorted_mc_indices[*]}" == "${sorted_populated[*]}" ]]; then + test_print_trc "All populated Memory Controllers successfully triggered errors" + else + die "Mismatch between populated MCs and decoded MCs" + fi +} + edac_test() { case $TEST_SCENARIO in check_edac_bus) - edac_bus=$(ls $EDAC_BUS) - if [ -n "$edac_bus" ]; then - test_print_trc "EDAC bus is found" - else + if [[ ! -d "$EDAC_BUS" ]]; then die "EDAC bus is not found" fi + edac_devices=$(ls "$EDAC_BUS"/devices/ 2>/dev/null) + if [[ -n "$edac_devices" ]]; then + test_print_trc "EDAC bus is found with devices: $edac_devices" + else + die "EDAC bus exists but no EDAC devices are registered" + fi ;; check_edac_driver) - test_print_trc "Check Intel i10nm edac driver" - lsmod | grep -q $EDAC_DRIVER - if ! lsmod | grep -q $EDAC_DRIVER; then - die "Intel i10nm edac driver is not loaded" + test_print_trc "Check Intel ${MC}_edac driver" + if ! lsmod | grep -q "${MC}_edac"; then + test_print_trc "${MC}_edac not loaded, attempting to load" + modprobe "${MC}_edac" || die "Failed to load ${MC}_edac driver" + fi + if lsmod | grep -q "${MC}_edac"; then + test_print_trc "Intel ${MC}_edac driver is loaded" else - test_print_trc "Intel i10nm edac driver is loaded" + die "Intel ${MC}_edac driver is not loaded" fi ;; + edac_addr_decode) + edac_addr_decode + ;; + edac_mc_index_change) + edac_test_error_inject + edac_mc_index_change + ;; + edac_mc_check_populated) + edac_test_error_inject_iomem + edac_mc_check_populated + ;; esac } @@ -63,4 +251,5 @@ while getopts :t:H arg; do esac done +get_edac_debug_path edac_test diff --git a/BM/edac/tests b/BM/edac/tests index 84f3bfc6..e63731aa 100644 --- a/BM/edac/tests +++ b/BM/edac/tests @@ -1,5 +1,8 @@ # This file collects Intel EDAC driver testcases which can run against -# on Intel 10nm series server +# Intel server platforms (i10nm: ICX/SPR/EMR/GNR/SRF/CWF, imh: DMR+) intel_edac.sh -t check_edac_bus intel_edac.sh -t check_edac_driver +intel_edac.sh -t edac_addr_decode +intel_edac.sh -t edac_mc_index_change +intel_edac.sh -t edac_mc_check_populated From 2ae6473e734cc70dc0ac524abde9e326a7a9a785 Mon Sep 17 00:00:00 2001 From: Yi Lai Date: Sun, 24 May 2026 21:11:11 -0400 Subject: [PATCH 2/3] edac: add retry_rd_err_log (RRL) test cases Add test cases to verify the EDAC driver's retry_rd_err_log module parameter functionality. Signed-off-by: Yi Lai (cherry picked from commit 3438e087be4276c10c01094c220cc8cd3cc2c91d) --- BM/edac/intel_edac.sh | 47 +++++++++++++++++++++++++++++++++++++++++++ BM/edac/tests | 2 ++ 2 files changed, 49 insertions(+) diff --git a/BM/edac/intel_edac.sh b/BM/edac/intel_edac.sh index 48258df4..884dc175 100755 --- a/BM/edac/intel_edac.sh +++ b/BM/edac/intel_edac.sh @@ -159,6 +159,47 @@ edac_mc_index_change() { fi } +edac_retry_rd_err_log_check() { + local mode=$1 + local MARKER dmesg_output + + if [[ -w /dev/kmsg ]]; then + MARKER="EDAC_RRL_MARKER_$(date +%s%N)" + echo "$MARKER" > /dev/kmsg + fi + + # Reload EDAC driver with retry_rd_err_log parameter + modprobe -r "${MC}_edac" 2>/dev/null + modprobe "${MC}_edac" retry_rd_err_log="$mode" || die "Failed to load ${MC}_edac with retry_rd_err_log=$mode" + + # Unload and reload to cover kernel teardown paths + modprobe -r "${MC}_edac" + modprobe "${MC}_edac" retry_rd_err_log="$mode" || die "Failed to reload ${MC}_edac with retry_rd_err_log=$mode" + + # Trigger EDAC address decode to exercise the RRL path + echo 0x12345 > "$EDAC_DEBUG_PATH" + + if [[ -w /dev/kmsg ]]; then + dmesg_output=$(dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER") + else + dmesg_output=$(dmesg | tail -n 1000) + fi + + if echo "$dmesg_output" | grep -q -e "retry_rd_err_log"; then + if [[ "$MC" == "imh" ]]; then + if echo "$dmesg_output" | grep -iq "SubChId"; then + test_print_trc "EDAC retry_rd_err_log=$mode and SubChId checked successfully" + else + die "retry_rd_err_log=$mode: Failed to find SubChId in dmesg (required for $MC)" + fi + else + test_print_trc "EDAC retry_rd_err_log=$mode checked successfully" + fi + else + die "Failed to find retry_rd_err_log in dmesg (mode=$mode)" + fi +} + edac_mc_check_populated() { local mc_indices sorted_mc_indices populated_indexes sorted_populated local SYSFS_EDAC_MC_DIR="/sys/devices/system/edac/mc" @@ -229,6 +270,12 @@ edac_test() { edac_test_error_inject_iomem edac_mc_check_populated ;; + edac_rrl_mode1) + edac_retry_rd_err_log_check 1 + ;; + edac_rrl_mode2) + edac_retry_rd_err_log_check 2 + ;; esac } diff --git a/BM/edac/tests b/BM/edac/tests index e63731aa..2f8c7a0a 100644 --- a/BM/edac/tests +++ b/BM/edac/tests @@ -6,3 +6,5 @@ intel_edac.sh -t check_edac_driver intel_edac.sh -t edac_addr_decode intel_edac.sh -t edac_mc_index_change intel_edac.sh -t edac_mc_check_populated +intel_edac.sh -t edac_rrl_mode1 +intel_edac.sh -t edac_rrl_mode2 From 2d41aeb57e9220729679b17e3b37f73ea55f5f40 Mon Sep 17 00:00:00 2001 From: Yi Lai Date: Fri, 29 May 2026 01:21:29 -0400 Subject: [PATCH 3/3] ras, edac: fix shellcheck warnings across shell scripts Signed-off-by: Yi Lai (cherry picked from commit d8f9e2ef7fbb4075e61a72f0ad79be6548c08b93) --- BM/edac/intel_edac.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/BM/edac/intel_edac.sh b/BM/edac/intel_edac.sh index 884dc175..5a856fd5 100755 --- a/BM/edac/intel_edac.sh +++ b/BM/edac/intel_edac.sh @@ -71,7 +71,7 @@ edac_test_error_inject() { SIZE_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo) SIZE_GB=$((SIZE_KB / 1024 / 1024)) tmp_addr_file="${PWD}/einj_edac.txt" - > "$tmp_addr_file" + true > "$tmp_addr_file" for ((i = 0; i < (SIZE_GB + TOLM) * 4; i += 4)); do addr_low=$(head -c 32 /dev/urandom | md5sum | head -c 7) @@ -106,7 +106,7 @@ edac_test_error_inject_iomem() { tmp_addr_file="${PWD}/einj_iomem.txt" iomem_tmp="${PWD}/iomem_tmp" - > "$tmp_addr_file" + true > "$tmp_addr_file" URANDOM=$(od -An -N4 -t uL /dev/urandom | tr -d " ") grep "System RAM" /proc/iomem | cut -d ':' -f1 > "$iomem_tmp" @@ -204,8 +204,8 @@ edac_mc_check_populated() { local mc_indices sorted_mc_indices populated_indexes sorted_populated local SYSFS_EDAC_MC_DIR="/sys/devices/system/edac/mc" - mc_indices=($(grep -oE "EDAC MC[0-9]+:" "$LOG" | sort -u | grep -oE "[0-9]+")) - sorted_mc_indices=($(printf "%s\n" "${mc_indices[@]}" | sort -n)) + mapfile -t mc_indices < <(grep -oE "EDAC MC[0-9]+:" "$LOG" | sort -u | grep -oE "[0-9]+") + mapfile -t sorted_mc_indices < <(printf "%s\n" "${mc_indices[@]}" | sort -n) [[ -d "$SYSFS_EDAC_MC_DIR" ]] || die "EDAC mc structure not found under $SYSFS_EDAC_MC_DIR" @@ -222,7 +222,7 @@ edac_mc_check_populated() { done [[ ${#populated_indexes[@]} -gt 0 ]] || die "No populated memory controllers found" - sorted_populated=($(printf "%s\n" "${populated_indexes[@]}" | sort -n)) + mapfile -t sorted_populated < <(printf "%s\n" "${populated_indexes[@]}" | sort -n) test_print_trc "Populated MCs from sysfs: ${sorted_populated[*]}" test_print_trc "Decoded MCs from log: ${sorted_mc_indices[*]}"