Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 20 additions & 4 deletions BM/edac/README.md
Original file line number Diff line number Diff line change
@@ -1,17 +1,33 @@
# EDAC

EDAC (Error Detection and Correction) framework provides support for detecting and correcting memory errors on systems with ECC (Error-Correcting Code) memory or other error detection mechanism.
EDAC (Error Detection and Correction) framework provides support for detecting and correcting memory errors on systems with ECC (Error-Correcting Code) memory or other error detection mechanisms.

Intel i10nm EDAC driver supports the Intel 10nm series server integrated memory controller.
Intel EDAC drivers supported:
- **i10nm_edac**: ICX, SPR, EMR, GNR, SRF, CWF and other 10nm+ server platforms (CPU family 6)
- **imh_edac**: Diamond Rapids (DMR) and future IMH platforms (CPU family 19)

The test script auto-detects the platform and selects the appropriate driver.

## Test Cases

| Test | Description |
|------|-------------|
| check_edac_bus | Verify EDAC bus exists and has registered devices |
| check_edac_driver | Verify EDAC driver is loaded (attempts modprobe if not) |
| edac_addr_decode | Test address decoding via EDAC debugfs interface |
| edac_mc_index_change | Inject errors and verify multiple MC indices in decode log |
| edac_mc_check_populated | Cross-check decoded MCs against sysfs populated MCs |

## Usage

You can run the cases one by one, e.g. command
Run cases individually:

```
./intel_edac.sh -t check_edac_bus
./intel_edac.sh -t edac_addr_decode
```
You also can run the cases together with runtests command, e.g.

Run all cases with runtests:

```
cd ..
Expand Down
258 changes: 247 additions & 11 deletions BM/edac/intel_edac.sh
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0-only
# Copyright (c) 2022 Intel Corporation
# Description: Test script for Intel i10nm EDAC driver, which supports 10nm series server
# Description: Test script for Intel EDAC drivers (i10nm_edac and imh_edac)
# EDAC: Error Detection and Correction
# @Author Yi Lai yi1.lai@intel.com

Expand All @@ -10,7 +10,6 @@ cd "$(dirname "$0")" 2>/dev/null || exit 1
source ../.env

EDAC_BUS="/sys/bus/edac"
EDAC_DRIVER=i10nm_edac

: "${CASE_NAME:=""}"

Expand All @@ -22,25 +21,261 @@ usage() {
__EOF
}

# Detect CPU family to determine EDAC driver variant and debugfs path
# Family 19 (0x13): IMH - Diamond Rapids (DMR) and future IMH platforms
# Family 6: i10nm - ICX, SPR, EMR, GNR, SRF, CWF and other 10nm+ server platforms
get_edac_debug_path() {
get_cpu_model
case $FML in
13)
MC="imh"
EDAC_DEBUG_PATH="/sys/kernel/debug/edac/imh_test/addr"
;;
*)
MC="i10nm"
EDAC_DEBUG_PATH="/sys/kernel/debug/edac/i10nm_test/addr"
;;
esac
LOG="${PWD}/dmesg.decoding.via.debugfs.${MC}_edac.log"
[[ -f "$LOG" ]] && rm -f "$LOG"
}

edac_addr_decode() {
local MARKER dmesg_output

if [[ -w /dev/kmsg ]]; then
MARKER="EDAC_CHECK_MARKER_$(date +%s%N)"
echo "$MARKER" > /dev/kmsg
fi

echo 0x12345 > "$EDAC_DEBUG_PATH"

if [[ -w /dev/kmsg ]]; then
dmesg_output=$(dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER")
else
dmesg_output=$(dmesg | tail -n 100)
fi

if echo "$dmesg_output" | grep -q -e "ADDR 0x12345"; then
test_print_trc "EDAC address decode successfully"
else
die "Failed to decode EDAC address"
fi
}

edac_test_error_inject() {
local MARKER
local TOLM=2
local SIZE_KB SIZE_GB tmp_addr_file addr_low addr_high

SIZE_KB=$(awk '/MemTotal/ {print $2}' /proc/meminfo)
SIZE_GB=$((SIZE_KB / 1024 / 1024))
tmp_addr_file="${PWD}/einj_edac.txt"
true > "$tmp_addr_file"

for ((i = 0; i < (SIZE_GB + TOLM) * 4; i += 4)); do
addr_low=$(head -c 32 /dev/urandom | md5sum | head -c 7)
addr_high=$(printf "%x" "$i")
echo "0x${addr_high}${addr_low}" >> "$tmp_addr_file"
done

if [[ -w /dev/kmsg ]]; then
MARKER="EDAC_INJECT_MARKER_$(date +%s%N)"
echo "$MARKER" > /dev/kmsg
fi

while read -r addr; do
echo "$addr" > "$EDAC_DEBUG_PATH"
done < "$tmp_addr_file"

if [[ -w /dev/kmsg ]]; then
dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER" >> "$LOG"
else
dmesg | tail -n 1000 >> "$LOG"
fi
rm -f "$tmp_addr_file"
}

edac_test_error_inject_iomem() {
local MARKER
local PAGESIZE=4096
local NUM_TESTADDR=40
local RANGE_SIZE_THR=500
local tmp_addr_file iomem_tmp URANDOM
local start_addr end_addr rand_addr test_pfn_base test_pfn test_addr

tmp_addr_file="${PWD}/einj_iomem.txt"
iomem_tmp="${PWD}/iomem_tmp"
true > "$tmp_addr_file"

URANDOM=$(od -An -N4 -t uL /dev/urandom | tr -d " ")
grep "System RAM" /proc/iomem | cut -d ':' -f1 > "$iomem_tmp"

while read -r line; do
start_addr=$((16#$(echo "$line" | awk -F '-' '{print $1}')))
end_addr=$((16#$(echo "$line" | awk -F '-' '{print $2}')))
# skip address < 1MB
((start_addr < 0x100000)) && continue
# skip small memory areas (<500MB)
(((end_addr - start_addr) < (RANGE_SIZE_THR * 0x100000))) && continue

rand_addr=$((start_addr + URANDOM % (end_addr - start_addr)))
if ((rand_addr + NUM_TESTADDR * PAGESIZE > end_addr)); then
rand_addr=$start_addr
fi
test_pfn_base=$((rand_addr / PAGESIZE))
for ((i = 1; i <= NUM_TESTADDR; i++)); do
test_pfn=$((test_pfn_base + i))
test_addr=$((test_pfn * PAGESIZE))
((test_addr > end_addr)) && break
printf "0x%lx\n" "$test_addr" >> "$tmp_addr_file"
done
done < "$iomem_tmp"

if [[ -w /dev/kmsg ]]; then
MARKER="EDAC_INJECT_IOMEM_MARKER_$(date +%s%N)"
echo "$MARKER" > /dev/kmsg
fi

while read -r addr; do
echo "$addr" > "$EDAC_DEBUG_PATH"
done < "$tmp_addr_file"

if [[ -w /dev/kmsg ]]; then
dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER" >> "$LOG"
else
dmesg | tail -n 1000 >> "$LOG"
fi
rm -f "$tmp_addr_file" "$iomem_tmp"
}

edac_mc_index_change() {
local mc_count
mc_count=$(grep -oE "EDAC MC[0-9]+:" "$LOG" | sort -u | wc -l)
if ((mc_count > 1)); then
test_print_trc "Decoding log contains $mc_count unique Memory Controller entries"
else
die "Decoding log contains only $mc_count Memory Controller entry"
fi
}

edac_retry_rd_err_log_check() {
local mode=$1
local MARKER dmesg_output

if [[ -w /dev/kmsg ]]; then
MARKER="EDAC_RRL_MARKER_$(date +%s%N)"
echo "$MARKER" > /dev/kmsg
fi

# Reload EDAC driver with retry_rd_err_log parameter
modprobe -r "${MC}_edac" 2>/dev/null
modprobe "${MC}_edac" retry_rd_err_log="$mode" || die "Failed to load ${MC}_edac with retry_rd_err_log=$mode"

# Unload and reload to cover kernel teardown paths
modprobe -r "${MC}_edac"
modprobe "${MC}_edac" retry_rd_err_log="$mode" || die "Failed to reload ${MC}_edac with retry_rd_err_log=$mode"

# Trigger EDAC address decode to exercise the RRL path
echo 0x12345 > "$EDAC_DEBUG_PATH"

if [[ -w /dev/kmsg ]]; then
dmesg_output=$(dmesg | sed -n "/$MARKER/,\$p" | grep -v "$MARKER")
else
dmesg_output=$(dmesg | tail -n 1000)
fi

if echo "$dmesg_output" | grep -q -e "retry_rd_err_log"; then
if [[ "$MC" == "imh" ]]; then
if echo "$dmesg_output" | grep -iq "SubChId"; then
test_print_trc "EDAC retry_rd_err_log=$mode and SubChId checked successfully"
else
die "retry_rd_err_log=$mode: Failed to find SubChId in dmesg (required for $MC)"
fi
else
test_print_trc "EDAC retry_rd_err_log=$mode checked successfully"
fi
else
die "Failed to find retry_rd_err_log in dmesg (mode=$mode)"
fi
}

edac_mc_check_populated() {
local mc_indices sorted_mc_indices populated_indexes sorted_populated
local SYSFS_EDAC_MC_DIR="/sys/devices/system/edac/mc"

mapfile -t mc_indices < <(grep -oE "EDAC MC[0-9]+:" "$LOG" | sort -u | grep -oE "[0-9]+")
mapfile -t sorted_mc_indices < <(printf "%s\n" "${mc_indices[@]}" | sort -n)

[[ -d "$SYSFS_EDAC_MC_DIR" ]] || die "EDAC mc structure not found under $SYSFS_EDAC_MC_DIR"

populated_indexes=()
for size_file in "$SYSFS_EDAC_MC_DIR"/mc*/size_mb; do
[[ -r "$size_file" ]] || continue
local size num mc_dir
size=$(cat "$size_file" 2>/dev/null)
if [[ -n "$size" ]] && ((size > 0)); then
mc_dir=$(dirname "$size_file")
num=$(basename "$mc_dir" | tr -d 'mc')
populated_indexes+=("$num")
fi
done

[[ ${#populated_indexes[@]} -gt 0 ]] || die "No populated memory controllers found"
mapfile -t sorted_populated < <(printf "%s\n" "${populated_indexes[@]}" | sort -n)

test_print_trc "Populated MCs from sysfs: ${sorted_populated[*]}"
test_print_trc "Decoded MCs from log: ${sorted_mc_indices[*]}"

if [[ "${sorted_mc_indices[*]}" == "${sorted_populated[*]}" ]]; then
test_print_trc "All populated Memory Controllers successfully triggered errors"
else
die "Mismatch between populated MCs and decoded MCs"
fi
}

edac_test() {
case $TEST_SCENARIO in
check_edac_bus)
edac_bus=$(ls $EDAC_BUS)
if [ -n "$edac_bus" ]; then
test_print_trc "EDAC bus is found"
else
if [[ ! -d "$EDAC_BUS" ]]; then
die "EDAC bus is not found"
fi
edac_devices=$(ls "$EDAC_BUS"/devices/ 2>/dev/null)
if [[ -n "$edac_devices" ]]; then
test_print_trc "EDAC bus is found with devices: $edac_devices"
else
die "EDAC bus exists but no EDAC devices are registered"
fi
;;
check_edac_driver)
test_print_trc "Check Intel i10nm edac driver"
lsmod | grep -q $EDAC_DRIVER
if ! lsmod | grep -q $EDAC_DRIVER; then
die "Intel i10nm edac driver is not loaded"
test_print_trc "Check Intel ${MC}_edac driver"
if ! lsmod | grep -q "${MC}_edac"; then
test_print_trc "${MC}_edac not loaded, attempting to load"
modprobe "${MC}_edac" || die "Failed to load ${MC}_edac driver"
fi
if lsmod | grep -q "${MC}_edac"; then
test_print_trc "Intel ${MC}_edac driver is loaded"
else
test_print_trc "Intel i10nm edac driver is loaded"
die "Intel ${MC}_edac driver is not loaded"
fi
;;
edac_addr_decode)
edac_addr_decode
;;
edac_mc_index_change)
edac_test_error_inject
edac_mc_index_change
;;
edac_mc_check_populated)
edac_test_error_inject_iomem
edac_mc_check_populated
;;
edac_rrl_mode1)
edac_retry_rd_err_log_check 1
;;
edac_rrl_mode2)
edac_retry_rd_err_log_check 2
;;
esac
}

Expand All @@ -63,4 +298,5 @@ while getopts :t:H arg; do
esac
done

get_edac_debug_path
edac_test
7 changes: 6 additions & 1 deletion BM/edac/tests
Original file line number Diff line number Diff line change
@@ -1,5 +1,10 @@
# This file collects Intel EDAC driver testcases which can run against
# on Intel 10nm series server
# Intel server platforms (i10nm: ICX/SPR/EMR/GNR/SRF/CWF, imh: DMR+)

intel_edac.sh -t check_edac_bus
intel_edac.sh -t check_edac_driver
intel_edac.sh -t edac_addr_decode
intel_edac.sh -t edac_mc_index_change
intel_edac.sh -t edac_mc_check_populated
intel_edac.sh -t edac_rrl_mode1
intel_edac.sh -t edac_rrl_mode2
Loading