Skip to content

Lock contention siyuan #15

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Nov 15, 2025
Merged

Lock contention siyuan #15

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0907191
automatically detects host & guest IOMMU config
siyuanchai1999 Nov 15, 2025
6721566
Remove the ftrace setup before warmup. We should do it after warmup
siyuanchai1999 Nov 15, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
72 changes: 52 additions & 20 deletions scripts/sosp24-experiments/vm_flows_exp.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ CLIENT_USE_PASS_AUTH=0 # 1 to use password, 0 to use identity file
CLIENT_SSH_IDENTITY_FILE="/home/schai/.ssh/id_rsa"

# off, shadow or nested
VIRT_TECH="off"
VIRT_TECH="nested"

function verify_virt_tech() {
local tech="$1"
Expand All @@ -62,6 +62,49 @@ function verify_virt_tech() {
fi
}

parse_iommu_mode() {
local cmdline="${1:-$(</proc/cmdline)}"
local cl
cl="$(printf '%s' "$cmdline" | tr '[:upper:]' '[:lower:]')"

# Passthrough (separate case)
if [[ "$cl" =~ (^|[[:space:]])(iommu=pt|iommu\.passthrough=(1|on|y|yes|true))($|[[:space:]]) ]]; then
echo passthrough
return
fi

# Off
if [[ "$cl" =~ (^|[[:space:]])(noiommu|iommu=off|intel_iommu=off|amd_iommu=off)($|[[:space:]]) ]]; then
echo off
return
fi

# Strict
if [[ "$cl" =~ (^|[[:space:]])iommu\.strict=(1|on|y|yes|true)($|[[:space:]]) ]] || \
[[ "$cl" =~ (^|[[:space:]])intel_iommu=([^[:space:]]*,)?strict([^[:space:]]*)($|[[:space:]]) ]] || \
[[ "$cl" =~ (^|[[:space:]])amd_iommu=([^[:space:]]*,)?strict([^[:space:]]*)($|[[:space:]]) ]]; then
echo strict
return
fi

# Lazy (non-strict)
if [[ "$cl" =~ (^|[[:space:]])iommu\.strict=(0|off|n|no|false)($|[[:space:]]) ]] || \
[[ "$cl" =~ (^|[[:space:]])intel_iommu=([^[:space:]]*,)?nonstrict([^[:space:]]*)($|[[:space:]]) ]] || \
[[ "$cl" =~ (^|[[:space:]])amd_iommu=([^[:space:]]*,)?nonstrict([^[:space:]]*)($|[[:space:]]) ]]; then
echo lazy
return
fi

# Explicitly enabled but no strictness specified → assume strict
if [[ "$cl" =~ (^|[[:space:]])(iommu=on|intel_iommu=on|amd_iommu=on)($|[[:space:]]) ]]; then
echo strict
return
fi

# Default if unspecified
echo strict
}

verify_virt_tech $VIRT_TECH

if [ "$CLIENT_USE_PASS_AUTH" -eq 1 ]; then
Expand All @@ -70,26 +113,15 @@ else
SSH_CLIENT_CMD="ssh -i $CLIENT_SSH_IDENTITY_FILE ${CLIENT_SSH_UNAME}@${CLIENT_SSH_HOST}"
fi

iommu_on=$(grep -o intel_iommu=on /proc/cmdline)
iommu_mode=$(sudo dmesg | grep -o 'iommu\.strict=[0-9]' | tail -n1 | cut -d= -f2)
iommu_config=""

if [ -z $iommu_on ]; then
iommu_config="host-strict-guest-off"
else
if [ "$iommu_mode" == "1" ]; then
echo "IOMMU is in STRICT mode, assuming the same for host"
iommu_config="host-strict-guest-on-$VIRT_TECH"
elif [ "$iommu_mode" == "0" ]; then
echo "IOMMU is in LAZY mode, assuming the same for host"
iommu_config="host-lazy-guest-on-$VIRT_TECH"
else
echo "IOMMU strictness could not be determined"
iommu_config="guest-on-$VIRT_TECH"
fi
fi
guest_cmdline=$(cat /proc/cmdline)
guest_iommu_config=$(parse_iommu_mode $guest_cmdline)
host_cmdline=$(ssh -i "$CLIENT_SSH_IDENTITY_FILE" "${HOST_UNAME}@${HOST_IP}" 'cat /proc/cmdline')
host_iommu_config=$(parse_iommu_mode $host_cmdline)

iommu_config="host-${guest_iommu_config}-guest-${host_iommu_config}-$VIRT_TECH"

echo "iommu_config: $iommu_config"
# exit 0
# pause the frame
sudo ethtool --pause $GUEST_INTF tx off rx off
$SSH_CLIENT_CMD "sudo ethtool --pause $CLIENT_INTF tx off rx off"
Expand All @@ -110,7 +142,7 @@ for socket_buf in 1; do
server_cores_mask=($(echo $server_cores | tr ',' '\n' | head -n $num_cores | tr '\n' ','))

format_i=$(printf "%02d\n" $i)
exp_name="${timestamp}-$(uname -r)-flow${format_i}-${iommu_config}-ringbuf-${ring_buffer}_sokcetbuf${socket_buf}_${num_cores}cores"
exp_name="${timestamp}-$(uname -r)-flow${format_i}-${iommu_config}-ringbuf-${ring_buffer}_sokcetbuf1_${num_cores}cores"
echo $exp_name

if [ "$DRY_RUN" -eq 1 ]; then
Expand Down
18 changes: 1 addition & 17 deletions scripts/vm-run-dctcp-tput-experiment.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -378,23 +378,7 @@ for ((j = 0; j < NUM_RUNS; j += 1)); do
sudo bash run-netapp-tput.sh --mode server -n "$GUEST_NUM_SERVERS" -N "$CLIENT_NUM_CLIENTS" -o "${EXP_NAME}-RUN-${j}" \
-p "$INIT_PORT" -c "$GUEST_CPU_MASK" &> "$guest_server_app_log_file" &
sleep 2 # Allow server app to initialize
cd - > /dev/null

# --- Ftrace Setup (Guest & Host) ---
log_info "Configuring GUEST ftrace for IOVA logging (Buffer: ${FTRACE_BUFFER_SIZE_KB}KB, Overwrite: ${FTRACE_OVERWRITE_ON_FULL})..."
sudo echo "$FTRACE_BUFFER_SIZE_KB" > /sys/kernel/debug/tracing/buffer_size_kb
sudo echo "$FTRACE_OVERWRITE_ON_FULL" > /sys/kernel/debug/tracing/options/overwrite
sudo echo > /sys/kernel/debug/tracing/trace # Clear buffer
sudo echo 1 > /sys/kernel/debug/tracing/tracing_on
log_info "GUEST IOVA ftrace is ON."

log_info "Configuring HOST ftrace for IOVA logging on $HOST_IP..."
$SSH_HOST_CMD \
"sudo bash -c 'sudo echo '$FTRACE_BUFFER_SIZE_KB' > /sys/kernel/debug/tracing/buffer_size_kb; \
sudo echo '$FTRACE_OVERWRITE_ON_FULL' > /sys/kernel/debug/tracing/options/overwrite; \
sudo echo > /sys/kernel/debug/tracing/trace; \
sudo echo 1 > /sys/kernel/debug/tracing/tracing_on'"
log_info "HOST IOVA ftrace is ON."
cd - > /dev/null

# --- Setup and Start Clients ---
log_info "Setting up and starting CLIENTS on $CLIENT_SSH_HOST..."
Expand Down