[HIP] Add blender test

External/HIP/CMakeLists.txt

@@ -1,6 +1,9 @@
 include(External)
 include(GPUTestVariant)
 llvm_externals_find(TEST_SUITE_HIP_ROOT "hip" "HIP prerequisites")
+message(STATUS "TEST_SUITE_HIP_ROOT: ${TEST_SUITE_HIP_ROOT}")
+get_filename_component(HIP_CLANG_PATH ${CMAKE_CXX_COMPILER} DIRECTORY)
+message(STATUS "HIP_CLANG_PATH: ${HIP_CLANG_PATH}")
 
 # Create targets for HIP tests that are part of the test suite.
 macro(create_local_hip_tests VariantSuffix)
@@ -35,6 +38,17 @@ macro(create_local_hip_tests VariantSuffix)
                           ${VariantOffload} ${VariantSuffix}
                           "${VariantCPPFLAGS}" "${VariantLibs}")
   endforeach()
+
+  # Add test for Blender.
+  configure_file(workload/blender/test_blender.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test_blender.sh @ONLY)
+  configure_file(workload/blender/verify_blender.sh.in ${CMAKE_CURRENT_BINARY_DIR}/verify_blender.sh @ONLY)
+  file(COPY utils/log_data.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+  file(COPY utils/compare_image.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+  file(COPY utils/requirements.txt DESTINATION "${CMAKE_CURRENT_BINARY_DIR}")
+  llvm_test_run(EXECUTABLE "/bin/bash" "test_blender.sh")
+  llvm_test_verify(/bin/bash verify_blender.sh %o)
+  llvm_add_test(blender.test test_blender.sh)
+  list(APPEND VARIANT_SIMPLE_TEST_TARGETS blender.test)
 endmacro()
 
 function(create_hip_test VariantSuffix)

External/HIP/lit.local.cfg

@@ -5,6 +5,15 @@ import os
 hip_env_vars = [
     "HIP_VISIBLE_DEVICES",
     "LD_LIBRARY_PATH",
+    "HIP_BLENDER_TEST_CCC_OVERRIDE_OPTIONS",
+    "HIP_BLENDER_TEST_PERF_THRESH",
+    "HIP_BLENDER_TEST_OPTIONS",
+    "HIP_BLENDER_TEST_BIN_DIR",
+    "HIP_BLENDER_TEST_SCENES_DIR",
+    "HIP_BLENDER_TEST_LOG_DIR",
+    "HIP_BLENDER_TEST_WORK_DIR",
+    "HIPCC_VERBOSE",
+    "HIP_CLANG_PATH",
 ]
 
 for var in hip_env_vars:

External/HIP/utils/compare_image.py

@@ -0,0 +1,73 @@
+#!/bin/python3
+import argparse
+import os
+import sys
+
+try:
+    import cv2
+    import numpy as np
+    from skimage.metrics import structural_similarity as ssim
+except ImportError:
+    print("One or more required packages are not installed. Please install them using the command:")
+    print("pip install -r requirements.txt")
+    sys.exit(1)
+
+def mse(imageA, imageB):
+    err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2)
+    err /= float(imageA.shape[0] * imageA.shape[1] * imageA.shape[2])
+    return err
+
+def compare_images(image_path1, image_path2, ssim_threshold=0.9, mse_threshold=1000):
+    image1 = cv2.imread(image_path1)
+    image2 = cv2.imread(image_path2)
+
+    if image1 is None or image2 is None:
+        raise ValueError("One or both of the image paths are invalid.")
+
+    if image1.shape != image2.shape:
+        image2 = cv2.resize(image2, (image1.shape[1], image1.shape[0]))
+
+    smaller_side = min(image1.shape[:2])
+    win_size = smaller_side if smaller_side % 2 == 1 else smaller_side - 1
+    win_size = max(win_size, 3)
+
+    ssim_index, diff = ssim(image1, image2, multichannel=True, full=True, win_size=win_size, channel_axis=2)
+    diff = (diff * 255).astype("uint8")
+
+    mse_value = mse(image1, image2)
+
+    # ssim_index is 'structural similarity index' (https://en.wikipedia.org/wiki/Structural_similarity_index_measure),
+    # which is a popular measure of image similarity. Since it only measures gray-scale images, mse_value is also used,
+    # which measures 'mean square error' (https://en.wikipedia.org/wiki/Mean_squared_error). Together they should be a
+    # robust way to spot image differences without false alarms.
+    if ssim_index < ssim_threshold or mse_value > mse_threshold:
+        return -1, ssim_index, mse_value, diff
+    else:
+        return 0, ssim_index, mse_value, None
+
+def main():
+    parser = argparse.ArgumentParser(description="Compare two images for similarity.")
+    parser.add_argument("--image", required=True, help="Path to the first image.")
+    parser.add_argument("--ref", required=True, help="Path to the reference image.")
+    parser.add_argument("--ssim-thresh", type=float, default=0.9, help="Threshold for the Structural Similarity Index (SSI).")
+    parser.add_argument("--mse-thresh", type=float, default=1000, help="Threshold for the Mean Squared Error (MSE).")
+    parser.add_argument("--quiet", action="store_true", help="Suppress output if specified.")
+
+    args = parser.parse_args()
+
+    result, ssim_index, mse_value, diff = compare_images(args.image, args.ref, args.ssim_thresh, args.mse_thresh)
+
+    if not args.quiet:
+        print(f"Result: {result}")
+        print(f"SSIM Index: {ssim_index:.3f}")
+        print(f"MSE Value: {mse_value:.3g}")
+
+    if result == -1:
+        image_path1_stem, image_path1_ext = os.path.splitext(args.image)
+        diff_image_path = f"{image_path1_stem}_diff{image_path1_ext}"
+        cv2.imwrite(diff_image_path, diff)
+        if not args.quiet:
+            print(f"Difference image saved to: {diff_image_path}")
+
+if __name__ == "__main__":
+    main()

External/HIP/utils/log_data.py

@@ -0,0 +1,72 @@
+#!/bin/python3
+import argparse
+import os
+import csv
+import time
+
+def parse_arguments():
+    parser = argparse.ArgumentParser(description='Record data and calculate statistics.')
+    parser.add_argument('--data', type=float, required=True, help='The data value to record.')
+    parser.add_argument('--log-file', type=str, required=True, help='The file to log data to.')
+    parser.add_argument('--label', type=str, required=True, help='The label for the data.')
+    parser.add_argument('--time-stamp', type=str, required=False, help='The timestamp for the data.')
+
+    args = parser.parse_args()
+    return args
+
+def read_existing_data(file_name):
+    data = []
+    if os.path.exists(file_name) and os.path.getsize(file_name) > 0:
+        with open(file_name, 'r') as file:
+            reader = csv.reader(file)
+            for row in reader:
+                if row and row[2].strip():
+                    try:
+                        data.append(float(row[2].strip()))
+                    except ValueError:
+                        continue
+    return data
+
+def calculate_average(data):
+    if not data:
+        return 0.0
+    non_zero_data = [d for d in data if d != 0]
+    if len(non_zero_data) == 0:
+        return 0.0
+    if len(non_zero_data) > 10:
+        non_zero_data = non_zero_data[-10:]
+    return sum(non_zero_data) / len(non_zero_data)
+
+def calculate_percentage_difference(new_value, average):
+    if average == 0:
+        return 0.0
+    return ((new_value - average) / average) * 100
+
+def append_data(file_name, time_stamp, label, data):
+    with open(file_name, 'a', newline='') as file:
+        writer = csv.writer(file)
+        writer.writerow([time_stamp, label, data])
+
+def main():
+    args = parse_arguments()
+
+    data = args.data
+    log_file = args.log_file
+    label = args.label
+    time_stamp = args.time_stamp if args.time_stamp else time.strftime("%Y-%m-%d %H:%M:%S")
+
+    existing_data = read_existing_data(log_file)
+    if not existing_data:
+        average = data
+        percentage_diff = 0.0
+    else:
+        average = calculate_average(existing_data)
+        percentage_diff = calculate_percentage_difference(data, average)
+
+    append_data(log_file, time_stamp, label, data)
+
+    print(f"Average of the last 10 non-zero data points: {average:.3g}")
+    print(f"Percentage difference from current data: {percentage_diff:.2f}%")
+
+if __name__ == "__main__":
+    main()

External/HIP/utils/requirements.txt

@@ -0,0 +1,3 @@
+opencv-python
+scikit-image
+numpy

External/HIP/workload/blender/test_blender.sh.in

@@ -0,0 +1,148 @@
+#!/bin/bash
+
+TEST_SUITE_HIP_ROOT=${TEST_SUITE_HIP_ROOT:-"@TEST_SUITE_HIP_ROOT@"}
+perf_thresh=${HIP_BLENDER_TEST_PERF_THRESH:-5}
+
+export CCC_OVERRIDE_OPTIONS=${HIP_BLENDER_TEST_CCC_OVERRIDE_OPTIONS:-"+-v"}
+
+export HIP_CLANG_PATH=${HIP_CLANG_PATH:-"@HIP_CLANG_PATH@"}
+export HIPCC_VERBOSE=${HIPCC_VERBOSE:-7}
+
+blender_options=${HIP_BLENDER_TEST_OPTIONS:-"-F PNG --debug-cycles -- --cycles-device HIP"}
+blender_dir=${HIP_BLENDER_TEST_BIN_DIR:-"$TEST_SUITE_HIP_ROOT/blender"}
+scene_dir=${HIP_BLENDER_TEST_SCENES_DIR:-"$TEST_SUITE_HIP_ROOT/Blender_Scenes"}
+log_dir=${HIP_BLENDER_TEST_LOG_DIR:-"$scene_dir/logs"}
+work_dir=${HIP_BLENDER_TEST_WORK_DIR:-.}
+summary_file="summary.txt"
+
+# Declare clang_hash as a global variable
+clang_hash=""
+
+get_clang_hash() {
+  clang_version_output=$($HIP_CLANG_PATH/clang -v 2>&1)
+  clang_hash=$(echo "$clang_version_output" | grep -oP '(?<=llvm-project )\w{8}')
+  echo "$clang_hash"
+}
+
+get_blender_version() {
+  blender_version_output=$($blender_dir/blender -v 2>&1)
+  blender_version=$(echo "$blender_version_output" | grep -oP 'Blender \K[0-9]+\.[0-9]+')
+  echo "$blender_version"
+}
+
+# disable pre-built kernels and enable local kernel build
+check_and_rename_lib() {
+  blender_version=$(get_blender_version)
+  major_minor_version=${blender_version}
+  lib_dir="$blender_dir/$major_minor_version/scripts/addons/cycles/lib"
+  if [[ -d "$lib_dir" ]]; then
+    mv "$lib_dir" "${lib_dir}.orig"
+  fi
+}
+
+log_kernel_compilation_time() {
+  blender_output=$1
+  kernel_compilation_time=$(sed -n 's/.*Kernel compilation finished in \([0-9]*\.[0-9]*\)s\./\1/p' $blender_output)
+  echo "Collected kernel compilation time: $kernel_compilation_time s"
+  if [[ ! -z "$kernel_compilation_time" ]]; then
+      mkdir -p "$log_dir"
+      kernel_log_file="$log_dir/kernel_compilation_time.log"
+    python3 log_data.py --data "$kernel_compilation_time" --label "$clang_hash" --log-file "$kernel_log_file"
+  fi
+}
+
+render() {
+  scene=$1
+  out_file=${scene##*/}
+  frame=${2:-1}
+  out_file_full=${out_file}_$(printf "%03d" $frame).png
+  input=$scene_dir/${scene}.blend
+  output=$scene_dir/out/${out_file}_
+  log_file="$log_dir/${out_file}.log"
+  mkdir -p "$log_dir"
+  echo "Render $input"
+
+  blender_output=$(mktemp)
+  timeout 300 $blender_dir/blender -b $input -o ${output}### -f $frame $blender_options 2>&1 | tee $blender_output
+  blender_return_code=${PIPESTATUS[0]}
+
+  average_time=$(grep -P "^\s*Path Tracing\s+\d+\.\d+\s+\d+\.\d+" $blender_output | awk '{print $4}')
+
+  log_kernel_compilation_time $blender_output
+
+  compare_output=$(mktemp)
+  timeout 300 python3 compare_image.py --image $scene_dir/out/${out_file_full} --ref $scene_dir/ref/${out_file_full} 2>&1 | tee $compare_output
+  compare_return_code=${PIPESTATUS[0]}
+
+  ssim=$(grep "SSIM Index:" $compare_output | awk '{print $3}')
+  mse=$(grep "MSE Value:" $compare_output | awk '{print $3}')
+
+  previous_average=""
+  percentage_difference=""
+  perf_regress=0
+
+  if [[ ! -z "$average_time" && "$average_time" != "0" ]]; then
+    log_output=$(python3 log_data.py --data "$average_time" --label "$clang_hash" --log-file "$log_file")
+
+    previous_average=$(echo "$log_output" | grep -oP '(?<=Average of the last 10 non-zero data points: )[^ ]+')
+    percentage_difference=$(echo "$log_output" | grep -oP '(?<=Percentage difference from current data: )[^%]+')
+    percentage_difference=${percentage_difference:-0}
+
+    if (( $(echo "$percentage_difference > $perf_thresh" | bc -l) )); then
+      perf_regress=1
+    fi
+  fi
+
+  echo "$scene $frame $blender_return_code $compare_return_code $perf_regress $average_time $previous_average $percentage_difference $ssim $mse" >> $summary_file
+
+  if [[ $blender_return_code -ne 0 || $compare_return_code -ne 0 || $perf_regress -eq 1 ]]; then
+    return 1
+  fi
+  return 0
+}
+
+run() {
+  cd $work_dir
+  echo "Begin Blender test."
+  hip_dir="$TEST_SUITE_HIP_ROOT"
+  if [[ ! -e "$hip_dir" ]]; then
+    echo "TEST_SUITE_HIP_ROOT=$TEST_SUITE_HIP_ROOT does not exist"
+    exit -1
+  fi
+  echo "TEST_SUITE_HIP_ROOT=$TEST_SUITE_HIP_ROOT"
+  if [[ ! -e "$blender_dir/blender" || ! -e "$scene_dir/scenes.txt" ]]; then
+    echo "Skip HIP Blender test since no blender or test scenes found."
+    echo "To set up HIP Blender test, download or build Blender from https://www.blender.org and install to External/hip/blender directory, and download Blender demo scenes and save to External/hip/Blender_scenes directory. Create a scenes.txt file under the Blender_scenes directory, with each line containing a scene file name and a frame number to render."
+    exit -1
+  fi
+
+  rm -rf ~/.cache/cycles
+  echo "Scene Frame Blender_Return_Code Compare_Return_Code Perf_Regress Average_Time Previous_Average Percentage_Difference SSIM MSE" > $summary_file
+
+  check_and_rename_lib
+
+  clang_hash=$(get_clang_hash)
+
+  all_passed=true
+
+  while IFS=' ' read -r scene frame; do
+    if [[ -z "$scene" || "$scene" == \#* ]]; then
+      continue
+    fi
+    if ! render "$scene" "$frame"; then
+      all_passed=false
+    fi
+  done < "$scene_dir/scenes.txt"
+
+  echo "HIP test summary:"
+  cat $summary_file
+
+  if $all_passed; then
+    echo "Blender test passes."
+  else
+    echo "Blender test fails."
+    return 1
+  fi
+}
+
+run

External/HIP/workload/blender/verify_blender.sh.in

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+grep "Blender test passes" $1
+ret=$?
+if [[ $ret -ne 0 ]]; then
+  cat $1
+fi
+if grep "Skip HIP Blender test since no blender or test scenes found" $1; then
+  exit 0
+fi
+exit $ret