From f55ebfadd8be2e5194df31a28ead89d752b0e1f1 Mon Sep 17 00:00:00 2001 From: "Yaxun (Sam) Liu" Date: Tue, 12 Dec 2023 11:14:28 -0500 Subject: [PATCH] [HIP] Add blender test Add test scripts to run blender with run-time compilation of HIP kernels, which can be used to test clang/llvm built by buildbot. --- External/HIP/CMakeLists.txt | 14 ++ External/HIP/lit.local.cfg | 9 ++ External/HIP/utils/compare_image.py | 73 +++++++++ External/HIP/utils/log_data.py | 72 +++++++++ External/HIP/utils/requirements.txt | 3 + .../HIP/workload/blender/test_blender.sh.in | 148 ++++++++++++++++++ .../HIP/workload/blender/verify_blender.sh.in | 11 ++ 7 files changed, 330 insertions(+) create mode 100644 External/HIP/utils/compare_image.py create mode 100644 External/HIP/utils/log_data.py create mode 100644 External/HIP/utils/requirements.txt create mode 100644 External/HIP/workload/blender/test_blender.sh.in create mode 100644 External/HIP/workload/blender/verify_blender.sh.in diff --git a/External/HIP/CMakeLists.txt b/External/HIP/CMakeLists.txt index dda0ad4f8d..d11add036d 100644 --- a/External/HIP/CMakeLists.txt +++ b/External/HIP/CMakeLists.txt @@ -1,6 +1,9 @@ include(External) include(GPUTestVariant) llvm_externals_find(TEST_SUITE_HIP_ROOT "hip" "HIP prerequisites") +message(STATUS "TEST_SUITE_HIP_ROOT: ${TEST_SUITE_HIP_ROOT}") +get_filename_component(HIP_CLANG_PATH ${CMAKE_CXX_COMPILER} DIRECTORY) +message(STATUS "HIP_CLANG_PATH: ${HIP_CLANG_PATH}") # Create targets for HIP tests that are part of the test suite. macro(create_local_hip_tests VariantSuffix) @@ -35,6 +38,17 @@ macro(create_local_hip_tests VariantSuffix) ${VariantOffload} ${VariantSuffix} "${VariantCPPFLAGS}" "${VariantLibs}") endforeach() + + # Add test for Blender. + configure_file(workload/blender/test_blender.sh.in ${CMAKE_CURRENT_BINARY_DIR}/test_blender.sh @ONLY) + configure_file(workload/blender/verify_blender.sh.in ${CMAKE_CURRENT_BINARY_DIR}/verify_blender.sh @ONLY) + file(COPY utils/log_data.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + file(COPY utils/compare_image.py DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + file(COPY utils/requirements.txt DESTINATION "${CMAKE_CURRENT_BINARY_DIR}") + llvm_test_run(EXECUTABLE "/bin/bash" "test_blender.sh") + llvm_test_verify(/bin/bash verify_blender.sh %o) + llvm_add_test(blender.test test_blender.sh) + list(APPEND VARIANT_SIMPLE_TEST_TARGETS blender.test) endmacro() function(create_hip_test VariantSuffix) diff --git a/External/HIP/lit.local.cfg b/External/HIP/lit.local.cfg index cff41e0128..e36ea3df46 100644 --- a/External/HIP/lit.local.cfg +++ b/External/HIP/lit.local.cfg @@ -5,6 +5,15 @@ import os hip_env_vars = [ "HIP_VISIBLE_DEVICES", "LD_LIBRARY_PATH", + "HIP_BLENDER_TEST_CCC_OVERRIDE_OPTIONS", + "HIP_BLENDER_TEST_PERF_THRESH", + "HIP_BLENDER_TEST_OPTIONS", + "HIP_BLENDER_TEST_BIN_DIR", + "HIP_BLENDER_TEST_SCENES_DIR", + "HIP_BLENDER_TEST_LOG_DIR", + "HIP_BLENDER_TEST_WORK_DIR", + "HIPCC_VERBOSE", + "HIP_CLANG_PATH", ] for var in hip_env_vars: diff --git a/External/HIP/utils/compare_image.py b/External/HIP/utils/compare_image.py new file mode 100644 index 0000000000..f4028c33e7 --- /dev/null +++ b/External/HIP/utils/compare_image.py @@ -0,0 +1,73 @@ +#!/bin/python3 +import argparse +import os +import sys + +try: + import cv2 + import numpy as np + from skimage.metrics import structural_similarity as ssim +except ImportError: + print("One or more required packages are not installed. Please install them using the command:") + print("pip install -r requirements.txt") + sys.exit(1) + +def mse(imageA, imageB): + err = np.sum((imageA.astype("float") - imageB.astype("float")) ** 2) + err /= float(imageA.shape[0] * imageA.shape[1] * imageA.shape[2]) + return err + +def compare_images(image_path1, image_path2, ssim_threshold=0.9, mse_threshold=1000): + image1 = cv2.imread(image_path1) + image2 = cv2.imread(image_path2) + + if image1 is None or image2 is None: + raise ValueError("One or both of the image paths are invalid.") + + if image1.shape != image2.shape: + image2 = cv2.resize(image2, (image1.shape[1], image1.shape[0])) + + smaller_side = min(image1.shape[:2]) + win_size = smaller_side if smaller_side % 2 == 1 else smaller_side - 1 + win_size = max(win_size, 3) + + ssim_index, diff = ssim(image1, image2, multichannel=True, full=True, win_size=win_size, channel_axis=2) + diff = (diff * 255).astype("uint8") + + mse_value = mse(image1, image2) + + # ssim_index is 'structural similarity index' (https://en.wikipedia.org/wiki/Structural_similarity_index_measure), + # which is a popular measure of image similarity. Since it only measures gray-scale images, mse_value is also used, + # which measures 'mean square error' (https://en.wikipedia.org/wiki/Mean_squared_error). Together they should be a + # robust way to spot image differences without false alarms. + if ssim_index < ssim_threshold or mse_value > mse_threshold: + return -1, ssim_index, mse_value, diff + else: + return 0, ssim_index, mse_value, None + +def main(): + parser = argparse.ArgumentParser(description="Compare two images for similarity.") + parser.add_argument("--image", required=True, help="Path to the first image.") + parser.add_argument("--ref", required=True, help="Path to the reference image.") + parser.add_argument("--ssim-thresh", type=float, default=0.9, help="Threshold for the Structural Similarity Index (SSI).") + parser.add_argument("--mse-thresh", type=float, default=1000, help="Threshold for the Mean Squared Error (MSE).") + parser.add_argument("--quiet", action="store_true", help="Suppress output if specified.") + + args = parser.parse_args() + + result, ssim_index, mse_value, diff = compare_images(args.image, args.ref, args.ssim_thresh, args.mse_thresh) + + if not args.quiet: + print(f"Result: {result}") + print(f"SSIM Index: {ssim_index:.3f}") + print(f"MSE Value: {mse_value:.3g}") + + if result == -1: + image_path1_stem, image_path1_ext = os.path.splitext(args.image) + diff_image_path = f"{image_path1_stem}_diff{image_path1_ext}" + cv2.imwrite(diff_image_path, diff) + if not args.quiet: + print(f"Difference image saved to: {diff_image_path}") + +if __name__ == "__main__": + main() diff --git a/External/HIP/utils/log_data.py b/External/HIP/utils/log_data.py new file mode 100644 index 0000000000..c22bc9e1ae --- /dev/null +++ b/External/HIP/utils/log_data.py @@ -0,0 +1,72 @@ +#!/bin/python3 +import argparse +import os +import csv +import time + +def parse_arguments(): + parser = argparse.ArgumentParser(description='Record data and calculate statistics.') + parser.add_argument('--data', type=float, required=True, help='The data value to record.') + parser.add_argument('--log-file', type=str, required=True, help='The file to log data to.') + parser.add_argument('--label', type=str, required=True, help='The label for the data.') + parser.add_argument('--time-stamp', type=str, required=False, help='The timestamp for the data.') + + args = parser.parse_args() + return args + +def read_existing_data(file_name): + data = [] + if os.path.exists(file_name) and os.path.getsize(file_name) > 0: + with open(file_name, 'r') as file: + reader = csv.reader(file) + for row in reader: + if row and row[2].strip(): + try: + data.append(float(row[2].strip())) + except ValueError: + continue + return data + +def calculate_average(data): + if not data: + return 0.0 + non_zero_data = [d for d in data if d != 0] + if len(non_zero_data) == 0: + return 0.0 + if len(non_zero_data) > 10: + non_zero_data = non_zero_data[-10:] + return sum(non_zero_data) / len(non_zero_data) + +def calculate_percentage_difference(new_value, average): + if average == 0: + return 0.0 + return ((new_value - average) / average) * 100 + +def append_data(file_name, time_stamp, label, data): + with open(file_name, 'a', newline='') as file: + writer = csv.writer(file) + writer.writerow([time_stamp, label, data]) + +def main(): + args = parse_arguments() + + data = args.data + log_file = args.log_file + label = args.label + time_stamp = args.time_stamp if args.time_stamp else time.strftime("%Y-%m-%d %H:%M:%S") + + existing_data = read_existing_data(log_file) + if not existing_data: + average = data + percentage_diff = 0.0 + else: + average = calculate_average(existing_data) + percentage_diff = calculate_percentage_difference(data, average) + + append_data(log_file, time_stamp, label, data) + + print(f"Average of the last 10 non-zero data points: {average:.3g}") + print(f"Percentage difference from current data: {percentage_diff:.2f}%") + +if __name__ == "__main__": + main() diff --git a/External/HIP/utils/requirements.txt b/External/HIP/utils/requirements.txt new file mode 100644 index 0000000000..995e8905b2 --- /dev/null +++ b/External/HIP/utils/requirements.txt @@ -0,0 +1,3 @@ +opencv-python +scikit-image +numpy diff --git a/External/HIP/workload/blender/test_blender.sh.in b/External/HIP/workload/blender/test_blender.sh.in new file mode 100644 index 0000000000..46476ee21f --- /dev/null +++ b/External/HIP/workload/blender/test_blender.sh.in @@ -0,0 +1,148 @@ +#!/bin/bash + +TEST_SUITE_HIP_ROOT=${TEST_SUITE_HIP_ROOT:-"@TEST_SUITE_HIP_ROOT@"} +perf_thresh=${HIP_BLENDER_TEST_PERF_THRESH:-5} + +export CCC_OVERRIDE_OPTIONS=${HIP_BLENDER_TEST_CCC_OVERRIDE_OPTIONS:-"+-v"} + +export HIP_CLANG_PATH=${HIP_CLANG_PATH:-"@HIP_CLANG_PATH@"} +export HIPCC_VERBOSE=${HIPCC_VERBOSE:-7} + +blender_options=${HIP_BLENDER_TEST_OPTIONS:-"-F PNG --debug-cycles -- --cycles-device HIP"} +blender_dir=${HIP_BLENDER_TEST_BIN_DIR:-"$TEST_SUITE_HIP_ROOT/blender"} +scene_dir=${HIP_BLENDER_TEST_SCENES_DIR:-"$TEST_SUITE_HIP_ROOT/Blender_Scenes"} +log_dir=${HIP_BLENDER_TEST_LOG_DIR:-"$scene_dir/logs"} +work_dir=${HIP_BLENDER_TEST_WORK_DIR:-.} +summary_file="summary.txt" + +# Declare clang_hash as a global variable +clang_hash="" + +get_clang_hash() { + clang_version_output=$($HIP_CLANG_PATH/clang -v 2>&1) + clang_hash=$(echo "$clang_version_output" | grep -oP '(?<=llvm-project )\w{8}') + echo "$clang_hash" +} + +get_blender_version() { + blender_version_output=$($blender_dir/blender -v 2>&1) + blender_version=$(echo "$blender_version_output" | grep -oP 'Blender \K[0-9]+\.[0-9]+') + echo "$blender_version" +} + +# disable pre-built kernels and enable local kernel build +check_and_rename_lib() { + blender_version=$(get_blender_version) + major_minor_version=${blender_version} + lib_dir="$blender_dir/$major_minor_version/scripts/addons/cycles/lib" + if [[ -d "$lib_dir" ]]; then + mv "$lib_dir" "${lib_dir}.orig" + fi +} + +log_kernel_compilation_time() { + blender_output=$1 + kernel_compilation_time=$(sed -n 's/.*Kernel compilation finished in \([0-9]*\.[0-9]*\)s\./\1/p' $blender_output) + echo "Collected kernel compilation time: $kernel_compilation_time s" + if [[ ! -z "$kernel_compilation_time" ]]; then + mkdir -p "$log_dir" + kernel_log_file="$log_dir/kernel_compilation_time.log" + python3 log_data.py --data "$kernel_compilation_time" --label "$clang_hash" --log-file "$kernel_log_file" + fi +} + +render() { + scene=$1 + out_file=${scene##*/} + frame=${2:-1} + out_file_full=${out_file}_$(printf "%03d" $frame).png + input=$scene_dir/${scene}.blend + output=$scene_dir/out/${out_file}_ + log_file="$log_dir/${out_file}.log" + mkdir -p "$log_dir" + echo "Render $input" + + blender_output=$(mktemp) + timeout 300 $blender_dir/blender -b $input -o ${output}### -f $frame $blender_options 2>&1 | tee $blender_output + blender_return_code=${PIPESTATUS[0]} + + average_time=$(grep -P "^\s*Path Tracing\s+\d+\.\d+\s+\d+\.\d+" $blender_output | awk '{print $4}') + + log_kernel_compilation_time $blender_output + + compare_output=$(mktemp) + timeout 300 python3 compare_image.py --image $scene_dir/out/${out_file_full} --ref $scene_dir/ref/${out_file_full} 2>&1 | tee $compare_output + compare_return_code=${PIPESTATUS[0]} + + ssim=$(grep "SSIM Index:" $compare_output | awk '{print $3}') + mse=$(grep "MSE Value:" $compare_output | awk '{print $3}') + + previous_average="" + percentage_difference="" + perf_regress=0 + + if [[ ! -z "$average_time" && "$average_time" != "0" ]]; then + log_output=$(python3 log_data.py --data "$average_time" --label "$clang_hash" --log-file "$log_file") + + previous_average=$(echo "$log_output" | grep -oP '(?<=Average of the last 10 non-zero data points: )[^ ]+') + percentage_difference=$(echo "$log_output" | grep -oP '(?<=Percentage difference from current data: )[^%]+') + percentage_difference=${percentage_difference:-0} + + if (( $(echo "$percentage_difference > $perf_thresh" | bc -l) )); then + perf_regress=1 + fi + fi + + echo "$scene $frame $blender_return_code $compare_return_code $perf_regress $average_time $previous_average $percentage_difference $ssim $mse" >> $summary_file + + if [[ $blender_return_code -ne 0 || $compare_return_code -ne 0 || $perf_regress -eq 1 ]]; then + return 1 + fi + return 0 +} + +run() { + cd $work_dir + echo "Begin Blender test." + hip_dir="$TEST_SUITE_HIP_ROOT" + if [[ ! -e "$hip_dir" ]]; then + echo "TEST_SUITE_HIP_ROOT=$TEST_SUITE_HIP_ROOT does not exist" + exit -1 + fi + echo "TEST_SUITE_HIP_ROOT=$TEST_SUITE_HIP_ROOT" + if [[ ! -e "$blender_dir/blender" || ! -e "$scene_dir/scenes.txt" ]]; then + echo "Skip HIP Blender test since no blender or test scenes found." + echo "To set up HIP Blender test, download or build Blender from https://www.blender.org and install to External/hip/blender directory, and download Blender demo scenes and save to External/hip/Blender_scenes directory. Create a scenes.txt file under the Blender_scenes directory, with each line containing a scene file name and a frame number to render." + exit -1 + fi + + rm -rf ~/.cache/cycles + echo "Scene Frame Blender_Return_Code Compare_Return_Code Perf_Regress Average_Time Previous_Average Percentage_Difference SSIM MSE" > $summary_file + + check_and_rename_lib + + clang_hash=$(get_clang_hash) + + all_passed=true + + while IFS=' ' read -r scene frame; do + if [[ -z "$scene" || "$scene" == \#* ]]; then + continue + fi + if ! render "$scene" "$frame"; then + all_passed=false + fi + done < "$scene_dir/scenes.txt" + + echo "HIP test summary:" + cat $summary_file + + if $all_passed; then + echo "Blender test passes." + else + echo "Blender test fails." + return 1 + fi +} + +run diff --git a/External/HIP/workload/blender/verify_blender.sh.in b/External/HIP/workload/blender/verify_blender.sh.in new file mode 100644 index 0000000000..25b3affe78 --- /dev/null +++ b/External/HIP/workload/blender/verify_blender.sh.in @@ -0,0 +1,11 @@ +#!/bin/bash + +grep "Blender test passes" $1 +ret=$? +if [[ $ret -ne 0 ]]; then + cat $1 +fi +if grep "Skip HIP Blender test since no blender or test scenes found" $1; then + exit 0 +fi +exit $ret \ No newline at end of file