Skip to content

Commit

Permalink
[aotinductor] Add a perf smoke test for AOTInductor
Browse files Browse the repository at this point in the history
Summary: To prevent perf regression like the one caused by #110510

ghstack-source-id: 329434d1fd74d36cede2096033da26615916606d
Pull Request resolved: #110972
  • Loading branch information
desertfire committed Oct 10, 2023
1 parent 5183760 commit c04e236
Show file tree
Hide file tree
Showing 2 changed files with 13 additions and 9 deletions.
10 changes: 7 additions & 3 deletions .ci/pytorch/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -485,9 +485,13 @@ test_inductor_torchbench_smoketest_perf() {
python benchmarks/dynamo/torchbench.py --device cuda --performance --backend inductor --float16 --training \
--batch-size-file "$(realpath benchmarks/dynamo/torchbench_models_list.txt)" --only hf_Bert \
--output "$TEST_REPORTS_DIR/inductor_training_smoketest.csv"
# the reference speedup value is hardcoded in check_hf_bert_perf_csv.py
# this value needs to be actively maintained to make this check useful
python benchmarks/dynamo/check_hf_bert_perf_csv.py -f "$TEST_REPORTS_DIR/inductor_training_smoketest.csv"
# The threshold value needs to be actively maintained to make this check useful
python benchmarks/dynamo/check_perf_csv.py -f "$TEST_REPORTS_DIR/inductor_training_smoketest.csv" -t 1.15

python benchmarks/dynamo/torchbench.py --device cuda --performance --bfloat16 --inference \
--export-aot-inductor --only nanogpt --output "$TEST_REPORTS_DIR/inductor_inference_smoketest.csv"
# The threshold value needs to be actively maintained to make this check useful
python benchmarks/dynamo/check_perf_csv.py -f "$TEST_REPORTS_DIR/inductor_inference_smoketest.csv" -t 5.5

# Check memory compression ratio for a few models
for test in hf_Albert timm_vision_transformer; do
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,7 @@
import pandas as pd


def check_hf_bert_perf_csv(filename):
def check_perf_csv(filename, threshold):
"""
Basic performance checking.
"""
Expand All @@ -16,10 +16,7 @@ def check_hf_bert_perf_csv(filename):
for _, row in df.iterrows():
model_name = row["name"]
speedup = row["speedup"]
# Reduce from 1.165 to 1.160, see https://github.com/pytorch/pytorch/issues/96530
# Reduce from 1.160 to 1.140 after a transformer version upgrade, see https://github.com/pytorch/benchmark/pull/1406
# The speedup is not backed to 1.16 after the extra graph break issue is fixed in transformer upstream
if speedup < 1.150:
if speedup < threshold:
failed.append(model_name)

print(f"{model_name:34} {speedup}")
Expand All @@ -39,5 +36,8 @@ def check_hf_bert_perf_csv(filename):
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("--file", "-f", type=str, help="csv file name")
parser.add_argument(
"--threshold", "-t", type=float, help="threshold speedup value to check against"
)
args = parser.parse_args()
check_hf_bert_perf_csv(args.file)
check_perf_csv(args.file, args.threshold)

0 comments on commit c04e236

Please sign in to comment.