From a20c8ff59363ff6d51babb548f1c1ac7f762dd48 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Wed, 12 Nov 2025 11:38:16 +0000 Subject: [PATCH 1/4] [LV] Pre-commit test for narrow-store-user-mask-op --- .../X86/narrow-to-single-scalar.ll | 32 +++++++++++++++++++ 1 file changed, 32 insertions(+) create mode 100644 llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll diff --git a/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll b/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll new file mode 100644 index 0000000000000..d643748e4c4d6 --- /dev/null +++ b/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll @@ -0,0 +1,32 @@ +; REQUIRES: asserts +; RUN: not --crash opt -p loop-vectorize -mcpu=skylake -S %s + +target triple = "x86_64-unknown-linux-gnu" + +@p = external global [3952 x i8], align 8 +@q = external global [3952 x i8], align 8 + +define void @narrow_store_user_mask_operand(i32 %x) { +entry: + br label %loop.ph + +loop.ph: + %iv = phi i64 [ 0, %entry ], [ %iv.next, %loop.tail ] + %x.pos = icmp sgt i32 %x, 0 + br i1 %x.pos, label %loop.body, label %loop.tail + +loop.body: + %ld.p = load double, ptr @p + %gep.q.iv = getelementptr double, ptr @q, i64 %iv + %gep.q.iv.8 = getelementptr i8, ptr %gep.q.iv, i64 -8 + store double %ld.p, ptr %gep.q.iv.8 + br label %loop.tail + +loop.tail: + %iv.next = add i64 %iv, 1 + %ec = icmp eq i64 %iv, 1 + br i1 %ec, label %exit, label %loop.ph + +exit: + ret void +} From c5ac8de5b170ff809a119d66ddc451f8da61831b Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Wed, 12 Nov 2025 12:23:56 +0000 Subject: [PATCH 2/4] [VPlan] Fix assert in store-user in narrowToSingleScalars Follow up on c2d4c7c18b96 ([VPlan] Permit more users in narrowToSingleScalars) to fix an assert related to WidenStore users of the recipe being narrowed in narrowToSingleScalars. --- .../Transforms/Vectorize/VPlanTransforms.cpp | 1 + .../X86/narrow-to-single-scalar.ll | 25 +++++++++++++++++-- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 7cef98f465715..bf9cffc5dfe31 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1437,6 +1437,7 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { // we checked the RepOrWidenR operand against // vputils::isSingleScalar. assert(RepOrWidenR == Store->getAddr() || + RepOrWidenR == Store->getMask() || vputils::isSingleScalar(Store->getStoredValue())); return true; } diff --git a/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll b/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll index d643748e4c4d6..94a05a67a0bdc 100644 --- a/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll +++ b/llvm/test/Transforms/LoopVectorize/X86/narrow-to-single-scalar.ll @@ -1,5 +1,5 @@ -; REQUIRES: asserts -; RUN: not --crash opt -p loop-vectorize -mcpu=skylake -S %s +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -p loop-vectorize -mcpu=skylake -S %s | FileCheck %s target triple = "x86_64-unknown-linux-gnu" @@ -7,6 +7,27 @@ target triple = "x86_64-unknown-linux-gnu" @q = external global [3952 x i8], align 8 define void @narrow_store_user_mask_operand(i32 %x) { +; CHECK-LABEL: define void @narrow_store_user_mask_operand( +; CHECK-SAME: i32 [[X:%.*]]) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: [[ENTRY:.*]]: +; CHECK-NEXT: br label %[[LOOP_PH:.*]] +; CHECK: [[LOOP_PH]]: +; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP_TAIL:.*]] ] +; CHECK-NEXT: [[X_POS:%.*]] = icmp sgt i32 [[X]], 0 +; CHECK-NEXT: br i1 [[X_POS]], label %[[LOOP_BODY:.*]], label %[[LOOP_TAIL]] +; CHECK: [[LOOP_BODY]]: +; CHECK-NEXT: [[LD_P:%.*]] = load double, ptr @p, align 8 +; CHECK-NEXT: [[GEP_Q_IV:%.*]] = getelementptr double, ptr @q, i64 [[IV]] +; CHECK-NEXT: [[GEP_Q_IV_8:%.*]] = getelementptr i8, ptr [[GEP_Q_IV]], i64 -8 +; CHECK-NEXT: store double [[LD_P]], ptr [[GEP_Q_IV_8]], align 8 +; CHECK-NEXT: br label %[[LOOP_TAIL]] +; CHECK: [[LOOP_TAIL]]: +; CHECK-NEXT: [[IV_NEXT]] = add i64 [[IV]], 1 +; CHECK-NEXT: [[EC:%.*]] = icmp eq i64 [[IV]], 1 +; CHECK-NEXT: br i1 [[EC]], label %[[EXIT:.*]], label %[[LOOP_PH]] +; CHECK: [[EXIT]]: +; CHECK-NEXT: ret void +; entry: br label %loop.ph From 9af51a28af547a0d972b766b939bdc54243c65db Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Wed, 12 Nov 2025 13:33:12 +0000 Subject: [PATCH 3/4] [VPlan] Tweak comment --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index bf9cffc5dfe31..11f80eac29950 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1433,8 +1433,8 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { if (auto *Store = dyn_cast(U)) { // VPWidenStore doesn't have users, and stores are always // profitable to widen: hence, permitting single-scalar stored - // values is an important leaf condition. The assert must hold as - // we checked the RepOrWidenR operand against + // values and mask operands is an important leaf condition. The + // assert must hold as we checked the RepOrWidenR operand against // vputils::isSingleScalar. assert(RepOrWidenR == Store->getAddr() || RepOrWidenR == Store->getMask() || From 7998e20255e6c959a03b67b31bedb1b510199f06 Mon Sep 17 00:00:00 2001 From: Ramkumar Ramachandra Date: Wed, 12 Nov 2025 16:09:56 +0000 Subject: [PATCH 4/4] [VPlan] NFC improvement --- llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp index 11f80eac29950..a0257e760f99a 100644 --- a/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp +++ b/llvm/lib/Transforms/Vectorize/VPlanTransforms.cpp @@ -1432,12 +1432,11 @@ static void narrowToSingleScalarRecipes(VPlan &Plan) { !all_of(RepOrWidenR->users(), [RepOrWidenR](const VPUser *U) { if (auto *Store = dyn_cast(U)) { // VPWidenStore doesn't have users, and stores are always - // profitable to widen: hence, permitting single-scalar stored - // values and mask operands is an important leaf condition. The - // assert must hold as we checked the RepOrWidenR operand against - // vputils::isSingleScalar. - assert(RepOrWidenR == Store->getAddr() || - RepOrWidenR == Store->getMask() || + // profitable to widen: hence, permitting address and mask + // operands, and single-scalar stored values is an important leaf + // condition. The assert must hold as we checked the RepOrWidenR + // operand against vputils::isSingleScalar. + assert(RepOrWidenR != Store->getStoredValue() || vputils::isSingleScalar(Store->getStoredValue())); return true; }