diff --git a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp index cf6d0ecab4f69..e1e24a99d0474 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineCalls.cpp @@ -318,18 +318,18 @@ Value *InstCombinerImpl::simplifyMaskedLoad(IntrinsicInst &II) { // * Single constant active lane -> store // * Narrow width by halfs excluding zero/undef lanes Instruction *InstCombinerImpl::simplifyMaskedStore(IntrinsicInst &II) { + Value *StorePtr = II.getArgOperand(1); + Align Alignment = cast(II.getArgOperand(2))->getAlignValue(); auto *ConstMask = dyn_cast(II.getArgOperand(3)); if (!ConstMask) return nullptr; // If the mask is all zeros, this instruction does nothing. - if (ConstMask->isNullValue()) + if (maskIsAllZeroOrUndef(ConstMask)) return eraseInstFromFunction(II); // If the mask is all ones, this is a plain vector store of the 1st argument. - if (ConstMask->isAllOnesValue()) { - Value *StorePtr = II.getArgOperand(1); - Align Alignment = cast(II.getArgOperand(2))->getAlignValue(); + if (maskIsAllOneOrUndef(ConstMask)) { StoreInst *S = new StoreInst(II.getArgOperand(0), StorePtr, false, Alignment); S->copyMetadata(II); @@ -389,7 +389,7 @@ Instruction *InstCombinerImpl::simplifyMaskedScatter(IntrinsicInst &II) { return nullptr; // If the mask is all zeros, a scatter does nothing. - if (ConstMask->isNullValue()) + if (maskIsAllZeroOrUndef(ConstMask)) return eraseInstFromFunction(II); // Vector splat address -> scalar store diff --git a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll index 8f7683419a82a..67ab167c189b4 100644 --- a/llvm/test/Transforms/InstCombine/masked_intrinsics.ll +++ b/llvm/test/Transforms/InstCombine/masked_intrinsics.ll @@ -16,6 +16,14 @@ define <2 x double> @load_zeromask(ptr %ptr, <2 x double> %passthru) { ret <2 x double> %res } +define <2 x double> @load_zero_withpoison_mask(ptr %ptr, <2 x double> %passthru) { +; CHECK-LABEL: @load_zero_withpoison_mask( +; CHECK-NEXT: ret <2 x double> [[PASSTHRU:%.*]] +; + %res = call <2 x double> @llvm.masked.load.v2f64.p0(ptr %ptr, i32 1, <2 x i1> , <2 x double> %passthru) + ret <2 x double> %res +} + define <2 x double> @load_onemask(ptr %ptr, <2 x double> %passthru) { ; CHECK-LABEL: @load_onemask( ; CHECK-NEXT: [[UNMASKEDLOAD:%.*]] = load <2 x double>, ptr [[PTR:%.*]], align 2 @@ -150,6 +158,14 @@ define void @store_zeromask(ptr %ptr, <2 x double> %val) { ret void } +define void @store_poisonmask(ptr %ptr, <2 x double> %val) { +; CHECK-LABEL: @store_poisonmask( +; CHECK-NEXT: ret void +; + call void @llvm.masked.store.v2f64.p0(<2 x double> %val, ptr %ptr, i32 4, <2 x i1> splat(i1 poison)) + ret void +} + define void @store_onemask(ptr %ptr, <2 x double> %val) { ; CHECK-LABEL: @store_onemask( ; CHECK-NEXT: store <2 x double> [[VAL:%.*]], ptr [[PTR:%.*]], align 4 @@ -159,6 +175,15 @@ define void @store_onemask(ptr %ptr, <2 x double> %val) { ret void } +define void @store_one_withpoison_mask(ptr %ptr, <2 x double> %val) { +; CHECK-LABEL: @store_one_withpoison_mask( +; CHECK-NEXT: store <2 x double> [[VAL:%.*]], ptr [[PTR:%.*]], align 4 +; CHECK-NEXT: ret void +; + call void @llvm.masked.store.v2f64.p0(<2 x double> %val, ptr %ptr, i32 4, <2 x i1> ) + ret void +} + define void @store_demandedelts(ptr %ptr, double %val) { ; CHECK-LABEL: @store_demandedelts( ; CHECK-NEXT: [[VALVEC1:%.*]] = insertelement <2 x double> poison, double [[VAL:%.*]], i64 0 @@ -189,6 +214,13 @@ define <2 x double> @gather_zeromask(<2 x ptr> %ptrs, <2 x double> %passthru) { ret <2 x double> %res } +define <2 x double> @gather_zero_withpoison_mask(<2 x ptr> %ptrs, <2 x double> %passthru) { +; CHECK-LABEL: @gather_zero_withpoison_mask( +; CHECK-NEXT: ret <2 x double> [[PASSTHRU:%.*]] +; + %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> , <2 x double> %passthru) + ret <2 x double> %res +} define <2 x double> @gather_onemask(<2 x ptr> %ptrs, <2 x double> %passthru) { ; CHECK-LABEL: @gather_onemask( @@ -199,6 +231,15 @@ define <2 x double> @gather_onemask(<2 x ptr> %ptrs, <2 x double> %passthru) { ret <2 x double> %res } +define <2 x double> @gather_one_withpoisonmask(<2 x ptr> %ptrs, <2 x double> %passthru) { +; CHECK-LABEL: @gather_one_withpoisonmask( +; CHECK-NEXT: [[RES:%.*]] = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> [[PTRS:%.*]], i32 4, <2 x i1> , <2 x double> [[PASSTHRU:%.*]]) +; CHECK-NEXT: ret <2 x double> [[RES]] +; + %res = call <2 x double> @llvm.masked.gather.v2f64.v2p0(<2 x ptr> %ptrs, i32 4, <2 x i1> , <2 x double> %passthru) + ret <2 x double> %res +} + define <4 x double> @gather_lane2(ptr %base, double %pt) { ; CHECK-LABEL: @gather_lane2( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[BASE:%.*]], <4 x i64> @@ -257,6 +298,23 @@ define void @scatter_zeromask(<2 x ptr> %ptrs, <2 x double> %val) { ret void } +define void @scatter_zero_withpoison_mask(<2 x ptr> %ptrs, <2 x double> %val) { +; CHECK-LABEL: @scatter_zero_withpoison_mask( +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> %val, <2 x ptr> %ptrs, i32 8, <2 x i1> ) + ret void +} + +define void @scatter_one_withpoison_mask(<2 x ptr> %ptrs, <2 x double> %val) { +; CHECK-LABEL: @scatter_one_withpoison_mask( +; CHECK-NEXT: call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> [[VAL:%.*]], <2 x ptr> [[PTRS:%.*]], i32 8, <2 x i1> ) +; CHECK-NEXT: ret void +; + call void @llvm.masked.scatter.v2f64.v2p0(<2 x double> %val, <2 x ptr> %ptrs, i32 8, <2 x i1> ) + ret void +} + define void @scatter_demandedelts(ptr %ptr, double %val) { ; CHECK-LABEL: @scatter_demandedelts( ; CHECK-NEXT: [[PTRS:%.*]] = getelementptr double, ptr [[PTR:%.*]], <2 x i64> diff --git a/llvm/test/Transforms/InstCombine/pr83947.ll b/llvm/test/Transforms/InstCombine/pr83947.ll index 1906502fdaf23..679230a46d369 100644 --- a/llvm/test/Transforms/InstCombine/pr83947.ll +++ b/llvm/test/Transforms/InstCombine/pr83947.ll @@ -24,7 +24,6 @@ define void @masked_scatter2() { define void @masked_scatter3() { ; CHECK-LABEL: define void @masked_scatter3() { -; CHECK-NEXT: store i32 0, ptr @c, align 4 ; CHECK-NEXT: ret void ; call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> undef) @@ -50,7 +49,6 @@ define void @masked_scatter5() { define void @masked_scatter6() { ; CHECK-LABEL: define void @masked_scatter6() { -; CHECK-NEXT: store i32 0, ptr @c, align 4 ; CHECK-NEXT: ret void ; call void @llvm.masked.scatter.v2i32.v2p0(<2 x i32> zeroinitializer, <2 x ptr> splat (ptr @c), i32 4, <2 x i1> )