-
Notifications
You must be signed in to change notification settings - Fork 15.3k
[CIR][X86] Implement lowering for AVX512 ktest builtins #169985
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change | ||||||||
|---|---|---|---|---|---|---|---|---|---|---|
|
|
@@ -757,14 +757,40 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID, | |||||||||
| case X86::BI__builtin_ia32_vpcomuw: | ||||||||||
| case X86::BI__builtin_ia32_vpcomud: | ||||||||||
| case X86::BI__builtin_ia32_vpcomuq: | ||||||||||
| cgm.errorNYI(expr->getSourceRange(), | ||||||||||
| std::string("unimplemented X86 builtin call: ") + | ||||||||||
| getContext().BuiltinInfo.getName(builtinID)); | ||||||||||
| return {}; | ||||||||||
| case X86::BI__builtin_ia32_kortestcqi: | ||||||||||
| case X86::BI__builtin_ia32_kortestchi: | ||||||||||
| case X86::BI__builtin_ia32_kortestcsi: | ||||||||||
| case X86::BI__builtin_ia32_kortestcdi: | ||||||||||
| case X86::BI__builtin_ia32_kortestcdi: { | ||||||||||
| mlir::Location loc = getLoc(expr->getExprLoc()); | ||||||||||
| cir::IntType ty = cast<cir::IntType>(ops[0].getType()); | ||||||||||
| cir::IntAttr allOnesAttr = | ||||||||||
| cir::IntAttr::get(ty, APInt::getAllOnes(ty.getWidth())); | ||||||||||
| cir::ConstantOp allOnesOp = builder.getConstant(loc, allOnesAttr); | ||||||||||
| mlir::Value orOp = emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops); | ||||||||||
| mlir::Value cmp = | ||||||||||
| cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allOnesOp); | ||||||||||
| return builder.createCast(cir::CastKind::bool_to_int, cmp, | ||||||||||
| cgm.convertType(expr->getType())); | ||||||||||
| } | ||||||||||
| case X86::BI__builtin_ia32_kortestzqi: | ||||||||||
| case X86::BI__builtin_ia32_kortestzhi: | ||||||||||
| case X86::BI__builtin_ia32_kortestzsi: | ||||||||||
| case X86::BI__builtin_ia32_kortestzdi: | ||||||||||
| case X86::BI__builtin_ia32_kortestzdi: { | ||||||||||
| mlir::Location loc = getLoc(expr->getExprLoc()); | ||||||||||
| cir::IntType ty = cast<cir::IntType>(ops[0].getType()); | ||||||||||
| cir::IntAttr allZerosAttr = | ||||||||||
| cir::IntAttr::get(ty, APInt::getZero(ty.getWidth())); | ||||||||||
| cir::ConstantOp allZerosOp = builder.getConstant(loc, allZerosAttr); | ||||||||||
|
Comment on lines
+785
to
+787
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
|
||||||||||
| mlir::Value orOp = emitX86MaskLogic(builder, loc, cir::BinOpKind::Or, ops); | ||||||||||
| mlir::Value cmp = | ||||||||||
| cir::CmpOp::create(builder, loc, cir::CmpOpKind::eq, orOp, allZerosOp); | ||||||||||
| return builder.createCast(cir::CastKind::bool_to_int, cmp, | ||||||||||
| cgm.convertType(expr->getType())); | ||||||||||
| } | ||||||||||
| case X86::BI__builtin_ia32_ktestcqi: | ||||||||||
| case X86::BI__builtin_ia32_ktestzqi: | ||||||||||
| case X86::BI__builtin_ia32_ktestchi: | ||||||||||
|
|
@@ -773,10 +799,6 @@ mlir::Value CIRGenFunction::emitX86BuiltinExpr(unsigned builtinID, | |||||||||
| case X86::BI__builtin_ia32_ktestzsi: | ||||||||||
| case X86::BI__builtin_ia32_ktestcdi: | ||||||||||
| case X86::BI__builtin_ia32_ktestzdi: | ||||||||||
| cgm.errorNYI(expr->getSourceRange(), | ||||||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. This shouldn't be falling through. Is part of your change missing here? |
||||||||||
| std::string("unimplemented X86 builtin call: ") + | ||||||||||
| getContext().BuiltinInfo.getName(builtinID)); | ||||||||||
| return {}; | ||||||||||
| case X86::BI__builtin_ia32_kaddqi: | ||||||||||
| return emitX86MaskAddLogic(builder, getLoc(expr->getExprLoc()), | ||||||||||
| "x86.avx512.kadd.b", ops); | ||||||||||
|
|
||||||||||
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -465,3 +465,127 @@ __mmask64 test_kmov_q(__mmask64 A) { | |||||
|
|
||||||
| return __builtin_ia32_kmovq(A); | ||||||
| } | ||||||
|
|
||||||
| unsigned char test_kortestc_mask32_u8(__mmask32 __A, __mmask32 __B) { | ||||||
| // CIR-LABEL: _kortestc_mask32_u8 | ||||||
| // CIR: [[ALL_ONES:%.*]] = cir.const #cir.int<4294967295> : !u32i | ||||||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
Suggested change
I prefer to have the |
||||||
| // CIR: [[LHS:%.*]] = cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>> | ||||||
| // CIR: [[RHS:%.*]] = cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR:%.*]] = cir.binop(or, [[LHS]], [[RHS]]) : !cir.vector<32 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR_INT:%.*]] = cir.cast bitcast [[OR]] : !cir.vector<32 x !cir.int<u, 1>> -> !u32i | ||||||
| // CIR: [[CMP:%.*]] = cir.cmp(eq, [[OR_INT]], [[ALL_ONES]]) : !u32i, !cir.bool | ||||||
| // CIR: [[B2I:%.*]] = cir.cast bool_to_int [[CMP]] : !cir.bool -> !s32i | ||||||
| // CIR: cir.cast integral [[B2I]] : !s32i -> !u8i | ||||||
|
|
||||||
| // LLVM-LABEL: _kortestc_mask32_u8 | ||||||
| // LLVM: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // LLVM: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // LLVM: [[OR:%.*]] = or <32 x i1> [[LHS]], [[RHS]] | ||||||
| // LLVM: [[CAST:%.*]] = bitcast <32 x i1> [[OR]] to i32 | ||||||
| // LLVM: [[CMP:%.*]] = icmp eq i32 [[CAST]], -1 | ||||||
| // LLVM: [[ZEXT:%.*]] = zext i1 [[CMP]] to i32 | ||||||
| // LLVM: trunc i32 [[ZEXT]] to i8 | ||||||
|
|
||||||
| // OGCG-LABEL: _kortestc_mask32_u8 | ||||||
| // OGCG: bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // OGCG: bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // OGCG: or <32 x i1> {{.*}}, {{.*}} | ||||||
| // OGCG: bitcast <32 x i1> {{.*}} to i32 | ||||||
| // OGCG: icmp eq i32 {{.*}}, -1 | ||||||
| // OGCG: zext i1 {{.*}} to i32 | ||||||
| // OGCG: trunc i32 {{.*}} to i8 | ||||||
|
Comment on lines
+490
to
+496
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Can you capture the values here as you have in the LLVM checks? |
||||||
| return _kortestc_mask32_u8(__A, __B); | ||||||
| } | ||||||
|
|
||||||
| unsigned char test_kortestc_mask64_u8(__mmask64 __A, __mmask64 __B) { | ||||||
| // CIR-LABEL: _kortestc_mask64_u8 | ||||||
| // CIR: [[ALL_ONES:%.*]] = cir.const #cir.int<18446744073709551615> : !u64i | ||||||
| // CIR: [[LHS:%.*]] = cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>> | ||||||
| // CIR: [[RHS:%.*]] = cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR:%.*]] = cir.binop(or, [[LHS]], [[RHS]]) : !cir.vector<64 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR_INT:%.*]] = cir.cast bitcast [[OR]] : !cir.vector<64 x !cir.int<u, 1>> -> !u64i | ||||||
| // CIR: [[CMP:%.*]] = cir.cmp(eq, [[OR_INT]], [[ALL_ONES]]) : !u64i, !cir.bool | ||||||
| // CIR: [[B2I:%.*]] = cir.cast bool_to_int [[CMP]] : !cir.bool -> !s32i | ||||||
| // CIR: cir.cast integral [[B2I]] : !s32i -> !u8i | ||||||
|
|
||||||
| // LLVM-LABEL: _kortestc_mask64_u8 | ||||||
| // LLVM: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // LLVM: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // LLVM: [[OR:%.*]] = or <64 x i1> [[LHS]], [[RHS]] | ||||||
| // LLVM: [[CAST:%.*]] = bitcast <64 x i1> [[OR]] to i64 | ||||||
| // LLVM: [[CMP:%.*]] = icmp eq i64 [[CAST]], -1 | ||||||
| // LLVM: [[ZEXT:%.*]] = zext i1 [[CMP]] to i32 | ||||||
| // LLVM: trunc i32 [[ZEXT]] to i8 | ||||||
|
|
||||||
| // OGCG-LABEL: _kortestc_mask64_u8 | ||||||
| // OGCG: bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // OGCG: bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // OGCG: or <64 x i1> {{.*}}, {{.*}} | ||||||
| // OGCG: bitcast <64 x i1> {{.*}} to i64 | ||||||
| // OGCG: icmp eq i64 {{.*}}, -1 | ||||||
| // OGCG: zext i1 {{.*}} to i32 | ||||||
| // OGCG: trunc i32 {{.*}} to i8 | ||||||
| return _kortestc_mask64_u8(__A, __B); | ||||||
| } | ||||||
|
|
||||||
| unsigned char test_kortestz_mask32_u8(__mmask32 __A, __mmask32 __B) { | ||||||
| // CIR-LABEL: _kortestz_mask32_u8 | ||||||
| // CIR: [[ZERO:%.*]] = cir.const #cir.int<0> : !u32i | ||||||
| // CIR: [[LHS:%.*]] = cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>> | ||||||
| // CIR: [[RHS:%.*]] = cir.cast bitcast {{.*}} : !u32i -> !cir.vector<32 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR:%.*]] = cir.binop(or, [[LHS]], [[RHS]]) : !cir.vector<32 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR_INT:%.*]] = cir.cast bitcast [[OR]] : !cir.vector<32 x !cir.int<u, 1>> -> !u32i | ||||||
| // CIR: [[CMP:%.*]] = cir.cmp(eq, [[OR_INT]], [[ZERO]]) : !u32i, !cir.bool | ||||||
| // CIR: [[B2I:%.*]] = cir.cast bool_to_int [[CMP]] : !cir.bool -> !s32i | ||||||
| // CIR: cir.cast integral [[B2I]] : !s32i -> !u8i | ||||||
|
|
||||||
| // LLVM-LABEL: _kortestz_mask32_u8 | ||||||
| // LLVM: [[LHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // LLVM: [[RHS:%.*]] = bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // LLVM: [[OR:%.*]] = or <32 x i1> [[LHS]], [[RHS]] | ||||||
| // LLVM: [[CAST:%.*]] = bitcast <32 x i1> [[OR]] to i32 | ||||||
| // LLVM: [[CMP:%.*]] = icmp eq i32 [[CAST]], 0 | ||||||
| // LLVM: [[ZEXT:%.*]] = zext i1 [[CMP]] to i32 | ||||||
| // LLVM: trunc i32 [[ZEXT]] to i8 | ||||||
|
|
||||||
| // OGCG-LABEL: _kortestz_mask32_u8 | ||||||
| // OGCG: bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // OGCG: bitcast i32 %{{.*}} to <32 x i1> | ||||||
| // OGCG: or <32 x i1> {{.*}}, {{.*}} | ||||||
| // OGCG: bitcast <32 x i1> {{.*}} to i32 | ||||||
| // OGCG: icmp eq i32 {{.*}}, 0 | ||||||
| // OGCG: zext i1 {{.*}} to i32 | ||||||
| // OGCG: trunc i32 {{.*}} to i8 | ||||||
| return _kortestz_mask32_u8(__A, __B); | ||||||
| } | ||||||
|
|
||||||
| unsigned char test_kortestz_mask64_u8(__mmask64 __A, __mmask64 __B) { | ||||||
| // CIR-LABEL: _kortestz_mask64_u8 | ||||||
| // CIR: [[ZERO:%.*]] = cir.const #cir.int<0> : !u64i | ||||||
| // CIR: [[LHS:%.*]] = cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>> | ||||||
| // CIR: [[RHS:%.*]] = cir.cast bitcast {{.*}} : !u64i -> !cir.vector<64 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR:%.*]] = cir.binop(or, [[LHS]], [[RHS]]) : !cir.vector<64 x !cir.int<u, 1>> | ||||||
| // CIR: [[OR_INT:%.*]] = cir.cast bitcast [[OR]] : !cir.vector<64 x !cir.int<u, 1>> -> !u64i | ||||||
| // CIR: [[CMP:%.*]] = cir.cmp(eq, [[OR_INT]], [[ZERO]]) : !u64i, !cir.bool | ||||||
| // CIR: [[B2I:%.*]] = cir.cast bool_to_int [[CMP]] : !cir.bool -> !s32i | ||||||
| // CIR: cir.cast integral [[B2I]] : !s32i -> !u8i | ||||||
|
|
||||||
| // LLVM-LABEL: _kortestz_mask64_u8 | ||||||
| // LLVM: [[LHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // LLVM: [[RHS:%.*]] = bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // LLVM: [[OR:%.*]] = or <64 x i1> [[LHS]], [[RHS]] | ||||||
| // LLVM: [[CAST:%.*]] = bitcast <64 x i1> [[OR]] to i64 | ||||||
| // LLVM: [[CMP:%.*]] = icmp eq i64 [[CAST]], 0 | ||||||
| // LLVM: [[ZEXT:%.*]] = zext i1 [[CMP]] to i32 | ||||||
| // LLVM: trunc i32 [[ZEXT]] to i8 | ||||||
|
|
||||||
| // OGCG-LABEL: _kortestz_mask64_u8 | ||||||
| // OGCG: bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // OGCG: bitcast i64 %{{.*}} to <64 x i1> | ||||||
| // OGCG: or <64 x i1> {{.*}}, {{.*}} | ||||||
| // OGCG: bitcast <64 x i1> {{.*}} to i64 | ||||||
| // OGCG: icmp eq i64 {{.*}}, 0 | ||||||
| // OGCG: zext i1 {{.*}} to i32 | ||||||
| // OGCG: trunc i32 {{.*}} to i8 | ||||||
| return _kortestz_mask64_u8(__A, __B); | ||||||
| } | ||||||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.