Skip to content

Commit

Permalink
[CostModel][X86] Improve ISD::CTTZ costs accounting for BSF/TZCNT imp…
Browse files Browse the repository at this point in the history 
…lementations
  • Loading branch information
@RKSimon
RKSimon committed Mar 13, 2020
1 parent ec3218d commit a2db388
Show file tree
Hide file tree
Showing 3 changed files with 244 additions and 172 deletions.
21 changes: 21 additions & 0 deletions llvm/lib/Target/X86/X86TargetTransformInfo.cpp
View file
Expand Up @@ -2133,6 +2133,14 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
{ ISD::FSQRT, MVT::f32, 28 }, // Pentium III from http://www.agner.org/
{ ISD::FSQRT, MVT::v4f32, 56 }, // Pentium III from http://www.agner.org/
};
static const CostTblEntry BMI64CostTbl[] = { // 64-bit targets
{ ISD::CTTZ, MVT::i64, 1 },
};
static const CostTblEntry BMI32CostTbl[] = { // 32 or 64-bit targets
{ ISD::CTTZ, MVT::i32, 1 },
{ ISD::CTTZ, MVT::i16, 1 },
{ ISD::CTTZ, MVT::i8, 1 },
};
static const CostTblEntry LZCNT64CostTbl[] = { // 64-bit targets
{ ISD::CTLZ, MVT::i64, 1 },
};
Expand All @@ -2152,6 +2160,7 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
static const CostTblEntry X64CostTbl[] = { // 64-bit targets
{ ISD::BITREVERSE, MVT::i64, 14 },
{ ISD::CTLZ, MVT::i64, 4 }, // BSR+XOR or BSR+XOR+CMOV
{ ISD::CTTZ, MVT::i64, 3 }, // TEST+BSF+CMOV/BRANCH
{ ISD::CTPOP, MVT::i64, 10 },
{ ISD::SADDO, MVT::i64, 1 },
{ ISD::UADDO, MVT::i64, 1 },
Expand All @@ -2163,6 +2172,9 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
{ ISD::CTLZ, MVT::i32, 4 }, // BSR+XOR or BSR+XOR+CMOV
{ ISD::CTLZ, MVT::i16, 4 }, // BSR+XOR or BSR+XOR+CMOV
{ ISD::CTLZ, MVT::i8, 4 }, // BSR+XOR or BSR+XOR+CMOV
{ ISD::CTTZ, MVT::i32, 3 }, // TEST+BSF+CMOV/BRANCH
{ ISD::CTTZ, MVT::i16, 3 }, // TEST+BSF+CMOV/BRANCH
{ ISD::CTTZ, MVT::i8, 3 }, // TEST+BSF+CMOV/BRANCH
{ ISD::CTPOP, MVT::i32, 8 },
{ ISD::CTPOP, MVT::i16, 9 },
{ ISD::CTPOP, MVT::i8, 7 },
Expand Down Expand Up @@ -2282,6 +2294,15 @@ int X86TTIImpl::getIntrinsicInstrCost(Intrinsic::ID IID, Type *RetTy,
if (const auto *Entry = CostTableLookup(SSE1CostTbl, ISD, MTy))
return LT.first * Entry->Cost;

if (ST->hasBMI()) {
if (ST->is64Bit())
if (const auto *Entry = CostTableLookup(BMI64CostTbl, ISD, MTy))
return LT.first * Entry->Cost;

if (const auto *Entry = CostTableLookup(BMI32CostTbl, ISD, MTy))
return LT.first * Entry->Cost;
}

if (ST->hasLZCNT()) {
if (ST->is64Bit())
if (const auto *Entry = CostTableLookup(LZCNT64CostTbl, ISD, MTy))
Expand Down
93 changes: 63 additions & 30 deletions llvm/test/Analysis/CostModel/X86/cttz.ll
View file
@@ -1,10 +1,11 @@
; NOTE: Assertions have been autogenerated by utils/update_analyze_test_checks.py
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+sse4.2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE42
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx | FileCheck %s -check-prefixes=CHECK,AVX,AVX1
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx2 | FileCheck %s -check-prefixes=CHECK,AVX,AVX2
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512f | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512F
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,AVX512,AVX512BW
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=-bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2,NOBMI
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+bmi,+sse2 | FileCheck %s -check-prefixes=CHECK,SSE,SSE2,BMI
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+bmi,+sse4.2 | FileCheck %s -check-prefixes=CHECK,BMI,SSE,SSE42
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+bmi,+avx | FileCheck %s -check-prefixes=CHECK,BMI,AVX,AVX1
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+bmi,+avx2 | FileCheck %s -check-prefixes=CHECK,BMI,AVX,AVX2
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+bmi,+avx512f | FileCheck %s -check-prefixes=CHECK,BMI,AVX512,AVX512F
; RUN: opt < %s -mtriple=x86_64-unknown-linux-gnu -cost-model -analyze -mattr=+bmi,+avx512vl,+avx512bw,+avx512dq | FileCheck %s -check-prefixes=CHECK,BMI,AVX512,AVX512BW

; Verify the cost of scalar trailing zero count instructions.

Expand All @@ -14,72 +15,104 @@ declare i16 @llvm.cttz.i16(i16, i1)
declare i8 @llvm.cttz.i8(i8, i1)

define i64 @var_cttz_i64(i64 %a) {
; CHECK-LABEL: 'var_cttz_i64'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
; NOBMI-LABEL: 'var_cttz_i64'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
;
; BMI-LABEL: 'var_cttz_i64'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 false)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
;
%cttz = call i64 @llvm.cttz.i64(i64 %a, i1 0)
ret i64 %cttz
}

define i64 @var_cttz_i64u(i64 %a) {
; CHECK-LABEL: 'var_cttz_i64u'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
; NOBMI-LABEL: 'var_cttz_i64u'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
;
; BMI-LABEL: 'var_cttz_i64u'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i64 @llvm.cttz.i64(i64 %a, i1 true)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i64 %cttz
;
%cttz = call i64 @llvm.cttz.i64(i64 %a, i1 1)
ret i64 %cttz
}

define i32 @var_cttz_i32(i32 %a) {
; CHECK-LABEL: 'var_cttz_i32'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
; NOBMI-LABEL: 'var_cttz_i32'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
;
; BMI-LABEL: 'var_cttz_i32'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 false)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
;
%cttz = call i32 @llvm.cttz.i32(i32 %a, i1 0)
ret i32 %cttz
}

define i32 @var_cttz_i32u(i32 %a) {
; CHECK-LABEL: 'var_cttz_i32u'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
; NOBMI-LABEL: 'var_cttz_i32u'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
;
; BMI-LABEL: 'var_cttz_i32u'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i32 @llvm.cttz.i32(i32 %a, i1 true)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i32 %cttz
;
%cttz = call i32 @llvm.cttz.i32(i32 %a, i1 1)
ret i32 %cttz
}

define i16 @var_cttz_i16(i16 %a) {
; CHECK-LABEL: 'var_cttz_i16'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
; NOBMI-LABEL: 'var_cttz_i16'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
;
; BMI-LABEL: 'var_cttz_i16'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 false)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
;
%cttz = call i16 @llvm.cttz.i16(i16 %a, i1 0)
ret i16 %cttz
}

define i16 @var_cttz_i16u(i16 %a) {
; CHECK-LABEL: 'var_cttz_i16u'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
; NOBMI-LABEL: 'var_cttz_i16u'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
;
; BMI-LABEL: 'var_cttz_i16u'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i16 @llvm.cttz.i16(i16 %a, i1 true)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i16 %cttz
;
%cttz = call i16 @llvm.cttz.i16(i16 %a, i1 1)
ret i16 %cttz
}

define i8 @var_cttz_i8(i8 %a) {
; CHECK-LABEL: 'var_cttz_i8'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
; NOBMI-LABEL: 'var_cttz_i8'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
;
; BMI-LABEL: 'var_cttz_i8'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 false)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
;
%cttz = call i8 @llvm.cttz.i8(i8 %a, i1 0)
ret i8 %cttz
}

define i8 @var_cttz_i8u(i8 %a) {
; CHECK-LABEL: 'var_cttz_i8u'
; CHECK-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
; CHECK-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
; NOBMI-LABEL: 'var_cttz_i8u'
; NOBMI-NEXT: Cost Model: Found an estimated cost of 3 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
; NOBMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
;
; BMI-LABEL: 'var_cttz_i8u'
; BMI-NEXT: Cost Model: Found an estimated cost of 1 for instruction: %cttz = call i8 @llvm.cttz.i8(i8 %a, i1 true)
; BMI-NEXT: Cost Model: Found an estimated cost of 0 for instruction: ret i8 %cttz
;
%cttz = call i8 @llvm.cttz.i8(i8 %a, i1 1)
ret i8 %cttz
Expand Down

0 comments on commit a2db388

Please sign in to comment.