diff --git a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp index b876fe0ec73f0..55d53210c3104 100644 --- a/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp +++ b/llvm/lib/Target/AArch64/AArch64TargetTransformInfo.cpp @@ -6042,6 +6042,22 @@ void AArch64TTIImpl::getUnrollingPreferences( UP.UpperBound = true; + // A loop can have a small maximum trip count while SCEV still cannot + // form an exact backedge count - typically a data-dependent exit, e.g. + // shifting a value until it reaches zero. Unlike for counted loops, the + // unrolled body keeps an exit test per iteration, and whether that pays + // off depends on how many iterations the loop usually runs, which is + // unknown at compile time; the code growth and extra branches are certain. + // Be conservative and hold such loops to a lower upper bound; 5 still lets + // smaller early-exit loops unroll. Also disable runtime unrolling, which + // would clamp the unroll count to the known maximum trip count and produce + // the same complete unroll. + if (L->getExitingBlock() && !SE.isBackedgeTakenCountMaxOrZero(L) && + isa(SE.getBackedgeTakenCount(L))) { + UP.MaxUpperBound = 5; + UP.Runtime = false; + } + // For inner loop, it is more likely to be a hot one, and the runtime check // can be promoted out from LICM pass, so the overhead is less, let's try // a larger threshold to unroll more loops. diff --git a/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll new file mode 100644 index 0000000000000..79e1d0b1bdcd7 --- /dev/null +++ b/llvm/test/Transforms/LoopUnroll/AArch64/unroll-max-upperbound-uncomputable-trip-count.ll @@ -0,0 +1,173 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 +; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 < %s | FileCheck %s --check-prefixes=COMMON,DEFAULT +; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 -unroll-max-upperbound=8 < %s | FileCheck %s --check-prefixes=COMMON,WIDE +; RUN: opt -S -passes=loop-unroll -mtriple=aarch64 -unroll-runtime < %s | FileCheck %s --check-prefixes=COMMON,WIDE + +; AArch64 caps upper-bound unrolling at a max trip count of 5 for loops whose +; exact trip count is not computable (data-dependent exits). A typical case is +; a varint-length helper: +; +; int varint_len(uint32_t serial_type) { +; uint64_t v = serial_type; +; int i; +; for (i = 1; (v >>= 7) != 0; i++) {} +; return i; +; } +; +; This loop runs at most 6 times, over the limit of 5, so it is not unrolled +; by default. Command-line flags are applied after the target preferences, so +; either flag restores unrolling: -unroll-max-upperbound=8 re-enables the +; upper-bound path, and -unroll-runtime overrides the target's Runtime = false, +; after which the runtime path clamps its count to the maximum trip count and +; produces the same complete unroll. +define i32 @varint_i32(i32 %serial_type) { +; DEFAULT-LABEL: define i32 @varint_i32( +; DEFAULT-SAME: i32 [[SERIAL_TYPE:%.*]]) { +; DEFAULT-NEXT: [[ENTRY:.*]]: +; DEFAULT-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE]] to i64 +; DEFAULT-NEXT: br label %[[LOOP:.*]] +; DEFAULT: [[LOOP]]: +; DEFAULT-NEXT: [[I:%.*]] = phi i32 [ 1, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ] +; DEFAULT-NEXT: [[V:%.*]] = phi i64 [ [[V0]], %[[ENTRY]] ], [ [[V_SHR:%.*]], %[[LOOP]] ] +; DEFAULT-NEXT: [[V_SHR]] = lshr i64 [[V]], 7 +; DEFAULT-NEXT: [[I_NEXT]] = add i32 [[I]], 1 +; DEFAULT-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0 +; DEFAULT-NEXT: br i1 [[CMP]], label %[[LOOP]], label %[[EXIT:.*]] +; DEFAULT: [[EXIT]]: +; DEFAULT-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[I]], %[[LOOP]] ] +; DEFAULT-NEXT: ret i32 [[I_LCSSA]] +; +; WIDE-LABEL: define i32 @varint_i32( +; WIDE-SAME: i32 [[SERIAL_TYPE:%.*]]) { +; WIDE-NEXT: [[ENTRY:.*:]] +; WIDE-NEXT: [[V0:%.*]] = zext i32 [[SERIAL_TYPE]] to i64 +; WIDE-NEXT: br label %[[LOOP:.*]] +; WIDE: [[LOOP]]: +; WIDE-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7 +; WIDE-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0 +; WIDE-NEXT: br i1 [[CMP]], label %[[LOOP_1:.*]], label %[[EXIT:.*]] +; WIDE: [[LOOP_1]]: +; WIDE-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7 +; WIDE-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0 +; WIDE-NEXT: br i1 [[CMP_1]], label %[[LOOP_2:.*]], label %[[EXIT]] +; WIDE: [[LOOP_2]]: +; WIDE-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7 +; WIDE-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0 +; WIDE-NEXT: br i1 [[CMP_2]], label %[[LOOP_3:.*]], label %[[EXIT]] +; WIDE: [[LOOP_3]]: +; WIDE-NEXT: [[V_SHR_3:%.*]] = lshr i64 [[V_SHR_2]], 7 +; WIDE-NEXT: [[CMP_3:%.*]] = icmp ne i64 [[V_SHR_3]], 0 +; WIDE-NEXT: br i1 [[CMP_3]], label %[[LOOP_4:.*]], label %[[EXIT]] +; WIDE: [[LOOP_4]]: +; WIDE-NEXT: [[V_SHR_4:%.*]] = lshr i64 [[V_SHR_3]], 7 +; WIDE-NEXT: [[CMP_4:%.*]] = icmp ne i64 [[V_SHR_4]], 0 +; WIDE-NEXT: br i1 [[CMP_4]], label %[[LOOP_5:.*]], label %[[EXIT]] +; WIDE: [[LOOP_5]]: +; WIDE-NEXT: br label %[[EXIT]] +; WIDE: [[EXIT]]: +; WIDE-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, %[[LOOP]] ], [ 2, %[[LOOP_1]] ], [ 3, %[[LOOP_2]] ], [ 4, %[[LOOP_3]] ], [ 5, %[[LOOP_4]] ], [ 6, %[[LOOP_5]] ] +; WIDE-NEXT: ret i32 [[I_LCSSA]] +; +entry: + %v0 = zext i32 %serial_type to i64 + br label %loop + +loop: + %i = phi i32 [ 1, %entry ], [ %i.next, %loop ] + %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ] + %v.shr = lshr i64 %v, 7 + %i.next = add i32 %i, 1 + %cmp = icmp ne i64 %v.shr, 0 + br i1 %cmp, label %loop, label %exit + +exit: + %i.lcssa = phi i32 [ %i, %loop ] + ret i32 %i.lcssa +} + +; The 16-bit version runs at most 4 times, under the limit of 5, so it is +; still unrolled by default. +define i32 @varint_i16(i16 %serial_type) { +; COMMON-LABEL: define i32 @varint_i16( +; COMMON-SAME: i16 [[SERIAL_TYPE:%.*]]) { +; COMMON-NEXT: [[ENTRY:.*:]] +; COMMON-NEXT: [[V0:%.*]] = zext i16 [[SERIAL_TYPE]] to i64 +; COMMON-NEXT: br label %[[LOOP:.*]] +; COMMON: [[LOOP]]: +; COMMON-NEXT: [[V_SHR:%.*]] = lshr i64 [[V0]], 7 +; COMMON-NEXT: [[CMP:%.*]] = icmp ne i64 [[V_SHR]], 0 +; COMMON-NEXT: br i1 [[CMP]], label %[[LOOP_1:.*]], label %[[EXIT:.*]] +; COMMON: [[LOOP_1]]: +; COMMON-NEXT: [[V_SHR_1:%.*]] = lshr i64 [[V_SHR]], 7 +; COMMON-NEXT: [[CMP_1:%.*]] = icmp ne i64 [[V_SHR_1]], 0 +; COMMON-NEXT: br i1 [[CMP_1]], label %[[LOOP_2:.*]], label %[[EXIT]] +; COMMON: [[LOOP_2]]: +; COMMON-NEXT: [[V_SHR_2:%.*]] = lshr i64 [[V_SHR_1]], 7 +; COMMON-NEXT: [[CMP_2:%.*]] = icmp ne i64 [[V_SHR_2]], 0 +; COMMON-NEXT: br i1 [[CMP_2]], label %[[LOOP_3:.*]], label %[[EXIT]] +; COMMON: [[LOOP_3]]: +; COMMON-NEXT: br label %[[EXIT]] +; COMMON: [[EXIT]]: +; COMMON-NEXT: [[I_LCSSA:%.*]] = phi i32 [ 1, %[[LOOP]] ], [ 2, %[[LOOP_1]] ], [ 3, %[[LOOP_2]] ], [ 4, %[[LOOP_3]] ] +; COMMON-NEXT: ret i32 [[I_LCSSA]] +; +entry: + %v0 = zext i16 %serial_type to i64 + br label %loop + +loop: + %i = phi i32 [ 1, %entry ], [ %i.next, %loop ] + %v = phi i64 [ %v0, %entry ], [ %v.shr, %loop ] + %v.shr = lshr i64 %v, 7 + %i.next = add i32 %i, 1 + %cmp = icmp ne i64 %v.shr, 0 + br i1 %cmp, label %loop, label %exit + +exit: + %i.lcssa = phi i32 [ %i, %loop ] + ret i32 %i.lcssa +} + +; This loop runs at most 6 times, and - because %n + 6 may wrap - either 6 or 1 +; times (MaxOrZero), never in between, so the option does not affect it. It is +; fully unrolled in both runs. +define i32 @max_or_zero(i32 %n, ptr %p) { +; COMMON-LABEL: define i32 @max_or_zero( +; COMMON-SAME: i32 [[N:%.*]], ptr [[P:%.*]]) { +; COMMON-NEXT: [[ENTRY:.*:]] +; COMMON-NEXT: [[END:%.*]] = add i32 [[N]], 6 +; COMMON-NEXT: br label %[[LOOP:.*]] +; COMMON: [[LOOP]]: +; COMMON-NEXT: store i32 [[N]], ptr [[P]], align 4 +; COMMON-NEXT: [[I_NEXT:%.*]] = add i32 [[N]], 1 +; COMMON-NEXT: [[C:%.*]] = icmp ult i32 [[I_NEXT]], [[END]] +; COMMON-NEXT: br i1 [[C]], label %[[LOOP_1:.*]], label %[[EXIT:.*]] +; COMMON: [[LOOP_1]]: +; COMMON-NEXT: store i32 [[I_NEXT]], ptr [[P]], align 4 +; COMMON-NEXT: [[I_NEXT_1:%.*]] = add i32 [[N]], 2 +; COMMON-NEXT: store i32 [[I_NEXT_1]], ptr [[P]], align 4 +; COMMON-NEXT: [[I_NEXT_2:%.*]] = add i32 [[N]], 3 +; COMMON-NEXT: store i32 [[I_NEXT_2]], ptr [[P]], align 4 +; COMMON-NEXT: [[I_NEXT_3:%.*]] = add i32 [[N]], 4 +; COMMON-NEXT: store i32 [[I_NEXT_3]], ptr [[P]], align 4 +; COMMON-NEXT: [[I_NEXT_4:%.*]] = add i32 [[N]], 5 +; COMMON-NEXT: store i32 [[I_NEXT_4]], ptr [[P]], align 4 +; COMMON-NEXT: br label %[[EXIT]] +; COMMON: [[EXIT]]: +; COMMON-NEXT: [[I_LCSSA:%.*]] = phi i32 [ [[N]], %[[LOOP]] ], [ [[I_NEXT_4]], %[[LOOP_1]] ] +; COMMON-NEXT: ret i32 [[I_LCSSA]] +; +entry: + %end = add i32 %n, 6 + br label %loop + +loop: + %i = phi i32 [ %n, %entry ], [ %i.next, %loop ] + store i32 %i, ptr %p + %i.next = add i32 %i, 1 + %c = icmp ult i32 %i.next, %end + br i1 %c, label %loop, label %exit + +exit: + ret i32 %i +}