Don't disable loop unroll for vectorized loops on AMDGPU target

We've got a performance regression after the https://reviews.llvm.org/D115261.
Despite the loop being vectorized unroll is still required.

Reviewed By: rampitec

Differential Revision: https://reviews.llvm.org/D149281

llvm/include/llvm/Analysis/TargetTransformInfo.h

@@ -569,6 +569,8 @@ class TargetTransformInfo {
     /// Don't allow loop unrolling to simulate more than this number of
     /// iterations when checking full unroll profitability
     unsigned MaxIterationsCountToAnalyze;
+    /// Don't disable runtime unroll for the loops which were vectorized.
+    bool UnrollVectorizedLoop = false;
   };
 
   /// Get target-customized preferences for the generic loop unrolling

llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp

@@ -113,6 +113,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
   // manipulations in average.
   UP.BEInsns += 3;
 
+  // We want to run unroll even for the loops which have been vectorized.
+  UP.UnrollVectorizedLoop = true;
+
   // TODO: Do we want runtime unrolling?
 
   // Maximum alloca size than can fit registers. Reserve 16 registers.

llvm/lib/Transforms/Vectorize/LoopVectorize.cpp

@@ -7743,7 +7743,10 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan(
     LoopVectorizeHints Hints(L, true, *ORE);
     Hints.setAlreadyVectorized();
   }
-  AddRuntimeUnrollDisableMetaData(L);
+  TargetTransformInfo::UnrollingPreferences UP;
+  TTI.getUnrollingPreferences(L, *PSE.getSE(), UP, ORE);
+  if (!UP.UnrollVectorizedLoop || CanonicalIVStartValue)
+    AddRuntimeUnrollDisableMetaData(L);
 
   // 3. Fix the vectorized code: take care of header phi's, live-outs,
   //    predication, updating analyses.

llvm/test/CodeGen/AMDGPU/vectorize-unroll-metadata.ll

@@ -0,0 +1,34 @@
+; RUN: opt -mtriple=amdgcn-- -mcpu=gfx90a -passes=loop-vectorize %s -S -o - | FileCheck %s
+
+; CHECK-LABEL: @test
+; CHECK-LABEL: vector.body:
+; CHECK: br i1 %{{[0-9]+}}, label %middle.block, label %vector.body, !llvm.loop !0
+; CHECK-LABEL: middle.block:
+; CHECK-LABEL: scalar.ph:
+; CHECK-LABEL: loop.body:
+; CHECK: br i1 %cond, label %exit, label %loop.body, !llvm.loop !2
+; CHECK: !0 = distinct !{!0, !1}
+; CHECK: !1 = !{!"llvm.loop.isvectorized", i32 1}
+; CHECK: !2 = distinct !{!2, !3, !1}
+; CHECK: !3 = !{!"llvm.loop.unroll.runtime.disable"}
+
+
+define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(3) %lds, i32 %n) {
+
+entry:
+  br label %loop.body
+
+loop.body:
+  %counter = phi i32 [0, %entry], [%inc, %loop.body]
+  %ptr_lds = getelementptr i32, ptr addrspace(3) %lds, i32 %counter
+  %val = load i32, ptr addrspace(3) %ptr_lds
+  %ptr_out = getelementptr i32, ptr addrspace(1) %out, i32 %counter
+  store i32 %val, ptr addrspace(1) %ptr_out
+  %inc = add i32 %counter, 1
+  %cond = icmp sge i32 %counter, %n
+  br i1 %cond, label  %exit, label %loop.body
+
+exit:
+  ret void
+}
+