[CodeGen] Generate follow-up metadata for loops with more than one tr…

…ansformation.

Before this patch, CGLoop would dump all transformations for a loop into
a single LoopID without encoding any order in which to apply them.
rL348944 added the possibility to encode a transformation order using
followup-attributes.

When a loop has more than one transformation, use the follow-up
attribute define the order in which they are applied. The emitted order
is the defacto order as defined by the current LLVM pass pipeline,
which is:

  LoopFullUnrollPass
  LoopDistributePass
  LoopVectorizePass
  LoopUnrollAndJamPass
  LoopUnrollPass
  MachinePipeliner

This patch should therefore not change the assembly output, assuming
that all explicit transformations can be applied, and no implicit
transformations in-between. In the former case,
WarnMissedTransformationsPass should emit a warning (except for
MachinePipeliner which is not implemented yet). The latter could be
avoided by adding 'llvm.loop.disable_nonforced' attributes.

Because LoopUnrollAndJamPass processes a loop nest, generation of the
MDNode is delayed to after the inner loop metadata have been processed.
A temporary LoopID is therefore used to annotate instructions and
RAUW'ed by the actual LoopID later.

Differential Revision: https://reviews.llvm.org/D57978

llvm-svn: 357415

clang/lib/CodeGen/CGLoopInfo.h

@@ -78,10 +78,11 @@ class LoopInfo {
 public:
   /// Construct a new LoopInfo for the loop with entry Header.
   LoopInfo(llvm::BasicBlock *Header, const LoopAttributes &Attrs,
-           const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc);
+           const llvm::DebugLoc &StartLoc, const llvm::DebugLoc &EndLoc,
+           LoopInfo *Parent);
 
   /// Get the loop id metadata for this loop.
-  llvm::MDNode *getLoopID() const { return LoopID; }
+  llvm::MDNode *getLoopID() const { return TempLoopID.get(); }
 
   /// Get the header block of this loop.
   llvm::BasicBlock *getHeader() const { return Header; }
@@ -92,15 +93,92 @@ class LoopInfo {
   /// Return this loop's access group or nullptr if it does not have one.
   llvm::MDNode *getAccessGroup() const { return AccGroup; }
 
+  /// Create the loop's metadata. Must be called after its nested loops have
+  /// been processed.
+  void finish();
+
 private:
   /// Loop ID metadata.
-  llvm::MDNode *LoopID;
+  llvm::TempMDTuple TempLoopID;
   /// Header block of this loop.
   llvm::BasicBlock *Header;
   /// The attributes for this loop.
   LoopAttributes Attrs;
   /// The access group for memory accesses parallel to this loop.
   llvm::MDNode *AccGroup = nullptr;
+  /// Start location of this loop.
+  llvm::DebugLoc StartLoc;
+  /// End location of this loop.
+  llvm::DebugLoc EndLoc;
+  /// The next outer loop, or nullptr if this is the outermost loop.
+  LoopInfo *Parent;
+  /// If this loop has unroll-and-jam metadata, this can be set by the inner
+  /// loop's LoopInfo to set the llvm.loop.unroll_and_jam.followup_inner
+  /// metadata.
+  llvm::MDNode *UnrollAndJamInnerFollowup = nullptr;
+
+  /// Create a LoopID without any transformations.
+  llvm::MDNode *
+  createLoopPropertiesMetadata(llvm::ArrayRef<llvm::Metadata *> LoopProperties);
+
+  /// Create a LoopID for transformations.
+  ///
+  /// The methods call each other in case multiple transformations are applied
+  /// to a loop. The transformation first to be applied will use LoopID of the
+  /// next transformation in its followup attribute.
+  ///
+  /// @param Attrs             The loop's transformations.
+  /// @param LoopProperties    Non-transformation properties such as debug
+  ///                          location, parallel accesses and disabled
+  ///                          transformations. These are added to the returned
+  ///                          LoopID.
+  /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
+  ///                          at least one transformation.
+  ///
+  /// @return A LoopID (metadata node) that can be used for the llvm.loop
+  ///         annotation or followup-attribute.
+  /// @{
+  llvm::MDNode *
+  createPipeliningMetadata(const LoopAttributes &Attrs,
+                           llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                           bool &HasUserTransforms);
+  llvm::MDNode *
+  createPartialUnrollMetadata(const LoopAttributes &Attrs,
+                              llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                              bool &HasUserTransforms);
+  llvm::MDNode *
+  createUnrollAndJamMetadata(const LoopAttributes &Attrs,
+                             llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                             bool &HasUserTransforms);
+  llvm::MDNode *
+  createLoopVectorizeMetadata(const LoopAttributes &Attrs,
+                              llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                              bool &HasUserTransforms);
+  llvm::MDNode *
+  createLoopDistributeMetadata(const LoopAttributes &Attrs,
+                               llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                               bool &HasUserTransforms);
+  llvm::MDNode *
+  createFullUnrollMetadata(const LoopAttributes &Attrs,
+                           llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                           bool &HasUserTransforms);
+  /// @}
+
+  /// Create a LoopID for this loop, including transformation-unspecific
+  /// metadata such as debug location.
+  ///
+  /// @param Attrs             This loop's attributes and transformations.
+  /// @param LoopProperties    Additional non-transformation properties to add
+  ///                          to the LoopID, such as transformation-specific
+  ///                          metadata that are not covered by @p Attrs.
+  /// @param HasUserTransforms [out] Set to true if the returned MDNode encodes
+  ///                          at least one transformation.
+  ///
+  /// @return A LoopID (metadata node) that can be used for the llvm.loop
+  ///         annotation.
+  llvm::MDNode *createMetadata(const LoopAttributes &Attrs,
+                               llvm::ArrayRef<llvm::Metadata *> LoopProperties,
+                               bool &HasUserTransforms);
 };
 
 /// A stack of loop information corresponding to loop nesting levels.

clang/test/CodeGenCXX/pragma-followup_inner.cpp

@@ -0,0 +1,42 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void followup_inner(int n, int *x) {
+#pragma unroll_and_jam(4)
+  for(int j = 0; j < n; j++) {
+#pragma clang loop pipeline_initiation_interval(10)
+#pragma clang loop unroll_count(4)
+#pragma clang loop vectorize(assume_safety)
+#pragma clang loop distribute(enable)
+    for(int i = 0; i < n; i++)
+      x[j+i*n] = 10;
+
+  }
+}
+
+
+// CHECK-LABEL: define void @followup_inner
+// CHECK: br label %for.cond1, !llvm.loop ![[INNERLOOP_3:[0-9]+]]
+// CHECK: br label %for.cond, !llvm.loop ![[OUTERLOOP_9:[0-9]+]]
+
+// CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{}
+
+// CHECK-DAG: ![[INNERLOOP_3:[0-9]+]] = distinct !{![[INNERLOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
+// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", !2}
+// CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true}
+// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true}
+
+// CHECK-DAG: ![[OUTERLOOP_9:[0-9]+]] = distinct !{![[OUTERLOOP_9:[0-9]+]], ![[UNROLLANDJAM_COUNT_10:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_COUNT_10:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPINNER_11:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_inner", !12}
+
+// CHECK-DAG: ![[LOOP_12:[0-9]+]] = distinct !{![[LOOP_12:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_COUNT_13:[0-9]+]], ![[UNROLL_FOLLOWUP_14:[0-9]+]]}
+// CHECK-DAG: ![[ISVECTORIZED_13:[0-9]+]] = !{!"llvm.loop.isvectorized"}
+// CHECK-DAG: ![[UNROLL_COUNT_13:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_14:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_15:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_15:[0-9]+]] = distinct !{![[LOOP_15:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_13:[0-9]+]], ![[UNROLL_DISABLE_16:[0-9]+]], ![[PIPELINE_17:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_DISABLE_16:[0-9]+]] = !{!"llvm.loop.unroll.disable"}
+// CHECK-DAG: ![[PIPELINE_17:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10}

clang/test/CodeGenCXX/pragma-followup_outer.cpp

@@ -0,0 +1,41 @@
+// RUN: %clang_cc1 -triple x86_64-apple-darwin -std=c++11 -emit-llvm -o - %s | FileCheck %s
+
+extern "C" void followup_outer(int n, int *x) {
+#pragma clang loop pipeline_initiation_interval(10)
+#pragma clang loop unroll_count(4)
+#pragma unroll_and_jam
+#pragma clang loop vectorize(assume_safety)
+#pragma clang loop distribute(enable)
+  for(int j = 0; j < n; j++) {
+    x[j] = 10;
+  }
+}
+
+
+// CHECK-LABEL: define void @followup_outer
+// CHECK: br label %for.cond, !llvm.loop ![[LOOP_3:[0-9]+]]
+
+// CHECK-DAG: ![[ACCESSGROUP_2:[0-9]+]] = distinct !{}
+
+// CHECK-DAG: ![[LOOP_3:[0-9]+]] = distinct !{![[LOOP_3:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[DISTRIBUTE_5:[0-9]+]], ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]]}
+// CHECK-DAG: ![[PARALLEL_ACCESSES_4:[0-9]+]] = !{!"llvm.loop.parallel_accesses", ![[ACCESSGROUP_2]]}
+// CHECK-DAG: ![[DISTRIBUTE_5:[0-9]+]] = !{!"llvm.loop.distribute.enable", i1 true}
+// CHECK-DAG: ![[DISTRIBUTE_FOLLOWUP_6:[0-9]+]] = !{!"llvm.loop.distribute.followup_all", ![[LOOP_7:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_7:[0-9]+]] = distinct !{![[LOOP_7:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[VECTORIZE_8:[0-9]+]], ![[VECTORIZE_FOLLOWUP_9:[0-9]+]]}
+// CHECK-DAG: ![[VECTORIZE_8:[0-9]+]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK-DAG: ![[VECTORIZE_FOLLOWUP_9:[0-9]+]] = !{!"llvm.loop.vectorize.followup_all", ![[LOOP_10:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_10:[0-9]+]] = distinct !{![[LOOP_10:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_12:[0-9]+]], ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]]}
+// CHECK-DAG: ![[ISVECTORIZED_11:[0-9]+]] = !{!"llvm.loop.isvectorized"}
+// CHECK-DAG: ![[UNROLLANDJAM_12:[0-9]+]] =  !{!"llvm.loop.unroll_and_jam.enable"}
+// CHECK-DAG: ![[UNROLLANDJAM_FOLLOWUPOUTER_13:[0-9]+]] = !{!"llvm.loop.unroll_and_jam.followup_outer", ![[LOOP_14:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_14:[0-9]+]] = distinct !{![[LOOP_14:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_COUNT_16:[0-9]+]], ![[UNROLL_FOLLOWUP_17:[0-9]+]]}
+// CHECK-DAG: ![[UNROLLANDJAM_DISABLE_15:[0-9]+]]  = !{!"llvm.loop.unroll_and_jam.disable"}
+// CHECK-DAG: ![[UNROLL_COUNT_16:[0-9]+]] = !{!"llvm.loop.unroll.count", i32 4}
+// CHECK-DAG: ![[UNROLL_FOLLOWUP_17:[0-9]+]] = !{!"llvm.loop.unroll.followup_all", ![[LOOP_18:[0-9]+]]}
+
+// CHECK-DAG: ![[LOOP_18:[0-9]+]] = distinct !{![[LOOP_18:[0-9]+]], ![[PARALLEL_ACCESSES_4:[0-9]+]], ![[ISVECTORIZED_11:[0-9]+]], ![[UNROLLANDJAM_DISABLE_15:[0-9]+]], ![[UNROLL_DISABLE_19:[0-9]+]], ![[INITIATIONINTERVAL_20:[0-9]+]]}
+// CHECK-DAG: ![[UNROLL_DISABLE_19:[0-9]+]]  = !{!"llvm.loop.unroll.disable"}
+// CHECK-DAG: ![[INITIATIONINTERVAL_20:[0-9]+]] = !{!"llvm.loop.pipeline.initiationinterval", i32 10}

clang/test/CodeGenCXX/pragma-loop-safety-imperfectly_nested.cpp

@@ -26,7 +26,7 @@ void vectorize_imperfectly_nested_test(int *List, int Length) {
 // CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
 // CHECK: ![[ACCESS_GROUP_LIST_3:[0-9]+]]  = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]}
 // CHECK: ![[ACCESS_GROUP_4]] = distinct !{}
-// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]}
+// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]]
 // CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]}
-// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]}
+// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]]
 // CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}

clang/test/CodeGenCXX/pragma-loop-safety-nested.cpp

@@ -21,7 +21,7 @@ void vectorize_nested_test(int *List, int Length) {
 // CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
 // CHECK: ![[ACCESS_GROUP_LIST_3]] = !{![[ACCESS_GROUP_2]], ![[ACCESS_GROUP_4:[0-9]+]]}
 // CHECK: ![[ACCESS_GROUP_4]] = distinct !{}
-// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_8:[0-9]+]]}
+// CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]], ![[PARALLEL_ACCESSES_8:[0-9]+]]
 // CHECK: ![[PARALLEL_ACCESSES_8]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_4]]}
-// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_10:[0-9]+]]}
+// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_10:[0-9]+]]
 // CHECK: ![[PARALLEL_ACCESSES_10]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}

clang/test/CodeGenCXX/pragma-loop-safety-outer.cpp

@@ -18,5 +18,5 @@ void vectorize_outer_test(int *List, int Length) {
 
 // CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
 // CHECK: ![[INNER_LOOPID]] = distinct !{![[INNER_LOOPID]],
-// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], {{.*}} ![[PARALLEL_ACCESSES_9:[0-9]+]]}
+// CHECK: ![[OUTER_LOOPID]] = distinct !{![[OUTER_LOOPID]], ![[PARALLEL_ACCESSES_9:[0-9]+]]
 // CHECK: ![[PARALLEL_ACCESSES_9]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}

clang/test/CodeGenCXX/pragma-loop-safety.cpp

@@ -47,12 +47,12 @@ void interleave_test(int *List, int Length) {
 }
 
 // CHECK: ![[ACCESS_GROUP_2]] = distinct !{}
-// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[PARALLEL_ACCESSES_7:[0-9]+]]}
+// CHECK: ![[LOOP1_HINTS]] = distinct !{![[LOOP1_HINTS]], ![[PARALLEL_ACCESSES_7:[0-9]+]], ![[UNROLL_DISABLE:[0-9]+]], ![[INTERLEAVE_1:[0-9]+]], ![[INTENABLE_1:[0-9]+]]}
+// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
+// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
 // CHECK: ![[INTERLEAVE_1]] = !{!"llvm.loop.interleave.count", i32 1}
 // CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
-// CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
-// CHECK: ![[PARALLEL_ACCESSES_7]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_2]]}
 // CHECK: ![[ACCESS_GROUP_8]] = distinct !{}
-// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]], ![[UNROLL_DISABLE]], ![[PARALLEL_ACCESSES_11:[0-9]+]]}
-// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
+// CHECK: ![[LOOP2_HINTS]] = distinct !{![[LOOP2_HINTS]], ![[PARALLEL_ACCESSES_11:[0-9]+]], ![[UNROLL_DISABLE]], ![[WIDTH_1:[0-9]+]], ![[INTENABLE_1]]}
 // CHECK: ![[PARALLEL_ACCESSES_11]] = !{!"llvm.loop.parallel_accesses", ![[ACCESS_GROUP_8]]}
+// CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}

clang/test/CodeGenCXX/pragma-loop.cpp

@@ -158,37 +158,60 @@ void template_test(double *List, int Length) {
   for_template_constant_expression_test<double, 2, 4, 8>(List, Length);
 }
 
-// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[WIDTH_4:.*]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[UNROLL_FULL:.*]], ![[DISTRIBUTE_ENABLE:.*]]}
-// CHECK: ![[WIDTH_4]] = !{!"llvm.loop.vectorize.width", i32 4}
-// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
-// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[LOOP_1]] = distinct !{![[LOOP_1]], ![[UNROLL_FULL:.*]]}
 // CHECK: ![[UNROLL_FULL]] = !{!"llvm.loop.unroll.full"}
-// CHECK: ![[DISTRIBUTE_ENABLE]] = !{!"llvm.loop.distribute.enable", i1 true}
-// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
-// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
+
+// CHECK: ![[LOOP_2]] = distinct !{![[LOOP_2]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_8:.*]], ![[INTERLEAVE_4:.*]]}
 // CHECK: ![[UNROLL_DISABLE]] = !{!"llvm.loop.unroll.disable"}
 // CHECK: ![[DISTRIBUTE_DISABLE]] = !{!"llvm.loop.distribute.enable", i1 false}
-// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]], ![[INTENABLE_1:.*]]}
+// CHECK: ![[WIDTH_8]] = !{!"llvm.loop.vectorize.width", i32 8}
+// CHECK: ![[INTERLEAVE_4]] = !{!"llvm.loop.interleave.count", i32 4}
+
+// CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[INTERLEAVE_4:.*]], ![[INTENABLE_1:.*]], ![[FOLLOWUP_VECTOR_3:.*]]}
+// CHECK: ![[INTENABLE_1]] = !{!"llvm.loop.vectorize.enable", i1 true}
+// CHECK: ![[FOLLOWUP_VECTOR_3]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_3:.*]]}
+// CHECK: ![[AFTER_VECTOR_3]] = distinct !{![[AFTER_VECTOR_3]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+// CHECK: ![[ISVECTORIZED]] = !{!"llvm.loop.isvectorized"}
 // CHECK: ![[UNROLL_8]] = !{!"llvm.loop.unroll.count", i32 8}
+
 // CHECK: ![[LOOP_4]] = distinct !{![[LOOP_4]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]]}
 // CHECK: ![[WIDTH_2]] = !{!"llvm.loop.vectorize.width", i32 2}
 // CHECK: ![[INTERLEAVE_2]] = !{!"llvm.loop.interleave.count", i32 2}
-// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[WIDTH_1:.*]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]]}
+
+// CHECK: ![[LOOP_5]] = distinct !{![[LOOP_5]], ![[UNROLL_DISABLE:.*]], ![[DISTRIBUTE_DISABLE:.*]], ![[WIDTH_1:.*]]}
 // CHECK: ![[WIDTH_1]] = !{!"llvm.loop.vectorize.width", i32 1}
-// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_6]] = distinct !{![[LOOP_6]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_6:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_6]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_6:.*]]}
+// CHECK: ![[AFTER_VECTOR_6]] = distinct !{![[AFTER_VECTOR_6]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
 // CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[WIDTH_5:.*]]}
 // CHECK: ![[WIDTH_5]] = !{!"llvm.loop.vectorize.width", i32 5}
+
 // CHECK: ![[LOOP_8]] = distinct !{![[LOOP_8]], ![[WIDTH_5:.*]]}
-// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[UNROLL_8:.*]]}
-// CHECK: ![[INTERLEAVE_8]] = !{!"llvm.loop.interleave.count", i32 8}
-// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[UNROLL_8:.*]]}
-// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[UNROLL_8:.*]]}
-// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[UNROLL_24:.*]]}
-// CHECK: ![[WIDTH_6]] = !{!"llvm.loop.vectorize.width", i32 6}
-// CHECK: ![[INTERLEAVE_10]] = !{!"llvm.loop.interleave.count", i32 10}
+
+// CHECK: ![[LOOP_9]] = distinct !{![[LOOP_9]], ![[WIDTH_8:.*]], ![[INTERLEAVE_8:.*]], ![[FOLLOWUP_VECTOR_9:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_9]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_9:.*]]}
+// CHECK: ![[AFTER_VECTOR_9]] = distinct !{![[AFTER_VECTOR_9]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_10]] = distinct !{![[LOOP_10]], ![[WIDTH_2:.*]], ![[INTERLEAVE_2:.*]], ![[FOLLOWUP_VECTOR_10:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_10]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_10:.*]]}
+// CHECK: ![[AFTER_VECTOR_10]] = distinct !{![[AFTER_VECTOR_10]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_11]] = distinct !{![[LOOP_11]], ![[WIDTH_2:.*]], ![[INTERLEAVE_4:.*]], ![[FOLLOWUP_VECTOR_11:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_11]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_11:.*]]}
+// CHECK: ![[AFTER_VECTOR_11]] = distinct !{![[AFTER_VECTOR_11]], ![[ISVECTORIZED:.*]], ![[UNROLL_8:.*]]}
+
+// CHECK: ![[LOOP_12]] = distinct !{![[LOOP_12]], ![[WIDTH_6:.*]], ![[INTERLEAVE_10:.*]], ![[FOLLOWUP_VECTOR_12:.*]]}
+// CHECK: ![[FOLLOWUP_VECTOR_12]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_12:.*]]}
+// CHECK: ![[AFTER_VECTOR_12]] = distinct !{![[AFTER_VECTOR_12]], ![[ISVECTORIZED:.*]], ![[UNROLL_24:.*]]}
 // CHECK: ![[UNROLL_24]] = !{!"llvm.loop.unroll.count", i32 24}
-// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[UNROLL_32:.*]]}
+
+// CHECK: ![[LOOP_13]] = distinct !{![[LOOP_13]], ![[WIDTH_8:.*]], ![[INTERLEAVE_16:.*]], ![[FOLLOWUP_VECTOR_13:.*]]}
 // CHECK: ![[INTERLEAVE_16]] = !{!"llvm.loop.interleave.count", i32 16}
+// CHECK: ![[FOLLOWUP_VECTOR_13]] = !{!"llvm.loop.vectorize.followup_all", ![[AFTER_VECTOR_13:.*]]}
+// CHECK: ![[AFTER_VECTOR_13]] = distinct !{![[AFTER_VECTOR_13]], ![[ISVECTORIZED:.*]], ![[UNROLL_32:.*]]}
 // CHECK: ![[UNROLL_32]] = !{!"llvm.loop.unroll.count", i32 32}
+
 // CHECK: ![[LOOP_14]] = distinct !{![[LOOP_14]], ![[WIDTH_10:.*]]}
 // CHECK: ![[WIDTH_10]] = !{!"llvm.loop.vectorize.width", i32 10}

clang/test/CodeGenCXX/pragma-unroll-and-jam.cpp

@@ -51,5 +51,5 @@ void clang_unroll_plus_nounroll_and_jam(int *List, int Length, int Value) {
 // CHECK: ![[UNJ_4]] = !{!"llvm.loop.unroll_and_jam.count", i32 4}
 // CHECK: ![[LOOP_3]] = distinct !{![[LOOP_3]], ![[UNJ_DISABLE:.*]]}
 // CHECK: ![[UNJ_DISABLE]] = !{!"llvm.loop.unroll_and_jam.disable"}
-// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNROLL_4:.*]], ![[UNJ_DISABLE:.*]]}
+// CHECK: ![[LOOP_7]] = distinct !{![[LOOP_7]], ![[UNJ_DISABLE:.*]], ![[UNROLL_4:.*]]}
 // CHECK: ![[UNROLL_4]] = !{!"llvm.loop.unroll.count", i32 4}