diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index 66a35f9bfaa53..53816d49de722 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -4748,18 +4748,6 @@ class slpvectorizer::BoUpSLP { return Res; } - /// Reordering permutes the operand columns of the tree entries and may - /// move an operand between the edges covered and not covered by copyable - /// scheduling data, making the computed dependency counts stale. Mark the - /// schedule data of the copyable-element instructions for recalculation - /// at the next bundle scheduling. - void markCopyableDepsForRecalc() { - for (const auto &P : ScheduleCopyableDataMapByInst) - if (ScheduleData *SD = - getScheduleData(const_cast(P.first))) - RecalcCopyableOperandDeps.insert(SD); - } - ScheduleCopyableData &addScheduleCopyableData(const EdgeInfo &EI, Instruction *I, int SchedulingRegionID, @@ -8219,8 +8207,6 @@ void BoUpSLP::reorderTopToBottom() { Mask, MaskOrder); } } - for (const auto &It : BlocksSchedules) - It.second->markCopyableDepsForRecalc(); } void BoUpSLP::buildReorderableOperands( @@ -8722,8 +8708,6 @@ void BoUpSLP::reorderBottomToTop(bool IgnoreReorder) { if (IgnoreReorder && !VectorizableTree.front()->ReorderIndices.empty() && VectorizableTree.front()->ReuseShuffleIndices.empty()) VectorizableTree.front()->ReorderIndices.clear(); - for (const auto &It : BlocksSchedules) - It.second->markCopyableDepsForRecalc(); } Instruction *BoUpSLP::getRootEntryInstruction(const TreeEntry &Entry) const { diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll deleted file mode 100644 index f7b2664e57b80..0000000000000 --- a/llvm/test/Transforms/SLPVectorizer/AArch64/recalc-copyable-deps-on-reorder.ll +++ /dev/null @@ -1,188 +0,0 @@ -; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6 -; RUN: opt -S --passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu < %s | FileCheck %s - -define void @test(ptr %0, ptr %1, double %2, ptr %3) { -; CHECK-LABEL: define void @test( -; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]], double [[TMP2:%.*]], ptr [[TMP3:%.*]]) { -; CHECK-NEXT: br i1 true, label %[[VECTOR_PH:.*]], label %[[DOTPREHEADER511:.*]] -; CHECK: [[VECTOR_PH]]: -; CHECK-NEXT: [[TMP5:%.*]] = load double, ptr [[TMP0]], align 8 -; CHECK-NEXT: [[TMP6:%.*]] = fmul double [[TMP5]], 0.000000e+00 -; CHECK-NEXT: [[TMP7:%.*]] = fadd double [[TMP6]], 0.000000e+00 -; CHECK-NEXT: [[TMP8:%.*]] = fmul double [[TMP6]], 0.000000e+00 -; CHECK-NEXT: [[TMP9:%.*]] = fmul double [[TMP8]], 0.000000e+00 -; CHECK-NEXT: [[TMP10:%.*]] = fadd double [[TMP2]], 0.000000e+00 -; CHECK-NEXT: [[TMP11:%.*]] = fadd double [[TMP10]], 1.000000e+00 -; CHECK-NEXT: [[TMP12:%.*]] = fadd double [[TMP9]], [[TMP11]] -; CHECK-NEXT: [[TMP13:%.*]] = fmul double [[TMP2]], 0.000000e+00 -; CHECK-NEXT: [[TMP14:%.*]] = fadd contract double [[TMP13]], 0.000000e+00 -; CHECK-NEXT: [[TMP15:%.*]] = fmul double [[TMP6]], [[TMP2]] -; CHECK-NEXT: [[TMP16:%.*]] = fadd contract double [[TMP15]], 0.000000e+00 -; CHECK-NEXT: [[TMP17:%.*]] = load double, ptr [[TMP1]], align 8 -; CHECK-NEXT: [[TMP18:%.*]] = fmul double [[TMP17]], 0.000000e+00 -; CHECK-NEXT: [[TMP19:%.*]] = fmul double [[TMP18]], [[TMP2]] -; CHECK-NEXT: [[TMP20:%.*]] = fmul double [[TMP2]], [[TMP19]] -; CHECK-NEXT: [[TMP21:%.*]] = fadd double [[TMP20]], 0.000000e+00 -; CHECK-NEXT: [[TMP22:%.*]] = load double, ptr [[TMP0]], align 8 -; CHECK-NEXT: [[TMP23:%.*]] = fmul double [[TMP22]], 0.000000e+00 -; CHECK-NEXT: [[TMP24:%.*]] = fmul double [[TMP2]], [[TMP23]] -; CHECK-NEXT: [[TMP25:%.*]] = fadd double [[TMP24]], 0.000000e+00 -; CHECK-NEXT: [[TMP26:%.*]] = fmul double [[TMP5]], 0.000000e+00 -; CHECK-NEXT: [[TMP27:%.*]] = fmul double [[TMP26]], 0.000000e+00 -; CHECK-NEXT: [[TMP28:%.*]] = fadd double [[TMP27]], 0.000000e+00 -; CHECK-NEXT: [[TMP29:%.*]] = fmul double [[TMP2]], [[TMP17]] -; CHECK-NEXT: [[TMP30:%.*]] = fadd double [[TMP29]], 0.000000e+00 -; CHECK-NEXT: br label %[[DOTPREHEADER511]] -; CHECK: [[_PREHEADER511:.*:]] -; CHECK-NEXT: [[DOTSROA_9_0:%.*]] = phi double [ [[TMP30]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4:%.*]] ] -; CHECK-NEXT: [[DOTSROA_7_0:%.*]] = phi double [ [[TMP28]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_6887_0:%.*]] = phi double [ [[TMP25]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_5_0:%.*]] = phi double [ [[TMP21]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_4886_0:%.*]] = phi double [ [[TMP16]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_3_0:%.*]] = phi double [ [[TMP14]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_2885_0:%.*]] = phi double [ [[TMP12]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: [[DOTSROA_0884_0:%.*]] = phi double [ [[TMP7]], %[[VECTOR_PH]] ], [ 0.000000e+00, [[TMP4]] ] -; CHECK-NEXT: br i1 true, label %[[BB31:.*]], [[DOTPREHEADER498:label %.*]] -; CHECK: [[BB31]]: -; CHECK-NEXT: [[TMP32:%.*]] = getelementptr i8, ptr [[TMP3]], i64 8 -; CHECK-NEXT: [[TMP33:%.*]] = load double, ptr [[TMP3]], align 8 -; CHECK-NEXT: [[TMP34:%.*]] = fmul double [[DOTSROA_0884_0]], [[TMP33]] -; CHECK-NEXT: [[TMP35:%.*]] = load double, ptr [[TMP32]], align 8 -; CHECK-NEXT: [[TMP36:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP35]] -; CHECK-NEXT: [[TMP37:%.*]] = fadd double [[TMP34]], [[TMP36]] -; CHECK-NEXT: [[TMP38:%.*]] = fmul double [[DOTSROA_4886_0]], 0.000000e+00 -; CHECK-NEXT: [[TMP39:%.*]] = fadd double [[TMP37]], [[TMP38]] -; CHECK-NEXT: [[TMP40:%.*]] = fadd double [[TMP39]], 0.000000e+00 -; CHECK-NEXT: [[TMP41:%.*]] = fadd double [[TMP40]], 1.000000e+00 -; CHECK-NEXT: [[TMP42:%.*]] = fmul double [[DOTSROA_2885_0]], [[TMP33]] -; CHECK-NEXT: [[TMP43:%.*]] = fmul double [[DOTSROA_3_0]], [[TMP35]] -; CHECK-NEXT: [[TMP44:%.*]] = fadd double [[TMP42]], [[TMP43]] -; CHECK-NEXT: [[TMP45:%.*]] = fmul double [[DOTSROA_5_0]], 0.000000e+00 -; CHECK-NEXT: [[TMP46:%.*]] = fadd double [[TMP44]], [[TMP45]] -; CHECK-NEXT: [[TMP47:%.*]] = fadd double [[TMP46]], [[TMP2]] -; CHECK-NEXT: [[TMP48:%.*]] = fadd double [[TMP47]], 0.000000e+00 -; CHECK-NEXT: [[TMP49:%.*]] = fmul double [[DOTSROA_4886_0]], [[TMP33]] -; CHECK-NEXT: [[TMP50:%.*]] = fmul double [[DOTSROA_5_0]], [[TMP35]] -; CHECK-NEXT: [[TMP51:%.*]] = fadd double [[TMP49]], [[TMP50]] -; CHECK-NEXT: [[TMP52:%.*]] = load double, ptr [[TMP0]], align 8 -; CHECK-NEXT: [[TMP53:%.*]] = fmul double [[DOTSROA_6887_0]], [[TMP52]] -; CHECK-NEXT: [[TMP54:%.*]] = fadd double [[TMP51]], [[TMP53]] -; CHECK-NEXT: [[TMP55:%.*]] = fmul double [[DOTSROA_9_0]], [[TMP2]] -; CHECK-NEXT: [[TMP56:%.*]] = fadd double [[TMP54]], [[TMP55]] -; CHECK-NEXT: [[TMP57:%.*]] = fadd double [[TMP56]], 0.000000e+00 -; CHECK-NEXT: [[TMP58:%.*]] = fmul double [[DOTSROA_7_0]], [[TMP33]] -; CHECK-NEXT: [[TMP59:%.*]] = fadd double [[TMP58]], [[TMP2]] -; CHECK-NEXT: [[TMP60:%.*]] = fmul double [[DOTSROA_9_0]], 0.000000e+00 -; CHECK-NEXT: [[TMP61:%.*]] = fadd double [[TMP59]], [[TMP60]] -; CHECK-NEXT: [[TMP62:%.*]] = fadd double [[TMP61]], 0.000000e+00 -; CHECK-NEXT: [[TMP63:%.*]] = fadd double [[TMP62]], 1.000000e+00 -; CHECK-NEXT: br [[DOTPREHEADER498]] -; CHECK: [[_PREHEADER498:.*:]] -; CHECK-NEXT: [[DOTSROA_0913_0:%.*]] = phi double [ [[TMP41]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_6915_0:%.*]] = phi double [ [[TMP48]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_10918_0:%.*]] = phi double [ [[TMP57]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_14921_0:%.*]] = phi double [ [[TMP63]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: [[DOTSROA_18924_0:%.*]] = phi double [ [[TMP33]], %[[BB31]] ], [ 0.000000e+00, %[[DOTPREHEADER511]] ] -; CHECK-NEXT: store double [[DOTSROA_0913_0]], ptr [[TMP1]], align 8 -; CHECK-NEXT: [[DOTSROA_6915_0__SROA_IDX916:%.*]] = getelementptr i8, ptr [[TMP1]], i64 8 -; CHECK-NEXT: store double [[DOTSROA_6915_0]], ptr [[DOTSROA_6915_0__SROA_IDX916]], align 8 -; CHECK-NEXT: [[DOTSROA_10918_0__SROA_IDX919:%.*]] = getelementptr i8, ptr [[TMP1]], i64 16 -; CHECK-NEXT: store double [[DOTSROA_10918_0]], ptr [[DOTSROA_10918_0__SROA_IDX919]], align 8 -; CHECK-NEXT: [[DOTSROA_14921_0__SROA_IDX922:%.*]] = getelementptr i8, ptr [[TMP1]], i64 24 -; CHECK-NEXT: store double [[DOTSROA_14921_0]], ptr [[DOTSROA_14921_0__SROA_IDX922]], align 8 -; CHECK-NEXT: store double [[DOTSROA_18924_0]], ptr [[TMP0]], align 8 -; CHECK-NEXT: ret void -; - br i1 true, label %vector.ph, label %.preheader511 - -vector.ph: - %5 = load double, ptr %0, align 8 - %6 = fmul double %5, 0.000000e+00 - %7 = fadd double %6, 0.000000e+00 - %8 = fmul double %6, 0.000000e+00 - %9 = fmul double %8, 0.000000e+00 - %10 = fadd double %2, 0.000000e+00 - %11 = fadd double %10, 1.000000e+00 - %12 = fadd double %9, %11 - %13 = fmul double %2, 0.000000e+00 - %14 = fadd contract double %13, 0.000000e+00 - %15 = fmul double %6, %2 - %16 = fadd contract double %15, 0.000000e+00 - %17 = load double, ptr %1, align 8 - %18 = fmul double %17, 0.000000e+00 - %19 = fmul double %18, %2 - %20 = fmul double %2, %19 - %21 = fadd double %20, 0.000000e+00 - %22 = load double, ptr %0, align 8 - %23 = fmul double %22, 0.000000e+00 - %24 = fmul double %2, %23 - %25 = fadd double %24, 0.000000e+00 - %26 = fmul double %5, 0.000000e+00 - %27 = fmul double %26, 0.000000e+00 - %28 = fadd double %27, 0.000000e+00 - %29 = fmul double %2, %17 - %30 = fadd double %29, 0.000000e+00 - br label %.preheader511 - -.preheader511: - %.sroa.9.0 = phi double [ %30, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.7.0 = phi double [ %28, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.6887.0 = phi double [ %25, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.5.0 = phi double [ %21, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.4886.0 = phi double [ %16, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.3.0 = phi double [ %14, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.2885.0 = phi double [ %12, %vector.ph ], [ 0.000000e+00, %4 ] - %.sroa.0884.0 = phi double [ %7, %vector.ph ], [ 0.000000e+00, %4 ] - br i1 true, label %31, label %.preheader498 - -31: - %32 = getelementptr i8, ptr %3, i64 8 - %33 = load double, ptr %3, align 8 - %34 = fmul double %.sroa.0884.0, %33 - %35 = load double, ptr %32, align 8 - %36 = fmul double %.sroa.2885.0, %35 - %37 = fadd double %34, %36 - %38 = fmul double %.sroa.4886.0, 0.000000e+00 - %39 = fadd double %37, %38 - %40 = fadd double %39, 0.000000e+00 - %41 = fadd double %40, 1.000000e+00 - %42 = fmul double %.sroa.2885.0, %33 - %43 = fmul double %.sroa.3.0, %35 - %44 = fadd double %42, %43 - %45 = fmul double %.sroa.5.0, 0.000000e+00 - %46 = fadd double %44, %45 - %47 = fadd double %46, %2 - %48 = fadd double %47, 0.000000e+00 - %49 = fmul double %.sroa.4886.0, %33 - %50 = fmul double %.sroa.5.0, %35 - %51 = fadd double %49, %50 - %52 = load double, ptr %0, align 8 - %53 = fmul double %.sroa.6887.0, %52 - %54 = fadd double %51, %53 - %55 = fmul double %.sroa.9.0, %2 - %56 = fadd double %54, %55 - %57 = fadd double %56, 0.000000e+00 - %58 = fmul double %.sroa.7.0, %33 - %59 = fadd double %58, %2 - %60 = fmul double %.sroa.9.0, 0.000000e+00 - %61 = fadd double %59, %60 - %62 = fadd double %61, 0.000000e+00 - %63 = fadd double %62, 1.000000e+00 - br label %.preheader498 - -.preheader498: - %.sroa.0913.0 = phi double [ %41, %31 ], [ 0.000000e+00, %.preheader511 ] - %.sroa.6915.0 = phi double [ %48, %31 ], [ 0.000000e+00, %.preheader511 ] - %.sroa.10918.0 = phi double [ %57, %31 ], [ 0.000000e+00, %.preheader511 ] - %.sroa.14921.0 = phi double [ %63, %31 ], [ 0.000000e+00, %.preheader511 ] - %.sroa.18924.0 = phi double [ %33, %31 ], [ 0.000000e+00, %.preheader511 ] - store double %.sroa.0913.0, ptr %1, align 8 - %.sroa.6915.0..sroa_idx916 = getelementptr i8, ptr %1, i64 8 - store double %.sroa.6915.0, ptr %.sroa.6915.0..sroa_idx916, align 8 - %.sroa.10918.0..sroa_idx919 = getelementptr i8, ptr %1, i64 16 - store double %.sroa.10918.0, ptr %.sroa.10918.0..sroa_idx919, align 8 - %.sroa.14921.0..sroa_idx922 = getelementptr i8, ptr %1, i64 24 - store double %.sroa.14921.0, ptr %.sroa.14921.0..sroa_idx922, align 8 - store double %.sroa.18924.0, ptr %0, align 8 - ret void -}