@@ -2557,13 +2557,19 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2557
2557
}
2558
2558
2559
2559
const std::string getAsStr () const override {
2560
- unsigned TotalBlocks = 0 , InitialThreadBlocks = 0 ;
2560
+ unsigned TotalBlocks = 0 , InitialThreadBlocks = 0 , AlignedBlocks = 0 ;
2561
2561
for (auto &It : BEDMap) {
2562
+ if (!It.getFirst ())
2563
+ continue ;
2562
2564
TotalBlocks++;
2563
2565
InitialThreadBlocks += It.getSecond ().IsExecutedByInitialThreadOnly ;
2566
+ AlignedBlocks += It.getSecond ().IsReachedFromAlignedBarrierOnly &&
2567
+ It.getSecond ().IsReachingAlignedBarrierOnly ;
2564
2568
}
2565
2569
return " [AAExecutionDomain] " + std::to_string (InitialThreadBlocks) + " /" +
2566
- std::to_string (TotalBlocks) + " executed by initial thread only" ;
2570
+ std::to_string (AlignedBlocks) + " of " +
2571
+ std::to_string (TotalBlocks) +
2572
+ " executed by initial thread / aligned" ;
2567
2573
}
2568
2574
2569
2575
// / See AbstractAttribute::trackStatistics().
@@ -2586,7 +2592,7 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2586
2592
2587
2593
SmallPtrSet<CallBase *, 16 > DeletedBarriers;
2588
2594
auto HandleAlignedBarrier = [&](CallBase *CB) {
2589
- const ExecutionDomainTy &ED = CEDMap[CB ];
2595
+ const ExecutionDomainTy &ED = CEDMap[{CB, PRE} ];
2590
2596
if (!ED.IsReachedFromAlignedBarrierOnly ||
2591
2597
ED.EncounteredNonLocalSideEffect )
2592
2598
return ;
@@ -2619,7 +2625,7 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2619
2625
// The final aligned barrier (LastCB) reaching the kernel end was
2620
2626
// removed already. This means we can go one step further and remove
2621
2627
// the barriers encoutered last before (LastCB).
2622
- const ExecutionDomainTy &LastED = CEDMap[LastCB];
2628
+ const ExecutionDomainTy &LastED = CEDMap[{ LastCB, PRE} ];
2623
2629
Worklist.append (LastED.AlignedBarriers .begin (),
2624
2630
LastED.AlignedBarriers .end ());
2625
2631
}
@@ -2652,12 +2658,12 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2652
2658
// / Merge all information from \p PredED into the successor \p ED. If
2653
2659
// / \p InitialEdgeOnly is set, only the initial edge will enter the block
2654
2660
// / represented by \p ED from this predecessor.
2655
- void mergeInPredecessor (Attributor &A, ExecutionDomainTy &ED,
2661
+ bool mergeInPredecessor (Attributor &A, ExecutionDomainTy &ED,
2656
2662
const ExecutionDomainTy &PredED,
2657
2663
bool InitialEdgeOnly = false );
2658
2664
2659
2665
// / Accumulate information for the entry block in \p EntryBBED.
2660
- void handleCallees (Attributor &A, ExecutionDomainTy &EntryBBED);
2666
+ bool handleCallees (Attributor &A, ExecutionDomainTy &EntryBBED);
2661
2667
2662
2668
// / See AbstractAttribute::updateImpl.
2663
2669
ChangeStatus updateImpl (Attributor &A) override ;
@@ -2667,6 +2673,7 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2667
2673
bool isExecutedByInitialThreadOnly (const BasicBlock &BB) const override {
2668
2674
if (!isValidState ())
2669
2675
return false ;
2676
+ assert (BB.getParent () == getAnchorScope () && " Block is out of scope!" );
2670
2677
return BEDMap.lookup (&BB).IsExecutedByInitialThreadOnly ;
2671
2678
}
2672
2679
@@ -2688,7 +2695,7 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2688
2695
if (CB != &I && AlignedBarriers.contains (const_cast <CallBase *>(CB))) {
2689
2696
break ;
2690
2697
}
2691
- const auto &It = CEDMap.find (CB );
2698
+ const auto &It = CEDMap.find ({CB, PRE} );
2692
2699
if (It == CEDMap.end ())
2693
2700
continue ;
2694
2701
if (!It->getSecond ().IsReachingAlignedBarrierOnly )
@@ -2708,26 +2715,12 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2708
2715
if (CB != &I && AlignedBarriers.contains (const_cast <CallBase *>(CB))) {
2709
2716
break ;
2710
2717
}
2711
- const auto &It = CEDMap.find (CB );
2718
+ const auto &It = CEDMap.find ({CB, POST} );
2712
2719
if (It == CEDMap.end ())
2713
2720
continue ;
2714
- if (!AA::isNoSyncInst (A, *CB, *this )) {
2715
- if (It->getSecond ().IsReachedFromAlignedBarrierOnly ) {
2716
- break ;
2717
- }
2718
- return false ;
2719
- }
2720
-
2721
- Function *Callee = CB->getCalledFunction ();
2722
- if (!Callee || Callee->isDeclaration ())
2723
- return false ;
2724
- const auto &EDAA = A.getAAFor <AAExecutionDomain>(
2725
- *this , IRPosition::function (*Callee), DepClassTy::OPTIONAL);
2726
- if (!EDAA.getState ().isValidState ())
2727
- return false ;
2728
- if (!EDAA.getFunctionExecutionDomain ().IsReachedFromAlignedBarrierOnly )
2729
- return false ;
2730
- break ;
2721
+ if (It->getSecond ().IsReachedFromAlignedBarrierOnly )
2722
+ break ;
2723
+ return false ;
2731
2724
} while ((CurI = CurI->getPrevNonDebugInstruction ()));
2732
2725
2733
2726
if (!CurI) {
@@ -2750,10 +2743,11 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2750
2743
" No request should be made against an invalid state!" );
2751
2744
return BEDMap.lookup (&BB);
2752
2745
}
2753
- ExecutionDomainTy getExecutionDomain (const CallBase &CB) const override {
2746
+ std::pair<ExecutionDomainTy, ExecutionDomainTy>
2747
+ getExecutionDomain (const CallBase &CB) const override {
2754
2748
assert (isValidState () &&
2755
2749
" No request should be made against an invalid state!" );
2756
- return CEDMap.lookup (&CB) ;
2750
+ return { CEDMap.lookup ({ &CB, PRE}), CEDMap. lookup ({&CB, POST})} ;
2757
2751
}
2758
2752
ExecutionDomainTy getFunctionExecutionDomain () const override {
2759
2753
assert (isValidState () &&
@@ -2810,12 +2804,21 @@ struct AAExecutionDomainFunction : public AAExecutionDomain {
2810
2804
// / Mapping containing information about the function for other AAs.
2811
2805
ExecutionDomainTy InterProceduralED;
2812
2806
2807
+ enum Direction { PRE = 0 , POST = 1 };
2813
2808
// / Mapping containing information per block.
2814
2809
DenseMap<const BasicBlock *, ExecutionDomainTy> BEDMap;
2815
- DenseMap<const CallBase *, ExecutionDomainTy> CEDMap;
2810
+ DenseMap<PointerIntPair<const CallBase *, 1 , Direction>, ExecutionDomainTy>
2811
+ CEDMap;
2816
2812
SmallSetVector<CallBase *, 16 > AlignedBarriers;
2817
2813
2818
2814
ReversePostOrderTraversal<Function *> *RPOT = nullptr ;
2815
+
2816
+ // / Set \p R to \V and report true if that changed \p R.
2817
+ static bool setAndRecord (bool &R, bool V) {
2818
+ bool Eq = (R == V);
2819
+ R = V;
2820
+ return !Eq;
2821
+ }
2819
2822
};
2820
2823
2821
2824
void AAExecutionDomainFunction::mergeInPredecessorBarriersAndAssumptions (
@@ -2827,26 +2830,33 @@ void AAExecutionDomainFunction::mergeInPredecessorBarriersAndAssumptions(
2827
2830
ED.addAlignedBarrier (A, *AB);
2828
2831
}
2829
2832
2830
- void AAExecutionDomainFunction::mergeInPredecessor (
2833
+ bool AAExecutionDomainFunction::mergeInPredecessor (
2831
2834
Attributor &A, ExecutionDomainTy &ED, const ExecutionDomainTy &PredED,
2832
2835
bool InitialEdgeOnly) {
2833
- ED.IsExecutedByInitialThreadOnly =
2834
- InitialEdgeOnly || (PredED.IsExecutedByInitialThreadOnly &&
2835
- ED.IsExecutedByInitialThreadOnly );
2836
-
2837
- ED.IsReachedFromAlignedBarrierOnly = ED.IsReachedFromAlignedBarrierOnly &&
2838
- PredED.IsReachedFromAlignedBarrierOnly ;
2839
- ED.EncounteredNonLocalSideEffect =
2840
- ED.EncounteredNonLocalSideEffect | PredED.EncounteredNonLocalSideEffect ;
2836
+
2837
+ bool Changed = false ;
2838
+ Changed |=
2839
+ setAndRecord (ED.IsExecutedByInitialThreadOnly ,
2840
+ InitialEdgeOnly || (PredED.IsExecutedByInitialThreadOnly &&
2841
+ ED.IsExecutedByInitialThreadOnly ));
2842
+
2843
+ Changed |= setAndRecord (ED.IsReachedFromAlignedBarrierOnly ,
2844
+ ED.IsReachedFromAlignedBarrierOnly &&
2845
+ PredED.IsReachedFromAlignedBarrierOnly );
2846
+ Changed |= setAndRecord (ED.EncounteredNonLocalSideEffect ,
2847
+ ED.EncounteredNonLocalSideEffect |
2848
+ PredED.EncounteredNonLocalSideEffect );
2849
+ // Do not track assumptions and barriers as part of Changed.
2841
2850
if (ED.IsReachedFromAlignedBarrierOnly )
2842
2851
mergeInPredecessorBarriersAndAssumptions (A, ED, PredED);
2843
2852
else
2844
2853
ED.clearAssumeInstAndAlignedBarriers ();
2854
+ return Changed;
2845
2855
}
2846
2856
2847
- void AAExecutionDomainFunction::handleCallees (Attributor &A,
2857
+ bool AAExecutionDomainFunction::handleCallees (Attributor &A,
2848
2858
ExecutionDomainTy &EntryBBED) {
2849
- SmallVector<ExecutionDomainTy> CallSiteEDs;
2859
+ SmallVector<std::pair< ExecutionDomainTy, ExecutionDomainTy>, 4 > CallSiteEDs;
2850
2860
auto PredForCallSite = [&](AbstractCallSite ACS) {
2851
2861
const auto &EDAA = A.getAAFor <AAExecutionDomain>(
2852
2862
*this , IRPosition::function (*ACS.getInstruction ()->getFunction ()),
@@ -2863,9 +2873,10 @@ void AAExecutionDomainFunction::handleCallees(Attributor &A,
2863
2873
if (A.checkForAllCallSites (PredForCallSite, *this ,
2864
2874
/* RequiresAllCallSites */ true ,
2865
2875
AllCallSitesKnown)) {
2866
- for (const auto &CSED : CallSiteEDs) {
2867
- mergeInPredecessor (A, EntryBBED, CSED);
2868
- ExitED.IsReachingAlignedBarrierOnly &= CSED.IsReachingAlignedBarrierOnly ;
2876
+ for (const auto &[CSInED, CSOutED] : CallSiteEDs) {
2877
+ mergeInPredecessor (A, EntryBBED, CSInED);
2878
+ ExitED.IsReachingAlignedBarrierOnly &=
2879
+ CSOutED.IsReachingAlignedBarrierOnly ;
2869
2880
}
2870
2881
2871
2882
} else {
@@ -2885,10 +2896,17 @@ void AAExecutionDomainFunction::handleCallees(Attributor &A,
2885
2896
}
2886
2897
}
2887
2898
2899
+ bool Changed = false ;
2888
2900
auto &FnED = BEDMap[nullptr ];
2889
- FnED.IsReachedFromAlignedBarrierOnly &=
2890
- EntryBBED.IsReachedFromAlignedBarrierOnly ;
2891
- FnED.IsReachingAlignedBarrierOnly &= ExitED.IsReachingAlignedBarrierOnly ;
2901
+ Changed |= setAndRecord (FnED.IsReachedFromAlignedBarrierOnly ,
2902
+ FnED.IsReachedFromAlignedBarrierOnly &
2903
+ EntryBBED.IsReachedFromAlignedBarrierOnly );
2904
+ Changed |= setAndRecord (FnED.IsReachingAlignedBarrierOnly ,
2905
+ FnED.IsReachingAlignedBarrierOnly &
2906
+ ExitED.IsReachingAlignedBarrierOnly );
2907
+ Changed |= setAndRecord (FnED.IsExecutedByInitialThreadOnly ,
2908
+ EntryBBED.IsExecutedByInitialThreadOnly );
2909
+ return Changed;
2892
2910
}
2893
2911
2894
2912
ChangeStatus AAExecutionDomainFunction::updateImpl (Attributor &A) {
@@ -2902,27 +2920,23 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
2902
2920
if (CB)
2903
2921
Changed |= AlignedBarriers.insert (CB);
2904
2922
// First, update the barrier ED kept in the separate CEDMap.
2905
- auto &CallED = CEDMap[CB];
2906
- mergeInPredecessor (A, CallED, ED);
2923
+ auto &CallInED = CEDMap[{CB, PRE}];
2924
+ Changed |= mergeInPredecessor (A, CallInED, ED);
2925
+ CallInED.IsReachingAlignedBarrierOnly = true ;
2907
2926
// Next adjust the ED we use for the traversal.
2908
2927
ED.EncounteredNonLocalSideEffect = false ;
2909
2928
ED.IsReachedFromAlignedBarrierOnly = true ;
2910
2929
// Aligned barrier collection has to come last.
2911
2930
ED.clearAssumeInstAndAlignedBarriers ();
2912
2931
if (CB)
2913
2932
ED.addAlignedBarrier (A, *CB);
2933
+ auto &CallOutED = CEDMap[{CB, POST}];
2934
+ Changed |= mergeInPredecessor (A, CallOutED, ED);
2914
2935
};
2915
2936
2916
2937
auto &LivenessAA =
2917
2938
A.getAAFor <AAIsDead>(*this , getIRPosition (), DepClassTy::OPTIONAL);
2918
2939
2919
- // Set \p R to \V and report true if that changed \p R.
2920
- auto SetAndRecord = [&](bool &R, bool V) {
2921
- bool Eq = (R == V);
2922
- R = V;
2923
- return !Eq;
2924
- };
2925
-
2926
2940
auto &OMPInfoCache = static_cast <OMPInformationCache &>(A.getInfoCache ());
2927
2941
2928
2942
Function *F = getAnchorScope ();
@@ -2940,7 +2954,7 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
2940
2954
ExecutionDomainTy ED;
2941
2955
// Propagate "incoming edges" into information about this block.
2942
2956
if (IsEntryBB) {
2943
- handleCallees (A, ED);
2957
+ Changed |= handleCallees (A, ED);
2944
2958
} else {
2945
2959
// For live non-entry blocks we only propagate
2946
2960
// information via live edges.
@@ -3009,8 +3023,8 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
3009
3023
3010
3024
// Record how we entered the call, then accumulate the effect of the
3011
3025
// call in ED for potential use by the callee.
3012
- auto &CallED = CEDMap[CB ];
3013
- mergeInPredecessor (A, CallED , ED);
3026
+ auto &CallInED = CEDMap[{CB, PRE} ];
3027
+ Changed |= mergeInPredecessor (A, CallInED , ED);
3014
3028
3015
3029
// If we have a sync-definition we can check if it starts/ends in an
3016
3030
// aligned barrier. If we are unsure we assume any sync breaks
@@ -3022,7 +3036,6 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
3022
3036
if (EDAA.getState ().isValidState ()) {
3023
3037
const auto &CalleeED = EDAA.getFunctionExecutionDomain ();
3024
3038
ED.IsReachedFromAlignedBarrierOnly =
3025
- CallED.IsReachedFromAlignedBarrierOnly =
3026
3039
CalleeED.IsReachedFromAlignedBarrierOnly ;
3027
3040
AlignedBarrierLastInBlock = ED.IsReachedFromAlignedBarrierOnly ;
3028
3041
if (IsNoSync || !CalleeED.IsReachedFromAlignedBarrierOnly )
@@ -3031,20 +3044,27 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
3031
3044
else
3032
3045
ED.EncounteredNonLocalSideEffect =
3033
3046
CalleeED.EncounteredNonLocalSideEffect ;
3034
- if (!CalleeED.IsReachingAlignedBarrierOnly )
3047
+ if (!CalleeED.IsReachingAlignedBarrierOnly ) {
3048
+ Changed |=
3049
+ setAndRecord (CallInED.IsReachingAlignedBarrierOnly , false );
3035
3050
SyncInstWorklist.push_back (&I);
3051
+ }
3036
3052
if (CalleeED.IsReachedFromAlignedBarrierOnly )
3037
3053
mergeInPredecessorBarriersAndAssumptions (A, ED, CalleeED);
3054
+ auto &CallOutED = CEDMap[{CB, POST}];
3055
+ Changed |= mergeInPredecessor (A, CallOutED, ED);
3038
3056
continue ;
3039
3057
}
3040
3058
}
3041
- if (!IsNoSync)
3042
- ED.IsReachedFromAlignedBarrierOnly =
3043
- CallED.IsReachedFromAlignedBarrierOnly = false ;
3059
+ if (!IsNoSync) {
3060
+ ED.IsReachedFromAlignedBarrierOnly = false ;
3061
+ Changed |= setAndRecord (CallInED.IsReachingAlignedBarrierOnly , false );
3062
+ SyncInstWorklist.push_back (&I);
3063
+ }
3044
3064
AlignedBarrierLastInBlock &= ED.IsReachedFromAlignedBarrierOnly ;
3045
3065
ED.EncounteredNonLocalSideEffect |= !CB->doesNotAccessMemory ();
3046
- if (!IsNoSync)
3047
- SyncInstWorklist. push_back (&I );
3066
+ auto &CallOutED = CEDMap[{CB, POST}];
3067
+ Changed |= mergeInPredecessor (A, CallOutED, ED );
3048
3068
}
3049
3069
3050
3070
if (!I.mayHaveSideEffects () && !I.mayReadFromMemory ())
@@ -3083,12 +3103,14 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
3083
3103
if (!isa<UnreachableInst>(BB.getTerminator ()) &&
3084
3104
!BB.getTerminator ()->getNumSuccessors ()) {
3085
3105
3086
- mergeInPredecessor (A, InterProceduralED, ED);
3106
+ Changed |= mergeInPredecessor (A, InterProceduralED, ED);
3087
3107
3088
3108
auto &FnED = BEDMap[nullptr ];
3089
3109
if (!FnED.IsReachingAlignedBarrierOnly ) {
3090
3110
IsEndAndNotReachingAlignedBarriersOnly = true ;
3091
3111
SyncInstWorklist.push_back (BB.getTerminator ());
3112
+ auto &BBED = BEDMap[&BB];
3113
+ Changed |= setAndRecord (BBED.IsReachingAlignedBarrierOnly , false );
3092
3114
}
3093
3115
if (IsKernel)
3094
3116
HandleAlignedBarrier (nullptr , ED);
@@ -3120,34 +3142,37 @@ ChangeStatus AAExecutionDomainFunction::updateImpl(Attributor &A) {
3120
3142
while (!SyncInstWorklist.empty ()) {
3121
3143
Instruction *SyncInst = SyncInstWorklist.pop_back_val ();
3122
3144
Instruction *CurInst = SyncInst;
3123
- bool HitAlignedBarrier = false ;
3145
+ bool HitAlignedBarrierOrKnownEnd = false ;
3124
3146
while ((CurInst = CurInst->getPrevNode ())) {
3125
3147
auto *CB = dyn_cast<CallBase>(CurInst);
3126
3148
if (!CB)
3127
3149
continue ;
3128
- auto &CallED = CEDMap[CB ];
3129
- if (SetAndRecord (CallED .IsReachingAlignedBarrierOnly , false ))
3150
+ auto &CallOutED = CEDMap[{CB, POST} ];
3151
+ if (setAndRecord (CallOutED .IsReachingAlignedBarrierOnly , false ))
3130
3152
Changed = true ;
3131
- HitAlignedBarrier = AlignedBarriers.count (CB);
3132
- if (HitAlignedBarrier)
3153
+ auto &CallInED = CEDMap[{CB, PRE}];
3154
+ HitAlignedBarrierOrKnownEnd =
3155
+ AlignedBarriers.count (CB) || !CallInED.IsReachingAlignedBarrierOnly ;
3156
+ if (HitAlignedBarrierOrKnownEnd)
3133
3157
break ;
3134
3158
}
3135
- if (HitAlignedBarrier )
3159
+ if (HitAlignedBarrierOrKnownEnd )
3136
3160
continue ;
3137
3161
BasicBlock *SyncBB = SyncInst->getParent ();
3138
3162
for (auto *PredBB : predecessors (SyncBB)) {
3139
3163
if (LivenessAA.isEdgeDead (PredBB, SyncBB))
3140
3164
continue ;
3141
3165
if (!Visited.insert (PredBB))
3142
3166
continue ;
3143
- SyncInstWorklist.push_back (PredBB->getTerminator ());
3144
3167
auto &PredED = BEDMap[PredBB];
3145
- if (SetAndRecord (PredED.IsReachingAlignedBarrierOnly , false ))
3168
+ if (setAndRecord (PredED.IsReachingAlignedBarrierOnly , false )) {
3146
3169
Changed = true ;
3170
+ SyncInstWorklist.push_back (PredBB->getTerminator ());
3171
+ }
3147
3172
}
3148
3173
if (SyncBB != &EntryBB)
3149
3174
continue ;
3150
- if (SetAndRecord (InterProceduralED.IsReachingAlignedBarrierOnly , false ))
3175
+ if (setAndRecord (InterProceduralED.IsReachingAlignedBarrierOnly , false ))
3151
3176
Changed = true ;
3152
3177
}
3153
3178
0 commit comments