Skip to content

Commit

Permalink
[rs4gc] don't duplicate existing values which are provably base pointers
Browse files Browse the repository at this point in the history 
RS4GC needs to rewrite the IR to ensure that every relocated pointer has an associated base pointer. The existing code isn't particularly smart about avoiding duplication of existing IR when it turns out the original pointer we were asked to materialize a base pointer for is itself a base pointer.

This patch adds a stage to the algorithm which prunes nodes proven (with a simple forward dataflow fixed point) to be base pointers from the list of nodes considered for duplication. This does require changing some of the later invariants slightly, that's probably the riskiest part of the change.

Differential Revision: D98122
  • Loading branch information
@preames
preames committed Mar 16, 2021
1 parent 40bc309 commit 5cabf47
Show file tree
Hide file tree
Showing 10 changed files with 145 additions and 174 deletions.
62 changes: 44 additions & 18 deletions llvm/lib/Transforms/Scalar/RewriteStatepointsForGC.cpp
View file
Original file line number Diff line number Diff line change
Expand Up @@ -900,14 +900,51 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
}
#endif

// Iterate forward through the value graph pruning any node from the state
// list where all of the inputs are base pointers. The purpose of this is to
// reuse existing values when the derived pointer we were asked to materialize
// a base pointer for happens to be a base pointer itself. (Or a sub-graph
// feeding it does.)
SmallVector<Value *> ToRemove;
do {
ToRemove.clear();
for (auto Pair : States) {
Value *BDV = Pair.first;
auto canPruneInput = [&](Value *V) {
Value *BDV = findBaseOrBDV(V, Cache);
if (V->stripPointerCasts() != BDV)
return false;
// The assumption is that anything not in the state list is
// propagates a base pointer.
return States.count(BDV) == 0;
};

bool CanPrune = true;
visitBDVOperands(BDV, [&](Value *Op) {
CanPrune = CanPrune && canPruneInput(Op);
});
if (CanPrune)
ToRemove.push_back(BDV);
}
for (Value *V : ToRemove) {
States.erase(V);
// Cache the fact V is it's own base for later usage.
Cache[V] = V;
}
} while (!ToRemove.empty());

// Did we manage to prove that Def itself must be a base pointer?
if (!States.count(Def))
return Def;

// Return a phi state for a base defining value. We'll generate a new
// base state for known bases and expect to find a cached state otherwise.
auto GetStateForBDV = [&](Value *BaseValue, Value *Input) {
if (isKnownBaseResult(BaseValue) && areBothVectorOrScalar(BaseValue, Input))
return BDVState(BaseValue, BDVState::Base, BaseValue);
auto I = States.find(BaseValue);
assert(I != States.end() && "lookup failed!");
return I->second;
if (I != States.end())
return I->second;
assert(areBothVectorOrScalar(BaseValue, Input));
return BDVState(BaseValue, BDVState::Base, BaseValue);
};

bool Progress = true;
Expand Down Expand Up @@ -1071,7 +1108,8 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
auto getBaseForInput = [&](Value *Input, Instruction *InsertPt) {
Value *BDV = findBaseOrBDV(Input, Cache);
Value *Base = nullptr;
if (isKnownBaseResult(BDV) && areBothVectorOrScalar(BDV, Input)) {
if (!States.count(BDV)) {
assert(areBothVectorOrScalar(BDV, Input));
Base = BDV;
} else {
// Either conflict or base.
Expand Down Expand Up @@ -1203,14 +1241,6 @@ static Value *findBasePointer(Value *I, DefiningValueMapTy &Cache) {
<< (Cache.count(BDV) ? Cache[BDV]->getName().str() : "none")
<< " to: " << Base->getName() << "\n");

if (Cache.count(BDV)) {
assert(isKnownBaseResult(Base) &&
"must be something we 'know' is a base pointer");
// Once we transition from the BDV relation being store in the Cache to
// the base relation being stored, it must be stable
assert((!isKnownBaseResult(Cache[BDV]) || Cache[BDV] == Base) &&
"base relation should be stable");
}
Cache[BDV] = Base;
}
assert(Cache.count(Def));
Expand Down Expand Up @@ -3016,11 +3046,7 @@ static void recomputeLiveInValues(GCPtrLivenessData &RevisedLivenessData,
// We may have base pointers which are now live that weren't before. We need
// to update the PointerToBase structure to reflect this.
for (auto V : Updated)
if (Info.PointerToBase.insert({V, V}).second) {
assert(isKnownBaseResult(V) &&
"Can't find base for unexpected live value!");
continue;
}
Info.PointerToBase.insert({V, V});

#ifndef NDEBUG
for (auto V : Updated)
Expand Down
114 changes: 54 additions & 60 deletions llvm/test/Transforms/RewriteStatepointsForGC/base-inference.ll
View file

Large diffs are not rendered by default.

7 changes: 2 additions & 5 deletions llvm/test/Transforms/RewriteStatepointsForGC/base-pointers-4.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,9 @@ define void @test(i32 %condition) gc "statepoint-example" {
; CHECK: dest_c:
; CHECK-NEXT: br label [[MERGE]]
; CHECK: merge:
; CHECK-NEXT: [[OBJ_TO_CONSUME_BASE:%.*]] = phi i64 addrspace(1)* [ [[TMP0]], [[DEST_A]] ], [ null, [[DEST_B]] ], [ null, [[DEST_C]] ], !is_base_value !0
; CHECK-NEXT: [[OBJ_TO_CONSUME:%.*]] = phi i64 addrspace(1)* [ [[TMP0]], [[DEST_A]] ], [ null, [[DEST_B]] ], [ null, [[DEST_C]] ]
; CHECK-NEXT: [[STATEPOINT_TOKEN1:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* [[OBJ_TO_CONSUME]], i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[OBJ_TO_CONSUME_BASE]], i64 addrspace(1)* [[OBJ_TO_CONSUME]]) ]
; CHECK-NEXT: [[OBJ_TO_CONSUME_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 0)
; CHECK-NEXT: [[OBJ_TO_CONSUME_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_TO_CONSUME_BASE_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: [[OBJ_TO_CONSUME_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 1)
; CHECK-NEXT: [[STATEPOINT_TOKEN1:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @consume_obj, i32 1, i32 0, i64 addrspace(1)* [[OBJ_TO_CONSUME]], i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[OBJ_TO_CONSUME]]) ]
; CHECK-NEXT: [[OBJ_TO_CONSUME_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN1]], i32 0, i32 0)
; CHECK-NEXT: [[OBJ_TO_CONSUME_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_TO_CONSUME_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: br label [[MERGE_SPLIT:%.*]]
; CHECK: merge.split:
Expand Down
33 changes: 10 additions & 23 deletions llvm/test/Transforms/RewriteStatepointsForGC/base-pointers.ll
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,24 +41,19 @@ define i64 addrspace(1)* @test1(i32 %caller, i8 addrspace(1)* %a, i8 addrspace(1
; CHECK-NEXT: br i1 undef, label [[LEFT:%.*]], label [[RIGHT:%.*]]
; CHECK: left:
; CHECK-NEXT: [[A_CAST:%.*]] = bitcast i8 addrspace(1)* [[A:%.*]] to i64 addrspace(1)*
; CHECK-NEXT: [[CAST:%.*]] = bitcast i8 addrspace(1)* [[A]] to i64 addrspace(1)*
; CHECK-NEXT: switch i32 [[UNKNOWN:%.*]], label [[RIGHT]] [
; CHECK-NEXT: i32 0, label [[MERGE:%.*]]
; CHECK-NEXT: i32 1, label [[MERGE]]
; CHECK-NEXT: i32 5, label [[MERGE]]
; CHECK-NEXT: ]
; CHECK: right:
; CHECK-NEXT: [[B_CAST:%.*]] = bitcast i8 addrspace(1)* [[B:%.*]] to i64 addrspace(1)*
; CHECK-NEXT: [[CAST1:%.*]] = bitcast i8 addrspace(1)* [[B]] to i64 addrspace(1)*
; CHECK-NEXT: br label [[MERGE]]
; CHECK: merge:
; CHECK-NEXT: [[VALUE_BASE:%.*]] = phi i64 addrspace(1)* [ [[CAST]], [[LEFT]] ], [ [[CAST]], [[LEFT]] ], [ [[CAST]], [[LEFT]] ], [ [[CAST1]], [[RIGHT]] ], !is_base_value !0
; CHECK-NEXT: [[VALUE:%.*]] = phi i64 addrspace(1)* [ [[A_CAST]], [[LEFT]] ], [ [[A_CAST]], [[LEFT]] ], [ [[A_CAST]], [[LEFT]] ], [ [[B_CAST]], [[RIGHT]] ]
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* [[VALUE]], i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[VALUE]], i64 addrspace(1)* [[VALUE_BASE]]) ]
; CHECK-NEXT: [[VALUE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void (i64 addrspace(1)*)*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidp1i64f(i64 2882400000, i32 0, void (i64 addrspace(1)*)* @parse_point, i32 1, i32 0, i64 addrspace(1)* [[VALUE]], i32 0, i32 0) [ "deopt"(i32 0, i32 0, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[VALUE]]) ]
; CHECK-NEXT: [[VALUE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
; CHECK-NEXT: [[VALUE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[VALUE_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: [[VALUE_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
; CHECK-NEXT: [[VALUE_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[VALUE_BASE_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: ret i64 addrspace(1)* [[VALUE_RELOCATED_CASTED]]
;
entry:
Expand Down Expand Up @@ -146,13 +141,10 @@ define i64 addrspace(1)* @test3(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1
; CHECK: taken:
; CHECK-NEXT: br label [[MERGE]]
; CHECK: merge:
; CHECK-NEXT: [[BDV_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[TAKEN]] ], !is_base_value !0
; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ]], [[ENTRY]] ], [ [[OBJ2]], [[TAKEN]] ]
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BDV_BASE]]) ]
; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[TAKEN]] ]
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: [[BDV_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
; CHECK-NEXT: [[BDV_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_BASE_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]
;
entry:
Expand All @@ -175,11 +167,9 @@ define i64 addrspace(1)* @test4(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1
; CHECK-NEXT: br label [[MERGE]]
; CHECK: merge:
; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ]], [[TAKEN]] ]
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[OBJ]]) ]
; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: [[OBJ_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
; CHECK-NEXT: [[OBJ_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[OBJ_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]
;
entry:
Expand All @@ -199,15 +189,12 @@ define i64 addrspace(1)* @test5(i1 %cnd, i64 addrspace(1)* %obj, i64 addrspace(1
; CHECK-NEXT: entry:
; CHECK-NEXT: br label [[MERGE:%.*]]
; CHECK: merge:
; CHECK-NEXT: [[BDV_BASE:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[MERGE]] ], !is_base_value !0
; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ]], [[ENTRY]] ], [ [[OBJ2]], [[MERGE]] ]
; CHECK-NEXT: [[BDV:%.*]] = phi i64 addrspace(1)* [ [[OBJ:%.*]], [[ENTRY:%.*]] ], [ [[OBJ2:%.*]], [[MERGE]] ]
; CHECK-NEXT: br i1 [[CND:%.*]], label [[MERGE]], label [[NEXT:%.*]]
; CHECK: next:
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]], i64 addrspace(1)* [[BDV_BASE]]) ]
; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 0)
; CHECK-NEXT: [[STATEPOINT_TOKEN:%.*]] = call token (i64, i32, void ()*, i32, i32, ...) @llvm.experimental.gc.statepoint.p0f_isVoidf(i64 2882400000, i32 0, void ()* @foo, i32 0, i32 0, i32 0, i32 0) [ "deopt"(i32 0, i32 -1, i32 0, i32 0, i32 0), "gc-live"(i64 addrspace(1)* [[BDV]]) ]
; CHECK-NEXT: [[BDV_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 0, i32 0)
; CHECK-NEXT: [[BDV_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: [[BDV_BASE_RELOCATED:%.*]] = call coldcc i8 addrspace(1)* @llvm.experimental.gc.relocate.p1i8(token [[STATEPOINT_TOKEN]], i32 1, i32 1)
; CHECK-NEXT: [[BDV_BASE_RELOCATED_CASTED:%.*]] = bitcast i8 addrspace(1)* [[BDV_BASE_RELOCATED]] to i64 addrspace(1)*
; CHECK-NEXT: ret i64 addrspace(1)* [[BDV_RELOCATED_CASTED]]
;
entry:
Expand Down
Loading

0 comments on commit 5cabf47

Please sign in to comment.