This repository has been archived by the owner on Aug 3, 2020. It is now read-only.
-
Notifications
You must be signed in to change notification settings - Fork 27
/
writebarrier.go
313 lines (282 loc) · 9.08 KB
/
writebarrier.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package ssa
import (
"cmd/internal/src"
"fmt"
)
// writebarrier expands write barrier ops (StoreWB, MoveWB, etc.) into
// branches and runtime calls, like
//
// if writeBarrier.enabled {
// writebarrierptr(ptr, val)
// } else {
// *ptr = val
// }
//
// If ptr is an address of a stack slot, write barrier will be removed
// and a normal store will be used.
// A sequence of WB stores for many pointer fields of a single type will
// be emitted together, with a single branch.
//
// Expanding WB ops introduces new control flows, and we would need to
// split a block into two if there were values after WB ops, which would
// require scheduling the values. To avoid this complexity, when building
// SSA, we make sure that WB ops are always at the end of a block. We do
// this before fuse as it may merge blocks. It also helps to reduce
// number of blocks as fuse merges blocks introduced in this phase.
func writebarrier(f *Func) {
var sb, sp, wbaddr *Value
var writebarrierptr, typedmemmove, typedmemclr interface{} // *gc.Sym
var storeWBs, others []*Value
var wbs *sparseSet
for _, b := range f.Blocks { // range loop is safe since the blocks we added contain no WB stores
valueLoop:
for i, v := range b.Values {
switch v.Op {
case OpStoreWB, OpMoveWB, OpMoveWBVolatile, OpZeroWB:
if IsStackAddr(v.Args[0]) {
switch v.Op {
case OpStoreWB:
v.Op = OpStore
case OpMoveWB, OpMoveWBVolatile:
v.Op = OpMove
v.Aux = nil
case OpZeroWB:
v.Op = OpZero
v.Aux = nil
}
continue
}
if wbaddr == nil {
// initalize global values for write barrier test and calls
// find SB and SP values in entry block
initln := f.Entry.Pos
for _, v := range f.Entry.Values {
if v.Op == OpSB {
sb = v
}
if v.Op == OpSP {
sp = v
}
}
if sb == nil {
sb = f.Entry.NewValue0(initln, OpSB, f.Config.fe.TypeUintptr())
}
if sp == nil {
sp = f.Entry.NewValue0(initln, OpSP, f.Config.fe.TypeUintptr())
}
wbsym := &ExternSymbol{Typ: f.Config.fe.TypeBool(), Sym: f.Config.fe.Syslook("writeBarrier").(fmt.Stringer)}
wbaddr = f.Entry.NewValue1A(initln, OpAddr, f.Config.fe.TypeUInt32().PtrTo(), wbsym, sb)
writebarrierptr = f.Config.fe.Syslook("writebarrierptr")
typedmemmove = f.Config.fe.Syslook("typedmemmove")
typedmemclr = f.Config.fe.Syslook("typedmemclr")
wbs = f.newSparseSet(f.NumValues())
defer f.retSparseSet(wbs)
}
pos := v.Pos
// there may be a sequence of WB stores in the current block. find them.
storeWBs = storeWBs[:0]
others = others[:0]
wbs.clear()
for _, w := range b.Values[i:] {
if w.Op == OpStoreWB || w.Op == OpMoveWB || w.Op == OpMoveWBVolatile || w.Op == OpZeroWB {
storeWBs = append(storeWBs, w)
wbs.add(w.ID)
} else {
others = append(others, w)
}
}
// make sure that no value in this block depends on WB stores
for _, w := range b.Values {
if w.Op == OpStoreWB || w.Op == OpMoveWB || w.Op == OpMoveWBVolatile || w.Op == OpZeroWB {
continue
}
for _, a := range w.Args {
if wbs.contains(a.ID) {
f.Fatalf("value %v depends on WB store %v in the same block %v", w, a, b)
}
}
}
// find the memory before the WB stores
// this memory is not a WB store but it is used in a WB store.
var mem *Value
for _, w := range storeWBs {
a := w.Args[len(w.Args)-1]
if wbs.contains(a.ID) {
continue
}
if mem != nil {
b.Fatalf("two stores live simultaneously: %s, %s", mem, a)
}
mem = a
}
b.Values = append(b.Values[:i], others...) // move WB ops out of this block
bThen := f.NewBlock(BlockPlain)
bElse := f.NewBlock(BlockPlain)
bEnd := f.NewBlock(b.Kind)
bThen.Pos = pos
bElse.Pos = pos
bEnd.Pos = pos
// set up control flow for end block
bEnd.SetControl(b.Control)
bEnd.Likely = b.Likely
for _, e := range b.Succs {
bEnd.Succs = append(bEnd.Succs, e)
e.b.Preds[e.i].b = bEnd
}
// set up control flow for write barrier test
// load word, test word, avoiding partial register write from load byte.
flag := b.NewValue2(pos, OpLoad, f.Config.fe.TypeUInt32(), wbaddr, mem)
const0 := f.ConstInt32(pos, f.Config.fe.TypeUInt32(), 0)
flag = b.NewValue2(pos, OpNeq32, f.Config.fe.TypeBool(), flag, const0)
b.Kind = BlockIf
b.SetControl(flag)
b.Likely = BranchUnlikely
b.Succs = b.Succs[:0]
b.AddEdgeTo(bThen)
b.AddEdgeTo(bElse)
bThen.AddEdgeTo(bEnd)
bElse.AddEdgeTo(bEnd)
memThen := mem
memElse := mem
for _, w := range storeWBs {
var val *Value
ptr := w.Args[0]
siz := w.AuxInt
typ := w.Aux // only non-nil for MoveWB, MoveWBVolatile, ZeroWB
var op Op
var fn interface{} // *gc.Sym
switch w.Op {
case OpStoreWB:
op = OpStore
fn = writebarrierptr
val = w.Args[1]
case OpMoveWB, OpMoveWBVolatile:
op = OpMove
fn = typedmemmove
val = w.Args[1]
case OpZeroWB:
op = OpZero
fn = typedmemclr
}
// then block: emit write barrier call
memThen = wbcall(pos, bThen, fn, typ, ptr, val, memThen, sp, sb, w.Op == OpMoveWBVolatile)
// else block: normal store
if op == OpZero {
memElse = bElse.NewValue2I(pos, op, TypeMem, siz, ptr, memElse)
} else {
memElse = bElse.NewValue3I(pos, op, TypeMem, siz, ptr, val, memElse)
}
}
// merge memory
// Splice memory Phi into the last memory of the original sequence,
// which may be used in subsequent blocks. Other memories in the
// sequence must be dead after this block since there can be only
// one memory live.
last := storeWBs[0]
if len(storeWBs) > 1 {
// find the last store
last = nil
wbs.clear() // we reuse wbs to record WB stores that is used in another WB store
for _, w := range storeWBs {
wbs.add(w.Args[len(w.Args)-1].ID)
}
for _, w := range storeWBs {
if wbs.contains(w.ID) {
continue
}
if last != nil {
b.Fatalf("two stores live simultaneously: %s, %s", last, w)
}
last = w
}
}
bEnd.Values = append(bEnd.Values, last)
last.Block = bEnd
last.reset(OpPhi)
last.Type = TypeMem
last.AddArg(memThen)
last.AddArg(memElse)
for _, w := range storeWBs {
if w != last {
w.resetArgs()
}
}
for _, w := range storeWBs {
if w != last {
f.freeValue(w)
}
}
if f.Config.fe.Debug_wb() {
f.Config.Warnl(pos, "write barrier")
}
break valueLoop
}
}
}
}
// wbcall emits write barrier runtime call in b, returns memory.
// if valIsVolatile, it moves val into temp space before making the call.
func wbcall(pos src.XPos, b *Block, fn interface{}, typ interface{}, ptr, val, mem, sp, sb *Value, valIsVolatile bool) *Value {
config := b.Func.Config
var tmp GCNode
if valIsVolatile {
// Copy to temp location if the source is volatile (will be clobbered by
// a function call). Marshaling the args to typedmemmove might clobber the
// value we're trying to move.
t := val.Type.ElemType()
tmp = config.fe.Auto(t)
aux := &AutoSymbol{Typ: t, Node: tmp}
mem = b.NewValue1A(pos, OpVarDef, TypeMem, tmp, mem)
tmpaddr := b.NewValue1A(pos, OpAddr, t.PtrTo(), aux, sp)
siz := MakeSizeAndAlign(t.Size(), t.Alignment()).Int64()
mem = b.NewValue3I(pos, OpMove, TypeMem, siz, tmpaddr, val, mem)
val = tmpaddr
}
// put arguments on stack
off := config.ctxt.FixedFrameSize()
if typ != nil { // for typedmemmove
taddr := b.NewValue1A(pos, OpAddr, config.fe.TypeUintptr(), typ, sb)
off = round(off, taddr.Type.Alignment())
arg := b.NewValue1I(pos, OpOffPtr, taddr.Type.PtrTo(), off, sp)
mem = b.NewValue3I(pos, OpStore, TypeMem, ptr.Type.Size(), arg, taddr, mem)
off += taddr.Type.Size()
}
off = round(off, ptr.Type.Alignment())
arg := b.NewValue1I(pos, OpOffPtr, ptr.Type.PtrTo(), off, sp)
mem = b.NewValue3I(pos, OpStore, TypeMem, ptr.Type.Size(), arg, ptr, mem)
off += ptr.Type.Size()
if val != nil {
off = round(off, val.Type.Alignment())
arg = b.NewValue1I(pos, OpOffPtr, val.Type.PtrTo(), off, sp)
mem = b.NewValue3I(pos, OpStore, TypeMem, val.Type.Size(), arg, val, mem)
off += val.Type.Size()
}
off = round(off, config.PtrSize)
// issue call
mem = b.NewValue1A(pos, OpStaticCall, TypeMem, fn, mem)
mem.AuxInt = off - config.ctxt.FixedFrameSize()
if valIsVolatile {
mem = b.NewValue1A(pos, OpVarKill, TypeMem, tmp, mem) // mark temp dead
}
return mem
}
// round to a multiple of r, r is a power of 2
func round(o int64, r int64) int64 {
return (o + r - 1) &^ (r - 1)
}
// IsStackAddr returns whether v is known to be an address of a stack slot
func IsStackAddr(v *Value) bool {
for v.Op == OpOffPtr || v.Op == OpAddPtr || v.Op == OpPtrIndex || v.Op == OpCopy {
v = v.Args[0]
}
switch v.Op {
case OpSP:
return true
case OpAddr:
return v.Args[0].Op == OpSP
}
return false
}