-
Notifications
You must be signed in to change notification settings - Fork 0
/
mgcwork.go
583 lines (532 loc) · 16.5 KB
/
mgcwork.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package runtime
import (
"runtime/internal/atomic"
"runtime/internal/sys"
"unsafe"
)
const (
_WorkbufSize = 2048 // in bytes; larger values result in less contention
// workbufAlloc is the number of bytes to allocate at a time
// for new workbufs. This must be a multiple of pageSize and
// should be a multiple of _WorkbufSize.
//
// Larger values reduce workbuf allocation overhead. Smaller
// values reduce heap fragmentation.
workbufAlloc = 32 << 10
)
// throwOnGCWork causes any operations that add pointers to a gcWork
// buffer to throw.
//
// TODO(austin): This is a temporary debugging measure for issue
// #27993. To be removed before release.
var throwOnGCWork bool
func init() {
if workbufAlloc%pageSize != 0 || workbufAlloc%_WorkbufSize != 0 {
throw("bad workbufAlloc")
}
}
// Garbage collector work pool abstraction.
//
// This implements a producer/consumer model for pointers to grey
// objects. A grey object is one that is marked and on a work
// queue. A black object is marked and not on a work queue.
//
// Write barriers, root discovery, stack scanning, and object scanning
// produce pointers to grey objects. Scanning consumes pointers to
// grey objects, thus blackening them, and then scans them,
// potentially producing new pointers to grey objects.
// A gcWork provides the interface to produce and consume work for the
// garbage collector.
//
// A gcWork can be used on the stack as follows:
//
// (preemption must be disabled)
// gcw := &getg().m.p.ptr().gcw
// .. call gcw.put() to produce and gcw.tryGet() to consume ..
//
// It's important that any use of gcWork during the mark phase prevent
// the garbage collector from transitioning to mark termination since
// gcWork may locally hold GC work buffers. This can be done by
// disabling preemption (systemstack or acquirem).
type gcWork struct {
// wbuf1 and wbuf2 are the primary and secondary work buffers.
//
// This can be thought of as a stack of both work buffers'
// pointers concatenated. When we pop the last pointer, we
// shift the stack up by one work buffer by bringing in a new
// full buffer and discarding an empty one. When we fill both
// buffers, we shift the stack down by one work buffer by
// bringing in a new empty buffer and discarding a full one.
// This way we have one buffer's worth of hysteresis, which
// amortizes the cost of getting or putting a work buffer over
// at least one buffer of work and reduces contention on the
// global work lists.
//
// wbuf1 is always the buffer we're currently pushing to and
// popping from and wbuf2 is the buffer that will be discarded
// next.
//
// Invariant: Both wbuf1 and wbuf2 are nil or neither are.
wbuf1, wbuf2 *workbuf
// Bytes marked (blackened) on this gcWork. This is aggregated into work.bytesMarked by dispose.
// 此gcWork上标记(变黑)的字节。这将聚合到由dispose标记的work.Bytes中。
bytesMarked uint64
// Scan work performed on this gcWork. This is aggregated into
// gcController by dispose and may also be flushed by callers.
// 已扫描数量
scanWork int64
// flushedWork indicates that a non-empty work buffer was
// flushed to the global work list since the last gcMarkDone
// termination check. Specifically, this indicates that this
// gcWork may have communicated work to another gcWork.
flushedWork bool
// pauseGen causes put operations to spin while pauseGen ==
// gcWorkPauseGen if debugCachedWork is true.
pauseGen uint32
// putGen is the pauseGen of the last putGen.
putGen uint32
// pauseStack is the stack at which this P was paused if
// debugCachedWork is true.
pauseStack [16]uintptr
}
// Most of the methods of gcWork are go:nowritebarrierrec because the
// write barrier itself can invoke gcWork methods but the methods are
// not generally re-entrant. Hence, if a gcWork method invoked the
// write barrier while the gcWork was in an inconsistent state, and
// the write barrier in turn invoked a gcWork method, it could
// permanently corrupt the gcWork.
func (w *gcWork) init() {
w.wbuf1 = getempty()
wbuf2 := trygetfull()
if wbuf2 == nil {
wbuf2 = getempty()
}
w.wbuf2 = wbuf2
}
func (w *gcWork) checkPut(ptr uintptr, ptrs []uintptr) {
if debugCachedWork {
alreadyFailed := w.putGen == w.pauseGen
w.putGen = w.pauseGen
if m := getg().m; m.locks > 0 || m.mallocing != 0 || m.preemptoff != "" || m.p.ptr().status != _Prunning {
// If we were to spin, the runtime may
// deadlock: the condition above prevents
// preemption (see newstack), which could
// prevent gcMarkDone from finishing the
// ragged barrier and releasing the spin.
return
}
for atomic.Load(&gcWorkPauseGen) == w.pauseGen {
}
if throwOnGCWork {
printlock()
if alreadyFailed {
println("runtime: checkPut already failed at this generation")
}
println("runtime: late gcWork put")
if ptr != 0 {
gcDumpObject("ptr", ptr, ^uintptr(0))
}
for _, ptr := range ptrs {
gcDumpObject("ptrs", ptr, ^uintptr(0))
}
println("runtime: paused at")
for _, pc := range w.pauseStack {
if pc == 0 {
break
}
f := findfunc(pc)
if f.valid() {
// Obviously this doesn't
// relate to ancestor
// tracebacks, but this
// function prints what we
// want.
printAncestorTracebackFuncInfo(f, pc)
} else {
println("\tunknown PC ", hex(pc), "\n")
}
}
throw("throwOnGCWork")
}
}
}
// put enqueues a pointer for the garbage collector to trace.
// obj must point to the beginning of a heap object or an oblet.
//go:nowritebarrierrec
/*
put不仅尝试将对象放入wbuf1,还会再wbuf1满的时候,尝试更换wbuf1 wbuf2的角色,都满的话,则想全局进行申请,
并将满的队列上交到全局队列
*/
func (w *gcWork) put(obj uintptr) {
w.checkPut(obj, nil)
flushed := false
wbuf := w.wbuf1
if wbuf == nil {
// 如果wbuf1不存在,则初始化wbuf1 wbuf2两个队列
w.init()
wbuf = w.wbuf1
// wbuf is empty at this point.
} else if wbuf.nobj == len(wbuf.obj) {
// wbuf1满了,更换wbuf1 wbuf2的角色
w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1
wbuf = w.wbuf1
if wbuf.nobj == len(wbuf.obj) {
// 更换角色后,wbuf1也满了,说明两个队列都满了
// 把 wbuf1上交全局并获取一个空的队列
putfull(wbuf)
w.flushedWork = true
wbuf = getempty()
w.wbuf1 = wbuf
// 设置队列上交的标志位
flushed = true
}
}
wbuf.obj[wbuf.nobj] = obj
wbuf.nobj++
// If we put a buffer on full, let the GC controller know so
// it can encourage more workers to run. We delay this until
// the end of put so that w is in a consistent state, since
// enlistWorker may itself manipulate w.
// 此时全局已经有标记满的队列,GC controller选择调度更多work进行工作
if flushed && gcphase == _GCmark {
gcController.enlistWorker()
}
}
// putFast does a put and reports whether it can be done quickly
// otherwise it returns false and the caller needs to call put.
//go:nowritebarrierrec
/*
work有wbuf1 wbuf2两个队列用于保存灰色对象,首先会往wbuf1队列里加入灰色对象,wbuf1满了后,交换wbuf1和wbuf2,
这时wbuf2便晋升为wbuf1,继续存放灰色对象,两个队列都满了,则向全局进行申请
*/
func (w *gcWork) putFast(obj uintptr) bool {
w.checkPut(obj, nil)
wbuf := w.wbuf1
if wbuf == nil {
// 没有申请缓存队列,返回false
return false
} else if wbuf.nobj == len(wbuf.obj) {
// wbuf1队列满了,返回false
return false
}
// 向未满wbuf1队列中加入对象
wbuf.obj[wbuf.nobj] = obj
wbuf.nobj++
return true
}
// putBatch performs a put on every pointer in obj. See put for
// constraints on these pointers.
//
//go:nowritebarrierrec
func (w *gcWork) putBatch(obj []uintptr) {
if len(obj) == 0 {
return
}
w.checkPut(0, obj)
flushed := false
wbuf := w.wbuf1
if wbuf == nil {
w.init()
wbuf = w.wbuf1
}
for len(obj) > 0 {
for wbuf.nobj == len(wbuf.obj) {
putfull(wbuf)
w.flushedWork = true
w.wbuf1, w.wbuf2 = w.wbuf2, getempty()
wbuf = w.wbuf1
flushed = true
}
n := copy(wbuf.obj[wbuf.nobj:], obj)
wbuf.nobj += n
obj = obj[n:]
}
if flushed && gcphase == _GCmark {
gcController.enlistWorker()
}
}
// tryGet dequeues a pointer for the garbage collector to trace.
//
// If there are no pointers remaining in this gcWork or in the global
// queue, tryGet returns 0. Note that there may still be pointers in
// other gcWork instances or other caches.
//go:nowritebarrierrec
func (w *gcWork) tryGet() uintptr {
wbuf := w.wbuf1
if wbuf == nil {
w.init()
wbuf = w.wbuf1
// wbuf is empty at this point.
}
if wbuf.nobj == 0 {
// wbuf1空了,更换wbuf1 wbuf2的角色
w.wbuf1, w.wbuf2 = w.wbuf2, w.wbuf1
wbuf = w.wbuf1
// 原wbuf2也是空的,尝试从全局队列获取一个满的队列
if wbuf.nobj == 0 {
owbuf := wbuf
wbuf = trygetfull()
// 获取不到,则返回
if wbuf == nil {
return 0
}
// 把空的队列上传到全局空队列,并把获取的满的队列,作为自身的wbuf1
putempty(owbuf)
w.wbuf1 = wbuf
}
}
wbuf.nobj--
return wbuf.obj[wbuf.nobj]
}
// tryGetFast dequeues a pointer for the garbage collector to trace
// if one is readily available. Otherwise it returns 0 and
// the caller is expected to call tryGet().
//go:nowritebarrierrec
func (w *gcWork) tryGetFast() uintptr {
wbuf := w.wbuf1
if wbuf == nil {
return 0
}
if wbuf.nobj == 0 {
return 0
}
wbuf.nobj--
return wbuf.obj[wbuf.nobj]
}
// dispose returns any cached pointers to the global queue.
// The buffers are being put on the full queue so that the
// write barriers will not simply reacquire them before the
// GC can inspect them. This helps reduce the mutator's
// ability to hide pointers during the concurrent mark phase.
//
//go:nowritebarrierrec
func (w *gcWork) dispose() {
if wbuf := w.wbuf1; wbuf != nil {
if wbuf.nobj == 0 {
putempty(wbuf)
} else {
putfull(wbuf)
w.flushedWork = true
}
w.wbuf1 = nil
wbuf = w.wbuf2
if wbuf.nobj == 0 {
putempty(wbuf)
} else {
putfull(wbuf)
w.flushedWork = true
}
w.wbuf2 = nil
}
if w.bytesMarked != 0 {
// dispose happens relatively infrequently. If this
// atomic becomes a problem, we should first try to
// dispose less and if necessary aggregate in a per-P
// counter.
atomic.Xadd64(&work.bytesMarked, int64(w.bytesMarked))
w.bytesMarked = 0
}
if w.scanWork != 0 {
atomic.Xaddint64(&gcController.scanWork, w.scanWork)
w.scanWork = 0
}
}
// balance moves some work that's cached in this gcWork back on the global queue.
// 将本地一部分工作放回全局队列中
//go:nowritebarrierrec
func (w *gcWork) balance() {
if w.wbuf1 == nil {
// 这里wbuf1 wbuf2队列还没有初始化
return
}
// 如果wbuf2不为空,则上交到全局,并获取一个空队列给wbuf2
if wbuf := w.wbuf2; wbuf.nobj != 0 {
w.checkPut(0, wbuf.obj[:wbuf.nobj])
putfull(wbuf)
w.flushedWork = true
w.wbuf2 = getempty()
} else if wbuf := w.wbuf1; wbuf.nobj > 4 {
w.checkPut(0, wbuf.obj[:wbuf.nobj])
// 把未满的wbuf1分成两半,并把其中一半上交到全局队列
w.wbuf1 = handoff(wbuf)
w.flushedWork = true // handoff did putfull
} else {
return
}
// We flushed a buffer to the full list, so wake a worker.
// 全局队列有满的队列了,其他work可以工作了
if gcphase == _GCmark {
gcController.enlistWorker()
}
}
// empty reports whether w has no mark work available.
//go:nowritebarrierrec
func (w *gcWork) empty() bool {
return w.wbuf1 == nil || (w.wbuf1.nobj == 0 && w.wbuf2.nobj == 0)
}
// Internally, the GC work pool is kept in arrays in work buffers.
// The gcWork interface caches a work buffer until full (or empty) to
// avoid contending on the global work buffer lists.
type workbufhdr struct {
node lfnode // must be first
nobj int
}
//go:notinheap
type workbuf struct {
workbufhdr
// account for the above fields
obj [(_WorkbufSize - unsafe.Sizeof(workbufhdr{})) / sys.PtrSize]uintptr
}
// workbuf factory routines. These funcs are used to manage the
// workbufs.
// If the GC asks for some work these are the only routines that
// make wbufs available to the GC.
func (b *workbuf) checknonempty() {
if b.nobj == 0 {
throw("workbuf is empty")
}
}
func (b *workbuf) checkempty() {
if b.nobj != 0 {
throw("workbuf is not empty")
}
}
// getempty pops an empty work buffer off the work.empty list,
// allocating new buffers if none are available.
//go:nowritebarrier
func getempty() *workbuf {
var b *workbuf
if work.empty != 0 {
b = (*workbuf)(work.empty.pop())
if b != nil {
b.checkempty()
}
}
if b == nil {
// Allocate more workbufs.
var s *mspan
if work.wbufSpans.free.first != nil {
lock(&work.wbufSpans.lock)
s = work.wbufSpans.free.first
if s != nil {
work.wbufSpans.free.remove(s)
work.wbufSpans.busy.insert(s)
}
unlock(&work.wbufSpans.lock)
}
if s == nil {
systemstack(func() {
s = mheap_.allocManual(workbufAlloc/pageSize, &memstats.gc_sys)
})
if s == nil {
throw("out of memory")
}
// Record the new span in the busy list.
lock(&work.wbufSpans.lock)
work.wbufSpans.busy.insert(s)
unlock(&work.wbufSpans.lock)
}
// Slice up the span into new workbufs. Return one and
// put the rest on the empty list.
for i := uintptr(0); i+_WorkbufSize <= workbufAlloc; i += _WorkbufSize {
newb := (*workbuf)(unsafe.Pointer(s.base() + i))
newb.nobj = 0
lfnodeValidate(&newb.node)
if i == 0 {
b = newb
} else {
putempty(newb)
}
}
}
return b
}
// putempty puts a workbuf onto the work.empty list.
// Upon entry this go routine owns b. The lfstack.push relinquishes ownership.
//go:nowritebarrier
func putempty(b *workbuf) {
b.checkempty()
work.empty.push(&b.node)
}
// putfull puts the workbuf on the work.full list for the GC.
// putfull accepts partially full buffers so the GC can avoid competing
// with the mutators for ownership of partially full buffers.
//go:nowritebarrier
func putfull(b *workbuf) {
b.checknonempty()
work.full.push(&b.node)
}
// trygetfull tries to get a full or partially empty workbuffer.
// If one is not immediately available return nil
//go:nowritebarrier
func trygetfull() *workbuf {
b := (*workbuf)(work.full.pop())
if b != nil {
b.checknonempty()
return b
}
return b
}
//go:nowritebarrier
func handoff(b *workbuf) *workbuf {
// Make new buffer with half of b's pointers.
b1 := getempty()
n := b.nobj / 2
b.nobj -= n
b1.nobj = n
memmove(unsafe.Pointer(&b1.obj[0]), unsafe.Pointer(&b.obj[b.nobj]), uintptr(n)*unsafe.Sizeof(b1.obj[0]))
// Put b on full list - let first half of b get stolen.
putfull(b)
return b1
}
// prepareFreeWorkbufs moves busy workbuf spans to free list so they
// can be freed to the heap. This must only be called when all
// workbufs are on the empty list.
func prepareFreeWorkbufs() {
lock(&work.wbufSpans.lock)
if work.full != 0 {
throw("cannot free workbufs when work.full != 0")
}
// Since all workbufs are on the empty list, we don't care
// which ones are in which spans. We can wipe the entire empty
// list and move all workbuf spans to the free list.
work.empty = 0
work.wbufSpans.free.takeAll(&work.wbufSpans.busy)
unlock(&work.wbufSpans.lock)
}
// freeSomeWbufs frees some workbufs back to the heap and returns
// true if it should be called again to free more.
// freeSomeWbufs 释放一些 workbufs 回到堆中,如果需要再次调用则返回 true
func freeSomeWbufs(preemptible bool) bool {
// 每个 span 需要 ~1–2 µs
const batchSize = 64 // ~1–2 µs per span.
lock(&work.wbufSpans.lock)
// 如果此时在标记阶段、或者 wbufSpans 为空,则不需要进行释放
// 因为标记阶段 workbufs 需要被标记,而 workbufs 为空则更不需要释放
if gcphase != _GCoff || work.wbufSpans.free.isEmpty() {
unlock(&work.wbufSpans.lock)
return false
}
systemstack(func() {
gp := getg().m.curg
// 清扫一批 span,64 个,大约 ~1–2 µs
// 在需要被抢占时停止、在清扫完毕后停止
for i := 0; i < batchSize && !(preemptible && gp.preempt); i++ {
span := work.wbufSpans.free.first
if span == nil {
break
}
// 将 span 移除 wbufSpans 的空闲链表中
work.wbufSpans.free.remove(span)
// 将 span 归还到 mheap 中
mheap_.freeManual(span, &memstats.gc_sys)
}
})
// workbufs 的空闲 span 列表尚未清空,还需要更多清扫
more := !work.wbufSpans.free.isEmpty()
unlock(&work.wbufSpans.lock)
return more
}