diff --git a/src/runtime/malloc.go b/src/runtime/malloc.go index 77a5a387687ef..eaf8db72208cd 100644 --- a/src/runtime/malloc.go +++ b/src/runtime/malloc.go @@ -976,6 +976,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { throw("malloc called with no P") } } + var span *mspan var x unsafe.Pointer noscan := typ == nil || typ.ptrdata == 0 if size <= maxSmallSize { @@ -1028,10 +1029,10 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { return x } // Allocate a new maxTinySize block. - span := c.alloc[tinySpanClass] + span = c.alloc[tinySpanClass] v := nextFreeFast(span) if v == 0 { - v, _, shouldhelpgc = c.nextFree(tinySpanClass) + v, span, shouldhelpgc = c.nextFree(tinySpanClass) } x = unsafe.Pointer(v) (*[2]uint64)(x)[0] = 0 @@ -1052,7 +1053,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { } size = uintptr(class_to_size[sizeclass]) spc := makeSpanClass(sizeclass, noscan) - span := c.alloc[spc] + span = c.alloc[spc] v := nextFreeFast(span) if v == 0 { v, span, shouldhelpgc = c.nextFree(spc) @@ -1063,15 +1064,14 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { } } } else { - var s *mspan shouldhelpgc = true systemstack(func() { - s = largeAlloc(size, needzero, noscan) + span = largeAlloc(size, needzero, noscan) }) - s.freeindex = 1 - s.allocCount = 1 - x = unsafe.Pointer(s.base()) - size = s.elemsize + span.freeindex = 1 + span.allocCount = 1 + x = unsafe.Pointer(span.base()) + size = span.elemsize } var scanSize uintptr @@ -1112,7 +1112,7 @@ func mallocgc(size uintptr, typ *_type, needzero bool) unsafe.Pointer { // This may be racing with GC so do it atomically if there can be // a race marking the bit. if gcphase != _GCoff { - gcmarknewobject(uintptr(x), size, scanSize) + gcmarknewobject(span, uintptr(x), size, scanSize) } if raceenabled { diff --git a/src/runtime/mgcmark.go b/src/runtime/mgcmark.go index dafb4634b4204..fe988c46d9c46 100644 --- a/src/runtime/mgcmark.go +++ b/src/runtime/mgcmark.go @@ -1627,11 +1627,21 @@ func gcDumpObject(label string, obj, off uintptr) { // //go:nowritebarrier //go:nosplit -func gcmarknewobject(obj, size, scanSize uintptr) { +func gcmarknewobject(span *mspan, obj, size, scanSize uintptr) { if useCheckmark { // The world should be stopped so this should not happen. throw("gcmarknewobject called while doing checkmark") } - markBitsForAddr(obj).setMarked() + + // Mark object. + objIndex := span.objIndex(obj) + span.markBitsForIndex(objIndex).setMarked() + + // Mark span. + arena, pageIdx, pageMask := pageIndexOf(span.base()) + if arena.pageMarks[pageIdx]&pageMask == 0 { + atomic.Or8(&arena.pageMarks[pageIdx], pageMask) + } + gcw := &getg().m.p.ptr().gcw gcw.bytesMarked += uint64(size) gcw.scanWork += int64(scanSize) diff --git a/src/runtime/mwbbuf.go b/src/runtime/mwbbuf.go index f444452bab502..632769c11450d 100644 --- a/src/runtime/mwbbuf.go +++ b/src/runtime/mwbbuf.go @@ -296,6 +296,13 @@ func wbBufFlush1(_p_ *p) { continue } mbits.setMarked() + + // Mark span. + arena, pageIdx, pageMask := pageIndexOf(span.base()) + if arena.pageMarks[pageIdx]&pageMask == 0 { + atomic.Or8(&arena.pageMarks[pageIdx], pageMask) + } + if span.spanclass.noscan() { gcw.bytesMarked += uint64(span.elemsize) continue