undo CL 104200047 / 318b04f28372

Breaks windows and race detector.
TBR=rsc

««« original CL description
runtime: stack allocator, separate from mallocgc

In order to move malloc to Go, we need to have a
separate stack allocator.  If we run out of stack
during malloc, malloc will not be available
to allocate a new stack.

Stacks are the last remaining FlagNoGC objects in the
GC heap.  Once they are out, we can get rid of the
distinction between the allocated/blockboundary bits.
(This will be in a separate change.)

Fixes #7468
Fixes #7424

LGTM=rsc, dvyukov
R=golang-codereviews, dvyukov, khr, dave, rsc
CC=golang-codereviews
https://golang.org/cl/104200047
»»»

TBR=rsc
CC=golang-codereviews
https://golang.org/cl/101570044

src/pkg/runtime/malloc.h

@@ -116,12 +116,6 @@ enum
 	MaxMHeapList = 1<<(20 - PageShift),	// Maximum page length for fixed-size list in MHeap.
 	HeapAllocChunk = 1<<20,		// Chunk size for heap growth
 
-	// Per-P, per order stack segment cache size.
-	StackCacheSize = 32*1024,
-	// Number of orders that get caching.  Order 0 is StackMin
-	// and each successive order is twice as large.
-	NumStackOrders = 3,
-
 	// Number of bits in page to span calculations (4k pages).
 	// On Windows 64-bit we limit the arena to 32GB or 35 bits (see below for reason).
 	// On other 64-bit platforms, we limit the arena to 128GB, or 37 bits.
@@ -253,8 +247,8 @@ struct MStats
 
 	// Statistics about allocation of low-level fixed-size structures.
 	// Protected by FixAlloc locks.
-	uint64	stacks_inuse;	// this number is included in heap_inuse above
-	uint64	stacks_sys;	// always 0 in mstats
+	uint64	stacks_inuse;	// bootstrap stacks
+	uint64	stacks_sys;
 	uint64	mspan_inuse;	// MSpan structures
 	uint64	mspan_sys;
 	uint64	mcache_inuse;	// MCache structures
@@ -311,13 +305,6 @@ struct MCacheList
 	uint32 nlist;
 };
 
-typedef struct StackFreeList StackFreeList;
-struct StackFreeList
-{
-	MLink *list;  // linked list of free stacks
-	uintptr size; // total size of stacks in list
-};
-
 // Per-thread (in Go, per-P) cache for small objects.
 // No locking needed because it is per-thread (per-P).
 struct MCache
@@ -333,9 +320,6 @@ struct MCache
 	// The rest is not accessed on every malloc.
 	MSpan*	alloc[NumSizeClasses];	// spans to allocate from
 	MCacheList free[NumSizeClasses];// lists of explicitly freed objects
-
-	StackFreeList stackcache[NumStackOrders];
-
 	// Local allocator stats, flushed during GC.
 	uintptr local_nlookup;		// number of pointer lookups
 	uintptr local_largefree;	// bytes freed for large objects (>MaxSmallSize)
@@ -346,7 +330,6 @@ struct MCache
 MSpan*	runtime·MCache_Refill(MCache *c, int32 sizeclass);
 void	runtime·MCache_Free(MCache *c, MLink *p, int32 sizeclass, uintptr size);
 void	runtime·MCache_ReleaseAll(MCache *c);
-void	runtime·stackcache_clear(MCache *c);
 
 // MTypes describes the types of blocks allocated within a span.
 // The compression field describes the layout of the data.
@@ -426,8 +409,7 @@ struct SpecialProfile
 // An MSpan is a run of pages.
 enum
 {
-	MSpanInUse = 0, // allocated for garbage collected heap
-	MSpanStack,     // allocated for use by stack allocator
+	MSpanInUse = 0,
 	MSpanFree,
 	MSpanListHead,
 	MSpanDead,
@@ -543,16 +525,15 @@ extern MHeap runtime·mheap;
 
 void	runtime·MHeap_Init(MHeap *h);
 MSpan*	runtime·MHeap_Alloc(MHeap *h, uintptr npage, int32 sizeclass, bool large, bool needzero);
-MSpan*	runtime·MHeap_AllocStack(MHeap *h, uintptr npage);
 void	runtime·MHeap_Free(MHeap *h, MSpan *s, int32 acct);
-void	runtime·MHeap_FreeStack(MHeap *h, MSpan *s);
 MSpan*	runtime·MHeap_Lookup(MHeap *h, void *v);
 MSpan*	runtime·MHeap_LookupMaybe(MHeap *h, void *v);
 void	runtime·MGetSizeClassInfo(int32 sizeclass, uintptr *size, int32 *npages, int32 *nobj);
 void*	runtime·MHeap_SysAlloc(MHeap *h, uintptr n);
 void	runtime·MHeap_MapBits(MHeap *h);
 void	runtime·MHeap_MapSpans(MHeap *h);
 void	runtime·MHeap_Scavenger(void);
+void	runtime·MHeap_SplitSpan(MHeap *h, MSpan *s);
 
 void*	runtime·mallocgc(uintptr size, uintptr typ, uint32 flag);
 void*	runtime·persistentalloc(uintptr size, uintptr align, uint64 *stat);

src/pkg/runtime/mcache.c

@@ -43,7 +43,6 @@ void
 runtime·freemcache(MCache *c)
 {
 	runtime·MCache_ReleaseAll(c);
-	runtime·stackcache_clear(c);
 	runtime·lock(&runtime·mheap);
 	runtime·purgecachedstats(c);
 	runtime·FixAlloc_Free(&runtime·mheap.cachealloc, c);

src/pkg/runtime/mcentral.c

@@ -263,8 +263,6 @@ MCentral_Grow(MCentral *c)
 	runtime·unlock(c);
 	runtime·MGetSizeClassInfo(c->sizeclass, &size, &npages, &n);
 	s = runtime·MHeap_Alloc(&runtime·mheap, npages, c->sizeclass, 0, 1);
-	if(s->next != nil || s->prev != nil)
-		runtime·throw("internal error: MSpan should not be in a list");
 	if(s == nil) {
 		// TODO(rsc): Log out of memory
 		runtime·lock(c);

src/pkg/runtime/mem.go

@@ -30,7 +30,7 @@ type MemStats struct {
 	// Low-level fixed-size structure allocator statistics.
 	//	Inuse is bytes used now.
 	//	Sys is bytes obtained from system.
-	StackInuse  uint64 // bytes used by stack allocator
+	StackInuse  uint64 // bootstrap stacks
 	StackSys    uint64
 	MSpanInuse  uint64 // mspan structures
 	MSpanSys    uint64

src/pkg/runtime/mgc0.c

@@ -1252,12 +1252,12 @@ markroot(ParFor *desc, uint32 i)
 			SpecialFinalizer *spf;
 
 			s = allspans[spanidx];
-			if(s->state != MSpanInUse)
-				continue;
 			if(s->sweepgen != sg) {
 				runtime·printf("sweep %d %d\n", s->sweepgen, sg);
 				runtime·throw("gc: unswept span");
 			}
+			if(s->state != MSpanInUse)
+				continue;
 			// The garbage collector ignores type pointers stored in MSpan.types:
 			//  - Compiler-generated types are stored outside of heap.
 			//  - The reflect package has runtime-generated types cached in its data structures.
@@ -2124,7 +2124,6 @@ flushallmcaches(void)
 		if(c==nil)
 			continue;
 		runtime·MCache_ReleaseAll(c);
-		runtime·stackcache_clear(c);
 	}
 }
 
@@ -2134,12 +2133,14 @@ runtime·updatememstats(GCStats *stats)
 	M *mp;
 	MSpan *s;
 	int32 i;
-	uint64 smallfree;
+	uint64 stacks_inuse, smallfree;
 	uint64 *src, *dst;
 
 	if(stats)
 		runtime·memclr((byte*)stats, sizeof(*stats));
+	stacks_inuse = 0;
 	for(mp=runtime·allm; mp; mp=mp->alllink) {
+		stacks_inuse += mp->stackinuse*FixedStack;
 		if(stats) {
 			src = (uint64*)&mp->gcstats;
 			dst = (uint64*)stats;
@@ -2148,6 +2149,7 @@ runtime·updatememstats(GCStats *stats)
 			runtime·memclr((byte*)&mp->gcstats, sizeof(mp->gcstats));
 		}
 	}
+	mstats.stacks_inuse = stacks_inuse;
 	mstats.mcache_inuse = runtime·mheap.cachealloc.inuse;
 	mstats.mspan_inuse = runtime·mheap.spanalloc.inuse;
 	mstats.sys = mstats.heap_sys + mstats.stacks_sys + mstats.mspan_sys +
@@ -2507,12 +2509,6 @@ runtime·ReadMemStats(MStats *stats)
 	// Size of the trailing by_size array differs between Go and C,
 	// NumSizeClasses was changed, but we can not change Go struct because of backward compatibility.
 	runtime·memcopy(runtime·sizeof_C_MStats, stats, &mstats);
-
-	// Stack numbers are part of the heap numbers, separate those out for user consumption
-	stats->stacks_sys = stats->stacks_inuse;
-	stats->heap_inuse -= stats->stacks_inuse;
-	stats->heap_sys -= stats->stacks_inuse;
-
 	g->m->gcing = 0;
 	g->m->locks++;
 	runtime·semrelease(&runtime·worldsema);