Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: a Windows application launched via Steam sometimes freezes #71242

Open
hajimehoshi opened this issue Jan 13, 2025 · 30 comments
Open

runtime: a Windows application launched via Steam sometimes freezes #71242

hajimehoshi opened this issue Jan 13, 2025 · 30 comments
Labels
BugReport Issues describing a possible bug in the Go implementation. compiler/runtime Issues related to the Go compiler and/or runtime. NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. OS-Windows
Milestone

Comments

@hajimehoshi
Copy link
Member

hajimehoshi commented Jan 13, 2025

Go version

go version go1.23.2 windows/amd64

Output of go env in your module/workspace:

set GO111MODULE=
set GOARCH=amd64
set GOBIN=
set GOCACHE=C:\Users\hajimehoshi\AppData\Local\go-build
set GOENV=C:\Users\hajimehoshi\AppData\Roaming\go\env
set GOEXE=.exe
set GOEXPERIMENT=
set GOFLAGS=
set GOHOSTARCH=amd64
set GOHOSTOS=windows
set GOINSECURE=
set GOMODCACHE=C:\Users\hajimehoshi\go\pkg\mod
set GONOPROXY=
set GONOSUMDB=
set GOOS=windows
set GOPATH=C:\Users\hajimehoshi\go
set GOPRIVATE=
set GOPROXY=https://proxy.golang.org,direct
set GOROOT=C:\Program Files\Go
set GOSUMDB=sum.golang.org
set GOTMPDIR=
set GOTOOLCHAIN=auto
set GOTOOLDIR=C:\Program Files\Go\pkg\tool\windows_amd64
set GOVCS=
set GOVERSION=go1.23.2
set GODEBUG=
set GOTELEMETRY=local
set GOTELEMETRYDIR=C:\Users\hajimehoshi\AppData\Roaming\go\telemetry
set GCCGO=gccgo
set GOAMD64=v1
set AR=ar
set CC=gcc
set CXX=g++
set CGO_ENABLED=1
set GOMOD=C:\Users\hajimehoshi\ebiten\go.mod
set GOWORK=
set CGO_CFLAGS=-O2 -g
set CGO_CPPFLAGS=
set CGO_CXXFLAGS=-O2 -g
set CGO_FFLAGS=-O2 -g
set CGO_LDFLAGS=-O2 -g
set PKG_CONFIG=pkg-config
set GOGCCFLAGS=-m64 -mthreads -Wl,--no-gc-sections -fmessage-length=0 -ffile-prefix-map=C:\Users\HAJIME~1\AppData\Local\Temp\go-build1125585036=/tmp/go-build -gno-record-gcc-switches

What did you do?

Compile this Go program to an execute file (with -ldflags="-H=windowsgui")

EDIT: I could minized the case further. See #71242 (comment)

package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"syscall"
	"time"
	"unsafe"
)

const (
	WS_OVERLAPPEDWINDOW = 0x00000000 | 0x00C00000 | 0x00080000 | 0x00040000 | 0x00020000 | 0x00010000
	CW_USEDEFAULT       = ^0x7fffffff
	SW_SHOW             = 5
	WM_DESTROY          = 2
)

type (
	ATOM      uint16
	HANDLE    uintptr
	HINSTANCE HANDLE
	HICON     HANDLE
	HCURSOR   HANDLE
	HBRUSH    HANDLE
	HWND      HANDLE
	HMENU     HANDLE
)

type WNDCLASSEX struct {
	Size       uint32
	Style      uint32
	WndProc    uintptr
	ClsExtra   int32
	WndExtra   int32
	Instance   HINSTANCE
	Icon       HICON
	Cursor     HCURSOR
	Background HBRUSH
	MenuName   *uint16
	ClassName  *uint16
	IconSm     HICON
}

type RECT struct {
	Left, Top, Right, Bottom int32
}

type POINT struct {
	X, Y int32
}

type MSG struct {
	Hwnd    HWND
	Message uint32
	WParam  uintptr
	LParam  uintptr
	Time    uint32
	Pt      POINT
}

func GetModuleHandle(modulename *uint16) HINSTANCE {
	r, _, _ := syscall.SyscallN(procGetModuleHandle.Addr(), uintptr(unsafe.Pointer(modulename)))
	return HINSTANCE(r)
}

func RegisterClassEx(w *WNDCLASSEX) ATOM {
	r, _, _ := syscall.SyscallN(procRegisterClassEx.Addr(), uintptr(unsafe.Pointer(w)))
	return ATOM(r)
}

func CreateWindowEx(exStyle uint, className, windowName *uint16,
	style uint, x, y, width, height int, parent HWND, menu HMENU,
	instance HINSTANCE, param unsafe.Pointer) HWND {
	r, _, _ := syscall.SyscallN(procCreateWindowEx.Addr(), uintptr(exStyle), uintptr(unsafe.Pointer(className)),
		uintptr(unsafe.Pointer(windowName)), uintptr(style), uintptr(x), uintptr(y), uintptr(width), uintptr(height),
		uintptr(parent), uintptr(menu), uintptr(instance), uintptr(param))
	return HWND(r)
}

func AdjustWindowRect(rect *RECT, style uint, menu bool) bool {
	var iMenu uintptr
	if menu {
		iMenu = 1
	}
	r, _, _ := syscall.SyscallN(procAdjustWindowRect.Addr(), uintptr(unsafe.Pointer(rect)), uintptr(style), iMenu)
	return r != 0
}

func ShowWindow(hwnd HWND, cmdshow int) bool {
	r, _, _ := syscall.SyscallN(procShowWindow.Addr(), uintptr(hwnd), uintptr(cmdshow))
	return r != 0
}

func GetMessage(msg *MSG, hwnd HWND, msgFilterMin, msgFilterMax uint32) int {
	r, _, _ := syscall.SyscallN(procGetMessage.Addr(), uintptr(unsafe.Pointer(msg)), uintptr(hwnd), uintptr(msgFilterMin), uintptr(msgFilterMax))
	return int(r)
}

func TranslateMessage(msg *MSG) bool {
	r, _, _ := syscall.SyscallN(procTranslateMessage.Addr(), uintptr(unsafe.Pointer(msg)))
	return r != 0
}

func DispatchMessage(msg *MSG) uintptr {
	r, _, _ := syscall.SyscallN(procDispatchMessage.Addr(), uintptr(unsafe.Pointer(msg)))
	return r
}

func DefWindowProc(hwnd HWND, msg uint32, wParam, lParam uintptr) uintptr {
	r, _, _ := syscall.SyscallN(procDefWindowProc.Addr(), uintptr(hwnd), uintptr(msg), wParam, lParam)
	return r
}

func PostQuitMessage(exitCode int) {
	syscall.SyscallN(procPostQuitMessage.Addr(), uintptr(exitCode))
}

var (
	kernel32            = syscall.NewLazyDLL("kernel32.dll")
	procGetModuleHandle = kernel32.NewProc("GetModuleHandleW")

	user32               = syscall.NewLazyDLL("user32.dll")
	procRegisterClassEx  = user32.NewProc("RegisterClassExW")
	procCreateWindowEx   = user32.NewProc("CreateWindowExW")
	procAdjustWindowRect = user32.NewProc("AdjustWindowRect")
	procShowWindow       = user32.NewProc("ShowWindow")
	procGetMessage       = user32.NewProc("GetMessageW")
	procTranslateMessage = user32.NewProc("TranslateMessage")
	procDispatchMessage  = user32.NewProc("DispatchMessageW")
	procDefWindowProc    = user32.NewProc("DefWindowProcW")
	procPostQuitMessage  = user32.NewProc("PostQuitMessage")
)

func init() {
	runtime.LockOSThread()
}

func main() {
	className, err := syscall.UTF16PtrFromString("Sample Window Class")
	if err != nil {
		panic(err)
	}
	inst := GetModuleHandle(className)

	wc := WNDCLASSEX{
		Size:      uint32(unsafe.Sizeof(WNDCLASSEX{})),
		WndProc:   syscall.NewCallback(wndProc),
		Instance:  inst,
		ClassName: className,
	}

	RegisterClassEx(&wc)

	wr := RECT{
		Left:   0,
		Top:    0,
		Right:  320,
		Bottom: 240,
	}
	title, err := syscall.UTF16PtrFromString("My Title")
	if err != nil {
		panic(err)
	}
	AdjustWindowRect(&wr, WS_OVERLAPPEDWINDOW, false)
	hwnd := CreateWindowEx(
		0, className,
		title,
		WS_OVERLAPPEDWINDOW,
		CW_USEDEFAULT, CW_USEDEFAULT, int(wr.Right-wr.Left), int(wr.Bottom-wr.Top),
		0, 0, inst, nil,
	)
	if hwnd == 0 {
		panic(syscall.GetLastError())
	}

	ShowWindow(hwnd, SW_SHOW)

	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	go func() {
		f, err := os.Create("log.txt")
		if err != nil {
			panic(err)
		}
		defer f.Close()

		log.SetOutput(f)

		for {
			time.Sleep(time.Second)
			var gcStats debug.GCStats
			debug.ReadGCStats(&gcStats)
			log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
			if err := f.Sync(); err != nil {
				panic(err)
			}
		}
	}()

	var msg MSG
	for GetMessage(&msg, 0, 0, 0) != 0 {
		TranslateMessage(&msg)
		DispatchMessage(&msg)
	}
}

func wndProc(hwnd HWND, msg uint32, wparam, lparam uintptr) uintptr {
	switch msg {
	case WM_DESTROY:
		PostQuitMessage(0)
	}
	return DefWindowProc(hwnd, msg, wparam, lparam)
}

Replace an exe file in a Steam game with the compiled exe file, and run it via Steam client.

What did you see happen?

The application sometimes freezes for more than 10 seconds. For example, I saw this log

2025/01/13 23:23:41 LastGC: 2025-01-13 23:23:41.6060533 +0900 JST, NumGC: 51, PauseTotal: 1.3186ms
2025/01/13 23:23:42 LastGC: 2025-01-13 23:23:42.6470915 +0900 JST, NumGC: 102, PauseTotal: 4.5945ms
2025/01/13 23:23:43 LastGC: 2025-01-13 23:23:43.6434896 +0900 JST, NumGC: 152, PauseTotal: 7.8752ms
2025/01/13 23:23:44 LastGC: 2025-01-13 23:23:44.6481645 +0900 JST, NumGC: 204, PauseTotal: 10.224ms
2025/01/13 23:23:45 LastGC: 2025-01-13 23:23:45.6448025 +0900 JST, NumGC: 255, PauseTotal: 12.5067ms
2025/01/13 23:23:46 LastGC: 2025-01-13 23:23:46.6584686 +0900 JST, NumGC: 309, PauseTotal: 14.2881ms
2025/01/13 23:23:47 LastGC: 2025-01-13 23:23:47.6550747 +0900 JST, NumGC: 362, PauseTotal: 17.056ms
2025/01/13 23:23:48 LastGC: 2025-01-13 23:23:48.6681264 +0900 JST, NumGC: 413, PauseTotal: 18.5012ms
2025/01/13 23:23:49 LastGC: 2025-01-13 23:23:49.6577372 +0900 JST, NumGC: 464, PauseTotal: 21.2877ms
2025/01/13 23:23:50 LastGC: 2025-01-13 23:23:50.6675317 +0900 JST, NumGC: 514, PauseTotal: 24.9508ms
2025/01/13 23:23:51 LastGC: 2025-01-13 23:23:51.6642908 +0900 JST, NumGC: 563, PauseTotal: 27.2671ms
2025/01/13 23:23:52 LastGC: 2025-01-13 23:23:52.6696781 +0900 JST, NumGC: 612, PauseTotal: 29.6692ms
2025/01/13 23:23:53 LastGC: 2025-01-13 23:23:53.6818947 +0900 JST, NumGC: 661, PauseTotal: 31.9968ms
2025/01/13 23:23:54 LastGC: 2025-01-13 23:23:54.6830572 +0900 JST, NumGC: 711, PauseTotal: 34.9958ms
2025/01/13 23:23:55 LastGC: 2025-01-13 23:23:55.6889185 +0900 JST, NumGC: 761, PauseTotal: 38.2468ms
2025/01/13 23:23:56 LastGC: 2025-01-13 23:23:56.6869067 +0900 JST, NumGC: 813, PauseTotal: 41.5747ms
2025/01/13 23:23:57 LastGC: 2025-01-13 23:23:57.6920325 +0900 JST, NumGC: 863, PauseTotal: 45.5415ms
2025/01/13 23:24:18 LastGC: 2025-01-13 23:23:58.2810119 +0900 JST, NumGC: 894, PauseTotal: 47.2387ms
2025/01/13 23:24:19 LastGC: 2025-01-13 23:24:19.3442472 +0900 JST, NumGC: 945, PauseTotal: 51.3869ms
2025/01/13 23:24:20 LastGC: 2025-01-13 23:24:20.3460036 +0900 JST, NumGC: 995, PauseTotal: 54.0004ms
2025/01/13 23:24:21 LastGC: 2025-01-13 23:24:21.3371656 +0900 JST, NumGC: 1047, PauseTotal: 55.3437ms
2025/01/13 23:24:22 LastGC: 2025-01-13 23:24:22.344327 +0900 JST, NumGC: 1098, PauseTotal: 56.9757ms
2025/01/13 23:24:23 LastGC: 2025-01-13 23:24:23.3523815 +0900 JST, NumGC: 1147, PauseTotal: 61.8229ms
2025/01/13 23:24:24 LastGC: 2025-01-13 23:24:24.3560493 +0900 JST, NumGC: 1200, PauseTotal: 64.7476ms
2025/01/13 23:24:25 LastGC: 2025-01-13 23:24:25.3552534 +0900 JST, NumGC: 1250, PauseTotal: 67.5551ms

You can see a freeze happens between 22:23:57 and 22:24:18.

What did you expect to see?

The application doesn't freeze.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 13, 2025

This was originally reported at hajimehoshi/ebiten#3181 by @corfe83.

  • A Steam overlay might cause this issue but we are not sure. This freeze happens even when the Steam overlay is disabled.
  • This freeze never happens when GC is suspended by debug.SetGCPercent(-1)
  • This freeze never happens when the application is launched without Steam.

@gabyhelp gabyhelp added the BugReport Issues describing a possible bug in the Go implementation. label Jan 13, 2025
@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 13, 2025

I could reproduce this even with a console application (without -ldflags="-H=windowsgui"). runtime.LockOSThread is not needed. Replacing an exe in a Steam game is still required.

package main

import (
	"log"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}
2025/01/14 00:19:14 LastGC: 2025-01-14 00:19:14.3262459 +0900 JST, NumGC: 51, PauseTotal: 1.016ms
2025/01/14 00:19:15 LastGC: 2025-01-14 00:19:15.3519275 +0900 JST, NumGC: 100, PauseTotal: 6.9976ms
2025/01/14 00:19:16 LastGC: 2025-01-14 00:19:16.3609673 +0900 JST, NumGC: 148, PauseTotal: 11.6081ms
2025/01/14 00:19:17 LastGC: 2025-01-14 00:19:17.3715984 +0900 JST, NumGC: 197, PauseTotal: 15.5458ms
2025/01/14 00:19:18 LastGC: 2025-01-14 00:19:18.3726086 +0900 JST, NumGC: 245, PauseTotal: 17.6441ms
2025/01/14 00:19:19 LastGC: 2025-01-14 00:19:19.3673871 +0900 JST, NumGC: 293, PauseTotal: 19.8266ms
2025/01/14 00:19:20 LastGC: 2025-01-14 00:19:20.3597451 +0900 JST, NumGC: 341, PauseTotal: 25.4462ms
2025/01/14 00:19:21 LastGC: 2025-01-14 00:19:21.3536769 +0900 JST, NumGC: 388, PauseTotal: 30.0621ms
2025/01/14 00:19:37 LastGC: 2025-01-14 00:19:37.1879618 +0900 JST, NumGC: 424, PauseTotal: 30.2376ms
2025/01/14 00:19:38 LastGC: 2025-01-14 00:19:38.1708792 +0900 JST, NumGC: 468, PauseTotal: 31.8384ms
2025/01/14 00:19:39 LastGC: 2025-01-14 00:19:39.1731811 +0900 JST, NumGC: 515, PauseTotal: 33.0891ms
2025/01/14 00:19:40 LastGC: 2025-01-14 00:19:40.1890122 +0900 JST, NumGC: 562, PauseTotal: 33.7643ms
2025/01/14 00:19:41 LastGC: 2025-01-14 00:19:41.1767277 +0900 JST, NumGC: 608, PauseTotal: 34.0499ms

A freeze occurs between 00:19:21 and 00:19:37

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 13, 2025

Apparently most of the threads got stuck at NtWaitForSingleObject when freezing, but I have no idea what was going on there...

Thread 0x4 (gameoverlayrenderer64!OverlayHookD3D3 seems a hooked function by Steam)
Image

Thread 0x5
Image

@hajimehoshi hajimehoshi changed the title a Windows application launched via Steam sometimes freezes runtime: a Windows application launched via Steam sometimes freezes Jan 13, 2025
@gopherbot gopherbot added the compiler/runtime Issues related to the Go compiler and/or runtime. label Jan 13, 2025
@mknyszek mknyszek added the NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. label Jan 13, 2025
@mknyszek mknyszek added this to the Backlog milestone Jan 13, 2025
@mknyszek
Copy link
Contributor

CC @golang/runtime

@qmuntal
Copy link
Member

qmuntal commented Jan 13, 2025

Does this issue reproduce with Go 1.22?

@hajimehoshi
Copy link
Member Author

Does this issue reproduce with Go 1.22?

Yes, I could reproduce this with Go 1.22.9.

@mknyszek
Copy link
Contributor

mknyszek commented Jan 13, 2025

Do you have a stack trace for each thread, for example from a debugger?

My guess based on the information provided would be that the threads are stuck in some GC-related thing (spinning up GC mark workers? that only happens on the first GC if GOMAXPROCS doesn't ever change) on a note (https://cs.opensource.google/go/go/+/master:src/runtime/os_windows.go;l=676;drc=4f881115d4067bda8a236aabcae8c41cdd13b4d0). I couldn't tell you why that would be a problem, since this happens all the time.

Alternatively, the threads are going to sleep for STW, which is also based on a note (https://cs.opensource.google/go/go/+/master:src/runtime/proc.go;l=1612;drc=f025d19e7b3f0c66242760c213cc2b54cb100f69), and perhaps something is preventing the thread stopping the world from continuing promptly. Note that the thread which is stopping-the-world spins, sleeping in 100ms increments (but is awoken early when the last thread goes to sleep). So it might be that too. We could confirm/deny this by disabling the GC but calling runtime.ReadMemStats at some regular interval. This also happens all the time without issue, though, so I also don't really have a guess as to why this would be a problem running under Steam.

@hajimehoshi
Copy link
Member Author

Do you have a stack trace for each thread, for example from a debugger?

I'm afraid I'm not familiar with WinDbg. How can I get the stack traces as a file? I'll try tomorrow.

Also I can insert printlns in the runtime by -overlay. I'll try this later too.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

I dumped stack traces by x64dbg:

Stack traces by x64dbg
Thread ID           Address          To               From             Size       Party  Comment                                      
10876 - Main Thread                                                                      
                    0000009585FFF798 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    0000009585FFF838 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    0000009585FFF9A8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009585FFF9D8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    0000009585FFF9F8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009585FFFA30 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009585FFFA48 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    0000009585FFFAB0 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    0000009585FFFAE8 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    0000009585FFFB18 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    0000009585FFFC90 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    0000009585FFFCC8 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    0000009585FFFD20 000000000083A7F3 000000000080EDCB 2A7A12A1F0 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C000129F10 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C000129F30 00000000007E7FC9 000000000083572E 90         User   innovation2007_windows_amd64.000000000083572E
                    000000C000129FC0 00000000007E7EA5 00000000007E7FC9 18         User   innovation2007_windows_amd64.00000000007E7FC9
                    000000C000129FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000129FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
26956                                                                                    
                    0000009586FFF778 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    0000009586FFF818 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    0000009586FFF988 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586FFF9B8 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586FFF9E0 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586FFFA18 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586FFFA30 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    0000009586FFFA98 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    0000009586FFFAD0 000000000080BD51 00000000007DB9D2 20         User   innovation2007_windows_amd64.00000000007DB9D2
                    0000009586FFFAF0 000000000080A73D 000000000080BD51 28         User   innovation2007_windows_amd64.000000000080BD51
                    0000009586FFFB18 000000000080A68A 000000000080A73D 28         User   innovation2007_windows_amd64.000000000080A73D
                    0000009586FFFB40 000000000083A765 000000000080A68A 8          User   innovation2007_windows_amd64.000000000080A68A
                    0000009586FFFB48 000000000083E477 000000000083A765 8          User   innovation2007_windows_amd64.000000000083A765
                    0000009586FFFB50 0000000000000000 000000000083E477            User   innovation2007_windows_amd64.000000000083E477
20696                                                                                    
                    00000095861FF8D8 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                    00000095861FFBB8 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                    00000095861FFBE8 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                    00000095861FFC68 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
29680                                                                                    
                    00000095865FF9F8 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                    00000095865FFCD8 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                    00000095865FFD08 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                    00000095865FFD88 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
12772                                                                                    
                    00000095863FFC28 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                    00000095863FFF08 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                    00000095863FFF38 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                    00000095863FFFB8 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
21908                                                                                    
                    00000095867FF328 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095867FF3C8 00007FFD45D11D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                    00000095867FF438 000000000083E029 00007FFD45D11D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                    00000095867FF5A8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095867FF5D8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    00000095867FF5F8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095867FF630 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095867FF648 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    00000095867FFBA8 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    00000095867FFBC0 0000000000813A36 0000000000813D35 60         User   innovation2007_windows_amd64.0000000000813D35
                    00000095867FFC20 000000000081379A 0000000000813A36 80         User   innovation2007_windows_amd64.0000000000813A36
                    00000095867FFCA0 000000000080A73D 000000000081379A 28         User   innovation2007_windows_amd64.000000000081379A
                    00000095867FFCC8 000000000080A68A 000000000080A73D 28         User   innovation2007_windows_amd64.000000000080A73D
                    00000095867FFCF0 000000000083A765 000000000080A68A 8          User   innovation2007_windows_amd64.000000000080A68A
                    00000095867FFCF8 000000000083E477 000000000083A765 8          User   innovation2007_windows_amd64.000000000083A765
                    00000095867FFD00 0000000000000000 000000000083E477            User   innovation2007_windows_amd64.000000000083E477
29272                                                                                    
                    00000095869FFB78 00000000007EA1B4 00000000007EC202 40         User   innovation2007_windows_amd64.00000000007EC202
                    00000095869FFBB8 00000000007E9F27 00000000007EA1B4 A8         User   innovation2007_windows_amd64.00000000007EA1B4
                    00000095869FFC60 00000000007EBF14 00000000007E9F27 68         User   innovation2007_windows_amd64.00000000007E9F27
                    00000095869FFCC8 00000000007E8407 00000000007EBF14 50         User   innovation2007_windows_amd64.00000000007EBF14
                    00000095869FFD18 000000000083A869 00000000007E8407 2A79728218 User   innovation2007_windows_amd64.00000000007E8407
                    000000C000127F30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C000127FC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C000127FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000127FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
12632                                                                                    
                    0000009586BFF208 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    0000009586BFF2A8 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    0000009586BFF418 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586BFF448 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586BFF468 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586BFF4A0 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586BFF4B8 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    0000009586BFF520 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    0000009586BFF558 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    0000009586BFF588 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    0000009586BFF700 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    0000009586BFF738 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    0000009586BFF790 000000000083A7F3 000000000080EDCB 2A79534780 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C000133F10 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C000133F30 00000000007E7FC9 000000000083572E 90         User   innovation2007_windows_amd64.000000000083572E
                    000000C000133FC0 00000000007E7EA5 00000000007E7FC9 18         User   innovation2007_windows_amd64.00000000007E7FC9
                    000000C000133FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000133FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
12836                                                                                    
                    0000009586DFEE48 00007FFDB8D9BEAD 00007FFDBB9907A4 30         System ntdll.NtReleaseMutant+14
                    0000009586DFEE78 00007FFD45D11EC0 00007FFDB8D9BEAD 70         User   kernelbase.ReleaseMutex+D
                    0000009586DFEEE8 000000000083E029 00007FFD45D11EC0 170        User   gameoverlayrenderer64.OverlayHookD3D3+27380
                    0000009586DFF058 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586DFF088 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586DFF0B0 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586DFF0E8 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586DFF100 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    0000009586DFF660 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    0000009586DFF678 00000000007EDC96 0000000000813D35 30         User   innovation2007_windows_amd64.0000000000813D35
                    0000009586DFF6A8 00000000007F3825 00000000007EDC96 18         User   innovation2007_windows_amd64.00000000007EDC96
                    0000009586DFF6C0 00000000007EBC87 00000000007F3825 68         User   innovation2007_windows_amd64.00000000007F3825
                    0000009586DFF728 00000000007E843E 00000000007EBC87 50         User   innovation2007_windows_amd64.00000000007EBC87
                    0000009586DFF778 000000000083A869 00000000007E843E 2A7932E7B8 User   innovation2007_windows_amd64.00000000007E843E
                    000000C00012DF30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C00012DFC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C00012DFD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C00012DFE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
22360                                                                                    
                    00000095871FF598 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095871FF638 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    00000095871FF7A8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095871FF7D8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    00000095871FF7F8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095871FF830 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095871FF848 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    00000095871FF8B0 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    00000095871FF8E8 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    00000095871FF918 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    00000095871FFA90 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    00000095871FFAC8 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    00000095871FFB20 000000000083A7F3 000000000080EDCB 2A78F2C3F0 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C00012BF10 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C00012BF30 00000000007E7FC9 000000000083572E 90         User   innovation2007_windows_amd64.000000000083572E
                    000000C00012BFC0 00000000007E7EA5 00000000007E7FC9 18         User   innovation2007_windows_amd64.00000000007E7FC9
                    000000C00012BFD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C00012BFE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
1584                                                                                     
                    00000095873FF388 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095873FF428 00007FFD45D11D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                    00000095873FF498 000000000083E029 00007FFD45D11D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                    00000095873FF608 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095873FF638 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    00000095873FF660 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095873FF698 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095873FF6B0 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    00000095873FFC10 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    00000095873FFC28 00000000007EDC96 0000000000813D35 30         User   innovation2007_windows_amd64.0000000000813D35
                    00000095873FFC58 00000000007F3825 00000000007EDC96 18         User   innovation2007_windows_amd64.00000000007EDC96
                    00000095873FFC70 00000000007EBC87 00000000007F3825 68         User   innovation2007_windows_amd64.00000000007F3825
                    00000095873FFCD8 00000000007E8407 00000000007EBC87 50         User   innovation2007_windows_amd64.00000000007EBC87
                    00000095873FFD28 000000000083A869 00000000007E8407 2A78D36208 User   innovation2007_windows_amd64.00000000007E8407
                    000000C000135F30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C000135FC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C000135FD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C000135FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
10740                                                                                    
                    00000095875FF488 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095875FF528 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    00000095875FF698 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095875FF6C8 000000000083C4C1 000000000083C40D 20         User   innovation2007_windows_amd64.000000000083C40D
                    00000095875FF6E8 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095875FF720 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095875FF738 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    00000095875FF7A0 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    00000095875FF7D8 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    00000095875FF808 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    00000095875FF980 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    00000095875FF9B8 000000000080EDCB 000000000080E9B1 58         User   innovation2007_windows_amd64.000000000080E9B1
                    00000095875FFA10 000000000083A7F3 000000000080EDCB 2A78B322B8 User   innovation2007_windows_amd64.000000000080EDCB
                    000000C000131CC8 000000000083572E 000000000083A7F3 20         User   innovation2007_windows_amd64.000000000083A7F3
                    000000C000131CE8 00000000007EAF67 000000000083572E 38         User   innovation2007_windows_amd64.000000000083572E
                    000000C000131D20 00000000007EA88A 00000000007EAF67 60         User   innovation2007_windows_amd64.00000000007EAF67
                    000000C000131D80 00000000007DCDF4 00000000007EA88A 28         User   innovation2007_windows_amd64.00000000007EA88A
                    000000C000131DA8 00000000008314DE 00000000007DCDF4 A0         User   innovation2007_windows_amd64.00000000007DCDF4
                    000000C000131E48 00000000007DCEE5 00000000008314DE 28         User   innovation2007_windows_amd64.00000000008314DE
                    000000C000131E70 0000000000878527 00000000007DCEE5 D8         User   innovation2007_windows_amd64.00000000007DCEE5
                    000000C000131F48 000000000080737D 0000000000878527 90         User   innovation2007_windows_amd64.0000000000878527
                    000000C000131FD8 000000000083C821 000000000080737D 8          User   innovation2007_windows_amd64.000000000080737D
                    000000C000131FE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821
29196                                                                                    
                    00000095877FF1C8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                    00000095877FF268 000000000083E029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                    00000095877FF3D8 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    00000095877FF408 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    00000095877FF430 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    00000095877FF468 0000000000801CD6 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    00000095877FF480 0000000000800DC5 0000000000801CD6 68         User   innovation2007_windows_amd64.0000000000801CD6
                    00000095877FF4E8 00000000007DB9D2 0000000000800DC5 38         User   innovation2007_windows_amd64.0000000000800DC5
                    00000095877FF520 000000000080BE6C 00000000007DB9D2 30         User   innovation2007_windows_amd64.00000000007DB9D2
                    00000095877FF550 000000000080D8DC 000000000080BE6C 178        User   innovation2007_windows_amd64.000000000080BE6C
                    00000095877FF6C8 000000000080E9B1 000000000080D8DC 38         User   innovation2007_windows_amd64.000000000080D8DC
                    00000095877FF700 000000000080A777 000000000080E9B1 28         User   innovation2007_windows_amd64.000000000080E9B1
                    00000095877FF728 000000000080A68A 000000000080A777 28         User   innovation2007_windows_amd64.000000000080A777
                    00000095877FF750 000000000083A765 000000000080A68A 8          User   innovation2007_windows_amd64.000000000080A68A
                    00000095877FF758 000000000083E477 000000000083A765 8          User   innovation2007_windows_amd64.000000000083A765
                    00000095877FF760 0000000000000000 000000000083E477            User   innovation2007_windows_amd64.000000000083E477

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

I also dumped goroutine stacks by delve

Goroutine Dumps
  Goroutine 1 - User: C:/Users/hajimehoshi/ebiten/examples/test/main.go:29 main.main (0x878527) [GC assist wait]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007eaf67 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007eaf67 in runtime.gcParkAssist
            at C:/Program Files/Go/src/runtime/mgcmark.go:737
        3  0x00000000007ea88a in runtime.gcAssistAlloc
            at C:/Program Files/Go/src/runtime/mgcmark.go:577
        4  0x00000000007dcdf4 in runtime.deductAssistCredit
            at C:/Program Files/Go/src/runtime/malloc.go:1349
        5  0x00000000008314de in runtime.mallocgc
            at C:/Program Files/Go/src/runtime/malloc.go:1037
        6  0x00000000007dcee5 in runtime.newobject
            at C:/Program Files/Go/src/runtime/malloc.go:1386
        7  0x0000000000878527 in main.main
            at C:/Users/hajimehoshi/ebiten/examples/test/main.go:29
        8  0x000000000080737d in runtime.main
            at C:/Program Files/Go/src/runtime/proc.go:272
        9  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 2 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [force gc (idle) 53406931548000]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x0000000000807698 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x0000000000807698 in runtime.forcegchelper
            at C:/Program Files/Go/src/runtime/proc.go:337
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 3 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [GC sweep wait 53437762463500]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007f153f in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007f153f in runtime.bgsweep
            at C:/Program Files/Go/src/runtime/mgcsweep.go:317
        3  0x00000000007e5e25 in runtime.gcenable.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:203
        4  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 4 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [GC scavenge wait]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007eef29 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007eef29 in runtime.(*scavengerState).park
            at C:/Program Files/Go/src/runtime/mgcscavenge.go:425
        3  0x00000000007ef4b9 in runtime.bgscavenge
            at C:/Program Files/Go/src/runtime/mgcscavenge.go:658
        4  0x00000000007e5dc5 in runtime.gcenable.gowrap2
            at C:/Program Files/Go/src/runtime/mgc.go:204
        5  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 5 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [finalizer wait 53406931548000]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e4ee7 in runtime.runfinq
            at C:/Program Files/Go/src/runtime/mfinal.go:193
        2  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 18 - User: C:/Users/hajimehoshi/ebiten/examples/test/main.go:22 main.main.func1 (0x8787a5) [GC assist wait]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007eaf67 in runtime.goparkunlock
            at C:/Program Files/Go/src/runtime/proc.go:430
        2  0x00000000007eaf67 in runtime.gcParkAssist
            at C:/Program Files/Go/src/runtime/mgcmark.go:737
        3  0x00000000007ea88a in runtime.gcAssistAlloc
            at C:/Program Files/Go/src/runtime/mgcmark.go:577
        4  0x00000000007dcdf4 in runtime.deductAssistCredit
            at C:/Program Files/Go/src/runtime/malloc.go:1349
        5  0x00000000008314de in runtime.mallocgc
            at C:/Program Files/Go/src/runtime/malloc.go:1037
        6  0x0000000000836609 in runtime.makeslice
            at C:/Program Files/Go/src/runtime/slice.go:116
        7  0x00000000008787a5 in main.main.func1
            at C:/Users/hajimehoshi/ebiten/examples/test/main.go:22
        8  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 19 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53406931548000]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 20 - User: :0 ??? (0x7ffdbb990424) (thread 31972) [GC mark termination 53437762463500]
        0  0x00007ffdbb990424 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)
  Goroutine 21 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53435067473800]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 22 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53437762463500]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 23 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53437077056700]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 24 - User: :0 ??? (0x7ffdbb9907a4) (thread 24128) [GC mark termination]
        0  0x00007ffdbb9907a4 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)
  Goroutine 25 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call 53437717344800]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
  Goroutine 26 - User: C:/Program Files/Go/src/runtime/proc.go:425 runtime.gopark (0x83572e) [debug call]
        0  0x000000000083572e in runtime.gopark
            at C:/Program Files/Go/src/runtime/proc.go:425
        1  0x00000000007e7fc9 in runtime.gcBgMarkWorker
            at C:/Program Files/Go/src/runtime/mgc.go:1363
        2  0x00000000007e7ea5 in runtime.gcBgMarkStartWorkers.gowrap1
            at C:/Program Files/Go/src/runtime/mgc.go:1279
        3  0x000000000083c821 in runtime.goexit
            at C:/Program Files/Go/src/runtime/asm_amd64.s:1700
[14 goroutines]
Go source
package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	/*for _, env := range os.Environ() {
		log.Println(env)
	}*/
	/*if runtime.GOOS == "windows" && os.Getenv("SteamClientLaunch") == "1" {
		runtime.GOMAXPROCS(max(1, min(2, runtime.NumCPU()-1)))
	}*/
	log.Println(runtime.Version())
	log.Printf("PID: %d", os.Getpid())
	go func() {
		for {
			_ = make([]byte, 256*1024)
			time.Sleep(time.Millisecond)
		}
	}()

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}

@hajimehoshi
Copy link
Member Author

My guess based on the information provided would be that the threads are stuck in some GC-related thing (spinning up GC mark workers? that only happens on the first GC if GOMAXPROCS doesn't ever change) on a note (https://cs.opensource.google/go/go/+/master:src/runtime/os_windows.go;l=676;drc=4f881115d4067bda8a236aabcae8c41cdd13b4d0). I couldn't tell you why that would be a problem, since this happens all the time.

Adding println affects the behavior apparently, so I am not sure this was a correct observation, but I saw goroutines got stuck at the stdcall2 with _WaitForSingleObject.

Alternatively, the threads are going to sleep for STW, which is also based on a note (https://cs.opensource.google/go/go/+/master:src/runtime/proc.go;l=1612;drc=f025d19e7b3f0c66242760c213cc2b54cb100f69), and perhaps something is preventing the thread stopping the world from continuing promptly. Note that the thread which is stopping-the-world spins, sleeping in 100ms increments (but is awoken early when the last thread goes to sleep). So it might be that too. We could confirm/deny this by disabling the GC but calling runtime.ReadMemStats at some regular interval. This also happens all the time without issue, though, so I also don't really have a guess as to why this would be a problem running under Steam.

STW was invoked on GOMAXPROCS in my case, but this was not related to the freeze. Freezing happens regardless of STW.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	debug.SetGCPercent(-1)
	go func() {
		for {
			println("runtime.GC start")
			runtime.GC()
			println("runtime.GC end")
			time.Sleep(5 * time.Millisecond)
		}
	}()

	log.Println(runtime.Version())
	log.Printf("PID: %d", os.Getpid())
	for i := 0; i < 5; i++ {
		go func() {
			for {
				_ = make([]byte, 256*1024)
				time.Sleep(time.Millisecond)
			}
		}()
	}

	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}

The above program manually calls runtime.GC. I run it with the Steam client and found that the goroutine for GC sometimes stops at runtime.GC. runtime.GC sometimes took more than 10 seconds. I've added printlns in runtime.GC implementation and found the locations where the goroutine got stuck seemed inconsistent. I guess the Go scheduler sometimes failed to assign a thread to the goroutine. And in the worst case, all the goroutines suspended.

@prattmic
Copy link
Member

Thanks for the details.

In the x64dbg stack traces, it looks like every thread is blocked, which seems suspicious. This thread seems particularly suspicious to me:

12836                                                                                    
                    0000009586DFEE48 00007FFDB8D9BEAD 00007FFDBB9907A4 30         System ntdll.NtReleaseMutant+14
                    0000009586DFEE78 00007FFD45D11EC0 00007FFDB8D9BEAD 70         User   kernelbase.ReleaseMutex+D
                    0000009586DFEEE8 000000000083E029 00007FFD45D11EC0 170        User   gameoverlayrenderer64.OverlayHookD3D3+27380
                    0000009586DFF058 000000000083C40D 000000000083E029 30         User   innovation2007_windows_amd64.000000000083E029
                    0000009586DFF088 000000000083C4C1 000000000083C40D 28         User   innovation2007_windows_amd64.000000000083C40D
                    0000009586DFF0B0 0000000000801B9B 000000000083C4C1 38         User   innovation2007_windows_amd64.000000000083C4C1
                    0000009586DFF0E8 0000000000801C56 0000000000801B9B 18         User   innovation2007_windows_amd64.0000000000801B9B
                    0000009586DFF100 0000000000802592 0000000000801C56 560        User   innovation2007_windows_amd64.0000000000801C56
                    0000009586DFF660 0000000000813D35 0000000000802592 18         User   innovation2007_windows_amd64.0000000000802592
                    0000009586DFF678 00000000007EDC96 0000000000813D35 30         User   innovation2007_windows_amd64.0000000000813D35
                    0000009586DFF6A8 00000000007F3825 00000000007EDC96 18         User   innovation2007_windows_amd64.00000000007EDC96
                    0000009586DFF6C0 00000000007EBC87 00000000007F3825 68         User   innovation2007_windows_amd64.00000000007F3825
                    0000009586DFF728 00000000007E843E 00000000007EBC87 50         User   innovation2007_windows_amd64.00000000007EBC87
                    0000009586DFF778 000000000083A869 00000000007E843E 2A7932E7B8 User   innovation2007_windows_amd64.00000000007E843E
                    000000C00012DF30 00000000007E80CC 000000000083A869 90         User   innovation2007_windows_amd64.000000000083A869
                    000000C00012DFC0 00000000007E7EA5 00000000007E80CC 18         User   innovation2007_windows_amd64.00000000007E80CC
                    000000C00012DFD8 000000000083C821 00000000007E7EA5 8          User   innovation2007_windows_amd64.00000000007E7EA5
                    000000C00012DFE0 0000000000000000 000000000083C821            User   innovation2007_windows_amd64.000000000083C821

@qmuntal is it normal for releasing a mutex to block? Or maybe this isn't actually blocked and we got lucky. @hajimehoshi could you collect another set of stack traces to see if they look similar?

It's unfortunate that we didn't get symbol names for the Go code itself. I tried building #71242 (comment) with Go 1.23.2, but the addresses don't even come close to matching up. @hajimehoshi could you try manually symbolizing some of these? I think go tool addr2line should do the trick. My guess is that the top few frames are runtime.semasleep, runtime.notesleep, runtime.mPark.

From the goroutine stacks, a GC is clearly running. A few goroutines are waiting to help out with GC assists (which is a bit odd). Two of them we can't see:

  Goroutine 20 - User: :0 ??? (0x7ffdbb990424) (thread 31972) [GC mark termination 53437762463500]
        0  0x00007ffdbb990424 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)

  Goroutine 24 - User: :0 ??? (0x7ffdbb9907a4) (thread 24128) [GC mark termination]
        0  0x00007ffdbb9907a4 in ???
            at ?:-1
            error: Only part of a ReadProcessMemory or WriteProcessMemory request was completed.
        (truncated)

These are perhaps stuck in GC mark termination, maybe in here?

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

could you collect another set of stack traces to see if they look similar?

Go program
package main

import (
	"log"
	"os"
	"runtime"
	"runtime/debug"
	"time"
)

func main() {
	for range 5 {
		go func() {
			for {
				_ = make([]byte, 256*1024)
				time.Sleep(time.Millisecond)
			}
		}()
	}

	log.Printf("Version: %s", runtime.Version())
	log.Printf("PID: %d", os.Getpid())
	for {
		time.Sleep(time.Second)
		var gcStats debug.GCStats
		debug.ReadGCStats(&gcStats)
		log.Printf("LastGC: %s, NumGC: %d, PauseTotal: %s", gcStats.LastGC, gcStats.NumGC, gcStats.PauseTotal)
	}
}
Stack traces by x64dbg
Thread ID          Address          To               From             Size       Party  Comment                                      
33152                                                                                   
                   00000072E57FF0A8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E57FF148 00007FFD949F1D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E57FF1B8 00000000003DE029 00007FFD949F1D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                   00000072E57FF328 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E57FF358 00000000003DC4C1 00000000003DC40D 28         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E57FF380 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E57FF3B8 00000000003A1C56 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E57FF3D0 00000000003A2592 00000000003A1C56 560        User   innovation2007_windows_amd64.00000000003A1C56
                   00000072E57FF930 00000000003B3D35 00000000003A2592 18         User   innovation2007_windows_amd64.00000000003A2592
                   00000072E57FF948 000000000038DC96 00000000003B3D35 30         User   innovation2007_windows_amd64.00000000003B3D35
                   00000072E57FF978 0000000000393825 000000000038DC96 18         User   innovation2007_windows_amd64.000000000038DC96
                   00000072E57FF990 000000000038BC87 0000000000393825 68         User   innovation2007_windows_amd64.0000000000393825
                   00000072E57FF9F8 0000000000388407 000000000038BC87 50         User   innovation2007_windows_amd64.000000000038BC87
                   00000072E57FFA48 00000000003DA869 0000000000388407 4D1ACD64E8 User   innovation2007_windows_amd64.0000000000388407
                   000000C0004D5F30 00000000003880CC 00000000003DA869 90         User   innovation2007_windows_amd64.00000000003DA869
                   000000C0004D5FC0 0000000000387EA5 00000000003880CC 18         User   innovation2007_windows_amd64.00000000003880CC
                   000000C0004D5FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004D5FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
26976                                                                                   
                   00000072E49FFAB8 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                   00000072E49FFD98 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                   00000072E49FFDC8 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                   00000072E49FFE48 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
9184 - Main Thread                                                                      
                   00000072E47FF928 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E47FF9C8 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E47FFB38 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E47FFB68 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E47FFB88 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E47FFBC0 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E47FFBD8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E47FFC40 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E47FFC78 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E47FFCA8 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E47FFE20 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E47FFE58 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E47FFEB0 00000000003DA7F3 00000000003AEDCB 4D1BCD8060 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0004D7F10 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0004D7F30 0000000000387FC9 00000000003D572E 90         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0004D7FC0 0000000000387EA5 0000000000387FC9 18         User   innovation2007_windows_amd64.0000000000387FC9
                   000000C0004D7FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004D7FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
7300                                                                                    
                   00000072E4BFF678 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                   00000072E4BFF958 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                   00000072E4BFF988 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                   00000072E4BFFA08 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
34092                                                                                   
                   00000072E55FF078 00007FFDB8D9BEAD 00007FFDBB9907A4 30         System ntdll.NtReleaseMutant+14
                   00000072E55FF0A8 00007FFD949F1EC0 00007FFDB8D9BEAD 70         User   kernelbase.ReleaseMutex+D
                   00000072E55FF118 00000000003DE029 00007FFD949F1EC0 170        User   gameoverlayrenderer64.OverlayHookD3D3+27380
                   00000072E55FF288 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E55FF2B8 00000000003DC4C1 00000000003DC40D 28         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E55FF2E0 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E55FF318 00000000003A1C56 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E55FF330 00000000003A2592 00000000003A1C56 560        User   innovation2007_windows_amd64.00000000003A1C56
                   00000072E55FF890 00000000003B3D35 00000000003A2592 18         User   innovation2007_windows_amd64.00000000003A2592
                   00000072E55FF8A8 000000000038DC96 00000000003B3D35 30         User   innovation2007_windows_amd64.00000000003B3D35
                   00000072E55FF8D8 0000000000393825 000000000038DC96 18         User   innovation2007_windows_amd64.000000000038DC96
                   00000072E55FF8F0 000000000038BC87 0000000000393825 68         User   innovation2007_windows_amd64.0000000000393825
                   00000072E55FF958 0000000000388407 000000000038BC87 50         User   innovation2007_windows_amd64.000000000038BC87
                   00000072E55FF9A8 00000000003DA869 0000000000388407 4D1AEDC588 User   innovation2007_windows_amd64.0000000000388407
                   000000C0004DBF30 00000000003880CC 00000000003DA869 90         User   innovation2007_windows_amd64.00000000003DA869
                   000000C0004DBFC0 0000000000387EA5 00000000003880CC 18         User   innovation2007_windows_amd64.00000000003880CC
                   000000C0004DBFD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004DBFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
29616                                                                                   
                   00000072E4FFEC88 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E4FFED28 00007FFD949F1D4F 00007FFDB8D99CEE 70         User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E4FFED98 00000000003DE029 00007FFD949F1D4F 170        User   gameoverlayrenderer64.OverlayHookD3D3+2720F
                   00000072E4FFEF08 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E4FFEF38 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E4FFEF58 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E4FFEF90 00000000003A1C56 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E4FFEFA8 00000000003A2592 00000000003A1C56 560        User   innovation2007_windows_amd64.00000000003A1C56
                   00000072E4FFF508 00000000003B3D35 00000000003A2592 18         User   innovation2007_windows_amd64.00000000003A2592
                   00000072E4FFF520 00000000003B3A36 00000000003B3D35 60         User   innovation2007_windows_amd64.00000000003B3D35
                   00000072E4FFF580 00000000003B379A 00000000003B3A36 80         User   innovation2007_windows_amd64.00000000003B3A36
                   00000072E4FFF600 00000000003AA73D 00000000003B379A 28         User   innovation2007_windows_amd64.00000000003B379A
                   00000072E4FFF628 00000000003AA68A 00000000003AA73D 28         User   innovation2007_windows_amd64.00000000003AA73D
                   00000072E4FFF650 00000000003DA765 00000000003AA68A 8          User   innovation2007_windows_amd64.00000000003AA68A
                   00000072E4FFF658 00000000003DE477 00000000003DA765 8          User   innovation2007_windows_amd64.00000000003DA765
                   00000072E4FFF660 0000000000000000 00000000003DE477            User   innovation2007_windows_amd64.00000000003DE477
28532                                                                                   
                   00000072E51FF6F8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E51FF798 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E51FF908 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E51FF938 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E51FF958 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E51FF990 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E51FF9A8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E51FFA10 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E51FFA48 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E51FFA78 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E51FFBF0 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E51FFC28 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E51FFC80 00000000003DA7F3 00000000003AEDCB 4D1AEC0188 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0000BFE08 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0000BFE28 000000000038AF67 00000000003D572E 38         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0000BFE60 000000000038A88A 000000000038AF67 60         User   innovation2007_windows_amd64.000000000038AF67
                   000000C0000BFEC0 000000000037CDF4 000000000038A88A 28         User   innovation2007_windows_amd64.000000000038A88A
                   000000C0000BFEE8 00000000003D14DE 000000000037CDF4 A0         User   innovation2007_windows_amd64.000000000037CDF4
                   000000C0000BFF88 00000000003D6609 00000000003D14DE 28         User   innovation2007_windows_amd64.00000000003D14DE
                   000000C0000BFFB0 00000000004184E5 00000000003D6609 28         User   innovation2007_windows_amd64.00000000003D6609
                   000000C0000BFFD8 00000000003DC821 00000000004184E5 8          User   innovation2007_windows_amd64.00000000004184E5
                   000000C0000BFFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
35804                                                                                   
                   00000072E53FF7E8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E53FF888 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E53FF9F8 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E53FFA28 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E53FFA48 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E53FFA80 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E53FFA98 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E53FFB00 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E53FFB38 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E53FFB68 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E53FFCE0 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E53FFD18 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E53FFD70 00000000003DA7F3 00000000003AEDCB 4D1B0DA1A0 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0004D9F10 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0004D9F30 0000000000387FC9 00000000003D572E 90         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0004D9FC0 0000000000387EA5 0000000000387FC9 18         User   innovation2007_windows_amd64.0000000000387FC9
                   000000C0004D9FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004D9FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
34976                                                                                   
                   00000072E4DFF768 00007FFDBB92586E 00007FFDBB993FF4 2E0        System ntdll.NtWaitForWorkViaWorkerFactory+14
                   00000072E4DFFA48 00007FFDBA13259D 00007FFDBB92586E 30         System ntdll.RtlClearThreadWorkOnBehalfTicket+35E
                   00000072E4DFFA78 00007FFDBB94AF38 00007FFDBA13259D 80         System kernel32.BaseThreadInitThunk+1D
                   00000072E4DFFAF8 0000000000000000 00007FFDBB94AF38            User   ntdll.RtlUserThreadStart+28
17840                                                                                   
                   00000072E59FF410 00000000003A5B1E 00000000003DC3C6 78         User   innovation2007_windows_amd64.00000000003DC3C6
                   00000072E59FF488 000000000038A0A5 00000000003A5B1E 50         User   innovation2007_windows_amd64.00000000003A5B1E
                   00000072E59FF4D8 0000000000389D65 000000000038A0A5 A8         User   innovation2007_windows_amd64.000000000038A0A5
                   00000072E59FF580 000000000038BF14 0000000000389D65 68         User   innovation2007_windows_amd64.0000000000389D65
                   00000072E59FF5E8 000000000038843E 000000000038BF14 50         User   innovation2007_windows_amd64.000000000038BF14
                   00000072E59FF638 00000000003DA869 000000000038843E 4D1AACA8F8 User   innovation2007_windows_amd64.000000000038843E
                   000000C0004C9F30 00000000003880CC 00000000003DA869 90         User   innovation2007_windows_amd64.00000000003DA869
                   000000C0004C9FC0 0000000000387EA5 00000000003880CC 18         User   innovation2007_windows_amd64.00000000003880CC
                   000000C0004C9FD8 00000000003DC821 0000000000387EA5 8          User   innovation2007_windows_amd64.0000000000387EA5
                   000000C0004C9FE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
35732                                                                                   
                   00000072E5BFF3B8 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E5BFF458 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E5BFF5C8 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E5BFF5F8 00000000003DC4C1 00000000003DC40D 28         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E5BFF620 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E5BFF658 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E5BFF670 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E5BFF6D8 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E5BFF710 00000000003ABD51 000000000037B9D2 20         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E5BFF730 00000000003AA73D 00000000003ABD51 28         User   innovation2007_windows_amd64.00000000003ABD51
                   00000072E5BFF758 00000000003AA68A 00000000003AA73D 28         User   innovation2007_windows_amd64.00000000003AA73D
                   00000072E5BFF780 00000000003DA765 00000000003AA68A 8          User   innovation2007_windows_amd64.00000000003AA68A
                   00000072E5BFF788 00000000003DE477 00000000003DA765 8          User   innovation2007_windows_amd64.00000000003DA765
                   00000072E5BFF790 0000000000000000 00000000003DE477            User   innovation2007_windows_amd64.00000000003DE477
11764                                                                                   
                   00000072E5DFF608 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E5DFF6A8 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E5DFF818 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E5DFF848 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E5DFF868 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E5DFF8A0 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E5DFF8B8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E5DFF920 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E5DFF958 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E5DFF988 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E5DFFB00 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E5DFFB38 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E5DFFB90 00000000003DA7F3 00000000003AEDCB 4D1A27C150 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C00007BCE0 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C00007BD00 000000000038AF67 00000000003D572E 38         User   innovation2007_windows_amd64.00000000003D572E
                   000000C00007BD38 000000000038A88A 000000000038AF67 60         User   innovation2007_windows_amd64.000000000038AF67
                   000000C00007BD98 000000000037CDF4 000000000038A88A 28         User   innovation2007_windows_amd64.000000000038A88A
                   000000C00007BDC0 00000000003D14DE 000000000037CDF4 A0         User   innovation2007_windows_amd64.000000000037CDF4
                   000000C00007BE60 000000000037CEE5 00000000003D14DE 28         User   innovation2007_windows_amd64.00000000003D14DE
                   000000C00007BE88 000000000041827B 000000000037CEE5 C0         User   innovation2007_windows_amd64.000000000037CEE5
                   000000C00007BF48 00000000003A737D 000000000041827B 90         User   innovation2007_windows_amd64.000000000041827B
                   000000C00007BFD8 00000000003DC821 00000000003A737D 8          User   innovation2007_windows_amd64.00000000003A737D
                   000000C00007BFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821
27512                                                                                   
                   00000072E5FFF538 00007FFDB8D99CEE 00007FFDBB990424 A0         System ntdll.NtWaitForSingleObject+14
                   00000072E5FFF5D8 00000000003DE029 00007FFDB8D99CEE 170        User   kernelbase.WaitForSingleObjectEx+8E
                   00000072E5FFF748 00000000003DC40D 00000000003DE029 30         User   innovation2007_windows_amd64.00000000003DE029
                   00000072E5FFF778 00000000003DC4C1 00000000003DC40D 20         User   innovation2007_windows_amd64.00000000003DC40D
                   00000072E5FFF798 00000000003A1B9B 00000000003DC4C1 38         User   innovation2007_windows_amd64.00000000003DC4C1
                   00000072E5FFF7D0 00000000003A1CD6 00000000003A1B9B 18         User   innovation2007_windows_amd64.00000000003A1B9B
                   00000072E5FFF7E8 00000000003A0DC5 00000000003A1CD6 68         User   innovation2007_windows_amd64.00000000003A1CD6
                   00000072E5FFF850 000000000037B9D2 00000000003A0DC5 38         User   innovation2007_windows_amd64.00000000003A0DC5
                   00000072E5FFF888 00000000003ABE6C 000000000037B9D2 30         User   innovation2007_windows_amd64.000000000037B9D2
                   00000072E5FFF8B8 00000000003AD8DC 00000000003ABE6C 178        User   innovation2007_windows_amd64.00000000003ABE6C
                   00000072E5FFFA30 00000000003AE9B1 00000000003AD8DC 38         User   innovation2007_windows_amd64.00000000003AD8DC
                   00000072E5FFFA68 00000000003AEDCB 00000000003AE9B1 58         User   innovation2007_windows_amd64.00000000003AE9B1
                   00000072E5FFFAC0 00000000003DA7F3 00000000003AEDCB 4D1A0BC348 User   innovation2007_windows_amd64.00000000003AEDCB
                   000000C0000BBE08 00000000003D572E 00000000003DA7F3 20         User   innovation2007_windows_amd64.00000000003DA7F3
                   000000C0000BBE28 000000000038AF67 00000000003D572E 38         User   innovation2007_windows_amd64.00000000003D572E
                   000000C0000BBE60 000000000038A88A 000000000038AF67 60         User   innovation2007_windows_amd64.000000000038AF67
                   000000C0000BBEC0 000000000037CDF4 000000000038A88A 28         User   innovation2007_windows_amd64.000000000038A88A
                   000000C0000BBEE8 00000000003D14DE 000000000037CDF4 A0         User   innovation2007_windows_amd64.000000000037CDF4
                   000000C0000BBF88 00000000003D6609 00000000003D14DE 28         User   innovation2007_windows_amd64.00000000003D14DE
                   000000C0000BBFB0 00000000004184E5 00000000003D6609 28         User   innovation2007_windows_amd64.00000000003D6609
                   000000C0000BBFD8 00000000003DC821 00000000004184E5 8          User   innovation2007_windows_amd64.00000000004184E5
                   000000C0000BBFE0 0000000000000000 00000000003DC821            User   innovation2007_windows_amd64.00000000003DC821

I'll try addr2line soon.

@hajimehoshi
Copy link
Member Author

could you try manually symbolizing some of these? I think go tool addr2line should do the trick. My guess is that the top few frames are runtime.semasleep, runtime.notesleep, runtime.mPark.

Unfortunately I failed to resolve the addresses:

Get-Content .\stacktraces.txt | go tool addr2line .\innovation2007_windows_amd64.exe

The result was

?
?:0
?
?:0
?
?:0
?
?:0
?
?:0
...

Am I missing something?

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

By the way, in order to reproduce this, it is possible to do this for free:

  1. Install Steam on Windows
  2. Install a free game like https://store.steampowered.com/app/1710390 (This is my game for demonstration so feel free to do experiments)
  3. Open the location of the exe file
  4. Replace the exe file with the compiled Go program
  5. Run the game from the Steam client.

@prattmic
Copy link
Member

Thanks, looks like that run is also sitting in ReleaseMutex.

Unfortunately I failed to resolve the addresses: ... Am I missing something?

addr2line requires that each line contains only an address, so you'll need to edit the stack trace file to remove everything else.

@hajimehoshi
Copy link
Member Author

addr2line requires that each line contains only an address, so you'll need to edit the stack trace file to remove everything else.

Hmm, addr2line still didn't resolve the file like this:

00000000003DE029
00000000003DC40D
00000000003DC4C1
00000000003A1B9B

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

I am not sure this would be helpful, but I objdumped the binary and searched the binaries in the suspicous stack traces. I realized the addresses are 'shifted'.

Stack traces with ReleaseMutex (I newly dumped this):

                                 000000BE991FF218  00007FFDB8D9BEAD  00007FFDBB9907A4  30                System    ntdll.NtReleaseMutant+14
                                 000000BE991FF248  00007FFDAFEE1EC0  00007FFDB8D9BEAD  70                User      kernelbase.ReleaseMutex+D
                                 000000BE991FF2B8  00000000007FE029  00007FFDAFEE1EC0  170               User      gameoverlayrenderer64.OverlayHookD3D3+27380
                                 000000BE991FF428  00000000007FC40D  00000000007FE029  30                User      innovation2007_windows_amd64.00000000007FE029
                                 000000BE991FF458  00000000007FC4C1  00000000007FC40D  28                User      innovation2007_windows_amd64.00000000007FC40D
                                 000000BE991FF480  00000000007C1B9B  00000000007FC4C1  38                User      innovation2007_windows_amd64.00000000007FC4C1
                                 000000BE991FF4B8  00000000007C1C56  00000000007C1B9B  18                User      innovation2007_windows_amd64.00000000007C1B9B
                                 000000BE991FF4D0  00000000007C2592  00000000007C1C56  560               User      innovation2007_windows_amd64.00000000007C1C56
                                 000000BE991FFA30  00000000007D3D35  00000000007C2592  18                User      innovation2007_windows_amd64.00000000007C2592
                                 000000BE991FFA48  00000000007ADC96  00000000007D3D35  30                User      innovation2007_windows_amd64.00000000007D3D35
                                 000000BE991FFA78  00000000007B3825  00000000007ADC96  18                User      innovation2007_windows_amd64.00000000007ADC96
                                 000000BE991FFA90  00000000007ABC87  00000000007B3825  68                User      innovation2007_windows_amd64.00000000007B3825
                                 000000BE991FFAF8  00000000007A8407  00000000007ABC87  50                User      innovation2007_windows_amd64.00000000007ABC87
                                 000000BE991FFB48  00000000007FA869  00000000007A8407  166FD83E8         User      innovation2007_windows_amd64.00000000007A8407
                                 000000C0001D7F30  00000000007A80CC  00000000007FA869  90                User      innovation2007_windows_amd64.00000000007FA869
                                 000000C0001D7FC0  00000000007A7EA5  00000000007A80CC  18                User      innovation2007_windows_amd64.00000000007A80CC
                                 000000C0001D7FD8  00000000007FC821  00000000007A7EA5  8                 User      innovation2007_windows_amd64.00000000007A7EA5
                                 000000C0001D7FE0  0000000000000000  00000000007FC821                    User      innovation2007_windows_amd64.00000000007FC821

00000000007FE029:

TEXT runtime.asmstdcall.abi0(SB) C:/Program Files/Go/src/runtime/sys_windows_amd64.s
  sys_windows_amd64.s:20	0x46dfa0		55			PUSHQ BP			
  sys_windows_amd64.s:20	0x46dfa1		4889e5			MOVQ SP, BP			
  sys_windows_amd64.s:20	0x46dfa4		4883ec10		SUBQ $0x10, SP			
  sys_windows_amd64.s:21	0x46dfa8		4889e0			MOVQ SP, AX			
  sys_windows_amd64.s:22	0x46dfab		4883e4f0		ANDQ $-0x10, SP			
  sys_windows_amd64.s:23	0x46dfaf		4889442408		MOVQ AX, 0x8(SP)		
  sys_windows_amd64.s:24	0x46dfb4		48890c24		MOVQ CX, 0(SP)			
  sys_windows_amd64.s:26	0x46dfb8		488b01			MOVQ 0(CX), AX			
  sys_windows_amd64.s:27	0x46dfbb		488b7110		MOVQ 0x10(CX), SI		
  sys_windows_amd64.s:28	0x46dfbf		488b4908		MOVQ 0x8(CX), CX		
  sys_windows_amd64.s:31	0x46dfc3		65488b3c2530000000	MOVQ GS:0x30, DI		
  sys_windows_amd64.s:32	0x46dfcc		c7476800000000		MOVL $0x0, 0x68(DI)		
  sys_windows_amd64.s:34	0x46dfd3		4881ec50010000		SUBQ $0x150, SP			
  sys_windows_amd64.s:37	0x46dfda		83f900			CMPL CX, $0x0			
  sys_windows_amd64.s:37	0x46dfdd		7448			JE 0x46e027			
  sys_windows_amd64.s:38	0x46dfdf		83f901			CMPL CX, $0x1			
  sys_windows_amd64.s:38	0x46dfe2		743b			JE 0x46e01f			
  sys_windows_amd64.s:39	0x46dfe4		83f902			CMPL CX, $0x2			
  sys_windows_amd64.s:39	0x46dfe7		742d			JE 0x46e016			
  sys_windows_amd64.s:40	0x46dfe9		83f903			CMPL CX, $0x3			
  sys_windows_amd64.s:40	0x46dfec		741f			JE 0x46e00d			
  sys_windows_amd64.s:41	0x46dfee		83f904			CMPL CX, $0x4			
  sys_windows_amd64.s:41	0x46dff1		7411			JE 0x46e004			
  sys_windows_amd64.s:44	0x46dff3		83f92a			CMPL CX, $0x2a			
  sys_windows_amd64.s:45	0x46dff6		7e02			JLE 0x46dffa			
  sys_windows_amd64.s:46	0x46dff8		cd03			INT $0x3			
  sys_windows_amd64.s:49	0x46dffa		4889e7			MOVQ SP, DI			
  sys_windows_amd64.s:50	0x46dffd		fc			CLD				
  sys_windows_amd64.s:51	0x46dffe		f348a5			REP; MOVSQ DS:0(SI), ES:0(DI)	
  sys_windows_amd64.s:52	0x46e001		4889e6			MOVQ SP, SI			
  sys_windows_amd64.s:60	0x46e004		4c8b4e18		MOVQ 0x18(SI), R9		
  sys_windows_amd64.s:61	0x46e008		66490f6ed9		MOVQ R9, X3			
  sys_windows_amd64.s:63	0x46e00d		4c8b4610		MOVQ 0x10(SI), R8		
  sys_windows_amd64.s:64	0x46e011		66490f6ed0		MOVQ R8, X2			
  sys_windows_amd64.s:66	0x46e016		488b5608		MOVQ 0x8(SI), DX		
  sys_windows_amd64.s:67	0x46e01a		66480f6eca		MOVQ DX, X1			
  sys_windows_amd64.s:69	0x46e01f		488b0e			MOVQ 0(SI), CX			
  sys_windows_amd64.s:70	0x46e022		66480f6ec1		MOVQ CX, X0			
  sys_windows_amd64.s:74	0x46e027		ffd0			CALL AX				
  sys_windows_amd64.s:76	0x46e029		4881c450010000		ADDQ $0x150, SP			<-- Start Here
  sys_windows_amd64.s:79	0x46e030		488b0c24		MOVQ 0(SP), CX			
  sys_windows_amd64.s:80	0x46e034		488b642408		MOVQ 0x8(SP), SP		
  sys_windows_amd64.s:81	0x46e039		48894118		MOVQ AX, 0x18(CX)		
  sys_windows_amd64.s:85	0x46e03d		660fd64120		MOVQ X0, 0x20(CX)		
  sys_windows_amd64.s:88	0x46e042		65488b3c2530000000	MOVQ GS:0x30, DI		
  sys_windows_amd64.s:89	0x46e04b		8b4768			MOVL 0x68(DI), AX		
  sys_windows_amd64.s:90	0x46e04e		48894128		MOVQ AX, 0x28(CX)		
  sys_windows_amd64.s:92	0x46e052		4883c410		ADDQ $0x10, SP			
  sys_windows_amd64.s:92	0x46e056		5d			POPQ BP				
  sys_windows_amd64.s:92	0x46e057		c3			RET	<-- End Here

00000000007FC40D:

TEXT runtime.asmcgocall_landingpad.abi0(SB) C:/Program Files/Go/src/runtime/asm_amd64.s
  asm_amd64.s:858	0x46c400		55			PUSHQ BP		
  asm_amd64.s:858	0x46c401		4889e5			MOVQ SP, BP		
  asm_amd64.s:862	0x46c404		4883ec20		SUBQ $0x20, SP		
  asm_amd64.s:871	0x46c408		4889d9			MOVQ BX, CX		
  asm_amd64.s:872	0x46c40b		ffd0			CALL AX			
  asm_amd64.s:875	0x46c40d		90			NOPL			<-- Start Here
  asm_amd64.s:876	0x46c40e		4883c420		ADDQ $0x20, SP		
  asm_amd64.s:877	0x46c412		5d			POPQ BP			
  asm_amd64.s:877	0x46c413		c3			RET			<-- End Here
  asm_amd64.s:880	0x46c414		4889df			MOVQ BX, DI		
  asm_amd64.s:881	0x46c417		ffe0			JMP AX			
  :-1			0x46c419		cc			INT $0x3		
  :-1			0x46c41a		cc			INT $0x3		
  :-1			0x46c41b		cc			INT $0x3		
  :-1			0x46c41c		cc			INT $0x3		
  :-1			0x46c41d		cc			INT $0x3		
  :-1			0x46c41e		cc			INT $0x3		
  :-1			0x46c41f		cc			INT $0x3

00000000007FC4C1:

TEXT runtime.asmcgocall.abi0(SB) C:/Program Files/Go/src/runtime/asm_amd64.s
  asm_amd64.s:887	0x46c420		55			PUSHQ BP					
  asm_amd64.s:887	0x46c421		4889e5			MOVQ SP, BP					
  asm_amd64.s:888	0x46c424		488b442410		MOVQ 0x10(SP), AX				
  asm_amd64.s:889	0x46c429		488b5c2418		MOVQ 0x18(SP), BX				
  asm_amd64.s:891	0x46c42e		4889e2			MOVQ SP, DX					
  asm_amd64.s:897	0x46c431		488b0d38c81500		MOVQ runtime.tls_g(SB), CX			
  asm_amd64.s:897	0x46c438		65488b09		MOVQ GS:0(CX), CX				
  asm_amd64.s:898	0x46c43c		488b39			MOVQ 0(CX), DI					
  asm_amd64.s:899	0x46c43f		4883ff00		CMPQ DI, $0x0					
  asm_amd64.s:900	0x46c443		7462			JE 0x46c4a7					
  asm_amd64.s:901	0x46c445		4c8b4730		MOVQ 0x30(DI), R8				
  asm_amd64.s:902	0x46c449		498b7050		MOVQ 0x50(R8), SI				
  asm_amd64.s:903	0x46c44d		4839f7			CMPQ DI, SI					
  asm_amd64.s:904	0x46c450		7455			JE 0x46c4a7					
  asm_amd64.s:905	0x46c452		498b30			MOVQ 0(R8), SI					
  asm_amd64.s:906	0x46c455		4839f7			CMPQ DI, SI					
  asm_amd64.s:907	0x46c458		744d			JE 0x46c4a7					
  asm_amd64.s:912	0x46c45a		e8c1d4ffff		CALL gosave_systemstack_switch(SB)		
  asm_amd64.s:913	0x46c45f		488931			MOVQ SI, 0(CX)					
  asm_amd64.s:914	0x46c462		488b6638		MOVQ 0x38(SI), SP				
  asm_amd64.s:917	0x46c466		4883ec10		SUBQ $0x10, SP					
  asm_amd64.s:918	0x46c46a		4883e4f0		ANDQ $-0x10, SP					
  asm_amd64.s:919	0x46c46e		48897c2408		MOVQ DI, 0x8(SP)				
  asm_amd64.s:920	0x46c473		488b7f08		MOVQ 0x8(DI), DI				
  asm_amd64.s:921	0x46c477		4829d7			SUBQ DX, DI					
  asm_amd64.s:922	0x46c47a		48893c24		MOVQ DI, 0(SP)					
  asm_amd64.s:923	0x46c47e		e87dffffff		CALL runtime.asmcgocall_landingpad.abi0(SB)	
  asm_amd64.s:926	0x46c483		488b0de6c71500		MOVQ runtime.tls_g(SB), CX			
  asm_amd64.s:926	0x46c48a		65488b09		MOVQ GS:0(CX), CX				
  asm_amd64.s:927	0x46c48e		488b7c2408		MOVQ 0x8(SP), DI				
  asm_amd64.s:928	0x46c493		488b7708		MOVQ 0x8(DI), SI				
  asm_amd64.s:929	0x46c497		482b3424		SUBQ 0(SP), SI					
  asm_amd64.s:930	0x46c49b		488939			MOVQ DI, 0(CX)					
  asm_amd64.s:931	0x46c49e		4889f4			MOVQ SI, SP					
  asm_amd64.s:933	0x46c4a1		89442420		MOVL AX, 0x20(SP)				
  asm_amd64.s:934	0x46c4a5		5d			POPQ BP						
  asm_amd64.s:934	0x46c4a6		c3			RET						
  asm_amd64.s:947	0x46c4a7		4883ec10		SUBQ $0x10, SP					
  asm_amd64.s:948	0x46c4ab		4883e4f0		ANDQ $-0x10, SP					
  asm_amd64.s:949	0x46c4af		48c744240800000000	MOVQ $0x0, 0x8(SP)				
  asm_amd64.s:950	0x46c4b8		48891424		MOVQ DX, 0(SP)					
  asm_amd64.s:951	0x46c4bc		e83fffffff		CALL runtime.asmcgocall_landingpad.abi0(SB)	
  asm_amd64.s:952	0x46c4c1		488b3424		MOVQ 0(SP), SI					<-- Start Here
  asm_amd64.s:953	0x46c4c5		4889f4			MOVQ SI, SP					
  asm_amd64.s:954	0x46c4c8		89442420		MOVL AX, 0x20(SP)				
  asm_amd64.s:955	0x46c4cc		5d			POPQ BP						
  asm_amd64.s:955	0x46c4cd		c3			RET						<-- End Here
  :-1			0x46c4ce		cc			INT $0x3					
  :-1			0x46c4cf		cc			INT $0x3					
  :-1			0x46c4d0		cc			INT $0x3					
  :-1			0x46c4d1		cc			INT $0x3					
  :-1			0x46c4d2		cc			INT $0x3					
  :-1			0x46c4d3		cc			INT $0x3					
  :-1			0x46c4d4		cc			INT $0x3					
  :-1			0x46c4d5		cc			INT $0x3					
  :-1			0x46c4d6		cc			INT $0x3					
  :-1			0x46c4d7		cc			INT $0x3					
  :-1			0x46c4d8		cc			INT $0x3					
  :-1			0x46c4d9		cc			INT $0x3					
  :-1			0x46c4da		cc			INT $0x3					
  :-1			0x46c4db		cc			INT $0x3					
  :-1			0x46c4dc		cc			INT $0x3					
  :-1			0x46c4dd		cc			INT $0x3					
  :-1			0x46c4de		cc			INT $0x3					
  :-1			0x46c4df		cc			INT $0x3					

00000000007C1B9B

TEXT runtime.stdcall(SB) C:/Program Files/Go/src/runtime/os_windows.go
  os_windows.go:956	0x431b20		55			PUSHQ BP				
  os_windows.go:956	0x431b21		4889e5			MOVQ SP, BP				
  os_windows.go:956	0x431b24		4883ec28		SUBQ $0x28, SP				
  os_windows.go:957	0x431b28		498b4e30		MOVQ 0x30(R14), CX			
  os_windows.go:959	0x431b2c		488981e0020000		MOVQ AX, 0x2e0(CX)			
  os_windows.go:961	0x431b33		83b9e000000000		CMPL 0xe0(CX), $0x0			
  os_windows.go:961	0x431b3a		7438			JE 0x431b74				
  os_windows.go:961	0x431b3c		4883b91803000000	CMPQ 0x318(CX), $0x0			
  os_windows.go:961	0x431b44		752a			JNE 0x431b70				
  runtime2.go:269	0x431b46		4c89f0			MOVQ R14, AX				
  os_windows.go:963	0x431b49		90			NOPL					
  runtime2.go:269	0x431b4a		48898120030000		MOVQ AX, 0x320(CX)			
  os_windows.go:964	0x431b51		488b442430		MOVQ 0x30(SP), AX			
  os_windows.go:964	0x431b56		48898110030000		MOVQ AX, 0x310(CX)			
  os_windows.go:967	0x431b5d		488d442438		LEAQ 0x38(SP), AX			
  os_windows.go:967	0x431b62		48898118030000		MOVQ AX, 0x318(CX)			
  os_windows.go:967	0x431b69		b801000000		MOVL $0x1, AX				
  os_windows.go:968	0x431b6e		eb06			JMP 0x431b76				
  os_windows.go:968	0x431b70		31c0			XORL AX, AX				
  os_windows.go:961	0x431b72		eb02			JMP 0x431b76				
  os_windows.go:961	0x431b74		31c0			XORL AX, AX				
  os_windows.go:958	0x431b76		48894c2420		MOVQ CX, 0x20(SP)			
  os_windows.go:971	0x431b7b		8844241f		MOVB AL, 0x1f(SP)			
  os_windows.go:970	0x431b7f		488b15aa1b1500		MOVQ runtime.asmstdcallAddr(SB), DX	
  os_windows.go:970	0x431b86		48891424		MOVQ DX, 0(SP)				
  os_windows.go:970	0x431b8a		488d91e0020000		LEAQ 0x2e0(CX), DX			
  os_windows.go:970	0x431b91		4889542408		MOVQ DX, 0x8(SP)			
  os_windows.go:970	0x431b96		e885a80300		CALL runtime.asmcgocall.abi0(SB)	
  os_windows.go:970	0x431b9b		450f57ff		XORPS X15, X15				<-- Start Here
  os_windows.go:970	0x431b9f		4c8b35ca701900		MOVQ runtime.tls_g(SB), R14		
  os_windows.go:970	0x431ba6		654d8b36		MOVQ GS:0(R14), R14			
  os_windows.go:970	0x431baa		4d8b36			MOVQ 0(R14), R14			
  os_windows.go:971	0x431bad		0fb644241f		MOVZX 0x1f(SP), AX			
  os_windows.go:971	0x431bb2		84c0			TESTL AL, AL				
  os_windows.go:971	0x431bb4		7412			JE 0x431bc8				
  os_windows.go:972	0x431bb6		488b4c2420		MOVQ 0x20(SP), CX			
  os_windows.go:972	0x431bbb		48c7811803000000000000	MOVQ $0x0, 0x318(CX)			
  os_windows.go:972	0x431bc6		eb05			JMP 0x431bcd				
  os_windows.go:974	0x431bc8		488b4c2420		MOVQ 0x20(SP), CX			
  os_windows.go:974	0x431bcd		488b81f8020000		MOVQ 0x2f8(CX), AX			
  os_windows.go:974	0x431bd4		4883c428		ADDQ $0x28, SP				
  os_windows.go:974	0x431bd8		5d			POPQ BP					
  os_windows.go:974	0x431bd9		c3			RET					<-- End Here
  :-1			0x431bda		cc			INT $0x3				
  :-1			0x431bdb		cc			INT $0x3				
  :-1			0x431bdc		cc			INT $0x3				
  :-1			0x431bdd		cc			INT $0x3				
  :-1			0x431bde		cc			INT $0x3				
  :-1			0x431bdf		cc			INT $0x3

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 14, 2025

Here is the resolved symbols in the stack trace below gameoverlayrenderer64.OverlayHookD3D3+27380:

runtime.asmstdcall
C:/Program Files/Go/src/runtime/sys_windows_amd64.s:76
runtime.asmcgocall_landingpad
C:/Program Files/Go/src/runtime/asm_amd64.s:875
runtime.asmcgocall
C:/Program Files/Go/src/runtime/asm_amd64.s:952
runtime.stdcall
C:/Program Files/Go/src/runtime/os_windows.go:970
runtime.stdcall1
C:/Program Files/Go/src/runtime/os_windows.go:991
runtime.preemptM
C:/Program Files/Go/src/runtime/os_windows.go:1351
runtime.preemptone
C:/Program Files/Go/src/runtime/proc.go:6328
runtime.(*gcControllerState).enlistWorker
C:/Program Files/Go/src/runtime/mgcpacer.go:723
runtime.(*gcWork).balance
C:/Program Files/Go/src/runtime/mgcwork.go:306
runtime.gcDrain
C:/Program Files/Go/src/runtime/mgcwork.go:235
runtime.gcBgMarkWorker.func2
C:/Program Files/Go/src/runtime/mgc.go:1459
runtime.systemstack
C:/Program Files/Go/src/runtime/asm_amd64.s:517
runtime.gcBgMarkWorker
C:/Program Files/Go/src/runtime/mgc.go:1423
runtime.gcBgMarkStartWorkers.gowrap1
C:/Program Files/Go/src/runtime/mgc.go:1279
runtime.goexit
C:/Program Files/Go/src/runtime/asm_amd64.s:1701

@prattmic
Copy link
Member

Apologies for the bad instructions, I don't use addr2line much.

That stack is really interesting. On Windows, asynchronous preemption uses the Windows thread suspend/resume APIs. It seems that this thread is getting stuck in ResumeThread. Maybe the Steam overlay messes with this API somehow?

@mknyszek
Copy link
Contributor

@hajimehoshi Can you reproduce with GODEBUG=asyncpreemptoff=1?

@hajimehoshi
Copy link
Member Author

@hajimehoshi Can you reproduce with GODEBUG=asyncpreemptoff=1?

I couldn't reproduce this with GODEBUG=asyncpreemptoff=1. I tried 10 times and watched each trial reached 10,000 GCs without any issues.

@hajimehoshi
Copy link
Member Author

hajimehoshi commented Jan 15, 2025

By the way, is there a compile option to specify asyncpreemptoff=1 by default? //go:debug didn't recognize asyncpreemptoff=1 unfortunately. For Steam games, it is not feasible to ask users to set an environment variable GODEBUG for this, so I'd like to specify this at the compile time. The only way to do this is using -overlay to rewrite os_windows.go, but is my understanding correct?

Of course, it would be the best that the runtime is fixed to suppress this issue.

There seem multiple situations where asyncpreemptoff=1 is a workaround.

EDIT: -ldflags="-X=runtime.godebugDefault=asyncpreemptoff=1" seemed to work as intended.

@mknyszek
Copy link
Contributor

//go:debug didn't recognize asyncpreemptoff=1 unfortunately.

I think we should fix that, though I'm glad you found another workaround with -ldflags. Maybe that makes more sense long-term? This issue seems to be related to the specific DLL injection (?) that Steam does (gameoverlayrenderer64.OverlayHookD3D3+27380 is pretty fishy). It would be unfortunate to disable async preemption for all users of a given package, since it does provide tangible latency benefits.

@hajimehoshi
Copy link
Member Author

I think we should fix that

Yeah, so would it be possible for //go:debug to accept asyncpreemptoff=1? I'm happy to file this if needed. This would be much better and less hacky than -ldflags. Also, I think this would be very helpful for various situations where this settings is required.

It would be unfortunate to disable async preemption for all users of a given package, since it does provide tangible latency benefits.

I mean, I want to disable async preemption when I build my games for Steam Windows. So I would specify asyncpreemptoff=1 for a main package (with a build tag). I don't intend to disable it for all the users of my package.

@prattmic
Copy link
Member

Yeah, so would it be possible for //go:debug to accept asyncpreemptoff=1? I'm happy to file this if needed.

I think this is reasonable, please do file an issue.

//go:debug only has an effect in main packages anyways, so applying to all users of a package isn't a problem.

That said, I don't think this is the final resolution. Steam seems to be messing with our process in a way that breaks certain Windows API calls. I think the next step is to report this bug to Valve, as current evidence points to a bug in whatever Steam is doing.

@hajimehoshi
Copy link
Member Author

I think this is reasonable, please do file an issue.

Thanks! #71283

That said, I don't think this is the final resolution. Steam seems to be messing with our process in a way that breaks certain Windows API calls. I think the next step is to report this bug to Valve, as current evidence points to a bug in whatever Steam is doing.

I agree. I've already reported the current situation https://steamcommunity.com/discussions/forum/0/595138100650327297/.

I'll try to make a simple C program that invokes SuspendThread and ResumeThread and tries to cause the freeze when I have time.

I appreciate all of your quick responses!

@mknyszek
Copy link
Contributor

mknyszek commented Jan 15, 2025

Apologies, the //go:debug stuff is new to me. There's more than just GODEBUG setting that we should allow; perhaps we should file a new issue capturing all of them. EDIT: I'm clearly behind, #71283 already exists. 😅 Thanks!

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Labels
BugReport Issues describing a possible bug in the Go implementation. compiler/runtime Issues related to the Go compiler and/or runtime. NeedsInvestigation Someone must examine and confirm this is a valid issue and not a duplicate of an existing one. OS-Windows
Projects
Development

No branches or pull requests

6 participants