runtime: (on linux/mips64le) bad pointer in frame main.invoke at 0xc000091da8: 0x7

[tianon]

Over in https://github.com/docker-library/golang we maintain builds of a weekly snapshot of tip across the various architectures supported by the program. Sometime in between our snapshot from two weeks ago (6a08e80) and last week (861c90c), we started seeing build failures on linux/mips64le (6a08e80...861c90c; build succeeds on 20250831 and fails on 20250906; I have not been able to reproduce this while cross-compiling, only natively):

+ export GOCACHE=/tmp/gocache GOHOSTOS=linux GOHOSTARCH=mips64le
+ cd /usr/local/go/src
+ ./make.bash
Building Go cmd/dist using /usr/local/goroot-bootstrap. (go1.25.1 linux/mips64le)
Building Go toolchain1 using /usr/local/goroot-bootstrap.
Building Go bootstrap cmd/go (go_bootstrap) using Go toolchain1.
Building Go toolchain2 using go_bootstrap and Go toolchain1.
runtime: bad pointer in frame main.invoke at 0xc00018bda8: 0x7
fatal error: invalid pointer found on stack

runtime stack:
runtime.throw({0x6b4e9b, 0x1e})
	/usr/local/go/src/runtime/panic.go:1227 +0x4c fp=0x7ffffbf1f208 sp=0x7ffffbf1f1e0 pc=0xb9f54
runtime.adjustpointers(0xc00018bce0, 0x7ffffbf1f2c0, 0x7ffffbf1f380, {0x9ff4b0, 0xa16840})
	/usr/local/go/src/runtime/stack.go:681 +0x288 fp=0x7ffffbf1f250 sp=0x7ffffbf1f208 pc=0x93f98
runtime.adjustframe(0x7ffffbf1f320, 0x7ffffbf1f380)
	/usr/local/go/src/runtime/stack.go:738 +0xe4 fp=0x7ffffbf1f2e0 sp=0x7ffffbf1f250 pc=0x94084
runtime.copystack(0xc0000021c0, 0x4000)
	/usr/local/go/src/runtime/stack.go:976 +0x364 fp=0x7ffffbf1f3c8 sp=0x7ffffbf1f2e0 pc=0x94874
runtime.newstack()
	/usr/local/go/src/runtime/stack.go:1168 +0x56c fp=0x7ffffbf1f4f8 sp=0x7ffffbf1f3c8 pc=0x94f8c
runtime.morestack()
	/usr/local/go/src/runtime/asm_mips64x.s:275 +0x78 fp=0x7ffffbf1f500 sp=0x7ffffbf1f4f8 pc=0xc1348

goroutine 1 gp=0xc0000021c0 m=0 mp=0xa49300 [copystack]:
cmd/vendor/golang.org/x/mod/modfile.(*File).add(0xc0001820d0, 0xc00018acb8, 0x0, 0xc0000b8280, {0xc0000a0500, 0x6}, {0xc0000ac690, 0x1, 0x1}, 0xc00018b370, ...)
	/usr/local/go/src/cmd/vendor/golang.org/x/mod/modfile/rule.go:339 +0x20 fp=0xc00018ac20 sp=0xc00018ac20 pc=0x2d36f8
cmd/vendor/golang.org/x/mod/modfile.parseToFile({0xc0000a6480, 0x23}, {0xc00009c400, 0x1a, 0x200}, 0xc00018b370, 0x1)
	/usr/local/go/src/cmd/vendor/golang.org/x/mod/modfile/rule.go:292 +0x228 fp=0xc00018ade8 sp=0xc00018ac20 pc=0x2d2c08
cmd/vendor/golang.org/x/mod/modfile.Parse(...)
	/usr/local/go/src/cmd/vendor/golang.org/x/mod/modfile/rule.go:255
cmd/go/internal/modload.ReadModFile({0xc0000a6480, 0x23}, 0xc00018b370)
	/usr/local/go/src/cmd/go/internal/modload/modfile.go:45 +0x194 fp=0xc00018af10 sp=0xc00018ade8 pc=0x3e14ec
cmd/go/internal/modload.loadModFile({0x78b418, 0xa66860}, 0xc0000ba200)
	/usr/local/go/src/cmd/go/internal/modload/init.go:961 +0xd1c fp=0xc00018b4a0 sp=0xc00018af10 pc=0x3c1d94
cmd/go/internal/modload.LoadPackages({0x78b418, 0xa66860}, {{0x0, 0x0}, 0x0, 0x0, 0x0, {0x0, 0x0}, 0x0, ...}, ...)
	/usr/local/go/src/cmd/go/internal/modload/load.go:367 +0x42c fp=0xc00018b7b8 sp=0xc00018b4a0 pc=0x3d18fc
cmd/go/internal/load.PackagesAndErrors({0x78b418, 0xa66860}, {0x0, 0x0, 0x0, 0x1, 0x0, 0x0}, {0xc00001e0b0, 0x5, ...})
	/usr/local/go/src/cmd/go/internal/load/pkg.go:2928 +0x364 fp=0xc00018bb20 sp=0xc00018b7b8 pc=0x4467c4
cmd/go/internal/work.runInstall({0x78b418, 0xa66860}, 0xa38160, {0xc00001e0b0, 0x5, 0x5})
	/usr/local/go/src/cmd/go/internal/work/build.go:699 +0x2d8 fp=0xc00018bbf0 sp=0xc00018bb20 pc=0x4c45c0
main.invoke(0xa38160, {0xc00001e090, 0x7, 0x7})
	/usr/local/go/src/cmd/go/main.go:341 +0x8a4 fp=0xc00018bdb0 sp=0xc00018bbf0 pc=0x5bdac4
main.main()
	/usr/local/go/src/cmd/go/main.go:220 +0xf84 fp=0xc00018bf60 sp=0xc00018bdb0 pc=0x5bc96c
runtime.main()
	/usr/local/go/src/runtime/proc.go:289 +0x3e8 fp=0xc00018bfd8 sp=0xc00018bf60 pc=0x6faf0
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc00018bfd8 sp=0xc00018bfd8 pc=0xc3214

goroutine 2 gp=0xc000002700 m=nil [force gc (idle)]:
runtime.gopark(0x6f03a0, 0xa46060, 0x11, 0xa, 0x1)
	/usr/local/go/src/runtime/proc.go:464 +0x140 fp=0xc00004afa8 sp=0xc00004af90 pc=0xba100
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:470
runtime.forcegchelper()
	/usr/local/go/src/runtime/proc.go:377 +0x13c fp=0xc00004afd8 sp=0xc00004afa8 pc=0x7000c
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc00004afd8 sp=0xc00004afd8 pc=0xc3214
created by runtime.init.6 in goroutine 1
	/usr/local/go/src/runtime/proc.go:365 +0x48

goroutine 3 gp=0xc000002c40 m=nil [GC sweep wait]:
runtime.gopark(0x6f03a0, 0xa463c0, 0xc, 0x9, 0x1)
	/usr/local/go/src/runtime/proc.go:464 +0x140 fp=0xc00004b788 sp=0xc00004b770 pc=0xba100
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:470
runtime.bgsweep(0xc00006a000)
	/usr/local/go/src/runtime/mgcsweep.go:279 +0x108 fp=0xc00004b7c8 sp=0xc00004b788 pc=0x53388
runtime.gcenable.gowrap1()
	/usr/local/go/src/runtime/mgc.go:212 +0x40 fp=0xc00004b7d8 sp=0xc00004b7c8 pc=0x41a60
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc00004b7d8 sp=0xc00004b7d8 pc=0xc3214
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:212 +0xc4

goroutine 4 gp=0xc000002e00 m=nil [GC scavenge wait]:
runtime.gopark(0x6f03a0, 0xa47200, 0xd, 0xa, 0x2)
	/usr/local/go/src/runtime/proc.go:464 +0x140 fp=0xc00004bf80 sp=0xc00004bf68 pc=0xba100
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:470
runtime.(*scavengerState).park(0xa47200)
	/usr/local/go/src/runtime/mgcscavenge.go:425 +0x9c fp=0xc00004bfa8 sp=0xc00004bf80 pc=0x4ffc4
runtime.bgscavenge(0xc00006a000)
	/usr/local/go/src/runtime/mgcscavenge.go:653 +0x80 fp=0xc00004bfc8 sp=0xc00004bfa8 pc=0x50700
runtime.gcenable.gowrap2()
	/usr/local/go/src/runtime/mgc.go:213 +0x40 fp=0xc00004bfd8 sp=0xc00004bfc8 pc=0x41a10
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc00004bfd8 sp=0xc00004bfd8 pc=0xc3214
created by runtime.gcenable in goroutine 1
	/usr/local/go/src/runtime/mgc.go:213 +0x130

goroutine 5 gp=0xc0000036c0 m=nil [GOMAXPROCS updater (idle)]:
runtime.gopark(0x6f03a0, 0xa45fc0, 0x12, 0xa, 0x1)
	/usr/local/go/src/runtime/proc.go:464 +0x140 fp=0xc00004c780 sp=0xc00004c768 pc=0xba100
runtime.goparkunlock(...)
	/usr/local/go/src/runtime/proc.go:470
runtime.updateMaxProcsGoroutine()
	/usr/local/go/src/runtime/proc.go:6686 +0x1a4 fp=0xc00004c7d8 sp=0xc00004c780 pc=0x84d54
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc00004c7d8 sp=0xc00004c7d8 pc=0xc3214
created by runtime.defaultGOMAXPROCSUpdateEnable in goroutine 1
	/usr/local/go/src/runtime/proc.go:6674 +0x8c

goroutine 6 gp=0xc000003880 m=nil [finalizer wait]:
runtime.gopark(0x6f00b0, 0xa66a40, 0x10, 0xa, 0x1)
	/usr/local/go/src/runtime/proc.go:464 +0x140 fp=0xc00004a728 sp=0xc00004a710 pc=0xba100
runtime.runFinalizers()
	/usr/local/go/src/runtime/mfinal.go:210 +0x194 fp=0xc00004a7d8 sp=0xc00004a728 pc=0x403cc
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc00004a7d8 sp=0xc00004a7d8 pc=0xc3214
created by runtime.createfing in goroutine 1
	/usr/local/go/src/runtime/mfinal.go:172 +0xa8

goroutine 17 gp=0xc000082380 m=nil [runnable]:
runtime.runCleanups()
	/usr/local/go/src/runtime/mcleanup.go:633 fp=0xc0000467d8 sp=0xc0000467d8 pc=0x3b688
runtime.goexit({})
	/usr/local/go/src/runtime/asm_mips64x.s:664 +0x4 fp=0xc0000467d8 sp=0xc0000467d8 pc=0xc3214
created by runtime.(*cleanupQueue).createGs in goroutine 1
	/usr/local/go/src/runtime/mcleanup.go:589 +0x194
go tool dist: FAILED: /usr/local/go/pkg/tool/linux_mips64le/go_bootstrap install -pgo=off cmd/asm cmd/cgo cmd/compile cmd/link cmd/preprofile: exit status 2

I'm happy to provide more information/testing as might be helpful to narrow down/fix it, but I'm definitely outside my depth with this error/backtrace. 👍

[gabyhelp]

Related Issues

• build: crashes during compilation on mips64le hw #16848 (closed)
• runtime: problem with preemption on windows arm32 #68552 (closed)
• runtime: build fails when run via QEMU for linux/amd64 running on linux/arm64 #69255 (closed)
• cmd/compile/internal/ssa: SIGSEGV building 1.16 toolchain2 on openbsd/mips64 #44332 (closed)
• runtime: AIX build broken with fatal error 'taggedPointerPack invalid packing' #73515 (closed)
• cmd/compile: go1.8 failed to install from source on mips64le #19140 (closed)
• cmd/compile: tip cannot bootstrap go 1.6 #19084 (closed)
• runtime: debugCheckBP = true crashes compiler compilation on Windows #41995
• cmd/compile: bootstrap glitch with version checking on min/max usage #65886
• runtime: frame pointer check fails on arm64 for C-to-Go calls #59401

_{(Emoji vote if this was helpful or unhelpful; more detailed feedback welcome in this discussion.)}

[mknyszek]

CC @golang/mips

I'll note that our mips builders are currently in pretty bad shape, and picking a random build, I see exactly the same failure: https://build.golang.org/log/e7dfa9a088f2bbc07b39bf59106fe81d6edf44ec

[tianon]

I ran a (very slow) git bisect and it's probably not a huge shock that the culprit is the only obviously mips-specific commit in the range, bb48272:

commit bb48272e24ca38676442afc20afb6e0b9a91048b
Author: Julian Zhu <jz531210@gmail.com>
Date:   Thu Sep 4 21:38:47 2025 +0800

    cmd/compile: simplify zerorange on mips64
    
    Change-Id: I488b55a21eaaf74373c2789a34bf9b3945ced072
    Reviewed-on: https://go-review.googlesource.com/c/go/+/700936
    Reviewed-by: Michael Pratt <mpratt@google.com>
    Reviewed-by: Keith Randall <khr@golang.org>
    Reviewed-by: Keith Randall <khr@google.com>
    LUCI-TryBot-Result: Go LUCI <golang-scoped@luci-project-accounts.iam.gserviceaccount.com>
    Auto-Submit: Keith Randall <khr@golang.org>

 src/cmd/compile/internal/mips64/ggen.go | 38 ++++++---------------------------
 1 file changed, 7 insertions(+), 31 deletions(-)

cc @HeliC829

[randall77]

Looking at that CL, I suspect there is a +8 that is missing from the old to the new code (old line 21).
bb48272#diff-9caaf8369990a0171c60ffdf6f9b557f3269e745873fd11fe0aab227855bc204L19

Could you try replacing off with off+8 in the new code and see if that fixes it?

[tianon]

Confirmed, the following patch on current master (eb7c67f) builds successfully!

diff --git a/src/cmd/compile/internal/mips64/ggen.go b/src/cmd/compile/internal/mips64/ggen.go
index 0740d9ab..740d68e3 100644
--- a/src/cmd/compile/internal/mips64/ggen.go
+++ b/src/cmd/compile/internal/mips64/ggen.go
@@ -17,7 +17,7 @@ func zerorange(pp *objw.Progs, p *obj.Prog, off, cnt int64, _ *uint32) *obj.Prog
 	}
 
 	for cnt != 0 {
-		p = pp.Append(p, mips.AMOVV, obj.TYPE_REG, mips.REGZERO, 0, obj.TYPE_MEM, mips.REGSP, off)
+		p = pp.Append(p, mips.AMOVV, obj.TYPE_REG, mips.REGZERO, 0, obj.TYPE_MEM, mips.REGSP, off+8)
 		cnt -= int64(types.PtrSize)
 		off += int64(types.PtrSize)
 	}

[gopherbot]

Change https://go.dev/cl/703937 mentions this issue: cmd/compile: fix mips zerorange implementation