Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: fatal error: unknown caller pc #44096

Open
xqzhang2015 opened this issue Feb 4, 2021 · 12 comments
Open

runtime: fatal error: unknown caller pc #44096

xqzhang2015 opened this issue Feb 4, 2021 · 12 comments

Comments

@xqzhang2015
Copy link

@xqzhang2015 xqzhang2015 commented Feb 4, 2021

What version of Go are you using (go version)?

$ go version
go version go1.15.3 linux/amd64

Does this issue reproduce with the latest release?

Not sure. Randomly crashed, no core file generated, and can't manually reproduce. And it's a PROD env, I haven't tried the latest Go version.

What operating system and processor architecture are you using (go env)?

go env Output
$ go env

GOARCH="amd64"
GOBIN=""
GOEXE=""
GOFLAGS=""
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GOINSECURE=""
GONOPROXY=""
GONOSUMDB=""
GOOS="linux"
GCCGO="gccgo"
AR="ar"
CC="gcc"
CXX="g++"
CGO_ENABLED="1"
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"

What did you do?

Can't reproduce now and a huge codebase.

What did you expect to see?

Not crash

What did you see instead?

back trace Output
runtime: unexpected return pc for runtime.sigpanic called from 0x89a8bbb9071581e4
stack: frame={sp:0xc006e4dd48, fp:0xc006e4dd78} stack=[0xc006e4d000,0xc006e4e000)
000000c006e4dc48:  000000000046a2e0   0000000000450bf3  
000000c006e4dc58:  000000c006e4dd48  000000c006430900 
000000c006e4dc68:  0000000000000000  000000c006e4dd38 
000000c006e4dc78:  0000000000439c05   000000c006430900 
000000c006e4dc88:  0000000000450bf3   000000c006e4dd48 
000000c006e4dc98:  0000000000004002  0000000000000000 
000000c006e4dca8:  0000000000000000  abc7a11dcb5e5c6b 
000000c006e4dcb8:  fb87fc4b0820e295  d396eac478514bd9 
000000c006e4dcc8:  2031f41adbb60851  000000c006430900 
000000c006e4dcd8:  f2f79aedd2c66ef7  861f82748f6fe4d2 
000000c006e4dce8:  a62e766e54d9ec83  9654578e104bd5fa 
000000c006e4dcf8:  000000c006430920  0000000000000000 
000000c006e4dd08:  0000000000d534e0  000000000168d290 
000000c006e4dd18:  0000000000000000  0000000000000000 
000000c006e4dd28:  0000000000000000  0000000000000000 
000000c006e4dd38:  000000c006e4dd68  0000000000450bf3  
000000c006e4dd48: <0000000000d534e0  000000000168d290 
000000c006e4dd58:  7aa8d45d4434b5bd  0000000000ffffff 
000000c006e4dd68:  0000000000ffffff !89a8bbb9071581e4 
000000c006e4dd78: >2f4a498cc5376f4d  0000000000000001 
000000c006e4dd88:  c200000000000000  08090a0b0c0d0e0f 
000000c006e4dd98:  0001020304050607  000000c006e4ddd0 
000000c006e4dda8:  0000000000000000  000000c00085bc80 
000000c006e4ddb8:  0000000000000001  0000000000000000 
000000c006e4ddc8:  000000c00085bc80  00000a8c46505853 
000000c006e4ddd8:  00000000000002ff  0000000000000a88 
000000c006e4dde8:  0000000000000000  0000000000000000 
000000c006e4ddf8:  0000000000000000  0000000000000203 
000000c006e4de08:  0000000000000000  0000000000000000 
000000c006e4de18:  0000000000000000  0000000000000000 
000000c006e4de28:  0000000000000000  0000000000000000 
000000c006e4de38:  0000000000000000  0000000000000000 
000000c006e4de48:  0000000000000000  0000000000000000 
000000c006e4de58:  0000000000000000  0000000000000000 
000000c006e4de68:  0000000000000000  0000000000000000 
fatal error: unknown caller pc

runtime stack:
runtime.throw(0xe9452a, 0x11)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0x7efdcbffc988 sp=0x7efdcbffc958 pc=0x43a432
runtime.gentraceback(0x450bf3, 0xc006e4dd48, 0x0, 0xc006430900, 0x0, 0x0, 0x7fffffff, 0x7efdcbffccb0, 0x0, 0x0, ...)
/usr/local/go/src/runtime/traceback.go:273 +0x1aec fp=0x7efdcbffcc58 sp=0x7efdcbffc988 pc=0x46194c
runtime.addOneOpenDeferFrame.func1()
/usr/local/go/src/runtime/panic.go:721 +0x91 fp=0x7efdcbffccd8 sp=0x7efdcbffcc58 pc=0x46a371
runtime.systemstack(0x7efd4c002550)
/usr/local/go/src/runtime/asm_amd64.s:370 +0x66 fp=0x7efdcbffcce0 sp=0x7efdcbffccd8 pc=0x471446
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1116 fp=0x7efdcbffcce8 sp=0x7efdcbffcce0 pc=0x43f600

goroutine 16188876 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc006e4dc40 sp=0xc006e4dc38 pc=0x4713c0
runtime.addOneOpenDeferFrame(0xc006430900, 0x450bf3, 0xc006e4dd48)
/usr/local/go/src/runtime/panic.go:720 +0x7b fp=0xc006e4dc80 sp=0xc006e4dc40 pc=0x4394fb
panic(0xd534e0, 0x168d290)
/usr/local/go/src/runtime/panic.go:929 +0x125 fp=0xc006e4dd48 sp=0xc006e4dc80 pc=0x439c05
runtime: unexpected return pc for runtime.sigpanic called from 0x89a8bbb9071581e4
stack: frame={sp:0xc006e4dd48, fp:0xc006e4dd78} stack=[0xc006e4d000,0xc006e4e000)
000000c006e4dc48: 000000000046a2e0 <runtime.addOneOpenDeferFrame.func1+0> 0000000000450bf3 <runtime.sigpanic+1043>
000000c006e4dc58: 000000c006e4dd48 000000c006430900
000000c006e4dc68: 0000000000000000 000000c006e4dd38
000000c006e4dc78: 0000000000439c05 <runtime.gopanic+293> 000000c006430900
000000c006e4dc88: 0000000000450bf3 <runtime.sigpanic+1043> 000000c006e4dd48
000000c006e4dc98: 0000000000004002 0000000000000000
000000c006e4dca8: 0000000000000000 abc7a11dcb5e5c6b
000000c006e4dcb8: fb87fc4b0820e295 d396eac478514bd9
000000c006e4dcc8: 2031f41adbb60851 000000c006430900
000000c006e4dcd8: f2f79aedd2c66ef7 861f82748f6fe4d2
000000c006e4dce8: a62e766e54d9ec83 9654578e104bd5fa
000000c006e4dcf8: 000000c006430920 0000000000000000
000000c006e4dd08: 0000000000d534e0 000000000168d290
000000c006e4dd18: 0000000000000000 0000000000000000
000000c006e4dd28: 0000000000000000 0000000000000000
000000c006e4dd38: 000000c006e4dd68 0000000000450bf3 <runtime.sigpanic+1043>
000000c006e4dd48: <0000000000d534e0 000000000168d290
000000c006e4dd58: 7aa8d45d4434b5bd 0000000000ffffff
000000c006e4dd68: 0000000000ffffff !89a8bbb9071581e4
000000c006e4dd78: >2f4a498cc5376f4d 0000000000000001
000000c006e4dd88: c200000000000000 08090a0b0c0d0e0f
000000c006e4dd98: 0001020304050607 000000c006e4ddd0
000000c006e4dda8: 0000000000000000 000000c00085bc80
000000c006e4ddb8: 0000000000000001 0000000000000000
000000c006e4ddc8: 000000c00085bc80 00000a8c46505853
000000c006e4ddd8: 00000000000002ff 0000000000000a88
000000c006e4dde8: 0000000000000000 0000000000000000
000000c006e4ddf8: 0000000000000000 0000000000000203
000000c006e4de08: 0000000000000000 0000000000000000
000000c006e4de18: 0000000000000000 0000000000000000
000000c006e4de28: 0000000000000000 0000000000000000
000000c006e4de38: 0000000000000000 0000000000000000
000000c006e4de48: 0000000000000000 0000000000000000
000000c006e4de58: 0000000000000000 0000000000000000
000000c006e4de68: 0000000000000000 0000000000000000
runtime.panicmem(...)
/usr/local/go/src/runtime/panic.go:212
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:720 +0x413 fp=0xc006e4dd78 sp=0xc006e4dd48 pc=0x450bf3
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1709 +0xcdc

goroutine 1 [chan receive, 779 minutes]:
runtime.gopark(0xecf018, 0xc001c34cb8, 0xec170e, 0x2)
/usr/local/go/src/runtime/proc.go:306 +0xe5 fp=0xc001440cc8 sp=0xc001440ca8 pc=0x43d025
runtime.chanrecv(0xc001c34c60, 0xc000eefdc8, 0x16eb401, 0xecf3d8)
/usr/local/go/src/runtime/chan.go:577 +0x36f fp=0xc001440d58 sp=0xc001440cc8 pc=0x40930f
runtime.chanrecv1(0xc001c34c60, 0xc000eefdc8)
/usr/local/go/src/runtime/chan.go:439 +0x2b fp=0xc001440d88 sp=0xc001440d58 pc=0x408f4b
main.sigHandler()
/XXX/main.go:95 +0x10b fp=0xc001440e38 sp=0xc001440d88 pc=0xc8608b
main.main()
/XXX/main.go:78 +0x3f1 fp=0xc001440f88 sp=0xc001440e38 pc=0xc859f1
runtime.main()
/usr/local/go/src/runtime/proc.go:204 +0x209 fp=0xc001440fe0 sp=0xc001440f88 pc=0x43cc29
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc001440fe8 sp=0xc001440fe0 pc=0x4731a1

@davecheney
Copy link
Contributor

@davecheney davecheney commented Feb 4, 2021

This may be caused by memory corruption. Have you tried running your program under the race detector? See https://blog.golang.org/race-detector .

Loading

@bcmills
Copy link
Member

@bcmills bcmills commented Feb 4, 2021

The call to runtime.addOneOpenDeferFrame in the traceback suggests corruption of the defer stack, which could be related to #43941 (comment) and the associated issues.

(CC @mdempsky @danscales)

On the other hand, a similar symptom can also occur if the program has a C dependency that registers a signal handler without SA_ONSTACK, which can cause memory corruption due to the C signal handler overrunning the (very small) goroutine stack; see #36287.

Loading

@mdempsky
Copy link
Member

@mdempsky mdempsky commented Feb 4, 2021

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 4, 2021

@davecheney I ever built with race condition detection enabled but crash doesn't happen again with such bin. I'm also confused about that.

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 4, 2021

@bcmills thanks for the comment.
for the defer issue, is there a way to detect it? It seems my App doesn't crash after enabling race detecting.
for the C signal handler, I practically use cgo to call a C library and will check it.

Loading

@mdempsky
Copy link
Member

@mdempsky mdempsky commented Feb 4, 2021

@xqzhang2015 A fix for #43882 is included in Go 1.16rc1 and will be included in the upcoming Go 1.16 release, likely within the next couple weeks. If this is related to #43882, which seems likely to me, then upgrading to Go 1.16rc1 or Go 1.16 (once it's released) should address the problem. If you see the issue again with Go 1.16rc1 or Go 1.16 though, that would be interesting.

I'm afraid there's not much else we're able to do at the moment with your report, unless you're able to reproduce the failure.

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 4, 2021

@mdempsky totally understands that. And all your comments are very helpful.
I will try to dig it according to these clues and try to reproduce the crash.

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 7, 2021

@bcmills I didn't find any signal handler in use on the C side.

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 7, 2021

I use goreplay to capture and replay the traffic during crash, then the crash could be reproduced but not stable.
It seems not a specific HTTP request leads to such crash. Instead, it seems because of a special condition.

Interesting thing is that panic during panic happens, which is in the bt below.

New back trace
runtime: unexpected return pc for crypto/tls.(*Conn).readRecordOrCCS called from 0xc006709c00
stack: frame={sp:0xc0067096c8, fp:0xc006709b28} stack=[0xc006709000,0xc00670a000)
000000c0067095c8:  0000000000000000  0000000000000000 
000000c0067095d8:  0000000000000000  0000000000000010 
000000c0067095e8:  0000000000000018  00007f1620f0d360 
000000c0067095f8:  000000c006709668  00000000005154d1  
000000c006709608:  000000c005ee45e0  000000c006d68000 
000000c006709618:  0000000000000f3b  0000000000000f3b 
000000c006709628:  00000000000004b6  0000000000000f36 
000000c006709638:  000000c006709668  0000000000000000 
000000c006709648:  0000000000000020  0000000000d922a0 
000000c006709658:  0000000000000000  000000c005ee45e0 
000000c006709668:  000000c0067096b8  0000000000666073  
000000c006709678:  000000c005613780  0000000000fb6a00 
000000c006709688:  000000c005ee45e0  000000000040e1e5  
000000c006709698:  0000000000d66720  0000000000e3f7e0 
000000c0067096a8:  0000000000000005  000000c005613780 
000000c0067096b8:  000000c006709b18  0000000000662e95  
000000c0067096c8: <000000c005613500  0000000000fb72e0 
000000c0067096d8:  000000c0067c05c8  0000000000000005 
000000c0067096e8:  000000c0067c05c8  00007f1619156700 
000000c0067096f8:  0000000000000200  00007f16192e5fff 
000000c006709708:  0000000000000800  00007f1619156900 
000000c006709718:  0000000100000000  00007f16192e5fff 
000000c006709728:  0000000000000800  0000000000203000 
000000c006709738:  0000000000000800  0000000000000800 
000000c006709748:  0000000000000580  00007f1619156900 
000000c006709758:  00007f16192e5fff  000000c0067097a8 
000000c006709768:  000000000041b99e   00007f1619156700 
000000c006709778:  0020300000000000  00007f16192e5fff 
000000c006709788:  00007f15efc74980  00007f15efc74980 
000000c006709798:  0000000000000580  0000000000000002 
000000c0067097a8:  000000c006709820  000000000041b485  
000000c0067097b8:  00007f15efc74980  0000000000000000 
000000c0067097c8:  0000000000000003  0000000000000002 
000000c0067097d8:  0000000000000001  000000000051ad6b  
000000c0067097e8:  000000c006d6933f  0000000000000166 
000000c0067097f8:  0000000000000cc1  0000000000000008 
000000c006709808:  0000000000000001  0000000000000030 
000000c006709818:  0000000000203000  000000c006709890 
000000c006709828:  000000000051af34   0000000000203000 
000000c006709838:  0000000000203000  0000000000203000 
000000c006709848:  0000000000000002  0000000000000cc1 
000000c006709858:  0000000000000000  0000000000000000 
000000c006709868:  0000000000000000  000000c006709910 
000000c006709878:  00000000004104d0   000000c00031fd70 
000000c006709888:  0000000000000030  0000000000000028 
000000c006709898:  000000c0056137a8  000000c001e51200 
000000c0067098a8:  000000c006d6933f  0000000000203001 
000000c0067098b8:  0000000000000020  0000000000000028 
000000c0067098c8:  0000000000203001  0000000000203001 
000000c0067098d8:  0000000000203001  000000c000780400 
000000c0067098e8:  0000000000000000  00000000ca152938 
000000c0067098f8:  0000000000000008  000000c006709978 
000000c006709908:  000000c0067099a8  0000000000203000 
000000c006709918:  000000c000d7c020  0000000000000010 
000000c006709928:  0000000000203000  0000000000203000 
000000c006709938:  0000000000203000  000000c0067099d8 
000000c006709948:  00007f15efc31908  0000000000000008 
000000c006709958:  0000000000000010  00007f1620f0d360 
000000c006709968:  000000c006709a08  00000000004104d0  
000000c006709978:  000000c00031fdd0  0000000000000030 
000000c006709988:  0000000000000028  0000000000dddfe0 
000000c006709998:  0000000000000008  0000000000e3cea0 
000000c0067099a8:  00007f15efc2b180  0000000000000028 
000000c0067099b8:  0000000000000028  00007f1620f0d360 
000000c0067099c8:  000000c00031fdd0  0000000000417c3b  
000000c0067099d8:  000000c000780400  0000000000000000 
000000c0067099e8:  0000000000000038  0000000000000001 
000000c0067099f8:  77d220971a80dc27  0000000000415a3b  
000000c006709a08:  000000c006709a58  000000007d9504a8 
000000c006709a18:  0000000000000000  0000000000000000 
000000c006709a28:  000000c006709a58  000000000047039b  
000000c006709a38:  00007f161d406370  0000000000000001 
000000c006709a48:  0000000000000000  000000c007c94000 
000000c006709a58:  000000c000000000  0000000000008000 
000000c006709a68:  000000c00670a801  0000000000000004 
000000c006709a78:  0000000000fe15c0  0000000000000212 
000000c006709a88:  00000000ed3cee2e  000000000000000e 
000000c006709a98:  000000c0057f2b40  0000000000000200 
000000c006709aa8:  00000000000001c3  000000c00177ae00 
000000c006709ab8:  000000c00670a798  000000c00004f800 
000000c006709ac8:  000000000000001f  ffffffffffffffe0 
000000c006709ad8:  ffffffffffffffff  000000c00670a748 
000000c006709ae8:  000000000048a65b   0000000000000212 
000000c006709af8:  0000000000000033  0000000000000000 
000000c006709b08:  0000000000000000  0000000000000000 
000000c006709b18:  0000000000000000 !000000c006709c00 
000000c006709b28: >000000c005613500  0000000000000000 
000000c006709b38:  0000000000000000  0000000000000000 
000000c006709b48:  010000000043d0dc  000000c006709b58 
000000c006709b58:  000000000046a740   000000c001def380 
000000c006709b68:  0000000000000000  0000000000ecf3d8 
000000c006709b78:  000000c006709be8  000000000074f3f7  
000000c006709b88:  000000c005613500  000000c006d69000 
000000c006709b98:  0000000000001000  0000000000001000 
000000c006709ba8:  0000000000000000  0000000000000000 
000000c006709bb8:  0000000000000000  000000c006709cc2 
000000c006709bc8:  000000c006709c10  0000000000000003 
000000c006709bd8:  0000000000000000  0000000000000005 
000000c006709be8:  000000c006709c38  0000000000519fa5  
000000c006709bf8:  000000c0067d4c60  000000000000037f 
000000c006709c08:  00007f161b0cb5e8  0000000000000000 
000000c006709c18:  0000ffff00001fa0  0000000000000000 
fatal error: unknown caller pc

runtime stack:
runtime.throw(0xe9452a, 0x11)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0x7f15383f9690 sp=0x7f15383f9660 pc=0x43a432
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00020fe00, 0x0, 0x0, 0x7fffffff, 0x7f15383f9b40, 0x0, 0x0, ...)
/usr/local/go/src/runtime/traceback.go:273 +0x1aec fp=0x7f15383f9960 sp=0x7f15383f9690 pc=0x46194c
runtime.scanstack(0xc00020fe00, 0xc00003f698)
/usr/local/go/src/runtime/mgcmark.go:844 +0x17c fp=0x7f15383f9b68 sp=0x7f15383f9960 pc=0x4228bc
runtime.markroot.func1()
/usr/local/go/src/runtime/mgcmark.go:245 +0xc6 fp=0x7f15383f9bb8 sp=0x7f15383f9b68 pc=0x468a46
runtime.markroot(0xc00003f698, 0xc0000000c8)
/usr/local/go/src/runtime/mgcmark.go:218 +0x310 fp=0x7f15383f9c40 sp=0x7f15383f9bb8 pc=0x421750
runtime.gcDrain(0xc00003f698, 0x3)
/usr/local/go/src/runtime/mgcmark.go:1109 +0x118 fp=0x7f15383f9c98 sp=0x7f15383f9c40 pc=0x4233b8
runtime.gcBgMarkWorker.func2()
/usr/local/go/src/runtime/mgc.go:1958 +0x87 fp=0x7f15383f9cd8 sp=0x7f15383f9c98 pc=0x468847
runtime.systemstack(0x7f1545c00120)
/usr/local/go/src/runtime/asm_amd64.s:370 +0x66 fp=0x7f15383f9ce0 sp=0x7f15383f9cd8 pc=0x471446
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1116 fp=0x7f15383f9ce8 sp=0x7f15383f9ce0 pc=0x43f600

goroutine 55 [GC worker (idle)]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_amd64.s:330 fp=0xc00025c760 sp=0xc00025c758 pc=0x4713c0
runtime.gcBgMarkWorker(0xc00003e000)
/usr/local/go/src/runtime/mgc.go:1945 +0x1be fp=0xc00025c7d8 sp=0xc00025c760 pc=0x42007e
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc00025c7e0 sp=0xc00025c7d8 pc=0x4731a1
created by runtime.gcBgMarkStartWorkers
/usr/local/go/src/runtime/mgc.go:1839 +0x77

goroutine 1 [chan receive, 7 minutes]:
runtime.gopark(0xecf018, 0xc00044ce98, 0xec170e, 0x2)
/usr/local/go/src/runtime/proc.go:306 +0xe5 fp=0xc0072a5cc8 sp=0xc0072a5ca8 pc=0x43d025
runtime.chanrecv(0xc00044ce40, 0xc000adbdc8, 0x16eb401, 0xecf3d8)
/usr/local/go/src/runtime/chan.go:577 +0x36f fp=0xc0072a5d58 sp=0xc0072a5cc8 pc=0x40930f
runtime.chanrecv1(0xc00044ce40, 0xc000adbdc8)
/usr/local/go/src/runtime/chan.go:439 +0x2b fp=0xc0072a5d88 sp=0xc0072a5d58 pc=0x408f4b
main.sigHandler()
/XXX/src/go/src/YYY/main.go:95 +0x10b fp=0xc0072a5e38 sp=0xc0072a5d88 pc=0xc8608b
main.main()
/XXX/src/go/src/YYY/main.go:78 +0x3f1 fp=0xc0072a5f88 sp=0xc0072a5e38 pc=0xc859f1
runtime.main()
/usr/local/go/src/runtime/proc.go:204 +0x209 fp=0xc0072a5fe0 sp=0xc0072a5f88 pc=0x43cc29
runtime.goexit()
/usr/local/go/src/runtime/asm_amd64.s:1374 +0x1 fp=0xc0072a5fe8 sp=0xc0072a5fe0 pc=0x4731a1

// ...

goroutine 112496 [IO wait (scan), 1 minutes]:
runtime.gopark(0xecf210, 0x7f15ec84da60, 0xc006701b02, 0x5)
/usr/local/go/src/runtime/proc.go:306 +0xe5 fp=0xc0067093e8 sp=0xc0067093c8 pc=0x43d025
runtime.netpollblock(0x7f15ec84da38, 0xffffff0000000072, 0xfb8720)
/usr/local/go/src/runtime/netpoll.go:436 +0x9b fp=0xc006709420 sp=0xc0067093e8 pc=0x43577b
internal/poll.runtime_pollWait(0x7f15ec84da38, 0x72, 0xfb8720)
/usr/local/go/src/runtime/netpoll.go:220 +0x55 fp=0xc006709448 sp=0xc006709420 pc=0x46d335
internal/poll.(*pollDesc).wait(0xc0097d9418, 0x72, 0xfb8700, 0x16a2260, 0x0)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:87 +0x45 fp=0xc006709478 sp=0xc006709448 pc=0x4e3d85
internal/poll.(*pollDesc).waitRead(...)
/usr/local/go/src/internal/poll/fd_poll_runtime.go:92
internal/poll.(*FD).Read(0xc0097d9400, 0xc006d68000, 0xf3b, 0xf3b, 0x0, 0x0, 0x0)
/usr/local/go/src/internal/poll/fd_unix.go:159 +0x1a5 fp=0xc0067094e8 sp=0xc006709478 pc=0x4e4dc5
net.(*netFD).Read(0xc0097d9400, 0xc006d68000, 0xf3b, 0xf3b, 0x203001, 0x66123b, 0xc005613660)
/usr/local/go/src/net/fd_posix.go:55 +0x4f fp=0xc006709548 sp=0xc0067094e8 pc=0x5faacf
net.(*conn).Read(0xc0067c05c8, 0xc006d68000, 0xf3b, 0xf3b, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:182 +0x8e fp=0xc0067095a8 sp=0xc006709548 pc=0x60feee
crypto/tls.(*atLeastReader).Read(0xc005ee45e0, 0xc006d68000, 0xf3b, 0xf3b, 0x4b6, 0xf36, 0xc006709668)
/usr/local/go/src/crypto/tls/conn.go:779 +0x62 fp=0xc006709608 sp=0xc0067095a8 pc=0x665e22
bytes.(*Buffer).ReadFrom(0xc005613780, 0xfb6a00, 0xc005ee45e0, 0x40e1e5, 0xd66720, 0xe3f7e0)
/usr/local/go/src/bytes/buffer.go:204 +0xb1 fp=0xc006709678 sp=0xc006709608 pc=0x5154d1
crypto/tls.(*Conn).readFromUntil(0xc005613500, 0xfb72e0, 0xc0067c05c8, 0x5, 0xc0067c05c8, 0x7f1619156700)
/usr/local/go/src/crypto/tls/conn.go:801 +0xf3 fp=0xc0067096c8 sp=0xc006709678 pc=0x666073
fatal error: unexpected signal during runtime execution
panic during panic
[signal SIGSEGV: segmentation violation code=0x1 addr=0x118 pc=0x46125e]

runtime stack:
runtime.throw(0xeabc22, 0x2a)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0x7f15383f9060 sp=0x7f15383f9030 pc=0x43a432
runtime.sigpanic()
/usr/local/go/src/runtime/signal_unix.go:704 +0x4ac fp=0x7f15383f9090 sp=0x7f15383f9060 pc=0x450c8c
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00020fe00, 0x0, 0x0, 0x64, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/runtime/traceback.go:261 +0x13fe fp=0x7f15383f9360 sp=0x7f15383f9090 pc=0x46125e
runtime.traceback1(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00020fe00, 0x0)
/usr/local/go/src/runtime/traceback.go:736 +0xf2 fp=0x7f15383f9508 sp=0x7f15383f9360 pc=0x462332
runtime.traceback(...)
/usr/local/go/src/runtime/traceback.go:690
runtime.tracebackothers(0xc001439980)
/usr/local/go/src/runtime/traceback.go:944 +0x165 fp=0x7f15383f9578 sp=0x7f15383f9508 pc=0x463365
runtime.dopanic_m(0xc001439980, 0x43a432, 0x7f15383f9660, 0x1)
/usr/local/go/src/runtime/panic.go:1322 +0x2bf fp=0x7f15383f95e0 sp=0x7f15383f9578 pc=0x43ab7f
runtime.fatalthrow.func1()
/usr/local/go/src/runtime/panic.go:1171 +0x5f fp=0x7f15383f9628 sp=0x7f15383f95e0 pc=0x46a49f
runtime.fatalthrow()
/usr/local/go/src/runtime/panic.go:1168 +0x57 fp=0x7f15383f9660 sp=0x7f15383f9628 pc=0x43a617
runtime.throw(0xe9452a, 0x11)
/usr/local/go/src/runtime/panic.go:1116 +0x72 fp=0x7f15383f9690 sp=0x7f15383f9660 pc=0x43a432
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00020fe00, 0x0, 0x0, 0x7fffffff, 0x7f15383f9b40, 0x0, 0x0, ...)
/usr/local/go/src/runtime/traceback.go:273 +0x1aec fp=0x7f15383f9960 sp=0x7f15383f9690 pc=0x46194c
runtime.scanstack(0xc00020fe00, 0xc00003f698)
/usr/local/go/src/runtime/mgcmark.go:844 +0x17c fp=0x7f15383f9b68 sp=0x7f15383f9960 pc=0x4228bc
runtime.markroot.func1()
/usr/local/go/src/runtime/mgcmark.go:245 +0xc6 fp=0x7f15383f9bb8 sp=0x7f15383f9b68 pc=0x468a46
runtime.markroot(0xc00003f698, 0xc0000000c8)
/usr/local/go/src/runtime/mgcmark.go:218 +0x310 fp=0x7f15383f9c40 sp=0x7f15383f9bb8 pc=0x421750
runtime.gcDrain(0xc00003f698, 0x3)
/usr/local/go/src/runtime/mgcmark.go:1109 +0x118 fp=0x7f15383f9c98 sp=0x7f15383f9c40 pc=0x4233b8
runtime.gcBgMarkWorker.func2()
/usr/local/go/src/runtime/mgc.go:1958 +0x87 fp=0x7f15383f9cd8 sp=0x7f15383f9c98 pc=0x468847
runtime.systemstack(0x7f1545c00120)
/usr/local/go/src/runtime/asm_amd64.s:370 +0x66 fp=0x7f15383f9ce0 sp=0x7f15383f9cd8 pc=0x471446
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1116 fp=0x7f15383f9ce8 sp=0x7f15383f9ce0 pc=0x43f600

Loading

@mdempsky
Copy link
Member

@mdempsky mdempsky commented Feb 7, 2021

@xqzhang2015 Glad to hear you're able to reproduce the crash. Are you able to test with Go 1.16rc1 to see if the crash still reproduces? Alternatively, you can try cherry-picking just a5a5e2c back to Go 1.15.3.

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 8, 2021

@mdempsky Thanks and will try Go 1.16rc1

Loading

@xqzhang2015
Copy link
Author

@xqzhang2015 xqzhang2015 commented Feb 8, 2021

With Go 1.16rc1, there is similar crash and backtrace.

The good news is that we narrow down the issue scope to cgo(calling C library) usage. In the app, I use cgo to call C API and use one flag to control C debug log level and some other business logic, but not related to a signal handler.

One suspicious point is something like the stack bytes size, because the crash only happens when the flag is enabled, which could dump more debug logs to stack, then will be collected by Go side(by a pointer). Will verify it.

bt with Go 1.16rc1
runtime: unexpected return pc for crypto/tls.(*Conn).readRecordOrCCS called from 0x0
stack: frame={sp:0xc000871998, fp:0xc000871c10} stack=[0xc000871000,0xc000872000)
000000c000871898:  0000000000000000  0000000000000018 
000000c0008718a8:  0000000000d5d340  0000000000000000 
000000c0008718b8:  0000000000000000  000000c000871938 
000000c0008718c8:  0000000000519ede   000000c00000ed08 
000000c0008718d8:  000000c005680000  0000000000001756 
000000c0008718e8:  0000000000001756  0000000000000008 
000000c0008718f8:  000000c000101800  0000000000000000 
000000c000871908:  0000000000000000  00000000004111d8  
000000c000871918:  0000000000000018  0000000000000000 
000000c000871928:  0000000000e07701  000000c00000ed08 
000000c000871938:  000000c000871988  0000000000658f33  
000000c000871948:  000000c00179d078  0000000000f7d5a0 
000000c000871958:  000000c00000ed08  000000000040e805  
000000c000871968:  0000000000d318e0  0000000000e07700 
000000c000871978:  0000000000000005  000000c00179d078 
000000c000871988:  000000c000871c00  0000000000655d75  
000000c000871998: <000000c00179ce00  0000000000f7de80 
000000c0008719a8:  000000c00627c0b0  0000000000000005 
000000c0008719b8:  000000c00627c0b0  0000000000203002 
000000c0008719c8:  0000000000203002  3ddd431c2076c8a9 
000000c0008719d8:  000000c000101800  0000000000000000 
000000c0008719e8:  0000000100871a18  00000000004111d8  
000000c0008719f8:  0000000000000008  0000000000410a7b  
000000c000871a08:  000000c006db8580  0000000000203000 
000000c000871a18:  0000000000000010  000000c00179d0a0 
000000c000871a28:  0000000000203000  0000000000203000 
000000c000871a38:  0000000000203000  0000000000000008 
000000c000871a48:  0000000000000010  00007fab433a71d8 
000000c000871a58:  0000000000000010  000000c006db8580 
000000c000871a68:  000000c000871af0  0000000000410a7b  
000000c000871a78:  000000c001231320  0000000000000030 
000000c000871a88:  0000000000000028  0000000000da8980 
000000c000871a98:  0000000000000010  0000000000e04a80 
000000c000871aa8:  00007fab12a5d338  0000000000000028 
000000c000871ab8:  0000000000000028  00007fab433a71d8 
000000c000871ac8:  0000000000000030  000000c001231320 
000000c000871ad8:  0000000000000018  000000c000101800 
000000c000871ae8:  0000000000415e3b   000000c000871b40 
000000c000871af8:  00000000859b02ee  0000000000000000 
000000c000871b08:  0000000000000000  000000c000871b40 
000000c000871b18:  000000000047285b   000188c96d2c6334 
000000c000871b28:  0000000004d25f8f  04d25f8f00d11580 
000000c000871b38:  00007fab3f8ae370  0000000000000001 
000000c000871b48:  0000000000000000  000000c000012000 
000000c000871b58:  000000c000000000  0000000000008000 
000000c000871b68:  000000000000001f  0000000000000004 
000000c000871b78:  000000000000001f  0000000000000206 
000000c000871b88:  00000000acf85492  000000000000000e 
000000c000871b98:  0000000000000086  ffffffffffffffff 
000000c000871ba8:  00000000000001d2  000000c0006190a0 
000000c000871bb8:  000000c000872870  000000c00003e000 
000000c000871bc8:  000000000000001f  ffffffffffffffe0 
000000c000871bd8:  ffffffffffffffff  000000c000872820 
000000c000871be8:  000000000048c7fb   0000000000000206 
000000c000871bf8:  0000000000000033  0000000000000000 
000000c000871c08: !0000000000000000 >0000000000000000 
000000c000871c18:  0000000000000000  000000c000871d00 
000000c000871c28:  000000000043df1c   010000c000871c38 
000000c000871c38:  000000000046cb40   000000c001d76180 
000000c000871c48:  0000000000000004  000000c00179cf58 
000000c000871c58:  0000000000e95610  000000c000871cd0 
000000c000871c68:  0000000000000000  000000c00179ce00 
000000c000871c78:  000000c000e7c000  0000000000001000 
000000c000871c88:  0000000000001000  0000000000000000 
000000c000871c98:  0000000000000000  0000000000000000 
000000c000871ca8:  000000c000871cf8  0000000000000003 
000000c000871cb8:  0000000000000000  0000000000000000 
000000c000871cc8:  0000000000000005  000000c000871d20 
000000c000871cd8:  000000000051e9c8   000000c005e2a000 
000000c000871ce8:  000000c000e7c000  0000000000001000 
000000c000871cf8:  0000000000001000  000000000000037f 
000000c000871d08:  00007fab3d5735e8 
fatal error: unknown caller pc

runtime stack:
runtime.throw(0xe5bf38, 0x11)
/XXX/go1.16rc1/src/runtime/panic.go:1117 +0x72 fp=0x7faa6e7f9638 sp=0x7faa6e7f9608 pc=0x43b1f2
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00821a600, 0x0, 0x0, 0x7fffffff, 0x7faa6e7f9b38, 0x0, 0x0, ...)
/XXX/go1.16rc1/src/runtime/traceback.go:261 +0x1a56 fp=0x7faa6e7f9958 sp=0x7faa6e7f9638 pc=0x463bd6
runtime.scanstack(0xc00821a600, 0xc000050e98)
/XXX/go1.16rc1/src/runtime/mgcmark.go:749 +0x17c fp=0x7faa6e7f9b60 sp=0x7faa6e7f9958 pc=0x42355c
runtime.markroot.func1()
/XXX/go1.16rc1/src/runtime/mgcmark.go:233 +0xc6 fp=0x7faa6e7f9bb0 sp=0x7faa6e7f9b60 pc=0x46ae26
runtime.markroot(0xc000050e98, 0x55d)
/XXX/go1.16rc1/src/runtime/mgcmark.go:206 +0x33e fp=0x7faa6e7f9c40 sp=0x7faa6e7f9bb0 pc=0x4223de
runtime.gcDrain(0xc000050e98, 0x3)
/XXX/go1.16rc1/src/runtime/mgcmark.go:1014 +0x118 fp=0x7faa6e7f9c98 sp=0x7faa6e7f9c40 pc=0x424078
runtime.gcBgMarkWorker.func2()
/XXX/go1.16rc1/src/runtime/mgc.go:1980 +0x88 fp=0x7faa6e7f9cd8 sp=0x7faa6e7f9c98 pc=0x46ac08
runtime.systemstack(0x7faa758000f0)
/XXX/go1.16rc1/src/runtime/asm_amd64.s:379 +0x66 fp=0x7faa6e7f9ce0 sp=0x7faa6e7f9cd8 pc=0x473806
runtime.mstart()
/XXX/go1.16rc1/src/runtime/proc.go:1246 fp=0x7faa6e7f9ce8 sp=0x7faa6e7f9ce0 pc=0x4404c0

goroutine 81 [GC worker (idle)]:
runtime.systemstack_switch()
/XXX/go1.16rc1/src/runtime/asm_amd64.s:339 fp=0xc0005d5f60 sp=0xc0005d5f58 pc=0x473780
runtime.gcBgMarkWorker()
/XXX/go1.16rc1/src/runtime/mgc.go:1967 +0x1c7 fp=0xc0005d5fe0 sp=0xc0005d5f60 pc=0x420ca7
runtime.goexit()
/XXX/go1.16rc1/src/runtime/asm_amd64.s:1371 +0x1 fp=0xc0005d5fe8 sp=0xc0005d5fe0 pc=0x4755c1
created by runtime.gcBgMarkStartWorkers
/XXX/go1.16rc1/src/runtime/mgc.go:1835 +0x37

// ...

fatal error: unexpected signal during runtime execution
panic during panic
[signal SIGSEGV: segmentation violation code=0x1 addr=0x118 pc=0x463529]

runtime stack:
runtime.throw(0xe72916, 0x2a)
/XXX/go1.16rc1/src/runtime/panic.go:1117 +0x72 fp=0x7faa6e7f8fb8 sp=0x7faa6e7f8f88 pc=0x43b1f2
runtime.sigpanic()
/XXX/go1.16rc1/src/runtime/signal_unix.go:718 +0x2e5 fp=0x7faa6e7f8ff0 sp=0x7faa6e7f8fb8 pc=0x4529c5
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00821a600, 0x0, 0x0, 0x64, 0x0, 0x0, 0x0, ...)
/XXX/go1.16rc1/src/runtime/traceback.go:249 +0x13a9 fp=0x7faa6e7f9310 sp=0x7faa6e7f8ff0 pc=0x463529
runtime.traceback1(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00821a600, 0x0)
/XXX/go1.16rc1/src/runtime/traceback.go:732 +0xf2 fp=0x7faa6e7f94b8 sp=0x7faa6e7f9310 pc=0x4645b2
runtime.traceback(...)
/XXX/go1.16rc1/src/runtime/traceback.go:686
runtime.tracebackothers(0xc000510600)
/XXX/go1.16rc1/src/runtime/traceback.go:951 +0x148 fp=0x7faa6e7f9520 sp=0x7faa6e7f94b8 pc=0x4655c8
runtime.dopanic_m(0xc000510600, 0x43b1f2, 0x7faa6e7f9608, 0x1)
/XXX/go1.16rc1/src/runtime/panic.go:1317 +0x2a8 fp=0x7faa6e7f9588 sp=0x7faa6e7f9520 pc=0x43b928
runtime.fatalthrow.func1()
/XXX/go1.16rc1/src/runtime/panic.go:1172 +0x5f fp=0x7faa6e7f95d0 sp=0x7faa6e7f9588 pc=0x46c89f
runtime.fatalthrow()
/XXX/go1.16rc1/src/runtime/panic.go:1169 +0x57 fp=0x7faa6e7f9608 sp=0x7faa6e7f95d0 pc=0x43b3d7
runtime.throw(0xe5bf38, 0x11)
/XXX/go1.16rc1/src/runtime/panic.go:1117 +0x72 fp=0x7faa6e7f9638 sp=0x7faa6e7f9608 pc=0x43b1f2
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc00821a600, 0x0, 0x0, 0x7fffffff, 0x7faa6e7f9b38, 0x0, 0x0, ...)
/XXX/go1.16rc1/src/runtime/traceback.go:261 +0x1a56 fp=0x7faa6e7f9958 sp=0x7faa6e7f9638 pc=0x463bd6
runtime.scanstack(0xc00821a600, 0xc000050e98)
/XXX/go1.16rc1/src/runtime/mgcmark.go:749 +0x17c fp=0x7faa6e7f9b60 sp=0x7faa6e7f9958 pc=0x42355c
runtime.markroot.func1()
/XXX/go1.16rc1/src/runtime/mgcmark.go:233 +0xc6 fp=0x7faa6e7f9bb0 sp=0x7faa6e7f9b60 pc=0x46ae26
runtime.markroot(0xc000050e98, 0x55d)
/XXX/go1.16rc1/src/runtime/mgcmark.go:206 +0x33e fp=0x7faa6e7f9c40 sp=0x7faa6e7f9bb0 pc=0x4223de
runtime.gcDrain(0xc000050e98, 0x3)
/XXX/go1.16rc1/src/runtime/mgcmark.go:1014 +0x118 fp=0x7faa6e7f9c98 sp=0x7faa6e7f9c40 pc=0x424078
runtime.gcBgMarkWorker.func2()
/XXX/go1.16rc1/src/runtime/mgc.go:1980 +0x88 fp=0x7faa6e7f9cd8 sp=0x7faa6e7f9c98 pc=0x46ac08
runtime.systemstack(0x7faa758000f0)
/XXX/go1.16rc1/src/runtime/asm_amd64.s:379 +0x66 fp=0x7faa6e7f9ce0 sp=0x7faa6e7f9cd8 pc=0x473806
runtime.mstart()
/XXX/go1.16rc1/src/runtime/proc.go:1246 fp=0x7faa6e7f9ce8 sp=0x7faa6e7f9ce0 pc=0x4404c0

Loading

@seankhliao seankhliao changed the title fatal error: unknown caller pc runtime: fatal error: unknown caller pc Jun 18, 2021
Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Linked pull requests

Successfully merging a pull request may close this issue.

None yet
4 participants