Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

runtime: unexpected return pc for runtime.sigpanic called from 0x0 #36287

Open
thundercw opened this issue Dec 27, 2019 · 7 comments
Open

runtime: unexpected return pc for runtime.sigpanic called from 0x0 #36287

thundercw opened this issue Dec 27, 2019 · 7 comments

Comments

@thundercw
Copy link

@thundercw thundercw commented Dec 27, 2019

What version of Go are you using (go version)?

$ go version
go version go1.13.5 linux/amd64

Does this issue reproduce with the latest release?

YES

What operating system and processor architecture are you using (go env)?

go env Output
$ go env
GO111MODULE=""
GOARCH="amd64"
GOBIN=""
GOCACHE="/home/etcd/.cache/go-build"
GOENV="/home/etcd/.config/go/env"
GOEXE=""
GOFLAGS=""
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GONOPROXY=""
GONOSUMDB=""
GOOS="linux"
GOPATH="/home/etcd/go"
GOPRIVATE=""
GOPROXY="https://proxy.golang.org,direct"
GOROOT="/usr/local/go"
GOSUMDB="sum.golang.org"
GOTMPDIR=""
GOTOOLDIR="/usr/local/go/pkg/tool/linux_amd64"
GCCGO="gccgo"
AR="ar"
CC="gcc"
CXX="g++"
CGO_ENABLED="1"
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build524004963=/tmp/go-build -gno-record-gcc-switches"

What did you do?

My code is written in c, I use cgo to call the etcd go client
I have past the tools test, include -rance and GODEBUG=cgocheck=2;

What did you expect to see?

no panic

What did you see instead?

runtime: unexpected return pc for runtime.gopark called from 0x0
stack: frame={sp:0x10c00005bf40, fp:0x10c00005bf60} stack=[0x10c00005b800,0x10c00005c000)
000010c00005be40: 0000000000000000 0000000000000000
000010c00005be50: 0000000000000000 0000000000000000
000010c00005be60: 0000000000000000 0000000000000000
000010c00005be70: 0000000000000000 0000000000000000
000010c00005be80: 0000000000000000 0000000000000000
000010c00005be90: 0000000000000000 0000000000000000
000010c00005bea0: 0000000000000000 0000000000000000
000010c00005beb0: 0000000000000000 0000000000000000
000010c00005bec0: 0000000001109488 0000000000000002
000010c00005bed0: 000010c000091500 000010c000062700
000010c00005bee0: 0000000000000000 000010c00005bf30
000010c00005bef0: ffffffffffffffff 000010c00012f140
000010c00005bf00: 0000000000000001 0000000000000000
000010c00005bf10: 0000000000000006 0000000000000050
000010c00005bf20: 0000000000000040 00007fadedc026d0
000010c00005bf30: 000010c00005bf50 000000000064d9f6 <runtime.gopark+230>
000010c00005bf40: <0000000000000000 0000000000000000
000010c00005bf50: 0000000000000000 !0000000000000000
000010c00005bf60: >0000000000000000 0000000000000000
000010c00005bf70: 0000000000000000 0000000000000000
000010c00005bf80: 0000000000000000 0000000000000000
000010c00005bf90: 0000000000000000 0002a16d2c169e29
000010c00005bfa0: 00000000001cf23b 0000000000000000
000010c00005bfb0: 000010c00009e040 0000000001109250
000010c00005bfc0: 0000000000000000 0000000000000000
000010c00005bfd0: 0000000000000000 0000000000000000
000010c00005bfe0: 0000000000000000 0000000000000000
000010c00005bff0: 0000000000000000 0000000000000000
fatal error: unknown caller pc
runtime: unexpected return pc for runtime.sigpanic called from 0x0
stack: frame={sp:0x10c00005cf60, fp:0x10c00005cf68} stack=[0x10c00005c800,0x10c00005d000)
000010c00005ce60: 0000000000000000 0000000000000000
000010c00005ce70: 0000000000000000 0000000000000000
000010c00005ce80: 0000000000000000 0000000000000000
000010c00005ce90: 0000000000000000 0000000000000000
000010c00005cea0: 0000000000000000 0000000000000000
000010c00005ceb0: 0000000000000000 0000000000000000
000010c00005cec0: 0000000001109320 0000000000000002
000010c00005ced0: 000010c000091e00 000010c00008e000
000010c00005cee0: 00007fade97903d0 000010c00005cf30
000010c00005cef0: 00000000000b4c58 ffffffffff5fd140
000010c00005cf00: 0000000000000005 0000000000000005
000010c00005cf10: 0000000000ff6a40 0000000000000000
000010c00005cf20: 0000000000000000 00000000014c9b60
000010c00005cf30: 000010c00005cf50 000000000064d9f6 <runtime.gopark+230>
000010c00005cf40: 0000000000000000 0000000000000000
000010c00005cf50: 0000000000000000 00000000006610b6 <runtime.sigpanic+1158>
000010c00005cf60: <0000000000000000 >0000000000000000
000010c00005cf70: 0000000000000000 0000000000000000
000010c00005cf80: 0000000000000000 0000000000000000
000010c00005cf90: 0002a16d2b3df782 0000000000000000
000010c00005cfa0: 000010c000091e00 000010c000369ad0
000010c00005cfb0: 0000000000676190 <runtime.gcBgMarkWorker.func2+0> 000010c000091e00
000010c00005cfc0: 0000000000000000 0000000000000000
000010c00005cfd0: 0000000000000000 0000000000000000
000010c00005cfe0: 0000000000000000 0000000000000000
000010c00005cff0: 0000000000000000 0000000000000000
fatal error: unknown caller pc

runtime stack:
runtime.throw(0xbb7338, 0x11)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:774 +0x74 fp=0x7fadecfead88 sp=0x7fadecfead58 pc=0x64bc44
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x10c000091500, 0x0, 0x0, 0x7fffffff, 0x7fadecfeb0f0, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:273 +0x19e9 fp=0x7fadecfeb060 sp=0x7fadecfead88 pc=0x66f9a9
runtime.scanstack(0x10c000091500, 0x10c000031270)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:711 +0x161 fp=0x7fadecfeb258 sp=0x7fadecfeb060 pc=0x63be91
runtime.scang(0x10c000091500, 0x10c000031270)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:886 +0x1e5 fp=0x7fadecfeb2a8 sp=0x7fadecfeb258 pc=0x64f855
runtime.markroot.func1()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:221 +0x71 fp=0x7fadecfeb2e8 sp=0x7fadecfeb2a8 pc=0x6763b1
runtime.markroot(0x10c000031270, 0x7fad0000000d)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:202 +0x2f5 fp=0x7fadecfeb368 sp=0x7fadecfeb2e8 pc=0x63af15
runtime.gcDrain(0x10c000031270, 0x3)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:915 +0x114 fp=0x7fadecfeb3c0 sp=0x7fadecfeb368 pc=0x63c7b4
runtime.gcBgMarkWorker.func2()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1913 +0x82 fp=0x7fadecfeb400 sp=0x7fadecfeb3c0 pc=0x676212
runtime.systemstack(0x679038)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:370 +0x63 fp=0x7fadecfeb408 sp=0x7fadecfeb400 pc=0x679193
runtime.mstart()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:1146 fp=0x7fadecfeb410 sp=0x7fadecfeb408 pc=0x650280

goroutine 83 [GC worker (idle)]:
runtime.systemstack_switch()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:330 fp=0x10c0001a0760 sp=0x10c0001a0758 pc=0x679120
runtime.gcBgMarkWorker(0x10c000030000)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1900 +0x1c5 fp=0x10c0001a07d8 sp=0x10c0001a0760 pc=0x638995
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c0001a07e0 sp=0x10c0001a07d8 pc=0x67b221
created by runtime.gcBgMarkStartWorkers
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1794 +0x79

goroutine 17 [runnable, locked to thread]:
math/big.(*Int).Mul(0x10c000238a38, 0x10c000238a78, 0x10c000238a38, 0x10c000238a18)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/math/big/int.go:158 +0x186 fp=0x10c0002388a0 sp=0x10c000238898 pc=0x73bb56
math/big.lehmerUpdate(0x10c000238a78, 0x10c000238a98, 0x10c000238a58, 0x10c000238a38, 0x10c000238a18, 0x10c0002389f8, 0x1b32fcd, 0x7063b55, 0xe460131, 0x3afaaf5e, ...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/math/big/int.go:598 +0x171 fp=0x10c0002388d8 sp=0x10c0002388a0 pc=0x73d631
math/big.(*Int).lehmerGCD(0x10c000238c18, 0x10c000238bb8, 0x0, 0x10c0004da880, 0x10c0004da860, 0x243000)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/math/big/int.go:672 +0x406 fp=0x10c000238b08 sp=0x10c0002388d8 pc=0x73e0d6
math/big.(*Int).GCD(0x10c000238c18, 0x10c000238bb8, 0x0, 0x10c0004da880, 0x10c0004da860, 0x628a86)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/math/big/int.go:519 +0xe0 fp=0x10c000238b48 sp=0x10c000238b08 pc=0x73d2e0
math/big.(*Int).ModInverse(0x10c0004da980, 0x10c0004da880, 0x10c0004da860, 0x10c0004da960)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/math/big/int.go:773 +0x162 fp=0x10c000238c48 sp=0x10c000238b48 pc=0x73ed62
crypto/rsa.(*PrivateKey).Precompute(0x10c0004bc2a0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/crypto/rsa/rsa.go:442 +0x18f fp=0x10c000238cf8 sp=0x10c000238c48 pc=0x7914cf
crypto/x509.ParsePKCS1PrivateKey(0x10c0004dd500, 0x92c, 0x92c, 0x14c2680, 0x7, 0x7)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/crypto/x509/pkcs1.go:94 +0x2d9 fp=0x10c000238d90 sp=0x10c000238cf8 pc=0x80c5b9
crypto/x509.ParsePKCS8PrivateKey(0x10c0004dc000, 0x946, 0x96c, 0x0, 0x1116e60, 0x10c0004d6490, 0x10c0004de000)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/crypto/x509/pkcs8.go:46 +0x2df fp=0x10c000238e70 sp=0x10c000238d90 pc=0x80ccef
crypto/tls.parsePrivateKey(0x10c0004dc000, 0x946, 0x96c, 0x10c0004f6000, 0x0, 0x0, 0x200)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/crypto/tls/tls.go:298 +0x78 fp=0x10c000238eb8 sp=0x10c000238e70 pc=0x867388
crypto/tls.X509KeyPair(0x10c0004cda65, 0x0, 0x200, 0x10c0004d4cc8, 0x0, 0x200, 0x0, 0x0, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/crypto/tls/tls.go:254 +0x579 fp=0x10c0002395b8 sp=0x10c000238eb8 pc=0x866349
github.com/coreos/etcd/pkg/tlsutil.NewCert(0x10c000168050, 0x43, 0x10c0001680a0, 0x43, 0x11072f0, 0x10c00034d06e, 0x2, 0xffffffffffffffff)
/home/zhangtianyi/XuanYuan/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/pkg/tlsutil/tlsutil.go:67 +0x196 fp=0x10c000239710 sp=0x10c0002395b8 pc=0xb2b2d6
github.com/coreos/etcd/pkg/transport.TLSInfo.baseConfig(0x10c000168050, 0x43, 0x10c0001680a0, 0x43, 0x10c000168000, 0x43, 0x0, 0x0, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/pkg/transport/listener.go:171 +0x182 fp=0x10c000239798 sp=0x10c000239710 pc=0xb2b802
github.com/coreos/etcd/pkg/transport.TLSInfo.ClientConfig(0x10c000168050, 0x43, 0x10c0001680a0, 0x43, 0x10c000168000, 0x43, 0x0, 0x0, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/pkg/transport/listener.go:250 +0x3ae fp=0x10c0002399a8 sp=0x10c000239798 pc=0xb2c1ce
main.newTransport(0x10c0000a04e0, 0x1, 0x10c0000a04e0, 0x67b2e1)
/home/zhangtianyi/XuanYuan/Code/src/cm/etcdapi/go_etcd_api/etcd_api_client.go:643 +0xe3 fp=0x10c000239b30 sp=0x10c0002399a8 pc=0xb42e73
main.newClient(0x10c000000001, 0x10c000239c01, 0x8, 0x10c000239c20, 0x10c000239c88)
/home/zhangtianyi/XuanYuan/Code/src/cm/etcdapi/go_etcd_api/etcd_api_client.go:691 +0x6f fp=0x10c000239c38 sp=0x10c000239b30 pc=0xb4328f
main.newKeyAPI(0x1, 0x10c00033c003, 0x10c0000a04e0, 0x10c0000d4f00, 0x1f)
/home/zhangtianyi/XuanYuan/Code/src/cm/etcdapi/go_etcd_api/etcd_api_client.go:717 +0x2f fp=0x10c000239c98 sp=0x10c000239c38 pc=0xb434af
main.etcd_get_client(0x10c000000001, 0x7fade631ddd0, 0x7fade631dc39, 0x7fade631dc48, 0x7fade631dc50, 0x0)
/home/zhangtianyi/XuanYuan/Code/src/cm/etcdapi/go_etcd_api/etcd_api_client.go:268 +0x115 fp=0x10c000239de0 sp=0x10c000239c98 pc=0xb3fc75
main._cgoexpwrap_58851cf6732e_etcd_get_client(0x1, 0x7fade631ddd0, 0x7fade631dc39, 0x7fade631dc48, 0x7fade631dc50, 0x7fade631e1f0)
_cgo_gotypes.go:158 +0x53 fp=0x10c000239e20 sp=0x10c000239de0 pc=0xb3eaf3
runtime.call64(0x0, 0x7fade631db10, 0x7fade631dba0, 0x30)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:540 +0x3d fp=0x10c000239e70 sp=0x10c000239e20 pc=0x67953d
runtime.cgocallbackg1(0x0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/cgocall.go:314 +0x1bb fp=0x10c000239f58 sp=0x10c000239e70 pc=0x61f82b
runtime.cgocallbackg(0x0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/cgocall.go:191 +0xc7 fp=0x10c000239fc0 sp=0x10c000239f58 pc=0x61f5d7
runtime.cgocallback_gofunc(0x0, 0x0, 0x0, 0x0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:793 +0x9a fp=0x10c000239fe0 sp=0x10c000239fc0 pc=0x67aaca
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c000239fe8 sp=0x10c000239fe0 pc=0x67b221

goroutine 2 [force gc (idle)]:
runtime.gopark(0x1109488, 0x2e26800, 0x1411, 0x1)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:304 +0xe6 fp=0x10c00005efb0 sp=0x10c00005ef90 pc=0x64d9f6
runtime.goparkunlock(...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:310
runtime.forcegchelper()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:253 +0xbb fp=0x10c00005efe0 sp=0x10c00005efb0 pc=0x64d89b
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c00005efe8 sp=0x10c00005efe0 pc=0x67b221
created by runtime.init.5
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:242 +0x37

goroutine 18 [GC sweep wait]:
runtime.gopark(0x1109488, 0x2e269a0, 0x140c, 0x1)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:304 +0xe6 fp=0x10c00005a7a8 sp=0x10c00005a788 pc=0x64d9f6
runtime.goparkunlock(...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:310
runtime.bgsweep(0x10c00009c000)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcsweep.go:89 +0x135 fp=0x10c00005a7d8 sp=0x10c00005a7a8 pc=0x63f0f5
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c00005a7e0 sp=0x10c00005a7d8 pc=0x67b221
created by runtime.gcenable
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:210 +0x5e

goroutine 19 [GC scavenge wait]:
runtime.gopark(0x1109488, 0x2e26ec0, 0x140d, 0x1)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:304 +0xe6 fp=0x10c00005af40 sp=0x10c00005af20 pc=0x64d9f6
runtime.goparkunlock(...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:310
runtime.bgscavenge(0x10c00009c000)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcscavenge.go:374 +0x3b7 fp=0x10c00005afd8 sp=0x10c00005af40 pc=0x63e9b7
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c00005afe0 sp=0x10c00005afd8 pc=0x67b221
created by runtime.gcenable
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:211 +0x80

goroutine 20 [finalizer wait]:
runtime.gopark(0x1109488, 0x2e46628, 0x1410, 0x1)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:304 +0xe6 fp=0x10c00005e758 sp=0x10c00005e738 pc=0x64d9f6
runtime.goparkunlock(...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:310
runtime.runfinq()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mfinal.go:175 +0xa7 fp=0x10c00005e7e0 sp=0x10c00005e758 pc=0x634e87
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c00005e7e8 sp=0x10c00005e7e0 pc=0x67b221
created by runtime.createfing
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mfinal.go:156 +0x63

goroutine 13 [GC worker (idle)]:
runtime.gopark(0x1109320, 0x10c000026f60, 0x1418, 0x0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:304 +0xe6 fp=0x10c00005b760 sp=0x10c00005b740 pc=0x64d9f6
runtime.gcBgMarkWorker(0x10c000052f00)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1846 +0x105 fp=0x10c00005b7d8 sp=0x10c00005b760 pc=0x6388d5
runtime.goexit()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:1357 +0x1 fp=0x10c00005b7e0 sp=0x10c00005b7d8 pc=0x67b221
created by runtime.gcBgMarkStartWorkers
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1794 +0x79

goroutine 22 [timer goroutine (idle) (scan)]:
fatal error: unexpected signal during runtime execution
panic during panic
[signal SIGSEGV: segmentation violation code=0x1 addr=0x118 pc=0x66f2ca]

runtime stack:
runtime.throw(0xbc3c9c, 0x2a)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:774 +0x74 fp=0x7fadecfea720 sp=0x7fadecfea6f0 pc=0x64bc44
runtime.sigpanic()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/signal_unix.go:378 +0x480 fp=0x7fadecfea750 sp=0x7fadecfea720 pc=0x6610b0
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x10c000091500, 0x0, 0x0, 0x64, 0x0, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:261 +0x130a fp=0x7fadecfeaa28 sp=0x7fadecfea750 pc=0x66f2ca
runtime.traceback1(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x10c000091500, 0x0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:722 +0xf2 fp=0x7fadecfeabd0 sp=0x7fadecfeaa28 pc=0x6702f2
runtime.traceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x10c000091500)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:676 +0x54 fp=0x7fadecfeac08 sp=0x7fadecfeabd0 pc=0x670124
runtime.tracebackothers(0x2e278c0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:929 +0x1ac fp=0x7fadecfeac70 sp=0x7fadecfeac08 pc=0x67132c
runtime.dopanic_m(0x2e278c0, 0x64bc44, 0x7fadecfead58, 0x1)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:974 +0x2a4 fp=0x7fadecfeacd8 sp=0x7fadecfeac70 pc=0x64c334
runtime.fatalthrow.func1()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:829 +0x61 fp=0x7fadecfead20 sp=0x7fadecfeacd8 pc=0x677d01
runtime.fatalthrow()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:826 +0x59 fp=0x7fadecfead58 sp=0x7fadecfead20 pc=0x64be19
runtime.throw(0xbb7338, 0x11)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:774 +0x74 fp=0x7fadecfead88 sp=0x7fadecfead58 pc=0x64bc44
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x10c000091500, 0x0, 0x0, 0x7fffffff, 0x7fadecfeb0f0, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:273 +0x19e9 fp=0x7fadecfeb060 sp=0x7fadecfead88 pc=0x66f9a9
runtime.scanstack(0x10c000091500, 0x10c000031270)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:711 +0x161 fp=0x7fadecfeb258 sp=0x7fadecfeb060 pc=0x63be91
runtime.scang(0x10c000091500, 0x10c000031270)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:886 +0x1e5 fp=0x7fadecfeb2a8 sp=0x7fadecfeb258 pc=0x64f855
runtime.markroot.func1()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:221 +0x71 fp=0x7fadecfeb2e8 sp=0x7fadecfeb2a8 pc=0x6763b1
runtime.markroot(0x10c000031270, 0x7fad0000000d)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:202 +0x2f5 fp=0x7fadecfeb368 sp=0x7fadecfeb2e8 pc=0x63af15
runtime.gcDrain(0x10c000031270, 0x3)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:915 +0x114 fp=0x7fadecfeb3c0 sp=0x7fadecfeb368 pc=0x63c7b4
runtime.gcBgMarkWorker.func2()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1913 +0x82 fp=0x7fadecfeb400 sp=0x7fadecfeb3c0 pc=0x676212
runtime.systemstack(0x679038)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:370 +0x63 fp=0x7fadecfeb408 sp=0x7fadecfeb400 pc=0x679193
runtime.mstart()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:1146 fp=0x7fadecfeb410 sp=0x7fadecfeb408 pc=0x650280

runtime stack:
runtime.throw(0xbb7338, 0x11)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/panic.go:774 +0x74 fp=0x7fade5a1bd60 sp=0x7fade5a1bd30 pc=0x64bc44
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0x10c000091e00, 0x0, 0x0, 0x7fffffff, 0x7fade5a1c0c8, 0x0, 0x0, ...)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/traceback.go:273 +0x19e9 fp=0x7fade5a1c038 sp=0x7fade5a1bd60 pc=0x66f9a9
runtime.scanstack(0x10c000091e00, 0x10c000035c70)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:711 +0x161 fp=0x7fade5a1c230 sp=0x7fade5a1c038 pc=0x63be91
runtime.scang(0x10c000091e00, 0x10c000035c70)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:886 +0x1e5 fp=0x7fade5a1c280 sp=0x7fade5a1c230 pc=0x64f855
runtime.markroot.func1()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:221 +0x71 fp=0x7fade5a1c2c0 sp=0x7fade5a1c280 pc=0x6763b1
runtime.markroot(0x10c000035c70, 0x7fad00000012)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:202 +0x2f5 fp=0x7fade5a1c340 sp=0x7fade5a1c2c0 pc=0x63af15
runtime.gcDrain(0x10c000035c70, 0x7)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgcmark.go:915 +0x114 fp=0x7fade5a1c398 sp=0x7fade5a1c340 pc=0x63c7b4
runtime.gcBgMarkWorker.func2()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/mgc.go:1936 +0x16d fp=0x7fade5a1c3d8 sp=0x7fade5a1c398 pc=0x6762fd
runtime.systemstack(0x7fade5a1d9c0)
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/asm_amd64.s:370 +0x63 fp=0x7fade5a1c3e0 sp=0x7fade5a1c3d8 pc=0x679193
runtime.mstart()
/home/zhangtianyi/XuanYuan/buildtools/euleros2.0_sp2_x86_64/go/src/runtime/proc.go:1146 fp=0x7fade5a1c3e8 sp=0x7fade5a1c3e0 pc=0x650280

@toothrot
Copy link
Contributor

@toothrot toothrot commented Jan 7, 2020

@thundercw Is there any way to see the code that causes this failure? Does it fail consistently?

If you run it with GOTRACEBACK=crash, it will generate more information about the stack as well as a core dump, which may be useful for your debugging.

@thundercw
Copy link
Author

@thundercw thundercw commented Jan 21, 2020

@toothrot I'm sorry, The code can't be see out of the company, but I have located the panic reason. In the C code defines the SIGCHID function, SIGCHID is triggered very frequent, and lead the go code panic. I tested the other signals, such as SIGALRM, will also trigger a panic.
Signal processing function defined in the C code is unavoidable, how should I write the CGO code to avoid this problem?

In addition what is the real reason of this problem?

Thanks!

@josharian
Copy link
Contributor

@josharian josharian commented Jan 21, 2020

Could this be #35777?

@josharian
Copy link
Contributor

@josharian josharian commented Jan 21, 2020

(If so, things to try would be using Go 1.14 beta 1, and upgrading your kernel version to one of the known fixed versions.)

@thundercw
Copy link
Author

@thundercw thundercw commented Jan 29, 2020

@josharian
The linux kernel I used is "Linux version 3.10.0-327.36.58.4.x86_64";
The issue is reproducible: CGO program with user defined signal handler in C code and trigger it very frequently.

@thundercw
Copy link
Author

@thundercw thundercw commented Feb 4, 2020

I used Go 1.14 beta 1 for test,it also paniced.

runtime: unexpected return pc for runtime.gopark called from 0x0
stack: frame={sp:0xc000061788, fp:0xc0000617a8} stack=[0xc000061000,0xc000061800)
000000c000061688:  00007f08a4261b9e <runtime.goready+94>  0000034446505853
000000c000061698:  0000000000000007  0000000000000340
000000c0000616a8:  0000000000000000  0000000000000000
000000c0000616b8:  0000000000000000  0000000000000003
000000c0000616c8:  0000000000000000  0000000000000000
000000c0000616d8:  0000000000000000  0000000000000000
000000c0000616e8:  0000000000000000  0000000000000000
000000c0000616f8:  0000000000000000  0000000000000000
000000c000061708:  0000000000000000  0000000000000000
000000c000061718:  0000000000000000  0000000000000000
000000c000061728:  0000000000000000  0000000000000000
000000c000061738:  0000000000000000  0000000000000000
000000c000061748:  0000000000000000  0000000000000000
000000c000061758:  0000000000000000  0000000000000000
000000c000061768:  0000000000000000  0000000000000000
000000c000061778:  0000000000000000  0000000000000000
000000c000061788: <0000000000000000  0000000000000000
000000c000061798:  0000000000000000 !0000000000000000
000000c0000617a8: >0000000000000000  0000000000000000
000000c0000617b8:  0000000000000000  0000000000000000
000000c0000617c8:  0000000000000000  0000000000000000
000000c0000617d8:  0000000000000000  0000000000000000
000000c0000617e8:  0000000000000000  0000000000000000
000000c0000617f8:  0000000000000000
fatal error: unknown caller pc

runtime stack:
runtime.throw(0x7f08a472d0e5, 0x11)
        /home/cw/local/go/src/runtime/panic.go:1112 +0x74
runtime.gentraceback(0xffffffffffffffff, 0xffffffffffffffff, 0x0, 0xc000000f00, 0x0, 0x0, 0x7fffffff, 0x7f087b42ebb8, 0x0, 0x0, ...)
        /home/cw/local/go/src/runtime/traceback.go:273 +0x19ff
runtime.scanstack(0xc000000f00, 0xc000035690)
        /home/cw/local/go/src/runtime/mgcmark.go:739 +0x162
runtime.markroot.func1()
        /home/cw/local/go/src/runtime/mgcmark.go:226 +0xc3
runtime.markroot(0xc000035690, 0x8)
        /home/cw/local/go/src/runtime/mgcmark.go:199 +0x2f5
runtime.gcDrain(0xc000035690, 0x7)
        /home/cw/local/go/src/runtime/mgcmark.go:999 +0x10b
runtime.gcBgMarkWorker.func2()
        /home/cw/local/go/src/runtime/mgc.go:1970 +0x16b
runtime.systemstack(0x0)
        /home/cw/local/go/src/runtime/asm_amd64.s:370 +0x63
runtime.mstart()
        /home/cw/local/go/src/runtime/proc.go:1077

goroutine 77 [GC worker (idle)]:
runtime.systemstack_switch()
        /home/cw/local/go/src/runtime/asm_amd64.s:330 fp=0xc000416f60 sp=0xc000416f58 pc=0x7f08a428f5d0
runtime.gcBgMarkWorker(0xc000034000)
        /home/cw/local/go/src/runtime/mgc.go:1934 +0x1c6 fp=0xc000416fd8 sp=0xc000416f60 pc=0x7f08a4246da6
runtime.goexit()
        /home/cw/local/go/src/runtime/asm_amd64.s:1375 +0x1 fp=0xc000416fe0 sp=0xc000416fd8 pc=0x7f08a4291711
created by runtime.gcBgMarkStartWorkers
        /home/cw/local/go/src/runtime/mgc.go:1828 +0x79

goroutine 17 [select, locked to thread]:
github.com/coreos/etcd/client.(*simpleHTTPClient).Do(0xc0001ea140, 0x7f08a4b587e0, 0xc00031b5c0, 0x7f08a4b4be60, 0x7f08a4eff330, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /home/cw/gauss/etcd_test3/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/client/client.go:564 +0x2fa
github.com/coreos/etcd/client.(*redirectFollowingHTTPClient).Do(0xc000521120, 0x7f08a4b587e0, 0xc00031b5c0, 0x7f08a4b4be60, 0x7f08a4eff330, 0xc0001a60a8, 0xf, 0x0, 0x0, 0x0, ...)
        /home/cw/gauss/etcd_test3/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/client/client.go:636 +0xaf
github.com/coreos/etcd/client.(*httpClusterClient).Do(0xc00031b560, 0x7f08a4b587e0, 0xc00031b5c0, 0x7f08a4b4be60, 0x7f08a4eff330, 0x7f08a4b3e158, 0xc000307c78, 0x7f08a42ba9bd, 0x7f08a4b587a0, 0xc0000260c0, ...)
        /home/cw/gauss/etcd_test3/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/client/client.go:365 +0x2ea
github.com/coreos/etcd/client.(*httpMembersAPI).List(0xc000410a60, 0x7f08a4b587e0, 0xc00031b5c0, 0x7f08a4b587e0, 0xc00031b5c0, 0xc000410a80, 0xc000307cf0, 0x7f08a42ba880)
        /home/cw/gauss/etcd_test3/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/client/members.go:119 +0x7c
main.etcd_cluster_state_client(0x7f0800000001, 0x7f0878be9ea0, 0x7f0878be9e9f, 0x7f0878be9e40, 0x0)
        /home/cw/gauss/etcd_test3/etcdapi/go_etcd_api/etcd_api_client.go:615 +0x211
main._cgoexpwrap_bea9c23aea74_etcd_cluster_state_client(0x1, 0x7f0878be9ea0, 0x7f0878be9e9f, 0x7f0878be9e40, 0x0)
        _cgo_gotypes.go:222 +0x49

goroutine 775 [runnable]:
net/http.(*Transport).dialConnFor(0xc0001c4280, 0xc00055c210)
        /home/cw/local/go/src/net/http/transport.go:1354
created by net/http.(*Transport).queueForDial
        /home/cw/local/go/src/net/http/transport.go:1326 +0x400

goroutine 771 [select]:
net/http.(*Transport).getConn(0xc0001c4280, 0xc00009ff20, 0x0, 0xc0001be640, 0x5, 0xc0000272b0, 0xf, 0x0, 0x0, 0x0, ...)
        /home/cw/local/go/src/net/http/transport.go:1283 +0x57d
net/http.(*Transport).roundTrip(0xc0001c4280, 0xc000556000, 0x7f08a470871a, 0x7f08a4b3d701, 0xc0007207b0)
        /home/cw/local/go/src/net/http/transport.go:544 +0x72a
net/http.(*Transport).RoundTrip(0xc0001c4280, 0xc000556000, 0x7f08a4b3deb0, 0x63, 0x200)
        /home/cw/local/go/src/net/http/roundtrip.go:17 +0x37
github.com/coreos/etcd/client.(*simpleHTTPClient).Do.func1(0xc0001ea140, 0xc000556000, 0xc00031b620)
        /home/cw/gauss/etcd_test3/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/client/client.go:556 +0x43
created by github.com/coreos/etcd/client.(*simpleHTTPClient).Do
        /home/cw/gauss/etcd_test3/open_source/etcd/etcd-3.3.6/cmd/vendor/src/github.com/coreos/etcd/client/client.go:555 +0x216

@thundercw
Copy link
Author

@thundercw thundercw commented Feb 29, 2020

@toothrot @josharian I simplified the code, and use go 1.13.3 to reproduce the problem. Increasing the frequency of the semaphore trigger increases the frequency of the problem. Please help me to find out the problem, thanks.

go version

go version go1.13.3 linux/amd64

go env

GO111MODULE=""
GOARCH="amd64"
GOBIN=""
GOCACHE="/home/cw/.cache/go-build"
GOENV="/home/cw/.config/go/env"
GOEXE=""
GOFLAGS=""
GOHOSTARCH="amd64"
GOHOSTOS="linux"
GONOPROXY=""
GONOSUMDB=""
GOOS="linux"
GOPATH="/home/cw/gpath"
GOPRIVATE=""
GOPROXY="https://proxy.golang.org,direct"
GOROOT="/home/cw/local/go"
GOSUMDB="sum.golang.org"
GOTMPDIR=""
GOTOOLDIR="/home/cw/local/go/pkg/tool/linux_amd64"
GCCGO="/usr/bin/gccgo"
AR="ar"
CC="/home/cw/local/gcc8.2/gcc/bin/gcc"
CXX="/home/cw/local/gcc8.2/gcc/bin/g++"
CGO_ENABLED="1"
GOMOD=""
CGO_CFLAGS="-g -O2"
CGO_CPPFLAGS=""
CGO_CXXFLAGS="-g -O2"
CGO_FFLAGS="-g -O2"
CGO_LDFLAGS="-g -O2"
PKG_CONFIG="pkg-config"
GOGCCFLAGS="-fPIC -m64 -pthread -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build950978664=/tmp/go-build -gno-record-gcc-switches"

linux kernel

cat /proc/version
Linux version 3.10.0-327.36.58.4.x86_64

the cgo code

package main

import "C"
import (
        "time"
)

//export cgo_open
func cgo_open(n *C.char) int {
        C.GoString(n)
        time.Sleep(200 * time.Millisecond)
        return 0
}

//export cgo_close
func cgo_close(id int) *C.char {
        time.Sleep(300 * time.Millisecond)
        return C.CString("n")
}

func main() {
}

C Code

#include <stdlib.h>
#include <stdio.h>
#include <unistd.h>
#include <pthread.h>
#include <signal.h>
#include "cgo/libcgo.h"

pid_t pid = 0;
void handler(int arg){}

static void * cgo_thread(void *arg)
{
        while(true) {
        int id = cgo_open("for test******************************************");
        char* name = cgo_close(id);
        //printf("%s\n",name);
        free(name);
                //usleep(1000);
        }
        return 0;
}

static void * signal_thread(void *arg)
{
        while(true) {
                int ret = kill(pid, SIGALRM);
                ret = kill(pid, SIGCHLD);
                usleep(1000);
        }
        return 0;
}

#define ThreadNum 100
int main() {

    (void)signal(SIGALRM, handler);
        (void)signal(SIGCHLD, handler);

        pid = getpid();
        int err;
        pthread_t  thr_id[ThreadNum] = { 0 };
        int i = 0;
        for(; i < 10; ++i) {
      if ((err = pthread_create(thr_id + i, NULL, cgo_thread, NULL) != 0)) {
        printf("pthread_create err: %d\n",err);
      }
    }

        for(; i < ThreadNum; ++i) {
                if ((err = pthread_create(thr_id + i, NULL, signal_thread, NULL) != 0)) {
          printf("pthread_create err: %d\n",err);
                }
        }

        void *res = NULL;
        for (i = 0; i < ThreadNum; ++i) {
                if (thr_id[i] != 0) {
                        int ret = pthread_join(thr_id[i], &res);
                }
        }

        return 0;
}

Call Stack

panic.txt

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
Projects
None yet
Linked pull requests

Successfully merging a pull request may close this issue.

None yet
3 participants
You can’t perform that action at this time.