-
Notifications
You must be signed in to change notification settings - Fork 18.4k
Closed
Labels
FrozenDueToAgeNeedsFixThe path to resolution is known, but the work has not been done.The path to resolution is known, but the work has not been done.release-blocker
Milestone
Description
What version of Go are you using (go version
)?
$ go version go version go1.12.5 linux/amd6
Does this issue reproduce with the latest release?
Yes, also tested with
$ go1.13rc1 version
go version go1.13rc1 linux/amd64
What operating system and processor architecture are you using (go env
)?
go env
Output
$ go env GOARCH="amd64" GOBIN="" GOCACHE="/usr/local/google/home/howardjohn/.cache/go-build" GOEXE="" GOFLAGS="" GOHOSTARCH="amd64" GOHOSTOS="linux" GOOS="linux" GOPATH="/usr/local/google/home/howardjohn/go" GOPROXY="" GORACE="" GOROOT="/usr/lib/google-golang" GOTMPDIR="" GOTOOLDIR="/usr/lib/google-golang/pkg/tool/linux_amd64" GCCGO="gccgo" CC="clang" CXX="clang++" CGO_ENABLED="1" GOMOD="" CGO_CFLAGS="-g -O2" CGO_CPPFLAGS="" CGO_CXXFLAGS="-g -O2" CGO_FFLAGS="-g -O2" CGO_LDFLAGS="-g -O2" PKG_CONFIG="pkg-config" GOGCCFLAGS="-fPIC -m64 -pthread -fno-caret-diagnostics -Qunused-arguments -fmessage-length=0 -fdebug-prefix-map=/tmp/go-build754955383=/tmp/go-build -gno-record-gcc-switches"
What did you do?
First, captured a heap profile of an application. The profile is pretty large, 94kb (I think that is large at least...). Then ran go tool pprof --http
and went to the source tab. Website displays ERR_EMPTY_RESPONSE
and I see crashes in the pprof tool.
Crash Logs
With Go 1.12
$ go tool pprof -http=:7777 /usr/local/google/home/howardjohn/pprof/pprof.pilot-discovery.alloc_objects.alloc_space.inuse_objects.inuse_space.033.pb.gz
[14676:14698:0823/082909.884994:ERROR:browser_process_sub_thread.cc(203)] Waited 8 ms for network service
2019/08/23 08:29:19 http: panic serving 127.0.0.1:57902: runtime error: index out of range
goroutine 55 [running]:
net/http.(*conn).serve.func1(0xc000c381e0)
/usr/lib/google-golang/src/net/http/server.go:1769 +0x139
panic(0xa3e520, 0xfd73d0)
/usr/lib/google-golang/src/runtime/panic.go:522 +0x1b5
cmd/vendor/golang.org/x/arch/x86/x86asm.decode1(0xc0092f8894, 0x2, 0x40a388, 0x40, 0x59000000, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/usr/lib/google-golang/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:426 +0x6bfe
cmd/vendor/golang.org/x/arch/x86/x86asm.Decode(...)
/usr/lib/google-golang/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:226
cmd/internal/objfile.disasm_x86(0xc0092f8894, 0x2, 0x40a388, 0x16f9894, 0xc003efeb90, 0x40, 0xc0011007e0, 0x64, 0x50)
/usr/lib/google-golang/src/cmd/internal/objfile/disasm.go:300 +0x92
cmd/internal/objfile.disasm_amd64(0xc0092f8894, 0x2, 0x40a388, 0x16f9894, 0xc003efeb90, 0xbdd420, 0x1041bd8, 0xc001ea2900, 0x7, 0x1)
/usr/lib/google-golang/src/cmd/internal/objfile/disasm.go:296 +0x5c
cmd/internal/objfile.(*Disasm).Decode(0xc000b95200, 0x16e5cf0, 0x16f9896, 0x0, 0x0, 0x0, 0xc0004d2f78)
/usr/lib/google-golang/src/cmd/internal/objfile/disasm.go:275 +0x377
main.(*objTool).Disasm(0xc000097800, 0xc000c1bb80, 0x4c, 0x16e5cf0, 0x16f9896, 0x203000, 0x537a45, 0xc00189e6c0, 0x7f1bdc1900b4, 0xad1510)
/usr/lib/google-golang/src/cmd/pprof/pprof.go:180 +0xf2
cmd/vendor/github.com/google/pprof/driver.(*internalObjTool).Disasm(0xc000097810, 0xc000c1bb80, 0x4c, 0x16e5cf0, 0x16f9896, 0x1, 0xc002761360, 0x6, 0xc0004d3198, 0x5e8262)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/driver/driver.go:280 +0x7f
cmd/vendor/github.com/google/pprof/internal/report.assemblyPerSourceLine(0xc002952000, 0x1434a, 0x17800, 0xc00154ae70, 0x1, 0x1, 0xc00066f270, 0x4e, 0xbd0ce0, 0xc000097810, ...)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:251 +0xec
cmd/vendor/github.com/google/pprof/internal/report.PrintWebList(0xbcba20, 0xc001150300, 0xc0008f21e0, 0xbd0ce0, 0xc000097810, 0x32, 0x2, 0x0)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:208 +0xaa3
cmd/vendor/github.com/google/pprof/internal/driver.(*webInterface).source(0xc000bea900, 0xbd7160, 0xc000938000, 0xc0001eb100)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:386 +0x18c
net/http.HandlerFunc.ServeHTTP(0xc000bf2940, 0xbd7160, 0xc000938000, 0xc0001eb100)
/usr/lib/google-golang/src/net/http/server.go:1995 +0x44
cmd/vendor/github.com/google/pprof/internal/driver.defaultWebServer.func1(0xbd7160, 0xc000938000, 0xc0001eb100)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:173 +0x9c
net/http.HandlerFunc.ServeHTTP(0xc000beab20, 0xbd7160, 0xc000938000, 0xc0001eb100)
/usr/lib/google-golang/src/net/http/server.go:1995 +0x44
net/http.StripPrefix.func1(0xbd7160, 0xc000938000, 0xc0001eb000)
/usr/lib/google-golang/src/net/http/server.go:2034 +0x18d
net/http.HandlerFunc.ServeHTTP(0xc000c1d740, 0xbd7160, 0xc000938000, 0xc0001eb000)
/usr/lib/google-golang/src/net/http/server.go:1995 +0x44
net/http.(*ServeMux).ServeHTTP(0xc000bef500, 0xbd7160, 0xc000938000, 0xc0001eb000)
/usr/lib/google-golang/src/net/http/server.go:2375 +0x1d6
net/http.serverHandler.ServeHTTP(0xc000bec750, 0xbd7160, 0xc000938000, 0xc0001eb000)
/usr/lib/google-golang/src/net/http/server.go:2787 +0xa8
net/http.(*conn).serve(0xc000c381e0, 0xbd86e0, 0xc000bef5c0)
/usr/lib/google-golang/src/net/http/server.go:1878 +0x851
created by net/http.(*Server).Serve
/usr/lib/google-golang/src/net/http/server.go:2906 +0x31e
2019/08/23 08:29:26 http: panic serving 127.0.0.1:57922: runtime error: index out of range
goroutine 57 [running]:
net/http.(*conn).serve.func1(0xc0017175e0)
/usr/lib/google-golang/src/net/http/server.go:1769 +0x139
panic(0xa3e520, 0xfd73d0)
/usr/lib/google-golang/src/runtime/panic.go:522 +0x1b5
cmd/vendor/golang.org/x/arch/x86/x86asm.decode1(0xc0092f8894, 0x2, 0x40a388, 0x40, 0x59000000, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/usr/lib/google-golang/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:426 +0x6bfe
cmd/vendor/golang.org/x/arch/x86/x86asm.Decode(...)
/usr/lib/google-golang/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:226
cmd/internal/objfile.disasm_x86(0xc0092f8894, 0x2, 0x40a388, 0x16f9894, 0xc000096a90, 0x40, 0xc0011007e0, 0x64, 0x50)
/usr/lib/google-golang/src/cmd/internal/objfile/disasm.go:300 +0x92
cmd/internal/objfile.disasm_amd64(0xc0092f8894, 0x2, 0x40a388, 0x16f9894, 0xc000096a90, 0xbdd420, 0x1041bd8, 0xc005588130, 0x7, 0x1)
/usr/lib/google-golang/src/cmd/internal/objfile/disasm.go:296 +0x5c
cmd/internal/objfile.(*Disasm).Decode(0xc000b95200, 0x16e5cf0, 0x16f9896, 0x0, 0x0, 0x0, 0xc0004cef78)
/usr/lib/google-golang/src/cmd/internal/objfile/disasm.go:275 +0x377
main.(*objTool).Disasm(0xc000097800, 0xc00fd73270, 0x4c, 0x16e5cf0, 0x16f9896, 0x203000, 0x537a45, 0xc000f25500, 0x7f1bded2c0e6, 0xad1501)
/usr/lib/google-golang/src/cmd/pprof/pprof.go:180 +0xf2
cmd/vendor/github.com/google/pprof/driver.(*internalObjTool).Disasm(0xc000097810, 0xc00fd73270, 0x4c, 0x16e5cf0, 0x16f9896, 0x1, 0xc03fe84560, 0x6, 0xc0004cf198, 0x5e8262)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/driver/driver.go:280 +0x7f
cmd/vendor/github.com/google/pprof/internal/report.assemblyPerSourceLine(0xc043a30000, 0x1434a, 0x17800, 0xc043b2e388, 0x1, 0x1, 0xc00fe34960, 0x4e, 0xbd0ce0, 0xc000097810, ...)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:251 +0xec
cmd/vendor/github.com/google/pprof/internal/report.PrintWebList(0xbcba20, 0xc013a508a0, 0xc00fe1df40, 0xbd0ce0, 0xc000097810, 0x32, 0x2, 0x0)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:208 +0xaa3
cmd/vendor/github.com/google/pprof/internal/driver.(*webInterface).source(0xc000bea900, 0xbd7160, 0xc00050a000, 0xc000b14300)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:386 +0x18c
net/http.HandlerFunc.ServeHTTP(0xc000bf2940, 0xbd7160, 0xc00050a000, 0xc000b14300)
/usr/lib/google-golang/src/net/http/server.go:1995 +0x44
cmd/vendor/github.com/google/pprof/internal/driver.defaultWebServer.func1(0xbd7160, 0xc00050a000, 0xc000b14300)
/usr/lib/google-golang/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:173 +0x9c
net/http.HandlerFunc.ServeHTTP(0xc000beab20, 0xbd7160, 0xc00050a000, 0xc000b14300)
/usr/lib/google-golang/src/net/http/server.go:1995 +0x44
net/http.StripPrefix.func1(0xbd7160, 0xc00050a000, 0xc000b14200)
/usr/lib/google-golang/src/net/http/server.go:2034 +0x18d
net/http.HandlerFunc.ServeHTTP(0xc000c1d740, 0xbd7160, 0xc00050a000, 0xc000b14200)
/usr/lib/google-golang/src/net/http/server.go:1995 +0x44
net/http.(*ServeMux).ServeHTTP(0xc000bef500, 0xbd7160, 0xc00050a000, 0xc000b14200)
/usr/lib/google-golang/src/net/http/server.go:2375 +0x1d6
net/http.serverHandler.ServeHTTP(0xc000bec750, 0xbd7160, 0xc00050a000, 0xc000b14200)
/usr/lib/google-golang/src/net/http/server.go:2787 +0xa8
net/http.(*conn).serve(0xc0017175e0, 0xbd86e0, 0xc001728080)
/usr/lib/google-golang/src/net/http/server.go:1878 +0x851
created by net/http.(*Server).Serve
/usr/lib/google-golang/src/net/http/server.go:2906 +0x31e
With go 1.13 rc1:
$ go1.13rc1 tool pprof -http=:7777 /usr/local/google/home/howardjohn/pprof/pprof.pilot-discovery.alloc_objects.alloc_space.inuse_objects.inuse_space.033.pb.gz
Serving web UI on http://localhost:7777
[38354:38371:0823/083852.889258:ERROR:browser_process_sub_thread.cc(203)] Waited 3 ms for network service
2019/08/23 08:39:05 http: panic serving 127.0.0.1:33616: runtime error: index out of range [2] with length 2
goroutine 50 [running]:
net/http.(*conn).serve.func1(0xc0009c8000)
/usr/local/go/src/net/http/server.go:1767 +0x139
panic(0x9921a0, 0xc000d490c0)
/usr/local/go/src/runtime/panic.go:679 +0x1b2
cmd/vendor/golang.org/x/arch/x86/x86asm.decode1(0xc008c42894, 0x2, 0x40a388, 0x40, 0x59000000, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:426 +0x6fa0
cmd/vendor/golang.org/x/arch/x86/x86asm.Decode(...)
/usr/local/go/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:226
cmd/internal/objfile.disasm_x86(0xc008c42894, 0x2, 0x40a388, 0x16f9894, 0xc0025decf0, 0x40, 0xc0005987e0, 0x64, 0x50)
/usr/local/go/src/cmd/internal/objfile/disasm.go:300 +0x91
cmd/internal/objfile.disasm_amd64(0xc008c42894, 0x2, 0x40a388, 0x16f9894, 0xc0025decf0, 0xadeb80, 0xe9d5c8, 0xc001b31ea0, 0x7, 0x1)
/usr/local/go/src/cmd/internal/objfile/disasm.go:296 +0x5c
cmd/internal/objfile.(*Disasm).Decode(0xc00167c080, 0x16e5cf0, 0x16f9896, 0x0, 0x0, 0x0, 0xc00068eee0)
/usr/local/go/src/cmd/internal/objfile/disasm.go:275 +0x39a
main.(*objTool).Disasm(0xc000061840, 0xc0006c75e0, 0x4c, 0x16e5cf0, 0x16f9896, 0x203000, 0x4227c5, 0xc0028c0840, 0x7fd7243bc065, 0x10)
/usr/local/go/src/cmd/pprof/pprof.go:180 +0xf2
cmd/vendor/github.com/google/pprof/driver.(*internalObjTool).Disasm(0xc000061850, 0xc0006c75e0, 0x4c, 0x16e5cf0, 0x16f9896, 0x1, 0xc002c4d340, 0x6, 0xc00068f100, 0x4e6ca1)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/driver/driver.go:280 +0x7f
cmd/vendor/github.com/google/pprof/internal/report.assemblyPerSourceLine(0xc002e3e000, 0x1434a, 0x17800, 0xc000bd2900, 0x1, 0x1, 0xc000feacd0, 0x4e, 0xad2900, 0xc000061850, ...)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:251 +0xec
cmd/vendor/github.com/google/pprof/internal/report.PrintWebList(0xaceb80, 0xc001260120, 0xc000443be0, 0xad2900, 0xc000061850, 0x32, 0x2, 0x0)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:208 +0xaa0
cmd/vendor/github.com/google/pprof/internal/driver.(*webInterface).source(0xc000326240, 0xad8b60, 0xc0007cc000, 0xc0009d8000)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:397 +0x18b
net/http.HandlerFunc.ServeHTTP(0xc0004e2240, 0xad8b60, 0xc0007cc000, 0xc0009d8000)
/usr/local/go/src/net/http/server.go:2007 +0x44
cmd/vendor/github.com/google/pprof/internal/driver.defaultWebServer.func1(0xad8b60, 0xc0007cc000, 0xc0009d8000)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:177 +0x9c
net/http.HandlerFunc.ServeHTTP(0xc0003264c0, 0xad8b60, 0xc0007cc000, 0xc0009d8000)
/usr/local/go/src/net/http/server.go:2007 +0x44
net/http.StripPrefix.func1(0xad8b60, 0xc0007cc000, 0xc000426600)
/usr/local/go/src/net/http/server.go:2046 +0x18a
net/http.HandlerFunc.ServeHTTP(0xc000bcb200, 0xad8b60, 0xc0007cc000, 0xc000426600)
/usr/local/go/src/net/http/server.go:2007 +0x44
net/http.(*ServeMux).ServeHTTP(0xc000bd4940, 0xad8b60, 0xc0007cc000, 0xc000426600)
/usr/local/go/src/net/http/server.go:2387 +0x1bd
net/http.serverHandler.ServeHTTP(0xc0005cf500, 0xad8b60, 0xc0007cc000, 0xc000426600)
/usr/local/go/src/net/http/server.go:2802 +0xa4
net/http.(*conn).serve(0xc0009c8000, 0xada0e0, 0xc000585c80)
/usr/local/go/src/net/http/server.go:1890 +0x875
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2927 +0x38e
2019/08/23 08:39:12 http: panic serving 127.0.0.1:33706: runtime error: index out of range [2] with length 2
goroutine 39 [running]:
net/http.(*conn).serve.func1(0xc000191540)
/usr/local/go/src/net/http/server.go:1767 +0x139
panic(0x9921a0, 0xc0001bcda0)
/usr/local/go/src/runtime/panic.go:679 +0x1b2
cmd/vendor/golang.org/x/arch/x86/x86asm.decode1(0xc008c42894, 0x2, 0x40a388, 0x40, 0x59000000, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:426 +0x6fa0
cmd/vendor/golang.org/x/arch/x86/x86asm.Decode(...)
/usr/local/go/src/cmd/vendor/golang.org/x/arch/x86/x86asm/decode.go:226
cmd/internal/objfile.disasm_x86(0xc008c42894, 0x2, 0x40a388, 0x16f9894, 0xc0004e2d70, 0x40, 0xc0005987e0, 0x64, 0x50)
/usr/local/go/src/cmd/internal/objfile/disasm.go:300 +0x91
cmd/internal/objfile.disasm_amd64(0xc008c42894, 0x2, 0x40a388, 0x16f9894, 0xc0004e2d70, 0xadeb80, 0xe9d5c8, 0xc0058330d0, 0x7, 0x1)
/usr/local/go/src/cmd/internal/objfile/disasm.go:296 +0x5c
cmd/internal/objfile.(*Disasm).Decode(0xc00167c080, 0x16e5cf0, 0x16f9896, 0x0, 0x0, 0x0, 0xc000692ee0)
/usr/local/go/src/cmd/internal/objfile/disasm.go:275 +0x39a
main.(*objTool).Disasm(0xc000061840, 0xc003ef9270, 0x4c, 0x16e5cf0, 0x16f9896, 0x203000, 0xc002a80280, 0x0, 0x0, 0xc00007e380)
/usr/local/go/src/cmd/pprof/pprof.go:180 +0xf2
cmd/vendor/github.com/google/pprof/driver.(*internalObjTool).Disasm(0xc000061850, 0xc003ef9270, 0x4c, 0x16e5cf0, 0x16f9896, 0x1, 0xc00d9ae540, 0x6, 0xc000693100, 0x4e6ca1)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/driver/driver.go:280 +0x7f
cmd/vendor/github.com/google/pprof/internal/report.assemblyPerSourceLine(0xc00dba0000, 0x1434a, 0x17800, 0xc00013fd98, 0x1, 0x1, 0xc003fc2960, 0x4e, 0xad2900, 0xc000061850, ...)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:251 +0xec
cmd/vendor/github.com/google/pprof/internal/report.PrintWebList(0xaceb80, 0xc004218de0, 0xc003f9df40, 0xad2900, 0xc000061850, 0x32, 0x2, 0x0)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/report/source.go:208 +0xaa0
cmd/vendor/github.com/google/pprof/internal/driver.(*webInterface).source(0xc000326240, 0xad8b60, 0xc0007cd960, 0xc0000e8500)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:397 +0x18b
net/http.HandlerFunc.ServeHTTP(0xc0004e2240, 0xad8b60, 0xc0007cd960, 0xc0000e8500)
/usr/local/go/src/net/http/server.go:2007 +0x44
cmd/vendor/github.com/google/pprof/internal/driver.defaultWebServer.func1(0xad8b60, 0xc0007cd960, 0xc0000e8500)
/usr/local/go/src/cmd/vendor/github.com/google/pprof/internal/driver/webui.go:177 +0x9c
net/http.HandlerFunc.ServeHTTP(0xc0003264c0, 0xad8b60, 0xc0007cd960, 0xc0000e8500)
/usr/local/go/src/net/http/server.go:2007 +0x44
net/http.StripPrefix.func1(0xad8b60, 0xc0007cd960, 0xc0000e8400)
/usr/local/go/src/net/http/server.go:2046 +0x18a
net/http.HandlerFunc.ServeHTTP(0xc000bcb200, 0xad8b60, 0xc0007cd960, 0xc0000e8400)
/usr/local/go/src/net/http/server.go:2007 +0x44
net/http.(*ServeMux).ServeHTTP(0xc000bd4940, 0xad8b60, 0xc0007cd960, 0xc0000e8400)
/usr/local/go/src/net/http/server.go:2387 +0x1bd
net/http.serverHandler.ServeHTTP(0xc0005cf500, 0xad8b60, 0xc0007cd960, 0xc0000e8400)
/usr/local/go/src/net/http/server.go:2802 +0xa4
net/http.(*conn).serve(0xc000191540, 0xada0e0, 0xc001438180)
/usr/local/go/src/net/http/server.go:1890 +0x875
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2927 +0x38e
Here is the attached profile, hopefully reproducible on other systems:
pprof.pilot-discovery.alloc_objects.alloc_space.inuse_objects.inuse_space.033.pb.gz
Metadata
Metadata
Assignees
Labels
FrozenDueToAgeNeedsFixThe path to resolution is known, but the work has not been done.The path to resolution is known, but the work has not been done.release-blocker