Segmentation fault while compacting during scrape

[smarterclayton]

Not sure this is necessarily a prometheus error (vs something deeper), but hit this on a large scraping cluster on beta 5:

Version info (this is carrying one patch openshift/prometheus but isn't related to this code path).

level=info ts=2017-10-19T15:35:52.495022548Z caller=main.go:214 msg="Starting prometheus" version="(version=2.0.0-beta.5, branch=HEAD, revision=17ec944bf574136287b743e57e3ded7219d259f0)"
level=info ts=2017-10-19T15:35:52.495089303Z caller=main.go:215 build_context="(go=go1.8.3, user=root@prometheus-binary-4-build, date=20171003-03:56:29)"
level=info ts=2017-10-19T15:35:52.495107178Z caller=main.go:216 host_details="(Linux 3.10.0-693.1.1.el7.x86_64 #1 SMP Thu Aug 3 08:15:31 EDT 2017 x86_64 prometheus-0 (none))"
level=info ts=2017-10-19T15:35:52.497403925Z caller=web.go:367 component=web msg="Start listening for connections" address=localhost:9090
level=info ts=2017-10-19T15:35:52.497415807Z caller=main.go:308 msg="Starting TSDB"
level=info ts=2017-10-19T15:35:52.497408597Z caller=targetmanager.go:68 component="target manager" msg="Starting target manager..."

ts=2017-10-19T15:25:15.395914428Z caller=compact.go:359 component=tsdb msg="compact blocks" count=1 mint=1508426520000 maxt=1508426640000
ts=2017-10-19T15:25:56.973940705Z caller=head.go:273 component=tsdb msg="head GC completed" duration=2.139604722s
ts=2017-10-19T15:26:10.453942467Z caller=head.go:282 component=tsdb msg="WAL truncation completed" duration=13.479944001s
ts=2017-10-19T15:26:12.063309535Z caller=compact.go:359 component=tsdb msg="compact blocks" count=3 mint=1508425560000 maxt=1508425920000
level=error ts=2017-10-19T15:26:33.286618434Z caller=scrape.go:774 component="target manager" scrape_pool=kubernetes-service-endpoints target=http://10.128.24.236:9779/metrics msg="stale report failed" err="out of bounds"
ts=2017-10-19T15:26:50.490873115Z caller=compact.go:359 component=tsdb msg="compact blocks" count=3 mint=1508425920000 maxt=1508426280000
ts=2017-10-19T15:27:27.221721376Z caller=compact.go:359 component=tsdb msg="compact blocks" count=3 mint=1508426280000 maxt=1508426640000
level=error ts=2017-10-19T15:27:27.649870605Z caller=scrape.go:774 component="target manager" scrape_pool=kubernetes-service-endpoints target=http://10.130.48.114:9779/metrics msg="stale report failed" err="out of bounds"
ts=2017-10-19T15:28:04.622281143Z caller=compact.go:359 component=tsdb msg="compact blocks" count=3 mint=1508425200000 maxt=1508426280000
unexpected fault address 0x7fa4774ab276
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x7fa4774ab276 pc=0x4591e7]

goroutine 549 [running]:
runtime.throw(0x1c3e328, 0x5)
	/usr/lib/golang/src/runtime/panic.go:596 +0x95 fp=0xc510f29208 sp=0xc510f291e8
runtime.sigpanic()
	/usr/lib/golang/src/runtime/signal_unix.go:297 +0x28c fp=0xc510f29258 sp=0xc510f29208
runtime.memeqbody()
	/usr/lib/golang/src/runtime/asm_amd64.s:1469 +0xa7 fp=0xc510f29260 sp=0xc510f29258
type..eq.github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/labels.Label(0xc510f29360, 0xc4d9917d38, 0x4dcddfd5cfd23445)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/labels/labels.go:1 +0xa4 fp=0xc510f29298 sp=0xc510f29260
runtime.mapaccess1(0x19a3ce0, 0xc420eb7d70, 0xc510f29360, 0xc4d9353f08)
	/usr/lib/golang/src/runtime/hashmap.go:348 +0x19e fp=0xc510f292e0 sp=0xc510f29298
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*memPostings).addFor(0xc42026b600, 0x29a390, 0xc510445bf0, 0x8, 0xc50d4c9710, 0x27)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/postings.go:64 +0x9d fp=0xc510f29390 sp=0xc510f292e0
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*memPostings).add(0xc42026b600, 0x29a390, 0xc5f0897900, 0x14, 0x14)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/postings.go:56 +0x87 fp=0xc510f293e0 sp=0xc510f29390
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*Head).getOrCreateWithID(0xc4202423c0, 0x29a390, 0x6df4f37b9ea2a328, 0xc5f0897900, 0x14, 0x14, 0x0, 0xc5f0897b00)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/head.go:873 +0x19c fp=0xc510f29480 sp=0xc510f293e0
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*Head).getOrCreate(0xc4202423c0, 0x6df4f37b9ea2a328, 0xc5f0897900, 0x14, 0x14, 0x11, 0x8)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/head.go:859 +0xc7 fp=0xc510f294d0 sp=0xc510f29480
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*headAppender).Add(0xc5399cd360, 0xc5f0897900, 0x14, 0x14, 0x15f353f968f, 0x402e000000000000, 0xb, 0x8, 0xd)
	/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb/head.go:393 +0xa2 fp=0xc510f29540 sp=0xc510f294d0
github.com/prometheus/prometheus/vendor/github.com/prometheus/tsdb.(*dbAppender).Add(0xc50b1627c0, 0xc5f0897900, 0x14, 0x14, 0x15f353f968f, 0x402e000000000000, 0xc5f0897900, 0xc5f0897400, 0xc5f0897900)
	<autogenerated>:89 +0x84 fp=0xc510f29598 sp=0xc510f29540
github.com/prometheus/prometheus/storage/tsdb.appender.Add(0x289e880, 0xc50b1627c0, 0xc5f0897900, 0x14, 0x14, 0x15f353f968f, 0x402e000000000000, 0x0, 0xd, 0xd)
	/go/src/github.com/prometheus/prometheus/storage/tsdb/tsdb.go:192 +0xa8 fp=0xc510f29630 sp=0xc510f29598
github.com/prometheus/prometheus/storage/tsdb.(*appender).Add(0xc47ab67310, 0xc5f0897900, 0x14, 0x14, 0x15f353f968f, 0x402e000000000000, 0x7, 0x8, 0xc5399cd3b0)
	<autogenerated>:7 +0x94 fp=0xc510f29690 sp=0xc510f29630
github.com/prometheus/prometheus/storage.(*fanoutAppender).Add(0xc581160c00, 0xc5f0897900, 0x14, 0x14, 0x15f353f968f, 0x402e000000000000, 0xc5f0897900, 0x14, 0x14)
	/go/src/github.com/prometheus/prometheus/storage/fanout.go:112 +0x7b fp=0xc510f29708 sp=0xc510f29690
github.com/prometheus/prometheus/retrieval.(*timeLimitAppender).Add(0xc50b1627e0, 0xc5f0897900, 0x14, 0x14, 0x15f353f968f, 0x402e000000000000, 0x4103400000000000, 0x0, 0x0)
	/go/src/github.com/prometheus/prometheus/retrieval/target.go:239 +0x7b fp=0xc510f29760 sp=0xc510f29708
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).append(0xc50c539d00, 0xc52f64a000, 0x3363b1, 0x61eb12, 0xed17abb80, 0x2712830d, 0x2960260, 0x0, 0x0, 0x0, ...)
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:881 +0x559 fp=0xc510f29c98 sp=0xc510f29760
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0xc50c539d00, 0xdf8475800, 0x2540be400, 0x0)
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:696 +0x560 fp=0xc510f2dfc0 sp=0xc510f29c98
runtime.goexit()
	/usr/lib/golang/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc510f2dfc8 sp=0xc510f2dfc0
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
	/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:314 +0x383

[grobie]

This has been fixed in rc.0. Please update to the latest release candidate (rc.1) and open a new issue if you find another panic.

[smarterclayton]

Found another one:

ts=2017-10-19T16:08:00.662477423Z caller=compact.go:359 component=tsdb msg="compact blocks" count=3 mint=1508428800000 maxt=1508429160000
ts=2017-10-19T16:09:00.402758595Z caller=compact.go:359 component=tsdb msg="compact blocks" count=1 mint=1508429160000 maxt=1508429280000
unexpected fault address 0x7f8c57cd4454
fatal error: fault
[signal SIGSEGV: segmentation violation code=0x1 addr=0x7f8c57cd4454 pc=0x458f70]

goroutine 63 [running]:
runtime.throw(0x1c3e328, 0x5)
        /usr/lib/golang/src/runtime/panic.go:596 +0x95 fp=0xc6829276e0 sp=0xc6829276c0
runtime.sigpanic()
        /usr/lib/golang/src/runtime/signal_unix.go:297 +0x28c fp=0xc682927730 sp=0xc6829276e0
runtime.aeshashbody()
        /usr/lib/golang/src/runtime/asm_amd64.s:984 +0x4b0 fp=0xc682927738 sp=0xc682927730
runtime.mapassign(0x19ac500, 0xc5f42e2180, 0xc682927818, 0xc589fdb288)
        /usr/lib/golang/src/runtime/hashmap.go:504 +0x84 fp=0xc6829277d8 sp=0xc682927738
github.com/prometheus/prometheus/pkg/labels.Labels.Map(0xc5c2034e00, 0xe, 0xe, 0x31d)
        /go/src/github.com/prometheus/prometheus/pkg/labels/labels.go:130 +0xb8 fp=0xc682927848 sp=0xc6829277d8
github.com/prometheus/prometheus/pkg/labels.Labels.MarshalJSON(0xc5c2034e00, 0xe, 0xe, 0x40db03, 0x19d6840, 0x1afd2a0, 0x1afd201, 0x7f8c6e98a1b8)
        /go/src/github.com/prometheus/prometheus/pkg/labels/labels.go:68 +0x43 fp=0xc682927890 sp=0xc682927848
github.com/prometheus/prometheus/pkg/labels.(*Labels).MarshalJSON(0xc5601d3ce0, 0x1afd2a0, 0xc5601d3ce0, 0x7f8c6e98a1b8, 0xc5601d3ce0, 0x73000101)
        <autogenerated>:5 +0x5f fp=0xc6829278e0 sp=0xc682927890
encoding/json.marshalerEncoder(0xc5128d8a50, 0x1afd2a0, 0xc6831796c0, 0x197, 0x100)
        /usr/lib/golang/src/encoding/json/encode.go:451 +0x9f fp=0xc682927950 sp=0xc6829278e0
encoding/json.(*structEncoder).encode(0xc6806843c0, 0xc5128d8a50, 0x1afd0e0, 0xc6831796c0, 0x199, 0x100)
        /usr/lib/golang/src/encoding/json/encode.go:645 +0x253 fp=0xc682927ab0 sp=0xc682927950
encoding/json.(*structEncoder).(encoding/json.encode)-fm(0xc5128d8a50, 0x1afd0e0, 0xc6831796c0, 0x199, 0xc683170100)
        /usr/lib/golang/src/encoding/json/encode.go:659 +0x64 fp=0xc682927af0 sp=0xc682927ab0
encoding/json.(*ptrEncoder).encode(0xc4f918f6d8, 0xc5128d8a50, 0x1a55aa0, 0xc58f462fa8, 0x196, 0x100)
        /usr/lib/golang/src/encoding/json/encode.go:786 +0xe3 fp=0xc682927b30 sp=0xc682927af0
encoding/json.(*ptrEncoder).(encoding/json.encode)-fm(0xc5128d8a50, 0x1a55aa0, 0xc58f462fa8, 0x196, 0x1a50100)
        /usr/lib/golang/src/encoding/json/encode.go:791 +0x64 fp=0xc682927b70 sp=0xc682927b30
encoding/json.(*arrayEncoder).encode(0xc4f918f6e0, 0xc5128d8a50, 0x1885800, 0xc5601d3be0, 0x97, 0xc5ca1f0100)
        /usr/lib/golang/src/encoding/json/encode.go:767 +0xf5 fp=0xc682927bc8 sp=0xc682927b70
encoding/json.(*arrayEncoder).(encoding/json.encode)-fm(0xc5128d8a50, 0x1885800, 0xc5601d3be0, 0x97, 0xc5aca10100)
        /usr/lib/golang/src/encoding/json/encode.go:774 +0x64 fp=0xc682927c08 sp=0xc682927bc8
encoding/json.(*sliceEncoder).encode(0xc4f918f6e8, 0xc5128d8a50, 0x1885800, 0xc5601d3be0, 0x97, 0x1880100)
        /usr/lib/golang/src/encoding/json/encode.go:741 +0xc1 fp=0xc682927c48 sp=0xc682927c08
encoding/json.(*sliceEncoder).(encoding/json.encode)-fm(0xc5128d8a50, 0x1885800, 0xc5601d3be0, 0x97, 0xc5601d0100)
        /usr/lib/golang/src/encoding/json/encode.go:753 +0x64 fp=0xc682927c88 sp=0xc682927c48
encoding/json.(*encodeState).reflectValue(0xc5128d8a50, 0x1885800, 0xc5601d3be0, 0x97, 0x100)
        /usr/lib/golang/src/encoding/json/encode.go:323 +0x82 fp=0xc682927cc0 sp=0xc682927c88
encoding/json.(*encodeState).marshal(0xc5128d8a50, 0x1885800, 0xc5601d3be0, 0x1880100, 0x0, 0x0)
        /usr/lib/golang/src/encoding/json/encode.go:296 +0xb8 fp=0xc682927cf8 sp=0xc682927cc0
encoding/json.Marshal(0x1885800, 0xc5601d3be0, 0x1885800, 0xc5601d3be0, 0xc57cf2d260, 0xc4b08f3e20, 0x429ea6)
        /usr/lib/golang/src/encoding/json/encode.go:161 +0x6e fp=0xc682927d40 sp=0xc682927cf8
github.com/prometheus/prometheus/notifier.(*Notifier).sendAll(0xc420125290, 0xc58f462fa0, 0x2, 0x2, 0x0)
        /go/src/github.com/prometheus/prometheus/notifier/notifier.go:416 +0xc8 fp=0xc682927f08 sp=0xc682927d40
github.com/prometheus/prometheus/notifier.(*Notifier).Run(0xc420125290)
        /go/src/github.com/prometheus/prometheus/notifier/notifier.go:315 +0x13f fp=0xc682927fd8 sp=0xc682927f08
runtime.goexit()
        /usr/lib/golang/src/runtime/asm_amd64.s:2197 +0x1 fp=0xc682927fe0 sp=0xc682927fd8
created by main.main
        /go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:330 +0x46e8

[smarterclayton]

Could not find an example of this elsewhere

[fabxc]

This one is also pre-rc.0? The segfaults can happen in a bunch of places but the root cause should be common.

[smarterclayton]

Yeah, same code level. Will update and report back.

[lock]

This thread has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.