Crash log

[RichiH]

At 8ad6ebc, return code 2.

alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
  build user:       adm@prometheus.space.net
  build date:       20160308-11:31:54
  go version:       1.5.1
INFO[0000] Loading configuration file                    file=/home/configs/alertmanager.yml source=main.go:144
fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0x63 pc=0x7f287c49767d]

runtime stack:
runtime.throw(0xba6a00, 0x2a)
        /usr/local/go/src/runtime/panic.go:527 +0x90
runtime.sigpanic()
        /usr/local/go/src/runtime/sigpanic_unix.go:12 +0x5a

goroutine 376 [syscall, locked to thread]:
runtime.cgocall(0x4059d0, 0xc8206c49d8, 0xc800000000)
        /usr/local/go/src/runtime/cgocall.go:120 +0x11b fp=0xc8206c4980 sp=0xc8206c4950
net._C2func_getaddrinfo(0x7f2874000b20, 0x0, 0xc8205819b0, 0xc8204c7a60, 0x0, 0x0, 0x0)
        ??:0 +0x4d fp=0xc8206c49d8 sp=0xc8206c4980
net.cgoLookupIPCNAME(0xc82018e250, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/cgo_unix.go:111 +0x1ac fp=0xc8206c4b00 sp=0xc8206c49d8
net.cgoLookupIP(0xc82018e250, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x6d2035206e616874)
        /usr/local/go/src/net/cgo_unix.go:163 +0x56 fp=0xc8206c4b58 sp=0xc8206c4b00
net.lookupIP(0xc82018e250, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup_unix.go:67 +0x94 fp=0xc8206c4bc8 sp=0xc8206c4b58
net.glob.func15(0xbecac8, 0xc82018e250, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/hook.go:10 +0x4d fp=0xc8206c4c08 sp=0xc8206c4bc8
net.lookupIPMerge.func1(0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup.go:68 +0x71 fp=0xc8206c4c88 sp=0xc8206c4c08
internal/singleflight.(*Group).doCall(0x1018470, 0xc820907a40, 0xc82018e250, 0x9, 0xc8206c4e80)
        /usr/local/go/src/internal/singleflight/singleflight.go:93 +0x2c fp=0xc8206c4d38 sp=0xc8206c4c88
internal/singleflight.(*Group).Do(0x1018470, 0xc82018e250, 0x9, 0xc8206c4e80, 0x0, 0x0, 0x0, 0x0, 0x20666f2074656e2e)
        /usr/local/go/src/internal/singleflight/singleflight.go:63 +0x284 fp=0xc8206c4da8 sp=0xc8206c4d38
net.lookupIPMerge(0xc82018e250, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup.go:69 +0x9b fp=0xc8206c4eb8 sp=0xc8206c4da8
net.lookupIPDeadline(0xc82018e250, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup.go:91 +0xde fp=0xc8206c50f0 sp=0xc8206c4eb8
net.internetAddrList(0xb01800, 0x3, 0xc82018e250, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /usr/local/go/src/net/ipsock.go:252 +0x6ee fp=0xc8206c5258 sp=0xc8206c50f0
net.resolveAddrList(0xafd968, 0x4, 0xb01800, 0x3, 0xc82018e250, 0xc, 0x0, 0xc800000000, 0x0, 0x0, ...)
        /usr/local/go/src/net/dial.go:153 +0x466 fp=0xc8206c5330 sp=0xc8206c5258
net.(*Dialer).Dial(0xc8206c5558, 0xb01800, 0x3, 0xc82018e250, 0xc, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/dial.go:209 +0x124 fp=0xc8206c5510 sp=0xc8206c5330
net.Dial(0xb01800, 0x3, 0xc82018e250, 0xc, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/dial.go:186 +0x95 fp=0xc8206c55a8 sp=0xc8206c5510
net/smtp.Dial(0xc82018e250, 0xc, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/smtp/smtp.go:46 +0x59 fp=0xc8206c5620 sp=0xc8206c55a8
github.com/prometheus/alertmanager/notify.(*Email).Notify(0xc8201d5920, 0x7f287f737440, 0xc820580210, 0xc8207a5a00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:267 +0xa2 fp=0xc8206c5910 sp=0xc8206c5620
github.com/prometheus/alertmanager/notify.Build.func1.1(0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:99 +0x155 fp=0xc8206c5a40 sp=0xc8206c5910
github.com/prometheus/alertmanager/notify.NotifierFunc.Notify(0xc8201e55f0, 0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:70 +0x5a fp=0xc8206c5a80 sp=0xc8206c5a40
github.com/prometheus/alertmanager/notify.(*RetryNotifier).Notify(0xc8201d59a0, 0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:192 +0x1eb fp=0xc8206c5c10 sp=0xc8206c5a80
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e0dc0, 0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf fp=0xc8206c5cb8 sp=0xc8206c5c10
github.com/prometheus/alertmanager/notify.(*DedupingNotifier).Notify(0xc8201e0de0, 0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:290 +0x6b2 fp=0xc8206c5e20 sp=0xc8206c5cb8
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e0e00, 0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf fp=0xc8206c5ec8 sp=0xc8206c5e20
github.com/prometheus/alertmanager/notify.Fanout.Notify.func1(0x7f287f737440, 0xc820580210, 0xc8207a5200, 0x2a, 0x40, 0xc8208115e0, 0xc8205c7310, 0x7f2880fc26f
0, 0xc8201e0e00)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:150 +0x86 fp=0xc8206c5f58 sp=0xc8206c5ec8
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1696 +0x1 fp=0xc8206c5f60 sp=0xc8206c5f58
created by github.com/prometheus/alertmanager/notify.Fanout.Notify
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:155 +0x60d

goroutine 1 [chan receive, 9 minutes]:
main.main()
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:204 +0xf75

goroutine 17 [syscall, 9 minutes, locked to thread]:
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 7 [syscall, 9 minutes]:
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
        /usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 35 [chan receive, 9 minutes]:
main.main.func4(0xc8204a6060, 0xc82001aeb0)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:199 +0x4d
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:202 +0xf52

goroutine 13 [chan receive, 9 minutes]:
database/sql.(*DB).connectionOpener(0xc8200acd20)
        /usr/local/go/src/database/sql/sql.go:634 +0x45
created by database/sql.Open
        /usr/local/go/src/database/sql/sql.go:481 +0x336

goroutine 16 [select, 1 minutes]:
main.(*Dispatcher).run(0xc82009a750, 0x7f287eef5150, 0xc8204ce000)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:150 +0x8ea
main.(*Dispatcher).Run(0xc82009a750)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:59 +0x1de
created by main.main.func3
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:175 +0x55a

goroutine 102 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7f287f737008, 0x72, 0xc82000e1f0)
        /usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc820117aa0, 0x72, 0x0, 0x0)
        /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc820117aa0, 0x0, 0x0)
        /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc820117a40, 0x0, 0x7f287f737100, 0xc8206cc140)
        /usr/local/go/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820118010, 0x4f26a0, 0x0, 0x0)
        /usr/local/go/src/net/tcpsock_posix.go:254 +0x4d
net/http.tcpKeepAliveListener.Accept(0xc820118010, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:2135 +0x41
net/http.(*Server).Serve(0xc820114120, 0x7f287f7370c8, 0xc820118010, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:1887 +0xb3
net/http.(*Server).ListenAndServe(0xc820114120, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:1877 +0x136
net/http.ListenAndServe(0xaf9a98, 0x5, 0x7f287f736030, 0xc8201e0ee0, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:1967 +0x8f
main.listen(0xc8201e0ee0)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:255 +0x73
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:189 +0xcb8

goroutine 103 [select, 9 minutes, locked to thread]:
runtime.gopark(0xbecf90, 0xc820030728, 0xb00f88, 0x6, 0x18, 0x2)
        /usr/local/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc820030728, 0x0, 0x18)
        /usr/local/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc820030728)
        /usr/local/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
        /usr/local/go/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 114 [chan receive, 9 minutes]:
github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe.func1(0xc820014d80, 0x0, 0xc8204a80c0, 0x0, 0x0, 0x0, 0xc8204a8120)
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:97 +0x165
created by github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:98 +0x1d8

goroutine 169 [semacquire]:
sync.runtime_Semacquire(0xc8205c731c)
        /usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*WaitGroup).Wait(0xc8205c7310)
        /usr/local/go/src/sync/waitgroup.go:126 +0xb4
github.com/prometheus/alertmanager/notify.Fanout.Notify(0xc8201e55c0, 0x7f287f737440, 0xc8206ca510, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:158 +0x63f
github.com/prometheus/alertmanager/notify.Router.Notify(0xc8201e5560, 0x7f287f737440, 0xc8206ca510, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:313 +0x28d
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e0e80, 0x7f287f737440, 0xc8206ca510, 0xc8207a5200, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf
github.com/prometheus/alertmanager/notify.(*SilenceNotifier).Notify(0xc8201e5860, 0x7f287f737440, 0xc8206ca510, 0xc8201e9c00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:348 +0x22b
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e0ea0, 0x7f287f737440, 0xc8206ca510, 0xc8201e9c00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc8201e58f0, 0x7f287f737440, 0xc8206ca510, 0xc8201538c0, 0x2a, 0x2a, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:383 +0x22b
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e0ec0, 0x7f287f737440, 0xc8206ca510, 0xc8201538c0, 0x2a, 0x2a, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf
main.(*Dispatcher).processAlert.func1(0x7f287f737440, 0xc8206ca510, 0xc8201538c0, 0x2a, 0x2a, 0xc8206cc1a0)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:236 +0x8a
main.(*aggrGroup).run.func1(0xc8201538c0, 0x2a, 0x2a, 0x2)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:336 +0x5b
main.(*aggrGroup).flush(0xc8204ea000, 0xc82005bec8)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:400 +0x466
main.(*aggrGroup).run(0xc8204ea000, 0xc820568c80)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:337 +0xd53
created by main.(*Dispatcher).processAlert
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f

goroutine 377 [select]:
github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.(*Ticker).run(0xc820843980)
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:55 +0x20b
created by github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.NewTicker
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:32 +0x127

[RichiH]

I don't know what causes it, but it happens with a certain level of regularity.

Jftr, Prometheus itself is compiled at prometheus/prometheus@2f151d0

alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
  build user:       adm@prometheus.space.net
  build date:       20160308-11:31:54
  go version:       1.5.1
INFO[0000] Loading configuration file                    file=/home/configs/alertmanager.yml source=main.go:144



fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0x63 pc=0x7fa32c4e967d]

runtime stack:
runtime.throw(0xba6a00, 0x2a)
        /usr/local/go/src/runtime/panic.go:527 +0x90
runtime.sigpanic()
        /usr/local/go/src/runtime/sigpanic_unix.go:12 +0x5a

goroutine 336 [syscall, locked to thread]:
runtime.cgocall(0x4059d0, 0xc8208389d8, 0xc800000000)
        /usr/local/go/src/runtime/cgocall.go:120 +0x11b fp=0xc820838980 sp=0xc820838950
net._C2func_getaddrinfo(0x7fa3380008c0, 0x0, 0xc8204ec210, 0xc820190e00, 0x0, 0x0, 0x0)
        ??:0 +0x4d fp=0xc8208389d8 sp=0xc820838980
net.cgoLookupIPCNAME(0xc8200debe0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/cgo_unix.go:111 +0x1ac fp=0xc820838b00 sp=0xc8208389d8
net.cgoLookupIP(0xc8200debe0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x8)
        /usr/local/go/src/net/cgo_unix.go:163 +0x56 fp=0xc820838b58 sp=0xc820838b00
net.lookupIP(0xc8200debe0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup_unix.go:67 +0x94 fp=0xc820838bc8 sp=0xc820838b58
net.glob.func15(0xbecac8, 0xc8200debe0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/hook.go:10 +0x4d fp=0xc820838c08 sp=0xc820838bc8
net.lookupIPMerge.func1(0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup.go:68 +0x71 fp=0xc820838c88 sp=0xc820838c08
internal/singleflight.(*Group).doCall(0x1018470, 0xc820831720, 0xc8200debe0, 0x9, 0xc820838e80)
        /usr/local/go/src/internal/singleflight/singleflight.go:93 +0x2c fp=0xc820838d38 sp=0xc820838c88
internal/singleflight.(*Group).Do(0x1018470, 0xc8200debe0, 0x9, 0xc820838e80, 0x0, 0x0, 0x0, 0x0, 0xdeaddeaddeaddead)
        /usr/local/go/src/internal/singleflight/singleflight.go:63 +0x284 fp=0xc820838da8 sp=0xc820838d38
net.lookupIPMerge(0xc8200debe0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup.go:69 +0x9b fp=0xc820838eb8 sp=0xc820838da8
net.lookupIPDeadline(0xc8200debe0, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/lookup.go:91 +0xde fp=0xc8208390f0 sp=0xc820838eb8
net.internetAddrList(0xb01800, 0x3, 0xc8200debe0, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /usr/local/go/src/net/ipsock.go:252 +0x6ee fp=0xc820839258 sp=0xc8208390f0
net.resolveAddrList(0xafd968, 0x4, 0xb01800, 0x3, 0xc8200debe0, 0xc, 0x0, 0xc800000000, 0x0, 0x0, ...)
        /usr/local/go/src/net/dial.go:153 +0x466 fp=0xc820839330 sp=0xc820839258
net.(*Dialer).Dial(0xc820839558, 0xb01800, 0x3, 0xc8200debe0, 0xc, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/dial.go:209 +0x124 fp=0xc820839510 sp=0xc820839330
net.Dial(0xb01800, 0x3, 0xc8200debe0, 0xc, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/dial.go:186 +0x95 fp=0xc8208395a8 sp=0xc820839510
net/smtp.Dial(0xc8200debe0, 0xc, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/smtp/smtp.go:46 +0x59 fp=0xc820839620 sp=0xc8208395a8
github.com/prometheus/alertmanager/notify.(*Email).Notify(0xc820170440, 0x7fa33c01f070, 0xc8207cb110, 0xc8200abc00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:267 +0xa2 fp=0xc820839910 sp=0xc820839620
github.com/prometheus/alertmanager/notify.Build.func1.1(0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:99 +0x155 fp=0xc820839a40 sp=0xc820839910
github.com/prometheus/alertmanager/notify.NotifierFunc.Notify(0xc820149da0, 0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:70 +0x5a fp=0xc820839a80 sp=0xc820839a40
github.com/prometheus/alertmanager/notify.(*RetryNotifier).Notify(0xc8201704c0, 0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:192 +0x1eb fp=0xc820839c10 sp=0xc820839a80
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc820123b80, 0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf fp=0xc820839cb8 sp=0xc820839c10
github.com/prometheus/alertmanager/notify.(*DedupingNotifier).Notify(0xc820123ba0, 0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:290 +0x6b2 fp=0xc820839e20 sp=0xc820839cb8
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc820123bc0, 0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf fp=0xc820839ec8 sp=0xc820839e20
github.com/prometheus/alertmanager/notify.Fanout.Notify.func1(0x7fa33c01f070, 0xc8207cb110, 0xc8200ab400, 0x2a, 0x40, 0xc8204ae4e0, 0xc8204a8420, 0x7fa33e8e98c
8, 0xc820123bc0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:150 +0x86 fp=0xc820839f58 sp=0xc820839ec8
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1696 +0x1 fp=0xc820839f60 sp=0xc820839f58
created by github.com/prometheus/alertmanager/notify.Fanout.Notify
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:155 +0x60d

goroutine 1 [chan receive, 14 minutes]:
main.main()
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:204 +0xf75

goroutine 17 [syscall, 14 minutes, locked to thread]:
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 10 [syscall, 14 minutes]:
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:22 +0x18
created by os/signal.init.1
        /usr/local/go/src/os/signal/signal_unix.go:28 +0x37

goroutine 16 [chan receive, 14 minutes]:
database/sql.(*DB).connectionOpener(0xc8200a1ae0)
        /usr/local/go/src/database/sql/sql.go:634 +0x45
created by database/sql.Open
        /usr/local/go/src/database/sql/sql.go:481 +0x336

goroutine 98 [chan receive, 14 minutes]:
github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe.func1(0xc820015200, 0x0, 0xc8201be240, 0xc8201bc400, 0x2a, 0x40, 0xc8201be2a0)
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:97 +0x165
created by github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:98 +0x1d8

goroutine 19 [select, 1 minutes]:
main.(*Dispatcher).run(0xc82008e990, 0x7fa33d0611f0, 0xc8204c8120)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:150 +0x8ea
main.(*Dispatcher).Run(0xc82008e990)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:59 +0x1de
created by main.main.func3
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:175 +0x55a

goroutine 88 [IO wait, 1 minutes]:
net.runtime_pollWait(0x7fa33e8f2fa0, 0x72, 0xc82000e1f0)
        /usr/local/go/src/runtime/netpoll.go:157 +0x60
net.(*pollDesc).Wait(0xc8202bddb0, 0x72, 0x0, 0x0)
        /usr/local/go/src/net/fd_poll_runtime.go:73 +0x3a
net.(*pollDesc).WaitRead(0xc8202bddb0, 0x0, 0x0)
        /usr/local/go/src/net/fd_poll_runtime.go:78 +0x36
net.(*netFD).accept(0xc8202bdd50, 0x0, 0x7fa33e8f3098, 0xc82053da20)
        /usr/local/go/src/net/fd_unix.go:408 +0x27c
net.(*TCPListener).AcceptTCP(0xc820038650, 0x4f26a0, 0x0, 0x0)
        /usr/local/go/src/net/tcpsock_posix.go:254 +0x4d
net/http.tcpKeepAliveListener.Accept(0xc820038650, 0x0, 0x0, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:2135 +0x41
net/http.(*Server).Serve(0xc820137f80, 0x7fa33e8f3060, 0xc820038650, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:1887 +0xb3
net/http.(*Server).ListenAndServe(0xc820137f80, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:1877 +0x136
net/http.ListenAndServe(0xaf9a98, 0x5, 0x7fa33e8f1fc8, 0xc820123ca0, 0x0, 0x0)
        /usr/local/go/src/net/http/server.go:1967 +0x8f
main.listen(0xc820123ca0)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:255 +0x73
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:189 +0xcb8

goroutine 89 [select, 14 minutes, locked to thread]:
runtime.gopark(0xbecf90, 0xc820030f28, 0xb00f88, 0x6, 0x4ceb18, 0x2)
        /usr/local/go/src/runtime/proc.go:185 +0x163
runtime.selectgoImpl(0xc820030f28, 0x0, 0x18)
        /usr/local/go/src/runtime/select.go:392 +0xa64
runtime.selectgo(0xc820030f28)
        /usr/local/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
        /usr/local/go/src/runtime/signal1_unix.go:227 +0x353
runtime.goexit()
        /usr/local/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 90 [chan receive, 14 minutes]:
main.main.func4(0xc820137da0, 0xc82001b360)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:199 +0x4d
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:202 +0xf52

goroutine 337 [select]:
github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.(*Ticker).run(0xc8208b9c40)
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:55 +0x20b
created by github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.NewTicker
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:32 +0x127

goroutine 268 [semacquire]:
sync.runtime_Semacquire(0xc8204a842c)
        /usr/local/go/src/runtime/sema.go:43 +0x26
sync.(*WaitGroup).Wait(0xc8204a8420)
        /usr/local/go/src/sync/waitgroup.go:126 +0xb4
github.com/prometheus/alertmanager/notify.Fanout.Notify(0xc820149d70, 0x7fa33c01f070, 0xc8207797d0, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:158 +0x63f
github.com/prometheus/alertmanager/notify.Router.Notify(0xc820149d10, 0x7fa33c01f070, 0xc8207797d0, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:313 +0x28d
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc820123c40, 0x7fa33c01f070, 0xc8207797d0, 0xc8200ab400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf
github.com/prometheus/alertmanager/notify.(*SilenceNotifier).Notify(0xc8201ca030, 0x7fa33c01f070, 0xc8207797d0, 0xc82018b000, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:348 +0x22b
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc820123c60, 0x7fa33c01f070, 0xc8207797d0, 0xc82018b000, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc8201ca0c0, 0x7fa33c01f070, 0xc8207797d0, 0xc82077ab00, 0x2a, 0x2a, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:383 +0x22b
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc820123c80, 0x7fa33c01f070, 0xc8207797d0, 0xc82077ab00, 0x2a, 0x2a, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x1bf
main.(*Dispatcher).processAlert.func1(0x7fa33c01f070, 0xc8207797d0, 0xc82077ab00, 0x2a, 0x2a, 0xc8207a2de0)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:236 +0x8a
main.(*aggrGroup).run.func1(0xc82077ab00, 0x2a, 0x2a, 0x2)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:336 +0x5b
main.(*aggrGroup).flush(0xc8201b4180, 0xc8205edec8)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:400 +0x466
main.(*aggrGroup).run(0xc8201b4180, 0xc8207602b0)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:337 +0xd53
created by main.(*Dispatcher).processAlert
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x42f

[fabxc]

Could you recompile the Alertmanager version you are using with the -race flag set.
If there are any races happening, it will be logged.

[RichiH]

% git diff
diff --git a/scripts/build.sh b/scripts/build.sh
index 0eede67..13ec597 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -43,6 +43,6 @@ fi
 export GO15VENDOREXPERIMENT="1"

 echo " >   alertmanager"
-go build -ldflags "${ldflags}" -o alertmanager${ext} ${repo_path}
+go build -race -ldflags "${ldflags}" -o alertmanager${ext} ${repo_path}

 exit 0
% make
>> formatting code
>> building binaries
 >   alertmanager
>> running tests
ok      github.com/prometheus/alertmanager      3.173s
?       github.com/prometheus/alertmanager/config       [no test files]
ok      github.com/prometheus/alertmanager/notify       0.019s
?       github.com/prometheus/alertmanager/provider     [no test files]
?       github.com/prometheus/alertmanager/provider/sqlite      [no test files]
?       github.com/prometheus/alertmanager/template     [no test files]
?       github.com/prometheus/alertmanager/template/internal/deftmpl    [no test files]
?       github.com/prometheus/alertmanager/test [no test files]
--- FAIL: TestRepeat (1.35s)
        acceptance.go:126: AM on 0.0.0.0:43676
        acceptance.go:185: exit status 1
        acceptance.go:190:
                collector "webhook":

                interval [2,2.5]
                ---
                - test[1f56cb8][active][1:]
                  [ � ]
                interval [3,3.5]
                ---
                - test[1f56cb8][active][1:]
                  [ � ]
                interval [4,4.5]
                ---
                - test[1f56cb8][active][1:3]
                  [ � ]

                Expected total of 3 alerts, got 0
                received:

        acceptance.go:194: stdout:
                alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
                  build user:       adm@prometheus.space.net
                  build date:       20160308-15:54:13
                  go version:       1.5.1
                alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
                  build user:       adm@prometheus.space.net
                  build date:       20160308-15:54:13
                  go version:       1.5.1
        acceptance.go:195: stderr:
                time="2016-03-08T15:55:23+01:00" level=info msg="Loading configuration file" file="/tmp/am_test871001559/config.yml" source="main.go:144"
                time="2016-03-08T15:55:24+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:24+01:00" level=info msg="Received SIGTERM, exiting gracefully..." source="main.go:206"
                time="2016-03-08T15:55:24+01:00" level=info msg="Loading configuration file" file="/tmp/am_test871001559/config.yml" source="main.go:144"
                time="2016-03-08T15:55:24+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:24+01:00" level=fatal msg="listen tcp 0.0.0.0:43676: bind: address already in use" source="main.go:256"
--- FAIL: TestMergeAlerts (6.91s)
        acceptance.go:126: AM on 0.0.0.0:41162
        acceptance.go:185: exit status 1
        acceptance.go:190:
                collector "webhook":

                interval [7.7,8]
                ---
                - test[1f56cb8][active][5.3:]
                  [ � ]
                interval [2,2.5]
                ---
                - test[1f56cb8][active][1:]
                  [ � ]
                interval [3,3.5]
                ---
                - test[1f56cb8][active][1:]
                  [ � ]
                interval [4,4.5]
                ---
                - test[1f56cb8][active][1:]
                  [ � ]
                interval [5,5.5]
                ---
                - test[1f56cb8][resolved][1:4.8]
                  [ � ]
                interval [6,6.5]
                ---
                - test[1f56cb8][active][5.3:]
                  [ � ]

                Expected total of 6 alerts, got 5
                received:
                @ 2.07035473
                - test[1f56cb8][active][1:]
                @ 3.063289297
                - test[1f56cb8][active][1:]
                @ 4.067393217
                - test[1f56cb8][active][1:]
                @ 5.075701611
                - test[1f56cb8][resolved][1:4.8]
                @ 6.07053297
                - test[1f56cb8][active][5.314192328:]

        acceptance.go:194: stdout:
                alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
                  build user:       adm@prometheus.space.net
                  build date:       20160308-15:54:13
                  go version:       1.5.1
                alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
                  build user:       adm@prometheus.space.net
                  build date:       20160308-15:54:13
                  go version:       1.5.1
        acceptance.go:195: stderr:
                time="2016-03-08T15:55:23+01:00" level=info msg="Loading configuration file" file="/tmp/am_test018428234/config.yml" source="main.go:144"
                time="2016-03-08T15:55:24+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:24+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:25+01:00" level=debug msg="flushing [test[1f56cb8][active]]" aggrGroup=cbf29ce484222325 source="dispatch.go:398"
                time="2016-03-08T15:55:25+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=inhibit
                time="2016-03-08T15:55:25+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=silence
                time="2016-03-08T15:55:25+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=route
                time="2016-03-08T15:55:25+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=dedup
                time="2016-03-08T15:55:25+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=retry
                time="2016-03-08T15:55:25+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:26+01:00" level=debug msg="flushing [test[1f56cb8][active]]" aggrGroup=cbf29ce484222325 source="dispatch.go:398"
                time="2016-03-08T15:55:26+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=inhibit
                time="2016-03-08T15:55:26+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=silence
                time="2016-03-08T15:55:26+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=route
                time="2016-03-08T15:55:26+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=dedup
                time="2016-03-08T15:55:26+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=retry
                time="2016-03-08T15:55:26+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:27+01:00" level=debug msg="flushing [test[1f56cb8][active]]" aggrGroup=cbf29ce484222325 source="dispatch.go:398"
                time="2016-03-08T15:55:27+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=inhibit
                time="2016-03-08T15:55:27+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=silence
                time="2016-03-08T15:55:27+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=route
                time="2016-03-08T15:55:27+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=dedup
                time="2016-03-08T15:55:27+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=retry
                time="2016-03-08T15:55:27+01:00" level=debug msg="Received alert" alert=test[1f56cb8][resolved] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:28+01:00" level=debug msg="Received alert" alert=test[1f56cb8][resolved] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:28+01:00" level=debug msg="Received alert" alert=test[1f56cb8][resolved] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:28+01:00" level=debug msg="flushing [test[1f56cb8][resolved]]" aggrGroup=cbf29ce484222325 source="dispatch.go:398"
                time="2016-03-08T15:55:28+01:00" level=debug msg="notify [test[1f56cb8][resolved]]" source="notify.go:400" step=inhibit
                time="2016-03-08T15:55:28+01:00" level=debug msg="notify [test[1f56cb8][resolved]]" source="notify.go:400" step=silence
                time="2016-03-08T15:55:28+01:00" level=debug msg="notify [test[1f56cb8][resolved]]" source="notify.go:400" step=route
                time="2016-03-08T15:55:28+01:00" level=debug msg="notify [test[1f56cb8][resolved]]" source="notify.go:400" step=dedup
                time="2016-03-08T15:55:28+01:00" level=debug msg="notify [test[1f56cb8][resolved]]" source="notify.go:400" step=retry
                time="2016-03-08T15:55:28+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:29+01:00" level=debug msg="flushing [test[1f56cb8][active]]" aggrGroup=cbf29ce484222325 source="dispatch.go:398"
                time="2016-03-08T15:55:29+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=inhibit
                time="2016-03-08T15:55:29+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=silence
                time="2016-03-08T15:55:29+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=route
                time="2016-03-08T15:55:29+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=dedup
                time="2016-03-08T15:55:29+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=retry
                time="2016-03-08T15:55:29+01:00" level=info msg="Received SIGTERM, exiting gracefully..." source="main.go:206"
                time="2016-03-08T15:55:30+01:00" level=info msg="Loading configuration file" file="/tmp/am_test018428234/config.yml" source="main.go:144"
                time="2016-03-08T15:55:30+01:00" level=debug msg="Received alert" alert=test[1f56cb8][active] component=dispatcher source="dispatch.go:160"
                time="2016-03-08T15:55:30+01:00" level=debug msg="flushing [test[1f56cb8][active]]" aggrGroup=cbf29ce484222325 source="dispatch.go:398"
                time="2016-03-08T15:55:30+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=inhibit
                time="2016-03-08T15:55:30+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=silence
                time="2016-03-08T15:55:30+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=route
                time="2016-03-08T15:55:30+01:00" level=debug msg="notify [test[1f56cb8][active]]" source="notify.go:400" step=dedup
                time="2016-03-08T15:55:30+01:00" level=fatal msg="listen tcp 0.0.0.0:41162: bind: address already in use" source="main.go:256"
FAIL
FAIL    github.com/prometheus/alertmanager/test/acceptance      13.132s
ok      github.com/prometheus/alertmanager/types        0.119s
?       github.com/prometheus/alertmanager/ui   [no test files]
?       github.com/prometheus/alertmanager/version      [no test files]
make: *** [test] Error 1
% echo $?
2
%

[fabxc]

Yea the acceptance tests have some known races (have a fix sitting somewhere).
But the build succeeded, so it would be interesting to see whether the actual AM code exposes any races at runtime.

[RichiH]

Same compile flags:

diff --git a/scripts/build.sh b/scripts/build.sh
index 0eede67..13ec597 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -43,6 +43,6 @@ fi
 export GO15VENDOREXPERIMENT="1"

 echo " >   alertmanager"
-go build -ldflags "${ldflags}" -o alertmanager${ext} ${repo_path}
+go build -race -ldflags "${ldflags}" -o alertmanager${ext} ${repo_path}

 exit 0

Died after 776 seconds:

adm@prometheus [2] [7] (git)-[master] ~/gocode/src/github.com/prometheus/alertmanager % ./alertmanager -config.file=/home/configs/alertmanager.yml
alertmanager, version 0.1.0 (branch: master, revision: 8ad6ebc)
  build user:       adm@prometheus.space.net
  build date:       20160308-15:54:13
  go version:       1.5.1
INFO[0000] Loading configuration file                    file=/home/configs/alertmanager.yml source=main.go:144

fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0x63 pc=0x7fda6646d67d]

runtime stack:
runtime.throw(0xd4b8c0, 0x2a)
        /tmp/workdir/go/src/runtime/panic.go:527 +0x90
runtime.sigpanic()
        /tmp/workdir/go/src/runtime/sigpanic_unix.go:12 +0x5a

goroutine 227 [syscall, locked to thread]:
runtime.cgocall(0x4256f0, 0xc8205807a8, 0xc800000000)
        /tmp/workdir/go/src/runtime/cgocall.go:120 +0xc2 fp=0xc820580750 sp=0xc820580728
net._C2func_getaddrinfo(0x27256a0, 0x0, 0xc820436de0, 0xc8201689a8, 0xc800000000, 0x0, 0x0)
        ??:0 +0x6b fp=0xc8205807a8 sp=0xc820580750
net.cgoLookupIPCNAME(0xc820154690, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/cgo_unix.go:111 +0x222 fp=0xc8205808f0 sp=0xc8205807a8
net.cgoLookupIP(0xc820154690, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0xb00e20)
        /tmp/workdir/go/src/net/cgo_unix.go:163 +0x64 fp=0xc820580948 sp=0xc8205808f0
net.lookupIP(0xc820154690, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup_unix.go:67 +0xb9 fp=0xc8205809e0 sp=0xc820580948
net.glob.func15(0xdb0568, 0xc820154690, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/hook.go:10 +0x6e fp=0xc820580a48 sp=0xc8205809e0
net.lookupIPMerge.func1(0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup.go:68 +0x99 fp=0xc820580ac8 sp=0xc820580a48
internal/singleflight.(*Group).doCall(0x1b8f9b0, 0xc8201dadc0, 0xc820154690, 0x9, 0xc820580d18)
        /tmp/workdir/go/src/internal/singleflight/singleflight.go:93 +0x3d fp=0xc820580b80 sp=0xc820580ac8
internal/singleflight.(*Group).Do(0x1b8f9b0, 0xc820154690, 0x9, 0xc820580d18, 0x0, 0x0, 0x0, 0x0, 0xbc)
        /tmp/workdir/go/src/internal/singleflight/singleflight.go:63 +0x3c8 fp=0xc820580c20 sp=0xc820580b80
net.lookupIPMerge(0xc820154690, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup.go:69 +0xd2 fp=0xc820580d50 sp=0xc820580c20
net.lookupIPDeadline(0xc820154690, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup.go:91 +0xef fp=0xc820580f88 sp=0xc820580d50
net.internetAddrList(0xca4f88, 0x3, 0xc820154690, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/workdir/go/src/net/ipsock.go:252 +0x775 fp=0xc820581120 sp=0xc820580f88
net.resolveAddrList(0xca10e8, 0x4, 0xca4f88, 0x3, 0xc820154690, 0xc, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/workdir/go/src/net/dial.go:153 +0x4be fp=0xc820581228 sp=0xc820581120
net.(*Dialer).Dial(0xc820581478, 0xca4f88, 0x3, 0xc820154690, 0xc, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/dial.go:209 +0x180 fp=0xc820581410 sp=0xc820581228
net.Dial(0xca4f88, 0x3, 0xc820154690, 0xc, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/dial.go:186 +0xa6 fp=0xc8205814c8 sp=0xc820581410
net/smtp.Dial(0xc820154690, 0xc, 0x18, 0x0, 0x0)
        /usr/local/go/src/net/smtp/smtp.go:46 +0x70 fp=0xc820581550 sp=0xc8205814c8
github.com/prometheus/alertmanager/notify.(*Email).Notify(0xc82013b3a0, 0x7fda7c8af070, 0xc820436510, 0xc820569660, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:267 +0xdd fp=0xc820581868 sp=0xc820581550
github.com/prometheus/alertmanager/notify.Build.func1.1(0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:99 +0x16b fp=0xc8205819b8 sp=0xc820581868
github.com/prometheus/alertmanager/notify.NotifierFunc.Notify(0xc8201f44e0, 0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:70 +0x72 fp=0xc820581a08 sp=0xc8205819b8
github.com/prometheus/alertmanager/notify.(*RetryNotifier).Notify(0xc82013b440, 0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:192 +0x21b fp=0xc820581b98 sp=0xc820581a08
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e2c80, 0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245 fp=0xc820581c50 sp=0xc820581b98
github.com/prometheus/alertmanager/notify.(*DedupingNotifier).Notify(0xc8201e2ca0, 0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:290 +0x9c0 fp=0xc820581e10 sp=0xc820581c50
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e2cc0, 0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245 fp=0xc820581ec8 sp=0xc820581e10
github.com/prometheus/alertmanager/notify.Fanout.Notify.func1(0x7fda7c8af070, 0xc820436510, 0xc820569080, 0x3, 0x4, 0xc8205690c0, 0xc8205260c0, 0x7fda7e97bbe8,
 0xc8201e2cc0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:150 +0x97 fp=0xc820581f58 sp=0xc820581ec8
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1696 +0x1 fp=0xc820581f60 sp=0xc820581f58
created by github.com/prometheus/alertmanager/notify.Fanout.Notify
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:155 +0x723

goroutine 1 [chan receive, 12 minutes]:
main.main()
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:204 +0x149b

goroutine 17 [syscall, 12 minutes, locked to thread]:
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 24 [syscall, 12 minutes]:
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:22 +0x26
created by os/signal.init.1
        /usr/local/go/src/os/signal/signal_unix.go:28 +0x45

goroutine 4 [chan receive, 12 minutes]:
main.main.func4(0xc820077500, 0xc8201505f0)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:199 +0x5b
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:202 +0x1478

goroutine 38 [chan receive, 12 minutes]:
database/sql.(*DB).connectionOpener(0xc82016e0a0)
        /usr/local/go/src/database/sql/sql.go:634 +0x66
created by database/sql.Open
        /usr/local/go/src/database/sql/sql.go:481 +0x439

goroutine 94 [IO wait, 3 minutes]:
net.runtime_pollWait(0x7fda7e981130, 0x72, 0x7fda7e973028)
        /tmp/workdir/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc8200fd8e0, 0x72, 0x0, 0x0)
        /tmp/workdir/go/src/net/fd_poll_runtime.go:73 +0x56
net.(*pollDesc).WaitRead(0xc8200fd8e0, 0x0, 0x0)
        /tmp/workdir/go/src/net/fd_poll_runtime.go:78 +0x44
net.(*netFD).accept(0xc8200fd880, 0x0, 0x7fda7e981228, 0xc8204d7760)
        /tmp/workdir/go/src/net/fd_unix.go:408 +0x2f6
net.(*TCPListener).AcceptTCP(0xc82009c410, 0xc820041d50, 0x0, 0x0)
        /tmp/workdir/go/src/net/tcpsock_posix.go:254 +0x77
net/http.tcpKeepAliveListener.Accept(0xc82009c410, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:2135 +0x4f
net/http.(*Server).Serve(0xc8200776e0, 0x7fda7e9811f0, 0xc82009c410, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:1887 +0xc4
net/http.(*Server).ListenAndServe(0xc8200776e0, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:1877 +0x175
net/http.ListenAndServe(0xc9d1f8, 0x5, 0x7fda7e980158, 0xc8201e2da0, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:1967 +0xe3
main.listen(0xc8201e2da0)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:255 +0xba
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:189 +0x1177

goroutine 26 [select]:
main.(*Dispatcher).run(0xc8200a6630, 0x7fda7d8f1168, 0xc8204f3d40)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:150 +0xa59
main.(*Dispatcher).Run(0xc8200a6630)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:59 +0x2b6
created by main.main.func3
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:175 +0x77a

goroutine 95 [select, 12 minutes, locked to thread]:
runtime.gopark(0xdb0aa8, 0xc82002df28, 0xca4710, 0x6, 0x4d5918, 0x2)
        /tmp/workdir/go/src/runtime/proc.go:185 +0x11d
runtime.selectgoImpl(0xc82002df28, 0xc82002df14, 0x0)
        /tmp/workdir/go/src/runtime/select.go:392 +0xb9d
runtime.selectgo(0xc82002df28)
        /tmp/workdir/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
        /tmp/workdir/go/src/runtime/signal1_unix.go:227 +0x2ea
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 98 [chan receive, 12 minutes]:
github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe.func1(0xc8201621c0, 0x0, 0xc820076cc0, 0xc8204ec000, 0x2a, 0x40, 0xc820076d20)
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:97 +0x189
created by github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:98 +0x24b

goroutine 138 [semacquire]:
sync.runtime_Semacquire(0xc8205260cc)
        /tmp/workdir/go/src/runtime/sema.go:43 +0x26
sync.(*WaitGroup).Wait(0xc8205260c0)
        /tmp/workdir/go/src/sync/waitgroup.go:126 +0x118
github.com/prometheus/alertmanager/notify.Fanout.Notify(0xc8201f44b0, 0x7fda7c8af070, 0xc82051c0c0, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:158 +0x755
github.com/prometheus/alertmanager/notify.Router.Notify(0xc8201f4450, 0x7fda7c8af070, 0xc82051c0c0, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:313 +0x308
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e2d40, 0x7fda7c8af070, 0xc82051c0c0, 0xc820569080, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245
github.com/prometheus/alertmanager/notify.(*SilenceNotifier).Notify(0xc8201f4750, 0x7fda7c8af070, 0xc82051c0c0, 0xc820568f80, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:348 +0x37d
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e2d60, 0x7fda7c8af070, 0xc82051c0c0, 0xc820568f80, 0x3, 0x4, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc8201f47e0, 0x7fda7c8af070, 0xc82051c0c0, 0xc82050a040, 0x3, 0x3, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:383 +0x37d
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201e2d80, 0x7fda7c8af070, 0xc82051c0c0, 0xc82050a040, 0x3, 0x3, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245
main.(*Dispatcher).processAlert.func1(0x7fda7c8af070, 0xc82051c0c0, 0xc82050a040, 0x3, 0x3, 0xc82050a060)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:236 +0xd5
main.(*aggrGroup).run.func1(0xc82050a040, 0x3, 0x3, 0x2)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:336 +0x69
main.(*aggrGroup).flush(0xc820166000, 0xc8200b9ec8)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:400 +0x638
main.(*aggrGroup).run(0xc820166000, 0xc8207f0090)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:337 +0x10ee
created by main.(*Dispatcher).processAlert
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x578

goroutine 228 [select]:
github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.(*Ticker).run(0xc8201dd340)
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:55 +0x27e
created by github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.NewTicker
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:32 +0x1b1

[RichiH]

Also happens with 04fbfb9:

alertmanager, version 0.1.0 (branch: master, revision: 04fbfb9)
  build user:       adm@prometheus.space.net
  build date:       20160314-15:52:23
  go version:       1.5.1
INFO[0000] Loading configuration file                    file=/home/configs/alertmanager.yml source=main.go:144
fatal error: unexpected signal during runtime execution
[signal 0xb code=0x1 addr=0x63 pc=0x7fe3a83e967d]

runtime stack:
runtime.throw(0xd4bc20, 0x2a)
        /tmp/workdir/go/src/runtime/panic.go:527 +0x90
runtime.sigpanic()
        /tmp/workdir/go/src/runtime/sigpanic_unix.go:12 +0x5a

goroutine 339 [syscall, locked to thread]:
runtime.cgocall(0x4256f0, 0xc8206e07a8, 0xc800000000)
        /tmp/workdir/go/src/runtime/cgocall.go:120 +0xc2 fp=0xc8206e0750 sp=0xc8206e0728
net._C2func_getaddrinfo(0x7fe38c003d30, 0x0, 0xc82065a270, 0xc820280728, 0xc800000000, 0x0, 0x0)
        ??:0 +0x6b fp=0xc8206e07a8 sp=0xc8206e0750
net.cgoLookupIPCNAME(0xc82000e790, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/cgo_unix.go:111 +0x222 fp=0xc8206e08f0 sp=0xc8206e07a8
net.cgoLookupIP(0xc82000e790, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc8206e09a0)
        /tmp/workdir/go/src/net/cgo_unix.go:163 +0x64 fp=0xc8206e0948 sp=0xc8206e08f0
net.lookupIP(0xc82000e790, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup_unix.go:67 +0xb9 fp=0xc8206e09e0 sp=0xc8206e0948
net.glob.func15(0xdb0908, 0xc82000e790, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/hook.go:10 +0x6e fp=0xc8206e0a48 sp=0xc8206e09e0
net.lookupIPMerge.func1(0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup.go:68 +0x99 fp=0xc8206e0ac8 sp=0xc8206e0a48
internal/singleflight.(*Group).doCall(0x1b909b0, 0xc82078fbd0, 0xc82000e790, 0x9, 0xc8206e0d18)
        /tmp/workdir/go/src/internal/singleflight/singleflight.go:93 +0x3d fp=0xc8206e0b80 sp=0xc8206e0ac8
internal/singleflight.(*Group).Do(0x1b909b0, 0xc82000e790, 0x9, 0xc8206e0d18, 0x0, 0x0, 0x0, 0x0, 0xc8206e0c80)
        /tmp/workdir/go/src/internal/singleflight/singleflight.go:63 +0x3c8 fp=0xc8206e0c20 sp=0xc8206e0b80
net.lookupIPMerge(0xc82000e790, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup.go:69 +0xd2 fp=0xc8206e0d50 sp=0xc8206e0c20
net.lookupIPDeadline(0xc82000e790, 0x9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/lookup.go:91 +0xef fp=0xc8206e0f88 sp=0xc8206e0d50
net.internetAddrList(0xca52a8, 0x3, 0xc82000e790, 0xc, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/workdir/go/src/net/ipsock.go:252 +0x775 fp=0xc8206e1120 sp=0xc8206e0f88
net.resolveAddrList(0xca1408, 0x4, 0xca52a8, 0x3, 0xc82000e790, 0xc, 0x0, 0x0, 0x0, 0x0, ...)
        /tmp/workdir/go/src/net/dial.go:153 +0x4be fp=0xc8206e1228 sp=0xc8206e1120
net.(*Dialer).Dial(0xc8206e1478, 0xca52a8, 0x3, 0xc82000e790, 0xc, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/dial.go:209 +0x180 fp=0xc8206e1410 sp=0xc8206e1228
net.Dial(0xca52a8, 0x3, 0xc82000e790, 0xc, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/dial.go:186 +0xa6 fp=0xc8206e14c8 sp=0xc8206e1410
net/smtp.Dial(0xc82000e790, 0xc, 0x18, 0x0, 0x0)
        /usr/local/go/src/net/smtp/smtp.go:46 +0x70 fp=0xc8206e1550 sp=0xc8206e14c8
github.com/prometheus/alertmanager/notify.(*Email).Notify(0xc8201f27d0, 0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3c00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:267 +0xdd fp=0xc8206e1868 sp=0xc8206e1550
github.com/prometheus/alertmanager/notify.Build.func1.1(0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:99 +0x16b fp=0xc8206e19b8 sp=0xc8206e1868
github.com/prometheus/alertmanager/notify.NotifierFunc.Notify(0xc8202064b0, 0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/impl.go:70 +0x72 fp=0xc8206e1a08 sp=0xc8206e19b8
github.com/prometheus/alertmanager/notify.(*RetryNotifier).Notify(0xc8201f2870, 0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:192 +0x21b fp=0xc8206e1b98 sp=0xc8206e1a08
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201fc5a0, 0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245 fp=0xc8206e1c50 sp=0xc8206e1b98
github.com/prometheus/alertmanager/notify.(*DedupingNotifier).Notify(0xc8201fc5c0, 0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:290 +0x9c0 fp=0xc8206e1e10 sp=0xc8206e1c50
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201fc5e0, 0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245 fp=0xc8206e1ec8 sp=0xc8206e1e10
github.com/prometheus/alertmanager/notify.Fanout.Notify.func1(0x7fe3b12f3120, 0xc82066f0b0, 0xc8200d3400, 0x2a, 0x40, 0xc820596f40, 0xc820645f10, 0x7fe3b12f2bd
8, 0xc8201fc5e0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:150 +0x97 fp=0xc8206e1f58 sp=0xc8206e1ec8
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1696 +0x1 fp=0xc8206e1f60 sp=0xc8206e1f58
created by github.com/prometheus/alertmanager/notify.Fanout.Notify
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:155 +0x723

goroutine 1 [chan receive, 7 minutes]:
main.main()
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:204 +0x149b

goroutine 17 [syscall, 7 minutes, locked to thread]:
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 36 [syscall, 7 minutes]:
os/signal.loop()
        /usr/local/go/src/os/signal/signal_unix.go:22 +0x26
created by os/signal.init.1
        /usr/local/go/src/os/signal/signal_unix.go:28 +0x45

goroutine 29 [select, 7 minutes, locked to thread]:
runtime.gopark(0xdb0e48, 0xc820184728, 0xca4a30, 0x6, 0x18, 0x2)
        /tmp/workdir/go/src/runtime/proc.go:185 +0x11d
runtime.selectgoImpl(0xc820184728, 0xc820184714, 0x0)
        /tmp/workdir/go/src/runtime/select.go:392 +0xb9d
runtime.selectgo(0xc820184728)
        /tmp/workdir/go/src/runtime/select.go:212 +0x12
runtime.ensureSigM.func1()
        /tmp/workdir/go/src/runtime/signal1_unix.go:227 +0x2ea
runtime.goexit()
        /tmp/workdir/go/src/runtime/asm_amd64.s:1696 +0x1

goroutine 5 [chan receive, 7 minutes]:
database/sql.(*DB).connectionOpener(0xc8201980a0)
        /usr/local/go/src/database/sql/sql.go:634 +0x66
created by database/sql.Open
        /usr/local/go/src/database/sql/sql.go:481 +0x439

goroutine 28 [IO wait]:
net.runtime_pollWait(0x7fe3b0249588, 0x72, 0x7fe3b0288028)
        /tmp/workdir/go/src/runtime/netpoll.go:157 +0x63
net.(*pollDesc).Wait(0xc82024e8b0, 0x72, 0x0, 0x0)
        /tmp/workdir/go/src/net/fd_poll_runtime.go:73 +0x56
net.(*pollDesc).WaitRead(0xc82024e8b0, 0x0, 0x0)
        /tmp/workdir/go/src/net/fd_poll_runtime.go:78 +0x44
net.(*netFD).accept(0xc82024e850, 0x0, 0x7fe3b0249680, 0xc820520340)
        /tmp/workdir/go/src/net/fd_unix.go:408 +0x2f6
net.(*TCPListener).AcceptTCP(0xc8200fc228, 0xc8204dcd50, 0x0, 0x0)
        /tmp/workdir/go/src/net/tcpsock_posix.go:254 +0x77
net/http.tcpKeepAliveListener.Accept(0xc8200fc228, 0x0, 0x0, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:2135 +0x4f
net/http.(*Server).Serve(0xc82011a840, 0x7fe3b0249648, 0xc8200fc228, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:1887 +0xc4
net/http.(*Server).ListenAndServe(0xc82011a840, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:1877 +0x175
net/http.ListenAndServe(0xc9d518, 0x5, 0x7fe3b02485b0, 0xc8201fc6c0, 0x0, 0x0)
        /tmp/workdir/go/src/net/http/server.go:1967 +0xe3
main.listen(0xc8201fc6c0)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:255 +0xba
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:189 +0x1177

goroutine 7 [select]:
main.(*Dispatcher).run(0xc82019e480, 0x7fe3b0290c48, 0xc82051bfa0)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:150 +0xa59
main.(*Dispatcher).Run(0xc82019e480)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:59 +0x2b6
created by main.main.func3
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:175 +0x77a

goroutine 98 [chan receive, 7 minutes]:
main.main.func4(0xc820076d80, 0xc82001a460)
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:199 +0x5b
created by main.main
        /home/gocode/src/github.com/prometheus/alertmanager/main.go:202 +0x1478

goroutine 30 [chan receive, 7 minutes]:
github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe.func1(0xc8200142c0, 0x0, 0xc820076a20, 0xc8200d3000, 0x2a, 0x40, 0xc820076a80)
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:97 +0x189
created by github.com/prometheus/alertmanager/provider/sqlite.(*Alerts).Subscribe
        /home/gocode/src/github.com/prometheus/alertmanager/provider/sqlite/sqlite.go:98 +0x24b

goroutine 170 [semacquire]:
sync.runtime_Semacquire(0xc820645f1c)
        /tmp/workdir/go/src/runtime/sema.go:43 +0x26
sync.(*WaitGroup).Wait(0xc820645f10)
        /tmp/workdir/go/src/sync/waitgroup.go:126 +0x118
github.com/prometheus/alertmanager/notify.Fanout.Notify(0xc820206480, 0x7fe3b12f3120, 0xc82073a1b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:158 +0x755
github.com/prometheus/alertmanager/notify.Router.Notify(0xc820206420, 0x7fe3b12f3120, 0xc82073a1b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:313 +0x308
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201fc660, 0x7fe3b12f3120, 0xc82073a1b0, 0xc8200d3400, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245
github.com/prometheus/alertmanager/notify.(*SilenceNotifier).Notify(0xc820206720, 0x7fe3b12f3120, 0xc82073a1b0, 0xc820116a00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:348 +0x37d
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201fc680, 0x7fe3b12f3120, 0xc82073a1b0, 0xc820116a00, 0x2a, 0x40, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245
github.com/prometheus/alertmanager/notify.(*InhibitNotifier).Notify(0xc8202067b0, 0x7fe3b12f3120, 0xc82073a1b0, 0xc820099a20, 0x2a, 0x2a, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:383 +0x37d
github.com/prometheus/alertmanager/notify.(*LogNotifier).Notify(0xc8201fc6a0, 0x7fe3b12f3120, 0xc82073a1b0, 0xc820099a20, 0x2a, 0x2a, 0x0, 0x0)
        /home/gocode/src/github.com/prometheus/alertmanager/notify/notify.go:403 +0x245
main.(*Dispatcher).processAlert.func1(0x7fe3b12f3120, 0xc82073a1b0, 0xc820099a20, 0x2a, 0x2a, 0xc820596ba0)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:236 +0xd5
main.(*aggrGroup).run.func1(0xc820099a20, 0x2a, 0x2a, 0x2)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:336 +0x69
main.(*aggrGroup).flush(0xc8206bda80, 0xc8206c5ec8)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:400 +0x638
main.(*aggrGroup).run(0xc8206bda80, 0xc8206da930)
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:337 +0x10ee
created by main.(*Dispatcher).processAlert
        /home/gocode/src/github.com/prometheus/alertmanager/dispatch.go:241 +0x578

goroutine 340 [select]:
github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.(*Ticker).run(0xc820647500)
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:55 +0x27e
created by github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff.NewTicker
        /home/gocode/src/github.com/prometheus/alertmanager/vendor/github.com/cenkalti/backoff/ticker.go:32 +0x1b1

[matthiasr]

It's segfaulting while trying to call into the libc for name resolution. Is there possibly a mismatch between the libraries in your build and runtime environments? Maybe try with netgo?

[RichiH]

All binaries are built on the same machine as they are running on, so other than the Go dependencies which are pulled in automagically, this sounds unlikely.

Shouldn't it be possible to catch something like that instead of just keeling over and dying, though?

[matthiasr]

No, this is not a recoverable condition. An error handler could not do anything but keel over and die either.

[RichiH]

@matthiasr What do you mean "Maybe try with netgo"? I built this on the same machine I am running it on. 22b37c8 crashes as well in case you are interested in another dump.

[matthiasr]

@RichiH depending on the version of Go, when CGO is enabled (which it needs to be for Alertmanager), it will use the system's libc.so to resolve DNS – and from the stack trace, the SEGFAULT happens while within this code path, probably while in the C library. I don't know what's happening there, but I do know that glibc does funky things like trying to load yet another shared library. It's curious that only Alertmanager tickles this, but unless we find out why it's going to be tricky to work around.

However, one possible workaround would be to avoid the libc altogether and use Go's builtin resolver – from what I gather, you need to add -tags netgo -pkgdir some_dir to the build line. The former disables the libc name resolution, the latter makes it put intermediate libraries in some_dir which should, if I read golang/go#10210 right, make it re-build the Go standard library respecting the netgo tag.

Alternatively, maybe try with Go 1.6? Sorry I can't be more concretely helpful here. What distribution is this on, and how did you install Go?

[RichiH]

@matthiasr OK, thanks. I can try building with netgo, but I would like to test Go 1.6 first; if that fixes it bumping the requirements would suffice.

This is a Fedora system with Go 1.4 installed via packages and Go 1.5 via that automagic requirement handling. I tried bumping the auto-downloader to 1.6 via

adm@prometheus [2] [7] (git)-[master] ~/gocode/src/github.com/prometheus/alertmanager % git diff
diff --git a/scripts/goenv.sh b/scripts/goenv.sh
index 801bcdf..a8c9735 100755
--- a/scripts/goenv.sh
+++ b/scripts/goenv.sh
@@ -14,8 +14,8 @@
 goroot="$1"
 gopath="$2"

-go_version_min="1.5"
-go_version_install="1.5.1"
+go_version_min="1.6"
+go_version_install="1.6.0"

 vernum() {
        printf "%03d%03d%03d" $(echo "$1" | tr '.' ' ')
adm@prometheus [2] [7] (git)-[master] ~/gocode/src/github.com/prometheus/alertmanager % make
[...]

but that didn't pull in 1.6 automagically.

[RichiH]

Update: Go 1.6 didn't fix it.