prometheus crash case

[jinhang]

[jinhang]

Can someone tell me how to deal with crash?

[jinhang]

time="2016-12-21T15:51:26+08:00" level=info msg="Starting prometheus (version=1.2.3, branch=master, revision=c1eee5b0da2540b9dfd2f70752015b0fce83b616)" source="main.go:75"
time="2016-12-21T15:51:26+08:00" level=info msg="Build context (go=go1.7.3, user=root@d8eb84e17a12, date=20161103-21:49:39)" source="main.go:76"
time="2016-12-21T15:51:26+08:00" level=info msg="Loading configuration file prometheus.yml" source="main.go:247"
time="2016-12-21T15:51:26+08:00" level=info msg="Loading series map and head chunks..." source="storage.go:354"
time="2016-12-21T15:51:28+08:00" level=info msg="40705 series loaded." source="storage.go:359"
time="2016-12-21T15:51:28+08:00" level=info msg="Listening on :9090" source="web.go:240"
time="2016-12-21T15:51:28+08:00" level=info msg="Starting target manager..." source="targetmanager.go:76"
time="2016-12-21T15:56:28+08:00" level=info msg="Checkpointing in-memory metrics and chunks..." source="persistence.go:549"
time="2016-12-21T15:56:38+08:00" level=info msg="Done checkpointing in-memory metrics and chunks in 9.6899689s." source="persistence.go:573"
time="2016-12-21T16:01:38+08:00" level=info msg="Checkpointing in-memory metrics and chunks..." source="persistence.go:549"
time="2016-12-21T16:01:47+08:00" level=info msg="Done checkpointing in-memory metrics and chunks in 8.6828682s." source="persistence.go:573"
runtime: out of memory: cannot allocate 720896-byte block (1342177280 in use)
fatal error: out of memory

runtime stack:
runtime.throw(0xcc6f3c, 0xd)
/usr/local/go/src/runtime/panic.go:566 +0x7f
runtime.largeAlloc(0xa0484, 0x3d839901, 0x4573a0)
/usr/local/go/src/runtime/malloc.go:776 +0xb7
runtime.mallocgc.func1()
/usr/local/go/src/runtime/malloc.go:669 +0x31
runtime.systemstack(0x136d0a00)
/usr/local/go/src/runtime/asm_386.s:323 +0x5e
runtime.mstart()
/usr/local/go/src/runtime/proc.go:1079

goroutine 38137 [running]:
runtime.systemstack_switch()
/usr/local/go/src/runtime/asm_386.s:277 fp=0x6504b918 sp=0x6504b914
runtime.mallocgc(0xa0484, 0xc09ac0, 0x25ea5001, 0xa)
/usr/local/go/src/runtime/malloc.go:670 +0x7b2 fp=0x6504b980 sp=0x6504b918
runtime.newobject(0xc09ac0, 0x0)
/usr/local/go/src/runtime/malloc.go:785 +0x2c fp=0x6504b994 sp=0x6504b980
compress/flate.NewWriter(0x555501a0, 0x3cf2be10, 0xffffffff, 0xa, 0x0, 0x0)
/usr/local/go/src/compress/flate/deflate.go:660 +0x2f fp=0x6504b9b4 sp=0x6504b994
compress/gzip.(*Writer).Write(0x25ea5080, 0x5e65e000, 0xd739, 0x11c78, 0x64246e80, 0x0, 0x0)
/usr/local/go/src/compress/gzip/gzip.go:190 +0x4cb fp=0x6504ba10 sp=0x6504b9b4
github.com/prometheus/prometheus/util/httputil.(*compressedResponseWriter).Write(0x6123c6a0, 0x5e65e000, 0xd739, 0x11c78, 0x11c78, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:40 +0x56 fp=0x6504ba38 sp=0x6504ba10
github.com/prometheus/prometheus/web/api/v1.respond(0x1093e00, 0x6123c6a0, 0xb48840, 0x60ad1450)
/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:330 +0x129 fp=0x6504ba78 sp=0x6504ba38
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x1093e00, 0x6123c6a0, 0x5ea03b80)
/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:113 +0x9e fp=0x6504ba9c sp=0x6504ba78
net/http.HandlerFunc.ServeHTTP(0x1387abe8, 0x1093e00, 0x6123c6a0, 0x5ea03b80)
/usr/local/go/src/net/http/server.go:1726 +0x34 fp=0x6504baac sp=0x6504ba9c
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x1091ae0, 0x1387abe8, 0x1093ee0, 0x3cf2be10, 0x5ea03b80)
/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90 +0x6e fp=0x6504bad4 sp=0x6504baac
github.com/prometheus/prometheus/util/httputil.(*CompressionHandler).ServeHTTP(0x1387abf0, 0x1093ee0, 0x3cf2be10, 0x5ea03b80)
:5 +0x9b fp=0x6504baf0 sp=0x6504bad4
net/http.(Handler).ServeHTTP-fm(0x1093ee0, 0x3cf2be10, 0x5ea03b80)
/go/src/github.com/prometheus/prometheus/web/web.go:173 +0x45 fp=0x6504bb0c sp=0x6504baf0
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x10941e0, 0x4cb11dd0, 0x5ea03b80)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2a4 fp=0x6504bbcc sp=0x6504bb0c
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x10941e0, 0x4cb11dd0, 0x5ea03b80, 0x0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83 +0x34b fp=0x6504bc80 sp=0x6504bbcc
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0x13937e80, 0x10941e0, 0x4cb11dd0, 0x5ea03b80)
/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x148 fp=0x6504bd20 sp=0x6504bc80
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0x136c5080, 0x10941e0, 0x4cb11dd0, 0x5ea03b80)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:125 +0x3b fp=0x6504bd34 sp=0x6504bd20
net/http.serverHandler.ServeHTTP(0x3a4d82d0, 0x10941e0, 0x4cb11dd0, 0x5ea03b80)
/usr/local/go/src/net/http/server.go:2202 +0x136 fp=0x6504bd5c sp=0x6504bd34
net/http.(*conn).serve(0x4bae77c0, 0x1094c60, 0x4a2481c0)
/usr/local/go/src/net/http/server.go:1579 +0xe86 fp=0x6504bf88 sp=0x6504bd5c
runtime.goexit()
/usr/local/go/src/runtime/asm_386.s:1612 +0x1 fp=0x6504bf8c sp=0x6504bf88
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2293 +0x489

goroutine 1 [select, 15 minutes]:
main.Main(0x0)
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:227 +0x1bdb
main.main()
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:43 +0x1a

goroutine 12 [syscall, 15 minutes]:
os/signal.signal_recv(0x438253)
/usr/local/go/src/runtime/sigqueue.go:116 +0x11e
os/signal.loop()
/usr/local/go/src/os/signal/signal_unix.go:22 +0x1a
created by os/signal.init.1
/usr/local/go/src/os/signal/signal_unix.go:28 +0x36

goroutine 115 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0x138a8100)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x4b3
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x55c

goroutine 136 [select, 15 minutes]:
main.Main.func2(0x137ab640, 0x137ab600, 0x13720b40, 0x5, 0x6, 0x136c0980)
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:162 +0x2da
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:176 +0x11da

goroutine 135 [select]:
github.com/prometheus/prometheus/rules.(*Group).run(0x136ca840)
/go/src/github.com/prometheus/prometheus/rules/manager.go:168 +0x218
github.com/prometheus/prometheus/rules.(*Manager).ApplyConfig.func1.1(0x139389c0, 0x136ca840)
/go/src/github.com/prometheus/prometheus/rules/manager.go:425 +0x40
created by github.com/prometheus/prometheus/rules.(*Manager).ApplyConfig.func1
/go/src/github.com/prometheus/prometheus/rules/manager.go:426 +0x8b

goroutine 137 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0x138c8420)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x22f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1da

goroutine 116 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0x138a8100)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x1ef
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x579

goroutine 117 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0x138a8100)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x649
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x6f8

goroutine 118 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0x138a8100)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x1f6
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x715

goroutine 119 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0x138c86e0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x22f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1da

goroutine 120 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0x138a8300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x4b3
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x55c

goroutine 121 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0x138a8300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x1ef
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x579

goroutine 122 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0x138a8300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x649
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x6f8

goroutine 123 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0x138a8300)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x1f6
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x715

goroutine 124 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0x138c8580)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x22f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1da

goroutine 125 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0x138a8200)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x4b3
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x55c

goroutine 126 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0x138a8200)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x1ef
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x579

goroutine 127 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0x138a8200)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x649
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x6f8

goroutine 128 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0x138a8200)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x1f6
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x715

goroutine 145 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.(*BufferPool).drain(0x138c8b00)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:206 +0x22f
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util.NewBufferPool
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/util/buffer_pool.go:237 +0x1da

goroutine 146 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).compactionError(0x138a8700)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:90 +0x4b3
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:139 +0x55c

goroutine 147 [select]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mpoolDrain(0x138a8700)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_state.go:96 +0x1ef
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:140 +0x579

goroutine 148 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).tCompaction(0x138a8700)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:804 +0x649
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:146 +0x6f8

goroutine 149 [select, 15 minutes]:
github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.(*DB).mCompaction(0x138a8700)
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db_compaction.go:751 +0x1f6
created by github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb.openDB
/go/src/github.com/prometheus/prometheus/vendor/github.com/syndtr/goleveldb/leveldb/db.go:147 +0x715

goroutine 150 [select]:
github.com/prometheus/prometheus/storage/local.(*persistence).processIndexingQueue(0x13704090)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:1278 +0x9c2
github.com/prometheus/prometheus/storage/local.(*persistence).run(0x13704090)
/go/src/github.com/prometheus/prometheus/storage/local/persistence.go:291 +0x21
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:344 +0x106

goroutine 46 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleEvictList(0x13986000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1016 +0x3bd
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:367 +0x348

goroutine 47 [select, 15 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleQuarantine(0x13986000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1626 +0x3b9
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:368 +0x365

goroutine 48 [select, 15 minutes]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).logThrottling(0x13986000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:887 +0x6d1
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:369 +0x382

goroutine 161 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).loop(0x13986000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1233 +0x54b
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:370 +0x39f

goroutine 159 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526a20, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 177 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).waitForNextFP(0x13986000, 0x9ef4, 0x2c000000, 0x3feff972, 0x1)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1120 +0x218
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughMemoryFingerprints.func1(0x3a4f2040, 0x13986000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1160 +0x21e
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughMemoryFingerprints
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1170 +0x5a

goroutine 178 [select]:
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).waitForNextFP(0x13986000, 0x0, 0x0, 0x3ff00000, 0x0)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1120 +0x218
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughArchivedFingerprints.func1(0x3a4f2080, 0x13986000)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1193 +0x22d
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).cycleThroughArchivedFingerprints
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1213 +0x5a

goroutine 166 [select, 15 minutes]:
github.com/prometheus/prometheus/notifier.(*Notifier).Run(0x1391f380)
/go/src/github.com/prometheus/prometheus/notifier/notifier.go:181 +0x241
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:207 +0x164d

goroutine 141 [select, 15 minutes]:
github.com/prometheus/prometheus/retrieval.(*targetSet).runScraping(0x3a4f4240, 0x33da00f8, 0x2658f4a0)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:237 +0x236
github.com/prometheus/prometheus/retrieval.(*TargetManager).reload.func1(0x136cdd60, 0x3a4f4240)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:120 +0x3c
created by github.com/prometheus/prometheus/retrieval.(*TargetManager).reload
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:122 +0x1b9

goroutine 168 [semacquire, 15 minutes]:
sync.runtime_Semacquire(0x136cdda4)
/usr/local/go/src/runtime/sema.go:47 +0x26
sync.(*WaitGroup).Wait(0x136cdd98)
/usr/local/go/src/sync/waitgroup.go:131 +0xa4
github.com/prometheus/prometheus/retrieval.(*TargetManager).Run(0x136cdd60)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:85 +0x17c
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:213 +0x16d3

goroutine 169 [IO wait]:
net.runtime_pollWait(0x44ae0f98, 0x72, 0x575b84c4)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x3a087170, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x3a0870d8, 0xcc208f, 0x8, 0x575b84e0, 0x1dda09a0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).acceptOne(0x3a0870a0, 0x1dda08c0, 0x2, 0x2, 0x3a0870d8, 0xd3e2cf, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:529 +0x1d4
net.(*netFD).accept(0x3a0870a0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:559 +0x133
net.(*TCPListener).accept(0x136fa300, 0xe8e54a3f, 0x0, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:132 +0x2d
net.(*TCPListener).AcceptTCP(0x136fa300, 0x457b70, 0x0, 0x0)
/usr/local/go/src/net/tcpsock.go:209 +0x8f
net/http.tcpKeepAliveListener.Accept(0x136fa300, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2608 +0x3d
net/http.(*Server).Serve(0x3a4d82d0, 0x1094620, 0x136fa300, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2273 +0x216
net/http.(*Server).ListenAndServe(0x3a4d82d0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:2219 +0xd9
github.com/prometheus/prometheus/web.(*Handler).Run(0x136c0980)
/go/src/github.com/prometheus/prometheus/web/web.go:246 +0x1e5
created by main.Main
/go/src/github.com/prometheus/prometheus/cmd/prometheus/main.go:220 +0x1733

goroutine 138 [select, 15 minutes]:
github.com/prometheus/prometheus/retrieval.(*targetSet).runScraping(0x3a4f4150, 0x33da00f8, 0x2658f0a0)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:237 +0x236
github.com/prometheus/prometheus/retrieval.(*TargetManager).reload.func1(0x136cdd60, 0x3a4f4150)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:120 +0x3c
created by github.com/prometheus/prometheus/retrieval.(*TargetManager).reload
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:122 +0x1b9

goroutine 151 [select, 15 minutes]:
github.com/prometheus/prometheus/retrieval.(*targetSet).runScraping(0x3a5260f0, 0x33da00f8, 0x34c92280)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:237 +0x236
github.com/prometheus/prometheus/retrieval.(*TargetManager).reload.func1(0x136cdd60, 0x3a5260f0)
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:120 +0x3c
created by github.com/prometheus/prometheus/retrieval.(*TargetManager).reload
/go/src/github.com/prometheus/prometheus/retrieval/targetmanager.go:122 +0x1b9

goroutine 38181 [runnable, locked to thread]:
syscall.Syscall(0x76d730af, 0x3, 0x28c, 0x5e4d17b8, 0x5e4d17b4, 0x2, 0x434c98, 0x3a52432c)
/usr/local/go/src/runtime/syscall_windows.go:163 +0x4a
syscall.getsockname(0x28c, 0x5e4d17b8, 0x5e4d17b4, 0x0, 0x0)
/usr/local/go/src/syscall/zsyscall_windows.go:1443 +0x6f
syscall.Getsockname(0x28c, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/syscall/syscall_windows.go:736 +0x76
net.(*netFD).dial(0x2067a540, 0x1094ce0, 0x50561a80, 0x1096e60, 0x0, 0x1096e60, 0x3e82f620, 0x0, 0x0)
/usr/local/go/src/net/sock_posix.go:146 +0x293
net.socket(0x1094ce0, 0x50561a80, 0xcbdaf6, 0x3, 0x2, 0x1, 0x0, 0x0, 0x1096e60, 0x0, ...)
/usr/local/go/src/net/sock_posix.go:89 +0x248
net.internetSocket(0x1094ce0, 0x50561a80, 0xcbdaf6, 0x3, 0x1096e60, 0x0, 0x1096e60, 0x3e82f620, 0x1, 0x0, ...)
/usr/local/go/src/net/ipsock_posix.go:158 +0xd0
net.doDialTCP(0x1094ce0, 0x50561a80, 0xcbdaf6, 0x3, 0x0, 0x3e82f620, 0x0, 0x0, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:58 +0x98
net.dialTCP(0x1094ce0, 0x50561a80, 0xcbdaf6, 0x3, 0x0, 0x3e82f620, 0xb2d05e00, 0x0, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:54 +0xa4
net.dialSingle(0x1094ce0, 0x50561a80, 0x4e60f130, 0x1093100, 0x3e82f620, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/dial.go:501 +0x5e0
net.dialSerial(0x1094ce0, 0x50561a80, 0x4e60f130, 0x5772c380, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/dial.go:469 +0x7b3
net.(*Dialer).DialContext(0x10f8200, 0x1094ce0, 0x50561a80, 0xcbdaf6, 0x3, 0x577364b7, 0x11, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/dial.go:351 +0x939
net/http.(*Transport).dial(0x3a51c0a0, 0x1094ce0, 0x50561a80, 0xcbdaf6, 0x3, 0x577364b7, 0x11, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/http/transport.go:835 +0x194
net/http.(*Transport).dialConn(0x3a51c0a0, 0x1094ce0, 0x50561a80, 0x0, 0x577364b0, 0x4, 0x577364b7, 0x11, 0x1092860, 0x0, ...)
/usr/local/go/src/net/http/transport.go:967 +0x199a
net/http.(*Transport).getConn.func4(0x3a51c0a0, 0x1094ce0, 0x50561a80, 0x3e82f5c0, 0x50561b40)
/usr/local/go/src/net/http/transport.go:885 +0x48
created by net/http.(*Transport).getConn
/usr/local/go/src/net/http/transport.go:887 +0x3a0

goroutine 289 [select]:
net/http.(*Transport).getConn(0x3a51c0a0, 0x5e4abe40, 0x0, 0x5e4b4870, 0x4, 0x5e4b4877, 0x11, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:890 +0x997
net/http.(*Transport).RoundTrip(0x3a51c0a0, 0x49e8aa00, 0x4, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:367 +0xa97
github.com/prometheus/prometheus/util/httputil.(*basicAuthRoundTripper).RoundTrip(0x34c92140, 0x49e8aa00, 0x34c92140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/client.go:105 +0xfd
net/http.send(0x49e8a980, 0x1090420, 0x34c92140, 0x0, 0x0, 0x0, 0x0, 0x5e4abe10, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:256 +0x51b
net/http.(*Client).send(0x34c92160, 0x49e8a980, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:146 +0x127
net/http.(*Client).doFollowingRedirects(0x34c92160, 0x49e8a980, 0xd41b20, 0x3, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:528 +0x95f
net/http.(*Client).Do(0x34c92160, 0x49e8a980, 0x46435000, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:184 +0xc1
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x44ae10a0, 0x46435000, 0x34c92160, 0x49e8a900, 0x89, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x86
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x3a5d28e8, 0x44ae10a0, 0x46435000, 0xcfec2f11, 0xe, 0x14f97f04, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:320 +0x150
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e360, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 171 [chan send]:
github.com/prometheus/prometheus/storage/local.(*seriesMap).fpIter.func1(0x138ed120, 0x3a5a0240)
/go/src/github.com/prometheus/prometheus/storage/local/series.go:133 +0xb6
created by github.com/prometheus/prometheus/storage/local.(*seriesMap).fpIter
/go/src/github.com/prometheus/prometheus/storage/local/series.go:138 +0x5a

goroutine 292 [IO wait]:
net.runtime_pollWait(0x44ae0908, 0x72, 0x59f78a94)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x1dda0530, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x1dda0498, 0xcc1130, 0x7, 0xd41980, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x1dda0460, 0x483c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x52ff8ef8, 0x483c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*persistConn).Read(0x4cb10090, 0x483c0000, 0x1000, 0x1000, 0x430c11, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1261 +0x180
bufio.(*Reader).Read(0x52ffa1b0, 0x483c0000, 0x1000, 0x1000, 0x41393f, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:199 +0xce
io.(*LimitedReader).Read(0x425a1a10, 0x483c0000, 0x1000, 0x1000, 0x5d2af8b0, 0x0, 0x0)
/usr/local/go/src/io/io.go:436 +0xce
net/http.(*body).readLocked(0x425959e0, 0x483c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:651 +0x7f
net/http.(*body).Read(0x425959e0, 0x483c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transfer.go:643 +0xe8
net/http.(*bodyEOFSignal).Read(0x42595a10, 0x483c0000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1974 +0x130
bufio.(*Reader).fill(0x42595a70)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).ReadByte(0x42595a70, 0x4478fc, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:231 +0x6a
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.(*TextParser).skipBlankTab(0x35a6f1e8)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt/text_parse.go:525 +0x24
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.(*TextParser).startLabelValue(0x35a6f1e8, 0x4783e270)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt/text_parse.go:308 +0x32
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.(*TextParser).(github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.startLabelValue)-fm(0x4783e270)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt/text_parse.go:302 +0x20
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.(*TextParser).TextToMetricFamilies(0x35a6f1e8, 0x1090ce0, 0x42595a10, 0x0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt/text_parse.go:102 +0x1f1
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.(*textDecoder).Decode(0x35a6f1e0, 0x42595a4c, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt/decode.go:126 +0x63
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt.(*SampleDecoder).Decode(0x42595a40, 0x5d2afd1c, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/expfmt/decode.go:153 +0x59
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x3a5d2918, 0x44ae10a0, 0x3bb99d40, 0xcfec2f10, 0xe, 0x35940dc4, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:342 +0x5a5
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e5a0, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 158 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a5269c0, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 157 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526960, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 291 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e4e0, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 296 [select]:
net/http.(*persistConn).roundTrip(0x5217b320, 0x468d1840, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1840 +0xba1
net/http.(*Transport).RoundTrip(0x3a51c0a0, 0x4387dc00, 0x4, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:380 +0xc82
github.com/prometheus/prometheus/util/httputil.(*basicAuthRoundTripper).RoundTrip(0x34c92140, 0x4387dc00, 0x34c92140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/client.go:105 +0xfd
net/http.send(0x4387db80, 0x1090420, 0x34c92140, 0x0, 0x0, 0x0, 0x0, 0x468d1810, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:256 +0x51b
net/http.(*Client).send(0x34c92160, 0x4387db80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:146 +0x127
net/http.(*Client).doFollowingRedirects(0x34c92160, 0x4387db80, 0xd41b20, 0x3, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:528 +0x95f
net/http.(*Client).Do(0x34c92160, 0x4387db80, 0x5d0ebf40, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:184 +0xc1
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x44ae10a0, 0x5d0ebf40, 0x34c92160, 0x4387db00, 0x89, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x86
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x3a5d2958, 0x44ae10a0, 0x5d0ebf40, 0xcfec2f11, 0xe, 0xf7db004, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:320 +0x150
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e8a0, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 294 [select]:
net/http.(*persistConn).roundTrip(0x5217b440, 0x468d1970, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1840 +0xba1
net/http.(*Transport).RoundTrip(0x3a51c0a0, 0x4387df00, 0x4, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:380 +0xc82
github.com/prometheus/prometheus/util/httputil.(*basicAuthRoundTripper).RoundTrip(0x34c92140, 0x4387df00, 0x34c92140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/client.go:105 +0xfd
net/http.send(0x4387de80, 0x1090420, 0x34c92140, 0x0, 0x0, 0x0, 0x0, 0x468d1940, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:256 +0x51b
net/http.(*Client).send(0x34c92160, 0x4387de80, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:146 +0x127
net/http.(*Client).doFollowingRedirects(0x34c92160, 0x4387de80, 0xd41b20, 0x3, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:528 +0x95f
net/http.(*Client).Do(0x34c92160, 0x4387de80, 0x46dcc540, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:184 +0xc1
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x44ae10a0, 0x46dcc540, 0x34c92160, 0x4387de00, 0x89, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x86
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x3a5d2938, 0x44ae10a0, 0x46dcc540, 0xcfec2f11, 0xe, 0xf8cf244, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:320 +0x150
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e720, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 293 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e660, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 290 [select]:
net/http.(*persistConn).roundTrip(0x5217b290, 0x468d17a0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1840 +0xba1
net/http.(*Transport).RoundTrip(0x3a51c0a0, 0x4387da80, 0x4, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:380 +0xc82
github.com/prometheus/prometheus/util/httputil.(*basicAuthRoundTripper).RoundTrip(0x34c92140, 0x4387da80, 0x34c92140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/client.go:105 +0xfd
net/http.send(0x4387da00, 0x1090420, 0x34c92140, 0x0, 0x0, 0x0, 0x0, 0x468d1770, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:256 +0x51b
net/http.(*Client).send(0x34c92160, 0x4387da00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:146 +0x127
net/http.(*Client).doFollowingRedirects(0x34c92160, 0x4387da00, 0xd41b20, 0x3, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:528 +0x95f
net/http.(*Client).Do(0x34c92160, 0x4387da00, 0x5d0ebc40, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:184 +0xc1
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x44ae10a0, 0x5d0ebc40, 0x34c92160, 0x4387d980, 0x89, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x86
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x3a5d28f8, 0x44ae10a0, 0x5d0ebc40, 0xcfec2f11, 0xe, 0xf7db004, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:320 +0x150
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e420, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 295 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e7e0, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 223 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e0c0, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 224 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e2a0, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 38196 [chan send]:
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0x4f3fdc00, 0x43953680, 0xf7)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x14d
created by github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x14d

goroutine 160 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526a80, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 273 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526ae0, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 274 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526b40, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 275 [select]:
net/http.(*persistConn).roundTrip(0x5217b3b0, 0x468d18d0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1840 +0xba1
net/http.(*Transport).RoundTrip(0x3a51c140, 0x4387dd80, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:380 +0xc82
github.com/prometheus/prometheus/util/httputil.(*basicAuthRoundTripper).RoundTrip(0x34c92300, 0x4387dd80, 0x34c92300, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/client.go:105 +0xfd
net/http.send(0x4387dd00, 0x1090420, 0x34c92300, 0x0, 0x0, 0x0, 0x0, 0x468d18a0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:256 +0x51b
net/http.(*Client).send(0x34c92320, 0x4387dd00, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:146 +0x127
net/http.(*Client).doFollowingRedirects(0x34c92320, 0x4387dd00, 0xd41b20, 0x3, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:528 +0x95f
net/http.(*Client).Do(0x34c92320, 0x4387dd00, 0x46dcc240, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:184 +0xc1
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x44ae10a0, 0x46dcc240, 0x34c92320, 0x4387dc80, 0x89, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x86
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x136dee18, 0x44ae10a0, 0x46dcc240, 0xcfec2f11, 0xe, 0xf8cf244, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:320 +0x150
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526ba0, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 276 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526c00, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 277 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526c60, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 278 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a526cc0, 0x540be400, 0x2, 0x540be400, 0x2, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 297 [select]:
net/http.(*Transport).getConn(0x3a51c0a0, 0x577329a0, 0x0, 0x577364b0, 0x4, 0x577364b7, 0x11, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:890 +0x997
net/http.(*Transport).RoundTrip(0x3a51c0a0, 0x62514400, 0x4, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:367 +0xa97
github.com/prometheus/prometheus/util/httputil.(*basicAuthRoundTripper).RoundTrip(0x34c92140, 0x62514400, 0x34c92140, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/util/httputil/client.go:105 +0xfd
net/http.send(0x62514380, 0x1090420, 0x34c92140, 0x0, 0x0, 0x0, 0x0, 0x57732970, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:256 +0x51b
net/http.(*Client).send(0x34c92160, 0x62514380, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:146 +0x127
net/http.(*Client).doFollowingRedirects(0x34c92160, 0x62514380, 0xd41b20, 0x3, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:528 +0x95f
net/http.(*Client).Do(0x34c92160, 0x62514380, 0x50561a80, 0x0, 0x0)
/usr/local/go/src/net/http/client.go:184 +0xc1
github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp.Do(0x44ae10a0, 0x50561a80, 0x34c92160, 0x62514300, 0x89, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/golang.org/x/net/context/ctxhttp/ctxhttp.go:30 +0x86
github.com/prometheus/prometheus/retrieval.(*targetScraper).scrape(0x3a5d2968, 0x44ae10a0, 0x50561a80, 0xcfec2f11, 0xe, 0x14f97f04, 0x10f81c0, 0x0, 0x0, 0x0, ...)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:320 +0x150
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60e960, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:420 +0x590
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 298 [select]:
github.com/prometheus/prometheus/retrieval.(*scrapeLoop).run(0x3a60ea20, 0xb2d05e00, 0x0, 0xb2d05e00, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:433 +0x791
created by github.com/prometheus/prometheus/retrieval.(*scrapePool).sync
/go/src/github.com/prometheus/prometheus/retrieval/scrape.go:233 +0x41a

goroutine 38175 [select]:
net/http.(*persistConn).writeLoop(0x5217b440)
/usr/local/go/src/net/http/transport.go:1646 +0x2ba
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1063 +0xdc5

goroutine 38226 [IO wait]:
net.runtime_pollWait(0x44ae0d40, 0x72, 0x6123c684)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x212a2a70, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x212a29d8, 0xcc1130, 0x7, 0xd41980, 0x2, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x212a29a0, 0x3f6d5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x3cf2bd68, 0x3f6d5000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*persistConn).Read(0x5217b3b0, 0x3f6d5000, 0x1000, 0x1000, 0x4127ae, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1261 +0x180
bufio.(*Reader).fill(0x6123e240)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).Peek(0x6123e240, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:129 +0x8f
net/http.(*persistConn).readLoop(0x5217b3b0)
/usr/local/go/src/net/http/transport.go:1418 +0x183
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1062 +0xda5

goroutine 38168 [IO wait]:
net.runtime_pollWait(0x44ae0ea8, 0x72, 0x5773294c)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x1cc408b0, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x1cc40818, 0xcc1130, 0x7, 0xd41980, 0x3, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x1cc407e0, 0x58415000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x4a0569f8, 0x58415000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*connReader).Read(0x4a054c80, 0x58415000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:586 +0x189
bufio.(*Reader).fill(0x4a04d080)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).ReadSlice(0x4a04d080, 0xd3e40a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:330 +0x1b4
bufio.(*Reader).ReadLine(0x4a04d080, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:359 +0x4b
net/textproto.(*Reader).readLineSlice(0x4a054ce0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/textproto/reader.go:55 +0x59
net/textproto.(*Reader).ReadLine(0x4a054ce0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/textproto/reader.go:36 +0x39
net/http.readRequest(0x4a04d080, 0xffffff00, 0x62514480, 0x0, 0x0)
/usr/local/go/src/net/http/request.go:793 +0x9a
net/http.(*conn).readRequest(0x45e8cf40, 0x1094c60, 0x4a054cc0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:765 +0x3c0
net/http.(*conn).serve(0x45e8cf40, 0x1094c60, 0x4a054cc0)
/usr/local/go/src/net/http/server.go:1532 +0x96c
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2293 +0x489

goroutine 38219 [IO wait]:
net.runtime_pollWait(0x44ae0ae8, 0x77, 0x5e4abde8)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x212a2c30, 0x77, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f8, 0x212a2be4, 0xcc2fbe, 0x9, 0xd41994, 0x46435280, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).connect(0x212a2b60, 0x1094ce0, 0x46435000, 0x1091120, 0x5e4b6740, 0x1091120, 0x5e4b6720, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:362 +0x613
net.(*netFD).dial(0x212a2b60, 0x1094ce0, 0x46435000, 0x1096e60, 0x0, 0x1096e60, 0x5e4b6700, 0x0, 0x0)
/usr/local/go/src/net/sock_posix.go:137 +0x248
net.socket(0x1094ce0, 0x46435000, 0xcbdaf6, 0x3, 0x2, 0x1, 0x0, 0x0, 0x1096e60, 0x0, ...)
/usr/local/go/src/net/sock_posix.go:89 +0x248
net.internetSocket(0x1094ce0, 0x46435000, 0xcbdaf6, 0x3, 0x1096e60, 0x0, 0x1096e60, 0x5e4b6700, 0x1, 0x0, ...)
/usr/local/go/src/net/ipsock_posix.go:158 +0xd0
net.doDialTCP(0x1094ce0, 0x46435000, 0xcbdaf6, 0x3, 0x0, 0x5e4b6700, 0x0, 0x0, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:58 +0x98
net.dialTCP(0x1094ce0, 0x46435000, 0xcbdaf6, 0x3, 0x0, 0x5e4b6700, 0xb2d05e00, 0x0, 0x0)
/usr/local/go/src/net/tcpsock_posix.go:54 +0xa4
net.dialSingle(0x1094ce0, 0x46435000, 0x5dd2c910, 0x1093100, 0x5e4b6700, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/dial.go:501 +0x5e0
net.dialSerial(0x1094ce0, 0x46435000, 0x5dd2c910, 0x5e4b81a0, 0x1, 0x1, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/dial.go:469 +0x7b3
net.(*Dialer).DialContext(0x10f8200, 0x1094ce0, 0x46435000, 0xcbdaf6, 0x3, 0x5e4b4877, 0x11, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/dial.go:351 +0x939
net/http.(*Transport).dial(0x3a51c0a0, 0x1094ce0, 0x46435000, 0xcbdaf6, 0x3, 0x5e4b4877, 0x11, 0x0, 0x0, 0x0, ...)
/usr/local/go/src/net/http/transport.go:835 +0x194
net/http.(*Transport).dialConn(0x3a51c0a0, 0x1094ce0, 0x46435000, 0x0, 0x5e4b4870, 0x4, 0x5e4b4877, 0x11, 0x1092860, 0x0, ...)
/usr/local/go/src/net/http/transport.go:967 +0x199a
net/http.(*Transport).getConn.func4(0x3a51c0a0, 0x1094ce0, 0x46435000, 0x5e4b66a0, 0x464350c0)
/usr/local/go/src/net/http/transport.go:885 +0x48
created by net/http.(*Transport).getConn
/usr/local/go/src/net/http/transport.go:887 +0x3a0

goroutine 38133 [select]:
net/http.(*persistConn).readLoop(0x4cb10090)
/usr/local/go/src/net/http/transport.go:1541 +0xe13
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1062 +0xda5

goroutine 38171 [runnable]:
encoding/json.(*encodeState).marshal(0x5318c000, 0xc18d20, 0x5317d2d8, 0xc10100, 0x5317d2d8, 0x6535f448)
/usr/local/go/src/encoding/json/encode.go:268
encoding/json.Marshal(0xc18d20, 0x5317d2d8, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/encoding/json/encode.go:145 +0x85
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/model.SamplePair.MarshalJSON(0x205cb640, 0x159, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/model/value.go:89 +0x105
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/model.(*SamplePair).MarshalJSON(0x5317d2c0, 0x0, 0x0, 0x0, 0x0, 0x0)
:53 +0x9b
encoding/json.marshalerEncoder(0x45f6cd20, 0xc47d20, 0x5e4f0300, 0x199, 0xc40100)
/usr/local/go/src/encoding/json/encode.go:431 +0xee
encoding/json.(*arrayEncoder).encode(0x58b32880, 0x45f6cd20, 0xb5f240, 0x62ff3b64, 0x197, 0x600100)
/usr/local/go/src/encoding/json/encode.go:723 +0xf5
encoding/json.(*arrayEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xb5f240, 0x62ff3b64, 0x197, 0x100)
/usr/local/go/src/encoding/json/encode.go:730 +0x52
encoding/json.(*sliceEncoder).encode(0x58b32890, 0x45f6cd20, 0xb5f240, 0x62ff3b64, 0x197, 0x100)
/usr/local/go/src/encoding/json/encode.go:697 +0xa5
encoding/json.(*sliceEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xb5f240, 0x62ff3b64, 0x197, 0x100)
/usr/local/go/src/encoding/json/encode.go:709 +0x52
encoding/json.(*structEncoder).encode(0x58c1e520, 0x45f6cd20, 0xc31940, 0x62ff3b60, 0x199, 0x100)
/usr/local/go/src/encoding/json/encode.go:601 +0x248
encoding/json.(*structEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xc31940, 0x62ff3b60, 0x199, 0x62ff0100)
/usr/local/go/src/encoding/json/encode.go:615 +0x52
encoding/json.(*ptrEncoder).encode(0x58b328a8, 0x45f6cd20, 0xbc5d80, 0x43c1b5a8, 0x196, 0x100)
/usr/local/go/src/encoding/json/encode.go:742 +0xd1
encoding/json.(*ptrEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xbc5d80, 0x43c1b5a8, 0x196, 0xbc0100)
/usr/local/go/src/encoding/json/encode.go:747 +0x52
encoding/json.(*arrayEncoder).encode(0x58b328b8, 0x45f6cd20, 0xc5c5e0, 0x62ff3db0, 0x97, 0x100)
/usr/local/go/src/encoding/json/encode.go:723 +0xf5
encoding/json.(*arrayEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xc5c5e0, 0x62ff3db0, 0x97, 0xc50100)
/usr/local/go/src/encoding/json/encode.go:730 +0x52
encoding/json.(*sliceEncoder).encode(0x58b328c8, 0x45f6cd20, 0xc5c5e0, 0x62ff3db0, 0x97, 0x100)
/usr/local/go/src/encoding/json/encode.go:697 +0xa5
encoding/json.(*sliceEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xc5c5e0, 0x62ff3db0, 0x97, 0x100)
/usr/local/go/src/encoding/json/encode.go:709 +0x52
encoding/json.(*encodeState).reflectValue(0x45f6cd20, 0xc5c5e0, 0x62ff3db0, 0x97, 0x62ff0100)
/usr/local/go/src/encoding/json/encode.go:307 +0x6a
encoding/json.interfaceEncoder(0x45f6cd20, 0xbf12a0, 0x62ff3dd4, 0x194, 0x100)
/usr/local/go/src/encoding/json/encode.go:573 +0xcc
encoding/json.(*structEncoder).encode(0x57d1ed00, 0x45f6cd20, 0xc10ba0, 0x62ff3dd0, 0x199, 0x100)
/usr/local/go/src/encoding/json/encode.go:601 +0x248
encoding/json.(*structEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xc10ba0, 0x62ff3dd0, 0x199, 0x62ff0100)
/usr/local/go/src/encoding/json/encode.go:615 +0x52
encoding/json.(*ptrEncoder).encode(0x57d26158, 0x45f6cd20, 0xb48840, 0x62ff3dd0, 0x16, 0x100)
/usr/local/go/src/encoding/json/encode.go:742 +0xd1
encoding/json.(*ptrEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xb48840, 0x62ff3dd0, 0x16, 0x100)
/usr/local/go/src/encoding/json/encode.go:747 +0x52
encoding/json.(*encodeState).reflectValue(0x45f6cd20, 0xb48840, 0x62ff3dd0, 0x16, 0x62ff0100)
/usr/local/go/src/encoding/json/encode.go:307 +0x6a
encoding/json.interfaceEncoder(0x45f6cd20, 0xba2a60, 0x6300bc68, 0x194, 0x100)
/usr/local/go/src/encoding/json/encode.go:573 +0xcc
encoding/json.(*structEncoder).encode(0x57d1ec40, 0x45f6cd20, 0xc3ab80, 0x6300bc60, 0x199, 0xffff0100)
/usr/local/go/src/encoding/json/encode.go:601 +0x248
encoding/json.(*structEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xc3ab80, 0x6300bc60, 0x199, 0x63000100)
/usr/local/go/src/encoding/json/encode.go:615 +0x52
encoding/json.(*ptrEncoder).encode(0x57d26118, 0x45f6cd20, 0xb48880, 0x6300bc60, 0x16, 0x100)
/usr/local/go/src/encoding/json/encode.go:742 +0xd1
encoding/json.(*ptrEncoder).(encoding/json.encode)-fm(0x45f6cd20, 0xb48880, 0x6300bc60, 0x16, 0x63000100)
/usr/local/go/src/encoding/json/encode.go:747 +0x52
encoding/json.(*encodeState).reflectValue(0x45f6cd20, 0xb48880, 0x6300bc60, 0x16, 0x100)
/usr/local/go/src/encoding/json/encode.go:307 +0x6a
encoding/json.(*encodeState).marshal(0x45f6cd20, 0xb48880, 0x6300bc60, 0x100, 0x0, 0x0)
/usr/local/go/src/encoding/json/encode.go:280 +0xa5
encoding/json.Marshal(0xb48880, 0x6300bc60, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/encoding/json/encode.go:145 +0x85
github.com/prometheus/prometheus/web/api/v1.respond(0x1093e00, 0x50617060, 0xb48840, 0x62ff3dd0)
/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:326 +0xdc
github.com/prometheus/prometheus/web/api/v1.(*API).Register.func1.1(0x1093e00, 0x50617060, 0x4f3fdc00)
/go/src/github.com/prometheus/prometheus/web/api/v1/api.go:113 +0x9e
net/http.HandlerFunc.ServeHTTP(0x1387abe8, 0x1093e00, 0x50617060, 0x4f3fdc00)
/usr/local/go/src/net/http/server.go:1726 +0x34
github.com/prometheus/prometheus/util/httputil.CompressionHandler.ServeHTTP(0x1091ae0, 0x1387abe8, 0x1093ee0, 0x5061d3d0, 0x4f3fdc00)
/go/src/github.com/prometheus/prometheus/util/httputil/compression.go:90 +0x6e
github.com/prometheus/prometheus/util/httputil.(*CompressionHandler).ServeHTTP(0x1387abf0, 0x1093ee0, 0x5061d3d0, 0x4f3fdc00)
:5 +0x9b
net/http.(Handler).ServeHTTP-fm(0x1093ee0, 0x5061d3d0, 0x4f3fdc00)
/go/src/github.com/prometheus/prometheus/web/web.go:173 +0x45
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1(0x10941e0, 0x60331830, 0x4f3fdc00)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:287 +0x2a4
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).handle.func1(0x10941e0, 0x60331830, 0x4f3fdc00, 0x0, 0x0, 0x0)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:83 +0x34b
github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter.(*Router).ServeHTTP(0x13937e80, 0x10941e0, 0x60331830, 0x4f3fdc00)
/go/src/github.com/prometheus/prometheus/vendor/github.com/julienschmidt/httprouter/router.go:299 +0x148
github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route.(*Router).ServeHTTP(0x136c5080, 0x10941e0, 0x60331830, 0x4f3fdc00)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/common/route/route.go:125 +0x3b
net/http.serverHandler.ServeHTTP(0x3a4d82d0, 0x10941e0, 0x60331830, 0x4f3fdc00)
/usr/local/go/src/net/http/server.go:2202 +0x136
net/http.(*conn).serve(0x45e8d1c0, 0x1094c60, 0x3b6b0460)
/usr/local/go/src/net/http/server.go:1579 +0xe86
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2293 +0x489

goroutine 38228 [IO wait]:
net.runtime_pollWait(0x44ae0b60, 0x72, 0x6123c688)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x212a28b0, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x212a2818, 0xcc1130, 0x7, 0xd41980, 0x5, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x212a27e0, 0x3f6dd000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x3cf2bd90, 0x3f6dd000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*persistConn).Read(0x5217b290, 0x3f6dd000, 0x1000, 0x1000, 0x1, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1261 +0x180
bufio.(*Reader).fill(0x6123e270)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).Peek(0x6123e270, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:129 +0x8f
net/http.(*persistConn).readLoop(0x5217b290)
/usr/local/go/src/net/http/transport.go:1418 +0x183
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1062 +0xda5

goroutine 38176 [IO wait]:
net.runtime_pollWait(0x44ae0e30, 0x72, 0x6123c680)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x212a2990, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x212a28f8, 0xcc1130, 0x7, 0xd41980, 0x5d036060, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x212a28c0, 0x3f6d1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x3cf2bd00, 0x3f6d1000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*persistConn).Read(0x5217b320, 0x3f6d1000, 0x1000, 0x1000, 0x539ead80, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1261 +0x180
bufio.(*Reader).fill(0x6123e210)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).Peek(0x6123e210, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:129 +0x8f
net/http.(*persistConn).readLoop(0x5217b320)
/usr/local/go/src/net/http/transport.go:1418 +0x183
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1062 +0xda5

goroutine 38230 [chan send]:
github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.computeApproximateRequestSize(0x5ea03b80, 0x3d1b5380, 0xf4)
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:316 +0x14d
created by github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus.InstrumentHandlerFuncWithOpts.func1
/go/src/github.com/prometheus/prometheus/vendor/github.com/prometheus/client_golang/prometheus/http.go:275 +0x14d

goroutine 38182 [runnable]:
net.(*netFD).connect.func2(0x1094ce0, 0x50561a80, 0x2067a540, 0x50561d00)
/usr/local/go/src/net/fd_windows.go:350 +0x10c
created by net.(*netFD).connect
/usr/local/go/src/net/fd_windows.go:358 +0x5e3

goroutine 38136 [IO wait]:
net.runtime_pollWait(0x44ae0a70, 0x72, 0x45f66a7c)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x1dda0290, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x1dda01f8, 0xcc1130, 0x7, 0xd41980, 0x3, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x1dda01c0, 0x3db18000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x4331a1e0, 0x3db18000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*connReader).Read(0x43318460, 0x3db18000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:586 +0x189
bufio.(*Reader).fill(0x433151d0)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).ReadSlice(0x433151d0, 0xd3e40a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:330 +0x1b4
bufio.(*Reader).ReadLine(0x433151d0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:359 +0x4b
net/textproto.(*Reader).readLineSlice(0x433184c0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/textproto/reader.go:55 +0x59
net/textproto.(*Reader).ReadLine(0x433184c0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/textproto/reader.go:36 +0x39
net/http.readRequest(0x433151d0, 0x3ff00, 0x494da680, 0x0, 0x0)
/usr/local/go/src/net/http/request.go:793 +0x9a
net/http.(*conn).readRequest(0x4bae7540, 0x1094c60, 0x433184a0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:765 +0x3c0
net/http.(*conn).serve(0x4bae7540, 0x1094c60, 0x433184a0)
/usr/local/go/src/net/http/server.go:1532 +0x96c
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2293 +0x489

goroutine 38140 [IO wait]:
net.runtime_pollWait(0x44ae0db8, 0x72, 0x45f66a78)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x1cc40530, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x1cc40498, 0xcc1130, 0x7, 0xd41980, 0x3, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x1cc40460, 0x5216c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x4a244868, 0x5216c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*connReader).Read(0x4a2484e0, 0x5216c000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:586 +0x189
bufio.(*Reader).fill(0x3db24f30)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).ReadSlice(0x3db24f30, 0xd3e40a, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:330 +0x1b4
bufio.(*Reader).ReadLine(0x3db24f30, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:359 +0x4b
net/textproto.(*Reader).readLineSlice(0x5c8b17c0, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/textproto/reader.go:55 +0x59
net/textproto.(*Reader).ReadLine(0x5c8b17c0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/net/textproto/reader.go:36 +0x39
net/http.readRequest(0x3db24f30, 0x0, 0x494da600, 0x0, 0x0)
/usr/local/go/src/net/http/request.go:793 +0x9a
net/http.(*conn).readRequest(0x4bae7ac0, 0x1094c60, 0x4a248520, 0x0, 0x0, 0x0)
/usr/local/go/src/net/http/server.go:765 +0x3c0
net/http.(*conn).serve(0x4bae7ac0, 0x1094c60, 0x4a248520)
/usr/local/go/src/net/http/server.go:1532 +0x96c
created by net/http.(*Server).Serve
/usr/local/go/src/net/http/server.go:2293 +0x489

goroutine 38174 [IO wait]:
net.runtime_pollWait(0x44ae0cc8, 0x72, 0x6123c67c)
/usr/local/go/src/runtime/netpoll.go:160 +0x55
net.(*pollDesc).wait(0x212a2b50, 0x72, 0x0, 0x0)
/usr/local/go/src/net/fd_poll_runtime.go:73 +0x35
net.(*ioSrv).ExecIO(0x136fa2f0, 0x212a2ab8, 0xcc1130, 0x7, 0xd41980, 0x2, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:171 +0x122
net.(*netFD).Read(0x212a2a80, 0x49b0f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/fd_windows.go:426 +0x129
net.(*conn).Read(0x3cf2bcd8, 0x49b0f000, 0x1000, 0x1000, 0x0, 0x0, 0x0)
/usr/local/go/src/net/net.go:173 +0xa9
net/http.(*persistConn).Read(0x5217b440, 0x49b0f000, 0x1000, 0x1000, 0x4127ae, 0x0, 0x0)
/usr/local/go/src/net/http/transport.go:1261 +0x180
bufio.(*Reader).fill(0x6123e1e0)
/usr/local/go/src/bufio/bufio.go:97 +0x172
bufio.(*Reader).Peek(0x6123e1e0, 0x1, 0x0, 0x0, 0x0, 0x0, 0x0)
/usr/local/go/src/bufio/bufio.go:129 +0x8f
net/http.(*persistConn).readLoop(0x5217b440)
/usr/local/go/src/net/http/transport.go:1418 +0x183
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1062 +0xda5

goroutine 38220 [select]:
net.(*netFD).connect.func2(0x1094ce0, 0x46435000, 0x212a2b60, 0x46435280)
/usr/local/go/src/net/fd_windows.go:350 +0x10c
created by net.(*netFD).connect
/usr/local/go/src/net/fd_windows.go:358 +0x5e3

goroutine 38227 [select]:
net/http.(*persistConn).writeLoop(0x5217b3b0)
/usr/local/go/src/net/http/transport.go:1646 +0x2ba
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1063 +0xdc5

goroutine 38225 [select]:
net/http.(*persistConn).writeLoop(0x5217b320)
/usr/local/go/src/net/http/transport.go:1646 +0x2ba
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1063 +0xdc5

goroutine 38134 [select]:
net/http.(*persistConn).writeLoop(0x4cb10090)
/usr/local/go/src/net/http/transport.go:1646 +0x2ba
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1063 +0xdc5

goroutine 38229 [select]:
net/http.(*persistConn).writeLoop(0x5217b290)
/usr/local/go/src/net/http/transport.go:1646 +0x2ba
created by net/http.(*Transport).dialConn
/usr/local/go/src/net/http/transport.go:1063 +0xdc5

[jinhang]

-storage.local.memory-chunks 1G
-storage.lcoal.max-chunk-to-persist 0.5G
why used 1.25G，can not allocate 704K ,so OOM!

[jinhang]

should I increase -storage.local.memory-chunks to 3g?

[jinhang]

when I increase to 3G ,log is:
time="2016-12-21T16:16:15+08:00" level=error msg="Storage needs throttling. Scrapes and rule evaluations will be skipped." chunksToPersist=433329 maxChunksToPersist=536870912 maxToleratedMemChunks=-1181116006 memoryChunks=467574 source="storage.go:895"
panic: runtime error: makeslice: len out of range

goroutine 163 [running]:
panic(0xbf1ea0, 0x46935148)
/usr/local/go/src/runtime/panic.go:500 +0x331
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).maybeEvict(0x1379a240)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1057 +0x62
github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).handleEvictList(0x1379a240)
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:1022 +0x168
created by github.com/prometheus/prometheus/storage/local.(*MemorySeriesStorage).Start
/go/src/github.com/prometheus/prometheus/storage/local/storage.go:367 +0x348

[juliusv]

Your process / machine is running out of memory in the first crash (runtime: out of memory: cannot allocate 720896-byte block (1342177280 in use) fatal error: out of memory).

In the second one, Prometheus tries to allocate a Go slice that's larger than possible, not sure why exactly, but maybe it has something to do with very high flag values you provide.

Can you give the actual flags that you start Prometheus with? That is, is -storage.local.memory-chunks 1G really one billion (G = giga = billion)? That would be way too high. It should be set to a couple of millions at maximum, depending on how much memory you machine has.

[jinhang]

yes . my machine has 16GB, when I start Prometheus with -storage.local.memory-chunks 1GB

[jinhang]

default config is 1MB ,it occasionally occur OOM

[jinhang]

now 40551 series loaded, 11 machine scrape with 3s,11mahchine with 10s.

In the second one, Prometheus tries to allocate a Go slice that's larger than possible, not sure why exactly, but maybe it has something to do with very high flag values you provide.

is 8000000000000

[jinhang]

@juliusv

[jinhang]

@brian-brazil

[juliusv]

Sorry, but the numbers you mention mismatch or don't really make sense. -storage.local.memory-chunks does not take a GB (bytes) value, but a number of chunks. The maximum it should be set to is a couple of million (certainly not billions or more!). On a machine with only 16GB of RAM, it should probably not be set to more than 2 million or something like that.

"is 8000000000000" -> what do you mean with that? Certainly no flag value should be set to 8 trillion. If that is the value for -storage.local.memory-chunks, then Prometheus will try to keep 8 trillion chunks in memory before it evicts them. It is no wonder why you are getting out-of-memory crashes then.

Try a value in the lower millions, or even less. If you really only have 40551 series, you probably want to set this to only a couple of hundred thousand.

[jinhang]

so ,How to avoid out-of-memory ?

[timchenxiaoyu]

my prometheus is crash
version=1.6.1, branch=master, revision=4666df502c0e239ed4aa1d80abbbfb54f61b23c3
@juliusv
this is disk io

this is memory

picture show crush twice in 12PM ADN 8 AM

ul 19 08:12:38 slave-95 kernel: INFO: task prometheus:20402 blocked for more than 120 seconds.
Jul 19 08:12:38 slave-95 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Jul 19 08:12:38 slave-95 kernel: prometheus D ffffffff816896f0 0 20402 20376 0x00000080
Jul 19 08:12:38 slave-95 kernel: ffff8801dc7236a0 0000000000000086 ffff8801e5f68fb0 ffff8801dc723fd8
Jul 19 08:12:38 slave-95 kernel: ffff8801dc723fd8 ffff8801dc723fd8 ffff8801e5f68fb0 ffff88043fd56c40
Jul 19 08:12:38 slave-95 kernel: 0000000000000000 7fffffffffffffff ffff88043ff407e8 ffffffff816896f0
Jul 19 08:12:38 slave-95 kernel: Call Trace:
Jul 19 08:12:38 slave-95 kernel: [] ? bit_wait+0x50/0x50
Jul 19 08:12:38 slave-95 kernel: [] schedule+0x29/0x70
Jul 19 08:12:38 slave-95 kernel: [] schedule_timeout+0x239/0x2d0
Jul 19 08:12:38 slave-95 kernel: [] ? page_remove_rmap+0x104/0x160
Jul 19 08:12:38 slave-95 kernel: [] ? kvm_clock_get_cycles+0x1f/0x30
Jul 19 08:12:38 slave-95 kernel: [] ? ktime_get_ts64+0x4c/0xf0
Jul 19 08:12:38 slave-95 kernel: [] ? bit_wait+0x50/0x50
Jul 19 08:12:38 slave-95 kernel: [] io_schedule_timeout+0xae/0x130
Jul 19 08:12:38 slave-95 kernel: [] io_schedule+0x18/0x20
Jul 19 08:12:38 slave-95 kernel: [] bit_wait_io+0x11/0x50
Jul 19 08:12:38 slave-95 kernel: [] __wait_on_bit+0x65/0x90
Jul 19 08:12:38 slave-95 kernel: [] ? __mem_cgroup_uncharge_common+0xcd/0x2f0
Jul 19 08:12:38 slave-95 kernel: [] wait_on_page_bit+0x81/0xa0
Jul 19 08:12:38 slave-95 kernel: [] ? wake_bit_function+0x40/0x40
Jul 19 08:12:38 slave-95 kernel: [] shrink_page_list+0x6e2/0xb00
Jul 19 08:12:38 slave-95 kernel: [] shrink_inactive_list+0x1fa/0x630
Jul 19 08:12:38 slave-95 kernel: [] shrink_lruvec+0x385/0x770
Jul 19 08:12:38 slave-95 kernel: [] ? blk_mq_run_hw_queue+0xa5/0xd0
Jul 19 08:12:38 slave-95 kernel: [] shrink_zone+0x76/0x1a0
Jul 19 08:12:38 slave-95 kernel: [] do_try_to_free_pages+0xf0/0x4e0
Jul 19 08:12:38 slave-95 kernel: [] ? get_page_from_freelist+0x51a/0x9f0
Jul 19 08:12:38 slave-95 kernel: [] try_to_free_mem_cgroup_pages+0xca/0x160
Jul 19 08:12:38 slave-95 kernel: [] mem_cgroup_reclaim+0x4e/0x120
Jul 19 08:12:38 slave-95 kernel: [] __mem_cgroup_try_charge+0x40c/0x660
Jul 19 08:12:38 slave-95 kernel: [] mem_cgroup_charge_common+0x59/0xc0
Jul 19 08:12:38 slave-95 kernel: [] mem_cgroup_newpage_charge+0x26/0x30
Jul 19 08:12:38 slave-95 kernel: [] handle_mm_fault+0xc90/0xfe0
Jul 19 08:12:38 slave-95 kernel: [] ? do_sync_read+0x8d/0xd0
Jul 19 08:12:38 slave-95 kernel: [] __do_page_fault+0x154/0x450
Jul 19 08:12:38 slave-95 kernel: [] trace_do_page_fault+0x56/0x150
Jul 19 08:12:38 slave-95 kernel: [] do_async_page_fault+0x1b/0xd0
Jul 19 08:12:38 slave-95 kernel: [] async_page_fault+0x28/0x30
Jul 19 08:12:38 slave-95 kernel: INFO: task prometheus:20419 blocked for more than 120 seconds.
Jul 19 08:12:38 slave-95 kernel: "echo 0 > /proc/sys/kernel/hung_task_timeout_secs" disables this message.
Jul 19 08:12:38 slave-95 kernel: prometheus D 0000000000000000 0 20419 20376 0x00000080
Jul 19 08:12:38 slave-95 kernel: ffff88009bc93a68 0000000000000086 ffff8800a58c0000 ffff88009bc93fd8
Jul 19 08:12:38 slave-95 kernel: ffff88009bc93fd8 ffff88009bc93fd8 ffff8800a58c0000 ffff8800a58c0000
Jul 19 08:12:38 slave-95 kernel: ffff88042e85ea40 ffffffffffffffff ffff88042e85ea48 0000000000000000

[timchenxiaoyu]

my target is not change ,but ,why write io is continuously increase?

[lock]

This thread has been automatically locked since there has not been any recent activity after it was closed. Please open a new issue for related bugs.